diff --git a/src/db/db/dbConverters.h b/src/db/db/dbConverters.h index 69e06d22a..9d3580e0e 100644 --- a/src/db/db/dbConverters.h +++ b/src/db/db/dbConverters.h @@ -40,6 +40,8 @@ namespace db template struct DB_PUBLIC_TEMPLATE TransformationConverter { + typedef T value_type; + std::string to_string (const T &t) const { return t.to_string (); @@ -58,6 +60,8 @@ struct DB_PUBLIC_TEMPLATE TransformationConverter */ struct DB_PUBLIC LayoutLayerConverter { + typedef db::LayerProperties value_type; + std::string to_string (const db::LayerProperties &p) const { return p.to_string (); @@ -79,6 +83,8 @@ struct DB_PUBLIC LayoutLayerConverter template struct DB_PUBLIC_TEMPLATE PointConverter { + typedef P value_type; + std::string to_string (const P &p) const { return tl::to_string (p.x ()) + "," + tl::to_string (p.y ()); diff --git a/src/db/db/dbStream.h b/src/db/db/dbStream.h index 9cc5d4b1a..22e0ec43e 100644 --- a/src/db/db/dbStream.h +++ b/src/db/db/dbStream.h @@ -154,6 +154,7 @@ class StreamOptionsReadAdaptor { public: typedef tl::pass_by_ref_tag tag; + typedef tl::zero_cardinality_tag cardinality; StreamOptionsReadAdaptor () : mp_options (0), m_done (false) diff --git a/src/db/db/dbTechnology.h b/src/db/db/dbTechnology.h index cb0da8751..84ae82b7c 100644 --- a/src/db/db/dbTechnology.h +++ b/src/db/db/dbTechnology.h @@ -807,6 +807,7 @@ class TechnologyComponentReadAdaptor { public: typedef tl::pass_by_ref_tag tag; + typedef tl::single_cardinality_tag cardinality; TechnologyComponentReadAdaptor (const std::string &name) : m_name (name), mp_t (0), m_done (false) diff --git a/src/img/img/imgStream.cc b/src/img/img/imgStream.cc index f198c0088..bc2561dc7 100644 --- a/src/img/img/imgStream.cc +++ b/src/img/img/imgStream.cc @@ -359,6 +359,8 @@ namespace { struct PointConverter { + typedef db::DPoint value_type; + std::string to_string (const db::DPoint &p) const { return p.to_string (); @@ -373,6 +375,8 @@ namespace { struct ColorMapConverter { + typedef std::pair > value_type; + std::string to_string (const std::pair > &cm) const { std::string s; @@ -427,7 +431,7 @@ tl::XMLStruct s_img_structure ("image-data", tl::make_member (&ImageProxy::max_value, &ImageProxy::set_max_value, "max-value") + tl::make_element (&ImageProxy::data_mapping, &ImageProxy::set_data_mapping, "data-mapping", tl::make_element (&img::DataMapping::false_color_nodes, "color-map", - tl::make_member >, img::DataMapping::false_color_nodes_type::const_iterator, img::DataMapping::false_color_nodes_type, ColorMapConverter> (&img::DataMapping::false_color_nodes_type::begin, &img::DataMapping::false_color_nodes_type::end, &img::DataMapping::false_color_nodes_type::push_back, "color-map-entry", ColorMapConverter ()) + tl::make_member >, img::DataMapping::false_color_nodes_type::const_iterator, img::DataMapping::false_color_nodes_type, tl::XMLStringBasedConverter > (&img::DataMapping::false_color_nodes_type::begin, &img::DataMapping::false_color_nodes_type::end, &img::DataMapping::false_color_nodes_type::push_back, "color-map-entry", tl::XMLStringBasedConverter ()) ) + tl::make_member (&img::DataMapping::brightness, "brightness") + tl::make_member (&img::DataMapping::contrast, "contrast") + @@ -437,7 +441,7 @@ tl::XMLStruct s_img_structure ("image-data", tl::make_member (&img::DataMapping::blue_gain, "blue-gain") ) + tl::make_element (&ImageProxy::landmarks, &ImageProxy::set_landmarks, "landmarks", - tl::make_member (&img::Object::landmarks_type::begin, &img::Object::landmarks_type::end, &img::Object::landmarks_type::push_back, "landmark", PointConverter ()) + tl::make_member > (&img::Object::landmarks_type::begin, &img::Object::landmarks_type::end, &img::Object::landmarks_type::push_back, "landmark", tl::XMLStringBasedConverter ()) ) + tl::make_member (&ImageProxy::begin_byte_data, &ImageProxy::end_byte_data, &ImageProxy::push_byte_data, "byte-data") + tl::make_member (&ImageProxy::begin_data, &ImageProxy::end_data, &ImageProxy::push_data, "data") diff --git a/src/lay/lay/laySaltGrain.cc b/src/lay/lay/laySaltGrain.cc index e60ed87ca..2d57ac8c1 100644 --- a/src/lay/lay/laySaltGrain.cc +++ b/src/lay/lay/laySaltGrain.cc @@ -353,6 +353,8 @@ SaltGrain::valid_version (const std::string &v) struct TimeConverter { + typedef QDateTime value_type; + std::string to_string (const QDateTime &time) const { if (time.isNull ()) { @@ -374,6 +376,8 @@ struct TimeConverter struct ImageConverter { + typedef QImage value_type; + std::string to_string (const QImage &image) const { if (image.isNull ()) { @@ -416,10 +420,10 @@ SaltGrain::xml_elements () tl::make_member (&SaltGrain::license, &SaltGrain::set_license, "license") + tl::make_member (&SaltGrain::author, &SaltGrain::set_author, "author") + tl::make_member (&SaltGrain::author_contact, &SaltGrain::set_author_contact, "author-contact") + - tl::make_member (&SaltGrain::authored_time, &SaltGrain::set_authored_time, "authored-time", TimeConverter ()) + - tl::make_member (&SaltGrain::installed_time, &SaltGrain::set_installed_time, "installed-time", TimeConverter ()) + - tl::make_member (&SaltGrain::icon, &SaltGrain::set_icon, "icon", ImageConverter ()) + - tl::make_member (&SaltGrain::screenshot, &SaltGrain::set_screenshot, "screenshot", ImageConverter ()) + + tl::make_member (&SaltGrain::authored_time, &SaltGrain::set_authored_time, "authored-time", tl::XMLStringBasedConverter ()) + + tl::make_member (&SaltGrain::installed_time, &SaltGrain::set_installed_time, "installed-time", tl::XMLStringBasedConverter ()) + + tl::make_member (&SaltGrain::icon, &SaltGrain::set_icon, "icon", tl::XMLStringBasedConverter ()) + + tl::make_member (&SaltGrain::screenshot, &SaltGrain::set_screenshot, "screenshot", tl::XMLStringBasedConverter ()) + tl::make_element (&SaltGrain::begin_dependencies, &SaltGrain::end_dependencies, &SaltGrain::add_dependency, "depends", tl::make_member (&SaltGrainDependency::name, "name") + tl::make_member (&SaltGrainDependency::url, "url") + diff --git a/src/laybasic/laybasic/layDispatcher.cc b/src/laybasic/laybasic/layDispatcher.cc index b3a1f3b66..e55f8700e 100644 --- a/src/laybasic/laybasic/layDispatcher.cc +++ b/src/laybasic/laybasic/layDispatcher.cc @@ -108,6 +108,8 @@ Dispatcher::config_finalize () struct ConfigGetAdaptor { + typedef tl::zero_cardinality_tag cardinality; + ConfigGetAdaptor (const std::string &name) : mp_owner (0), m_done (false), m_name (name) { @@ -145,6 +147,8 @@ private: struct ConfigGetNullAdaptor { + typedef tl::zero_cardinality_tag cardinality; + ConfigGetNullAdaptor () { // .. nothing yet .. diff --git a/src/layui/layui/rdbMarkerBrowserDialog.cc b/src/layui/layui/rdbMarkerBrowserDialog.cc index 6299d3992..e38683505 100644 --- a/src/layui/layui/rdbMarkerBrowserDialog.cc +++ b/src/layui/layui/rdbMarkerBrowserDialog.cc @@ -387,7 +387,7 @@ BEGIN_PROTECTED } else { - rdb->save (rdb->filename ()); + rdb->save (rdb->filename (), rdb->binary ()); rdb->reset_modified (); } @@ -459,7 +459,7 @@ BEGIN_PROTECTED throw tl::Exception (tl::to_string (tr ("The current report database is not saved.\nSave it to some file with 'Save As', before saving it as waiver DB."))); } - rdb->write (rdb->filename () + ".w"); + rdb->write (rdb->filename () + ".w", rdb->binary ()); END_PROTECTED } @@ -475,11 +475,13 @@ BEGIN_PROTECTED if (rdb) { // prepare and open the file dialog - lay::FileDialog save_dialog (this, tl::to_string (QObject::tr ("Save Marker Database File")), "KLayout RDB files (*.lyrdb)"); + lay::FileDialog save_dialog (this, tl::to_string (QObject::tr ("Save Marker Database File")), tl::to_string (tr ("KLayout RDB files (*.lyrdb);;KLayout binary RDB files (*.rdb);;All files (*)"))); std::string fn (rdb->filename ()); if (save_dialog.get_save (fn)) { - rdb->save (fn); + bool binary = (tl::extension (fn) == "rdb"); + + rdb->save (fn, binary); rdb->reset_modified (); // update the RDB title strings diff --git a/src/lym/lym/lymMacro.cc b/src/lym/lym/lymMacro.cc index 5844a708b..257494805 100644 --- a/src/lym/lym/lymMacro.cc +++ b/src/lym/lym/lymMacro.cc @@ -150,6 +150,8 @@ bool Macro::del () struct Interpreter2s { + typedef Macro::Interpreter value_type; + std::string to_string (Macro::Interpreter i) const { switch (i) { @@ -199,7 +201,7 @@ static tl::XMLStruct xml_struct ("klayout-macro", tl::make_member (&Macro::show_in_menu, &Macro::set_show_in_menu, "show-in-menu") + tl::make_member (&Macro::group_name, &Macro::set_group_name, "group-name") + tl::make_member (&Macro::menu_path, &Macro::set_menu_path, "menu-path") + - tl::make_member (&Macro::interpreter, &Macro::set_interpreter, "interpreter", Interpreter2s ()) + + tl::make_member (&Macro::interpreter, &Macro::set_interpreter, "interpreter", tl::XMLStringBasedConverter ()) + tl::make_member (&Macro::dsl_interpreter, &Macro::set_dsl_interpreter, "dsl-interpreter-name") + tl::make_member (&Macro::text, &Macro::set_text, "text") + tl::make_member ("format") // for backward compatibility diff --git a/src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc b/src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc index 858b7d6a2..7cfd53278 100644 --- a/src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc +++ b/src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc @@ -166,6 +166,8 @@ GerberImportData::setup_importer (db::GerberImporter *importer) struct MountingConverter { + typedef GerberImportData::mounting_type value_type; + std::string to_string (GerberImportData::mounting_type m) const { return m == GerberImportData::MountingTop ? "top" : "bottom"; @@ -190,9 +192,9 @@ pcb_project_structure ("pcb-project", tl::make_member (&GerberImportData::border, "border") + tl::make_member (&GerberImportData::free_layer_mapping, "free-layer-mapping") + tl::make_element (&GerberImportData::layout_layers, "layout-layers", - tl::make_member::const_iterator, std::vector > (&std::vector::begin, &std::vector::end, &std::vector::push_back, "layout-layer", db::LayoutLayerConverter ()) + tl::make_member::const_iterator, std::vector > (&std::vector::begin, &std::vector::end, &std::vector::push_back, "layout-layer", tl::XMLStringBasedConverter ()) ) + - tl::make_member (&GerberImportData::mounting, "mounting", MountingConverter ()) + + tl::make_member (&GerberImportData::mounting, "mounting", tl::XMLStringBasedConverter ()) + tl::make_member (&GerberImportData::num_metal_layers, "num-metal-layers") + tl::make_member (&GerberImportData::num_via_types, "num-via-types") + tl::make_element (&GerberImportData::artwork_files, "artwork-files", @@ -217,11 +219,11 @@ pcb_project_structure ("pcb-project", ) + tl::make_element (&GerberImportData::reference_points, "reference-points", tl::make_element, std::vector >::const_iterator, std::vector > > (&std::vector >::begin, &std::vector >::end, &std::vector >::push_back, "reference-point", - tl::make_member (&std::pair ::first, "pcb", db::PointConverter ()) + - tl::make_member (&std::pair ::second, "layout", db::PointConverter ()) + tl::make_member (&std::pair ::first, "pcb", tl::XMLStringBasedConverter > ()) + + tl::make_member (&std::pair ::second, "layout", tl::XMLStringBasedConverter > ()) ) ) + - tl::make_member (&GerberImportData::explicit_trans, "explicit-trans", db::TransformationConverter ()) + + tl::make_member (&GerberImportData::explicit_trans, "explicit-trans", tl::XMLStringBasedConverter > ()) + tl::make_member (&GerberImportData::layer_properties_file, "layer-properties-file") + tl::make_member (&GerberImportData::num_circle_points, "num-circle-points") + tl::make_member (&GerberImportData::merge_flag, "merge-flag") + diff --git a/src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc b/src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc index 4a0c639fc..03587c323 100644 --- a/src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc +++ b/src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc @@ -49,6 +49,8 @@ static struct { struct ModeConverter { + typedef lay::StreamImportData::mode_type value_type; + std::string to_string (lay::StreamImportData::mode_type t) const { for (unsigned int i = 0; i < sizeof (mode_strings) / sizeof (mode_strings [0]); ++i) { @@ -82,6 +84,8 @@ static struct { struct LayerModeConverter { + typedef lay::StreamImportData::layer_mode_type value_type; + std::string to_string (lay::StreamImportData::layer_mode_type t) const { for (unsigned int i = 0; i < sizeof (layer_mode_strings) / sizeof (layer_mode_strings [0]); ++i) { @@ -144,8 +148,8 @@ static tl::XMLElementList xml_elements () ) + tl::make_member (&StreamImportData::topcell, "cell-name") + tl::make_member (&StreamImportData::layer_offset, "layer-offset") + - tl::make_member (&StreamImportData::layer_mode, "layer-mode", LayerModeConverter ()) + - tl::make_member (&StreamImportData::mode, "import-mode", ModeConverter ()) + + tl::make_member (&StreamImportData::layer_mode, "layer-mode", tl::XMLStringBasedConverter ()) + + tl::make_member (&StreamImportData::mode, "import-mode", tl::XMLStringBasedConverter ()) + tl::make_element (&StreamImportData::reference_points, "reference-points", tl::make_element (&ref_point_v::begin, &ref_point_v::end, &ref_point_v::push_back, "reference-point", tl::make_member (&ref_point::first, "p1") + diff --git a/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc b/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc index 87f900ac7..c2c952b83 100644 --- a/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc +++ b/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc @@ -24,50 +24,49 @@ #include "dbTechnology.h" #include "tlClassRegistry.h" -namespace tl -{ - /** - * @brief A specialization of the XMLConverter that is used to serialize the connection info - */ - - template <> - struct XMLStdConverter - { - std::string to_string (const db::NetTracerConnectionInfo &v) const - { - return v.to_string (); - } - - void from_string (const std::string &s, db::NetTracerConnectionInfo &v) const - { - tl::Extractor ex (s.c_str ()); - v.parse (ex); - } - }; - - /** - * @brief A specialization of the XMLConverter that is used to serialize the symbol info - */ - - template <> - struct XMLStdConverter - { - std::string to_string (const db::NetTracerSymbolInfo &v) const - { - return v.to_string (); - } - - void from_string (const std::string &s, db::NetTracerSymbolInfo &v) const - { - tl::Extractor ex (s.c_str ()); - v.parse (ex); - } - }; -} - namespace { +/** + * @brief A string converter for NetTracerConnectionInfo + */ + +struct NetTracerConnectionInfoConverter +{ + typedef db::NetTracerConnectionInfo value_type; + + std::string to_string (const db::NetTracerConnectionInfo &v) const + { + return v.to_string (); + } + + void from_string (const std::string &s, db::NetTracerConnectionInfo &v) const + { + tl::Extractor ex (s.c_str ()); + v.parse (ex); + } +}; + +/** + * @brief A specialization of the XMLConverter that is used to serialize the symbol info + */ + +struct NetTracerSymbolInfoConverter +{ + typedef db::NetTracerSymbolInfo value_type; + + std::string to_string (const db::NetTracerSymbolInfo &v) const + { + return v.to_string (); + } + + void from_string (const std::string &s, db::NetTracerSymbolInfo &v) const + { + tl::Extractor ex (s.c_str ()); + v.parse (ex); + } +}; + static const db::NetTracerConnectivity * get_fallback_default (const db::NetTracerTechnologyComponent &tc) { @@ -129,6 +128,7 @@ template struct FallbackXMLReadAdaptor { typedef tl::pass_by_ref_tag tag; + typedef tl::many_cardinality_tag cardinality; FallbackXMLReadAdaptor (Iter (db::NetTracerConnectivity::*begin) () const, Iter (db::NetTracerConnectivity::*end) () const) : mp_begin (begin), mp_end (end) @@ -191,14 +191,14 @@ public: tl::make_element ((NetTracerTechnologyComponent::const_iterator (NetTracerTechnologyComponent::*) () const) &NetTracerTechnologyComponent::begin, (NetTracerTechnologyComponent::const_iterator (NetTracerTechnologyComponent::*) () const) &NetTracerTechnologyComponent::end, (void (NetTracerTechnologyComponent::*) (const NetTracerConnectivity &)) &NetTracerTechnologyComponent::push_back, "stack", tl::make_member (&NetTracerConnectivity::name, &NetTracerConnectivity::set_name, "name") + tl::make_member (&NetTracerConnectivity::description, &NetTracerConnectivity::set_description, "description") + - tl::make_member ((NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin, (NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end, &NetTracerConnectivity::add, "connection") + - tl::make_member ((NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin_symbols, (NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end_symbols, &NetTracerConnectivity::add_symbol, "symbols") + tl::make_member ((NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin, (NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end, &NetTracerConnectivity::add, "connection", tl::XMLStringBasedConverter ()) + + tl::make_member ((NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin_symbols, (NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end_symbols, &NetTracerConnectivity::add_symbol, "symbols", tl::XMLStringBasedConverter ()) ) + // Fallback readers for migrating pre-0.28 setups to 0.28 and backward compatibility - tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStdConverter > ( + tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStringBasedConverter > ( FallbackXMLReadAdaptor (&NetTracerConnectivity::begin, &NetTracerConnectivity::end), FallbackXMLWriteAdaptor (&NetTracerConnectivity::add), "connection") + - tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStdConverter > ( + tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStringBasedConverter > ( FallbackXMLReadAdaptor (&NetTracerConnectivity::begin_symbols, &NetTracerConnectivity::end_symbols), FallbackXMLWriteAdaptor (&NetTracerConnectivity::add_symbol), "symbols") ); diff --git a/src/rdb/rdb/gsiDeclRdb.cc b/src/rdb/rdb/gsiDeclRdb.cc index d2db8c0d1..c3610bb11 100644 --- a/src/rdb/rdb/gsiDeclRdb.cc +++ b/src/rdb/rdb/gsiDeclRdb.cc @@ -1733,10 +1733,12 @@ Class decl_ReportDatabase ("rdb", "ReportDatabase", "The reader recognizes the format automatically and will choose the appropriate decoder. 'gzip' compressed files are uncompressed " "automatically.\n" ) + - gsi::method ("save", &rdb::Database::save, gsi::arg ("filename"), + gsi::method ("save", &rdb::Database::save, gsi::arg ("filename"), gsi::arg ("binary", false), "@brief Saves the database to the given file\n" "@param filename The file to which to save the database\n" - "The database is always saved in KLayout's XML-based format.\n" + "The database is saved in KLayout's XML-based format if 'binary' is false or a ProtocolBuffer-based binary format if 'binary' is true.\n" + "\n" + "The 'binary' argument has been added in version 0.29.7." ), "@brief The report database object\n" "A report database is organized around a set of items which are associated with cells and categories. " diff --git a/src/rdb/rdb/rdb.cc b/src/rdb/rdb/rdb.cc index bc7c9c841..8079d900f 100644 --- a/src/rdb/rdb/rdb.cc +++ b/src/rdb/rdb/rdb.cc @@ -278,7 +278,7 @@ template <> RDB_PUBLIC bool Value::is_shape () const } -bool ValueBase::compare (const ValueBase *a, const ValueBase *b) +bool ValueBase::compare (const ValueBase *a, const ValueBase *b) { // compare is the intrinsic compare of equal type and type index for different types. if (a->type_index () == b->type_index ()) { @@ -416,19 +416,19 @@ ValueBase::create_from_shape (const db::Shape &shape, const db::CplxTrans &trans // ------------------------------------------------------------------------------------------ // ValueWrapper implementation -std::string -ValueWrapper::to_string (const Database *rdb) const +std::string +ValueWrapper::to_string () const { std::string r; r.reserve (200); - if (tag_id () > 0 && rdb) { + if (tag_id () > 0) { r += "["; - const Tag &tag = rdb->tags ().tag (tag_id ()); - if (tag.is_user_tag ()) { + auto np = rdb::Tags::name_for_id (tag_id ()); + if (np.second) { r += "#"; } - r += tl::to_word_or_quoted_string (tag.name ()); + r += tl::to_word_or_quoted_string (np.first); r += "] "; } @@ -437,15 +437,15 @@ ValueWrapper::to_string (const Database *rdb) const return r; } -void -ValueWrapper::from_string (Database *rdb, const std::string &s) +void +ValueWrapper::from_string (const std::string &s) { tl::Extractor ex (s.c_str ()); - from_string (rdb, ex); + from_string (ex); } -void -ValueWrapper::from_string (Database *rdb, tl::Extractor &ex) +void +ValueWrapper::from_string (tl::Extractor &ex) { id_type tag_id = 0; @@ -453,10 +453,10 @@ ValueWrapper::from_string (Database *rdb, tl::Extractor &ex) bool user_tag = ex.test ("#"); - std::string tn; + std::string tn; ex.read_word_or_quoted (tn); - tag_id = rdb->tags ().tag (tn, user_tag).id (); + tag_id = rdb::Tags::id_for_name (tn, user_tag); ex.test ("]"); @@ -477,16 +477,16 @@ Values::operator= (const Values &d) } bool -Values::compare (const Values &other, const std::map &tag_map, const std::map &rev_tag_map) const +Values::compare (const Values &other, const std::set &common_tags) const { Values::const_iterator a = begin (), b = other.begin (); while (a != end () && b != other.end ()) { - id_type t12 = 0; + id_type t1 = 0; while (a != end () && a->tag_id () != 0) { - auto j = tag_map.find (a->tag_id ()); - if (j != tag_map.end ()) { - t12 = j->second; + auto j = common_tags.find (a->tag_id ()); + if (j != common_tags.end ()) { + t1 = a->tag_id (); break; } ++a; @@ -494,9 +494,9 @@ Values::compare (const Values &other, const std::map &tag_map, id_type t2 = 0; while (b != other.end () && b->tag_id () != 0) { - auto j = rev_tag_map.find (b->tag_id ()); - if (j != rev_tag_map.end ()) { - t2 = j->first; + auto j = common_tags.find (b->tag_id ()); + if (j != common_tags.end ()) { + t2 = b->tag_id (); break; } ++b; @@ -506,8 +506,8 @@ Values::compare (const Values &other, const std::map &tag_map, return b != other.end (); } - if (t12 != t2) { - return t12 < t2; + if (t1 != t2) { + return t1 < t2; } if (a->get () && b->get ()) { @@ -528,8 +528,8 @@ Values::compare (const Values &other, const std::map &tag_map, return b != other.end (); } -std::string -Values::to_string (const Database *rdb) const +std::string +Values::to_string () const { std::string r; r.reserve (200); @@ -540,22 +540,22 @@ Values::to_string (const Database *rdb) const r += ";"; } - r += v->to_string (rdb); + r += v->to_string (); } return r; } -void -Values::from_string (Database *rdb, const std::string &s) +void +Values::from_string (const std::string &s) { tl::Extractor ex (s.c_str ()); while (! ex.at_end ()) { ValueWrapper v; - v.from_string (rdb, ex); + v.from_string (ex); add (v); @@ -567,7 +567,7 @@ Values::from_string (Database *rdb, const std::string &s) // ------------------------------------------------------------------------------------------ // Cell implementation -void +void Cells::import_cell (const Cell &c) { Cell *cell; @@ -582,7 +582,7 @@ Cells::import_cell (const Cell &c) cell->references ().insert (*r); } } - + // ------------------------------------------------------------------------------------------ // Cell implementation @@ -615,7 +615,7 @@ Cell::~Cell () // .. nothing yet .. } -std::pair +std::pair Cell::path_to (id_type parent_cell_id, const Database *db) const { if (parent_cell_id == id ()) { @@ -626,7 +626,7 @@ Cell::path_to (id_type parent_cell_id, const Database *db) const } } -std::pair +std::pair Cell::path_to (id_type parent_cell_id, const Database *db, std::set &visited, const db::DCplxTrans &trans) const { for (reference_iterator r = references ().begin (); r != references ().end (); ++r) { @@ -654,15 +654,15 @@ Cell::path_to (id_type parent_cell_id, const Database *db, std::set &v return std::pair (false, db::DCplxTrans ()); } -void +void Cell::import_references (const References &references) { for (References::const_iterator r = references.begin (); r != references.end (); ++r) { m_references.insert (*r); } } - -std::string + +std::string Cell::qname () const { if (m_variant.empty ()) { @@ -687,7 +687,7 @@ References::References (Cell *cell) // .. nothing yet .. } -void +void References::set_database (Database *database) { mp_database = database; @@ -705,20 +705,20 @@ Reference::Reference (References *references) // .. nothing yet .. } -void +void Reference::set_trans_str (const std::string &s) { tl::Extractor ex (s.c_str ()); ex.read (m_trans); } -std::string +std::string Reference::trans_str () const { return m_trans.to_string (); } -void +void Reference::set_parent_cell_qname (const std::string &qname) { tl_assert (mp_database != 0); @@ -729,7 +729,7 @@ Reference::set_parent_cell_qname (const std::string &qname) m_parent_cell_id = cell->id (); } -std::string +std::string Reference::parent_cell_qname () const { tl_assert (mp_database != 0); @@ -787,7 +787,7 @@ Category::sub_categories () return *mp_sub_categories; } -std::string +std::string Category::path () const { std::vector path_elements; @@ -810,7 +810,7 @@ Category::path () const return r; } -void +void Category::import_sub_categories (Categories *categories) { if (mp_sub_categories) { @@ -826,7 +826,7 @@ Category::import_sub_categories (Categories *categories) } } -void +void Category::set_database (Database *database) { mp_database = database; @@ -838,14 +838,14 @@ Category::set_database (Database *database) // ------------------------------------------------------------------------------------------ // Categories implementation -void +void Categories::clear () { m_categories.clear (); m_categories_by_name.clear (); } -void +void Categories::add_category (Category *cat) { cat->set_database (mp_database.get ()); @@ -853,7 +853,7 @@ Categories::add_category (Category *cat) m_categories_by_name.insert (std::make_pair (cat->name (), cat)); } -void +void Categories::set_database (Database *database) { mp_database = database; @@ -863,7 +863,7 @@ Categories::set_database (Database *database) } Category * -Categories::category_by_name (const char *path) +Categories::category_by_name (const char *path) { std::string component; tl::Extractor ex (path); @@ -912,18 +912,71 @@ Categories::import_category (Category *category) // ------------------------------------------------------------------------------------------ // Tags implementation +static tl::Mutex s_tag_namespace_mutex; +static std::map, id_type> s_tag_namespace; +static std::vector > s_tags; + +const std::pair & +Tags::name_for_id (id_type tag_id) +{ + tl::MutexLocker locker (&s_tag_namespace_mutex); + tl_assert (tag_id > 0 && tag_id <= s_tags.size ()); + return s_tags [tag_id - 1]; +} + +id_type +Tags::id_for_name (const std::string &name, bool user_flag) +{ + tl::MutexLocker locker (&s_tag_namespace_mutex); + + auto k = std::make_pair (name, user_flag); + auto t = s_tag_namespace.find (k); + if (t != s_tag_namespace.end ()) { + return t->second; + } + + id_type id = s_tags.size () + 1; + s_tag_namespace [k] = id; + s_tags.push_back (k); + return id; +} + +Tag +Tags::tag_for_name (const std::string &name, bool user_flag) +{ + return Tag (id_for_name (name, user_flag), name, user_flag); +} + Tags::Tags () { // .. nothing yet .. } -void +void Tags::clear () { - m_ids_for_names.clear (); + m_tags_per_id.clear (); m_tags.clear (); } +void +Tags::remove_tag (id_type id) +{ + auto j = m_tags_per_id.find (id); + if (j != m_tags_per_id.end ()) { + + m_tags.erase (m_tags.begin () + j->second); + + for (auto i = m_tags_per_id.begin (); i != m_tags_per_id.end (); ++i) { + if (i->second > j->second) { + i->second -= 1; + } + } + m_tags_per_id.erase (id); + + } +} + const Tag & Tags::tag (const std::string &name, bool user_tag) const { @@ -933,26 +986,31 @@ Tags::tag (const std::string &name, bool user_tag) const Tag & Tags::tag (const std::string &name, bool user_tag) { - std::map , id_type>::const_iterator i = m_ids_for_names.find (std::make_pair (name, user_tag)); - if (i == m_ids_for_names.end ()) { - i = m_ids_for_names.insert (std::make_pair (std::make_pair (name, user_tag), m_tags.size () + 1)).first; - m_tags.push_back (Tag (i->second, name, user_tag)); + id_type id = id_for_name (name, user_tag); + auto i = m_tags_per_id.find (id); + if (i == m_tags_per_id.end ()) { + m_tags.push_back (Tag (id, name, user_tag)); + m_tags_per_id [id] = m_tags.size () - 1; + return m_tags.back (); + } else { + return m_tags [i->second]; } - return m_tags [i->second - 1]; } const Tag & Tags::tag (id_type id) const { - tl_assert (id < m_tags.size () + 1 && id > 0); - return m_tags [id - 1]; + auto i = m_tags_per_id.find (id); + tl_assert (i != m_tags_per_id.end ()); + return m_tags [i->second]; } Tag & Tags::tag (id_type id) { - tl_assert (id < m_tags.size () + 1 && id > 0); - return m_tags [id - 1]; + auto i = m_tags_per_id.find (id); + tl_assert (i != m_tags_per_id.end ()); + return m_tags [i->second]; } void @@ -962,10 +1020,10 @@ Tags::import_tag (const Tag &t) tt.set_description (t.description ()); } -bool +bool Tags::has_tag (const std::string &name, bool user_tag) const { - return m_ids_for_names.find (std::make_pair (name, user_tag)) != m_ids_for_names.end (); + return m_tags_per_id.find (id_for_name (name, user_tag)) != m_tags_per_id.end (); } // ------------------------------------------------------------------------------------------ @@ -1010,36 +1068,31 @@ Item &Item::operator= (const Item &d) return *this; } -void +void Item::add_tag (id_type tag_id) { - if (m_tag_ids.size () <= tag_id) { - m_tag_ids.resize (tag_id + 1, false); - } - m_tag_ids [tag_id] = true; + m_tag_ids.insert (tag_id); } -void +void Item::remove_tag (id_type tag_id) { - if (m_tag_ids.size () > tag_id) { - m_tag_ids [tag_id] = false; - } + m_tag_ids.erase (tag_id); } -void +void Item::remove_tags () { - m_tag_ids = std::vector (); + m_tag_ids.clear (); } -bool +bool Item::has_tag (id_type tag_id) const { - return m_tag_ids.size () > tag_id && m_tag_ids [tag_id]; + return m_tag_ids.find (tag_id) != m_tag_ids.end (); } -std::string +std::string Item::cell_qname () const { tl_assert (mp_database != 0); @@ -1048,7 +1101,7 @@ Item::cell_qname () const return cell->qname (); } -void +void Item::set_cell_qname (const std::string &qname) { tl_assert (mp_database != 0); @@ -1068,7 +1121,7 @@ Item::category_name () const return category->path (); } -void +void Item::set_category_name (const std::string &path) { tl_assert (mp_database != 0); @@ -1079,7 +1132,7 @@ Item::set_category_name (const std::string &path) m_category_id = category->id (); } -std::string +std::string Item::tag_str () const { tl_assert (mp_database != 0); @@ -1089,13 +1142,12 @@ Item::tag_str () const if (! m_tag_ids.empty ()) { - id_type tag_id = 0; - for (std::vector::const_iterator t = m_tag_ids.begin (); t != m_tag_ids.end (); ++t, ++tag_id) { + for (std::set::const_iterator t = m_tag_ids.begin (); t != m_tag_ids.end (); ++t) { if (*t) { if (! r.empty ()) { r += ","; } - const Tag &tag = mp_database->tags ().tag (tag_id); + const Tag &tag = mp_database->tags ().tag (*t); if (tag.is_user_tag ()) { r += "#"; } @@ -1108,7 +1160,7 @@ Item::tag_str () const return r; } -void +void Item::set_tag_str (const std::string &tags) { tl_assert (mp_database != 0); @@ -1126,7 +1178,7 @@ Item::set_tag_str (const std::string &tags) } #if defined(HAVE_QT) -void +void Item::set_image (const QImage &image) { if (image.isNull ()) { @@ -1314,7 +1366,7 @@ Database::import_tags (const Tags &tags) } } -void +void Database::import_categories (Categories *categories) { set_modified (); @@ -1325,7 +1377,7 @@ Database::import_categories (Categories *categories) mp_categories->set_database (this); } -void +void Database::import_cells (const Cells &cells) { set_modified (); @@ -1377,13 +1429,13 @@ Database::create_category (Categories *container, const std::string &name) } Category * -Database::category_by_name_non_const (const std::string &name) +Database::category_by_name_non_const (const std::string &name) { return mp_categories->category_by_name (name.c_str ()); } Category * -Database::category_by_id_non_const (id_type id) +Database::category_by_id_non_const (id_type id) { std::map ::const_iterator c = m_categories_by_id.find (id); if (c != m_categories_by_id.end ()) { @@ -1438,7 +1490,7 @@ Database::create_cell (const std::string &name, const std::string &variant, cons } else { new_cell = new Cell (++m_next_id, name, std::string (), layout_name); - + } m_cells.add_cell (new_cell); @@ -1446,7 +1498,7 @@ Database::create_cell (const std::string &name, const std::string &variant, cons m_cells_by_qname.insert (std::make_pair (new_cell->qname (), new_cell)); } else { - + new_cell = new Cell (++m_next_id, name, variant, layout_name); m_cells.add_cell (new_cell); m_cells_by_id.insert (std::make_pair (new_cell->id (), new_cell)); @@ -1474,7 +1526,7 @@ Database::create_cell (const std::string &name, const std::string &variant, cons } other_cell->set_variant (tl::to_string (variant_index + 1)); - + m_cells_by_qname.insert (std::make_pair (other_cell->qname (), other_cell)); } @@ -1520,21 +1572,21 @@ Database::cell_by_id_non_const (id_type id) } } -void +void Database::set_tag_description (id_type tag_id, const std::string &description) { set_modified (); tags_non_const ().tag (tag_id).set_description (description); } -void +void Database::add_item_tag (const Item *item, id_type tag) { set_modified (); const_cast (item)->add_tag (tag); } -void +void Database::remove_item_tag (const Item *item, id_type tag) { set_modified (); @@ -1549,7 +1601,7 @@ Database::set_item_comment (const Item *item, const std::string &comment) } #if defined(HAVE_QT) -void +void Database::set_item_image (const Item *item, const QImage &image) { set_modified (); @@ -1564,14 +1616,14 @@ Database::set_item_image_str (const Item *item, const std::string &image_str) const_cast (item)->set_image_str (image_str); } -void +void Database::set_item_multiplicity (const Item *item, size_t n) { set_modified (); const_cast (item)->set_multiplicity (n); } -void +void Database::set_item_visited (const Item *item_c, bool visited) { if (visited != item_c->visited ()) { @@ -1632,7 +1684,7 @@ Database::create_item (id_type cell_id, id_type category_id) static std::list empty_list; -std::pair +std::pair Database::items_by_cell_and_category (id_type cell_id, id_type category_id) const { std::map , std::list >::const_iterator i = m_items_by_cell_and_category_id.find (std::make_pair (cell_id, category_id)); @@ -1709,7 +1761,7 @@ Database::num_items (id_type cell_id, id_type category_id) const } } -size_t +size_t Database::num_items_visited (id_type cell_id, id_type category_id) const { std::map , size_t>::const_iterator n = m_num_items_visited_by_cell_and_category.find (std::make_pair (cell_id, category_id)); @@ -1727,6 +1779,7 @@ Database::clear () m_generator = ""; m_filename = ""; + m_binary = false; m_description = ""; m_name = ""; m_topcell = ""; @@ -1779,6 +1832,7 @@ Database::load (std::string fn) tl::log << "Loading RDB from " << fn; clear (); + set_binary (false); tl::InputStream stream (fn); @@ -1813,33 +1867,30 @@ namespace class ValueMapEntryCompare { public: - ValueMapEntryCompare (const std::map &tag2tag, const std::map &rev_tag2tag) + ValueMapEntryCompare (const std::set &common_tags) { - mp_tag2tag = &tag2tag; - mp_rev_tag2tag = &rev_tag2tag; + mp_common_tags = &common_tags; } bool operator() (const Item *a, const Item *b) const { - return a->values ().compare (b->values (), *mp_tag2tag, *mp_rev_tag2tag); + return a->values ().compare (b->values (), *mp_common_tags); } private: - const std::map *mp_tag2tag; - const std::map *mp_rev_tag2tag; + const std::set *mp_common_tags; }; class ValueMapEntry { public: ValueMapEntry () - : mp_tag2tag (0), mp_rev_tag2tag (0) + : mp_common_tags (0) { } - void build (const rdb::Database &rdb, id_type cell_id, id_type cat_id, const std::map &tag2tag, const std::map &rev_tag2tag) + void build (const rdb::Database &rdb, id_type cell_id, id_type cat_id, const std::set &common_tags) { - mp_tag2tag = &tag2tag; - mp_rev_tag2tag = &rev_tag2tag; + mp_common_tags = &common_tags; auto i2i = rdb.items_by_cell_and_category (cell_id, cat_id); @@ -1853,13 +1904,13 @@ namespace m_items.push_back ((*i).operator-> ()); } - ValueMapEntryCompare cmp (*mp_tag2tag, *mp_rev_tag2tag); + ValueMapEntryCompare cmp (*mp_common_tags); std::sort (m_items.begin (), m_items.end (), cmp); } const Item *find (const rdb::Item &item) const { - ValueMapEntryCompare cmp (*mp_tag2tag, *mp_rev_tag2tag); + ValueMapEntryCompare cmp (*mp_common_tags); auto i = std::lower_bound (m_items.begin (), m_items.end (), &item, cmp); if (i == m_items.end ()) { @@ -1875,8 +1926,7 @@ namespace public: std::vector m_items; - const std::map *mp_tag2tag; - const std::map *mp_rev_tag2tag; + const std::set *mp_common_tags; }; } @@ -1897,8 +1947,6 @@ Database::apply (const rdb::Database &other) { std::map cell2cell; std::map cat2cat; - std::map tag2tag; - std::map rev_tag2tag; for (auto c = other.cells ().begin (); c != other.cells ().end (); ++c) { // TODO: do we have a consistent scheme of naming variants? What requirements @@ -1913,16 +1961,15 @@ Database::apply (const rdb::Database &other) map_category (*c, *this, cat2cat); } - std::map tags_by_name; + std::set t1; for (auto c = tags ().begin_tags (); c != tags ().end_tags (); ++c) { - tags_by_name.insert (std::make_pair (c->name (), c->id ())); + t1.insert (c->id ()); } + std::set common_tags; for (auto c = other.tags ().begin_tags (); c != other.tags ().end_tags (); ++c) { - auto t = tags_by_name.find (c->name ()); - if (t != tags_by_name.end ()) { - tag2tag.insert (std::make_pair (t->second, c->id ())); - rev_tag2tag.insert (std::make_pair (c->id (), t->second)); + if (t1.find (c->id ()) != t1.end ()) { + common_tags.insert (c->id ()); } } @@ -1944,7 +1991,7 @@ Database::apply (const rdb::Database &other) auto vmap = value_map.find (std::make_pair (icell->second, icat->second)); if (vmap == value_map.end ()) { vmap = value_map.insert (std::make_pair (std::make_pair (icell->second, icat->second), ValueMapEntry ())).first; - vmap->second.build (other, icell->second, icat->second, tag2tag, rev_tag2tag); + vmap->second.build (other, icell->second, icat->second, common_tags); } // find a value in the reference DB diff --git a/src/rdb/rdb/rdb.h b/src/rdb/rdb/rdb.h index b5c4e98a8..1d8ce52a8 100644 --- a/src/rdb/rdb/rdb.h +++ b/src/rdb/rdb/rdb.h @@ -69,10 +69,10 @@ class Items; /** * @brief A report item's category * - * An item is member of exactly one category. This can be a check for example. + * An item is member of exactly one category. This can be a check for example. * A category is described by a name and a description string. An Id is provided * to reference this category from actual report items. - * Categories can be organized hierarchically for which a category collection + * Categories can be organized hierarchically for which a category collection * is provided and member of the individual category. * * A category can only be created by the database object, since the @@ -113,7 +113,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_id (id_type id) + void set_id (id_type id) { m_id = id; } @@ -131,7 +131,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_name (const std::string &d) + void set_name (const std::string &d) { m_name = d; } @@ -156,7 +156,7 @@ public: /** * @brief The description string (setter) */ - void set_description (const std::string &d) + void set_description (const std::string &d) { m_description = d; } @@ -182,7 +182,7 @@ public: /** * @brief The parent (owner) of this category (getter) */ - Category *parent () + Category *parent () { return mp_parent; } @@ -193,7 +193,7 @@ public: * This method is provided for persistency application only. It should not be used otherwise. * The Category object will take over ownership over the sub categories. */ - void import_sub_categories (Categories *categories); + void import_sub_categories (Categories *categories); /** * @brief Report the number of items @@ -243,7 +243,7 @@ private: * @brief Default constructor * * Creates a category object with empty name and description. - * This constructor is private to allow only the database to produce a + * This constructor is private to allow only the database to produce a * category object. */ Category (const std::string &name); @@ -263,9 +263,9 @@ private: { m_num_items_visited += d; } - + /** - * @brief Add an offset to the number of items + * @brief Add an offset to the number of items */ void add_to_num_items (int d) { @@ -280,7 +280,7 @@ private: m_num_items = 0; m_num_items_visited = 0; } - + /** * @brief Set the database reference */ @@ -294,10 +294,10 @@ private: /** * @brief The collection of categories * - * A generic collection of categories used for the root node and + * A generic collection of categories used for the root node and * sub-category nodes. */ -class RDB_PUBLIC Categories +class RDB_PUBLIC Categories { public: typedef tl::shared_collection::const_iterator const_iterator; @@ -328,39 +328,39 @@ public: /** * @brief Iterate the categories inside this collection (begin iterator) */ - const_iterator begin () const - { - return m_categories.begin (); + const_iterator begin () const + { + return m_categories.begin (); } /** * @brief Iterate the categories inside this collection (end iterator) */ - const_iterator end () const - { - return m_categories.end (); + const_iterator end () const + { + return m_categories.end (); } /** * @brief Iterate the categories inside this collection (begin iterator) */ - iterator begin () - { - return m_categories.begin (); + iterator begin () + { + return m_categories.begin (); } /** * @brief Iterate the categories inside this collection (end iterator) */ - iterator end () - { - return m_categories.end (); + iterator end () + { + return m_categories.end (); } /** * @brief Find a category by name * - * The name is actually a path expression which specifies the category starting from the given + * The name is actually a path expression which specifies the category starting from the given * node with a '.' separated path notation. I.e. 'a.b' is the sub-category 'b' of category 'a'. * If no such category can be found, 0 is returned. */ @@ -372,7 +372,7 @@ public: /** * @brief Find a category by name (non-const version) * - * The name is actually a path expression which specifies the category starting from the given + * The name is actually a path expression which specifies the category starting from the given * node with a '.' separated path notation. I.e. 'a.b' is the sub-category 'b' of category 'a'. * If no such category can be found, 0 is returned. */ @@ -389,7 +389,7 @@ public: * This method is provided for persistency application only. It should not be used otherwise. * This will take over ownership over the category. */ - void import_category (Category *category); + void import_category (Category *category); /** * @brief Gets the database reference @@ -404,7 +404,7 @@ public: friend class Category; tl::shared_collection m_categories; - std::map m_categories_by_name; + std::map m_categories_by_name; tl::weak_ptr mp_database; Categories () @@ -423,7 +423,7 @@ public: * A value has a value (as the name says) and an optional tag id. Tag id's identify * the value and make the value a named one. */ -class RDB_PUBLIC ValueBase +class RDB_PUBLIC ValueBase { public: ValueBase () @@ -497,14 +497,14 @@ public: return m_value; } - C &value () + C &value () { return m_value; } - int type_index () const - { - return type_index_of (); + int type_index () const + { + return type_index_of (); } bool compare (const ValueBase *other) const; @@ -545,7 +545,7 @@ RDB_PUBLIC_TEMPLATE ValueBase *make_value (const db::object_with_properties & /** * @brief A class encapsulating ValueBase pointer */ -class RDB_PUBLIC ValueWrapper +class RDB_PUBLIC ValueWrapper { public: /** @@ -602,7 +602,7 @@ public: /** * @brief Get the pointer */ - ValueBase *get () + ValueBase *get () { return mp_ptr; } @@ -637,19 +637,19 @@ public: } /** - * @brief Convert the values collection to a string + * @brief Convert the values collection to a string */ - std::string to_string (const Database *rdb = 0) const; + std::string to_string () const; /** * @brief Fill the values collection from the string */ - void from_string (Database *rdb, const std::string &s); + void from_string (const std::string &s); /** * @brief Fill the values collection from an extractor */ - void from_string (Database *rdb, tl::Extractor &ex); + void from_string (tl::Extractor &ex); private: ValueBase *mp_ptr; @@ -682,22 +682,19 @@ public: } /** - * @brief Assignment + * @brief Assignment */ Values &operator= (const Values &d); /** * @brief Compare two value sets (less operator) * - * This compare function will use the tag mapping provided by tag map ("this" tag id to "other" tag id). - * Values with tags not listed in the tag map will not be compared. + * This compare function will use the tags provide in "common_tags". Tags outside this set are ignored. * Untagged values (tag_id 0) will be compared always. * - * "rev_tag_map" needs to be the reverse of "tag_map". - * * The order of the values matters. */ - bool compare (const Values &other, const std::map &tag_map, const std::map &rev_tag_map) const; + bool compare (const Values &other, const std::set &common_tags) const; /** * @brief The const iterator (begin) @@ -718,7 +715,7 @@ public: /** * @brief The non-const iterator (begin) */ - iterator begin () + iterator begin () { return m_values.begin (); } @@ -768,14 +765,14 @@ public: } /** - * @brief Convert the values collection to a string + * @brief Convert the values collection to a string */ - std::string to_string (const Database *rdb) const; + std::string to_string () const; /** * @brief Fill the values collection from the string */ - void from_string (Database *rdb, const std::string &s); + void from_string (const std::string &s); private: std::list m_values; @@ -784,8 +781,8 @@ private: /** * @brief A report item * - * A report item is one information item in the report. - * The value of a report item is manyfold. Values can be keyed, + * A report item is one information item in the report. + * The value of a report item is manyfold. Values can be keyed, * i.e. multiple values can be present with different keys. * Each value can be of different types where the type is specified by a type Id. */ @@ -799,12 +796,12 @@ public: Item (Items *items); /** - * @brief Copy constructor + * @brief Copy constructor */ Item (const Item &d); /** - * @brief Assignment + * @brief Assignment */ Item &operator= (const Item &d); @@ -826,7 +823,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_cell_id (id_type id) + void set_cell_id (id_type id) { m_cell_id = id; } @@ -856,7 +853,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_category_id (id_type id) + void set_category_id (id_type id) { m_category_id = id; } @@ -1067,7 +1064,7 @@ private: size_t m_multiplicity; std::string m_comment; bool m_visited; - std::vector m_tag_ids; + std::set m_tag_ids; Database *mp_database; std::string m_image_str; @@ -1083,9 +1080,9 @@ private: }; /** - * @brief An item reference + * @brief An item reference * - * This is basically a wrapper for a pointer that correctly + * This is basically a wrapper for a pointer that correctly * maps const * and non-const * values through the operator-> * overloads. */ @@ -1151,7 +1148,7 @@ public: /** * @brief Construct an item list with a database reference */ - Items (Database *database) + Items (Database *database) : mp_database (database) { // .. nothing yet .. @@ -1160,33 +1157,33 @@ public: /** * @brief Iterate the items inside this collection (begin iterator) */ - const_iterator begin () const - { - return m_items.begin (); + const_iterator begin () const + { + return m_items.begin (); } /** * @brief Iterate the items inside this collection (end iterator) */ - const_iterator end () const - { - return m_items.end (); + const_iterator end () const + { + return m_items.end (); } /** * @brief Iterate the items inside this collection (non-const begin iterator) */ iterator begin () - { - return m_items.begin (); + { + return m_items.begin (); } /** * @brief Iterate the items inside this collection (non-const end iterator) */ iterator end () - { - return m_items.end (); + { + return m_items.end (); } /** @@ -1205,7 +1202,7 @@ public: * * This method is provided for persistency application only. It should not be used otherwise. */ - Item &back () + Item &back () { return m_items.back (); } @@ -1213,7 +1210,7 @@ public: /** * @brief Get the database reference */ - Database *database () + Database *database () { return mp_database; } @@ -1360,7 +1357,7 @@ private: }; /** - * @brief A collection of references + * @brief A collection of references */ class RDB_PUBLIC References { @@ -1403,7 +1400,7 @@ public: /** * @brief Begin iterator (non-const) */ - iterator begin () + iterator begin () { return m_references.begin (); } @@ -1411,7 +1408,7 @@ public: /** * @brief End iterator (non-const) */ - iterator end () + iterator end () { return m_references.end (); } @@ -1427,7 +1424,7 @@ public: /** * @brief Get the database reference */ - Database *database () + Database *database () { return mp_database; } @@ -1510,7 +1507,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_id (id_type id) + void set_id (id_type id) { m_id = id; } @@ -1528,7 +1525,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_name (const std::string &d) + void set_name (const std::string &d) { m_name = d; } @@ -1546,7 +1543,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_variant (const std::string &v) + void set_variant (const std::string &v) { m_variant = v; } @@ -1601,7 +1598,7 @@ public: /** * @brief The reference collection (non-const) */ - References &references () + References &references () { return m_references; } @@ -1614,10 +1611,10 @@ public: void import_references (const References &references); /** - * @brief Get one example transformation leading from this cell to a given parent cell + * @brief Get one example transformation leading from this cell to a given parent cell * * This method will try to determine one path from the given cell to the given parent - * cell and return the accumulated transformation for this path. + * cell and return the accumulated transformation for this path. * If no path is found, the first parameter of the returned pair is false, otherwise it's * true. */ @@ -1626,7 +1623,7 @@ public: /** * @brief Get the database reference */ - Database *database () + Database *database () { return mp_database; } @@ -1664,7 +1661,7 @@ private: } /** - * @brief Add an offset to the number of items + * @brief Add an offset to the number of items */ void add_to_num_items (int d) { @@ -1679,7 +1676,7 @@ private: m_num_items = 0; m_num_items_visited = 0; } - + /** * @brief Set the database reference */ @@ -1693,7 +1690,7 @@ private: }; /** - * @brief A collection of cells + * @brief A collection of cells */ class RDB_PUBLIC Cells { @@ -1753,7 +1750,7 @@ public: /** * @brief Begin iterator (non-const) */ - iterator begin () + iterator begin () { return m_cells.begin (); } @@ -1761,7 +1758,7 @@ public: /** * @brief End iterator (non-const) */ - iterator end () + iterator end () { return m_cells.end (); } @@ -1794,7 +1791,7 @@ public: /** * @brief Get the database reference (non-const) */ - Database *database () + Database *database () { return mp_database.get (); } @@ -1858,9 +1855,9 @@ public: /** * @brief Gets a flag indicating whether the tag is a user tag or a system tag - * + * * If this flag is false, the tag is a system tag used for tagging "waived" and - * similar conditions. Otherwise it is a user tag which can be used freely to + * similar conditions. Otherwise it is a user tag which can be used freely to * tag arbitrary conditions. */ bool is_user_tag () const @@ -1873,7 +1870,7 @@ public: * * See \is_user_tag for details. */ - void set_user_tag (bool user) + void set_user_tag (bool user) { m_is_user_tag = user; } @@ -1891,7 +1888,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_id (id_type id) + void set_id (id_type id) { m_id = id; } @@ -1910,7 +1907,7 @@ public: * The name of the tag must not be changed when the tag is already part of a Tags collection. * Otherwise, the tag collection becomes inconsistent. */ - void set_name (const std::string &name) + void set_name (const std::string &name) { m_name = name; } @@ -1931,7 +1928,7 @@ public: typedef tag_list_type::const_iterator const_iterator; /** - * @brief Default constructor for the tags list + * @brief Default constructor for the tags list * * This method is provided for persistency application only. It should not be used otherwise. */ @@ -1982,27 +1979,59 @@ public: * @brief Import a tag * * This method is provided for persistency application only. It should not be used otherwise. - * This will assign a new id to the tag and replace any tag with that + * This will assign a new id to the tag and replace any tag with that * name. */ void import_tag (const Tag &tag); /** * @brief Clear the collection of tags + * + * NOTE: this will not remove the tags from items or values, so the use cases for this method + * are limited. */ void clear (); + /** + * @brief Removes the tag with the given ID + * + * NOTE: this will not remove the tags from items or values, so the use cases for this method + * are limited. + */ + void remove_tag (id_type id); + + /** + * @brief Gets the name and user flag for a tag ID + * + * This method will assert if the tag ID is not valid in the global namespace. + */ + static const std::pair &name_for_id (id_type tag_id); + + /** + * @brief Gets the id for a given name and user flag + * + * This will pull the ID from the global namespace. + */ + static id_type id_for_name (const std::string &name, bool user_flag); + + /** + * @brief Gets a tag object for a given name and user flag + * + * This will create the tag from the global namespace. + */ + static Tag tag_for_name (const std::string &name, bool user_flag); + private: friend class Database; - mutable std::map , id_type> m_ids_for_names; - mutable std::vector m_tags; + mutable std::map m_tags_per_id; + mutable std::vector m_tags; }; /** * @brief The database object */ -class RDB_PUBLIC Database +class RDB_PUBLIC Database : public gsi::ObjectBase, public tl::Object { @@ -2025,7 +2054,7 @@ public: ~Database (); /** - * @brief Get the database description + * @brief Gets the database description */ const std::string &description () const { @@ -2033,7 +2062,7 @@ public: } /** - * @brief Set the database description + * @brief Sets the database description */ void set_description (const std::string &description) { @@ -2042,7 +2071,7 @@ public: } /** - * @brief Get the database original file + * @brief Gets the database original file * * The original file describes what original file the marker database * was derived from. @@ -2053,7 +2082,7 @@ public: } /** - * @brief Set the database original file + * @brief Sets the database original file */ void set_original_file (const std::string &original_file) { @@ -2062,7 +2091,7 @@ public: } /** - * @brief Get the database name + * @brief Gets the database name */ const std::string &name () const { @@ -2070,7 +2099,7 @@ public: } /** - * @brief Set the database name + * @brief Sets the database name */ void set_name (const std::string &name) { @@ -2078,7 +2107,7 @@ public: } /** - * @brief Get the file name + * @brief Gets the file name */ const std::string &filename () const { @@ -2086,7 +2115,7 @@ public: } /** - * @brief Set the file name + * @brief Sets the file name */ void set_filename (const std::string &filename) { @@ -2095,7 +2124,24 @@ public: } /** - * @brief Get the generator name + * @brief Gets a value indicating whether the database was saved to binary format + */ + bool binary () const + { + return m_binary; + } + + /** + * @brief Sets a value indicating whether the database was saved to binary format + */ + void set_binary (bool bin) + { + set_modified (); + m_binary = bin; + } + + /** + * @brief Gets the generator name */ const std::string &generator () const { @@ -2103,7 +2149,7 @@ public: } /** - * @brief Set the generator name + * @brief Sets the generator name */ void set_generator (const std::string &generator) { @@ -2112,7 +2158,7 @@ public: } /** - * @brief Set the top cell name + * @brief Sets the top cell name */ void set_top_cell_name (const std::string &topcell) { @@ -2121,7 +2167,7 @@ public: } /** - * @brief Return the top cell name + * @brief Returns the top cell name */ const std::string &top_cell_name () const { @@ -2129,7 +2175,7 @@ public: } /* - * @brief Get the reference to the tags collection (const version) + * @brief Gets the reference to the tags collection (const version) */ const Tags &tags () const { @@ -2137,14 +2183,14 @@ public: } /** - * @brief Import tags + * @brief Imports tags * * This method is provided for persistency application only. It should not be used otherwise. */ void import_tags (const Tags &tags); /** - * @brief Get the reference to the categories collection (const version) + * @brief Gets the reference to the categories collection (const version) */ const Categories &categories () const { @@ -2152,7 +2198,7 @@ public: } /** - * @brief Get the reference to the categories collection (non-const version) + * @brief Gets the reference to the categories collection (non-const version) */ Categories &categories_non_const () { @@ -2160,7 +2206,7 @@ public: } /** - * @brief Import categories + * @brief Imports categories * * This method is provided for persistency application only. It should not be used otherwise. * This will take over the ownership over the Categories object. @@ -2168,17 +2214,17 @@ public: void import_categories (Categories *categories); /** - * @brief Create a category and register it + * @brief Creates a category and register it */ Category *create_category (const std::string &name); /** - * @brief Create a category as a subcategory and register it + * @brief Creates a category as a subcategory and register it */ Category *create_category (Category *parent, const std::string &name); /** - * @brief Create a category as a subcategory in the container and register it + * @brief Creates a category as a subcategory in the container and register it * * Hint: this method does not set the parent properly and must not be used * under normal circumstances. It is provided as a internal method and @@ -2187,7 +2233,7 @@ public: Category *create_category (Categories *container, const std::string &name); /** - * @brief Get the category pointer for a category name (const version) + * @brief Gets the category pointer for a category name (const version) * * This method returns 0 if the category name is invalid. */ @@ -2197,7 +2243,7 @@ public: } /** - * @brief Get the category pointer for a category id (const version) + * @brief Gets the category pointer for a category id (const version) * * This method returns 0 if the category is invalid. */ @@ -2207,21 +2253,21 @@ public: } /** - * @brief Get the category pointer for a category name (non-const version) + * @brief Gets the category pointer for a category name (non-const version) * * This method returns 0 if the category name is invalid. */ Category *category_by_name_non_const (const std::string &name); /** - * @brief Get the category pointer for a category id (non-const version) + * @brief Gets the category pointer for a category id (non-const version) * * This method returns 0 if the category is invalid. */ Category *category_by_id_non_const (id_type id); /** - * @brief Access to the cell collection (const) + * @brief Gets the cell collection (const) */ const Cells &cells () const { @@ -2229,7 +2275,7 @@ public: } /** - * @brief Access to the cell collection + * @brief Gets the cell collection */ Cells &cells_non_const () { @@ -2237,14 +2283,14 @@ public: } /** - * @brief Import cells + * @brief Imports cells * * This method is provided for persistency application only. It should not be used otherwise. */ void import_cells (const Cells &cells); /** - * @brief Create a cell and register it + * @brief Creates a cell and registers it * * If a cell with that name already exists, a variant is created */ @@ -2254,10 +2300,10 @@ public: } /** - * @brief Create a cell variant and register it + * @brief Creates a cell variant and registers it * * A cell with name name/variant combination must not exist already. - * If the variant string is empty, this method behaves the same as the + * If the variant string is empty, this method behaves the same as the * method without variant. * * "layout_name" is the name of the cell in the layout. If empty, the layout @@ -2266,14 +2312,14 @@ public: Cell *create_cell (const std::string &name, const std::string &variant, const std::string &layout_name); /** - * @brief Get all variants registered for a given cell name (not qname!) + * @brief Gets all variants registered for a given cell name (not qname!) * * @return a vector of id's corresponding to the given variants or an empty vector if the name is not valid or the cell has no variants */ const std::vector &variants (const std::string &name); /** - * @brief Get the cell pointer for a cell name or name:variant combination (const version) + * @brief Gets the cell pointer for a cell name or name:variant combination (const version) * * This method returns 0 if the cell name or name:variant combination is invalid. */ @@ -2283,7 +2329,7 @@ public: } /** - * @brief Get the cell pointer for a cell id (const version) + * @brief Gets the cell pointer for a cell id (const version) * * This method returns 0 if the cell id is invalid. */ @@ -2293,21 +2339,21 @@ public: } /** - * @brief Get the cell pointer for a cell name or name:variant combination (non-const version) + * @brief Gets the cell pointer for a cell name or name:variant combination (non-const version) * * This method returns 0 if the cell name or name:variant combination is invalid. */ Cell *cell_by_qname_non_const (const std::string &qname); /** - * @brief Get the cell pointer for a cell id (non-const version) + * @brief Gets the cell pointer for a cell id (non-const version) * * This method returns 0 if the cell id is invalid. */ Cell *cell_by_id_non_const (id_type id); /** - * @brief Report the number of items in total + * @brief Reports the number of items in total */ size_t num_items () const { @@ -2315,7 +2361,7 @@ public: } /** - * @brief Report the number of items visited + * @brief Reports the number of items visited */ size_t num_items_visited () const { @@ -2323,37 +2369,37 @@ public: } /** - * @brief Report the number of items for a given cell and category id + * @brief Reports the number of items for a given cell and category id */ size_t num_items (id_type cell_id, id_type category_id) const; /** - * @brief Report the number of items visited + * @brief Reports the number of items visited */ size_t num_items_visited (id_type cell_id, id_type category_id) const; /** - * @brief Create a new item for the given cell and category (both given by id) + * @brief Creates a new item for the given cell and category (both given by id) */ Item *create_item (id_type cell_id, id_type category_id); /** - * @brief Set a tag's description + * @brief Sets a tag's description */ void set_tag_description (id_type tag_id, const std::string &description); /** - * @brief Set an item to visited state or reset the state + * @brief Sets an item to visited state or reset the state */ void set_item_visited (const Item *item, bool visited); /** - * @brief Add a tag to the given item + * @brief Adds a tag to the given item */ void add_item_tag (const Item *item, id_type tag); /** - * @brief Remove a tag from the given item + * @brief Removes a tag from the given item */ void remove_item_tag (const Item *item, id_type tag); @@ -2364,23 +2410,23 @@ public: #if defined(HAVE_QT) /** - * @brief Set the image of an item + * @brief Sets the image of an item */ void set_item_image (const Item *item, const QImage &image); #endif /** - * @brief Set the image string of an item + * @brief Sets the image string of an item */ void set_item_image_str (const Item *item, const std::string &image_str); /** - * @brief Set the multiplicity of an item + * @brief Sets the multiplicity of an item */ void set_item_multiplicity (const Item *item, size_t n); /** - * @brief Get the items collection (const version) + * @brief Gets the items collection (const version) */ const Items &items () const { @@ -2388,7 +2434,7 @@ public: } /** - * @brief Get the items collection (non-const version) + * @brief Gets the items collection (non-const version) */ Items &items_non_const () { @@ -2396,7 +2442,7 @@ public: } /** - * @brief Set the items collection + * @brief Sets the items collection * * This method is provided for persistency application only. It should not be used otherwise. * This will take ownership over the items collection. @@ -2404,32 +2450,32 @@ public: void set_items (Items *items); /** - * @brief Get an iterator pair that delivers the const items (ItemRef) for a given cell + * @brief Gets an iterator pair that delivers the const items (ItemRef) for a given cell */ - std::pair items_by_cell (id_type cell_id) const; + std::pair items_by_cell (id_type cell_id) const; /** - * @brief Get an iterator pair that delivers the non-const items (ItemRef) for a given cell + * @brief Gets an iterator pair that delivers the non-const items (ItemRef) for a given cell */ std::pair items_by_cell (id_type cell_id); /** - * @brief Get an iterator that delivers the const items (ItemRef) for a given category + * @brief Gets an iterator that delivers the const items (ItemRef) for a given category */ - std::pair items_by_category (id_type category_id) const; + std::pair items_by_category (id_type category_id) const; /** - * @brief Get an iterator that delivers the non-const items (ItemRef) for a given category + * @brief Gets an iterator that delivers the non-const items (ItemRef) for a given category */ std::pair items_by_category (id_type category_id); /** - * @brief Get an iterator that delivers the const items (ItemRef) for a given cell and category + * @brief Gets an iterator that delivers the const items (ItemRef) for a given cell and category */ - std::pair items_by_cell_and_category (id_type cell_id, id_type category_id) const; + std::pair items_by_cell_and_category (id_type cell_id, id_type category_id) const; /** - * @brief Get an iterator that delivers the non-const items (ItemRef) for a given cell and category + * @brief Gets an iterator that delivers the non-const items (ItemRef) for a given cell and category */ std::pair items_by_cell_and_category (id_type cell_id, id_type category_id); @@ -2442,27 +2488,27 @@ public: } /** - * @brief Reset the modified file + * @brief Resets the modified file */ - void reset_modified () + void reset_modified () { m_modified = false; } /** - * @brief Save the database to a file + * @brief Saves the database to a file */ - void save (const std::string &filename); + void save (const std::string &filename, bool binary = false); /** - * @brief Write the database to a file + * @brief Writes the database to a file * * This function is like "save", but does not update the file name attribute. */ - void write (const std::string &filename); + void write (const std::string &filename, bool binary = false); /** - * @brief Load the database from a file + * @brief Loads the database from a file * * Note: This clears the existing database. * The argument intentionally is a copy, so we can call @@ -2491,6 +2537,7 @@ public: private: std::string m_generator; std::string m_filename; + bool m_binary; std::string m_description; std::string m_original_file; std::string m_name; @@ -2515,7 +2562,7 @@ private: void clear (); - void set_modified () + void set_modified () { m_modified = true; } diff --git a/src/rdb/rdb/rdbFile.cc b/src/rdb/rdb/rdbFile.cc index dda333dba..61ad32ac8 100644 --- a/src/rdb/rdb/rdbFile.cc +++ b/src/rdb/rdb/rdbFile.cc @@ -38,104 +38,129 @@ namespace rdb struct ValueConverter { - ValueConverter (rdb::Database *rdb) - : mp_rdb (rdb) + typedef std::string pb_type; + + ValueConverter () { } std::string to_string (const ValueWrapper &value) const { - return value.to_string (mp_rdb); + return value.to_string (); } void from_string (const std::string &s, ValueWrapper &value) const { - value.from_string (mp_rdb, s); + value.from_string (s); } -private: - rdb::Database *mp_rdb; -}; + std::string pb_encode (const ValueWrapper &value) const + { + return value.to_string (); + } -static -tl::XMLElementList categories_format = - tl::make_element_with_parent_ref (&rdb::Categories::begin, &rdb::Categories::end, &rdb::Categories::import_category, "category", - tl::make_member (&rdb::Category::name, &rdb::Category::set_name, "name") + - tl::make_member (&rdb::Category::description, &rdb::Category::set_description, "description") + - tl::make_element_with_parent_ref (&rdb::Category::sub_categories, &rdb::Category::import_sub_categories, "categories", - &categories_format - ) - ) -; + void pb_decode (const std::string &s, ValueWrapper &value) const + { + value.from_string (s); + } +}; // generation of the RDB file XML structure static tl::XMLStruct -make_rdb_structure (rdb::Database *rdb) +make_rdb_structure () { - return tl::XMLStruct ("report-database", - tl::make_member (&rdb::Database::description, &rdb::Database::set_description, "description") + - tl::make_member (&rdb::Database::original_file, &rdb::Database::set_original_file, "original-file") + - tl::make_member (&rdb::Database::generator, &rdb::Database::set_generator, "generator") + - tl::make_member (&rdb::Database::top_cell_name, &rdb::Database::set_top_cell_name, "top-cell") + - tl::make_element (&rdb::Database::tags, &rdb::Database::import_tags, "tags", - tl::make_element (&rdb::Tags::begin_tags, &rdb::Tags::end_tags, &rdb::Tags::import_tag, "tag", - tl::make_member (&rdb::Tag::name, &rdb::Tag::set_name, "name") + - tl::make_member (&rdb::Tag::description, &rdb::Tag::set_description, "description") + static + tl::XMLElementList categories_format = + tl::make_element_with_parent_ref (&rdb::Categories::begin, &rdb::Categories::end, &rdb::Categories::import_category, "category#1", + tl::make_member (&rdb::Category::name, &rdb::Category::set_name, "name#1") + + tl::make_member (&rdb::Category::description, &rdb::Category::set_description, "description#2") + + tl::make_element_with_parent_ref (&rdb::Category::sub_categories, &rdb::Category::import_sub_categories, "categories#3", + &categories_format + ) + ) + ; + + return tl::XMLStruct ("report-database#0", + tl::make_member (&rdb::Database::description, &rdb::Database::set_description, "description#1") + + tl::make_member (&rdb::Database::original_file, &rdb::Database::set_original_file, "original-file#2") + + tl::make_member (&rdb::Database::generator, &rdb::Database::set_generator, "generator#3") + + tl::make_member (&rdb::Database::top_cell_name, &rdb::Database::set_top_cell_name, "top-cell#4") + + tl::make_element (&rdb::Database::tags, &rdb::Database::import_tags, "tags#5", + tl::make_element (&rdb::Tags::begin_tags, &rdb::Tags::end_tags, &rdb::Tags::import_tag, "tag#1", + tl::make_member (&rdb::Tag::name, &rdb::Tag::set_name, "name#1") + + tl::make_member (&rdb::Tag::description, &rdb::Tag::set_description, "description#2") ) ) + - tl::make_element_with_parent_ref (&rdb::Database::categories, &rdb::Database::import_categories, "categories", + tl::make_element_with_parent_ref (&rdb::Database::categories, &rdb::Database::import_categories, "categories#6", &categories_format ) + - tl::make_element_with_parent_ref (&rdb::Database::cells, &rdb::Database::import_cells, "cells", + tl::make_element_with_parent_ref (&rdb::Database::cells, &rdb::Database::import_cells, "cells#7", // must be sorted cells (children after parents)! - tl::make_element_with_parent_ref (&rdb::Cells::begin, &rdb::Cells::end, &rdb::Cells::import_cell, "cell", - tl::make_member (&rdb::Cell::name, &rdb::Cell::set_name, "name") + - tl::make_member (&rdb::Cell::variant, &rdb::Cell::set_variant, "variant") + - tl::make_member (&rdb::Cell::layout_name, &rdb::Cell::set_layout_name, "layout-name") + - tl::make_element_with_parent_ref (&rdb::Cell::references, &rdb::Cell::import_references, "references", - tl::make_element_with_parent_ref (&rdb::References::begin, &rdb::References::end, &rdb::References::insert, "ref", - tl::make_member (&rdb::Reference::parent_cell_qname, &rdb::Reference::set_parent_cell_qname, "parent") + - tl::make_member (&rdb::Reference::trans_str, &rdb::Reference::set_trans_str, "trans") + tl::make_element_with_parent_ref (&rdb::Cells::begin, &rdb::Cells::end, &rdb::Cells::import_cell, "cell#1", + tl::make_member (&rdb::Cell::name, &rdb::Cell::set_name, "name#1") + + tl::make_member (&rdb::Cell::variant, &rdb::Cell::set_variant, "variant#2") + + tl::make_member (&rdb::Cell::layout_name, &rdb::Cell::set_layout_name, "layout-name#3") + + tl::make_element_with_parent_ref (&rdb::Cell::references, &rdb::Cell::import_references, "references#4", + tl::make_element_with_parent_ref (&rdb::References::begin, &rdb::References::end, &rdb::References::insert, "ref#1", + tl::make_member (&rdb::Reference::parent_cell_qname, &rdb::Reference::set_parent_cell_qname, "parent#1") + + tl::make_member (&rdb::Reference::trans_str, &rdb::Reference::set_trans_str, "trans#2") ) ) ) ) + - tl::make_element_with_parent_ref (&rdb::Database::items, &rdb::Database::set_items, "items", - tl::make_element_with_parent_ref (&rdb::Items::begin, &rdb::Items::end, &rdb::Items::add_item, "item", - tl::make_member (&rdb::Item::tag_str, &rdb::Item::set_tag_str, "tags") + - tl::make_member (&rdb::Item::category_name, &rdb::Item::set_category_name, "category") + - tl::make_member (&rdb::Item::cell_qname, &rdb::Item::set_cell_qname, "cell") + - tl::make_member (&rdb::Item::visited, &rdb::Item::set_visited, "visited") + - tl::make_member (&rdb::Item::multiplicity, &rdb::Item::set_multiplicity, "multiplicity") + - tl::make_member (&rdb::Item::comment, &rdb::Item::set_comment, "comment") + - tl::make_member (&rdb::Item::image_str, &rdb::Item::set_image_str, "image") + - tl::make_element (&rdb::Item::values, &rdb::Item::set_values, "values", - tl::make_member (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value", ValueConverter (rdb)) - ) + tl::make_element_with_parent_ref (&rdb::Database::items, &rdb::Database::set_items, "items#8", + tl::make_element_with_parent_ref (&rdb::Items::begin, &rdb::Items::end, &rdb::Items::add_item, "item#1", + tl::make_member (&rdb::Item::tag_str, &rdb::Item::set_tag_str, "tags#1") + + tl::make_member (&rdb::Item::category_name, &rdb::Item::set_category_name, "category#2") + + tl::make_member (&rdb::Item::cell_qname, &rdb::Item::set_cell_qname, "cell#3") + + tl::make_member (&rdb::Item::visited, &rdb::Item::set_visited, "visited#4") + + tl::make_member (&rdb::Item::multiplicity, &rdb::Item::set_multiplicity, "multiplicity#5") + + tl::make_member (&rdb::Item::comment, &rdb::Item::set_comment, "comment#6") + + tl::make_member (&rdb::Item::image_str, &rdb::Item::set_image_str, "image#7") + + tl::make_element (&rdb::Item::values, &rdb::Item::set_values, "values#8", + tl::make_member (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value#1", ValueConverter ()) ) ) ) ); } +static tl::XMLStruct s_rdb_struct = make_rdb_structure (); +static tl::RegisteredClass rdb_struct_def (&s_rdb_struct, 0, "KLayout-RDB", false); + // ------------------------------------------------------------- // Implementation of rdb::Database::save and write // TODO: move this somewhere else - with generalized functionality void -rdb::Database::save (const std::string &fn) +rdb::Database::save (const std::string &fn, bool binary) { - write (fn); + write (fn, binary); set_filename (fn); + set_binary (binary); } void -rdb::Database::write (const std::string &fn) +rdb::Database::write (const std::string &fn, bool binary) { tl::OutputStream os (fn, tl::OutputStream::OM_Auto); - make_rdb_structure (this).write (os, *this); - if (tl::verbosity () >= 10) { - tl::log << "Saved RDB to " << fn; + if (binary) { + + tl::ProtocolBufferWriter writer (os); + s_rdb_struct.write (writer, *this); + + if (tl::verbosity () >= 10) { + tl::log << tl::to_string (tr ("Saved binary RDB to ")) << fn; + } + + } else { + + s_rdb_struct.write (os, *this); + + if (tl::verbosity () >= 10) { + tl::log << tl::to_string (tr ("Saved RDB to ")) << fn; + } + } } @@ -146,8 +171,8 @@ class StandardReader : public ReaderBase { public: - StandardReader (tl::InputStream &stream) - : m_input_stream (stream) + StandardReader (tl::InputStream &stream, bool binary) + : m_input_stream (stream), m_binary (binary) { // .. nothing yet .. } @@ -155,17 +180,30 @@ public: virtual void read (Database &db) { tl::SelfTimer timer (tl::verbosity () >= 11, "Reading marker database file"); - tl::XMLStreamSource in (m_input_stream, tl::to_string (tr ("Reading RDB"))); - make_rdb_structure (&db).parse (in, db); + + if (m_binary) { + + tl::ProtocolBufferReader reader (m_input_stream); + s_rdb_struct.parse (reader, db); + db.set_binary (true); + + } else { + + tl::XMLStreamSource in (m_input_stream, tl::to_string (tr ("Reading RDB"))); + s_rdb_struct.parse (in, db); + db.set_binary (false); + + } } virtual const char *format () const { - return "KLayout-RDB"; + return m_binary ? "KLayout-RDB-PB" : "KLayout-RDB"; } private: tl::InputStream &m_input_stream; + bool m_binary; }; class StandardFormatDeclaration @@ -193,10 +231,46 @@ class StandardFormatDeclaration virtual ReaderBase *create_reader (tl::InputStream &s) const { - return new StandardReader (s); + return new StandardReader (s, false); } }; static tl::RegisteredClass format_decl (new StandardFormatDeclaration (), 0, "KLayout-RDB"); +class BinaryFormatDeclaration + : public FormatDeclaration +{ + virtual std::string format_name () const { return "KLayout-RDB-PB"; } + virtual std::string format_desc () const { return "KLayout binary report database format"; } + virtual std::string file_format () const { return "KLayout binary RDB files (*.rdb *.rdb.gz)"; } + + virtual bool detect (tl::InputStream &stream) const + { + static const char header[] = { + // ProtocolBuffer wire format, LEN record with ID 0 and string "report-database". + 0x02, 0x0f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2d, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65 + }; + + const char *h = stream.get (sizeof (header)); + if (! h) { + return false; + } + + for (size_t i = 0; i < sizeof (header); ++i) { + if (h[i] != header[i]) { + return false; + } + } + + return true; + } + + virtual ReaderBase *create_reader (tl::InputStream &s) const + { + return new StandardReader (s, true); + } +}; + +static tl::RegisteredClass pb_format_decl (new BinaryFormatDeclaration (), 1, "KLayout-RDB-PB"); + } diff --git a/src/rdb/unit_tests/rdbTests.cc b/src/rdb/unit_tests/rdbTests.cc index 59acd89e8..730b5cb25 100644 --- a/src/rdb/unit_tests/rdbTests.cc +++ b/src/rdb/unit_tests/rdbTests.cc @@ -816,7 +816,18 @@ TEST(13_ApplyIgnoreUnknownTag) i2->values ().clear (); i2->add_value (std::string ("xyz"), vtag22); - // values with incompatible tags are ignored -> tag2 is applied + // values with incompatible tags are ignored, but vtag1 is a common tag. + // So far, nothing is applied as the we match abc[vtag1] vs. xyz[vtag2] which is both different value and tag- + db1.apply (db2); + + EXPECT_EQ (i1->tag_str (), ""); + + // NOTE: don't do this at home + const_cast (db2.tags ()).remove_tag (vtag21); + + // vtag1 is no longer a common tag -> we match abc[vtag1] vs. xyz[vtag2] where vtag1 is not known on the + // other side and vtag2 is not known on the self side. + // Hence, the values are ignored and the tag is applied. db1.apply (db2); EXPECT_EQ (i1->tag_str (), "tag2"); diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index b3386179b..ed9e57df2 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -29,6 +29,7 @@ SOURCES = \ tlObject.cc \ tlProgress.cc \ tlPixelBuffer.cc \ + tlProtocolBuffer.cc \ tlResources.cc \ tlScriptError.cc \ tlSleep.cc \ @@ -43,6 +44,7 @@ SOURCES = \ tlUnitTest.cc \ tlInt128Support.cc \ tlXMLParser.cc \ + tlXMLReader.cc \ tlXMLWriter.cc \ tlThreadedWorkers.cc \ tlThreads.cc \ @@ -86,6 +88,7 @@ HEADERS = \ tlObjectCollection.h \ tlProgress.h \ tlPixelBuffer.h \ + tlProtocolBuffer.h \ tlResources.h \ tlReuseVector.h \ tlScriptError.h \ @@ -110,6 +113,7 @@ HEADERS = \ tlInt128Support.h \ tlDefs.h \ tlXMLParser.h \ + tlXMLReader.h \ tlXMLWriter.h \ tlThreadedWorkers.h \ tlThreads.h \ diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc new file mode 100644 index 000000000..126b0ffdf --- /dev/null +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -0,0 +1,664 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "tlProtocolBuffer.h" +#include "tlLog.h" + +namespace tl +{ + +// ---------------------------------------------------------------------------------- + +ProtocolBufferReader::ProtocolBufferReader (tl::InputStream &input) + : mp_stream (&input), m_type (PBWireType (0)), m_pos (0), m_pos_before (0) +{ + // .. nothing yet .. +} + +int +ProtocolBufferReader::read_tag () +{ + m_type = PB_VARINT; + uint32_t value = 0; + read (value); + m_type = PBWireType (value & 7); + return int (value >> 3); +} + +void +ProtocolBufferReader::skip () +{ + if (m_type == PB_VARINT) { + + while ((*get (1) & 0x80) != 0) { + // continue reading until the byte's MSB is 0 + } + + } else if (m_type == PB_I64) { + + get (8); + + } else if (m_type == PB_I32) { + + get (4); + + } else if (m_type == PB_LEN) { + + size_t value = read_varint (); + skip_bytes (value); + + } +} + +void +ProtocolBufferReader::read (double &d) +{ + uint64_t value = 0; + read (value); + d = *reinterpret_cast (&value); +} + +void +ProtocolBufferReader::read (float &f) +{ + uint32_t value = 0; + read (value); + f = *reinterpret_cast (&value); +} + +void +ProtocolBufferReader::read (std::string &s) +{ + if (m_type != PB_LEN) { + error (tl::to_string (tr ("Expected a LEN wire type for a string"))); + } + + size_t value = read_varint (); + + s = std::string (); + s.reserve (value); + + const size_t chunk_size = 1024; + while (value > 0) { + size_t l = std::min (chunk_size, value); + s += std::string (get (l), l); + value -= l; + } +} + +void +ProtocolBufferReader::read (uint32_t &ui32) +{ + if (m_type == PB_VARINT) { + + pb_varint ui64 = read_varint (); + if (ui64 > std::numeric_limits::max ()) { + error (tl::to_string (tr ("32 bit value overflow"))); + } + + ui32 = uint32_t (ui64); + + } else if (m_type == PB_I32) { + + ui32 = 0; + const char *cp = get (sizeof (ui32)); + for (unsigned int i = 0; i < sizeof (ui32); ++i) { + ui32 <<= 8; + ui32 |= (unsigned char) cp [sizeof (ui32) - 1 - i]; + } + + } else { + error (tl::to_string (tr ("Expected a VARINT or I32 wire type"))); + } +} + +void +ProtocolBufferReader::read (int32_t &i32) +{ + uint32_t ui32; + read (ui32); + + if (m_type != PB_I32) { + if (ui32 & 1) { + i32 = -(int32_t (ui32 >> 1) + 1); + } else { + i32 = int32_t (ui32 >> 1); + } + } else { + i32 = int32_t (ui32); + } +} + +void +ProtocolBufferReader::read (uint64_t &ui64) +{ + if (m_type == PB_VARINT) { + + ui64 = read_varint (); + + } else if (m_type == PB_I64) { + + ui64 = 0; + const char *cp = get (sizeof (ui64)); + for (unsigned int i = 0; i < sizeof (ui64); ++i) { + ui64 <<= 8; + ui64 |= (unsigned char) cp [sizeof (ui64) - 1 - i]; + } + + } else { + error (tl::to_string (tr ("Expected a VARINT or I64 wire type"))); + } +} + +void +ProtocolBufferReader::read (int64_t &i64) +{ + uint64_t ui64; + read (ui64); + + if (m_type != PB_I64) { + if (ui64 & 1) { + i64 = -(int64_t (ui64 >> 1) + 1); + } else { + i64 = int64_t (ui64 >> 1); + } + } else { + i64 = int64_t (ui64); + } +} + +void +ProtocolBufferReader::read (bool &b) +{ + uint32_t ui32; + read (ui32); + b = (ui32 != 0); +} + +void +ProtocolBufferReader::open () +{ + if (m_type != PB_LEN) { + error (tl::to_string (tr ("Expected a LEN wire type for a submessage"))); + } + + size_t value = read_varint (); + if (! m_seq_counts.empty ()) { + // take out the following bytes from the current sequence + m_seq_counts.back () -= value; + } + m_seq_counts.push_back (value); +} + +void +ProtocolBufferReader::close () +{ + if (! m_seq_counts.empty ()) { + skip_bytes (m_seq_counts.back ()); + m_seq_counts.pop_back (); + } +} + +bool +ProtocolBufferReader::at_end () const +{ + if (m_seq_counts.empty ()) { + const char *cp = mp_stream->get (1); + if (cp) { + mp_stream->unget (1); + return false; + } else { + return true; + } + } else { + return m_seq_counts.back () == 0; + } +} + +const char * +ProtocolBufferReader::get (size_t n) +{ + m_pos_before = m_pos; + m_pos += n; + if (! m_seq_counts.empty ()) { + if (m_seq_counts.back () < n) { + error (tl::to_string (tr ("sequence underflow"))); + } + m_seq_counts.back () -= n; + } + + const char *cp = mp_stream->get (n); + if (! cp) { + error (tl::to_string (tr ("unexpected end of file"))); + } + + return cp; +} + +pb_varint +ProtocolBufferReader::read_varint () +{ + pb_varint v = 0; + unsigned int s = 0; + + while (true) { + const char *cp = get (1); + if (! cp) { + error (tl::to_string (tr ("unexpected end of file"))); + } + if (s + 7 > 64 && pb_varint ((unsigned char) (*cp & 0x7f)) >> (s + 7 - 64) != 0) { + error (tl::to_string (tr ("64 bit integer overflow"))); + } + v |= (pb_varint ((unsigned char) (*cp & 0x7f)) << s); + s += 7; + if ((*cp & 0x80) == 0) { + break; + } + } + + return v; +} + +void +ProtocolBufferReader::skip_bytes (size_t n) +{ + m_pos_before = m_pos; + m_pos += n; + if (! m_seq_counts.empty ()) { + if (m_seq_counts.back () < n) { + error (tl::to_string (tr ("sequence underflow"))); + } + m_seq_counts.back () -= n; + } + + const size_t chunk_size = 1024; + while (n > 0) { + size_t l = std::min (chunk_size, n); + if (! mp_stream->get (l)) { + error (tl::to_string (tr ("unexpected end of file"))); + } + n -= l; + } +} + +void +ProtocolBufferReader::error (const std::string &msg) +{ + throw ProtocolBufferReaderError (msg + tl::to_string (tr (", in: ")) + mp_stream->source (), m_pos_before); +} + +// ---------------------------------------------------------------------------------- +// Writer utilities + +inline uint32_t zigzag_encode (int32_t v) +{ + if (v < 0) { + return ((uint32_t (-v) - 1) << 1) + 1; + } else { + return uint32_t (v) << 1; + } +} + +inline uint64_t zigzag_encode (int64_t v) +{ + if (v < 0) { + return ((uint64_t (-v) - 1) << 1) + 1; + } else { + return uint64_t (v) << 1; + } +} + +inline void little_endian_encode (char *b, uint32_t v) +{ + for (unsigned int i = 0; i < sizeof (v); ++i) { + b[i] = (char) v; + v >>= 8; + } +} + +inline void little_endian_encode (char *b, uint64_t v) +{ + for (unsigned int i = 0; i < sizeof (v); ++i) { + b[i] = (char) v; + v >>= 8; + } +} + +inline size_t count_varint_bytes (pb_varint v) +{ + size_t n = 0; + while (true) { + ++n; + if (v < 0x80) { + break; + } + v >>= 7; + } + return n; +} + +const size_t max_varint_bytes = (sizeof (pb_varint) * 8) / 7 + 1; + +inline size_t varint_encode (char *b, pb_varint v) +{ + char *cp = b; + while (true) { + if (v < 0x80) { + *cp++ = char (v); + break; + } else { + *cp++ = (char (v) | 0x80); + } + v >>= 7; + } + return cp - b; +} + +// ---------------------------------------------------------------------------------- + +ProtocolBufferWriterBase::ProtocolBufferWriterBase () + : m_bytes_counted (0) +{ + // .. nothing yet .. +} + +ProtocolBufferWriterBase::~ProtocolBufferWriterBase () +{ + // .. nothing yet .. +} + +void ProtocolBufferWriterBase::write (int tag, float v) +{ + write (tag, *reinterpret_cast (&v), true); +} + +void ProtocolBufferWriterBase::write (int tag, double v) +{ + write (tag, *reinterpret_cast (&v), true); +} + +void ProtocolBufferWriterBase::write (int tag, int32_t v, bool fixed) +{ + if (fixed) { + write (tag, uint32_t (v), true); + } else { + write (tag, zigzag_encode (v), false); + } +} + +void ProtocolBufferWriterBase::write (int tag, int64_t v, bool fixed) +{ + if (fixed) { + write (tag, uint64_t (v), true); + } else { + write (tag, zigzag_encode (v), false); + } +} + +void ProtocolBufferWriterBase::write (int tag, bool b) +{ + write (tag, uint32_t (b ? 1 : 0)); +} + +void ProtocolBufferWriterBase::write (int tag, uint32_t v, bool fixed) +{ + if (fixed) { + write_varint (pb_varint ((tag << 3) + PB_I32), true); + write_fixed (v); + } else { + write_varint (pb_varint ((tag << 3) + PB_VARINT), true); + write_varint (pb_varint (v), false); + } +} + +void ProtocolBufferWriterBase::write (int tag, uint64_t v, bool fixed) +{ + if (fixed) { + write_varint (pb_varint ((tag << 3) + PB_I64), true); + write_fixed (v); + } else { + write_varint (pb_varint ((tag << 3) + PB_VARINT), true); + write_varint (pb_varint (v), false); + } +} + +void ProtocolBufferWriterBase::write (int tag, const std::string &s) +{ + write_varint (pb_varint ((tag << 3) + PB_LEN), true); + write_varint (s.size (), false); + write_bytes (s); +} + +bool ProtocolBufferWriterBase::is_counting () const +{ + return ! m_byte_counter_stack.empty (); +} + +void ProtocolBufferWriterBase::begin_seq (int tag, bool counting) +{ + if (counting) { + + if (is_counting ()) { + write_varint (pb_varint ((tag << 3) + PB_LEN), true); + } + + m_byte_counter_stack.push_back (0); + + } else { + + write_varint (pb_varint ((tag << 3) + PB_LEN), true); + write_varint (m_bytes_counted, false); + + } +} + +void ProtocolBufferWriterBase::end_seq () +{ + if (is_counting ()) { + + m_bytes_counted = m_byte_counter_stack.back (); + m_byte_counter_stack.pop_back (); + + // just for adding the required bytes + if (is_counting ()) { + m_byte_counter_stack.back () += m_bytes_counted; + write_varint (m_bytes_counted, false); + } + + } +} + +void ProtocolBufferWriterBase::add_bytes (size_t n) +{ + tl_assert (! m_byte_counter_stack.empty ()); + m_byte_counter_stack.back () += n; +} + +// ---------------------------------------------------------------------------------- + +ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream) + : mp_stream (&stream) +{ + // .. nothing yet .. +} + +void ProtocolBufferWriter::write_fixed (uint32_t v) +{ + if (is_counting ()) { + add_bytes (sizeof (v)); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + mp_stream->put (b, sizeof (v)); + } +} + +void ProtocolBufferWriter::write_fixed (uint64_t v) +{ + if (is_counting ()) { + add_bytes (sizeof (v)); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + mp_stream->put (b, sizeof (v)); + } +} + +void ProtocolBufferWriter::write_bytes (const std::string &s) +{ + if (is_counting ()) { + add_bytes (s.size ()); + } else { + mp_stream->put (s.c_str (), s.size ()); + } +} + +void ProtocolBufferWriter::write_varint (pb_varint v, bool /*id*/) +{ + if (is_counting ()) { + add_bytes (count_varint_bytes (v)); + } else { + char b [max_varint_bytes]; + size_t n = varint_encode (b, v); + mp_stream->put (b, n); + } +} + +// ---------------------------------------------------------------------------------- + +ProtocolBufferDumper::ProtocolBufferDumper () + : m_debug_pos (0) +{ + // .. nothing yet .. +} + +void ProtocolBufferDumper::write_fixed (uint32_t v) +{ + if (is_counting ()) { + add_bytes (sizeof (v)); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + dump (b, sizeof (v), "I32", tl::to_string (v)); + } +} + +void ProtocolBufferDumper::write_fixed (uint64_t v) +{ + if (is_counting ()) { + add_bytes (sizeof (v)); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + dump (b, sizeof (v), "I64", tl::to_string (v)); + } +} + +void ProtocolBufferDumper::write_bytes (const std::string &s) +{ + if (is_counting ()) { + add_bytes (s.size ()); + } else { + dump (s.c_str (), s.size (), "(string)", s); + } +} + +void +ProtocolBufferDumper::write_varint (pb_varint v, bool id) +{ + if (is_counting ()) { + + add_bytes (count_varint_bytes (v)); + + } else { + + char b[max_varint_bytes]; + size_t n = varint_encode (b, v); + + if (id) { + tl::PBWireType wt = tl::PBWireType (v & 7); + std::string wire_type; + if (wt == PB_EGROUP) { + wire_type = "EGROUP"; + } else if (wt == PB_SGROUP) { + wire_type = "SGROUP"; + } else if (wt == PB_VARINT) { + wire_type = "VARINT"; + } else if (wt == PB_I32) { + wire_type = "I32"; + } else if (wt == PB_I64) { + wire_type = "I64"; + } else if (wt == PB_LEN) { + wire_type = "LEN"; + } + dump (b, n, "(id)", "#" + tl::to_string (v >> 3) + " " + wire_type); + } else { + dump (b, n, "VARINT", tl::to_string (v)); + } + + } +} + +void +ProtocolBufferDumper::dump (const char *cp, size_t n, const std::string &type, const std::string &value) +{ + bool first = true; + size_t nn = n; + + while (n > 0) { + + std::string line; + if (first) { + line += tl::sprintf ("%08ld", m_debug_pos); + } else { + line += " "; + } + line += " "; + + for (unsigned int i = 0; i < 8; ++i) { + if (n > 0) { + line += tl::sprintf ("%02x", (unsigned int) ((unsigned char) *cp)); + ++cp; + --n; + } else { + line += " "; + } + line += " "; + } + + if (first) { + line += " "; + line += type; + line += " "; + line += value; + } + + tl::info << line; + + first = false; + + } + + m_debug_pos += nn; +} + +} diff --git a/src/tl/tl/tlProtocolBuffer.h b/src/tl/tl/tlProtocolBuffer.h new file mode 100644 index 000000000..1b8ab25ce --- /dev/null +++ b/src/tl/tl/tlProtocolBuffer.h @@ -0,0 +1,433 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef HDR_tlProtocolBuffer +#define HDR_tlProtocolBuffer + +#include "tlCommon.h" +#include "tlStream.h" + +namespace tl +{ + +// Representation of VARINT values +typedef uint64_t pb_varint; + +enum PBWireType +{ + PB_VARINT = 0, + PB_I64 = 1, + PB_LEN = 2, + PB_SGROUP = 3, + PB_EGROUP = 4, + PB_I32 = 5 +}; + +/** + * @brief An exception thrown by the ProtocolBuffer reader on a read error + */ +class TL_PUBLIC ProtocolBufferReaderError + : public tl::Exception +{ +public: + ProtocolBufferReaderError (const std::string &msg, size_t position) + : tl::Exception (msg), m_position (position) + { } + + virtual std::string msg () const + { + return basic_msg () + tl::to_string (tr (", at position ")) + tl::to_string (m_position); + } + +private: + size_t m_position; +}; + +/** + * @brief A reader for ProtocolBuffer files and streams + * + * This is a low-level decoder for ProtocolBuffer files. + * + * Use "read_tag" to read a new tag. Unknown tags must be skipped. + * Use "skip" to skip an entry. + * + * + */ +class TL_PUBLIC ProtocolBufferReaderBase +{ +public: + /** + * @brief Constructor + */ + ProtocolBufferReaderBase () + { + // .. nothing yet .. + } + + /** + * @brief Destructor + */ + virtual ~ProtocolBufferReaderBase () + { + // .. nothing yet .. + } + + /** + * @brief Reads a new tag + * This method will also set the current write type. + * @returns The message ID + */ + virtual int read_tag () = 0; + + /** + * @brief Gets the current wire type + */ + virtual PBWireType type () const = 0; + + /** + * @brief Skips the current tag + */ + virtual void skip () = 0; + + /** + * @brief Reads a floating-point value from the current message + * Throws a reader error if the current tag's value is not compatible with a double value. + */ + virtual void read (double &d) = 0; + + /** + * @brief Reads a floating-point value from the current message + * Throws a reader error if the current tag's value is not compatible with a float value. + */ + virtual void read (float &f) = 0; + + /** + * @brief Reads a string from the current message + * Throws a reader error if the current tag's value is not compatible with a string. + */ + virtual void read (std::string &s) = 0; + + /** + * @brief Reads a uint32_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a uint32_t. + */ + virtual void read (uint32_t &ui32) = 0; + + /** + * @brief Reads a int32_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a int32_t. + */ + virtual void read (int32_t &i32) = 0; + + /** + * @brief Reads a uint64_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a uint64_t. + */ + virtual void read (uint64_t &ui64) = 0; + + /** + * @brief Reads a int64_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a int64_t. + */ + virtual void read (int64_t &i64) = 0; + + /** + * @brief Reads a boolean value from the current message + * Throws a reader error if the current tag's value is not compatible with a bool. + */ + virtual void read (bool &b) = 0; + + /** + * @brief Opens a LEN sequence + * After calling "open", the parser will continue reading messages, but + * "at_end" will report true on the end of the sequence, not at the end of the + * file. + * This method will throw an exception if not in a message of LEN type. + */ + virtual void open () = 0; + + /** + * @brief Closes the LEN sequence + * This method will jump to the end of the sequence and continue reading + * messages from the previous block or file. + */ + virtual void close () = 0; + + /** + * @brief Returns true if at the end of the file or end of a block + */ + virtual bool at_end () const = 0; + + /** + * @brief Emits an error at the current position + */ + virtual void error (const std::string &msg) = 0; +}; + +/** + * @brief A reader for ProtocolBuffer files and streams + * + * This is the reader implementation for binary files + * as described here: + * https://protobuf.dev/programming-guides/encoding/ + */ +class TL_PUBLIC ProtocolBufferReader + : public ProtocolBufferReaderBase +{ +public: + /** + * @brief Creates a ProtocolBuffer parser from the given stream + */ + ProtocolBufferReader (tl::InputStream &input); + + /** + * @brief Reads a new tag + * This method will also set the current write type. + * @returns The message ID + */ + int read_tag (); + + /** + * @brief Gets the current wire type + */ + PBWireType type () const + { + return m_type; + } + + /** + * @brief Skips the current tag + */ + void skip (); + + /** + * @brief Reads a floating-point value from the current message + * Throws a reader error if the current tag's value is not compatible with a double value. + */ + void read (double &d); + + /** + * @brief Reads a floating-point value from the current message + * Throws a reader error if the current tag's value is not compatible with a float value. + */ + void read (float &f); + + /** + * @brief Reads a string from the current message + * Throws a reader error if the current tag's value is not compatible with a string. + */ + void read (std::string &s); + + /** + * @brief Reads a uint32_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a uint32_t. + */ + void read (uint32_t &ui32); + + /** + * @brief Reads a int32_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a int32_t. + */ + void read (int32_t &i32); + + /** + * @brief Reads a uint64_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a uint64_t. + */ + void read (uint64_t &ui64); + + /** + * @brief Reads a int64_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a int64_t. + */ + void read (int64_t &i64); + + /** + * @brief Reads a boolean value from the current message + * Throws a reader error if the current tag's value is not compatible with a bool. + */ + void read (bool &b); + + /** + * @brief Opens a LEN sequence + * After calling "open", the parser will continue reading messages, but + * "at_end" will report true on the end of the sequence, not at the end of the + * file. + * This method will throw an exception if not in a message of LEN type. + */ + void open (); + + /** + * @brief Closes the LEN sequence + * This method will jump to the end of the sequence and continue reading + * messages from the previous block or file. + */ + void close (); + + /** + * @brief Returns true if at the end of the file or end of a block + */ + bool at_end () const; + + /** + * @brief Emits an error at the current position + */ + void error (const std::string &msg); + +private: + tl::InputStream *mp_stream; + PBWireType m_type; + size_t m_pos; + size_t m_pos_before; + std::vector m_seq_counts; + + pb_varint read_varint (); + void skip_bytes (size_t n); + const char *get (size_t n); +}; + +/** + * @brief The Protocol buffer writer + * + * Scalar types are easy to write: just use the "write" methods. + * This includes strings. + * + * Submessages and packed sequences are special as byte counting is + * required. The writer uses a two-pass approach. This means: + * + * 1. On writing a submessage or packed repetition, call "begin_seq" + * with the message tag and "counting" set to true. + * 2. Write the elements using "write" or the submessage writing + * scheme recursively. + * 3. At the end of the sequence, call "end_seq". + * 4. if "is_counting()" is false, repeat steps 1 to 3 with + * "counting" set to false on "begin_seq". + */ +class TL_PUBLIC ProtocolBufferWriterBase +{ +public: + /** + * @brief Constructor + */ + ProtocolBufferWriterBase (); + + /** + * @brief Destructor + */ + virtual ~ProtocolBufferWriterBase (); + + /** + * @brief Writes a scalar element with the given value and tag + */ + + // implicit types + void write (int tag, float v); + void write (int tag, double v); + void write (int tag, int32_t v, bool fixed = false); + void write (int tag, int64_t v, bool fixed = false); + void write (int tag, uint32_t v, bool fixed = false); + void write (int tag, uint64_t v, bool fixed = false); + void write (int tag, bool b); + void write (int tag, const std::string &s); + + /** + * @brief Returns true if the writer is in counting mode + */ + bool is_counting () const; + + /** + * @brief Initiates a new sequence. See class documentation for details. + */ + void begin_seq (int tag, bool counting); + + /** + * @brief Ends a sequence. See class documentation for details. + */ + void end_seq (); + +protected: + virtual void write_bytes (const std::string &s) = 0; + virtual void write_fixed (uint32_t v) = 0; + virtual void write_fixed (uint64_t v) = 0; + virtual void write_varint (pb_varint v, bool id) = 0; + void add_bytes (size_t n); + +private: + size_t m_bytes_counted; + std::vector m_byte_counter_stack; +}; + +/** + * @brief A writer for ProtocolBuffer files and streams + * + * This is the writer implementation for binary files + * as described here: + * https://protobuf.dev/programming-guides/encoding/ + */ +class TL_PUBLIC ProtocolBufferWriter + : public ProtocolBufferWriterBase +{ +public: + /** + * @brief Creates a writer for the given stream + */ + ProtocolBufferWriter (tl::OutputStream &stream); + +protected: + virtual void write_bytes (const std::string &s); + virtual void write_fixed (uint32_t v); + virtual void write_fixed (uint64_t v); + virtual void write_varint (pb_varint v, bool id = false); + +private: + tl::OutputStream *mp_stream; +}; + +/** + * @brief A writer implementation that dumps the file content to tl::info + * + * This implementation does a halfway job of producing binary files, + * but only insofar it is needed for dumping the binary data. + */ +class TL_PUBLIC ProtocolBufferDumper + : public ProtocolBufferWriterBase +{ +public: + /** + * @brief Creates the writer + */ + ProtocolBufferDumper (); + +protected: + virtual void write_bytes (const std::string &s); + virtual void write_fixed (uint32_t v); + virtual void write_fixed (uint64_t v); + virtual void write_varint (pb_varint v, bool id = false); + void dump (const char *cp, size_t n, const std::string &type, const std::string &value); + +private: + size_t m_debug_pos; +}; + +} + +#endif diff --git a/src/tl/tl/tlXMLParser.cc b/src/tl/tl/tlXMLParser.cc index 44ea5d3fa..56f9f68bb 100644 --- a/src/tl/tl/tlXMLParser.cc +++ b/src/tl/tl/tlXMLParser.cc @@ -26,891 +26,11 @@ #include "tlLog.h" #include "tlAssert.h" #include "tlProgress.h" +#include "tlProtocolBuffer.h" #include #include -#if defined(HAVE_EXPAT) - -#include - -namespace tl -{ - -// -------------------------------------------------------------------- -// SourcePrivateData implementation - -class XMLSourcePrivateData -{ -public: - XMLSourcePrivateData (tl::InputStream *stream) - : mp_stream_holder (stream), - m_has_error (false) - { - mp_stream = stream; - } - - XMLSourcePrivateData (tl::InputStream *stream, const std::string &progress_message) - : mp_stream_holder (stream), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = stream; - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - } - - XMLSourcePrivateData (tl::InputStream &stream) - : m_has_error (false) - { - mp_stream = &stream; - } - - XMLSourcePrivateData (tl::InputStream &stream, const std::string &progress_message) - : mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = &stream; - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - } - - int read (char *data, size_t n) - { - try { - - if (mp_progress.get ()) { - mp_progress->set (mp_stream->pos ()); - } - - size_t n0 = n; - for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; --n) { - *data++ = *rd; - } - - return int (n0 - n); - - } catch (tl::Exception &ex) { - m_error = ex.msg (); - m_has_error = true; - return -1; - } - } - - bool has_error () const - { - return m_has_error; - } - - const std::string &error_msg () const - { - return m_error; - } - - void reset () - { - mp_stream->reset (); - } - -private: - std::unique_ptr mp_stream_holder; - tl::InputStream *mp_stream; - std::unique_ptr mp_progress; - bool m_has_error; - std::string m_error; -}; - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - delete mp_source; - mp_source = 0; -} - -void -XMLSource::reset () -{ - mp_source->reset (); -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &string) - : m_copy (string) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (m_copy.c_str (), string.size ())))); -} - -XMLStringSource::XMLStringSource (const char *cp) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, strlen (cp))))); -} - -XMLStringSource::XMLStringSource (const char *cp, size_t len) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, len)))); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (path), progress_message)); -} - -XMLFileSource::XMLFileSource (const std::string &path) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (path))); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) -{ - set_source (new XMLSourcePrivateData (s, progress_message)); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &s) -{ - set_source (new XMLSourcePrivateData (s)); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -void XMLCALL start_element_handler (void *user_data, const XML_Char *name, const XML_Char **atts); -void XMLCALL end_element_handler (void *user_data, const XML_Char *name); -void XMLCALL cdata_handler (void *user_data, const XML_Char *s, int len); - -static std::string get_lname (const std::string &name) -{ - size_t colon = name.find (':'); - if (colon != std::string::npos) { - return std::string (name, colon + 1, name.size () - colon - 1); - } else { - return name; - } -} - -class XMLParserPrivateData -{ -public: - XMLParserPrivateData () - : mp_struct_handler (0) - { - mp_parser = XML_ParserCreate ("UTF-8"); - tl_assert (mp_parser != NULL); - } - - ~XMLParserPrivateData () - { - if (mp_parser != NULL) { - XML_ParserFree (mp_parser); - } - } - - void start_element (const std::string &name) - { - try { - // TODO: Provide namespace URI? - mp_struct_handler->start_element (std::string (), get_lname (name), name); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void end_element (const std::string &name) - { - try { - // TODO: Provide namespace URI? - mp_struct_handler->end_element (std::string (), get_lname (name), name); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void cdata (const std::string &cdata) - { - try { - mp_struct_handler->characters (cdata); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void parse (tl::XMLSource &source, XMLStructureHandler &struct_handler) - { - m_has_error = false; - mp_struct_handler = &struct_handler; - - // Just in case we want to reuse it ... - XML_ParserReset (mp_parser, NULL); - XML_SetUserData (mp_parser, (void *) this); - XML_SetElementHandler (mp_parser, start_element_handler, end_element_handler); - XML_SetCharacterDataHandler (mp_parser, cdata_handler); - - const int chunk = 65536; - char buffer [chunk]; - - int n; - - do { - - n = source.source ()->read (buffer, chunk); - if (n < 0) { - break; - } - - XML_Status status = XML_Parse (mp_parser, buffer, n, n < chunk /*is final*/); - if (status == XML_STATUS_ERROR) { - m_has_error = true; - m_error = XML_ErrorString (XML_GetErrorCode (mp_parser)); - m_error_line = XML_GetErrorLineNumber (mp_parser); - m_error_column = XML_GetErrorColumnNumber (mp_parser); - } - - } while (n == chunk && !m_has_error); - } - - void check_error () - { - if (m_has_error) { - throw tl::XMLLocatedException (m_error, m_error_line, m_error_column); - } - } - -private: - void error (tl::Exception &ex) - { - m_has_error = true; - m_error_line = XML_GetCurrentLineNumber (mp_parser); - m_error_column = XML_GetCurrentColumnNumber (mp_parser); - m_error = ex.msg (); - } - - XML_Parser mp_parser; - XMLStructureHandler *mp_struct_handler; - bool m_has_error; - std::string m_error; - int m_error_line, m_error_column; -}; - -void start_element_handler (void *user_data, const XML_Char *name, const XML_Char ** /*atts*/) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->start_element (std::string (name)); -} - -void end_element_handler (void *user_data, const XML_Char *name) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->end_element (std::string (name)); -} - -void cdata_handler (void *user_data, const XML_Char *s, int len) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->cdata (std::string (s, size_t (len))); -} - - -XMLParser::XMLParser () - : mp_data (new XMLParserPrivateData ()) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - delete mp_data; - mp_data = 0; -} - -void -XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) -{ - mp_data->parse (source, struct_handler); - - // throws an exception if there is an error - mp_data->check_error (); -} - -bool -XMLParser::is_available () -{ - return true; -} - -} - -#elif defined(HAVE_QT) - -#include -#include -#include - -namespace tl -{ - -// -------------------------------------------------------------------- -// A SAX handler for the Qt implementation - -class SAXHandler - : public QXmlDefaultHandler -{ -public: - SAXHandler (XMLStructureHandler *sh); - - virtual bool characters (const QString &ch); - virtual bool endElement (const QString &namespaceURI, const QString &localName, const QString &qName); - virtual bool startElement (const QString &namespaceURI, const QString &localName, const QString &qName, const QXmlAttributes &atts); - virtual bool error (const QXmlParseException &exception); - virtual bool fatalError (const QXmlParseException &exception); - virtual bool warning (const QXmlParseException &exception); - virtual QString errorString () const; - - void setDocumentLocator (QXmlLocator *locator); - - const tl::XMLLocatedException *exception () const - { - return m_error.get (); - } - -private: - QXmlLocator *mp_locator; - XMLStructureHandler *mp_struct_handler; - std::unique_ptr m_error; - std::string m_error_string; -}; - -// -------------------------------------------------------------------------------------------------------- -// trureHandler implementation - -SAXHandler::SAXHandler (XMLStructureHandler *sh) - : QXmlDefaultHandler (), mp_locator (0), mp_struct_handler (sh) -{ - // .. nothing yet .. -} - -void -SAXHandler::setDocumentLocator (QXmlLocator *locator) -{ - mp_locator = locator; -} - -bool -SAXHandler::startElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname, const QXmlAttributes & /*atts*/) -{ - std::string uri (tl::to_string (qs_uri)); - std::string lname (tl::to_string (qs_lname)); - std::string qname (tl::to_string (qs_qname)); - - try { - mp_struct_handler->start_element (uri, lname, qname); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -bool -SAXHandler::endElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname) -{ - std::string uri (tl::to_string (qs_uri)); - std::string lname (tl::to_string (qs_lname)); - std::string qname (tl::to_string (qs_qname)); - - try { - mp_struct_handler->end_element (uri, lname, qname); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -bool -SAXHandler::characters (const QString &t) -{ - try { - mp_struct_handler->characters (tl::to_string (t)); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -QString -SAXHandler::errorString () const -{ - return tl::to_qstring (m_error_string); -} - - -bool -SAXHandler::error (const QXmlParseException &ex) -{ - m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); - // stop reading - return false; -} - -bool -SAXHandler::fatalError (const QXmlParseException &ex) -{ - m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); - // stop reading - return false; -} - -bool -SAXHandler::warning (const QXmlParseException &ex) -{ - tl::XMLLocatedException lex (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ()); - tl::warn << lex.msg (); - // continue - return true; -} - -// -------------------------------------------------------------------- -// SourcePrivateData implementation - -class XMLSourcePrivateData - : public QXmlInputSource -{ -public: - XMLSourcePrivateData () - : QXmlInputSource () - { - // .. nothing yet .. - } - - XMLSourcePrivateData (QIODevice *dev) - : QXmlInputSource (dev) - { - // .. nothing yet .. - } -}; - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - delete mp_source; - mp_source = 0; -} - -void -XMLSource::reset () -{ - mp_source->reset (); -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &string) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (string.c_str ())); - set_source (source); -} - -XMLStringSource::XMLStringSource (const char *cp) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (cp)); - set_source (source); -} - -XMLStringSource::XMLStringSource (const char *cp, size_t len) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (cp, int (len))); - set_source (source); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// StreamIODevice definition and implementation - -class StreamIODevice - : public QIODevice -{ -public: - StreamIODevice (tl::InputStream &stream) - : mp_stream (&stream), - mp_progress (0), - m_has_error (false) - { - open (QIODevice::ReadOnly); - } - - StreamIODevice (tl::InputStream &stream, const std::string &progress_message) - : mp_stream (&stream), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - open (QIODevice::ReadOnly); - } - - StreamIODevice (const std::string &path) - : mp_stream_holder (new tl::InputStream (path)), - mp_progress (0), - m_has_error (false) - { - mp_stream = mp_stream_holder.get (); - open (QIODevice::ReadOnly); - } - - StreamIODevice (const std::string &path, const std::string &progress_message) - : mp_stream_holder (new tl::InputStream (path)), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = mp_stream_holder.get (); - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - open (QIODevice::ReadOnly); - } - - ~StreamIODevice () - { - if (mp_progress) { - delete mp_progress; - mp_progress = 0; - } - } - - virtual bool isSequential () const - { - return true; - } - - qint64 writeData (const char *, qint64) - { - tl_assert (false); - } - - qint64 readData (char *data, qint64 n) - { - try { - - if (mp_progress) { - mp_progress->set (mp_stream->pos ()); - } - - qint64 n0 = n; - for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; ) { - // NOTE: we skip CR to compensate for Windows CRLF line terminators (issue #419). - if (*rd != '\r') { - *data++ = *rd; - --n; - } - } - - if (n0 == n) { - return -1; - } else { - return n0 - n; - } - - } catch (tl::Exception &ex) { - setErrorString (tl::to_qstring (ex.msg ())); - m_has_error = true; - return -1; - } - } - - bool has_error () const - { - return m_has_error; - } - -private: - tl::InputStream *mp_stream; - std::unique_ptr mp_stream_holder; - tl::AbsoluteProgress *mp_progress; - bool m_has_error; -}; - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -class XMLStreamSourcePrivateData - : public XMLSourcePrivateData -{ -public: - XMLStreamSourcePrivateData (StreamIODevice *io) - : XMLSourcePrivateData (io), mp_io (io) - { - // .. nothing yet .. - } - - virtual void fetchData () - { - QXmlInputSource::fetchData (); - - // This feature is actually missing in the original implementation: throw an exception on error - if (mp_io->has_error ()) { - throw tl::Exception (tl::to_string (mp_io->errorString ())); - } - } - -private: - std::unique_ptr mp_io; -}; - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path, progress_message))); -} - -XMLFileSource::XMLFileSource (const std::string &path) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path))); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s, progress_message))); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &s) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s))); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -class XMLParserPrivateData - : public QXmlSimpleReader -{ -public: - XMLParserPrivateData () : QXmlSimpleReader () { } -}; - -XMLParser::XMLParser () - : mp_data (new XMLParserPrivateData ()) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - delete mp_data; - mp_data = 0; -} - -void -XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) -{ - SAXHandler handler (&struct_handler); - - mp_data->setContentHandler (&handler); - mp_data->setErrorHandler (&handler); - - bool result = mp_data->parse (source.source (), false /*=not incremental*/); - if (! result && handler.exception ()) { - throw tl::XMLLocatedException (*handler.exception ()); - } -} - -bool -XMLParser::is_available () -{ - return true; -} - -} - -#else - -namespace tl -{ - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - // .. nothing yet .. -} - -void -XMLSource::reset () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &) -{ - tl_assert (false); -} - -XMLStringSource::XMLStringSource (const char *) -{ - tl_assert (false); -} - -XMLStringSource::XMLStringSource (const char *, size_t) -{ - tl_assert (false); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &, const std::string &) -{ - tl_assert (false); -} - -XMLFileSource::XMLFileSource (const std::string &) -{ - tl_assert (false); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &, const std::string &) -{ - tl_assert (false); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &) -{ - tl_assert (false); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -XMLParser::XMLParser () - : mp_data (0) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - // .. nothing yet .. -} - -void -XMLParser::parse (XMLSource &, XMLStructureHandler &) -{ - tl_assert (false); -} - -bool -XMLParser::is_available () -{ - return false; -} - -} - -#endif - namespace tl { // ----------------------------------------------------------------- @@ -942,8 +62,140 @@ XMLElementProxy::~XMLElementProxy () mp_ptr = 0; } +// ----------------------------------------------------------------- +// XMLElementList implementation + +static size_t s_oid = 0; + +XMLElementList::XMLElementList () + : m_oid (++s_oid) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementBase &e) + : m_oid (++s_oid) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +XMLElementList::XMLElementList (XMLElementBase *e) + : m_oid (++s_oid) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +XMLElementList::XMLElementList (const std::string &name, const XMLElementList &d) + : m_elements (d.m_elements), m_oid (d.m_oid), m_name (name) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementList &d) + : m_elements (d.m_elements), m_oid (d.m_oid), m_name (d.m_name) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementList &d, const XMLElementBase &e) + : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +XMLElementList::XMLElementList (const XMLElementList &d, XMLElementBase *e) + : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +void XMLElementList::append (const XMLElementBase &e) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +void XMLElementList::append (XMLElementBase *e) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +XMLElementList::iterator XMLElementList::begin () const +{ + return m_elements.begin (); +} + +XMLElementList::iterator XMLElementList::end () const +{ + return m_elements.end (); +} + +XMLElementList XMLElementList::empty () +{ + return XMLElementList (); +} + +// ----------------------------------------------------------------- // XMLElementBase implementation +static std::string parse_name (const std::string &n) +{ + auto hash = n.find ("#"); + if (hash != std::string::npos) { + return std::string (n, 0, hash); + } else { + return n; + } +} + +static int parse_tag (const std::string &n) +{ + auto hash = n.find ("#"); + if (hash != std::string::npos) { + tl::Extractor ex (n.c_str () + hash + 1); + int tag; + if (ex.try_read (tag)) { + return tag; + } + } + return -1; +} + +XMLElementBase::XMLElementBase (const std::string &name, const XMLElementList &children) + : m_name (parse_name (name)), m_tag (parse_tag (name)), mp_children (new XMLElementList (children)), m_owns_child_list (true) +{ + // .. nothing yet .. +} + +XMLElementBase::XMLElementBase (const std::string &name, const XMLElementList *children) + : m_name (parse_name (name)), m_tag (parse_tag (name)), mp_children (children), m_owns_child_list (false) +{ + // .. nothing yet .. +} + +XMLElementBase::XMLElementBase (const XMLElementBase &d) + : m_name (d.m_name), m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) +{ + if (m_owns_child_list) { + mp_children = new XMLElementList (*d.mp_children); + } else { + mp_children = d.mp_children; + } +} + +XMLElementBase::~XMLElementBase () +{ + if (m_owns_child_list) { + delete const_cast (mp_children); + mp_children = 0; + } +} + void XMLElementBase::write_indent (tl::OutputStream &os, int indent) { @@ -975,6 +227,127 @@ XMLElementBase::write_string (tl::OutputStream &os, const std::string &s) } } +bool +XMLElementBase::check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const +{ + if (m_name == "*") { + return true; + } else { + return m_name == lname; // no namespace currently + } +} + +XMLElementBase::Cardinality +XMLElementBase::cardinality () const +{ + return Zero; +} + +XMLElementBase::iterator +XMLElementBase::begin () const +{ + return mp_children->begin (); +} + +XMLElementBase::iterator +XMLElementBase::end () const +{ + return mp_children->end (); +} + +std::string +XMLElementBase::name4code () const +{ + std::string res; + const char *n = name ().c_str (); + + if (! isalpha (*n) && *n != '_') { + res += '_'; + } + + while (*n) { + if (*n == '-') { + res += '_'; + } else if (isalnum (*n) || *n == '_') { + res += *n; + } + ++n; + } + + return res; +} + +std::string +XMLElementBase::create_def (std::map > &messages) const +{ + std::string res; + + auto m = messages.find (oid ()); + if (m != messages.end ()) { + + res += "message " + m->second.second + " {\n"; + + for (auto i = begin (); i != end (); ++i) { + const XMLElementBase *e = i->get (); + Cardinality c = e->cardinality (); + std::string entry = e->create_def_entry (messages); + if (! entry.empty ()) { + res += " "; + if (c != Zero) { + if (c == Many) { + res += "repeated "; + } else { + res += "optional "; + } + res += entry + "\n"; + } + } + } + + res += "}"; + + } + + return res; +} + +void +XMLElementBase::collect_messages (std::map > &messages) const +{ + for (auto i = begin (); i != end (); ++i) { + i->get ()->collect_messages (messages); + } +} + +std::string +XMLElementBase::make_message_name () const +{ + std::string res = mp_children->name (); + if (! res.empty ()) { + return res; + } + + // Capitalize names + std::string n4c = name4code (); + + bool upcase = true; + + const char *n = n4c.c_str (); + while (*n) { + if (*n == '_') { + upcase = true; + } else if (upcase) { + res += toupper (*n); + upcase = false; + } else { + res += *n; + } + ++n; + } + + return res; +} + // -------------------------------------------------------------------------------------------------------- // trureHandler implementation @@ -1066,5 +439,76 @@ XMLWriterState::XMLWriterState () // .. nothing yet .. } +// -------------------------------------------------------------------- +// PBParser implementation + +PBParser::PBParser () +{ + // .. nothing yet .. +} + +PBParser::~PBParser () +{ + // .. nothing yet .. +} + +void +PBParser::parse (tl::ProtocolBufferReaderBase &reader, const XMLElementBase *root, XMLReaderState *reader_state) +{ + mp_state = reader_state; + parse_element (root, reader); +} + +void +PBParser::parse_element (const XMLElementBase *parent, tl::ProtocolBufferReaderBase &reader) +{ + while (! reader.at_end ()) { + + int tag = reader.read_tag (); + + const XMLElementBase *new_element = 0; + if (parent) { + for (XMLElementBase::iterator c = parent->begin (); c != parent->end (); ++c) { + if ((*c)->tag () == tag) { + new_element = (*c).get (); + break; + } + } + } + + if (! new_element) { + reader.skip (); + } else { + new_element->pb_create (parent, *mp_state); + new_element->pb_parse (this, reader); + new_element->pb_finish (parent, *mp_state); + } + + } +} + +void +PBParser::expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name) +{ + int tag = reader.read_tag (); + if (tag != name_tag) { + reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with ID %d (got %d)")), name_tag, tag)); + } + + std::string n; + reader.read (n); + if (n != name) { + reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with string '%s' (got '%s')")), name, n)); + } +} + +// -------------------------------------------------------------------- +// PBWriterState implementation + +PBWriterState::PBWriterState () +{ + // .. nothing yet .. +} + } diff --git a/src/tl/tl/tlXMLParser.h b/src/tl/tl/tlXMLParser.h index b53ad22c7..1c3b1e130 100644 --- a/src/tl/tl/tlXMLParser.h +++ b/src/tl/tl/tlXMLParser.h @@ -29,6 +29,8 @@ #include #include +#include "tlXMLReader.h" +#include "tlProtocolBuffer.h" #include "tlAssert.h" #include "tlInternational.h" #include "tlString.h" @@ -40,339 +42,12 @@ namespace tl { /** - * @brief A basic XML parser error exception class + * NOTE: This XML parser package also supports a ProtocolBuffer flavor. + * This allows binding the same scheme to efficient binary PB format. */ -class TL_PUBLIC XMLException : public tl::Exception -{ -public: - XMLException (const char *msg) - : Exception (tl::to_string (tr ("XML parser error: %s")).c_str ()), - m_msg (msg) - { - // .. nothing yet .. - } - - XMLException (const std::string &msg) - : Exception (fmt (-1, -1).c_str (), msg.c_str ()), - m_msg (msg) - { - // .. nothing yet .. - } - - /** - * @brief Raw (unprefixed) message of the XML parser - */ - const std::string & - raw_msg () const - { - return m_msg; - } - -protected: - XMLException (const std::string &msg, int line, int column) - : Exception (fmt (line, column).c_str (), msg.c_str (), line, column), - m_msg (msg) - { - // .. nothing yet .. - } - -private: - std::string m_msg; - - static std::string fmt (int line, int /*column*/) - { - if (line < 0) { - return tl::to_string (tr ("XML parser error: %s")).c_str (); - } else { - return tl::to_string (tr ("XML parser error: %s in line %d, column %d")).c_str (); - } - } -}; - -/** - * @brief A XML parser error exception class that additionally provides line and column information - */ - -class TL_PUBLIC XMLLocatedException : public XMLException -{ -public: - XMLLocatedException (const std::string &msg, int line, int column) - : XMLException (msg, line, column), - m_line (line), m_column (column) - { - // .. nothing yet .. - } - - /** - * @brief Line number information of the exception - */ - int line () const - { - return m_line; - } - - /** - * @brief Column number information of the exception - */ - int column () const - { - return m_column; - } - -private: - int m_line; - int m_column; -}; - -/** - * @brief An object wrapper base class for target object management - * - * Implementations of this class through the XMLReaderProxy templates - * manage pointers to certain objects. - */ - -class TL_PUBLIC XMLReaderProxyBase -{ -public: - XMLReaderProxyBase () { } - virtual ~XMLReaderProxyBase () { } - virtual void release () = 0; - virtual void detach () = 0; -}; - -/** - * @brief An object wrapper base class for target object management specialized to a certain class - */ - -template -class TL_PUBLIC_TEMPLATE XMLReaderProxy - : public XMLReaderProxyBase -{ -public: - XMLReaderProxy (Obj *obj, bool owns_obj) - : mp_obj (obj), m_owns_obj (owns_obj) - { } - - virtual ~XMLReaderProxy () { } - - virtual void release () - { - if (m_owns_obj && mp_obj) { - delete mp_obj; - } - mp_obj = 0; - } - - virtual void detach () - { - m_owns_obj = false; - } - - Obj *ptr () const - { - return mp_obj; - } - -private: - Obj *mp_obj; - bool m_owns_obj; -}; - -/** - * @brief Helper class: A class tag - */ - -template -struct XMLObjTag -{ - XMLObjTag() { } - typedef Obj obj; -}; - -/** - * @brief Helper class: The reader state - * - * The reader state mainly comprises of a stack of objects being parsed and - * a string in which to collect cdata. - */ - -class TL_PUBLIC XMLReaderState -{ -public: - /** - * @brief Default constructor - */ - XMLReaderState (); - - /** - * @brief Destructor - */ - ~XMLReaderState (); - - /** - * @brief Push a new object on the stack - */ - template - void push (XMLObjTag /*tag*/) - { - m_objects.push_back (new XMLReaderProxy (new Obj (), true)); - } - - /** - * @brief Push an existing object on the stack - */ - template - void push (Obj *obj) - { - m_objects.push_back (new XMLReaderProxy (obj, false)); - } - - /** - * @brief Push an existing object on the stack with the ownership flag - */ - template - void push (Obj *obj, bool owner) - { - m_objects.push_back (new XMLReaderProxy (obj, owner)); - } - - /** - * @brief Get the top object - */ - template - Obj *back (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - return (dynamic_cast &> (*m_objects.back ())).ptr (); - } - - /** - * @brief Get the top object and release - */ - template - Obj *detach_back (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - m_objects.back ()->detach (); - return (dynamic_cast &> (*m_objects.back ())).ptr (); - } - - /** - * @brief Pop an object from the stack - */ - template - void pop (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - m_objects.back ()->release (); - delete m_objects.back (); - m_objects.pop_back (); - } - - /** - * @brief Empty predicate: true, if no more object is on the stack - */ - bool empty () const - { - return m_objects.empty (); - } - - /** - * @brief Obtain the parent object from the stack - */ - template - Obj *parent (XMLObjTag /*tag*/) - { - tl_assert (m_objects.size () > 1); - return (dynamic_cast &> (*m_objects.end () [-2])).ptr (); - } - - /** - * @brief The cdata string collected - */ - std::string cdata; - -private: - std::vector m_objects; -}; - -// The opaque source type -class XMLSourcePrivateData; - -/** - * @brief A generic XML text source class - * - * This class is the base class providing input for - * the Qt XML parser and basically maps to a QXmlInputSource object - * for compatibility with the "libparsifal" branch. - */ - -class TL_PUBLIC XMLSource -{ -public: - XMLSource (); - ~XMLSource (); - - XMLSourcePrivateData *source () - { - return mp_source; - } - - void reset (); - -protected: - void set_source (XMLSourcePrivateData *source) - { - mp_source = source; - } - -private: - XMLSourcePrivateData *mp_source; -}; - -/** - * @brief A specialization of XMLSource to receive a string - */ - -class TL_PUBLIC XMLStringSource : public XMLSource -{ -public: - XMLStringSource (const std::string &string); - XMLStringSource (const char *cp); - XMLStringSource (const char *cp, size_t len); - ~XMLStringSource (); - -private: - std::string m_copy; -}; - -/** - * @brief A specialization of XMLSource to receive from a file - */ - -class TL_PUBLIC XMLFileSource : public XMLSource -{ -public: - XMLFileSource (const std::string &path); - XMLFileSource (const std::string &path, const std::string &progress_message); - ~XMLFileSource (); -}; - -/** - * @brief A generic stream source class - * - * This class implements a XML parser source from a tl::InputStream - */ - -class TL_PUBLIC XMLStreamSource : public XMLSource -{ -public: - XMLStreamSource (tl::InputStream &stream); - XMLStreamSource (tl::InputStream &stream, const std::string &progress_message); - ~XMLStreamSource (); -}; - +class XMLReaderState; +class XMLReaderState; // ----------------------------------------------------------------- // The C++ structure definition interface (for use cases see tlXMLParser.ut) @@ -387,6 +62,18 @@ struct pass_by_ref_tag { pass_by_ref_tag () { } }; +struct zero_cardinality_tag { + zero_cardinality_tag () { } +}; + +struct single_cardinality_tag { + single_cardinality_tag () { } +}; + +struct many_cardinality_tag { + many_cardinality_tag () { } +}; + /** * @brief The XML handler implementation * @@ -483,66 +170,208 @@ public: typedef std::list children_list; typedef children_list::const_iterator iterator; - XMLElementList () + XMLElementList (); + XMLElementList (const XMLElementBase &e); + XMLElementList (XMLElementBase *e); + XMLElementList (const std::string &name, const XMLElementList &d); + XMLElementList (const XMLElementList &d); + XMLElementList (const XMLElementList &d, const XMLElementBase &e); + XMLElementList (const XMLElementList &d, XMLElementBase *e); + + void append (const XMLElementBase &e); + void append (XMLElementBase *e); + + iterator begin () const; + iterator end () const; + + static XMLElementList empty (); + + size_t oid () const { - // .. nothing yet .. + return m_oid; } - XMLElementList (const XMLElementBase &e) + const std::string &name () const { - m_elements.push_back (XMLElementProxy (e)); - } - - XMLElementList (XMLElementBase *e) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - XMLElementList (const XMLElementList &d, const XMLElementBase &e) - : m_elements (d.m_elements) - { - m_elements.push_back (XMLElementProxy (e)); - } - - XMLElementList (const XMLElementList &d, XMLElementBase *e) - : m_elements (d.m_elements) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - void append (const XMLElementBase &e) - { - m_elements.push_back (XMLElementProxy (e)); - } - - void append (XMLElementBase *e) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - iterator begin () const - { - return m_elements.begin (); - } - - iterator end () const - { - return m_elements.end (); - } - - static XMLElementList empty () - { - return XMLElementList (); + return m_name; } private: std::list m_elements; + size_t m_oid; + std::string m_name; +}; + +/** + * @brief An object wrapper base class for target object management + * + * Implementations of this class through the XMLReaderProxy templates + * manage pointers to certain objects. + */ + +class TL_PUBLIC XMLReaderProxyBase +{ +public: + XMLReaderProxyBase () { } + virtual ~XMLReaderProxyBase () { } + virtual void release () = 0; + virtual void detach () = 0; +}; + +/** + * @brief An object wrapper base class for target object management specialized to a certain class + */ + +template +class TL_PUBLIC_TEMPLATE XMLReaderProxy + : public XMLReaderProxyBase +{ +public: + XMLReaderProxy (Obj *obj, bool owns_obj) + : mp_obj (obj), m_owns_obj (owns_obj) + { } + + virtual ~XMLReaderProxy () { } + + virtual void release () + { + if (m_owns_obj && mp_obj) { + delete mp_obj; + } + mp_obj = 0; + } + + virtual void detach () + { + m_owns_obj = false; + } + + Obj *ptr () const + { + return mp_obj; + } + +private: + Obj *mp_obj; + bool m_owns_obj; +}; + +/** + * @brief Helper class: A class tag + */ + +template +struct XMLObjTag +{ + XMLObjTag() { } + typedef Obj obj; +}; + +/** + * @brief Helper class: The reader state + * + * The reader state mainly comprises of a stack of objects being parsed and + * a string in which to collect cdata. + */ + +class TL_PUBLIC XMLReaderState +{ +public: + /** + * @brief Default constructor + */ + XMLReaderState (); + + /** + * @brief Destructor + */ + ~XMLReaderState (); + + /** + * @brief Push a new object on the stack + */ + template + void push (XMLObjTag /*tag*/) + { + m_objects.push_back (new XMLReaderProxy (new Obj (), true)); + } + + /** + * @brief Push an existing object on the stack + */ + template + void push (Obj *obj) + { + m_objects.push_back (new XMLReaderProxy (obj, false)); + } + + /** + * @brief Push an existing object on the stack with the ownership flag + */ + template + void push (Obj *obj, bool owner) + { + m_objects.push_back (new XMLReaderProxy (obj, owner)); + } + + /** + * @brief Get the top object + */ + template + Obj *back (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + return (dynamic_cast &> (*m_objects.back ())).ptr (); + } + + /** + * @brief Get the top object and release + */ + template + Obj *detach_back (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + m_objects.back ()->detach (); + return (dynamic_cast &> (*m_objects.back ())).ptr (); + } + + /** + * @brief Pop an object from the stack + */ + template + void pop (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + m_objects.back ()->release (); + delete m_objects.back (); + m_objects.pop_back (); + } + + /** + * @brief Empty predicate: true, if no more object is on the stack + */ + bool empty () const + { + return m_objects.empty (); + } + + /** + * @brief Obtain the parent object from the stack + */ + template + Obj *parent (XMLObjTag /*tag*/) + { + tl_assert (m_objects.size () > 1); + return (dynamic_cast &> (*m_objects.end () [-2])).ptr (); + } + + /** + * @brief The cdata string collected + */ + std::string cdata; + +private: + std::vector m_objects; }; /** @@ -592,6 +421,78 @@ private: std::vector m_objects; }; +/** + * @brief The PB parser class + * This is the main entry point. It will take a PB reader, the structure (in "root") and + * a reader state initialized with the top level object. + */ +class TL_PUBLIC PBParser +{ +public: + PBParser (); + ~PBParser (); + + void parse (tl::ProtocolBufferReaderBase &reader, const XMLElementBase *root, XMLReaderState *reader_state); + void parse_element (const XMLElementBase *parent, tl::ProtocolBufferReaderBase &reader); + + void expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name); + + XMLReaderState &reader_state () + { + return *mp_state; + } + +private: + XMLReaderState *mp_state; +}; + +/** + * @brief Helper class: A stack of const objects being written (PB binding) + */ + +class TL_PUBLIC PBWriterState +{ +public: + /** + * @brief Default constructor + */ + PBWriterState (); + + /** + * @brief Push a new object on the stack + */ + template + void push (const Obj *obj) + { + m_objects.push_back (obj); + } + + /** + * @brief Pop an object from the stack + */ + template + const Obj *pop (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + const Obj *obj = reinterpret_cast (m_objects.back ()); + m_objects.pop_back (); + return obj; + } + + /** + * @brief Obtain the parent object from the stack + */ + template + const Obj *back (XMLObjTag /*tag*/) + { + tl_assert (m_objects.size () > 0); + return reinterpret_cast (m_objects.end () [-1]); + } + +private: + std::vector m_objects; +}; + /** * @brief The XML element base object * @@ -606,35 +507,14 @@ class TL_PUBLIC XMLElementBase public: typedef XMLElementList::iterator iterator; - XMLElementBase (const std::string &name, const XMLElementList &children) - : m_name (name), mp_children (new XMLElementList (children)), m_owns_child_list (true) - { - // .. nothing yet .. - } + enum Cardinality { + Zero, Single, Many + }; - XMLElementBase (const std::string &name, const XMLElementList *children) - : m_name (name), mp_children (children), m_owns_child_list (false) - { - // .. nothing yet .. - } - - XMLElementBase (const XMLElementBase &d) - : m_name (d.m_name), m_owns_child_list (d.m_owns_child_list) - { - if (m_owns_child_list) { - mp_children = new XMLElementList (*d.mp_children); - } else { - mp_children = d.mp_children; - } - } - - virtual ~XMLElementBase () - { - if (m_owns_child_list) { - delete const_cast (mp_children); - mp_children = 0; - } - } + XMLElementBase (const std::string &name, const XMLElementList &children); + XMLElementBase (const std::string &name, const XMLElementList *children); + XMLElementBase (const XMLElementBase &d); + virtual ~XMLElementBase (); virtual XMLElementBase *clone () const = 0; @@ -648,36 +528,235 @@ public: static void write_indent (tl::OutputStream &os, int indent); static void write_string (tl::OutputStream &os, const std::string &s); + virtual void pb_create (const XMLElementBase *parent, XMLReaderState &objs) const = 0; + virtual void pb_parse (PBParser *, tl::ProtocolBufferReaderBase &) const = 0; + virtual void pb_finish (const XMLElementBase *parent, XMLReaderState &objs) const = 0; + + virtual void pb_write (const XMLElementBase *, tl::ProtocolBufferWriterBase &, PBWriterState &) const { } + + virtual Cardinality cardinality () const; + + size_t oid () const + { + return mp_children->oid (); + } + + int tag () const + { + return m_tag; + } + const std::string &name () const { return m_name; } - bool check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const + bool check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const; + + /** + * @brief Returns a name suitable for code + * Specifically, hyphens are replaced by underscores. + */ + std::string name4code () const; + + iterator begin () const; + iterator end () const; + + std::string create_def (std::map > &messages) const; + +protected: + virtual void collect_messages (std::map > &messages) const; + virtual std::string create_def_entry (std::map > &messages) const = 0; + + std::string make_message_name () const; + + static Cardinality get_cardinality (tl::zero_cardinality_tag) { - if (m_name == "*") { - return true; - } else { - return m_name == lname; // no namespace currently - } + return Zero; } - iterator begin () const + static Cardinality get_cardinality (tl::single_cardinality_tag) { - return mp_children->begin (); + return Single; } - iterator end () const + static Cardinality get_cardinality (tl::many_cardinality_tag) { - return mp_children->end (); + return Many; } private: std::string m_name; + int m_tag; const XMLElementList *mp_children; bool m_owns_child_list; }; +/** + * @brief Basic implementation for XMLElement and XMLElementWithParentRef + */ + +template +class TL_PUBLIC_TEMPLATE XMLElementImplBase + : public XMLElementBase +{ +public: + XMLElementImplBase (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) + : XMLElementBase (name, children), m_r (r), m_w (w) + { + // .. nothing yet .. + } + + XMLElementImplBase (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) + : XMLElementBase (name, children), m_r (r), m_w (w) + { + // .. nothing yet .. + } + + XMLElementImplBase (const XMLElementImplBase &d) + : XMLElementBase (d), m_r (d.m_r), m_w (d.m_w) + { + // .. nothing yet .. + } + + virtual void cdata (const std::string & /*cdata*/, XMLReaderState & /*objs*/) const + { + // .. nothing yet .. + } + + virtual void write (const XMLElementBase * /*parent*/, tl::OutputStream &os, int indent, XMLWriterState &objs) const + { + XMLObjTag parent_tag; + Read r (m_r); + r.start (*objs.back (parent_tag)); + while (! r.at_end ()) { + XMLElementBase::write_indent (os, indent); + os << "<" << this->name () << ">\n"; + typedef typename Read::tag read_tag_type; + read_tag_type read_tag; + write_obj (r (), os, indent, read_tag, objs); + XMLElementBase::write_indent (os, indent); + os << "name () << ">\n"; + r.next (); + } + } + + virtual bool has_any (XMLWriterState &objs) const + { + XMLObjTag parent_tag; + Read r (m_r); + r.start (*objs.back (parent_tag)); + return (! r.at_end ()); + } + + virtual void pb_parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const + { + reader.open (); + parser->parse_element (this, reader); + reader.close (); + } + + virtual void pb_write (const XMLElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const + { + XMLObjTag parent_tag; + + Read r (m_r); + r.start (*objs.back (parent_tag)); + while (! r.at_end ()) { + typedef typename Read::tag read_tag_type; + pb_write_obj (r (), tag (), writer, read_tag_type (), objs); + r.next (); + } + } + + virtual Cardinality cardinality () const + { + typedef typename Read::cardinality cardinality_type; + return get_cardinality (cardinality_type ()); + } + + virtual void collect_messages (std::map > &messages) const + { + if (messages.find (oid ()) == messages.end ()) { + messages [oid ()] = std::make_pair (this, make_message_name ()); + XMLElementBase::collect_messages (messages); + } + } + + virtual std::string create_def_entry (std::map > &messages) const + { + auto m = messages.find (oid ()); + if (m != messages.end ()) { + return m->second.second + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; + } else { + return std::string (); + } + } + +protected: + Read m_r; + Write m_w; + +private: + // this write helper is used if the reader delivers an object by value + void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, XMLWriterState &objs) const + { + XMLObjTag tag; + objs.push (&obj); + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, os, indent + 1, objs); + } + objs.pop (tag); + } + + void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, XMLWriterState &objs) const + { + XMLObjTag tag; + objs.push (&obj); + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, os, indent + 1, objs); + } + objs.pop (tag); + } + + // this write helper is used if the reader delivers an object by value + void pb_write_obj (Obj obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_value_tag, PBWriterState &objs) const + { + XMLObjTag self_tag; + + for (unsigned int pass = 0; pass < 2; ++pass) { + writer.begin_seq (tag, pass == 0); + objs.push (&obj); + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->pb_write (this, writer, objs); + } + objs.pop (self_tag); + writer.end_seq (); + if (! writer.is_counting ()) { + break; + } + } + } + + void pb_write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_ref_tag, PBWriterState &objs) const + { + XMLObjTag self_tag; + + for (unsigned int pass = 0; pass < 2; ++pass) { + writer.begin_seq (tag, pass == 0); + objs.push (&obj); + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->pb_write (this, writer, objs); + } + objs.pop (self_tag); + writer.end_seq (); + if (writer.is_counting ()) { + break; + } + } + } +}; + /** * @brief A XML child element * @@ -697,23 +776,23 @@ private: template class TL_PUBLIC_TEMPLATE XMLElement - : public XMLElementBase + : public XMLElementImplBase { public: XMLElement (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) - : XMLElementBase (name, children), m_r (r), m_w (w) + : XMLElementImplBase (r, w, name, children) { // .. nothing yet .. } XMLElement (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) - : XMLElementBase (name, children), m_r (r), m_w (w) + : XMLElementImplBase (r, w, name, children) { // .. nothing yet .. } XMLElement (const XMLElement &d) - : XMLElementBase (d), m_r (d.m_r), m_w (d.m_w) + : XMLElementImplBase (d) { // .. nothing yet .. } @@ -729,66 +808,25 @@ public: objs.push (tag); } - virtual void cdata (const std::string & /*cdata*/, XMLReaderState & /*objs*/) const - { - // .. nothing yet .. - } - virtual void finish (const XMLElementBase * /*parent*/, XMLReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const { XMLObjTag tag; XMLObjTag parent_tag; - m_w (*objs.parent (parent_tag), objs); + this->m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } - virtual void write (const XMLElementBase * /*parent*/, tl::OutputStream &os, int indent, XMLWriterState &objs) const - { - XMLObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - while (! r.at_end ()) { - XMLElementBase::write_indent (os, indent); - os << "<" << this->name () << ">\n"; - typedef typename Read::tag read_tag_type; - read_tag_type read_tag; - write_obj (r (), os, indent, read_tag, objs); - XMLElementBase::write_indent (os, indent); - os << "name () << ">\n"; - r.next (); - } - } - - virtual bool has_any (XMLWriterState &objs) const - { - XMLObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - return (! r.at_end ()); - } - -private: - Read m_r; - Write m_w; - - // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, XMLWriterState &objs) const + virtual void pb_create (const XMLElementBase *, XMLReaderState &objs) const { XMLObjTag tag; - objs.push (&obj); - for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } - objs.pop (tag); + objs.push (tag); } - void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, XMLWriterState &objs) const + virtual void pb_finish (const XMLElementBase * /*parent*/, XMLReaderState &objs) const { XMLObjTag tag; - objs.push (&obj); - for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } + XMLObjTag parent_tag; + this->m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } }; @@ -802,23 +840,23 @@ private: template class TL_PUBLIC_TEMPLATE XMLElementWithParentRef - : public XMLElementBase + : public XMLElementImplBase { public: XMLElementWithParentRef (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) - : XMLElementBase (name, children), m_r (r), m_w (w) + : XMLElementImplBase (r, w, name, children) { // .. nothing yet .. } XMLElementWithParentRef (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) - : XMLElementBase (name, children), m_r (r), m_w (w) + : XMLElementImplBase (r, w, name, children) { // .. nothing yet .. } XMLElementWithParentRef (const XMLElementWithParentRef &d) - : XMLElementBase (d), m_r (d.m_r), m_w (d.m_w) + : XMLElementImplBase (d) { // .. nothing yet .. } @@ -835,66 +873,26 @@ public: objs.push (new Obj (objs.back (parent_tag)), true); } - virtual void cdata (const std::string & /*cdata*/, XMLReaderState & /*objs*/) const - { - // .. nothing yet .. - } - virtual void finish (const XMLElementBase * /*parent*/, XMLReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const { XMLObjTag tag; XMLObjTag parent_tag; - m_w (*objs.parent (parent_tag), objs); + this->m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } - virtual void write (const XMLElementBase * /*parent*/, tl::OutputStream &os, int indent, XMLWriterState &objs) const - { - XMLObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - while (! r.at_end ()) { - XMLElementBase::write_indent (os, indent); - os << "<" << this->name () << ">\n"; - typedef typename Read::tag read_tag_type; - read_tag_type read_tag; - write_obj (r (), os, indent, read_tag, objs); - XMLElementBase::write_indent (os, indent); - os << "name () << ">\n"; - r.next (); - } - } - - virtual bool has_any (XMLWriterState &objs) const - { - XMLObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - return (! r.at_end ()); - } - -private: - Read m_r; - Write m_w; - - // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, XMLWriterState &objs) const + virtual void pb_create (const XMLElementBase *, XMLReaderState &objs) const { XMLObjTag tag; - objs.push (&obj); - for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } - objs.pop (tag); + XMLObjTag parent_tag; + objs.push (new Obj (objs.back (parent_tag)), true); } - void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, XMLWriterState &objs) const + virtual void pb_finish (const XMLElementBase * /*parent*/, XMLReaderState &objs) const { XMLObjTag tag; - objs.push (&obj); - for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } + XMLObjTag parent_tag; + this->m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } }; @@ -992,10 +990,278 @@ public: return (! r.at_end ()); } + virtual void pb_create (const XMLElementBase *, XMLReaderState &) const + { + // .. nothing yet .. + } + + virtual void pb_finish (const XMLElementBase *, XMLReaderState &) const + { + // .. nothing yet .. + } + + virtual void pb_parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const + { + XMLObjTag tag; + XMLObjTag parent_tag; + + XMLReaderState value_obj; + value_obj.push (tag); + + read (reader, *value_obj.back (tag)); + m_w (*parser->reader_state ().back (parent_tag), value_obj); + + value_obj.pop (tag); + } + + virtual void pb_write (const XMLElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const + { + XMLObjTag parent_tag; + Read r (m_r); + r.start (* objs.back (parent_tag)); + while (! r.at_end ()) { + write (writer, tag (), r ()); + r.next (); + } + } + + virtual Cardinality cardinality () const + { + typedef typename Read::cardinality cardinality_type; + return get_cardinality (cardinality_type ()); + } + + virtual void collect_messages (std::map > & /*messages*/) const + { + // no messages here. + } + + virtual std::string create_def_entry (std::map > & /*messages*/) const + { + const Value *v = 0; + return typestring (v) + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; + } + private: Read m_r; Write m_w; Converter m_c; + + // write incarnations + void write (tl::ProtocolBufferWriterBase &writer, int tag, float v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, double v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint8_t v) const + { + writer.write (tag, (uint32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int8_t v) const + { + writer.write (tag, (int32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint16_t v) const + { + writer.write (tag, (uint32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int16_t v) const + { + writer.write (tag, (int32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint32_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int32_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint64_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int64_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, bool v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, const std::string &v) const + { + writer.write (tag, v); + } + + template + void write (tl::ProtocolBufferWriterBase &writer, int tag, const T &v) const + { + writer.write (tag, m_c.pb_encode (v)); + } + + // read incarnations + void read (tl::ProtocolBufferReaderBase &reader, float &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, double &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, uint8_t &v) const + { + uint32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, int8_t &v) const + { + int32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, uint16_t &v) const + { + uint32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, int16_t &v) const + { + int32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, uint32_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, int32_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, uint64_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, int64_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, bool &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, std::string &v) const + { + reader.read (v); + } + + template + void read (tl::ProtocolBufferReaderBase &reader, T &v) const + { + typename Converter::pb_type vv; + reader.read (vv); + m_c.pb_decode (vv, v); + } + + // type strings + std::string typestring (const float *) const + { + return "float"; + } + + std::string typestring (const double *) const + { + return "double"; + } + + std::string typestring (const uint8_t *) const + { + return "uint32"; + } + + std::string typestring (const uint16_t *) const + { + return "uint32"; + } + + std::string typestring (const uint32_t *) const + { + return "uint32"; + } + + std::string typestring (const uint64_t *) const + { + return "uint64"; + } + + std::string typestring (const int8_t *) const + { + return "sint32"; + } + + std::string typestring (const int16_t *) const + { + return "sint32"; + } + + std::string typestring (const int32_t *) const + { + return "sint32"; + } + + std::string typestring (const int64_t *) const + { + return "sint64"; + } + + std::string typestring (const bool *) const + { + return "uint32"; + } + + std::string typestring (const std::string *) const + { + return "string"; + } + + template + std::string typestring (const T *) const + { + const typename Converter::pb_type *v = 0; + return typestring (v); + } }; /** @@ -1060,6 +1326,11 @@ public: return false; } + virtual void pb_create (const XMLElementBase *, XMLReaderState &) const { } + virtual void pb_parse (PBParser *, tl::ProtocolBufferReaderBase &) const { } + virtual void pb_finish (const XMLElementBase *, XMLReaderState &) const { } + virtual std::string create_def_entry (std::map > &) const { return std::string (); } + private: Write m_w; Converter m_c; @@ -1143,11 +1414,90 @@ public: tl_assert (rs.empty ()); } + /** + * @brief Serializes the given object (root) to the writer + */ + void write (tl::ProtocolBufferWriterBase &writer, const Obj &root) const + { + PBWriterState writer_state; + writer_state.push (& root); + + writer.write (tag (), name ()); + + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->pb_write (this, writer, writer_state); + } + } + + /** + * @brief Deserializes the given object (root) from the reader + */ + void parse (tl::ProtocolBufferReaderBase &reader, Obj &root) const + { + XMLObjTag self_tag; + XMLReaderState rs; + rs.push (&root); + PBParser h; + h.expect_header (reader, tag (), name ()); + h.parse (reader, this, &rs); + rs.pop (self_tag); + tl_assert (rs.empty ()); + } + + /** + * @brief Produces a definition for the protoc compiler + */ + std::string create_def () const + { + std::map > msgs; + collect_messages (msgs); + msgs[oid ()] = std::make_pair (this, make_message_name ()); + + std::string res = "// created from KLayout proto definition '" + name () + "'\n\n"; + res += "syntax = \"proto2\";"; + + for (auto i = msgs.begin (); i != msgs.end (); ++i) { + std::string entry = i->second.first->create_def (msgs); + if (! entry.empty ()) { + res += "\n"; + res += "\n"; + res += entry; + } + } + + return res; + } + private: virtual void write (const XMLElementBase*, tl::OutputStream &, int, XMLWriterState &) const { // .. see write (os) } + + virtual void pb_write (const XMLElementBase*, tl::ProtocolBufferWriterBase &, PBWriterState &) const + { + // disable base class implementation + } + + virtual void pb_parse (PBParser *, tl::ProtocolBufferReaderBase &) const + { + // disable base class implementation + } + + virtual void pb_create (const XMLElementBase *, XMLReaderState &) const + { + // disable base class implementation + } + + virtual void pb_finish (const XMLElementBase *, XMLReaderState &) const + { + // disable base class implementation + } + + virtual std::string create_def_entry (std::map > &) const + { + return std::string (); + } }; /** @@ -1267,6 +1617,7 @@ template struct XMLMemberDummyReadAdaptor { typedef pass_by_ref_tag tag; + typedef zero_cardinality_tag cardinality; XMLMemberDummyReadAdaptor () { @@ -1298,6 +1649,7 @@ template struct XMLMemberReadAdaptor { typedef pass_by_ref_tag tag; + typedef single_cardinality_tag cardinality; XMLMemberReadAdaptor (Value Parent::*member) : mp_member (member), mp_owner (0), m_done (false) @@ -1336,6 +1688,7 @@ template struct XMLMemberAccRefReadAdaptor { typedef pass_by_ref_tag tag; + typedef single_cardinality_tag cardinality; XMLMemberAccRefReadAdaptor (const Value &(Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1374,6 +1727,7 @@ template struct XMLMemberAccReadAdaptor { typedef pass_by_value_tag tag; + typedef single_cardinality_tag cardinality; XMLMemberAccReadAdaptor (Value (Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1412,6 +1766,7 @@ template struct XMLMemberIterReadAdaptor { typedef pass_by_ref_tag tag; + typedef many_cardinality_tag cardinality; XMLMemberIterReadAdaptor (Iter (Parent::*begin) () const, Iter (Parent::*end) () const) : mp_begin (begin), mp_end (end) @@ -1771,6 +2126,18 @@ make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end template struct XMLStdConverter { + typedef std::string pb_type; + + pb_type pb_encode (const Value &v) const + { + return tl::to_string (v); + } + + void pb_decode (const pb_type &s, Value &v) const + { + tl::from_string (s, v); + } + std::string to_string (const Value &v) const { return tl::to_string (v); @@ -1782,6 +2149,26 @@ struct XMLStdConverter } }; +/** + * @brief A helper class to convert a string converter to a XML converter + */ +template +struct XMLStringBasedConverter + : public StringConverter +{ + typedef std::string pb_type; + + pb_type pb_encode (const typename StringConverter::value_type &v) const + { + return StringConverter::to_string (v); + } + + void pb_decode (const pb_type &s, typename StringConverter::value_type &v) const + { + StringConverter::from_string (s, v); + } +}; + /** * @brief Utility: create a XMLMember object without read & write capability */ diff --git a/src/tl/tl/tlXMLReader.cc b/src/tl/tl/tlXMLReader.cc new file mode 100644 index 000000000..49f0509dc --- /dev/null +++ b/src/tl/tl/tlXMLReader.cc @@ -0,0 +1,913 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlXMLReader.h" +#include "tlXMLParser.h" +#include "tlString.h" +#include "tlLog.h" +#include "tlAssert.h" +#include "tlProgress.h" + +#include +#include + +#if defined(HAVE_EXPAT) + +#include + +namespace tl +{ + +// -------------------------------------------------------------------- +// SourcePrivateData implementation + +class XMLSourcePrivateData +{ +public: + XMLSourcePrivateData (tl::InputStream *stream) + : mp_stream_holder (stream), + m_has_error (false) + { + mp_stream = stream; + } + + XMLSourcePrivateData (tl::InputStream *stream, const std::string &progress_message) + : mp_stream_holder (stream), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = stream; + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + } + + XMLSourcePrivateData (tl::InputStream &stream) + : m_has_error (false) + { + mp_stream = &stream; + } + + XMLSourcePrivateData (tl::InputStream &stream, const std::string &progress_message) + : mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = &stream; + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + } + + int read (char *data, size_t n) + { + try { + + if (mp_progress.get ()) { + mp_progress->set (mp_stream->pos ()); + } + + size_t n0 = n; + for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; --n) { + *data++ = *rd; + } + + return int (n0 - n); + + } catch (tl::Exception &ex) { + m_error = ex.msg (); + m_has_error = true; + return -1; + } + } + + bool has_error () const + { + return m_has_error; + } + + const std::string &error_msg () const + { + return m_error; + } + + void reset () + { + mp_stream->reset (); + } + +private: + std::unique_ptr mp_stream_holder; + tl::InputStream *mp_stream; + std::unique_ptr mp_progress; + bool m_has_error; + std::string m_error; +}; + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + delete mp_source; + mp_source = 0; +} + +void +XMLSource::reset () +{ + mp_source->reset (); +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &string) + : m_copy (string) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (m_copy.c_str (), string.size ())))); +} + +XMLStringSource::XMLStringSource (const char *cp) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, strlen (cp))))); +} + +XMLStringSource::XMLStringSource (const char *cp, size_t len) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, len)))); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (path), progress_message)); +} + +XMLFileSource::XMLFileSource (const std::string &path) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (path))); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) +{ + set_source (new XMLSourcePrivateData (s, progress_message)); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &s) +{ + set_source (new XMLSourcePrivateData (s)); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +void XMLCALL start_element_handler (void *user_data, const XML_Char *name, const XML_Char **atts); +void XMLCALL end_element_handler (void *user_data, const XML_Char *name); +void XMLCALL cdata_handler (void *user_data, const XML_Char *s, int len); + +static std::string get_lname (const std::string &name) +{ + size_t colon = name.find (':'); + if (colon != std::string::npos) { + return std::string (name, colon + 1, name.size () - colon - 1); + } else { + return name; + } +} + +class XMLParserPrivateData +{ +public: + XMLParserPrivateData () + : mp_struct_handler (0) + { + mp_parser = XML_ParserCreate ("UTF-8"); + tl_assert (mp_parser != NULL); + } + + ~XMLParserPrivateData () + { + if (mp_parser != NULL) { + XML_ParserFree (mp_parser); + } + } + + void start_element (const std::string &name) + { + try { + // TODO: Provide namespace URI? + mp_struct_handler->start_element (std::string (), get_lname (name), name); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void end_element (const std::string &name) + { + try { + // TODO: Provide namespace URI? + mp_struct_handler->end_element (std::string (), get_lname (name), name); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void cdata (const std::string &cdata) + { + try { + mp_struct_handler->characters (cdata); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void parse (tl::XMLSource &source, XMLStructureHandler &struct_handler) + { + m_has_error = false; + mp_struct_handler = &struct_handler; + + // Just in case we want to reuse it ... + XML_ParserReset (mp_parser, NULL); + XML_SetUserData (mp_parser, (void *) this); + XML_SetElementHandler (mp_parser, start_element_handler, end_element_handler); + XML_SetCharacterDataHandler (mp_parser, cdata_handler); + + const int chunk = 65536; + char buffer [chunk]; + + int n; + + do { + + n = source.source ()->read (buffer, chunk); + if (n < 0) { + break; + } + + XML_Status status = XML_Parse (mp_parser, buffer, n, n < chunk /*is final*/); + if (status == XML_STATUS_ERROR) { + m_has_error = true; + m_error = XML_ErrorString (XML_GetErrorCode (mp_parser)); + m_error_line = XML_GetErrorLineNumber (mp_parser); + m_error_column = XML_GetErrorColumnNumber (mp_parser); + } + + } while (n == chunk && !m_has_error); + } + + void check_error () + { + if (m_has_error) { + throw tl::XMLLocatedException (m_error, m_error_line, m_error_column); + } + } + +private: + void error (tl::Exception &ex) + { + m_has_error = true; + m_error_line = XML_GetCurrentLineNumber (mp_parser); + m_error_column = XML_GetCurrentColumnNumber (mp_parser); + m_error = ex.msg (); + } + + XML_Parser mp_parser; + XMLStructureHandler *mp_struct_handler; + bool m_has_error; + std::string m_error; + int m_error_line, m_error_column; +}; + +void start_element_handler (void *user_data, const XML_Char *name, const XML_Char ** /*atts*/) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->start_element (std::string (name)); +} + +void end_element_handler (void *user_data, const XML_Char *name) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->end_element (std::string (name)); +} + +void cdata_handler (void *user_data, const XML_Char *s, int len) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->cdata (std::string (s, size_t (len))); +} + + +XMLParser::XMLParser () + : mp_data (new XMLParserPrivateData ()) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + delete mp_data; + mp_data = 0; +} + +void +XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) +{ + mp_data->parse (source, struct_handler); + + // throws an exception if there is an error + mp_data->check_error (); +} + +bool +XMLParser::is_available () +{ + return true; +} + +} + +#elif defined(HAVE_QT) + +#include +#include +#include + +namespace tl +{ + +// -------------------------------------------------------------------- +// A SAX handler for the Qt implementation + +class SAXHandler + : public QXmlDefaultHandler +{ +public: + SAXHandler (XMLStructureHandler *sh); + + virtual bool characters (const QString &ch); + virtual bool endElement (const QString &namespaceURI, const QString &localName, const QString &qName); + virtual bool startElement (const QString &namespaceURI, const QString &localName, const QString &qName, const QXmlAttributes &atts); + virtual bool error (const QXmlParseException &exception); + virtual bool fatalError (const QXmlParseException &exception); + virtual bool warning (const QXmlParseException &exception); + virtual QString errorString () const; + + void setDocumentLocator (QXmlLocator *locator); + + const tl::XMLLocatedException *exception () const + { + return m_error.get (); + } + +private: + QXmlLocator *mp_locator; + XMLStructureHandler *mp_struct_handler; + std::unique_ptr m_error; + std::string m_error_string; +}; + +// -------------------------------------------------------------------------------------------------------- +// trureHandler implementation + +SAXHandler::SAXHandler (XMLStructureHandler *sh) + : QXmlDefaultHandler (), mp_locator (0), mp_struct_handler (sh) +{ + // .. nothing yet .. +} + +void +SAXHandler::setDocumentLocator (QXmlLocator *locator) +{ + mp_locator = locator; +} + +bool +SAXHandler::startElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname, const QXmlAttributes & /*atts*/) +{ + std::string uri (tl::to_string (qs_uri)); + std::string lname (tl::to_string (qs_lname)); + std::string qname (tl::to_string (qs_qname)); + + try { + mp_struct_handler->start_element (uri, lname, qname); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +bool +SAXHandler::endElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname) +{ + std::string uri (tl::to_string (qs_uri)); + std::string lname (tl::to_string (qs_lname)); + std::string qname (tl::to_string (qs_qname)); + + try { + mp_struct_handler->end_element (uri, lname, qname); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +bool +SAXHandler::characters (const QString &t) +{ + try { + mp_struct_handler->characters (tl::to_string (t)); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +QString +SAXHandler::errorString () const +{ + return tl::to_qstring (m_error_string); +} + + +bool +SAXHandler::error (const QXmlParseException &ex) +{ + m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); + // stop reading + return false; +} + +bool +SAXHandler::fatalError (const QXmlParseException &ex) +{ + m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); + // stop reading + return false; +} + +bool +SAXHandler::warning (const QXmlParseException &ex) +{ + tl::XMLLocatedException lex (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ()); + tl::warn << lex.msg (); + // continue + return true; +} + +// -------------------------------------------------------------------- +// SourcePrivateData implementation + +class XMLSourcePrivateData + : public QXmlInputSource +{ +public: + XMLSourcePrivateData () + : QXmlInputSource () + { + // .. nothing yet .. + } + + XMLSourcePrivateData (QIODevice *dev) + : QXmlInputSource (dev) + { + // .. nothing yet .. + } +}; + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + delete mp_source; + mp_source = 0; +} + +void +XMLSource::reset () +{ + mp_source->reset (); +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &string) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (string.c_str ())); + set_source (source); +} + +XMLStringSource::XMLStringSource (const char *cp) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (cp)); + set_source (source); +} + +XMLStringSource::XMLStringSource (const char *cp, size_t len) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (cp, int (len))); + set_source (source); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// StreamIODevice definition and implementation + +class StreamIODevice + : public QIODevice +{ +public: + StreamIODevice (tl::InputStream &stream) + : mp_stream (&stream), + mp_progress (0), + m_has_error (false) + { + open (QIODevice::ReadOnly); + } + + StreamIODevice (tl::InputStream &stream, const std::string &progress_message) + : mp_stream (&stream), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + open (QIODevice::ReadOnly); + } + + StreamIODevice (const std::string &path) + : mp_stream_holder (new tl::InputStream (path)), + mp_progress (0), + m_has_error (false) + { + mp_stream = mp_stream_holder.get (); + open (QIODevice::ReadOnly); + } + + StreamIODevice (const std::string &path, const std::string &progress_message) + : mp_stream_holder (new tl::InputStream (path)), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = mp_stream_holder.get (); + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + open (QIODevice::ReadOnly); + } + + ~StreamIODevice () + { + if (mp_progress) { + delete mp_progress; + mp_progress = 0; + } + } + + virtual bool isSequential () const + { + return true; + } + + qint64 writeData (const char *, qint64) + { + tl_assert (false); + } + + qint64 readData (char *data, qint64 n) + { + try { + + if (mp_progress) { + mp_progress->set (mp_stream->pos ()); + } + + qint64 n0 = n; + for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; ) { + // NOTE: we skip CR to compensate for Windows CRLF line terminators (issue #419). + if (*rd != '\r') { + *data++ = *rd; + --n; + } + } + + if (n0 == n) { + return -1; + } else { + return n0 - n; + } + + } catch (tl::Exception &ex) { + setErrorString (tl::to_qstring (ex.msg ())); + m_has_error = true; + return -1; + } + } + + bool has_error () const + { + return m_has_error; + } + +private: + tl::InputStream *mp_stream; + std::unique_ptr mp_stream_holder; + tl::AbsoluteProgress *mp_progress; + bool m_has_error; +}; + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +class XMLStreamSourcePrivateData + : public XMLSourcePrivateData +{ +public: + XMLStreamSourcePrivateData (StreamIODevice *io) + : XMLSourcePrivateData (io), mp_io (io) + { + // .. nothing yet .. + } + + virtual void fetchData () + { + QXmlInputSource::fetchData (); + + // This feature is actually missing in the original implementation: throw an exception on error + if (mp_io->has_error ()) { + throw tl::Exception (tl::to_string (mp_io->errorString ())); + } + } + +private: + std::unique_ptr mp_io; +}; + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path, progress_message))); +} + +XMLFileSource::XMLFileSource (const std::string &path) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path))); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s, progress_message))); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &s) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s))); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +class XMLParserPrivateData + : public QXmlSimpleReader +{ +public: + XMLParserPrivateData () : QXmlSimpleReader () { } +}; + +XMLParser::XMLParser () + : mp_data (new XMLParserPrivateData ()) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + delete mp_data; + mp_data = 0; +} + +void +XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) +{ + SAXHandler handler (&struct_handler); + + mp_data->setContentHandler (&handler); + mp_data->setErrorHandler (&handler); + + bool result = mp_data->parse (source.source (), false /*=not incremental*/); + if (! result && handler.exception ()) { + throw tl::XMLLocatedException (*handler.exception ()); + } +} + +bool +XMLParser::is_available () +{ + return true; +} + +} + +#else + +namespace tl +{ + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + // .. nothing yet .. +} + +void +XMLSource::reset () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &) +{ + tl_assert (false); +} + +XMLStringSource::XMLStringSource (const char *) +{ + tl_assert (false); +} + +XMLStringSource::XMLStringSource (const char *, size_t) +{ + tl_assert (false); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &, const std::string &) +{ + tl_assert (false); +} + +XMLFileSource::XMLFileSource (const std::string &) +{ + tl_assert (false); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &, const std::string &) +{ + tl_assert (false); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &) +{ + tl_assert (false); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +XMLParser::XMLParser () + : mp_data (0) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + // .. nothing yet .. +} + +void +XMLParser::parse (XMLSource &, XMLStructureHandler &) +{ + tl_assert (false); +} + +bool +XMLParser::is_available () +{ + return false; +} + +} + +#endif diff --git a/src/tl/tl/tlXMLReader.h b/src/tl/tl/tlXMLReader.h new file mode 100644 index 000000000..3b8423e3d --- /dev/null +++ b/src/tl/tl/tlXMLReader.h @@ -0,0 +1,215 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlXMLReader +#define HDR_tlXMLReader + +#include "tlCommon.h" + +#include +#include + +#include "tlAssert.h" +#include "tlInternational.h" +#include "tlString.h" +#include "tlStream.h" + +class QIODevice; + +namespace tl +{ + +/** + * NOTE: This XML parser package also supports a ProtocolBuffer flavor. + * This allows binding the same scheme to efficient binary PB format. + */ + +class ProtocolBufferReaderBase; +class ProtocolBufferWriterBase; + +/** + * @brief A basic XML parser error exception class + */ + +class TL_PUBLIC XMLException : public tl::Exception +{ +public: + XMLException (const char *msg) + : Exception (tl::to_string (tr ("XML parser error: %s")).c_str ()), + m_msg (msg) + { + // .. nothing yet .. + } + + XMLException (const std::string &msg) + : Exception (fmt (-1, -1).c_str (), msg.c_str ()), + m_msg (msg) + { + // .. nothing yet .. + } + + /** + * @brief Raw (unprefixed) message of the XML parser + */ + const std::string & + raw_msg () const + { + return m_msg; + } + +protected: + XMLException (const std::string &msg, int line, int column) + : Exception (fmt (line, column).c_str (), msg.c_str (), line, column), + m_msg (msg) + { + // .. nothing yet .. + } + +private: + std::string m_msg; + + static std::string fmt (int line, int /*column*/) + { + if (line < 0) { + return tl::to_string (tr ("XML parser error: %s")).c_str (); + } else { + return tl::to_string (tr ("XML parser error: %s in line %d, column %d")).c_str (); + } + } +}; + +/** + * @brief A XML parser error exception class that additionally provides line and column information + */ + +class TL_PUBLIC XMLLocatedException : public XMLException +{ +public: + XMLLocatedException (const std::string &msg, int line, int column) + : XMLException (msg, line, column), + m_line (line), m_column (column) + { + // .. nothing yet .. + } + + /** + * @brief Line number information of the exception + */ + int line () const + { + return m_line; + } + + /** + * @brief Column number information of the exception + */ + int column () const + { + return m_column; + } + +private: + int m_line; + int m_column; +}; + +// The opaque source type +class XMLSourcePrivateData; + +/** + * @brief A generic XML text source class + * + * This class is the base class providing input for + * the Qt XML parser and basically maps to a QXmlInputSource object + * for compatibility with the "libparsifal" branch. + */ + +class TL_PUBLIC XMLSource +{ +public: + XMLSource (); + ~XMLSource (); + + XMLSourcePrivateData *source () + { + return mp_source; + } + + void reset (); + +protected: + void set_source (XMLSourcePrivateData *source) + { + mp_source = source; + } + +private: + XMLSourcePrivateData *mp_source; +}; + +/** + * @brief A specialization of XMLSource to receive a string + */ + +class TL_PUBLIC XMLStringSource : public XMLSource +{ +public: + XMLStringSource (const std::string &string); + XMLStringSource (const char *cp); + XMLStringSource (const char *cp, size_t len); + ~XMLStringSource (); + +private: + std::string m_copy; +}; + +/** + * @brief A specialization of XMLSource to receive from a file + */ + +class TL_PUBLIC XMLFileSource : public XMLSource +{ +public: + XMLFileSource (const std::string &path); + XMLFileSource (const std::string &path, const std::string &progress_message); + ~XMLFileSource (); +}; + +/** + * @brief A generic stream source class + * + * This class implements a XML parser source from a tl::InputStream + */ + +class TL_PUBLIC XMLStreamSource : public XMLSource +{ +public: + XMLStreamSource (tl::InputStream &stream); + XMLStreamSource (tl::InputStream &stream, const std::string &progress_message); + ~XMLStreamSource (); +}; + +} // namespace tl + +#endif + diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc new file mode 100644 index 000000000..000ffda67 --- /dev/null +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -0,0 +1,636 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "tlXMLParser.h" +#include "tlProtocolBuffer.h" +#include "tlUnitTest.h" +#include "tlFileUtils.h" + +#include +#include + +// Basic tests of reader and writer +TEST (1_BasicTypes) +{ + tl::OutputMemoryStream out; + + { + tl::OutputStream os (out); + tl::ProtocolBufferWriter writer (os); + writer.write (1, std::string ("xyz_abc")); + writer.write (2, float (1.5)); + writer.write (3, double (2.5)); + writer.write (4, true); + writer.write (5, int32_t (-100000)); + writer.write (6, int32_t (100000)); + writer.write (7, uint32_t (200000)); + writer.write (8, int64_t (-10000000000)); + writer.write (9, int64_t (10000000000)); + writer.write (10, uint64_t (20000000000)); + writer.write (11, int32_t (-100000), true); + writer.write (12, int32_t (100000), true); + writer.write (13, uint32_t (200000), true); + writer.write (14, int64_t (-10000000000), true); + writer.write (15, int64_t (10000000000), true); + writer.write (16, uint64_t (20000000000), true); + } + + { + tl::InputMemoryStream s2 (out.data (), out.size ()); + tl::InputStream is (s2); + tl::ProtocolBufferReader reader (is); + + std::string s; + bool b; + float f; + double d; + uint32_t ui32; + int32_t i32; + uint64_t ui64; + int64_t i64; + + EXPECT_EQ (reader.read_tag (), 1); + s = ""; + reader.read (s); + EXPECT_EQ (s, "xyz_abc"); + EXPECT_EQ (reader.read_tag (), 2); + f = 0; + reader.read (f); + EXPECT_EQ (f, 1.5); + EXPECT_EQ (reader.read_tag (), 3); + d = 0; + reader.read (d); + EXPECT_EQ (d, 2.5); + EXPECT_EQ (reader.read_tag (), 4); + b = false; + reader.read (b); + EXPECT_EQ (b, true); + EXPECT_EQ (reader.read_tag (), 5); + i32 = 0; + reader.read (i32); + EXPECT_EQ (i32, -100000); + EXPECT_EQ (reader.read_tag (), 6); + i32 = 0; + reader.read (i32); + EXPECT_EQ (i32, 100000); + EXPECT_EQ (reader.read_tag (), 7); + ui32 = 0; + reader.read (ui32); + EXPECT_EQ (ui32, 200000u); + EXPECT_EQ (reader.read_tag (), 8); + i64 = 0; + reader.read (i64); + EXPECT_EQ (i64, -10000000000); + EXPECT_EQ (reader.read_tag (), 9); + i64 = 0; + reader.read (i64); + EXPECT_EQ (i64, 10000000000); + EXPECT_EQ (reader.read_tag (), 10); + ui64 = 0; + reader.read (ui64); + EXPECT_EQ (ui64, 20000000000u); + EXPECT_EQ (reader.read_tag (), 11); + i32 = 0; + reader.read (i32); + EXPECT_EQ (i32, -100000); + EXPECT_EQ (reader.read_tag (), 12); + i32 = 0; + reader.read (i32); + EXPECT_EQ (i32, 100000); + EXPECT_EQ (reader.read_tag (), 13); + ui32 = 0; + reader.read (ui32); + EXPECT_EQ (ui32, 200000u); + EXPECT_EQ (reader.read_tag (), 14); + i64 = 0; + reader.read (i64); + EXPECT_EQ (i64, -10000000000); + EXPECT_EQ (reader.read_tag (), 15); + i64 = 0; + reader.read (i64); + EXPECT_EQ (i64, 10000000000); + EXPECT_EQ (reader.read_tag (), 16); + ui64 = 0; + reader.read (ui64); + EXPECT_EQ (ui64, 20000000000u); + EXPECT_EQ (reader.at_end (), true); + } +} + +struct Child { + Child () : txt (""), d(-1), live(true) { } + ~Child () { tl_assert (live); live = false; } + std::string txt; + double d; + bool live; + bool operator== (const Child &x) const { return txt == x.txt && fabs (d - x.d) < 1e-9 && children == x.children; } + std::vector children; + std::vector::const_iterator begin_children() const { return children.begin (); } + std::vector::const_iterator end_children() const { return children.end (); } + void add_child (const Child &c) { children.push_back (c); } + void add_child_ptr (Child *c) { children.push_back (*c); delete c; } +}; + +struct Root { + long m; + unsigned int mi; + bool b; + std::vector m_subs; + std::vector m_isubs; + std::vector m_children; + Child m_child; + + Root () : m(0), mi(0), b(false) { } + + bool operator== (const Root &x) const { return m == x.m && + mi == x.mi && m_subs == x.m_subs && + m_isubs == x.m_isubs && m_children == x.m_children + && m_child == x.m_child && b == x.b; } + + int get_mi () const { return mi; } + void set_mi (int i) { mi = i; } + void add_sub (const double &s) { + m_subs.push_back (s); + } + void add_isub (const int &s) { + m_isubs.push_back (s); + } + std::vector::const_iterator begin_subs () const { + return m_subs.begin (); + } + std::vector::const_iterator end_subs () const { + return m_subs.end (); + } + std::vector::const_iterator begin_isubs () const { + return m_isubs.begin (); + } + std::vector::const_iterator end_isubs () const { + return m_isubs.end (); + } + void add_child_ptr (Child *c) { + m_children.push_back (*c); + delete c; + } + void add_child (const Child &c) { + m_children.push_back (c); + } + std::vector::const_iterator begin_children () const { + return m_children.begin (); + } + std::vector::const_iterator end_children () const { + return m_children.end (); + } + void set_child (const Child &child) { m_child = child; } + const Child &get_child () const { return m_child; } +}; + +static tl::XMLElementList child_struct ("Child", + tl::make_member (&Child::txt, "txt#1") + + tl::make_member (&Child::d, "d#2") + + tl::make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children#3", &child_struct) +); + +static tl::XMLStruct structure ("pbtest-struct#88888888", + tl::make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, "sub#1") + + tl::make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, "isub#2") + + tl::make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "children#3", &child_struct) + + tl::make_element (&Root::get_child, &Root::set_child, "child#4", &child_struct) + + tl::make_member (&Root::m, "m#5") + + tl::make_member (&Root::get_mi, &Root::set_mi, "mi#6") + + tl::make_member (&Root::b, "b#7") +); + +static void build_struct (Root &root) +{ + root.add_sub (0.5); + root.add_sub (7.5); + root.add_isub (420000000); + root.m = -1700000; + root.set_mi (21); + root.b = true; + + Child c1; + c1.txt = "c1"; + c1.d = 1.0; + root.add_child (c1); + + Child c2; + c2.txt = "c2"; + c2.d = 2.0; + + Child c21; + c21.txt = "c21"; + c21.d = 2.1; + c2.add_child (c21); + + Child c22; + c22.txt = "c22"; + c22.d = 2.2; + c2.add_child (c22); + + Child c23; + c23.txt = "c23"; + c23.d = 2.3; + c2.add_child (c23); + + root.add_child (c2); + + Child sc; + sc.txt = "single"; + sc.d = 4.2e6; + root.set_child (sc); +} + +TEST (100_BasicStruct) +{ + Root root; + build_struct (root); + + std::string fn = tl::combine_path (tl::testtmp (), "pb_test.pb"); + + { + tl::OutputStream os (fn); + tl::ProtocolBufferWriter writer (os); + structure.write (writer, root); + /* + for debugging: + tl::ProtocolBufferDumper dumper; + structure.write (dumper, root); + */ + } + + root = Root (); + + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + // TODO: adjust + EXPECT_EQ (error, ""); + EXPECT_EQ (root.m_subs.size (), size_t (2)); + EXPECT_EQ (root.m_subs [0], 0.5); + EXPECT_EQ (root.m_subs [1], 7.5); + EXPECT_EQ (root.m_isubs.size (), size_t (1)); + EXPECT_EQ (root.m_isubs [0], 420000000); + EXPECT_EQ (root.m, -1700000); + EXPECT_EQ (root.b, true); + EXPECT_EQ (root.mi, (unsigned int) 21); + EXPECT_EQ (root.m_children.size (), size_t (2)); + EXPECT_EQ (root.m_children [0].txt, "c1"); + EXPECT_EQ (root.m_children [0].d, 1.0); + EXPECT_EQ (root.m_children [1].txt, "c2"); + EXPECT_EQ (root.m_children [1].d, 2.0); + EXPECT_EQ (root.m_children [1].end_children () - root.m_children [1].begin_children (), 3); + EXPECT_EQ (root.m_child.txt, "single"); + EXPECT_EQ (root.m_child.d, 4.2e6); + + // write .. + tl::OutputMemoryStream out; + + { + tl::OutputStream os (out); + tl::ProtocolBufferWriter writer (os); + structure.write (writer, root); + } + + // and read again. + Root rsave (root); + + try { + error.clear (); + tl::InputMemoryStream s2 (out.data (), out.size ()); + tl::InputStream is (s2); + root = Root (); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (root == rsave, true); + + // write empty object + out.clear (); + root = Root (); + + { + tl::OutputStream os (out); + tl::ProtocolBufferWriter writer (os); + structure.write (writer, root); + } + + // and read again. + try { + error.clear (); + tl::InputMemoryStream s2 (out.data (), out.size ()); + tl::InputStream is (s2); + root = Root (); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (root == Root (), true); +} + +struct TestClass +{ + enum enum_type { A, B, C }; + + TestClass () : e (A) { } + enum_type e; +}; + +struct TestClassEnumConverter +{ + typedef uint32_t pb_type; + + pb_type pb_encode (TestClass::enum_type e) const + { + switch (e) { + case TestClass::A: + return 17; + case TestClass::B: + return 18; + case TestClass::C: + return 19; + default: + return 0; + } + } + + void pb_decode (uint32_t value, TestClass::enum_type &e) const + { + switch (value) { + case 17: + e = TestClass::A; + break; + case 18: + e = TestClass::B; + break; + case 19: + e = TestClass::C; + break; + default: + e = TestClass::enum_type (0); + break; + } + } + + void from_string (const std::string &, TestClass::enum_type) const { } + std::string to_string (TestClass::enum_type) const { return std::string (); } +}; + +tl::XMLStruct tc_structure ("pbtest-tc#1", + tl::make_member (&TestClass::e, "e#2", TestClassEnumConverter ()) +); + +TEST (101_Converter) +{ + TestClass tc; + + tc.e = TestClass::A; + std::string fn = tl::combine_path (tl::testtmp (), "pb_101a.pb"); + + { + tl::OutputStream os (fn); + tl::ProtocolBufferWriter writer (os); + tc_structure.write (writer, tc); + } + + tc = TestClass (); + + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (tc.e, TestClass::A); + + tc.e = TestClass::B; + + fn = tl::combine_path (tl::testtmp (), "pb_101b.pb"); + + { + tl::OutputStream os (fn); + tl::ProtocolBufferWriter writer (os); + tc_structure.write (writer, tc); + } + + tc = TestClass (); + + error.clear (); + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (tc.e, TestClass::B); +} + +TEST (101_ExternalFiles) +{ + Root root_au; + build_struct (root_au); + + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct1.pb"); + + Root root; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (root == root_au, true); +} + +TEST (102_InvalidHeader) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct2_invalid_header.pb"); + + Root root; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 49), "Expected header field with ID 88888888 (got 4444)"); +} + +TEST (103_InvalidHeader) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct3_invalid_header.pb"); + + Root root; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 71), "Expected header field with string 'pbtest-struct' (got 'xxxxxxxxxxxxx')"); +} + +// "last one wins", unknown IDs are skipped +TEST (104_Deviations) +{ + Root root_au; + build_struct (root_au); + + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct4_deviations.pb"); + + Root root; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (root == root_au, true); +} + +TEST (105_Int32Overflow) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct5_overflow.pb"); + + TestClass tc; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 21), "32 bit value overflow"); +} + +TEST (106_InvalidType) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct6_invalid_type.pb"); + + TestClass tc; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 34), "Expected a VARINT or I32 wire type"); +} + +TEST (107_InvalidType) +{ + // uses I32 to represent enum - works too, not just VARINT + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct7_invalid_type.pb"); + + TestClass tc; + tc.e = TestClass::B; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (tc.e, TestClass::A); +} + +TEST (108_InvalidType) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct8_invalid_type.pb"); + + TestClass tc; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 34), "Expected a VARINT or I32 wire type"); +} + +TEST (109_DumpSpec) +{ + EXPECT_EQ (structure.create_def (), + "// created from KLayout proto definition 'pbtest-struct'\n" + "\n" + "syntax = \"proto2\";\n" + "\n" + "message Child {\n" + " optional string txt = 1;\n" + " optional double d = 2;\n" + " repeated Child children = 3;\n" + "}\n" + "\n" + "message PbtestStruct {\n" + " repeated double sub = 1;\n" + " repeated sint32 isub = 2;\n" + " repeated Child children = 3;\n" + " optional Child child = 4;\n" + " optional sint64 m = 5;\n" + " optional sint32 mi = 6;\n" + " optional uint32 b = 7;\n" + "}" + ); +} diff --git a/src/tl/unit_tests/unit_tests.pro b/src/tl/unit_tests/unit_tests.pro index 3d8b06676..3d3562342 100644 --- a/src/tl/unit_tests/unit_tests.pro +++ b/src/tl/unit_tests/unit_tests.pro @@ -33,6 +33,7 @@ SOURCES = \ tlObjectTests.cc \ tlOptionalTests.cc \ tlPixelBufferTests.cc \ + tlProtocolBufferTests.cc \ tlResourcesTests.cc \ tlReuseVectorTests.cc \ tlStableVectorTests.cc \ diff --git a/testdata/pb/struct1.pb b/testdata/pb/struct1.pb new file mode 100644 index 000000000..b786e0d3c Binary files /dev/null and b/testdata/pb/struct1.pb differ diff --git a/testdata/pb/struct2_invalid_header.pb b/testdata/pb/struct2_invalid_header.pb new file mode 100644 index 000000000..8fb936070 Binary files /dev/null and b/testdata/pb/struct2_invalid_header.pb differ diff --git a/testdata/pb/struct3_invalid_header.pb b/testdata/pb/struct3_invalid_header.pb new file mode 100644 index 000000000..0c1e687ef Binary files /dev/null and b/testdata/pb/struct3_invalid_header.pb differ diff --git a/testdata/pb/struct4_deviations.pb b/testdata/pb/struct4_deviations.pb new file mode 100644 index 000000000..d21f797c0 Binary files /dev/null and b/testdata/pb/struct4_deviations.pb differ diff --git a/testdata/pb/struct5_overflow.pb b/testdata/pb/struct5_overflow.pb new file mode 100644 index 000000000..14e328697 --- /dev/null +++ b/testdata/pb/struct5_overflow.pb @@ -0,0 +1,2 @@ + + pbtest-tcԝ? \ No newline at end of file diff --git a/testdata/pb/struct6_invalid_type.pb b/testdata/pb/struct6_invalid_type.pb new file mode 100644 index 000000000..4331c5da2 Binary files /dev/null and b/testdata/pb/struct6_invalid_type.pb differ diff --git a/testdata/pb/struct7_invalid_type.pb b/testdata/pb/struct7_invalid_type.pb new file mode 100644 index 000000000..bec039f51 Binary files /dev/null and b/testdata/pb/struct7_invalid_type.pb differ diff --git a/testdata/pb/struct8_invalid_type.pb b/testdata/pb/struct8_invalid_type.pb new file mode 100644 index 000000000..02d42f890 --- /dev/null +++ b/testdata/pb/struct8_invalid_type.pb @@ -0,0 +1,2 @@ + + pbtest-tcabc \ No newline at end of file diff --git a/testdata/ruby/rdbTest.rb b/testdata/ruby/rdbTest.rb index 91ecb9bf2..c388aae6a 100644 --- a/testdata/ruby/rdbTest.rb +++ b/testdata/ruby/rdbTest.rb @@ -595,7 +595,7 @@ class RDB_TestClass < TestBase assert_equal(db.is_modified?, false) tag_id = db.tag_id("x") - assert_equal(tag_id, 1) + assert_equal(db.tag_name(tag_id), "x") db.set_tag_description(tag_id, "xdesc") assert_equal(db.tag_description(tag_id), "xdesc")