Merge branch 'rdb-tag-rework' into xxx

src/db/db/dbConverters.h

@@ -40,6 +40,8 @@ namespace db
 template <class T>
 struct DB_PUBLIC_TEMPLATE TransformationConverter
 {
+  typedef T value_type;
+
   std::string to_string (const T &t) const
   {
     return t.to_string ();
@@ -58,6 +60,8 @@ struct DB_PUBLIC_TEMPLATE TransformationConverter
  */
 struct DB_PUBLIC LayoutLayerConverter
 {
+  typedef db::LayerProperties value_type;
+
   std::string to_string (const db::LayerProperties &p) const
   {
     return p.to_string ();
@@ -79,6 +83,8 @@ struct DB_PUBLIC LayoutLayerConverter
 template <class P>
 struct DB_PUBLIC_TEMPLATE PointConverter
 {
+  typedef P value_type;
+
   std::string to_string (const P &p) const
   {
     return tl::to_string (p.x ()) + "," + tl::to_string (p.y ());

src/db/db/dbStream.h

@@ -154,6 +154,7 @@ class StreamOptionsReadAdaptor
 {
 public:
   typedef tl::pass_by_ref_tag tag;
+  typedef tl::zero_cardinality_tag cardinality;
 
   StreamOptionsReadAdaptor ()
     : mp_options (0), m_done (false)

src/db/db/dbTechnology.h

@@ -807,6 +807,7 @@ class TechnologyComponentReadAdaptor
 {
 public:
   typedef tl::pass_by_ref_tag tag;
+  typedef tl::single_cardinality_tag cardinality;
 
   TechnologyComponentReadAdaptor (const std::string &name)
     : m_name (name), mp_t (0), m_done (false)

src/img/img/imgStream.cc

@@ -359,6 +359,8 @@ namespace {
 
   struct PointConverter
   {
+    typedef db::DPoint value_type;
+
     std::string to_string (const db::DPoint &p) const
     {
       return p.to_string ();
@@ -373,6 +375,8 @@ namespace {
 
   struct ColorMapConverter
   {
+    typedef std::pair<double, std::pair<tl::Color, tl::Color> > value_type;
+
     std::string to_string (const std::pair<double, std::pair<tl::Color, tl::Color> > &cm) const
     {
       std::string s;
@@ -427,7 +431,7 @@ tl::XMLStruct<ImageProxy> s_img_structure ("image-data",
   tl::make_member (&ImageProxy::max_value, &ImageProxy::set_max_value, "max-value") +
   tl::make_element (&ImageProxy::data_mapping, &ImageProxy::set_data_mapping, "data-mapping",
     tl::make_element (&img::DataMapping::false_color_nodes, "color-map",
-      tl::make_member<std::pair<double, std::pair<tl::Color, tl::Color> >, img::DataMapping::false_color_nodes_type::const_iterator, img::DataMapping::false_color_nodes_type, ColorMapConverter> (&img::DataMapping::false_color_nodes_type::begin, &img::DataMapping::false_color_nodes_type::end, &img::DataMapping::false_color_nodes_type::push_back, "color-map-entry", ColorMapConverter ())
+      tl::make_member<std::pair<double, std::pair<tl::Color, tl::Color> >, img::DataMapping::false_color_nodes_type::const_iterator, img::DataMapping::false_color_nodes_type, tl::XMLStringBasedConverter<ColorMapConverter> > (&img::DataMapping::false_color_nodes_type::begin, &img::DataMapping::false_color_nodes_type::end, &img::DataMapping::false_color_nodes_type::push_back, "color-map-entry", tl::XMLStringBasedConverter<ColorMapConverter> ())
     ) +
     tl::make_member (&img::DataMapping::brightness, "brightness") +
     tl::make_member (&img::DataMapping::contrast, "contrast") +
@@ -437,7 +441,7 @@ tl::XMLStruct<ImageProxy> s_img_structure ("image-data",
     tl::make_member (&img::DataMapping::blue_gain, "blue-gain")
   ) +
   tl::make_element (&ImageProxy::landmarks, &ImageProxy::set_landmarks, "landmarks",
-    tl::make_member<db::DPoint, img::Object::landmarks_type::const_iterator, img::Object::landmarks_type, PointConverter> (&img::Object::landmarks_type::begin, &img::Object::landmarks_type::end, &img::Object::landmarks_type::push_back, "landmark", PointConverter ())
+    tl::make_member<db::DPoint, img::Object::landmarks_type::const_iterator, img::Object::landmarks_type, tl::XMLStringBasedConverter<PointConverter> > (&img::Object::landmarks_type::begin, &img::Object::landmarks_type::end, &img::Object::landmarks_type::push_back, "landmark", tl::XMLStringBasedConverter<PointConverter> ())
   ) +
   tl::make_member (&ImageProxy::begin_byte_data, &ImageProxy::end_byte_data, &ImageProxy::push_byte_data, "byte-data") +
   tl::make_member (&ImageProxy::begin_data, &ImageProxy::end_data, &ImageProxy::push_data, "data")

src/lay/lay/laySaltGrain.cc

@@ -353,6 +353,8 @@ SaltGrain::valid_version (const std::string &v)
 
 struct TimeConverter
 {
+  typedef QDateTime value_type;
+
   std::string to_string (const QDateTime &time) const
   {
     if (time.isNull ()) {
@@ -374,6 +376,8 @@ struct TimeConverter
 
 struct ImageConverter
 {
+  typedef QImage value_type;
+
   std::string to_string (const QImage &image) const
   {
     if (image.isNull ()) {
@@ -416,10 +420,10 @@ SaltGrain::xml_elements ()
       tl::make_member (&SaltGrain::license, &SaltGrain::set_license, "license") +
       tl::make_member (&SaltGrain::author, &SaltGrain::set_author, "author") +
       tl::make_member (&SaltGrain::author_contact, &SaltGrain::set_author_contact, "author-contact") +
-      tl::make_member (&SaltGrain::authored_time, &SaltGrain::set_authored_time, "authored-time", TimeConverter ()) +
+      tl::make_member (&SaltGrain::authored_time, &SaltGrain::set_authored_time, "authored-time", tl::XMLStringBasedConverter<TimeConverter> ()) +
-      tl::make_member (&SaltGrain::installed_time, &SaltGrain::set_installed_time, "installed-time", TimeConverter ()) +
+      tl::make_member (&SaltGrain::installed_time, &SaltGrain::set_installed_time, "installed-time", tl::XMLStringBasedConverter<TimeConverter> ()) +
-      tl::make_member (&SaltGrain::icon, &SaltGrain::set_icon, "icon", ImageConverter ()) +
+      tl::make_member (&SaltGrain::icon, &SaltGrain::set_icon, "icon",  tl::XMLStringBasedConverter<ImageConverter> ()) +
-      tl::make_member (&SaltGrain::screenshot, &SaltGrain::set_screenshot, "screenshot", ImageConverter ()) +
+      tl::make_member (&SaltGrain::screenshot, &SaltGrain::set_screenshot, "screenshot", tl::XMLStringBasedConverter<ImageConverter> ()) +
       tl::make_element (&SaltGrain::begin_dependencies, &SaltGrain::end_dependencies, &SaltGrain::add_dependency, "depends",
         tl::make_member (&SaltGrainDependency::name, "name") +
         tl::make_member (&SaltGrainDependency::url, "url") +

src/laybasic/laybasic/layDispatcher.cc

@@ -108,6 +108,8 @@ Dispatcher::config_finalize ()
 
 struct ConfigGetAdaptor
 {
+  typedef tl::zero_cardinality_tag cardinality;
+
   ConfigGetAdaptor (const std::string &name)
     : mp_owner (0), m_done (false), m_name (name)
   {
@@ -145,6 +147,8 @@ private:
 
 struct ConfigGetNullAdaptor
 {
+  typedef tl::zero_cardinality_tag cardinality;
+
   ConfigGetNullAdaptor ()
   {
     // .. nothing yet ..

src/layui/layui/rdbMarkerBrowserDialog.cc

@@ -387,7 +387,7 @@ BEGIN_PROTECTED
 
       } else {
 
-        rdb->save (rdb->filename ());
+        rdb->save (rdb->filename (), rdb->binary ());
         rdb->reset_modified ();
 
       }
@@ -459,7 +459,7 @@ BEGIN_PROTECTED
     throw tl::Exception (tl::to_string (tr ("The current report database is not saved.\nSave it to some file with 'Save As', before saving it as waiver DB.")));
   }
 
-  rdb->write (rdb->filename () + ".w");
+  rdb->write (rdb->filename () + ".w", rdb->binary ());
 
 END_PROTECTED
 }
@@ -475,11 +475,13 @@ BEGIN_PROTECTED
     if (rdb) {
 
       //  prepare and open the file dialog
-      lay::FileDialog save_dialog (this, tl::to_string (QObject::tr ("Save Marker Database File")), "KLayout RDB files (*.lyrdb)");
+      lay::FileDialog save_dialog (this, tl::to_string (QObject::tr ("Save Marker Database File")), tl::to_string (tr ("KLayout RDB files (*.lyrdb);;KLayout binary RDB files (*.rdb);;All files (*)")));
       std::string fn (rdb->filename ());
       if (save_dialog.get_save (fn)) {
 
-        rdb->save (fn);
+        bool binary = (tl::extension (fn) == "rdb");
+
+        rdb->save (fn, binary);
         rdb->reset_modified ();
 
         //  update the RDB title strings

src/lym/lym/lymMacro.cc

@@ -150,6 +150,8 @@ bool Macro::del ()
 
 struct Interpreter2s
 {
+  typedef Macro::Interpreter value_type;
+
   std::string to_string (Macro::Interpreter i) const
   {
     switch (i) {
@@ -199,7 +201,7 @@ static tl::XMLStruct<lym::Macro> xml_struct ("klayout-macro",
   tl::make_member (&Macro::show_in_menu, &Macro::set_show_in_menu, "show-in-menu") +
   tl::make_member (&Macro::group_name, &Macro::set_group_name, "group-name") +
   tl::make_member (&Macro::menu_path, &Macro::set_menu_path, "menu-path") +
-  tl::make_member (&Macro::interpreter, &Macro::set_interpreter, "interpreter", Interpreter2s ()) +
+  tl::make_member (&Macro::interpreter, &Macro::set_interpreter, "interpreter", tl::XMLStringBasedConverter<Interpreter2s> ()) +
   tl::make_member (&Macro::dsl_interpreter, &Macro::set_dsl_interpreter, "dsl-interpreter-name") +
   tl::make_member (&Macro::text, &Macro::set_text, "text") +
   tl::make_member<Macro> ("format")  //  for backward compatibility

src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc

@@ -166,6 +166,8 @@ GerberImportData::setup_importer (db::GerberImporter *importer)
 
 struct MountingConverter
 {
+  typedef GerberImportData::mounting_type value_type;
+
   std::string to_string (GerberImportData::mounting_type m) const
   {
     return m == GerberImportData::MountingTop ? "top" : "bottom";
@@ -190,9 +192,9 @@ pcb_project_structure ("pcb-project",
   tl::make_member (&GerberImportData::border, "border") +
   tl::make_member (&GerberImportData::free_layer_mapping, "free-layer-mapping") +
   tl::make_element (&GerberImportData::layout_layers, "layout-layers",
-    tl::make_member<db::LayerProperties, std::vector<db::LayerProperties>::const_iterator, std::vector<db::LayerProperties> > (&std::vector<db::LayerProperties>::begin, &std::vector<db::LayerProperties>::end, &std::vector<db::LayerProperties>::push_back, "layout-layer", db::LayoutLayerConverter ())
+    tl::make_member<db::LayerProperties, std::vector<db::LayerProperties>::const_iterator, std::vector<db::LayerProperties> > (&std::vector<db::LayerProperties>::begin, &std::vector<db::LayerProperties>::end, &std::vector<db::LayerProperties>::push_back, "layout-layer", tl::XMLStringBasedConverter<db::LayoutLayerConverter> ())
   ) +
-  tl::make_member (&GerberImportData::mounting, "mounting", MountingConverter ()) +
+  tl::make_member (&GerberImportData::mounting, "mounting", tl::XMLStringBasedConverter<MountingConverter> ()) +
   tl::make_member (&GerberImportData::num_metal_layers, "num-metal-layers") +
   tl::make_member (&GerberImportData::num_via_types, "num-via-types") +
   tl::make_element (&GerberImportData::artwork_files, "artwork-files",
@@ -217,11 +219,11 @@ pcb_project_structure ("pcb-project",
   ) +
   tl::make_element (&GerberImportData::reference_points, "reference-points",
     tl::make_element<std::pair <db::DPoint, db::DPoint>, std::vector<std::pair <db::DPoint, db::DPoint> >::const_iterator, std::vector<std::pair <db::DPoint, db::DPoint> > > (&std::vector<std::pair <db::DPoint, db::DPoint> >::begin, &std::vector<std::pair <db::DPoint, db::DPoint> >::end, &std::vector<std::pair <db::DPoint, db::DPoint> >::push_back, "reference-point",
-      tl::make_member (&std::pair <db::DPoint, db::DPoint>::first, "pcb", db::PointConverter<db::DPoint> ()) +
+      tl::make_member (&std::pair <db::DPoint, db::DPoint>::first, "pcb", tl::XMLStringBasedConverter<db::PointConverter<db::DPoint> > ()) +
-      tl::make_member (&std::pair <db::DPoint, db::DPoint>::second, "layout", db::PointConverter<db::DPoint> ())
+      tl::make_member (&std::pair <db::DPoint, db::DPoint>::second, "layout", tl::XMLStringBasedConverter<db::PointConverter<db::DPoint> > ())
     )
   ) +
-  tl::make_member (&GerberImportData::explicit_trans, "explicit-trans", db::TransformationConverter<db::DCplxTrans> ()) +
+  tl::make_member (&GerberImportData::explicit_trans, "explicit-trans", tl::XMLStringBasedConverter<db::TransformationConverter<db::DCplxTrans> > ()) +
   tl::make_member (&GerberImportData::layer_properties_file, "layer-properties-file") +
   tl::make_member (&GerberImportData::num_circle_points, "num-circle-points") +
   tl::make_member (&GerberImportData::merge_flag, "merge-flag") +

src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc

@@ -49,6 +49,8 @@ static struct {
 
 struct ModeConverter
 {
+  typedef lay::StreamImportData::mode_type value_type;
+
   std::string to_string (lay::StreamImportData::mode_type t) const
   {
     for (unsigned int i = 0; i < sizeof (mode_strings) / sizeof (mode_strings [0]); ++i) {
@@ -82,6 +84,8 @@ static struct {
 
 struct LayerModeConverter
 {
+  typedef lay::StreamImportData::layer_mode_type value_type;
+
   std::string to_string (lay::StreamImportData::layer_mode_type t) const
   {
     for (unsigned int i = 0; i < sizeof (layer_mode_strings) / sizeof (layer_mode_strings [0]); ++i) {
@@ -144,8 +148,8 @@ static tl::XMLElementList xml_elements ()
     ) +
     tl::make_member (&StreamImportData::topcell, "cell-name") +
     tl::make_member (&StreamImportData::layer_offset, "layer-offset") +
-    tl::make_member (&StreamImportData::layer_mode, "layer-mode", LayerModeConverter ()) +
+    tl::make_member (&StreamImportData::layer_mode, "layer-mode", tl::XMLStringBasedConverter<LayerModeConverter> ()) +
-    tl::make_member (&StreamImportData::mode, "import-mode", ModeConverter ()) +
+    tl::make_member (&StreamImportData::mode, "import-mode", tl::XMLStringBasedConverter<ModeConverter> ()) +
     tl::make_element (&StreamImportData::reference_points, "reference-points",
       tl::make_element<ref_point, ref_point_v::const_iterator, ref_point_v> (&ref_point_v::begin, &ref_point_v::end, &ref_point_v::push_back, "reference-point",
         tl::make_member (&ref_point::first, "p1") +

src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc

@@ -24,50 +24,49 @@
 #include "dbTechnology.h"
 #include "tlClassRegistry.h"
 
-namespace tl
-{
-  /**
-   *  @brief A specialization of the XMLConverter that is used to serialize the connection info
-   */
-
-  template <>
-  struct XMLStdConverter<db::NetTracerConnectionInfo>
-  {
-    std::string to_string (const db::NetTracerConnectionInfo &v) const
-    {
-      return v.to_string ();
-    }
-
-    void from_string (const std::string &s, db::NetTracerConnectionInfo &v) const
-    {
-      tl::Extractor ex (s.c_str ());
-      v.parse (ex);
-    }
-  };
-
-  /**
-   *  @brief A specialization of the XMLConverter that is used to serialize the symbol info
-   */
-
-  template <>
-  struct XMLStdConverter<db::NetTracerSymbolInfo>
-  {
-    std::string to_string (const db::NetTracerSymbolInfo &v) const
-    {
-      return v.to_string ();
-    }
-
-    void from_string (const std::string &s, db::NetTracerSymbolInfo &v) const
-    {
-      tl::Extractor ex (s.c_str ());
-      v.parse (ex);
-    }
-  };
-}
-
 namespace
 {
 
+/**
+ *  @brief A string converter for NetTracerConnectionInfo
+ */
+
+struct NetTracerConnectionInfoConverter
+{
+  typedef db::NetTracerConnectionInfo value_type;
+
+  std::string to_string (const db::NetTracerConnectionInfo &v) const
+  {
+    return v.to_string ();
+  }
+
+  void from_string (const std::string &s, db::NetTracerConnectionInfo &v) const
+  {
+    tl::Extractor ex (s.c_str ());
+    v.parse (ex);
+  }
+};
+
+/**
+ *  @brief A specialization of the XMLConverter that is used to serialize the symbol info
+ */
+
+struct NetTracerSymbolInfoConverter
+{
+  typedef db::NetTracerSymbolInfo value_type;
+
+  std::string to_string (const db::NetTracerSymbolInfo &v) const
+  {
+    return v.to_string ();
+  }
+
+  void from_string (const std::string &s, db::NetTracerSymbolInfo &v) const
+  {
+    tl::Extractor ex (s.c_str ());
+    v.parse (ex);
+  }
+};
+
 static const db::NetTracerConnectivity *
 get_fallback_default (const db::NetTracerTechnologyComponent &tc)
 {
@@ -129,6 +128,7 @@ template <class Value, class Iter>
 struct FallbackXMLReadAdaptor
 {
   typedef tl::pass_by_ref_tag tag;
+  typedef tl::many_cardinality_tag cardinality;
 
   FallbackXMLReadAdaptor (Iter (db::NetTracerConnectivity::*begin) () const, Iter (db::NetTracerConnectivity::*end) () const)
     : mp_begin (begin), mp_end (end)
@@ -191,14 +191,14 @@ public:
       tl::make_element ((NetTracerTechnologyComponent::const_iterator (NetTracerTechnologyComponent::*) () const) &NetTracerTechnologyComponent::begin, (NetTracerTechnologyComponent::const_iterator (NetTracerTechnologyComponent::*) () const) &NetTracerTechnologyComponent::end, (void (NetTracerTechnologyComponent::*) (const NetTracerConnectivity &)) &NetTracerTechnologyComponent::push_back, "stack",
         tl::make_member (&NetTracerConnectivity::name, &NetTracerConnectivity::set_name, "name") +
         tl::make_member (&NetTracerConnectivity::description, &NetTracerConnectivity::set_description, "description") +
-        tl::make_member ((NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin, (NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end, &NetTracerConnectivity::add, "connection") +
+        tl::make_member ((NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin, (NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end, &NetTracerConnectivity::add, "connection", tl::XMLStringBasedConverter<NetTracerConnectionInfoConverter> ()) +
-        tl::make_member ((NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin_symbols, (NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end_symbols, &NetTracerConnectivity::add_symbol, "symbols")
+        tl::make_member ((NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin_symbols, (NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end_symbols, &NetTracerConnectivity::add_symbol, "symbols", tl::XMLStringBasedConverter<NetTracerSymbolInfoConverter> ())
       ) +
       //  Fallback readers for migrating pre-0.28 setups to 0.28 and backward compatibility
-      tl::XMLMember<NetTracerConnectionInfo, NetTracerTechnologyComponent, FallbackXMLReadAdaptor <NetTracerConnectionInfo, NetTracerConnectivity::const_iterator>, FallbackXMLWriteAdaptor <NetTracerConnectionInfo>, tl::XMLStdConverter <NetTracerConnectionInfo> > (
+      tl::XMLMember<NetTracerConnectionInfo, NetTracerTechnologyComponent, FallbackXMLReadAdaptor <NetTracerConnectionInfo, NetTracerConnectivity::const_iterator>, FallbackXMLWriteAdaptor <NetTracerConnectionInfo>, tl::XMLStringBasedConverter <NetTracerConnectionInfoConverter> > (
         FallbackXMLReadAdaptor <NetTracerConnectionInfo, NetTracerConnectivity::const_iterator> (&NetTracerConnectivity::begin, &NetTracerConnectivity::end),
         FallbackXMLWriteAdaptor <NetTracerConnectionInfo> (&NetTracerConnectivity::add), "connection") +
-      tl::XMLMember<NetTracerSymbolInfo, NetTracerTechnologyComponent, FallbackXMLReadAdaptor <NetTracerSymbolInfo, NetTracerConnectivity::const_symbol_iterator>, FallbackXMLWriteAdaptor <NetTracerSymbolInfo>, tl::XMLStdConverter <NetTracerSymbolInfo> > (
+      tl::XMLMember<NetTracerSymbolInfo, NetTracerTechnologyComponent, FallbackXMLReadAdaptor <NetTracerSymbolInfo, NetTracerConnectivity::const_symbol_iterator>, FallbackXMLWriteAdaptor <NetTracerSymbolInfo>, tl::XMLStringBasedConverter <NetTracerSymbolInfoConverter> > (
         FallbackXMLReadAdaptor <NetTracerSymbolInfo, NetTracerConnectivity::const_symbol_iterator> (&NetTracerConnectivity::begin_symbols, &NetTracerConnectivity::end_symbols),
         FallbackXMLWriteAdaptor <NetTracerSymbolInfo> (&NetTracerConnectivity::add_symbol), "symbols")
     );

src/rdb/rdb/gsiDeclRdb.cc

@@ -1733,10 +1733,12 @@ Class<rdb::Database> decl_ReportDatabase ("rdb", "ReportDatabase",
     "The reader recognizes the format automatically and will choose the appropriate decoder. 'gzip' compressed files are uncompressed "
     "automatically.\n"
   ) + 
-  gsi::method ("save", &rdb::Database::save, gsi::arg ("filename"),
+  gsi::method ("save", &rdb::Database::save, gsi::arg ("filename"), gsi::arg ("binary", false),
     "@brief Saves the database to the given file\n"
     "@param filename The file to which to save the database\n"
-    "The database is always saved in KLayout's XML-based format.\n"
+    "The database is saved in KLayout's XML-based format if 'binary' is false or a ProtocolBuffer-based binary format if 'binary' is true.\n"
+    "\n"
+    "The 'binary' argument has been added in version 0.29.7."
   ),
   "@brief The report database object\n"
   "A report database is organized around a set of items which are associated with cells and categories. "

src/rdb/rdb/rdb.cc

@@ -417,18 +417,18 @@ ValueBase::create_from_shape (const db::Shape &shape, const db::CplxTrans &trans
 //  ValueWrapper implementation
 
 std::string
-ValueWrapper::to_string (const Database *rdb) const
+ValueWrapper::to_string () const
 {
   std::string r;
   r.reserve (200);
 
-  if (tag_id () > 0 && rdb) {
+  if (tag_id () > 0) {
     r += "[";
-    const Tag &tag = rdb->tags ().tag (tag_id ());
+    auto np = rdb::Tags::name_for_id (tag_id ());
-    if (tag.is_user_tag ()) {
+    if (np.second) {
       r += "#";
     }
-    r += tl::to_word_or_quoted_string (tag.name ());
+    r += tl::to_word_or_quoted_string (np.first);
     r += "] ";
   }
 
@@ -438,14 +438,14 @@ ValueWrapper::to_string (const Database *rdb) const
 }
 
 void
-ValueWrapper::from_string (Database *rdb, const std::string &s)
+ValueWrapper::from_string (const std::string &s)
 {
   tl::Extractor ex (s.c_str ());
-  from_string (rdb, ex);
+  from_string (ex);
 }
 
 void
-ValueWrapper::from_string (Database *rdb, tl::Extractor &ex)
+ValueWrapper::from_string (tl::Extractor &ex)
 {
   id_type tag_id = 0;
 
@@ -456,7 +456,7 @@ ValueWrapper::from_string (Database *rdb, tl::Extractor &ex)
     std::string tn;
     ex.read_word_or_quoted (tn);
 
-    tag_id = rdb->tags ().tag (tn, user_tag).id ();
+    tag_id = rdb::Tags::id_for_name (tn, user_tag);
 
     ex.test ("]");
 
@@ -477,16 +477,16 @@ Values::operator= (const Values &d)
 }
 
 bool
-Values::compare (const Values &other, const std::map<id_type, id_type> &tag_map, const std::map<id_type, id_type> &rev_tag_map) const
+Values::compare (const Values &other, const std::set<id_type> &common_tags) const
 {
   Values::const_iterator a = begin (), b = other.begin ();
   while (a != end () && b != other.end ()) {
 
-    id_type t12 = 0;
+    id_type t1 = 0;
     while (a != end () && a->tag_id () != 0) {
-      auto j = tag_map.find (a->tag_id ());
+      auto j = common_tags.find (a->tag_id ());
-      if (j != tag_map.end ()) {
+      if (j != common_tags.end ()) {
-        t12 = j->second;
+        t1 = a->tag_id ();
         break;
       }
       ++a;
@@ -494,9 +494,9 @@ Values::compare (const Values &other, const std::map<id_type, id_type> &tag_map,
 
     id_type t2 = 0;
     while (b != other.end () && b->tag_id () != 0) {
-      auto j = rev_tag_map.find (b->tag_id ());
+      auto j = common_tags.find (b->tag_id ());
-      if (j != rev_tag_map.end ()) {
+      if (j != common_tags.end ()) {
-        t2 = j->first;
+        t2 = b->tag_id ();
         break;
       }
       ++b;
@@ -506,8 +506,8 @@ Values::compare (const Values &other, const std::map<id_type, id_type> &tag_map,
       return b != other.end ();
     }
 
-    if (t12 != t2) {
+    if (t1 != t2) {
-      return t12 < t2;
+      return t1 < t2;
     }
 
     if (a->get () && b->get ()) {
@@ -529,7 +529,7 @@ Values::compare (const Values &other, const std::map<id_type, id_type> &tag_map,
 }
 
 std::string
-Values::to_string (const Database *rdb) const
+Values::to_string () const
 {
   std::string r;
   r.reserve (200);
@@ -540,7 +540,7 @@ Values::to_string (const Database *rdb) const
       r += ";";
     }
 
-    r += v->to_string (rdb);
+    r += v->to_string ();
 
   }
 
@@ -548,14 +548,14 @@ Values::to_string (const Database *rdb) const
 }
 
 void
-Values::from_string (Database *rdb, const std::string &s) 
+Values::from_string (const std::string &s)
 {
   tl::Extractor ex (s.c_str ());
 
   while (! ex.at_end ()) {
 
     ValueWrapper v;
-    v.from_string (rdb, ex);
+    v.from_string (ex);
 
     add (v);
 
@@ -912,6 +912,41 @@ Categories::import_category (Category *category)
 // ------------------------------------------------------------------------------------------
 //  Tags implementation
 
+static tl::Mutex s_tag_namespace_mutex;
+static std::map<std::pair<std::string, bool>, id_type> s_tag_namespace;
+static std::vector<std::pair<std::string, bool> > s_tags;
+
+const std::pair<std::string, bool> &
+Tags::name_for_id (id_type tag_id)
+{
+  tl::MutexLocker locker (&s_tag_namespace_mutex);
+  tl_assert (tag_id > 0 && tag_id <= s_tags.size ());
+  return s_tags [tag_id - 1];
+}
+
+id_type
+Tags::id_for_name (const std::string &name, bool user_flag)
+{
+  tl::MutexLocker locker (&s_tag_namespace_mutex);
+
+  auto k = std::make_pair (name, user_flag);
+  auto t = s_tag_namespace.find (k);
+  if (t != s_tag_namespace.end ()) {
+    return t->second;
+  }
+
+  id_type id = s_tags.size () + 1;
+  s_tag_namespace [k] = id;
+  s_tags.push_back (k);
+  return id;
+}
+
+Tag
+Tags::tag_for_name (const std::string &name, bool user_flag)
+{
+  return Tag (id_for_name (name, user_flag), name, user_flag);
+}
+
 Tags::Tags ()
 {
   //  .. nothing yet ..
@@ -920,10 +955,28 @@ Tags::Tags ()
 void
 Tags::clear ()
 {
-  m_ids_for_names.clear ();
+  m_tags_per_id.clear ();
   m_tags.clear ();
 }
 
+void
+Tags::remove_tag (id_type id)
+{
+  auto j = m_tags_per_id.find (id);
+  if (j != m_tags_per_id.end ()) {
+
+    m_tags.erase (m_tags.begin () + j->second);
+
+    for (auto i = m_tags_per_id.begin (); i != m_tags_per_id.end (); ++i) {
+      if (i->second > j->second) {
+        i->second -= 1;
+      }
+    }
+    m_tags_per_id.erase (id);
+
+  }
+}
+
 const Tag &
 Tags::tag (const std::string &name, bool user_tag) const
 {
@@ -933,26 +986,31 @@ Tags::tag (const std::string &name, bool user_tag) const
 Tag &
 Tags::tag (const std::string &name, bool user_tag)
 {
-  std::map <std::pair<std::string, bool>, id_type>::const_iterator i = m_ids_for_names.find (std::make_pair (name, user_tag));
+  id_type id = id_for_name (name, user_tag);
-  if (i == m_ids_for_names.end ()) {
+  auto i = m_tags_per_id.find (id);
-    i = m_ids_for_names.insert (std::make_pair (std::make_pair (name, user_tag), m_tags.size () + 1)).first;
+  if (i == m_tags_per_id.end ()) {
-    m_tags.push_back (Tag (i->second, name, user_tag));
+    m_tags.push_back (Tag (id, name, user_tag));
+    m_tags_per_id [id] = m_tags.size () - 1;
+    return m_tags.back ();
+  } else {
+    return m_tags [i->second];
   }
-  return m_tags [i->second - 1];
 }
 
 const Tag &
 Tags::tag (id_type id) const
 {
-  tl_assert (id < m_tags.size () + 1 && id > 0);
+  auto i = m_tags_per_id.find (id);
-  return m_tags [id - 1];
+  tl_assert (i != m_tags_per_id.end ());
+  return m_tags [i->second];
 }
 
 Tag &
 Tags::tag (id_type id)
 {
-  tl_assert (id < m_tags.size () + 1 && id > 0);
+  auto i = m_tags_per_id.find (id);
-  return m_tags [id - 1];
+  tl_assert (i != m_tags_per_id.end ());
+  return m_tags [i->second];
 }
 
 void
@@ -965,7 +1023,7 @@ Tags::import_tag (const Tag &t)
 bool
 Tags::has_tag (const std::string &name, bool user_tag) const
 {
-  return m_ids_for_names.find (std::make_pair (name, user_tag)) != m_ids_for_names.end ();
+  return m_tags_per_id.find (id_for_name (name, user_tag)) != m_tags_per_id.end ();
 }
 
 // ------------------------------------------------------------------------------------------
@@ -1013,30 +1071,25 @@ Item &Item::operator= (const Item &d)
 void
 Item::add_tag (id_type tag_id)
 {
-  if (m_tag_ids.size () <= tag_id) {
+  m_tag_ids.insert (tag_id);
-    m_tag_ids.resize (tag_id + 1, false);
-  }
-  m_tag_ids [tag_id] = true;
 }
 
 void
 Item::remove_tag (id_type tag_id)
 {
-  if (m_tag_ids.size () > tag_id) {
+  m_tag_ids.erase (tag_id);
-    m_tag_ids [tag_id] = false;
-  }
 }
 
 void
 Item::remove_tags ()
 {
-  m_tag_ids = std::vector <bool> ();
+  m_tag_ids.clear ();
 }
 
 bool
 Item::has_tag (id_type tag_id) const
 {
-  return m_tag_ids.size () > tag_id && m_tag_ids [tag_id];
+  return m_tag_ids.find (tag_id) != m_tag_ids.end ();
 }
 
 std::string
@@ -1089,13 +1142,12 @@ Item::tag_str () const
 
   if (! m_tag_ids.empty ()) {
 
-    id_type tag_id = 0;
+    for (std::set<id_type>::const_iterator t = m_tag_ids.begin (); t != m_tag_ids.end (); ++t) {
-    for (std::vector<bool>::const_iterator t = m_tag_ids.begin (); t != m_tag_ids.end (); ++t, ++tag_id) {
       if (*t) {
         if (! r.empty ()) {
           r += ",";
         }
-        const Tag &tag = mp_database->tags ().tag (tag_id);
+        const Tag &tag = mp_database->tags ().tag (*t);
         if (tag.is_user_tag ()) {
           r += "#";
         }
@@ -1727,6 +1779,7 @@ Database::clear ()
 
   m_generator = "";
   m_filename = "";
+  m_binary = false;
   m_description = "";
   m_name = "";
   m_topcell = "";
@@ -1779,6 +1832,7 @@ Database::load (std::string fn)
   tl::log << "Loading RDB from " << fn;
 
   clear ();
+  set_binary (false);
 
   tl::InputStream stream (fn);
 
@@ -1813,33 +1867,30 @@ namespace
   class ValueMapEntryCompare
   {
   public:
-    ValueMapEntryCompare (const std::map<id_type, id_type> &tag2tag, const std::map<id_type, id_type> &rev_tag2tag)
+    ValueMapEntryCompare (const std::set<id_type> &common_tags)
     {
-      mp_tag2tag = &tag2tag;
+      mp_common_tags = &common_tags;
-      mp_rev_tag2tag = &rev_tag2tag;
     }
 
     bool operator() (const Item *a, const Item *b) const
     {
-      return a->values ().compare (b->values (), *mp_tag2tag, *mp_rev_tag2tag);
+      return a->values ().compare (b->values (), *mp_common_tags);
     }
 
   private:
-    const std::map<id_type, id_type> *mp_tag2tag;
+    const std::set<id_type> *mp_common_tags;
-    const std::map<id_type, id_type> *mp_rev_tag2tag;
   };
 
   class ValueMapEntry
   {
   public:
     ValueMapEntry ()
-      : mp_tag2tag (0), mp_rev_tag2tag (0)
+      : mp_common_tags (0)
     { }
 
-    void build (const rdb::Database &rdb, id_type cell_id, id_type cat_id, const std::map<id_type, id_type> &tag2tag, const std::map<id_type, id_type> &rev_tag2tag)
+    void build (const rdb::Database &rdb, id_type cell_id, id_type cat_id, const std::set<id_type> &common_tags)
     {
-      mp_tag2tag = &tag2tag;
+      mp_common_tags = &common_tags;
-      mp_rev_tag2tag = &rev_tag2tag;
 
       auto i2i = rdb.items_by_cell_and_category (cell_id, cat_id);
 
@@ -1853,13 +1904,13 @@ namespace
         m_items.push_back ((*i).operator-> ());
       }
 
-      ValueMapEntryCompare cmp (*mp_tag2tag, *mp_rev_tag2tag);
+      ValueMapEntryCompare cmp (*mp_common_tags);
       std::sort (m_items.begin (), m_items.end (), cmp);
     }
 
     const Item *find (const rdb::Item &item) const
     {
-      ValueMapEntryCompare cmp (*mp_tag2tag, *mp_rev_tag2tag);
+      ValueMapEntryCompare cmp (*mp_common_tags);
 
       auto i = std::lower_bound (m_items.begin (), m_items.end (), &item, cmp);
       if (i == m_items.end ()) {
@@ -1875,8 +1926,7 @@ namespace
 
   public:
     std::vector<const Item *> m_items;
-    const std::map<id_type, id_type> *mp_tag2tag;
+    const std::set<id_type> *mp_common_tags;
-    const std::map<id_type, id_type> *mp_rev_tag2tag;
   };
 }
 
@@ -1897,8 +1947,6 @@ Database::apply (const rdb::Database &other)
 {
   std::map<id_type, id_type> cell2cell;
   std::map<id_type, id_type> cat2cat;
-  std::map<id_type, id_type> tag2tag;
-  std::map<id_type, id_type> rev_tag2tag;
 
   for (auto c = other.cells ().begin (); c != other.cells ().end (); ++c) {
     //  TODO: do we have a consistent scheme of naming variants? What requirements
@@ -1913,16 +1961,15 @@ Database::apply (const rdb::Database &other)
     map_category (*c, *this, cat2cat);
   }
 
-  std::map<std::string, id_type> tags_by_name;
+  std::set<id_type> t1;
   for (auto c = tags ().begin_tags (); c != tags ().end_tags (); ++c) {
-    tags_by_name.insert (std::make_pair (c->name (), c->id ()));
+    t1.insert (c->id ());
   }
 
+  std::set<id_type> common_tags;
   for (auto c = other.tags ().begin_tags (); c != other.tags ().end_tags (); ++c) {
-    auto t = tags_by_name.find (c->name ());
+    if (t1.find (c->id ()) != t1.end ()) {
-    if (t != tags_by_name.end ()) {
+      common_tags.insert (c->id ());
-      tag2tag.insert (std::make_pair (t->second, c->id ()));
-      rev_tag2tag.insert (std::make_pair (c->id (), t->second));
     }
   }
 
@@ -1944,7 +1991,7 @@ Database::apply (const rdb::Database &other)
     auto vmap = value_map.find (std::make_pair (icell->second, icat->second));
     if (vmap == value_map.end ()) {
       vmap = value_map.insert (std::make_pair (std::make_pair (icell->second, icat->second), ValueMapEntry ())).first;
-      vmap->second.build (other, icell->second, icat->second, tag2tag, rev_tag2tag);
+      vmap->second.build (other, icell->second, icat->second, common_tags);
     }
 
     //  find a value in the reference DB

src/rdb/rdb/rdb.h

@@ -639,17 +639,17 @@ public:
   /**
    *  @brief Convert the values collection to a string
    */
-  std::string to_string (const Database *rdb = 0) const;
+  std::string to_string () const;
 
   /**
    *  @brief Fill the values collection from the string
    */
-  void from_string (Database *rdb, const std::string &s);  
+  void from_string (const std::string &s);
 
   /**
    *  @brief Fill the values collection from an extractor
    */
-  void from_string (Database *rdb, tl::Extractor &ex);  
+  void from_string (tl::Extractor &ex);
 
 private:
   ValueBase *mp_ptr;
@@ -689,15 +689,12 @@ public:
   /**
    *  @brief Compare two value sets (less operator)
    *
-   *  This compare function will use the tag mapping provided by tag map ("this" tag id to "other" tag id).
+   *  This compare function will use the tags provide in "common_tags". Tags outside this set are ignored.
-   *  Values with tags not listed in the tag map will not be compared.
    *  Untagged values (tag_id 0) will be compared always.
    *
-   *  "rev_tag_map" needs to be the reverse of "tag_map".
-   *
    *  The order of the values matters.
    */
-  bool compare (const Values &other, const std::map<id_type, id_type> &tag_map, const std::map<id_type, id_type> &rev_tag_map) const;
+  bool compare (const Values &other, const std::set<id_type> &common_tags) const;
 
   /**
    *  @brief The const iterator (begin)
@@ -770,12 +767,12 @@ public:
   /**
    *  @brief Convert the values collection to a string
    */
-  std::string to_string (const Database *rdb) const;
+  std::string to_string () const;
 
   /**
    *  @brief Fill the values collection from the string
    */
-  void from_string (Database *rdb, const std::string &s);  
+  void from_string (const std::string &s);
 
 private:
   std::list <ValueWrapper> m_values;
@@ -1067,7 +1064,7 @@ private:
   size_t m_multiplicity;
   std::string m_comment;
   bool m_visited;
-  std::vector <bool> m_tag_ids;
+  std::set <id_type> m_tag_ids;
   Database *mp_database;
   std::string m_image_str;
 
@@ -1989,14 +1986,46 @@ public:
 
   /**
    *  @brief Clear the collection of tags
+   *
+   *  NOTE: this will not remove the tags from items or values, so the use cases for this method
+   *  are limited.
    */
   void clear ();
 
+  /**
+   *  @brief Removes the tag with the given ID
+   *
+   *  NOTE: this will not remove the tags from items or values, so the use cases for this method
+   *  are limited.
+   */
+  void remove_tag (id_type id);
+
+  /**
+   *  @brief Gets the name and user flag for a tag ID
+   *
+   *  This method will assert if the tag ID is not valid in the global namespace.
+   */
+  static const std::pair<std::string, bool> &name_for_id (id_type tag_id);
+
+  /**
+   *  @brief Gets the id for a given name and user flag
+   *
+   *  This will pull the ID from the global namespace.
+   */
+  static id_type id_for_name (const std::string &name, bool user_flag);
+
+  /**
+   *  @brief Gets a tag object for a given name and user flag
+   *
+   *  This will create the tag from the global namespace.
+   */
+  static Tag tag_for_name (const std::string &name, bool user_flag);
+
 private:
   friend class Database;
 
-  mutable std::map <std::pair<std::string, bool>, id_type> m_ids_for_names;
+  mutable std::map<id_type, size_t> m_tags_per_id;
-  mutable std::vector <Tag> m_tags;
+  mutable std::vector<Tag> m_tags;
 };
 
 /**
@@ -2025,7 +2054,7 @@ public:
   ~Database ();
 
   /**
-   *  @brief Get the database description
+   *  @brief Gets the database description
    */
   const std::string &description () const
   {
@@ -2033,7 +2062,7 @@ public:
   }
 
   /**
-   *  @brief Set the database description
+   *  @brief Sets the database description
    */
   void set_description (const std::string &description)
   {
@@ -2042,7 +2071,7 @@ public:
   }
 
   /**
-   *  @brief Get the database original file
+   *  @brief Gets the database original file
    *
    *  The original file describes what original file the marker database
    *  was derived from.
@@ -2053,7 +2082,7 @@ public:
   }
 
   /**
-   *  @brief Set the database original file
+   *  @brief Sets the database original file
    */
   void set_original_file (const std::string &original_file)
   {
@@ -2062,7 +2091,7 @@ public:
   }
 
   /**
-   *  @brief Get the database name
+   *  @brief Gets the database name
    */
   const std::string &name () const
   {
@@ -2070,7 +2099,7 @@ public:
   }
 
   /**
-   *  @brief Set the database name
+   *  @brief Sets the database name
    */
   void set_name (const std::string &name)
   {
@@ -2078,7 +2107,7 @@ public:
   }
 
   /**
-   *  @brief Get the file name
+   *  @brief Gets the file name
    */
   const std::string &filename () const
   {
@@ -2086,7 +2115,7 @@ public:
   }
 
   /**
-   *  @brief Set the file name
+   *  @brief Sets the file name
    */
   void set_filename (const std::string &filename)
   {
@@ -2095,7 +2124,24 @@ public:
   }
 
   /**
-   *  @brief Get the generator name
+   *  @brief Gets a value indicating whether the database was saved to binary format
+   */
+  bool binary () const
+  {
+    return m_binary;
+  }
+
+  /**
+   *  @brief Sets a value indicating whether the database was saved to binary format
+   */
+  void set_binary (bool bin)
+  {
+    set_modified ();
+    m_binary = bin;
+  }
+
+  /**
+   *  @brief Gets the generator name
    */
   const std::string &generator () const
   {
@@ -2103,7 +2149,7 @@ public:
   }
 
   /**
-   *  @brief Set the generator name
+   *  @brief Sets the generator name
    */
   void set_generator (const std::string &generator)
   {
@@ -2112,7 +2158,7 @@ public:
   }
 
   /**
-   *  @brief Set the top cell name 
+   *  @brief Sets the top cell name
    */
   void set_top_cell_name (const std::string &topcell)
   {
@@ -2121,7 +2167,7 @@ public:
   }
 
   /**
-   *  @brief Return the top cell name
+   *  @brief Returns the top cell name
    */
   const std::string &top_cell_name () const
   {
@@ -2129,7 +2175,7 @@ public:
   }
 
   /*
-   *  @brief Get the reference to the tags collection (const version)
+   *  @brief Gets the reference to the tags collection (const version)
    */
   const Tags &tags () const
   {
@@ -2137,14 +2183,14 @@ public:
   }
 
   /**
-   *  @brief Import tags
+   *  @brief Imports tags
    *
    *  This method is provided for persistency application only. It should not be used otherwise.
    */
   void import_tags (const Tags &tags);
 
   /**
-   *  @brief Get the reference to the categories collection (const version)
+   *  @brief Gets the reference to the categories collection (const version)
    */
   const Categories &categories () const
   {
@@ -2152,7 +2198,7 @@ public:
   }
 
   /**
-   *  @brief Get the reference to the categories collection (non-const version)
+   *  @brief Gets the reference to the categories collection (non-const version)
    */
   Categories &categories_non_const ()
   {
@@ -2160,7 +2206,7 @@ public:
   }
 
   /**
-   *  @brief Import categories
+   *  @brief Imports categories
    *
    *  This method is provided for persistency application only. It should not be used otherwise.
    *  This will take over the ownership over the Categories object.
@@ -2168,17 +2214,17 @@ public:
   void import_categories (Categories *categories);
 
   /**
-   *  @brief Create a category and register it 
+   *  @brief Creates a category and register it
    */
   Category *create_category (const std::string &name);
 
   /**
-   *  @brief Create a category as a subcategory and register it 
+   *  @brief Creates a category as a subcategory and register it
    */
   Category *create_category (Category *parent, const std::string &name);
 
   /**
-   *  @brief Create a category as a subcategory in the container and register it 
+   *  @brief Creates a category as a subcategory in the container and register it
    *
    *  Hint: this method does not set the parent properly and must not be used
    *  under normal circumstances. It is provided as a internal method and
@@ -2187,7 +2233,7 @@ public:
   Category *create_category (Categories *container, const std::string &name);
 
   /**
-   *  @brief Get the category pointer for a category name (const version)
+   *  @brief Gets the category pointer for a category name (const version)
    *
    *  This method returns 0 if the category name is invalid.
    */
@@ -2197,7 +2243,7 @@ public:
   }
 
   /**
-   *  @brief Get the category pointer for a category id (const version)
+   *  @brief Gets the category pointer for a category id (const version)
    *
    *  This method returns 0 if the category is invalid.
    */
@@ -2207,21 +2253,21 @@ public:
   }
 
   /**
-   *  @brief Get the category pointer for a category name (non-const version)
+   *  @brief Gets the category pointer for a category name (non-const version)
    *
    *  This method returns 0 if the category name is invalid.
    */
   Category *category_by_name_non_const (const std::string &name);
 
   /**
-   *  @brief Get the category pointer for a category id (non-const version)
+   *  @brief Gets the category pointer for a category id (non-const version)
    *
    *  This method returns 0 if the category is invalid.
    */
   Category *category_by_id_non_const (id_type id);
 
   /**
-   *  @brief Access to the cell collection (const)
+   *  @brief Gets the cell collection (const)
    */
   const Cells &cells () const
   {
@@ -2229,7 +2275,7 @@ public:
   }
 
   /**
-   *  @brief Access to the cell collection
+   *  @brief Gets the cell collection
    */
   Cells &cells_non_const ()
   {
@@ -2237,14 +2283,14 @@ public:
   }
 
   /**
-   *  @brief Import cells
+   *  @brief Imports cells
    *
    *  This method is provided for persistency application only. It should not be used otherwise.
    */
   void import_cells (const Cells &cells);
 
   /**
-   *  @brief Create a cell and register it
+   *  @brief Creates a cell and registers it
    *
    *  If a cell with that name already exists, a variant is created
    */
@@ -2254,7 +2300,7 @@ public:
   }
 
   /**
-   *  @brief Create a cell variant and register it
+   *  @brief Creates a cell variant and registers it
    *
    *  A cell with name name/variant combination must not exist already.
    *  If the variant string is empty, this method behaves the same as the
@@ -2266,14 +2312,14 @@ public:
   Cell *create_cell (const std::string &name, const std::string &variant, const std::string &layout_name);
 
   /**
-   *  @brief Get all variants registered for a given cell name (not qname!)
+   *  @brief Gets all variants registered for a given cell name (not qname!)
    *
    *  @return a vector of id's corresponding to the given variants or an empty vector if the name is not valid or the cell has no variants
    */
   const std::vector <id_type> &variants (const std::string &name);
 
   /**
-   *  @brief Get the cell pointer for a cell name or name:variant combination (const version)
+   *  @brief Gets the cell pointer for a cell name or name:variant combination (const version)
    *
    *  This method returns 0 if the cell name or name:variant combination is invalid.
    */
@@ -2283,7 +2329,7 @@ public:
   }
 
   /**
-   *  @brief Get the cell pointer for a cell id (const version)
+   *  @brief Gets the cell pointer for a cell id (const version)
    *
    *  This method returns 0 if the cell id is invalid.
    */
@@ -2293,21 +2339,21 @@ public:
   }
 
   /**
-   *  @brief Get the cell pointer for a cell name or name:variant combination (non-const version)
+   *  @brief Gets the cell pointer for a cell name or name:variant combination (non-const version)
    *
    *  This method returns 0 if the cell name or name:variant combination is invalid.
    */
   Cell *cell_by_qname_non_const (const std::string &qname);
 
   /**
-   *  @brief Get the cell pointer for a cell id (non-const version)
+   *  @brief Gets the cell pointer for a cell id (non-const version)
    *
    *  This method returns 0 if the cell id is invalid.
    */
   Cell *cell_by_id_non_const (id_type id);
 
   /**
-   *  @brief Report the number of items in total
+   *  @brief Reports the number of items in total
    */
   size_t num_items () const
   {
@@ -2315,7 +2361,7 @@ public:
   }
 
   /**
-   *  @brief Report the number of items visited
+   *  @brief Reports the number of items visited
    */
   size_t num_items_visited () const
   {
@@ -2323,37 +2369,37 @@ public:
   }
 
   /**
-   *  @brief Report the number of items for a given cell and category id
+   *  @brief Reports the number of items for a given cell and category id
    */
   size_t num_items (id_type cell_id, id_type category_id) const;
 
   /**
-   *  @brief Report the number of items visited
+   *  @brief Reports the number of items visited
    */
   size_t num_items_visited (id_type cell_id, id_type category_id) const;
 
   /**
-   *  @brief Create a new item for the given cell and category (both given by id)
+   *  @brief Creates a new item for the given cell and category (both given by id)
    */
   Item *create_item (id_type cell_id, id_type category_id);
 
   /**
-   *  @brief Set a tag's description
+   *  @brief Sets a tag's description
    */
   void set_tag_description (id_type tag_id, const std::string &description);
 
   /**
-   *  @brief Set an item to visited state or reset the state
+   *  @brief Sets an item to visited state or reset the state
    */
   void set_item_visited (const Item *item, bool visited);
 
   /**
-   *  @brief Add a tag to the given item
+   *  @brief Adds a tag to the given item
    */
   void add_item_tag (const Item *item, id_type tag);
 
   /**
-   *  @brief Remove a tag from the given item
+   *  @brief Removes a tag from the given item
    */
   void remove_item_tag (const Item *item, id_type tag);
 
@@ -2364,23 +2410,23 @@ public:
 
 #if defined(HAVE_QT)
   /**
-   *  @brief Set the image of an item
+   *  @brief Sets the image of an item
    */
   void set_item_image (const Item *item, const QImage &image);
 #endif
 
   /**
-   *  @brief Set the image string of an item
+   *  @brief Sets the image string of an item
    */
   void set_item_image_str (const Item *item, const std::string &image_str);
 
   /**
-   *  @brief Set the multiplicity of an item
+   *  @brief Sets the multiplicity of an item
    */
   void set_item_multiplicity (const Item *item, size_t n);
 
   /**
-   *  @brief Get the items collection (const version)
+   *  @brief Gets the items collection (const version)
    */
   const Items &items () const
   {
@@ -2388,7 +2434,7 @@ public:
   }
 
   /**
-   *  @brief Get the items collection (non-const version)
+   *  @brief Gets the items collection (non-const version)
    */
   Items &items_non_const ()
   {
@@ -2396,7 +2442,7 @@ public:
   }
 
   /**
-   *  @brief Set the items collection
+   *  @brief Sets the items collection
    *
    *  This method is provided for persistency application only. It should not be used otherwise.
    *  This will take ownership over the items collection.
@@ -2404,32 +2450,32 @@ public:
   void set_items (Items *items);
 
   /**
-   *  @brief Get an iterator pair that delivers the const items (ItemRef) for a given cell
+   *  @brief Gets an iterator pair that delivers the const items (ItemRef) for a given cell
    */
   std::pair<const_item_ref_iterator, const_item_ref_iterator> items_by_cell (id_type cell_id) const;
 
   /**
-   *  @brief Get an iterator pair that delivers the non-const items (ItemRef) for a given cell
+   *  @brief Gets an iterator pair that delivers the non-const items (ItemRef) for a given cell
    */
   std::pair<item_ref_iterator, item_ref_iterator> items_by_cell (id_type cell_id);
 
   /**
-   *  @brief Get an iterator that delivers the const items (ItemRef) for a given category
+   *  @brief Gets an iterator that delivers the const items (ItemRef) for a given category
    */
   std::pair<const_item_ref_iterator, const_item_ref_iterator> items_by_category (id_type category_id) const;
 
   /**
-   *  @brief Get an iterator that delivers the non-const items (ItemRef) for a given category
+   *  @brief Gets an iterator that delivers the non-const items (ItemRef) for a given category
    */
   std::pair<item_ref_iterator, item_ref_iterator> items_by_category (id_type category_id);
 
   /**
-   *  @brief Get an iterator that delivers the const items (ItemRef) for a given cell and category
+   *  @brief Gets an iterator that delivers the const items (ItemRef) for a given cell and category
    */
   std::pair<const_item_ref_iterator, const_item_ref_iterator> items_by_cell_and_category (id_type cell_id, id_type category_id) const;
 
   /**
-   *  @brief Get an iterator that delivers the non-const items (ItemRef) for a given cell and category
+   *  @brief Gets an iterator that delivers the non-const items (ItemRef) for a given cell and category
    */
   std::pair<item_ref_iterator, item_ref_iterator> items_by_cell_and_category (id_type cell_id, id_type category_id);
 
@@ -2442,7 +2488,7 @@ public:
   }
 
   /**
-   *  @brief Reset the modified file
+   *  @brief Resets the modified file
    */
   void reset_modified ()
   {
@@ -2450,19 +2496,19 @@ public:
   }
 
   /**
-   *  @brief Save the database to a file
+   *  @brief Saves the database to a file
    */
-  void save (const std::string &filename);
+  void save (const std::string &filename, bool binary = false);
 
   /**
-   *  @brief Write the database to a file
+   *  @brief Writes the database to a file
    *
    *  This function is like "save", but does not update the file name attribute.
    */
-  void write (const std::string &filename);
+  void write (const std::string &filename, bool binary = false);
 
   /**
-   *  @brief Load the database from a file
+   *  @brief Loads the database from a file
    *
    *  Note: This clears the existing database.
    *  The argument intentionally is a copy, so we can call
@@ -2491,6 +2537,7 @@ public:
 private:
   std::string m_generator;
   std::string m_filename;
+  bool m_binary;
   std::string m_description;
   std::string m_original_file;
   std::string m_name;

src/rdb/rdb/rdbFile.cc

@@ -38,104 +38,129 @@ namespace rdb
 
 struct ValueConverter
 {
-  ValueConverter (rdb::Database *rdb)
+  typedef std::string pb_type;
-    : mp_rdb (rdb)
+
+  ValueConverter ()
   {
   }
 
   std::string to_string (const ValueWrapper &value) const
   {
-    return value.to_string (mp_rdb);
+    return value.to_string ();
   }
 
   void from_string (const std::string &s, ValueWrapper &value) const
   {
-    value.from_string (mp_rdb, s);
+    value.from_string (s);
   }
 
-private:
+  std::string pb_encode (const ValueWrapper &value) const
-  rdb::Database *mp_rdb;
+  {
-};
+    return value.to_string ();
+  }
 
-static 
+  void pb_decode (const std::string &s, ValueWrapper &value) const
-tl::XMLElementList categories_format = 
+  {
-  tl::make_element_with_parent_ref<rdb::Category, rdb::Categories::const_iterator, rdb::Categories> (&rdb::Categories::begin, &rdb::Categories::end, &rdb::Categories::import_category, "category", 
+    value.from_string (s);
-    tl::make_member<std::string, rdb::Category> (&rdb::Category::name, &rdb::Category::set_name, "name") + 
+  }
-    tl::make_member<std::string, rdb::Category> (&rdb::Category::description, &rdb::Category::set_description, "description") +
+};
-    tl::make_element_with_parent_ref<rdb::Categories, rdb::Category> (&rdb::Category::sub_categories, &rdb::Category::import_sub_categories, "categories",
-      &categories_format
-    ) 
-  ) 
-;
 
 //  generation of the RDB file XML structure
 static tl::XMLStruct <rdb::Database> 
-make_rdb_structure (rdb::Database *rdb)
+make_rdb_structure ()
 {
-  return tl::XMLStruct <rdb::Database>("report-database", 
+  static
-    tl::make_member<std::string, rdb::Database> (&rdb::Database::description, &rdb::Database::set_description, "description") +
+  tl::XMLElementList categories_format =
-    tl::make_member<std::string, rdb::Database> (&rdb::Database::original_file, &rdb::Database::set_original_file, "original-file") +
+    tl::make_element_with_parent_ref<rdb::Category, rdb::Categories::const_iterator, rdb::Categories> (&rdb::Categories::begin, &rdb::Categories::end, &rdb::Categories::import_category, "category#1",
-    tl::make_member<std::string, rdb::Database> (&rdb::Database::generator, &rdb::Database::set_generator, "generator") +
+      tl::make_member<std::string, rdb::Category> (&rdb::Category::name, &rdb::Category::set_name, "name#1") +
-    tl::make_member<std::string, rdb::Database> (&rdb::Database::top_cell_name, &rdb::Database::set_top_cell_name, "top-cell") +
+      tl::make_member<std::string, rdb::Category> (&rdb::Category::description, &rdb::Category::set_description, "description#2") +
-    tl::make_element<rdb::Tags, rdb::Database> (&rdb::Database::tags, &rdb::Database::import_tags, "tags",
+      tl::make_element_with_parent_ref<rdb::Categories, rdb::Category> (&rdb::Category::sub_categories, &rdb::Category::import_sub_categories, "categories#3",
-      tl::make_element<rdb::Tag, rdb::Tags::const_iterator, rdb::Tags> (&rdb::Tags::begin_tags, &rdb::Tags::end_tags, &rdb::Tags::import_tag, "tag", 
+        &categories_format
-        tl::make_member<std::string, rdb::Tag> (&rdb::Tag::name, &rdb::Tag::set_name, "name") + 
+      )
-        tl::make_member<std::string, rdb::Tag> (&rdb::Tag::description, &rdb::Tag::set_description, "description")
+    )
+  ;
+
+  return tl::XMLStruct <rdb::Database>("report-database#0",
+    tl::make_member<std::string, rdb::Database> (&rdb::Database::description, &rdb::Database::set_description, "description#1") +
+    tl::make_member<std::string, rdb::Database> (&rdb::Database::original_file, &rdb::Database::set_original_file, "original-file#2") +
+    tl::make_member<std::string, rdb::Database> (&rdb::Database::generator, &rdb::Database::set_generator, "generator#3") +
+    tl::make_member<std::string, rdb::Database> (&rdb::Database::top_cell_name, &rdb::Database::set_top_cell_name, "top-cell#4") +
+    tl::make_element<rdb::Tags, rdb::Database> (&rdb::Database::tags, &rdb::Database::import_tags, "tags#5",
+      tl::make_element<rdb::Tag, rdb::Tags::const_iterator, rdb::Tags> (&rdb::Tags::begin_tags, &rdb::Tags::end_tags, &rdb::Tags::import_tag, "tag#1",
+        tl::make_member<std::string, rdb::Tag> (&rdb::Tag::name, &rdb::Tag::set_name, "name#1") +
+        tl::make_member<std::string, rdb::Tag> (&rdb::Tag::description, &rdb::Tag::set_description, "description#2")
       ) 
     ) + 
-    tl::make_element_with_parent_ref<rdb::Categories, rdb::Database> (&rdb::Database::categories, &rdb::Database::import_categories, "categories",
+    tl::make_element_with_parent_ref<rdb::Categories, rdb::Database> (&rdb::Database::categories, &rdb::Database::import_categories, "categories#6",
       &categories_format
     ) +
-    tl::make_element_with_parent_ref<rdb::Cells, rdb::Database> (&rdb::Database::cells, &rdb::Database::import_cells, "cells",
+    tl::make_element_with_parent_ref<rdb::Cells, rdb::Database> (&rdb::Database::cells, &rdb::Database::import_cells, "cells#7",
       // must be sorted cells (children after parents)!
-      tl::make_element_with_parent_ref<rdb::Cell, rdb::Cells::const_iterator, rdb::Cells> (&rdb::Cells::begin, &rdb::Cells::end, &rdb::Cells::import_cell, "cell", 
+      tl::make_element_with_parent_ref<rdb::Cell, rdb::Cells::const_iterator, rdb::Cells> (&rdb::Cells::begin, &rdb::Cells::end, &rdb::Cells::import_cell, "cell#1",
-        tl::make_member<std::string, rdb::Cell> (&rdb::Cell::name, &rdb::Cell::set_name, "name") +
+        tl::make_member<std::string, rdb::Cell> (&rdb::Cell::name, &rdb::Cell::set_name, "name#1") +
-        tl::make_member<std::string, rdb::Cell> (&rdb::Cell::variant, &rdb::Cell::set_variant, "variant") +
+        tl::make_member<std::string, rdb::Cell> (&rdb::Cell::variant, &rdb::Cell::set_variant, "variant#2") +
-        tl::make_member<std::string, rdb::Cell> (&rdb::Cell::layout_name, &rdb::Cell::set_layout_name, "layout-name") +
+        tl::make_member<std::string, rdb::Cell> (&rdb::Cell::layout_name, &rdb::Cell::set_layout_name, "layout-name#3") +
-        tl::make_element_with_parent_ref<rdb::References, rdb::Cell> (&rdb::Cell::references, &rdb::Cell::import_references, "references",
+        tl::make_element_with_parent_ref<rdb::References, rdb::Cell> (&rdb::Cell::references, &rdb::Cell::import_references, "references#4",
-          tl::make_element_with_parent_ref<rdb::Reference, rdb::References::const_iterator, rdb::References> (&rdb::References::begin, &rdb::References::end, &rdb::References::insert, "ref", 
+          tl::make_element_with_parent_ref<rdb::Reference, rdb::References::const_iterator, rdb::References> (&rdb::References::begin, &rdb::References::end, &rdb::References::insert, "ref#1",
-            tl::make_member<std::string, rdb::Reference> (&rdb::Reference::parent_cell_qname, &rdb::Reference::set_parent_cell_qname, "parent") + 
+            tl::make_member<std::string, rdb::Reference> (&rdb::Reference::parent_cell_qname, &rdb::Reference::set_parent_cell_qname, "parent#1") +
-            tl::make_member<std::string, rdb::Reference> (&rdb::Reference::trans_str, &rdb::Reference::set_trans_str, "trans") 
+            tl::make_member<std::string, rdb::Reference> (&rdb::Reference::trans_str, &rdb::Reference::set_trans_str, "trans#2")
           )
         )
       )
     ) + 
-    tl::make_element_with_parent_ref<rdb::Items, rdb::Database> (&rdb::Database::items, &rdb::Database::set_items, "items",
+    tl::make_element_with_parent_ref<rdb::Items, rdb::Database> (&rdb::Database::items, &rdb::Database::set_items, "items#8",
-      tl::make_element_with_parent_ref<rdb::Item, rdb::Items::const_iterator, rdb::Items> (&rdb::Items::begin, &rdb::Items::end, &rdb::Items::add_item, "item", 
+      tl::make_element_with_parent_ref<rdb::Item, rdb::Items::const_iterator, rdb::Items> (&rdb::Items::begin, &rdb::Items::end, &rdb::Items::add_item, "item#1",
-        tl::make_member<std::string, rdb::Item> (&rdb::Item::tag_str, &rdb::Item::set_tag_str, "tags") + 
+        tl::make_member<std::string, rdb::Item> (&rdb::Item::tag_str, &rdb::Item::set_tag_str, "tags#1") +
-        tl::make_member<std::string, rdb::Item> (&rdb::Item::category_name, &rdb::Item::set_category_name, "category") + 
+        tl::make_member<std::string, rdb::Item> (&rdb::Item::category_name, &rdb::Item::set_category_name, "category#2") +
-        tl::make_member<std::string, rdb::Item> (&rdb::Item::cell_qname, &rdb::Item::set_cell_qname, "cell") +
+        tl::make_member<std::string, rdb::Item> (&rdb::Item::cell_qname, &rdb::Item::set_cell_qname, "cell#3") +
-        tl::make_member<bool, rdb::Item> (&rdb::Item::visited, &rdb::Item::set_visited, "visited") +
+        tl::make_member<bool, rdb::Item> (&rdb::Item::visited, &rdb::Item::set_visited, "visited#4") +
-        tl::make_member<size_t, rdb::Item> (&rdb::Item::multiplicity, &rdb::Item::set_multiplicity, "multiplicity") +
+        tl::make_member<size_t, rdb::Item> (&rdb::Item::multiplicity, &rdb::Item::set_multiplicity, "multiplicity#5") +
-        tl::make_member<std::string, rdb::Item> (&rdb::Item::comment, &rdb::Item::set_comment, "comment") +
+        tl::make_member<std::string, rdb::Item> (&rdb::Item::comment, &rdb::Item::set_comment, "comment#6") +
-        tl::make_member<std::string, rdb::Item> (&rdb::Item::image_str, &rdb::Item::set_image_str, "image") +
+        tl::make_member<std::string, rdb::Item> (&rdb::Item::image_str, &rdb::Item::set_image_str, "image#7") +
-        tl::make_element<rdb::Values, rdb::Item> (&rdb::Item::values, &rdb::Item::set_values, "values", 
+        tl::make_element<rdb::Values, rdb::Item> (&rdb::Item::values, &rdb::Item::set_values, "values#8",
-          tl::make_member<rdb::ValueWrapper, rdb::Values::const_iterator, rdb::Values> (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value", ValueConverter (rdb)) 
+          tl::make_member<rdb::ValueWrapper, rdb::Values::const_iterator, rdb::Values> (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value#1", ValueConverter ())        )
-        )
       )
     )
   );
 }
 
+static tl::XMLStruct <rdb::Database> s_rdb_struct = make_rdb_structure ();
+static tl::RegisteredClass<tl::XMLElementBase> rdb_struct_def (&s_rdb_struct, 0, "KLayout-RDB", false);
+
 // -------------------------------------------------------------
 //  Implementation of rdb::Database::save and write
 //  TODO: move this somewhere else - with generalized functionality
 
 void
-rdb::Database::save (const std::string &fn)
+rdb::Database::save (const std::string &fn, bool binary)
 {
-  write (fn);
+  write (fn, binary);
   set_filename (fn);
+  set_binary (binary);
 }
 
 void
-rdb::Database::write (const std::string &fn)
+rdb::Database::write (const std::string &fn, bool binary)
 {
   tl::OutputStream os (fn, tl::OutputStream::OM_Auto);
-  make_rdb_structure (this).write (os, *this);
 
-  if (tl::verbosity () >= 10) {
+  if (binary) {
-    tl::log << "Saved RDB to " << fn;
+
+    tl::ProtocolBufferWriter writer (os);
+    s_rdb_struct.write (writer, *this);
+
+    if (tl::verbosity () >= 10) {
+      tl::log << tl::to_string (tr ("Saved binary RDB to ")) << fn;
+    }
+
+  } else {
+
+    s_rdb_struct.write (os, *this);
+
+    if (tl::verbosity () >= 10) {
+      tl::log << tl::to_string (tr ("Saved RDB to ")) << fn;
+    }
+
   }
 }
 
@@ -146,8 +171,8 @@ class StandardReader
   : public ReaderBase
 {
 public:
-  StandardReader (tl::InputStream &stream)
+  StandardReader (tl::InputStream &stream, bool binary)
-    : m_input_stream (stream)
+    : m_input_stream (stream), m_binary (binary)
   {
     // .. nothing yet ..
   }
@@ -155,17 +180,30 @@ public:
   virtual void read (Database &db) 
   {
     tl::SelfTimer timer (tl::verbosity () >= 11, "Reading marker database file");
-    tl::XMLStreamSource in (m_input_stream, tl::to_string (tr ("Reading RDB")));
+
-    make_rdb_structure (&db).parse (in, db); 
+    if (m_binary) {
+
+      tl::ProtocolBufferReader reader (m_input_stream);
+      s_rdb_struct.parse (reader, db);
+      db.set_binary (true);
+
+    } else {
+
+      tl::XMLStreamSource in (m_input_stream, tl::to_string (tr ("Reading RDB")));
+      s_rdb_struct.parse (in, db);
+      db.set_binary (false);
+
+    }
   }
 
   virtual const char *format () const 
   {
-    return "KLayout-RDB";
+    return m_binary ? "KLayout-RDB-PB" : "KLayout-RDB";
   }
 
 private:
   tl::InputStream &m_input_stream;
+  bool m_binary;
 };
 
 class StandardFormatDeclaration 
@@ -193,10 +231,46 @@ class StandardFormatDeclaration
 
   virtual ReaderBase *create_reader (tl::InputStream &s) const 
   {
-    return new StandardReader (s);
+    return new StandardReader (s, false);
   }
 };
 
 static tl::RegisteredClass<rdb::FormatDeclaration> format_decl (new StandardFormatDeclaration (), 0, "KLayout-RDB");
 
+class BinaryFormatDeclaration
+  : public FormatDeclaration
+{
+  virtual std::string format_name () const { return "KLayout-RDB-PB"; }
+  virtual std::string format_desc () const { return "KLayout binary report database format"; }
+  virtual std::string file_format () const { return "KLayout binary RDB files (*.rdb *.rdb.gz)"; }
+
+  virtual bool detect (tl::InputStream &stream) const
+  {
+    static const char header[] = {
+      //  ProtocolBuffer wire format, LEN record with ID 0 and string "report-database".
+      0x02, 0x0f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2d, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65
+    };
+
+    const char *h = stream.get (sizeof (header));
+    if (! h) {
+      return false;
+    }
+
+    for (size_t i = 0; i < sizeof (header); ++i) {
+      if (h[i] != header[i]) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  virtual ReaderBase *create_reader (tl::InputStream &s) const
+  {
+    return new StandardReader (s, true);
+  }
+};
+
+static tl::RegisteredClass<rdb::FormatDeclaration> pb_format_decl (new BinaryFormatDeclaration (), 1, "KLayout-RDB-PB");
+
 }

src/rdb/unit_tests/rdbTests.cc

@@ -816,7 +816,18 @@ TEST(13_ApplyIgnoreUnknownTag)
   i2->values ().clear ();
   i2->add_value (std::string ("xyz"), vtag22);
 
-  //  values with incompatible tags are ignored -> tag2 is applied
+  //  values with incompatible tags are ignored, but vtag1 is a common tag.
+  //  So far, nothing is applied as the we match abc[vtag1] vs. xyz[vtag2] which is both different value and tag-
+  db1.apply (db2);
+
+  EXPECT_EQ (i1->tag_str (), "");
+
+  //  NOTE: don't do this at home
+  const_cast<rdb::Tags &> (db2.tags ()).remove_tag (vtag21);
+
+  //  vtag1 is no longer a common tag -> we match abc[vtag1] vs. xyz[vtag2] where vtag1 is not known on the
+  //  other side and vtag2 is not known on the self side.
+  //  Hence, the values are ignored and the tag is applied.
   db1.apply (db2);
 
   EXPECT_EQ (i1->tag_str (), "tag2");

src/tl/tl/tl.pro

@@ -29,6 +29,7 @@ SOURCES = \
     tlObject.cc \
     tlProgress.cc \
     tlPixelBuffer.cc \
+    tlProtocolBuffer.cc \
     tlResources.cc \
     tlScriptError.cc \
     tlSleep.cc \
@@ -43,6 +44,7 @@ SOURCES = \
     tlUnitTest.cc \
     tlInt128Support.cc \
     tlXMLParser.cc \
+    tlXMLReader.cc \
     tlXMLWriter.cc \
     tlThreadedWorkers.cc \
     tlThreads.cc \
@@ -86,6 +88,7 @@ HEADERS = \
     tlObjectCollection.h \
     tlProgress.h \
     tlPixelBuffer.h \
+    tlProtocolBuffer.h \
     tlResources.h \
     tlReuseVector.h \
     tlScriptError.h \
@@ -110,6 +113,7 @@ HEADERS = \
     tlInt128Support.h \
     tlDefs.h \
     tlXMLParser.h \
+    tlXMLReader.h \
     tlXMLWriter.h \
     tlThreadedWorkers.h \
     tlThreads.h \

src/tl/tl/tlProtocolBuffer.cc

@@ -0,0 +1,664 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2024 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#include "tlProtocolBuffer.h"
+#include "tlLog.h"
+
+namespace tl
+{
+
+// ----------------------------------------------------------------------------------
+
+ProtocolBufferReader::ProtocolBufferReader (tl::InputStream &input)
+  : mp_stream (&input), m_type (PBWireType (0)), m_pos (0), m_pos_before (0)
+{
+  //  .. nothing yet ..
+}
+
+int
+ProtocolBufferReader::read_tag ()
+{
+  m_type = PB_VARINT;
+  uint32_t value = 0;
+  read (value);
+  m_type = PBWireType (value & 7);
+  return int (value >> 3);
+}
+
+void
+ProtocolBufferReader::skip ()
+{
+  if (m_type == PB_VARINT) {
+
+    while ((*get (1) & 0x80) != 0) {
+      //  continue reading until the byte's MSB is 0
+    }
+
+  } else if (m_type == PB_I64) {
+
+    get (8);
+
+  } else if (m_type == PB_I32) {
+
+    get (4);
+
+  } else if (m_type == PB_LEN) {
+
+    size_t value = read_varint ();
+    skip_bytes (value);
+
+  }
+}
+
+void
+ProtocolBufferReader::read (double &d)
+{
+  uint64_t value = 0;
+  read (value);
+  d = *reinterpret_cast<const double *> (&value);
+}
+
+void
+ProtocolBufferReader::read (float &f)
+{
+  uint32_t value = 0;
+  read (value);
+  f = *reinterpret_cast<const float *> (&value);
+}
+
+void
+ProtocolBufferReader::read (std::string &s)
+{
+  if (m_type != PB_LEN) {
+    error (tl::to_string (tr ("Expected a LEN wire type for a string")));
+  }
+
+  size_t value = read_varint ();
+
+  s = std::string ();
+  s.reserve (value);
+
+  const size_t chunk_size = 1024;
+  while (value > 0) {
+    size_t l = std::min (chunk_size, value);
+    s += std::string (get (l), l);
+    value -= l;
+  }
+}
+
+void
+ProtocolBufferReader::read (uint32_t &ui32)
+{
+  if (m_type == PB_VARINT) {
+
+    pb_varint ui64 = read_varint ();
+    if (ui64 > std::numeric_limits<uint32_t>::max ()) {
+      error (tl::to_string (tr ("32 bit value overflow")));
+    }
+
+    ui32 = uint32_t (ui64);
+
+  } else if (m_type == PB_I32) {
+
+    ui32 = 0;
+    const char *cp = get (sizeof (ui32));
+    for (unsigned int i = 0; i < sizeof (ui32); ++i) {
+      ui32 <<= 8;
+      ui32 |= (unsigned char) cp [sizeof (ui32) - 1 - i];
+    }
+
+  } else {
+    error (tl::to_string (tr ("Expected a VARINT or I32 wire type")));
+  }
+}
+
+void
+ProtocolBufferReader::read (int32_t &i32)
+{
+  uint32_t ui32;
+  read (ui32);
+
+  if (m_type != PB_I32) {
+    if (ui32 & 1) {
+      i32 = -(int32_t (ui32 >> 1) + 1);
+    } else {
+      i32 = int32_t (ui32 >> 1);
+    }
+  } else {
+    i32 = int32_t (ui32);
+  }
+}
+
+void
+ProtocolBufferReader::read (uint64_t &ui64)
+{
+  if (m_type == PB_VARINT) {
+
+    ui64 = read_varint ();
+
+  } else if (m_type == PB_I64) {
+
+    ui64 = 0;
+    const char *cp = get (sizeof (ui64));
+    for (unsigned int i = 0; i < sizeof (ui64); ++i) {
+      ui64 <<= 8;
+      ui64 |= (unsigned char) cp [sizeof (ui64) - 1 - i];
+    }
+
+  } else {
+    error (tl::to_string (tr ("Expected a VARINT or I64 wire type")));
+  }
+}
+
+void
+ProtocolBufferReader::read (int64_t &i64)
+{
+  uint64_t ui64;
+  read (ui64);
+
+  if (m_type != PB_I64) {
+    if (ui64 & 1) {
+      i64 = -(int64_t (ui64 >> 1) + 1);
+    } else {
+      i64 = int64_t (ui64 >> 1);
+    }
+  } else {
+    i64 = int64_t (ui64);
+  }
+}
+
+void
+ProtocolBufferReader::read (bool &b)
+{
+  uint32_t ui32;
+  read (ui32);
+  b = (ui32 != 0);
+}
+
+void
+ProtocolBufferReader::open ()
+{
+  if (m_type != PB_LEN) {
+    error (tl::to_string (tr ("Expected a LEN wire type for a submessage")));
+  }
+
+  size_t value = read_varint ();
+  if (! m_seq_counts.empty ()) {
+    //  take out the following bytes from the current sequence
+    m_seq_counts.back () -= value;
+  }
+  m_seq_counts.push_back (value);
+}
+
+void
+ProtocolBufferReader::close ()
+{
+  if (! m_seq_counts.empty ()) {
+    skip_bytes (m_seq_counts.back ());
+    m_seq_counts.pop_back ();
+  }
+}
+
+bool
+ProtocolBufferReader::at_end () const
+{
+  if (m_seq_counts.empty ()) {
+    const char *cp = mp_stream->get (1);
+    if (cp) {
+      mp_stream->unget (1);
+      return false;
+    } else {
+      return true;
+    }
+  } else {
+    return m_seq_counts.back () == 0;
+  }
+}
+
+const char *
+ProtocolBufferReader::get (size_t n)
+{
+  m_pos_before = m_pos;
+  m_pos += n;
+  if (! m_seq_counts.empty ()) {
+    if (m_seq_counts.back () < n) {
+      error (tl::to_string (tr ("sequence underflow")));
+    }
+    m_seq_counts.back () -= n;
+  }
+
+  const char *cp = mp_stream->get (n);
+  if (! cp) {
+    error (tl::to_string (tr ("unexpected end of file")));
+  }
+
+  return cp;
+}
+
+pb_varint
+ProtocolBufferReader::read_varint ()
+{
+  pb_varint v = 0;
+  unsigned int s = 0;
+
+  while (true) {
+    const char *cp = get (1);
+    if (! cp) {
+      error (tl::to_string (tr ("unexpected end of file")));
+    }
+    if (s + 7 > 64 && pb_varint ((unsigned char) (*cp & 0x7f)) >> (s + 7 - 64) != 0) {
+      error (tl::to_string (tr ("64 bit integer overflow")));
+    }
+    v |= (pb_varint ((unsigned char) (*cp & 0x7f)) << s);
+    s += 7;
+    if ((*cp & 0x80) == 0) {
+      break;
+    }
+  }
+
+  return v;
+}
+
+void
+ProtocolBufferReader::skip_bytes (size_t n)
+{
+  m_pos_before = m_pos;
+  m_pos += n;
+  if (! m_seq_counts.empty ()) {
+    if (m_seq_counts.back () < n) {
+      error (tl::to_string (tr ("sequence underflow")));
+    }
+    m_seq_counts.back () -= n;
+  }
+
+  const size_t chunk_size = 1024;
+  while (n > 0) {
+    size_t l = std::min (chunk_size, n);
+    if (! mp_stream->get (l)) {
+      error (tl::to_string (tr ("unexpected end of file")));
+    }
+    n -= l;
+  }
+}
+
+void
+ProtocolBufferReader::error (const std::string &msg)
+{
+  throw ProtocolBufferReaderError (msg + tl::to_string (tr (", in: ")) + mp_stream->source (), m_pos_before);
+}
+
+// ----------------------------------------------------------------------------------
+//  Writer utilities
+
+inline uint32_t zigzag_encode (int32_t v)
+{
+  if (v < 0) {
+    return ((uint32_t (-v) - 1) << 1) + 1;
+  } else {
+    return uint32_t (v) << 1;
+  }
+}
+
+inline uint64_t zigzag_encode (int64_t v)
+{
+  if (v < 0) {
+    return ((uint64_t (-v) - 1) << 1) + 1;
+  } else {
+    return uint64_t (v) << 1;
+  }
+}
+
+inline void little_endian_encode (char *b, uint32_t v)
+{
+  for (unsigned int i = 0; i < sizeof (v); ++i) {
+    b[i] = (char) v;
+    v >>= 8;
+  }
+}
+
+inline void little_endian_encode (char *b, uint64_t v)
+{
+  for (unsigned int i = 0; i < sizeof (v); ++i) {
+    b[i] = (char) v;
+    v >>= 8;
+  }
+}
+
+inline size_t count_varint_bytes (pb_varint v)
+{
+  size_t n = 0;
+  while (true) {
+    ++n;
+    if (v < 0x80) {
+      break;
+    }
+    v >>= 7;
+  }
+  return n;
+}
+
+const size_t max_varint_bytes = (sizeof (pb_varint) * 8) / 7 + 1;
+
+inline size_t varint_encode (char *b, pb_varint v)
+{
+  char *cp = b;
+  while (true) {
+    if (v < 0x80) {
+      *cp++ = char (v);
+      break;
+    } else {
+      *cp++ = (char (v) | 0x80);
+    }
+    v >>= 7;
+  }
+  return cp - b;
+}
+
+// ----------------------------------------------------------------------------------
+
+ProtocolBufferWriterBase::ProtocolBufferWriterBase ()
+  : m_bytes_counted (0)
+{
+  //  .. nothing yet ..
+}
+
+ProtocolBufferWriterBase::~ProtocolBufferWriterBase ()
+{
+  //  .. nothing yet ..
+}
+
+void ProtocolBufferWriterBase::write (int tag, float v)
+{
+  write (tag, *reinterpret_cast<uint32_t *> (&v), true);
+}
+
+void ProtocolBufferWriterBase::write (int tag, double v)
+{
+  write (tag, *reinterpret_cast<uint64_t *> (&v), true);
+}
+
+void ProtocolBufferWriterBase::write (int tag, int32_t v, bool fixed)
+{
+  if (fixed) {
+    write (tag, uint32_t (v), true);
+  } else {
+    write (tag, zigzag_encode (v), false);
+  }
+}
+
+void ProtocolBufferWriterBase::write (int tag, int64_t v, bool fixed)
+{
+  if (fixed) {
+    write (tag, uint64_t (v), true);
+  } else {
+    write (tag, zigzag_encode (v), false);
+  }
+}
+
+void ProtocolBufferWriterBase::write (int tag, bool b)
+{
+  write (tag, uint32_t (b ? 1 : 0));
+}
+
+void ProtocolBufferWriterBase::write (int tag, uint32_t v, bool fixed)
+{
+  if (fixed) {
+    write_varint (pb_varint ((tag << 3) + PB_I32), true);
+    write_fixed (v);
+  } else {
+    write_varint (pb_varint ((tag << 3) + PB_VARINT), true);
+    write_varint (pb_varint (v), false);
+  }
+}
+
+void ProtocolBufferWriterBase::write (int tag, uint64_t v, bool fixed)
+{
+  if (fixed) {
+    write_varint (pb_varint ((tag << 3) + PB_I64), true);
+    write_fixed (v);
+  } else {
+    write_varint (pb_varint ((tag << 3) + PB_VARINT), true);
+    write_varint (pb_varint (v), false);
+  }
+}
+
+void ProtocolBufferWriterBase::write (int tag, const std::string &s)
+{
+  write_varint (pb_varint ((tag << 3) + PB_LEN), true);
+  write_varint (s.size (), false);
+  write_bytes (s);
+}
+
+bool ProtocolBufferWriterBase::is_counting () const
+{
+  return ! m_byte_counter_stack.empty ();
+}
+
+void ProtocolBufferWriterBase::begin_seq (int tag, bool counting)
+{
+  if (counting) {
+
+    if (is_counting ()) {
+      write_varint (pb_varint ((tag << 3) + PB_LEN), true);
+    }
+
+    m_byte_counter_stack.push_back (0);
+
+  } else {
+
+    write_varint (pb_varint ((tag << 3) + PB_LEN), true);
+    write_varint (m_bytes_counted, false);
+
+  }
+}
+
+void ProtocolBufferWriterBase::end_seq ()
+{
+  if (is_counting ()) {
+
+    m_bytes_counted = m_byte_counter_stack.back ();
+    m_byte_counter_stack.pop_back ();
+
+    //  just for adding the required bytes
+    if (is_counting ()) {
+      m_byte_counter_stack.back () += m_bytes_counted;
+      write_varint (m_bytes_counted, false);
+    }
+
+  }
+}
+
+void ProtocolBufferWriterBase::add_bytes (size_t n)
+{
+  tl_assert (! m_byte_counter_stack.empty ());
+  m_byte_counter_stack.back () += n;
+}
+
+// ----------------------------------------------------------------------------------
+
+ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream)
+  : mp_stream (&stream)
+{
+  //  .. nothing yet ..
+}
+
+void ProtocolBufferWriter::write_fixed (uint32_t v)
+{
+  if (is_counting ()) {
+    add_bytes (sizeof (v));
+  } else {
+    char b[sizeof (v)];
+    little_endian_encode (b, v);
+    mp_stream->put (b, sizeof (v));
+  }
+}
+
+void ProtocolBufferWriter::write_fixed (uint64_t v)
+{
+  if (is_counting ()) {
+    add_bytes (sizeof (v));
+  } else {
+    char b[sizeof (v)];
+    little_endian_encode (b, v);
+    mp_stream->put (b, sizeof (v));
+  }
+}
+
+void ProtocolBufferWriter::write_bytes (const std::string &s)
+{
+  if (is_counting ()) {
+    add_bytes (s.size ());
+  } else {
+    mp_stream->put (s.c_str (), s.size ());
+  }
+}
+
+void ProtocolBufferWriter::write_varint (pb_varint v, bool /*id*/)
+{
+  if (is_counting ()) {
+    add_bytes (count_varint_bytes (v));
+  } else {
+    char b [max_varint_bytes];
+    size_t n = varint_encode (b, v);
+    mp_stream->put (b, n);
+  }
+}
+
+// ----------------------------------------------------------------------------------
+
+ProtocolBufferDumper::ProtocolBufferDumper ()
+  : m_debug_pos (0)
+{
+  //  .. nothing yet ..
+}
+
+void ProtocolBufferDumper::write_fixed (uint32_t v)
+{
+  if (is_counting ()) {
+    add_bytes (sizeof (v));
+  } else {
+    char b[sizeof (v)];
+    little_endian_encode (b, v);
+    dump (b, sizeof (v), "I32", tl::to_string (v));
+  }
+}
+
+void ProtocolBufferDumper::write_fixed (uint64_t v)
+{
+  if (is_counting ()) {
+    add_bytes (sizeof (v));
+  } else {
+    char b[sizeof (v)];
+    little_endian_encode (b, v);
+    dump (b, sizeof (v), "I64", tl::to_string (v));
+  }
+}
+
+void ProtocolBufferDumper::write_bytes (const std::string &s)
+{
+  if (is_counting ()) {
+    add_bytes (s.size ());
+  } else {
+    dump (s.c_str (), s.size (), "(string)", s);
+  }
+}
+
+void
+ProtocolBufferDumper::write_varint (pb_varint v, bool id)
+{
+  if (is_counting ()) {
+
+    add_bytes (count_varint_bytes (v));
+
+  } else {
+
+    char b[max_varint_bytes];
+    size_t n = varint_encode (b, v);
+
+    if (id) {
+      tl::PBWireType wt = tl::PBWireType (v & 7);
+      std::string wire_type;
+      if (wt == PB_EGROUP) {
+        wire_type = "EGROUP";
+      } else if (wt == PB_SGROUP) {
+        wire_type = "SGROUP";
+      } else if (wt == PB_VARINT) {
+        wire_type = "VARINT";
+      } else if (wt == PB_I32) {
+        wire_type = "I32";
+      } else if (wt == PB_I64) {
+        wire_type = "I64";
+      } else if (wt == PB_LEN) {
+        wire_type = "LEN";
+      }
+      dump (b, n, "(id)", "#" + tl::to_string (v >> 3) + " " + wire_type);
+    } else {
+      dump (b, n, "VARINT", tl::to_string (v));
+    }
+
+  }
+}
+
+void
+ProtocolBufferDumper::dump (const char *cp, size_t n, const std::string &type, const std::string &value)
+{
+  bool first = true;
+  size_t nn = n;
+
+  while (n > 0) {
+
+    std::string line;
+    if (first) {
+      line += tl::sprintf ("%08ld", m_debug_pos);
+    } else {
+      line += "      ";
+    }
+    line += "  ";
+
+    for (unsigned int i = 0; i < 8; ++i) {
+      if (n > 0) {
+        line += tl::sprintf ("%02x", (unsigned int) ((unsigned char) *cp));
+        ++cp;
+        --n;
+      } else {
+        line += "  ";
+      }
+      line += " ";
+    }
+
+    if (first) {
+      line += " ";
+      line += type;
+      line += " ";
+      line += value;
+    }
+
+    tl::info << line;
+
+    first = false;
+
+  }
+
+  m_debug_pos += nn;
+}
+
+}

src/tl/tl/tlProtocolBuffer.h

@@ -0,0 +1,433 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2024 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#ifndef HDR_tlProtocolBuffer
+#define HDR_tlProtocolBuffer
+
+#include "tlCommon.h"
+#include "tlStream.h"
+
+namespace tl
+{
+
+//  Representation of VARINT values
+typedef uint64_t pb_varint;
+
+enum PBWireType
+{
+  PB_VARINT = 0,
+  PB_I64 = 1,
+  PB_LEN = 2,
+  PB_SGROUP = 3,
+  PB_EGROUP = 4,
+  PB_I32 = 5
+};
+
+/**
+ *  @brief An exception thrown by the ProtocolBuffer reader on a read error
+ */
+class TL_PUBLIC ProtocolBufferReaderError
+  : public tl::Exception
+{
+public:
+  ProtocolBufferReaderError (const std::string &msg, size_t position)
+    : tl::Exception (msg), m_position (position)
+  { }
+
+  virtual std::string msg () const
+  {
+    return basic_msg () + tl::to_string (tr (", at position ")) + tl::to_string (m_position);
+  }
+
+private:
+  size_t m_position;
+};
+
+/**
+ *  @brief A reader for ProtocolBuffer files and streams
+ *
+ *  This is a low-level decoder for ProtocolBuffer files.
+ *
+ *  Use "read_tag" to read a new tag. Unknown tags must be skipped.
+ *  Use "skip" to skip an entry.
+ *
+ *
+ */
+class TL_PUBLIC ProtocolBufferReaderBase
+{
+public:
+  /**
+   *  @brief Constructor
+   */
+  ProtocolBufferReaderBase ()
+  {
+    //  .. nothing yet ..
+  }
+
+  /**
+   *  @brief Destructor
+   */
+  virtual ~ProtocolBufferReaderBase ()
+  {
+    //  .. nothing yet ..
+  }
+
+  /**
+   *  @brief Reads a new tag
+   *  This method will also set the current write type.
+   *  @returns The message ID
+   */
+  virtual int read_tag () = 0;
+
+  /**
+   *  @brief Gets the current wire type
+   */
+  virtual PBWireType type () const = 0;
+
+  /**
+   *  @brief Skips the current tag
+   */
+  virtual void skip () = 0;
+
+  /**
+   *  @brief Reads a floating-point value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a double value.
+   */
+  virtual void read (double &d) = 0;
+
+  /**
+   *  @brief Reads a floating-point value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a float value.
+   */
+  virtual void read (float &f) = 0;
+
+  /**
+   *  @brief Reads a string from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a string.
+   */
+  virtual void read (std::string &s) = 0;
+
+  /**
+   *  @brief Reads a uint32_t value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a uint32_t.
+   */
+  virtual void read (uint32_t &ui32) = 0;
+
+  /**
+   *  @brief Reads a int32_t value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a int32_t.
+   */
+  virtual void read (int32_t &i32) = 0;
+
+  /**
+   *  @brief Reads a uint64_t value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a uint64_t.
+   */
+  virtual void read (uint64_t &ui64) = 0;
+
+  /**
+   *  @brief Reads a int64_t value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a int64_t.
+   */
+  virtual void read (int64_t &i64) = 0;
+
+  /**
+   *  @brief Reads a boolean value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a bool.
+   */
+  virtual void read (bool &b) = 0;
+
+  /**
+   *  @brief Opens a LEN sequence
+   *  After calling "open", the parser will continue reading messages, but
+   *  "at_end" will report true on the end of the sequence, not at the end of the
+   *  file.
+   *  This method will throw an exception if not in a message of LEN type.
+   */
+  virtual void open () = 0;
+
+  /**
+   *  @brief Closes the LEN sequence
+   *  This method will jump to the end of the sequence and continue reading
+   *  messages from the previous block or file.
+   */
+  virtual void close () = 0;
+
+  /**
+   *  @brief Returns true if at the end of the file or end of a block
+   */
+  virtual bool at_end () const = 0;
+
+  /**
+   *  @brief Emits an error at the current position
+   */
+  virtual void error (const std::string &msg) = 0;
+};
+
+/**
+ *  @brief A reader for ProtocolBuffer files and streams
+ *
+ *  This is the reader implementation for binary files
+ *  as described here:
+ *  https://protobuf.dev/programming-guides/encoding/
+ */
+class TL_PUBLIC ProtocolBufferReader
+  : public ProtocolBufferReaderBase
+{
+public:
+  /**
+   *  @brief Creates a ProtocolBuffer parser from the given stream
+   */
+  ProtocolBufferReader (tl::InputStream &input);
+
+  /**
+   *  @brief Reads a new tag
+   *  This method will also set the current write type.
+   *  @returns The message ID
+   */
+  int read_tag ();
+
+  /**
+   *  @brief Gets the current wire type
+   */
+  PBWireType type () const
+  {
+    return m_type;
+  }
+
+  /**
+   *  @brief Skips the current tag
+   */
+  void skip ();
+
+  /**
+   *  @brief Reads a floating-point value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a double value.
+   */
+  void read (double &d);
+
+  /**
+   *  @brief Reads a floating-point value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a float value.
+   */
+  void read (float &f);
+
+  /**
+   *  @brief Reads a string from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a string.
+   */
+  void read (std::string &s);
+
+  /**
+   *  @brief Reads a uint32_t value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a uint32_t.
+   */
+  void read (uint32_t &ui32);
+
+  /**
+   *  @brief Reads a int32_t value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a int32_t.
+   */
+  void read (int32_t &i32);
+
+  /**
+   *  @brief Reads a uint64_t value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a uint64_t.
+   */
+  void read (uint64_t &ui64);
+
+  /**
+   *  @brief Reads a int64_t value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a int64_t.
+   */
+  void read (int64_t &i64);
+
+  /**
+   *  @brief Reads a boolean value from the current message
+   *  Throws a reader error if the current tag's value is not compatible with a bool.
+   */
+  void read (bool &b);
+
+  /**
+   *  @brief Opens a LEN sequence
+   *  After calling "open", the parser will continue reading messages, but
+   *  "at_end" will report true on the end of the sequence, not at the end of the
+   *  file.
+   *  This method will throw an exception if not in a message of LEN type.
+   */
+  void open ();
+
+  /**
+   *  @brief Closes the LEN sequence
+   *  This method will jump to the end of the sequence and continue reading
+   *  messages from the previous block or file.
+   */
+  void close ();
+
+  /**
+   *  @brief Returns true if at the end of the file or end of a block
+   */
+  bool at_end () const;
+
+  /**
+   *  @brief Emits an error at the current position
+   */
+  void error (const std::string &msg);
+
+private:
+  tl::InputStream *mp_stream;
+  PBWireType m_type;
+  size_t m_pos;
+  size_t m_pos_before;
+  std::vector<size_t> m_seq_counts;
+
+  pb_varint read_varint ();
+  void skip_bytes (size_t n);
+  const char *get (size_t n);
+};
+
+/**
+ *  @brief The Protocol buffer writer
+ *
+ *  Scalar types are easy to write: just use the "write" methods.
+ *  This includes strings.
+ *
+ *  Submessages and packed sequences are special as byte counting is
+ *  required. The writer uses a two-pass approach. This means:
+ *
+ *  1. On writing a submessage or packed repetition, call "begin_seq"
+ *     with the message tag and "counting" set to true.
+ *  2. Write the elements using "write" or the submessage writing
+ *     scheme recursively.
+ *  3. At the end of the sequence, call "end_seq".
+ *  4. if "is_counting()" is false, repeat steps 1 to 3 with
+ *     "counting" set to false on "begin_seq".
+ */
+class TL_PUBLIC ProtocolBufferWriterBase
+{
+public:
+  /**
+   *  @brief Constructor
+   */
+  ProtocolBufferWriterBase ();
+
+  /**
+   *  @brief Destructor
+   */
+  virtual ~ProtocolBufferWriterBase ();
+
+  /**
+   *  @brief Writes a scalar element with the given value and tag
+   */
+
+  //  implicit types
+  void write (int tag, float v);
+  void write (int tag, double v);
+  void write (int tag, int32_t v, bool fixed = false);
+  void write (int tag, int64_t v, bool fixed = false);
+  void write (int tag, uint32_t v, bool fixed = false);
+  void write (int tag, uint64_t v, bool fixed = false);
+  void write (int tag, bool b);
+  void write (int tag, const std::string &s);
+
+  /**
+   *  @brief Returns true if the writer is in counting mode
+   */
+  bool is_counting () const;
+
+  /**
+   *  @brief Initiates a new sequence. See class documentation for details.
+   */
+  void begin_seq (int tag, bool counting);
+
+  /**
+   *  @brief Ends a sequence. See class documentation for details.
+   */
+  void end_seq ();
+
+protected:
+  virtual void write_bytes (const std::string &s) = 0;
+  virtual void write_fixed (uint32_t v) = 0;
+  virtual void write_fixed (uint64_t v) = 0;
+  virtual void write_varint (pb_varint v, bool id) = 0;
+  void add_bytes (size_t n);
+
+private:
+  size_t m_bytes_counted;
+  std::vector<size_t> m_byte_counter_stack;
+};
+
+/**
+ *  @brief A writer for ProtocolBuffer files and streams
+ *
+ *  This is the writer implementation for binary files
+ *  as described here:
+ *  https://protobuf.dev/programming-guides/encoding/
+ */
+class TL_PUBLIC ProtocolBufferWriter
+  : public ProtocolBufferWriterBase
+{
+public:
+  /**
+   *  @brief Creates a writer for the given stream
+   */
+  ProtocolBufferWriter (tl::OutputStream &stream);
+
+protected:
+  virtual void write_bytes (const std::string &s);
+  virtual void write_fixed (uint32_t v);
+  virtual void write_fixed (uint64_t v);
+  virtual void write_varint (pb_varint v, bool id = false);
+
+private:
+  tl::OutputStream *mp_stream;
+};
+
+/**
+ *  @brief A writer implementation that dumps the file content to tl::info
+ *
+ *  This implementation does a halfway job of producing binary files,
+ *  but only insofar it is needed for dumping the binary data.
+ */
+class TL_PUBLIC ProtocolBufferDumper
+  : public ProtocolBufferWriterBase
+{
+public:
+  /**
+   *  @brief Creates the writer
+   */
+  ProtocolBufferDumper ();
+
+protected:
+  virtual void write_bytes (const std::string &s);
+  virtual void write_fixed (uint32_t v);
+  virtual void write_fixed (uint64_t v);
+  virtual void write_varint (pb_varint v, bool id = false);
+  void dump (const char *cp, size_t n, const std::string &type, const std::string &value);
+
+private:
+  size_t m_debug_pos;
+};
+
+}
+
+#endif

src/tl/tl/tlXMLReader.cc

@@ -0,0 +1,913 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2024 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+
+#include "tlXMLReader.h"
+#include "tlXMLParser.h"
+#include "tlString.h"
+#include "tlLog.h"
+#include "tlAssert.h"
+#include "tlProgress.h"
+
+#include <cstring>
+#include <memory>
+
+#if defined(HAVE_EXPAT)
+
+#include <expat.h>
+
+namespace tl
+{
+
+// --------------------------------------------------------------------
+//  SourcePrivateData implementation
+
+class XMLSourcePrivateData
+{
+public:
+  XMLSourcePrivateData (tl::InputStream *stream)
+    : mp_stream_holder (stream),
+      m_has_error (false)
+  {
+    mp_stream = stream;
+  }
+
+  XMLSourcePrivateData (tl::InputStream *stream, const std::string &progress_message)
+    : mp_stream_holder (stream),
+      mp_progress (new AbsoluteProgress (progress_message, 100)),
+      m_has_error (false)
+  {
+    mp_stream = stream;
+    mp_progress->set_format (tl::to_string (tr ("%.0f MB")));
+    mp_progress->set_unit (1024 * 1024);
+  }
+
+  XMLSourcePrivateData (tl::InputStream &stream)
+    : m_has_error (false)
+  {
+    mp_stream = &stream;
+  }
+
+  XMLSourcePrivateData (tl::InputStream &stream, const std::string &progress_message)
+    : mp_progress (new AbsoluteProgress (progress_message, 100)),
+      m_has_error (false)
+  {
+    mp_stream = &stream;
+    mp_progress->set_format (tl::to_string (tr ("%.0f MB")));
+    mp_progress->set_unit (1024 * 1024);
+  }
+
+  int read (char *data, size_t n)
+  {
+    try {
+
+      if (mp_progress.get ()) {
+        mp_progress->set (mp_stream->pos ());
+      }
+
+      size_t n0 = n;
+      for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; --n) {
+        *data++ = *rd;
+      }
+
+      return int (n0 - n);
+
+    } catch (tl::Exception &ex) {
+      m_error = ex.msg ();
+      m_has_error = true;
+      return -1;
+    }
+  }
+
+  bool has_error () const
+  {
+    return m_has_error;
+  }
+
+  const std::string &error_msg () const
+  {
+    return m_error;
+  }
+
+  void reset ()
+  {
+    mp_stream->reset ();
+  }
+
+private:
+  std::unique_ptr<tl::InputStream> mp_stream_holder;
+  tl::InputStream *mp_stream;
+  std::unique_ptr<tl::AbsoluteProgress> mp_progress;
+  bool m_has_error;
+  std::string m_error;
+};
+
+// --------------------------------------------------------------------
+//  XMLSource implementation
+
+XMLSource::XMLSource ()
+  : mp_source (0)
+{
+  //  .. nothing yet ..
+}
+
+XMLSource::~XMLSource ()
+{
+  delete mp_source;
+  mp_source = 0;
+}
+
+void
+XMLSource::reset ()
+{
+  mp_source->reset ();
+}
+
+// --------------------------------------------------------------------
+//  XMLStringSource implementation
+
+XMLStringSource::XMLStringSource (const std::string &string)
+  : m_copy (string)
+{
+  set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (m_copy.c_str (), string.size ()))));
+}
+
+XMLStringSource::XMLStringSource (const char *cp)
+{
+  set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, strlen (cp)))));
+}
+
+XMLStringSource::XMLStringSource (const char *cp, size_t len)
+{
+  set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, len))));
+}
+
+XMLStringSource::~XMLStringSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLFileSource implementation
+
+XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message)
+{
+  set_source (new XMLSourcePrivateData (new tl::InputStream (path), progress_message));
+}
+
+XMLFileSource::XMLFileSource (const std::string &path)
+{
+  set_source (new XMLSourcePrivateData (new tl::InputStream (path)));
+}
+
+XMLFileSource::~XMLFileSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLStreamSource implementation
+
+XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message)
+{
+  set_source (new XMLSourcePrivateData (s, progress_message));
+}
+
+XMLStreamSource::XMLStreamSource (tl::InputStream &s)
+{
+  set_source (new XMLSourcePrivateData (s));
+}
+
+XMLStreamSource::~XMLStreamSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLParser implementation
+
+void XMLCALL start_element_handler (void *user_data, const XML_Char *name, const XML_Char **atts);
+void XMLCALL end_element_handler (void *user_data, const XML_Char *name);
+void XMLCALL cdata_handler (void *user_data, const XML_Char *s, int len);
+
+static std::string get_lname (const std::string &name)
+{
+  size_t colon = name.find (':');
+  if (colon != std::string::npos) {
+    return std::string (name, colon + 1, name.size () - colon - 1);
+  } else {
+    return name;
+  }
+}
+
+class XMLParserPrivateData
+{
+public:
+  XMLParserPrivateData ()
+    : mp_struct_handler (0)
+  {
+    mp_parser = XML_ParserCreate ("UTF-8");
+    tl_assert (mp_parser != NULL);
+  }
+
+  ~XMLParserPrivateData ()
+  {
+    if (mp_parser != NULL) {
+      XML_ParserFree (mp_parser);
+    }
+  }
+
+  void start_element (const std::string &name)
+  {
+    try {
+      //  TODO: Provide namespace URI?
+      mp_struct_handler->start_element (std::string (), get_lname (name), name);
+    } catch (tl::Exception &ex) {
+      error (ex);
+    }
+  }
+
+  void end_element (const std::string &name)
+  {
+    try {
+      //  TODO: Provide namespace URI?
+      mp_struct_handler->end_element (std::string (), get_lname (name), name);
+    } catch (tl::Exception &ex) {
+      error (ex);
+    }
+  }
+
+  void cdata (const std::string &cdata)
+  {
+    try {
+      mp_struct_handler->characters (cdata);
+    } catch (tl::Exception &ex) {
+      error (ex);
+    }
+  }
+
+  void parse (tl::XMLSource &source, XMLStructureHandler &struct_handler)
+  {
+    m_has_error = false;
+    mp_struct_handler = &struct_handler;
+
+    //  Just in case we want to reuse it ...
+    XML_ParserReset (mp_parser, NULL);
+    XML_SetUserData (mp_parser, (void *) this);
+    XML_SetElementHandler (mp_parser, start_element_handler, end_element_handler);
+    XML_SetCharacterDataHandler (mp_parser, cdata_handler);
+
+    const int chunk = 65536;
+    char buffer [chunk];
+
+    int n;
+
+    do {
+
+      n = source.source ()->read (buffer, chunk);
+      if (n < 0) {
+        break;
+      }
+
+      XML_Status status = XML_Parse (mp_parser, buffer, n, n < chunk /*is final*/);
+      if (status == XML_STATUS_ERROR) {
+        m_has_error = true;
+        m_error = XML_ErrorString (XML_GetErrorCode (mp_parser));
+        m_error_line = XML_GetErrorLineNumber (mp_parser);
+        m_error_column = XML_GetErrorColumnNumber (mp_parser);
+      }
+
+    } while (n == chunk && !m_has_error);
+  }
+
+  void check_error ()
+  {
+    if (m_has_error) {
+      throw tl::XMLLocatedException (m_error, m_error_line, m_error_column);
+    }
+  }
+
+private:
+  void error (tl::Exception &ex)
+  {
+    m_has_error = true;
+    m_error_line = XML_GetCurrentLineNumber (mp_parser);
+    m_error_column = XML_GetCurrentColumnNumber (mp_parser);
+    m_error = ex.msg ();
+  }
+
+  XML_Parser mp_parser;
+  XMLStructureHandler *mp_struct_handler;
+  bool m_has_error;
+  std::string m_error;
+  int m_error_line, m_error_column;
+};
+
+void start_element_handler (void *user_data, const XML_Char *name, const XML_Char ** /*atts*/)
+{
+  XMLParserPrivateData *d = reinterpret_cast<XMLParserPrivateData *> (user_data);
+  d->start_element (std::string (name));
+}
+
+void end_element_handler (void *user_data, const XML_Char *name)
+{
+  XMLParserPrivateData *d = reinterpret_cast<XMLParserPrivateData *> (user_data);
+  d->end_element (std::string (name));
+}
+
+void cdata_handler (void *user_data, const XML_Char *s, int len)
+{
+  XMLParserPrivateData *d = reinterpret_cast<XMLParserPrivateData *> (user_data);
+  d->cdata (std::string (s, size_t (len)));
+}
+
+
+XMLParser::XMLParser ()
+  : mp_data (new XMLParserPrivateData ())
+{
+  //  .. nothing yet ..
+}
+
+XMLParser::~XMLParser ()
+{
+  delete mp_data;
+  mp_data = 0;
+}
+
+void
+XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler)
+{
+  mp_data->parse (source, struct_handler);
+
+  //  throws an exception if there is an error
+  mp_data->check_error ();
+}
+
+bool
+XMLParser::is_available ()
+{
+  return true;
+}
+
+}
+
+#elif defined(HAVE_QT)
+
+#include <QFile>
+#include <QIODevice>
+#include <QXmlContentHandler>
+
+namespace tl
+{
+
+// --------------------------------------------------------------------
+//  A SAX handler for the Qt implementation
+
+class SAXHandler
+  : public QXmlDefaultHandler
+{
+public:
+  SAXHandler (XMLStructureHandler *sh);
+
+  virtual bool characters (const QString &ch);
+  virtual bool endElement (const QString &namespaceURI, const QString &localName, const QString &qName);
+  virtual bool startElement (const QString &namespaceURI, const QString &localName, const QString &qName, const QXmlAttributes &atts);
+  virtual bool error (const QXmlParseException &exception);
+  virtual bool fatalError (const QXmlParseException &exception);
+  virtual bool warning (const QXmlParseException &exception);
+  virtual QString errorString () const;
+
+  void setDocumentLocator (QXmlLocator *locator);
+
+  const tl::XMLLocatedException *exception () const
+  {
+    return m_error.get ();
+  }
+
+private:
+  QXmlLocator *mp_locator;
+  XMLStructureHandler *mp_struct_handler;
+  std::unique_ptr<tl::XMLLocatedException> m_error;
+  std::string m_error_string;
+};
+
+// --------------------------------------------------------------------------------------------------------
+//  trureHandler implementation
+
+SAXHandler::SAXHandler (XMLStructureHandler *sh)
+  : QXmlDefaultHandler (), mp_locator (0), mp_struct_handler (sh)
+{
+  // .. nothing yet ..
+}
+
+void
+SAXHandler::setDocumentLocator (QXmlLocator *locator)
+{
+  mp_locator = locator;
+}
+
+bool
+SAXHandler::startElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname, const QXmlAttributes & /*atts*/)
+{
+  std::string uri (tl::to_string (qs_uri));
+  std::string lname (tl::to_string (qs_lname));
+  std::string qname (tl::to_string (qs_qname));
+
+  try {
+    mp_struct_handler->start_element (uri, lname, qname);
+  } catch (tl::XMLException &ex) {
+    m_error_string = ex.raw_msg ();
+    return false;
+  } catch (tl::Exception &ex) {
+    m_error_string = ex.msg ();
+    return false;
+  }
+
+  //  successful
+  return true;
+}
+
+bool
+SAXHandler::endElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname)
+{
+  std::string uri (tl::to_string (qs_uri));
+  std::string lname (tl::to_string (qs_lname));
+  std::string qname (tl::to_string (qs_qname));
+
+  try {
+    mp_struct_handler->end_element (uri, lname, qname);
+  } catch (tl::XMLException &ex) {
+    m_error_string = ex.raw_msg ();
+    return false;
+  } catch (tl::Exception &ex) {
+    m_error_string = ex.msg ();
+    return false;
+  }
+
+  //  successful
+  return true;
+}
+
+bool
+SAXHandler::characters (const QString &t)
+{
+  try {
+    mp_struct_handler->characters (tl::to_string (t));
+  } catch (tl::XMLException &ex) {
+    m_error_string = ex.raw_msg ();
+    return false;
+  } catch (tl::Exception &ex) {
+    m_error_string = ex.msg ();
+    return false;
+  }
+
+  //  successful
+  return true;
+}
+
+QString
+SAXHandler::errorString () const
+{
+  return tl::to_qstring (m_error_string);
+}
+
+
+bool
+SAXHandler::error (const QXmlParseException &ex)
+{
+  m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ()));
+  //  stop reading
+  return false;
+}
+
+bool
+SAXHandler::fatalError (const QXmlParseException &ex)
+{
+  m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ()));
+  //  stop reading
+  return false;
+}
+
+bool
+SAXHandler::warning (const QXmlParseException &ex)
+{
+  tl::XMLLocatedException lex (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ());
+  tl::warn << lex.msg ();
+  //  continue
+  return true;
+}
+
+// --------------------------------------------------------------------
+//  SourcePrivateData implementation
+
+class XMLSourcePrivateData
+  : public QXmlInputSource
+{
+public:
+  XMLSourcePrivateData ()
+    : QXmlInputSource ()
+  {
+    //  .. nothing yet ..
+  }
+
+  XMLSourcePrivateData (QIODevice *dev)
+    : QXmlInputSource (dev)
+  {
+    //  .. nothing yet ..
+  }
+};
+
+// --------------------------------------------------------------------
+//  XMLSource implementation
+
+XMLSource::XMLSource ()
+  : mp_source (0)
+{
+  //  .. nothing yet ..
+}
+
+XMLSource::~XMLSource ()
+{
+  delete mp_source;
+  mp_source = 0;
+}
+
+void
+XMLSource::reset ()
+{
+  mp_source->reset ();
+}
+
+// --------------------------------------------------------------------
+//  XMLStringSource implementation
+
+XMLStringSource::XMLStringSource (const std::string &string)
+{
+  XMLSourcePrivateData *source = new XMLSourcePrivateData ();
+  source->setData (QByteArray (string.c_str ()));
+  set_source (source);
+}
+
+XMLStringSource::XMLStringSource (const char *cp)
+{
+  XMLSourcePrivateData *source = new XMLSourcePrivateData ();
+  source->setData (QByteArray (cp));
+  set_source (source);
+}
+
+XMLStringSource::XMLStringSource (const char *cp, size_t len)
+{
+  XMLSourcePrivateData *source = new XMLSourcePrivateData ();
+  source->setData (QByteArray (cp, int (len)));
+  set_source (source);
+}
+
+XMLStringSource::~XMLStringSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  StreamIODevice definition and implementation
+
+class StreamIODevice
+  : public QIODevice
+{
+public:
+  StreamIODevice (tl::InputStream &stream)
+    : mp_stream (&stream),
+      mp_progress (0),
+      m_has_error (false)
+  {
+    open (QIODevice::ReadOnly);
+  }
+
+  StreamIODevice (tl::InputStream &stream, const std::string &progress_message)
+    : mp_stream (&stream),
+      mp_progress (new AbsoluteProgress (progress_message, 100)),
+      m_has_error (false)
+  {
+    mp_progress->set_format (tl::to_string (tr ("%.0f MB")));
+    mp_progress->set_unit (1024 * 1024);
+    open (QIODevice::ReadOnly);
+  }
+
+  StreamIODevice (const std::string &path)
+    : mp_stream_holder (new tl::InputStream (path)),
+      mp_progress (0),
+      m_has_error (false)
+  {
+    mp_stream = mp_stream_holder.get ();
+    open (QIODevice::ReadOnly);
+  }
+
+  StreamIODevice (const std::string &path, const std::string &progress_message)
+    : mp_stream_holder (new tl::InputStream (path)),
+      mp_progress (new AbsoluteProgress (progress_message, 100)),
+      m_has_error (false)
+  {
+    mp_stream = mp_stream_holder.get ();
+    mp_progress->set_format (tl::to_string (tr ("%.0f MB")));
+    mp_progress->set_unit (1024 * 1024);
+    open (QIODevice::ReadOnly);
+  }
+
+  ~StreamIODevice ()
+  {
+    if (mp_progress) {
+      delete mp_progress;
+      mp_progress = 0;
+    }
+  }
+
+  virtual bool isSequential () const
+  {
+    return true;
+  }
+
+  qint64 writeData (const char *, qint64)
+  {
+    tl_assert (false);
+  }
+
+  qint64 readData (char *data, qint64 n)
+  {
+    try {
+
+      if (mp_progress) {
+        mp_progress->set (mp_stream->pos ());
+      }
+
+      qint64 n0 = n;
+      for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; ) {
+        //  NOTE: we skip CR to compensate for Windows CRLF line terminators (issue #419).
+        if (*rd != '\r') {
+          *data++ = *rd;
+          --n;
+        }
+      }
+
+      if (n0 == n) {
+        return -1;
+      } else {
+        return n0 - n;
+      }
+
+    } catch (tl::Exception &ex) {
+      setErrorString (tl::to_qstring (ex.msg ()));
+      m_has_error = true;
+      return -1;
+    }
+  }
+
+  bool has_error () const
+  {
+    return m_has_error;
+  }
+
+private:
+  tl::InputStream *mp_stream;
+  std::unique_ptr<tl::InputStream> mp_stream_holder;
+  tl::AbsoluteProgress *mp_progress;
+  bool m_has_error;
+};
+
+// --------------------------------------------------------------------
+//  XMLFileSource implementation
+
+class XMLStreamSourcePrivateData
+  : public XMLSourcePrivateData
+{
+public:
+  XMLStreamSourcePrivateData (StreamIODevice *io)
+    : XMLSourcePrivateData (io), mp_io (io)
+  {
+    //  .. nothing yet ..
+  }
+
+  virtual void fetchData ()
+  {
+    QXmlInputSource::fetchData ();
+
+    //  This feature is actually missing in the original implementation: throw an exception on error
+    if (mp_io->has_error ()) {
+      throw tl::Exception (tl::to_string (mp_io->errorString ()));
+    }
+  }
+
+private:
+  std::unique_ptr<StreamIODevice> mp_io;
+};
+
+// --------------------------------------------------------------------
+//  XMLFileSource implementation
+
+XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message)
+{
+  set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path, progress_message)));
+}
+
+XMLFileSource::XMLFileSource (const std::string &path)
+{
+  set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path)));
+}
+
+XMLFileSource::~XMLFileSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLStreamSource implementation
+
+XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message)
+{
+  set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s, progress_message)));
+}
+
+XMLStreamSource::XMLStreamSource (tl::InputStream &s)
+{
+  set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s)));
+}
+
+XMLStreamSource::~XMLStreamSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLParser implementation
+
+class XMLParserPrivateData
+  : public QXmlSimpleReader
+{
+public:
+  XMLParserPrivateData () : QXmlSimpleReader () { }
+};
+
+XMLParser::XMLParser ()
+  : mp_data (new XMLParserPrivateData ())
+{
+  //  .. nothing yet ..
+}
+
+XMLParser::~XMLParser ()
+{
+  delete mp_data;
+  mp_data = 0;
+}
+
+void
+XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler)
+{
+  SAXHandler handler (&struct_handler);
+
+  mp_data->setContentHandler (&handler);
+  mp_data->setErrorHandler (&handler);
+
+  bool result = mp_data->parse (source.source (), false /*=not incremental*/);
+  if (! result && handler.exception ()) {
+    throw tl::XMLLocatedException (*handler.exception ());
+  }
+}
+
+bool
+XMLParser::is_available ()
+{
+  return true;
+}
+
+}
+
+#else
+
+namespace tl
+{
+
+// --------------------------------------------------------------------
+//  XMLSource implementation
+
+XMLSource::XMLSource ()
+  : mp_source (0)
+{
+  //  .. nothing yet ..
+}
+
+XMLSource::~XMLSource ()
+{
+  //  .. nothing yet ..
+}
+
+void
+XMLSource::reset ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLStringSource implementation
+
+XMLStringSource::XMLStringSource (const std::string &)
+{
+  tl_assert (false);
+}
+
+XMLStringSource::XMLStringSource (const char *)
+{
+  tl_assert (false);
+}
+
+XMLStringSource::XMLStringSource (const char *, size_t)
+{
+  tl_assert (false);
+}
+
+XMLStringSource::~XMLStringSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLFileSource implementation
+
+XMLFileSource::XMLFileSource (const std::string &, const std::string &)
+{
+  tl_assert (false);
+}
+
+XMLFileSource::XMLFileSource (const std::string &)
+{
+  tl_assert (false);
+}
+
+XMLFileSource::~XMLFileSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLStreamSource implementation
+
+XMLStreamSource::XMLStreamSource (tl::InputStream &, const std::string &)
+{
+  tl_assert (false);
+}
+
+XMLStreamSource::XMLStreamSource (tl::InputStream &)
+{
+  tl_assert (false);
+}
+
+XMLStreamSource::~XMLStreamSource ()
+{
+  //  .. nothing yet ..
+}
+
+// --------------------------------------------------------------------
+//  XMLParser implementation
+
+XMLParser::XMLParser ()
+  : mp_data (0)
+{
+  //  .. nothing yet ..
+}
+
+XMLParser::~XMLParser ()
+{
+  //  .. nothing yet ..
+}
+
+void
+XMLParser::parse (XMLSource &, XMLStructureHandler &)
+{
+  tl_assert (false);
+}
+
+bool
+XMLParser::is_available ()
+{
+  return false;
+}
+
+}
+
+#endif

src/tl/tl/tlXMLReader.h

@@ -0,0 +1,215 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2024 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+
+#ifndef HDR_tlXMLReader
+#define HDR_tlXMLReader
+
+#include "tlCommon.h"
+
+#include <list>
+#include <vector>
+
+#include "tlAssert.h"
+#include "tlInternational.h"
+#include "tlString.h"
+#include "tlStream.h"
+
+class QIODevice;
+
+namespace tl
+{
+
+/**
+ *  NOTE: This XML parser package also supports a ProtocolBuffer flavor.
+ *  This allows binding the same scheme to efficient binary PB format.
+ */
+
+class ProtocolBufferReaderBase;
+class ProtocolBufferWriterBase;
+
+/**
+ *  @brief A basic XML parser error exception class
+ */
+
+class TL_PUBLIC XMLException : public tl::Exception
+{
+public:
+  XMLException (const char *msg)
+    : Exception (tl::to_string (tr ("XML parser error: %s")).c_str ()),
+      m_msg (msg)
+  {
+    //  .. nothing yet ..
+  }
+
+  XMLException (const std::string &msg)
+    : Exception (fmt (-1, -1).c_str (), msg.c_str ()),
+      m_msg (msg)
+  {
+    //  .. nothing yet ..
+  }
+
+  /**
+   *  @brief Raw (unprefixed) message of the XML parser
+   */
+  const std::string &
+  raw_msg () const
+  {
+    return m_msg;
+  }
+
+protected:
+  XMLException (const std::string &msg, int line, int column)
+    : Exception (fmt (line, column).c_str (), msg.c_str (), line, column),
+      m_msg (msg)
+  {
+    //  .. nothing yet ..
+  }
+
+private:
+  std::string m_msg;
+
+  static std::string fmt (int line, int /*column*/)
+  {
+    if (line < 0) {
+      return tl::to_string (tr ("XML parser error: %s")).c_str ();
+    } else {
+      return tl::to_string (tr ("XML parser error: %s in line %d, column %d")).c_str ();
+    }
+  }
+};
+
+/**
+ *  @brief A XML parser error exception class that additionally provides line and column information
+ */
+
+class TL_PUBLIC XMLLocatedException : public XMLException
+{
+public:
+  XMLLocatedException (const std::string &msg, int line, int column)
+    : XMLException (msg, line, column),
+      m_line (line), m_column (column)
+  {
+    //  .. nothing yet ..
+  }
+
+  /**
+   *  @brief Line number information of the exception
+   */
+  int line () const
+  {
+    return m_line;
+  }
+
+  /**
+   *  @brief Column number information of the exception
+   */
+  int column () const
+  {
+    return m_column;
+  }
+
+private:
+  int m_line;
+  int m_column;
+};
+
+//  The opaque source type
+class XMLSourcePrivateData;
+
+/**
+ *  @brief A generic XML text source class
+ *
+ *  This class is the base class providing input for
+ *  the Qt XML parser and basically maps to a QXmlInputSource object
+ *  for compatibility with the "libparsifal" branch.
+ */
+
+class TL_PUBLIC XMLSource
+{
+public:
+  XMLSource ();
+  ~XMLSource ();
+
+  XMLSourcePrivateData *source ()
+  {
+    return mp_source;
+  }
+
+  void reset ();
+
+protected:
+  void set_source (XMLSourcePrivateData *source)
+  {
+    mp_source = source;
+  }
+
+private:
+  XMLSourcePrivateData *mp_source;
+};
+
+/**
+ *  @brief A specialization of XMLSource to receive a string
+ */
+
+class TL_PUBLIC XMLStringSource : public XMLSource
+{
+public:
+  XMLStringSource (const std::string &string);
+  XMLStringSource (const char *cp);
+  XMLStringSource (const char *cp, size_t len);
+  ~XMLStringSource ();
+
+private:
+  std::string m_copy;
+};
+
+/**
+ *  @brief A specialization of XMLSource to receive from a file
+ */
+
+class TL_PUBLIC XMLFileSource : public XMLSource
+{
+public:
+  XMLFileSource (const std::string &path);
+  XMLFileSource (const std::string &path, const std::string &progress_message);
+  ~XMLFileSource ();
+};
+
+/**
+ *  @brief A generic stream source class
+ *
+ *  This class implements a XML parser source from a tl::InputStream
+ */
+
+class TL_PUBLIC XMLStreamSource : public XMLSource
+{
+public:
+  XMLStreamSource (tl::InputStream &stream);
+  XMLStreamSource (tl::InputStream &stream, const std::string &progress_message);
+  ~XMLStreamSource ();
+};
+
+} // namespace tl
+
+#endif
+

src/tl/unit_tests/tlProtocolBufferTests.cc

@@ -0,0 +1,636 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2024 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#include "tlXMLParser.h"
+#include "tlProtocolBuffer.h"
+#include "tlUnitTest.h"
+#include "tlFileUtils.h"
+
+#include <sstream>
+#include <cmath>
+
+//  Basic tests of reader and writer
+TEST (1_BasicTypes)
+{
+  tl::OutputMemoryStream out;
+
+  {
+    tl::OutputStream os (out);
+    tl::ProtocolBufferWriter writer (os);
+    writer.write (1, std::string ("xyz_abc"));
+    writer.write (2, float (1.5));
+    writer.write (3, double (2.5));
+    writer.write (4, true);
+    writer.write (5, int32_t (-100000));
+    writer.write (6, int32_t (100000));
+    writer.write (7, uint32_t (200000));
+    writer.write (8, int64_t (-10000000000));
+    writer.write (9, int64_t (10000000000));
+    writer.write (10, uint64_t (20000000000));
+    writer.write (11, int32_t (-100000), true);
+    writer.write (12, int32_t (100000), true);
+    writer.write (13, uint32_t (200000), true);
+    writer.write (14, int64_t (-10000000000), true);
+    writer.write (15, int64_t (10000000000), true);
+    writer.write (16, uint64_t (20000000000), true);
+  }
+
+  {
+    tl::InputMemoryStream s2 (out.data (), out.size ());
+    tl::InputStream is (s2);
+    tl::ProtocolBufferReader reader (is);
+
+    std::string s;
+    bool b;
+    float f;
+    double d;
+    uint32_t ui32;
+    int32_t i32;
+    uint64_t ui64;
+    int64_t i64;
+
+    EXPECT_EQ (reader.read_tag (), 1);
+    s = "";
+    reader.read (s);
+    EXPECT_EQ (s, "xyz_abc");
+    EXPECT_EQ (reader.read_tag (), 2);
+    f = 0;
+    reader.read (f);
+    EXPECT_EQ (f, 1.5);
+    EXPECT_EQ (reader.read_tag (), 3);
+    d = 0;
+    reader.read (d);
+    EXPECT_EQ (d, 2.5);
+    EXPECT_EQ (reader.read_tag (), 4);
+    b = false;
+    reader.read (b);
+    EXPECT_EQ (b, true);
+    EXPECT_EQ (reader.read_tag (), 5);
+    i32 = 0;
+    reader.read (i32);
+    EXPECT_EQ (i32, -100000);
+    EXPECT_EQ (reader.read_tag (), 6);
+    i32 = 0;
+    reader.read (i32);
+    EXPECT_EQ (i32, 100000);
+    EXPECT_EQ (reader.read_tag (), 7);
+    ui32 = 0;
+    reader.read (ui32);
+    EXPECT_EQ (ui32, 200000u);
+    EXPECT_EQ (reader.read_tag (), 8);
+    i64 = 0;
+    reader.read (i64);
+    EXPECT_EQ (i64, -10000000000);
+    EXPECT_EQ (reader.read_tag (), 9);
+    i64 = 0;
+    reader.read (i64);
+    EXPECT_EQ (i64, 10000000000);
+    EXPECT_EQ (reader.read_tag (), 10);
+    ui64 = 0;
+    reader.read (ui64);
+    EXPECT_EQ (ui64, 20000000000u);
+    EXPECT_EQ (reader.read_tag (), 11);
+    i32 = 0;
+    reader.read (i32);
+    EXPECT_EQ (i32, -100000);
+    EXPECT_EQ (reader.read_tag (), 12);
+    i32 = 0;
+    reader.read (i32);
+    EXPECT_EQ (i32, 100000);
+    EXPECT_EQ (reader.read_tag (), 13);
+    ui32 = 0;
+    reader.read (ui32);
+    EXPECT_EQ (ui32, 200000u);
+    EXPECT_EQ (reader.read_tag (), 14);
+    i64 = 0;
+    reader.read (i64);
+    EXPECT_EQ (i64, -10000000000);
+    EXPECT_EQ (reader.read_tag (), 15);
+    i64 = 0;
+    reader.read (i64);
+    EXPECT_EQ (i64, 10000000000);
+    EXPECT_EQ (reader.read_tag (), 16);
+    ui64 = 0;
+    reader.read (ui64);
+    EXPECT_EQ (ui64, 20000000000u);
+    EXPECT_EQ (reader.at_end (), true);
+  }
+}
+
+struct Child {
+  Child () : txt (""), d(-1), live(true) { }
+  ~Child () { tl_assert (live); live = false; }
+  std::string txt;
+  double d;
+  bool live;
+  bool operator== (const Child &x) const { return txt == x.txt && fabs (d - x.d) < 1e-9 && children == x.children; }
+  std::vector<Child> children;
+  std::vector<Child>::const_iterator begin_children() const { return children.begin (); }
+  std::vector<Child>::const_iterator end_children() const { return children.end (); }
+  void add_child (const Child &c) { children.push_back (c); }
+  void add_child_ptr (Child *c) { children.push_back (*c); delete c; }
+};
+
+struct Root {
+  long m;
+  unsigned int mi;
+  bool b;
+  std::vector<double> m_subs;
+  std::vector<int> m_isubs;
+  std::vector<Child> m_children;
+  Child m_child;
+
+  Root () : m(0), mi(0), b(false) { }
+
+  bool operator== (const Root &x) const { return m == x.m &&
+                                          mi == x.mi && m_subs == x.m_subs &&
+                                          m_isubs == x.m_isubs && m_children == x.m_children
+                                          && m_child == x.m_child && b == x.b; }
+
+  int get_mi () const { return mi; }
+  void set_mi (int i) { mi = i; }
+  void add_sub (const double &s) {
+    m_subs.push_back (s);
+  }
+  void add_isub (const int &s) {
+    m_isubs.push_back (s);
+  }
+  std::vector<double>::const_iterator begin_subs () const {
+    return m_subs.begin ();
+  }
+  std::vector<double>::const_iterator end_subs () const {
+    return m_subs.end ();
+  }
+  std::vector<int>::const_iterator begin_isubs () const {
+    return m_isubs.begin ();
+  }
+  std::vector<int>::const_iterator end_isubs () const {
+    return m_isubs.end ();
+  }
+  void add_child_ptr (Child *c) {
+    m_children.push_back (*c);
+    delete c;
+  }
+  void add_child (const Child &c) {
+    m_children.push_back (c);
+  }
+  std::vector<Child>::const_iterator begin_children () const {
+    return m_children.begin ();
+  }
+  std::vector<Child>::const_iterator end_children () const {
+    return m_children.end ();
+  }
+  void set_child (const Child &child) { m_child = child; }
+  const Child &get_child () const { return m_child; }
+};
+
+static tl::XMLElementList child_struct ("Child",
+  tl::make_member (&Child::txt, "txt#1") +
+  tl::make_member (&Child::d, "d#2") +
+  tl::make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children#3", &child_struct)
+);
+
+static tl::XMLStruct<Root> structure ("pbtest-struct#88888888",
+  tl::make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, "sub#1") +
+  tl::make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, "isub#2") +
+  tl::make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "children#3", &child_struct) +
+  tl::make_element (&Root::get_child, &Root::set_child, "child#4", &child_struct) +
+  tl::make_member (&Root::m, "m#5") +
+  tl::make_member (&Root::get_mi, &Root::set_mi, "mi#6") +
+  tl::make_member (&Root::b, "b#7")
+);
+
+static void build_struct (Root &root)
+{
+  root.add_sub (0.5);
+  root.add_sub (7.5);
+  root.add_isub (420000000);
+  root.m = -1700000;
+  root.set_mi (21);
+  root.b = true;
+
+  Child c1;
+  c1.txt = "c1";
+  c1.d = 1.0;
+  root.add_child (c1);
+
+  Child c2;
+  c2.txt = "c2";
+  c2.d = 2.0;
+
+  Child c21;
+  c21.txt = "c21";
+  c21.d = 2.1;
+  c2.add_child (c21);
+
+  Child c22;
+  c22.txt = "c22";
+  c22.d = 2.2;
+  c2.add_child (c22);
+
+  Child c23;
+  c23.txt = "c23";
+  c23.d = 2.3;
+  c2.add_child (c23);
+
+  root.add_child (c2);
+
+  Child sc;
+  sc.txt = "single";
+  sc.d = 4.2e6;
+  root.set_child (sc);
+}
+
+TEST (100_BasicStruct)
+{
+  Root root;
+  build_struct (root);
+
+  std::string fn = tl::combine_path (tl::testtmp (), "pb_test.pb");
+
+  {
+    tl::OutputStream os (fn);
+    tl::ProtocolBufferWriter writer (os);
+    structure.write (writer, root);
+    /*
+    for debugging:
+    tl::ProtocolBufferDumper dumper;
+    structure.write (dumper, root);
+    */
+  }
+
+  root = Root ();
+
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    structure.parse (reader, root);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  //  TODO: adjust
+  EXPECT_EQ (error, "");
+  EXPECT_EQ (root.m_subs.size (), size_t (2));
+  EXPECT_EQ (root.m_subs [0], 0.5);
+  EXPECT_EQ (root.m_subs [1], 7.5);
+  EXPECT_EQ (root.m_isubs.size (), size_t (1));
+  EXPECT_EQ (root.m_isubs [0], 420000000);
+  EXPECT_EQ (root.m, -1700000);
+  EXPECT_EQ (root.b, true);
+  EXPECT_EQ (root.mi, (unsigned int) 21);
+  EXPECT_EQ (root.m_children.size (), size_t (2));
+  EXPECT_EQ (root.m_children [0].txt, "c1");
+  EXPECT_EQ (root.m_children [0].d, 1.0);
+  EXPECT_EQ (root.m_children [1].txt, "c2");
+  EXPECT_EQ (root.m_children [1].d, 2.0);
+  EXPECT_EQ (root.m_children [1].end_children () - root.m_children [1].begin_children (), 3);
+  EXPECT_EQ (root.m_child.txt, "single");
+  EXPECT_EQ (root.m_child.d, 4.2e6);
+
+  //  write ..
+  tl::OutputMemoryStream out;
+
+  {
+    tl::OutputStream os (out);
+    tl::ProtocolBufferWriter writer (os);
+    structure.write (writer, root);
+  }
+
+  //  and read again.
+  Root rsave (root);
+
+  try {
+    error.clear ();
+    tl::InputMemoryStream s2 (out.data (), out.size ());
+    tl::InputStream is (s2);
+    root = Root ();
+    tl::ProtocolBufferReader reader (is);
+    structure.parse (reader, root);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (error, "");
+  EXPECT_EQ (root == rsave, true);
+
+  //  write empty object
+  out.clear ();
+  root = Root ();
+
+  {
+    tl::OutputStream os (out);
+    tl::ProtocolBufferWriter writer (os);
+    structure.write (writer, root);
+  }
+
+  //  and read again.
+  try {
+    error.clear ();
+    tl::InputMemoryStream s2 (out.data (), out.size ());
+    tl::InputStream is (s2);
+    root = Root ();
+    tl::ProtocolBufferReader reader (is);
+    structure.parse (reader, root);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (error, "");
+  EXPECT_EQ (root == Root (), true);
+}
+
+struct TestClass
+{
+  enum enum_type { A, B, C };
+
+  TestClass () : e (A) { }
+  enum_type e;
+};
+
+struct TestClassEnumConverter
+{
+  typedef uint32_t pb_type;
+
+  pb_type pb_encode (TestClass::enum_type e) const
+  {
+    switch (e) {
+    case TestClass::A:
+      return 17;
+    case TestClass::B:
+      return 18;
+    case TestClass::C:
+      return 19;
+    default:
+      return 0;
+    }
+  }
+
+  void pb_decode (uint32_t value, TestClass::enum_type &e) const
+  {
+    switch (value) {
+    case 17:
+      e = TestClass::A;
+      break;
+    case 18:
+      e = TestClass::B;
+      break;
+    case 19:
+      e = TestClass::C;
+      break;
+    default:
+      e = TestClass::enum_type (0);
+      break;
+    }
+  }
+
+  void from_string (const std::string &, TestClass::enum_type) const { }
+  std::string to_string (TestClass::enum_type) const { return std::string (); }
+};
+
+tl::XMLStruct<TestClass> tc_structure ("pbtest-tc#1",
+  tl::make_member (&TestClass::e, "e#2", TestClassEnumConverter ())
+);
+
+TEST (101_Converter)
+{
+  TestClass tc;
+
+  tc.e = TestClass::A;
+  std::string fn = tl::combine_path (tl::testtmp (), "pb_101a.pb");
+
+  {
+    tl::OutputStream os (fn);
+    tl::ProtocolBufferWriter writer (os);
+    tc_structure.write (writer, tc);
+  }
+
+  tc = TestClass ();
+
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    tc_structure.parse (reader, tc);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (tc.e, TestClass::A);
+
+  tc.e = TestClass::B;
+
+  fn = tl::combine_path (tl::testtmp (), "pb_101b.pb");
+
+  {
+    tl::OutputStream os (fn);
+    tl::ProtocolBufferWriter writer (os);
+    tc_structure.write (writer, tc);
+  }
+
+  tc = TestClass ();
+
+  error.clear ();
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    tc_structure.parse (reader, tc);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (tc.e, TestClass::B);
+}
+
+TEST (101_ExternalFiles)
+{
+  Root root_au;
+  build_struct (root_au);
+
+  std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct1.pb");
+
+  Root root;
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    structure.parse (reader, root);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (error, "");
+  EXPECT_EQ (root == root_au, true);
+}
+
+TEST (102_InvalidHeader)
+{
+  std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct2_invalid_header.pb");
+
+  Root root;
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    structure.parse (reader, root);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (std::string (error, 0, 49), "Expected header field with ID 88888888 (got 4444)");
+}
+
+TEST (103_InvalidHeader)
+{
+  std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct3_invalid_header.pb");
+
+  Root root;
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    structure.parse (reader, root);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (std::string (error, 0, 71), "Expected header field with string 'pbtest-struct' (got 'xxxxxxxxxxxxx')");
+}
+
+//  "last one wins", unknown IDs are skipped
+TEST (104_Deviations)
+{
+  Root root_au;
+  build_struct (root_au);
+
+  std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct4_deviations.pb");
+
+  Root root;
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    structure.parse (reader, root);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (error, "");
+  EXPECT_EQ (root == root_au, true);
+}
+
+TEST (105_Int32Overflow)
+{
+  std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct5_overflow.pb");
+
+  TestClass tc;
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    tc_structure.parse (reader, tc);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (std::string (error, 0, 21), "32 bit value overflow");
+}
+
+TEST (106_InvalidType)
+{
+  std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct6_invalid_type.pb");
+
+  TestClass tc;
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    tc_structure.parse (reader, tc);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (std::string (error, 0, 34), "Expected a VARINT or I32 wire type");
+}
+
+TEST (107_InvalidType)
+{
+  //  uses I32 to represent enum - works too, not just VARINT
+  std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct7_invalid_type.pb");
+
+  TestClass tc;
+  tc.e = TestClass::B;
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    tc_structure.parse (reader, tc);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (error, "");
+  EXPECT_EQ (tc.e, TestClass::A);
+}
+
+TEST (108_InvalidType)
+{
+  std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct8_invalid_type.pb");
+
+  TestClass tc;
+  std::string error;
+  try {
+    tl::InputStream is (fn);
+    tl::ProtocolBufferReader reader (is);
+    tc_structure.parse (reader, tc);
+  } catch (tl::Exception &ex) {
+    error = ex.msg ();
+  }
+
+  EXPECT_EQ (std::string (error, 0, 34), "Expected a VARINT or I32 wire type");
+}
+
+TEST (109_DumpSpec)
+{
+  EXPECT_EQ (structure.create_def (),
+    "// created from KLayout proto definition 'pbtest-struct'\n"
+    "\n"
+    "syntax = \"proto2\";\n"
+    "\n"
+    "message Child {\n"
+    "  optional string txt = 1;\n"
+    "  optional double d = 2;\n"
+    "  repeated Child children = 3;\n"
+    "}\n"
+    "\n"
+    "message PbtestStruct {\n"
+    "  repeated double sub = 1;\n"
+    "  repeated sint32 isub = 2;\n"
+    "  repeated Child children = 3;\n"
+    "  optional Child child = 4;\n"
+    "  optional sint64 m = 5;\n"
+    "  optional sint32 mi = 6;\n"
+    "  optional uint32 b = 7;\n"
+    "}"
+  );
+}

src/tl/unit_tests/unit_tests.pro

@@ -33,6 +33,7 @@ SOURCES = \
   tlObjectTests.cc \
   tlOptionalTests.cc \
   tlPixelBufferTests.cc \
+  tlProtocolBufferTests.cc \
   tlResourcesTests.cc \
   tlReuseVectorTests.cc \
   tlStableVectorTests.cc \

testdata/pb/struct5_overflow.pb

@@ -0,0 +1,2 @@
+
+	pbtest-tc€Ô<E282AC>ª?

testdata/pb/struct8_invalid_type.pb

@@ -0,0 +1,2 @@
+
+	pbtest-tcabc

testdata/ruby/rdbTest.rb

@@ -595,7 +595,7 @@ class RDB_TestClass < TestBase
     assert_equal(db.is_modified?, false)
 
     tag_id = db.tag_id("x")
-    assert_equal(tag_id, 1)
+    assert_equal(db.tag_name(tag_id), "x")
     db.set_tag_description(tag_id, "xdesc")
     assert_equal(db.tag_description(tag_id), "xdesc")