From 793c892b2b044991d635f2195e8a80ba707f38e0 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 1 Sep 2024 17:47:25 +0200
Subject: [PATCH 01/12] Reworking documentation of LayoutToNetlist

---
 src/db/db/gsiDeclDbLayoutToNetlist.cc | 248 +++++++++++++++++++-------
 1 file changed, 180 insertions(+), 68 deletions(-)

diff --git a/src/db/db/gsiDeclDbLayoutToNetlist.cc b/src/db/db/gsiDeclDbLayoutToNetlist.cc
index 8517787ba..52a4e9d9f 100644
--- a/src/db/db/gsiDeclDbLayoutToNetlist.cc
+++ b/src/db/db/gsiDeclDbLayoutToNetlist.cc
@@ -44,11 +44,6 @@ static db::LayoutToNetlist *make_l2n_from_existing_dss_with_layout (db::DeepShap
   return new db::LayoutToNetlist (dss, layout_index);
 }
 
-static db::LayoutToNetlist *make_l2n_from_existing_dss (db::DeepShapeStore *dss)
-{
-  return new db::LayoutToNetlist (dss);
-}
-
 static db::LayoutToNetlist *make_l2n_flat (const std::string &topcell_name, double dbu)
 {
   return new db::LayoutToNetlist (topcell_name, dbu);
@@ -225,12 +220,23 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
     "@brief Creates a new extractor connected to an original layout\n"
     "This constructor will attach the extractor to an original layout through the "
     "shape iterator.\n"
+    "\n"
+    "The shape iterator does not need to be an actual shape iterator. It is merely used to identify the original "
+    "layout and provides additional parameters such as the top cell to use and advanced options such as subtree pruning.\n"
+    "\n"
+    "You can construct a dummy iteraor usable for this purpose without layers using an empty layer set:\n"
+    "\n"
+    "@code\n"
+    "ly = ...  # external layout\n"
+    "l2n = RBA::LayoutToNetlist::new(RBA::RecursiveShapeIterator::new(ly, ly.top_cell, []))"
+    "..."
+    "@/code\n"
   ) +
   gsi::constructor ("new", &make_l2n_default,
     "@brief Creates a new and empty extractor object\n"
     "The main objective for this constructor is to create an object suitable for reading an annotated netlist.\n"
   ) +
-  gsi::constructor ("new", &make_l2n_from_existing_dss, gsi::arg ("dss"),
+  gsi::constructor ("new", &make_l2n_from_existing_dss_with_layout, gsi::arg ("dss"), gsi::arg ("layout_index", 0),
     "@brief Creates a new extractor object reusing an existing \\DeepShapeStore object\n"
     "This constructor can be used if there is a DSS object already from which the "
     "shapes can be taken. This version can only be used with \\register to "
@@ -241,14 +247,8 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
     "\n"
     "The extractor will not take ownership of the dss object unless you call \\keep_dss."
   ) +
-  gsi::constructor ("new", &make_l2n_from_existing_dss_with_layout, gsi::arg ("dss"), gsi::arg ("layout_index"),
-    "@brief Creates a new extractor object reusing an existing \\DeepShapeStore object\n"
-    "This constructor can be used if there is a DSS object already from which the "
-    "shapes can be taken. NOTE: in this case, the make_... functions will create "
-    "new layers inside this DSS. To register existing layers (regions) use \\register.\n"
-  ) +
   gsi::constructor ("new", &make_l2n_flat, gsi::arg ("topcell_name"), gsi::arg ("dbu"),
-    "@brief Creates a new extractor object with a flat DSS\n"
+    "@brief Creates a new, detached extractor object with a flat DSS\n"
     "@param topcell_name The name of the top cell of the internal flat layout\n"
     "@param dbu The database unit to use for the internal flat layout\n"
     "\n"
@@ -258,6 +258,25 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
     "\n"
     "The database unit is mandatory because the physical parameter extraction "
     "for devices requires this unit for translation of layout to physical dimensions.\n"
+    "\n"
+    "Example:\n"
+    "\n"
+    "@code\n"
+    "rmetal1 = ...  # an external Region object representing 'metal1' layer\n"
+    "rvia11  = ...  # an external Region object representing 'via1' layer\n"
+    "rmetal2 = ...  # an external Region object representing 'metal2' layer\n"
+    "\n"
+    "l2n = RBA::LayoutToNetlist::new(\"TOP_CELL\", 0.001)\n"
+    "# imports the external Regions as flat ones and assigns proper names\n"
+    "l2n.register(rmetal1, \"metal1\")\n"
+    "l2n.register(rvia1, \"via1\")\n"
+    "l2n.register(rmetal2, \"metal2\")\n"
+    "\n"
+    "# intra- and inter-layer connects:\n"
+    "l2n.connect(metal1)\n"
+    "l2n.connect(metal1, via1)\n"
+    "l2n.connect(metal2)\n"
+    "@/code\n"
   ) +
   gsi::method ("generator", &db::LayoutToNetlist::generator,
     "@brief Gets the generator string.\n"
@@ -268,9 +287,11 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
   ) +
   gsi::method ("dss", (db::DeepShapeStore &(db::LayoutToNetlist::*) ()) &db::LayoutToNetlist::dss,
     "@brief Gets a reference to the internal DSS object.\n"
+    "See the class description for details about the DSS object used inside the LayoutToNetlist object."
   ) +
   gsi::method ("keep_dss", &db::LayoutToNetlist::keep_dss,
-    "@brief Resumes ownership over the DSS object if created with an external one.\n"
+    "@brief Takes ownership of the DSS object if the LayoutToNetlist object was created with an external one.\n"
+    "See the class description for details about the DSS object used inside the LayoutToNetlist object."
   ) +
   gsi::method ("threads=", &db::LayoutToNetlist::set_threads, gsi::arg ("n"),
     "@brief Sets the number of threads to use for operations which support multiple threads\n"
@@ -329,13 +350,28 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
   ) +
   gsi::method ("layer_name", (std::string (db::LayoutToNetlist::*) (const db::ShapeCollection &region) const) &db::LayoutToNetlist::name, gsi::arg ("l"),
     "@brief Gets the name of the given layer\n"
+    "The layer is given by a \\ShapeCollection object which is either a \\Region or \\Texts object.\n"
+    "This \\Region or \\Texts object needs to be either one that was created by make_... or one that\n"
+    "was registered through \\register. This function will return the name that was given to that \\Region or \\Texts "
+    "during those operations."
+    "\n"
+    "You can use \\layer_by_name to retrieve the \\Region object of a layer from the layer index.\n"
   ) +
   gsi::method ("layer_index", (unsigned int (db::LayoutToNetlist::*) (const db::ShapeCollection &region) const) &db::LayoutToNetlist::layer_of<db::ShapeCollection>, gsi::arg ("l"),
     "@brief Gets the layer index for the given data object\n"
+    "This method is essentially identical to \\layer_of, but uses \\ShapeCollection, which is a polymorphic base class "
+    "for a variety of shape containers.\n"
+    "\n"
+    "The layer index returned is index of the corresponding layer inside the internal layout (see \\internal_layout).\n"
+    "The layer index is more handy to use for identifying a layer than a \\Region of \\Texts object, for example when using it as a key in hashes.\n"
+    "\n"
+    "You can use \\layer_by_index to retrieve the \\Region object of a layer from the layer index.\n"
+    "\n"
     "This method has been introduced in version 0.29.3.\n"
   ) +
   gsi::method ("layer_name", (std::string (db::LayoutToNetlist::*) (unsigned int) const) &db::LayoutToNetlist::name, gsi::arg ("l"),
     "@brief Gets the name of the given layer (by index)\n"
+    "See \\layer_index for a description of the layer index.\n"
   ) +
   gsi::method ("register", (unsigned int (db::LayoutToNetlist::*) (const db::ShapeCollection &collection, const std::string &)) &db::LayoutToNetlist::register_layer, gsi::arg ("l"), gsi::arg ("n", std::string ()),
     "@brief Names the given layer\n"
@@ -345,11 +381,16 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
     "Registering will copy the shapes into the LayoutToNetlist object in this step to enable "
     "netlist extraction.\n"
     "\n"
-    "Naming a layer allows the system to indicate the layer in various contexts, i.e. "
-    "when writing the data to a file. Named layers are also persisted inside the LayoutToNetlist object. "
-    "They are not discarded when the Region object is destroyed.\n"
+    "External \\Region or \\Texts objects that are registered are persisted. This means "
+    "the LayoutToNetlist object becomes owner of them and they are not discarded when the "
+    "Region or Text object is destroyed.\n"
     "\n"
-    "If required, the system will assign a name automatically."
+    "Naming a layer allows allows retrieving the layer later, for example after the LayoutToNetlist object\n"
+    "has been written to a file and restored from that file (during this process, the layer indexes will change).\n"
+    "\n"
+    "If no name is given, the system will assign a name automatically.\n"
+    "It is recommended to specify a name if it is required to identify the layer later - for example for "
+    "retrieving shapes from it.\n"
     "\n"
     "This method has been generalized in version 0.27. Starting with version 0.29.3, the index of the layer is returned.\n"
   ) +
@@ -360,6 +401,7 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
     "@brief Returns a list of indexes of the layers kept inside the LayoutToNetlist object.\n"
     "You can use \\layer_name to get the name from a layer index. You can use \\layer_info to get "
     "the \\LayerInfo object attached to a layer - if the layer is an original layer.\n"
+    "You can use \\layer_by_index to get the \\Region object for the layer by index.\n"
     "\n"
     "This method has been introduced in version 0.29.2.\n"
   ) +
@@ -369,70 +411,109 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
     "stream layer information where the original layer was taken from. Otherwise an empty \\LayerInfo object "
     "is returned.\n"
     "\n"
+    "The LayerInfo object is usually empty for derived layers - i.e. those which are computed through "
+    "boolean operations for example. It is recommended to assign names to such layers for easy identification later.\n"
+    "\n"
     "This method has been introduced in version 0.29.2.\n"
   ) +
   gsi::factory ("layer_by_name", &db::LayoutToNetlist::layer_by_name, gsi::arg ("name"),
     "@brief Gets a layer object for the given name.\n"
-    "The returned object is a copy which represents the named layer."
+    "The returned object is a new Region object representing the named layer. It will refer to a layer inside the "
+    "internal layout, or more specifically inside the \\DeepShapeStorage object (see \\dss and \\internal_layout).\n"
+    "The method returns 'nil' if the name is not a valid layer name.\n"
+    "See \\register and the make_... methods for a description of layer naming.\n"
   ) +
   gsi::factory ("layer_by_index", &db::LayoutToNetlist::layer_by_index, gsi::arg ("index"),
     "@brief Gets a layer object for the given index.\n"
-    "Only named layers can be retrieved with this method. "
-    "The returned object is a copy which represents the named layer."
+    "The returned object is a new Region object representing the layer with the given index. It will refer to a layer inside the "
+    "internal layout, or more specifically inside the \\DeepShapeStorage object (see \\dss and \\internal_layout).\n"
+    "The method returns 'nil' if the index is not a valid layer index."
   ) +
   gsi::method ("is_persisted?", &db::LayoutToNetlist::is_persisted<db::Region>, gsi::arg ("layer"),
     "@brief Returns true, if the given layer is a persisted region.\n"
     "Persisted layers are kept inside the LayoutToNetlist object and are not released "
-    "if their object is destroyed. Named layers are persisted, unnamed layers are not. "
-    "Only persisted, named layers can be put into \\connect."
+    "if their object is destroyed. Layers created with make_... or registered through \\register are persisted.\n"
+    "This basically applies to all layers, except intermediate layers that are potentially created as results of "
+    "operations between layers and which are not registered.\n"
   ) +
   gsi::method ("is_persisted?", &db::LayoutToNetlist::is_persisted<db::Texts>, gsi::arg ("layer"),
     "@brief Returns true, if the given layer is a persisted texts collection.\n"
     "Persisted layers are kept inside the LayoutToNetlist object and are not released "
-    "if their object is destroyed. Named layers are persisted, unnamed layers are not. "
-    "Only persisted, named layers can be put into \\connect.\n"
+    "if their object is destroyed. Layers created with make_... or registered through \\register are persisted.\n"
+    "This basically applies to all layers, except intermediate layers that are potentially created as results of "
+    "operations between layers and which are not registered.\n"
     "\n"
     "The variant for Texts collections has been added in version 0.27."
   ) +
   gsi::factory ("make_layer", (db::Region *(db::LayoutToNetlist::*) (const std::string &)) &db::LayoutToNetlist::make_layer, gsi::arg ("name", std::string ()),
     "@brief Creates a new, empty hierarchical region\n"
     "\n"
+    "This method will create a new, empty layer inside the internal layout and register it.\n"
+    "It returns a new Region object that represents the new layer. See the class description for more details.\n"
     "The name is optional. If given, the layer will already be named accordingly (see \\register).\n"
   ) +
   gsi::factory ("make_layer", (db::Region *(db::LayoutToNetlist::*) (unsigned int, const std::string &)) &db::LayoutToNetlist::make_layer, gsi::arg ("layer_index"), gsi::arg ("name", std::string ()),
     "@brief Creates a new hierarchical region representing an original layer\n"
     "'layer_index' is the layer index of the desired layer in the original layout.\n"
-    "This variant produces polygons and takes texts for net name annotation.\n"
+    "This variant produces polygons and takes texts for net name annotation as special, property-annotated polygons.\n"
     "A variant not taking texts is \\make_polygon_layer. A Variant only taking\n"
     "texts is \\make_text_layer.\n"
     "\n"
-    "The name is optional. If given, the layer will already be named accordingly (see \\register).\n"
+    "This method will basically create a copy of the original layer inside the internal layout - more specifically inside "
+    "the DSS (see \\dss and \\internal_layout). It returns a new Region object that represents this layer copy. "
+    "The new layer is already registered with the given name and can be used for \\connect for example.\n"
   ) +
   gsi::factory ("make_text_layer", &db::LayoutToNetlist::make_text_layer, gsi::arg ("layer_index"), gsi::arg ("name", std::string ()),
     "@brief Creates a new region representing an original layer taking texts only\n"
     "See \\make_layer for details.\n"
     "\n"
-    "The name is optional. If given, the layer will already be named accordingly (see \\register).\n"
-    "\n"
     "Starting with version 0.27, this method returns a \\Texts object."
   ) +
   gsi::factory ("make_polygon_layer", &db::LayoutToNetlist::make_polygon_layer, gsi::arg ("layer_index"), gsi::arg ("name", std::string ()),
-    "@brief Creates a new region representing an original layer taking polygons and texts\n"
+    "@brief Creates a new region representing an original layer taking polygons only\n"
     "See \\make_layer for details.\n"
-    "\n"
-    "The name is optional. If given, the layer will already be named accordingly (see \\register).\n"
   ) +
   gsi::method ("extract_devices", &db::LayoutToNetlist::extract_devices, gsi::arg ("extractor"), gsi::arg ("layers"),
     "@brief Extracts devices\n"
     "See the class description for more details.\n"
     "This method will run device extraction for the given extractor. The layer map is specific\n"
-    "for the extractor and uses the region objects derived with \\make_layer and its variants.\n"
+    "for the extractor and uses the Region objects derived with \\make_layer and its variants or Region "
+    "objects registered through \\register. The layer map keys are the inputs layers defined for the\n"
+    "specific extractor, but also the output layers where the extractor places markers for the device terminals.\n"
     "\n"
-    "In addition, derived regions can be passed too. Certain limitations apply. It's safe to use\n"
+    "In addition, derived regions can also be passed to the device extractor inside the layer map.\n"
+    "Certain limitations apply. It is usually safe to use\n"
     "boolean operations for deriving layers. Other operations are applicable as long as they are\n"
     "capable of delivering hierarchical layers.\n"
     "\n"
-    "If errors occur, the device extractor will contain theses errors.\n"
+    "Example:\n"
+    "\n"
+    "@code\n"
+    "ly = ... # original Layout\n"
+    "\n"
+    "l2n = RBA::LayoutToNetlist::new(RBA::RecursiveShapeIterator::new(ly, ly.top_cell, []))\n"
+    "rnwell   = l2n.make_layer(ly.layer(1, 0), \"nwell\" )\n"
+    "ractive  = l2n.make_layer(ly.layer(2, 0), \"active\" )\n"
+    "rpoly    = l2n.make_layer(ly.layer(3, 0), \"poly\" )\n"
+    "\n"
+    "rpactive = ractive & rnwell\n"
+    "rpgate   = rpactive & rpoly\n"
+    "rpsd     = rpactive - rpgate\n"
+    "\n"
+    "rnactive = ractive - rnwell\n"
+    "rngate   = rnactive & rpoly\n"
+    "rnsd     = rnactive - rngate\n"
+    "\n"
+    "# PMOS transistor device extraction\n"
+    "pmos_ex = RBA::DeviceExtractorMOS3Transistor::new(\"PMOS\")\n"
+    "l2n.extract_devices(pmos_ex, { \"SD\" => rpsd, \"G\" => rpgate, \"P\" => rpoly })\n"
+    "\n"
+    "# NMOS transistor device extraction\n"
+    "nmos_ex = RBA::DeviceExtractorMOS3Transistor::new(\"NMOS\")\n"
+    "l2n.extract_devices(nmos_ex, { \"SD\" => rnsd, \"G\" => rngate, \"P\" => rpoly })\n"
+    "@/code\n"
+    "\n"
+    "If errors occur, they will be logged inside the device extractor object and copied to the log of this LayoutToNetlist object (self).\n"
   ) +
   gsi::method ("reset_extracted", &db::LayoutToNetlist::reset_extracted,
     "@brief Resets the extracted netlist and enables re-extraction\n"
@@ -448,10 +529,12 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
   ) +
   gsi::method ("connect", (void (db::LayoutToNetlist::*) (const db::Region &)) &db::LayoutToNetlist::connect, gsi::arg ("l"),
     "@brief Defines an intra-layer connection for the given layer.\n"
-    "The layer is either an original layer created with \\make_includelayer and its variants or\n"
-    "a derived layer. Certain limitations apply. It's safe to use\n"
+    "The layer as a Region object, representing either an original layer created with \\make_layer and its variants or\n"
+    "a derived layer which was registered using \\register. Certain limitations apply. It's safe to use\n"
     "boolean operations for deriving layers. Other operations are applicable as long as they are\n"
-    "capable of delivering hierarchical layers.\n"
+    "capable of delivering hierarchical layers. Operations that introduce flat layers will create additonal pins\n"
+    "as connections need to be made from a subcell to the top cell. Hence, flat layers - or rather some with a bad hierarchy - should\n"
+    "be avoided in \\connect.\n"
   ) +
   gsi::method ("connect", (void (db::LayoutToNetlist::*) (const db::Region &, const db::Region &)) &db::LayoutToNetlist::connect, gsi::arg ("a"), gsi::arg ("b"),
     "@brief Defines an inter-layer connection for the given layers.\n"
@@ -544,12 +627,6 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
     "\n"
     "This attribute has been introduced in version 0.27.\n"
   ) +
-  gsi::method ("make_soft_connection_diodes=", &db::LayoutToNetlist::set_make_soft_connection_diodes, gsi::arg ("flag"),
-    "@hide"
-  ) +
-  gsi::method ("make_soft_connection_diodes", &db::LayoutToNetlist::make_soft_connection_diodes,
-    "@hide"
-  ) +
   gsi::method ("top_level_mode=", &db::LayoutToNetlist::set_top_level_mode, gsi::arg ("flag"),
     "@brief Sets a flag indicating whether top level mode is enabled.\n"
     "\n"
@@ -640,12 +717,16 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
     "The internal layout is where the LayoutToNetlist database stores the shapes for the nets. "
     "Usually you do not need to access this object - you must use \\build_net or \\shapes_of_net to "
     "retrieve the per-net shape information. If you access the internal layout, make sure you do not "
-    "modify it."
+    "modify it.\n"
+    "\n"
+    "See the class description for details about the internal layout object."
   ) +
   gsi::method_ext ("internal_top_cell", &l2n_internal_top_cell,
     "@brief Gets the internal top cell\n"
     "Usually it should not be required to obtain the internal cell. If you need to do so, make sure not to modify the cell as\n"
-    "the functionality of the netlist extractor depends on it."
+    "the functionality of the netlist extractor depends on it.\n"
+    "\n"
+    "See the class description for details about the internal layout object."
   ) +
   gsi::method ("layer_of", &db::LayoutToNetlist::layer_of<db::Region>, gsi::arg ("l"),
     "@brief Gets the internal layer for a given extraction layer\n"
@@ -950,57 +1031,88 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
    "This variant has been introduced in version 0.26.6.\n"
   ) +
   //  test API
+  gsi::method ("make_soft_connection_diodes=", &db::LayoutToNetlist::set_make_soft_connection_diodes, gsi::arg ("flag"), "@hide") +
+  gsi::method ("make_soft_connection_diodes", &db::LayoutToNetlist::make_soft_connection_diodes, "@hide") +
   gsi::method_ext ("dump_joined_net_names", &dump_joined_net_names, "@hide") +
   gsi::method_ext ("dump_joined_net_names_per_cell", &dump_joined_net_names_per_cell, "@hide") +
   gsi::method_ext ("dump_joined_nets", &dump_joined_nets, "@hide") +
   gsi::method_ext ("dump_joined_nets_per_cell", &dump_joined_nets_per_cell, "@hide")
   ,
-  "@brief A generic framework for extracting netlists from layouts\n"
+  "@brief A framework for extracting netlists from layouts\n"
   "\n"
-  "This class wraps various concepts from db::NetlistExtractor and db::NetlistDeviceExtractor\n"
-  "and more. It is supposed to provide a framework for extracting a netlist from a layout.\n"
+  "This class provides a framework for extracting a netlist from a layout.\n"
   "\n"
-  "The use model of this class consists of five steps which need to be executed in this order.\n"
+  "A LayoutToNetlist object extracts a netlist from an external \\Layout. To do so, it keeps "
+  "an internal copy with an optimized representation of the original layout. When a netlist is extracted "
+  "the net geometries can be recovered from that internal layout. In addition to that layout, it keeps "
+  "the extracted netlist. Netlist and internal layout form a pair and there are references between them. "
+  "For example, the \\Circuit objects from the netlist have an attribute \\cell_index, which tells what cell "
+  "from the internal layout the circuit was derived from. In the same way, subcircuit references refer to "
+  "cell instances and nets keep a reference to the shapes they were derived from.\n"
+  "\n"
+  "LayoutToNetlist can also operate in detached mode, when there is no external layout. In this mode, "
+  "layers are created inside the internal layout only. As there is no input hierarchy, operation is "
+  "necessarily flat in that case. Single \\Region and \\Texts shape collections can be introduced into "
+  "the LayoutToNetlist objects from external sources to populate the layers from the internal layout.\n"
+  "For detached mode, use the 'LayoutToNetlist(topcell, dbu)' constructor.\n"
+  "\n"
+  "Usually, the internal layout is stored inside an internal \\DeepShapeStore object, which supplies "
+  "additional services such as layer lifetime management and maintains the connection to and from "
+  "the external layout.\n"
+  "However, you can also use the extractor with an existing \\DeepShapeStore object.\n"
+  "In that case, this external \\DeepShapeStore object is used instead of the internal one.\n"
+  "\n"
+  "The LayoutToNetlist object can be persisted into a 'Layout to netlist database' file. This database "
+  "is a storage for both the netlist and the net or circuit geometries. When reading such file into "
+  "a new LayoutToNetlist object, there will be no connection to any external layout, but all the "
+  "essential netlist and geometry information will be available.\n"
+  "\n"
+  "The LayoutToNetlist object is also the entry point for netlist-driven algorithms such as antenna checks.\n"
+  "\n"
+  "The use model of the LayoutToNetlist object consists of five steps which need to be executed in this order.\n"
   "\n"
   "@ul\n"
-  "@li Configuration: in this step, the LayoutToNetlist object is created and\n"
+  "@li @b Configuration: @/b\n"
+  "    In this step, the LayoutToNetlist object is created and\n"
   "    if required, configured. Methods to be used in this step are \\threads=,\n"
   "    \\area_ratio= or \\max_vertex_count=. The constructor for the LayoutToNetlist\n"
   "    object receives a \\RecursiveShapeIterator object which basically supplies the\n"
-  "    hierarchy and the layout taken as input.\n"
+  "    hierarchy and the external layout taken as input. The constructor will initialize\n"
+  "    the internal layout and connect it to the external one.\n"
   "@/li\n"
-  "@li Preparation\n"
+  "@li @b Preparation: @/b\n"
   "    In this step, the device recognition and extraction layers are drawn from\n"
-  "    the framework. Derived can now be computed using boolean operations.\n"
-  "    Methods to use in this step are \\make_layer and its variants.\n"
+  "    the framework. Derived layers can now be computed using boolean operations.\n"
+  "    Methods to use in this step are \\make_layer and its variants. \\make_layer will either create\n"
+  "    a new, empty layer or pull a layer from the external layout into the internal layout.\n"
+  "    Derived layers are computed using the \\Region or \\Texts objects representing\n"
+  "    existing layers. If derived layers are to be used in connectivity, they\n"
+  "    need to be registered using \\register. This makes the LayoutToNetlist object the owner of "
+  "    the layer (the layer is said to be persisted then). Registered layers can or should be given a "
+  "    name. That helps indentifying them later.\n"
   "    Layer preparation is not necessarily required to happen before all\n"
   "    other steps. Layers can be computed shortly before they are required.\n"
   "@/li\n"
-  "@li Following the preparation, the devices can be extracted using \\extract_devices.\n"
+  "@li @b Device extraction: @/b\n"
+  "    Following the preparation, the devices can be extracted using \\extract_devices.\n"
   "    This method needs to be called for each device extractor required. Each time,\n"
-  "    a device extractor needs to be given plus a map of device layers. The device\n"
+  "    a device extractor needs to be given, plus a map of device layers. The device\n"
   "    layers are device extractor specific. Either original or derived layers\n"
   "    may be specified here. Layer preparation may happen between calls to \\extract_devices.\n"
   "@/li\n"
-  "@li Once the devices are derived, the netlist connectivity can be defined and the\n"
+  "@li @b Connectivity definition: @/b\n"
+  "    Once the devices are derived, the netlist connectivity can be defined and the\n"
   "    netlist extracted. The connectivity is defined with \\connect and its\n"
   "    flavours. The actual netlist extraction happens with \\extract_netlist.\n"
   "@/li\n"
-  "@li After netlist extraction, the information is ready to be retrieved.\n"
+  "@li @b Netlist extraction: @/b\n"
+  "    After netlist extraction, the information is ready to be retrieved.\n"
   "    The produced netlist is available with \\netlist. The Shapes of a\n"
   "    specific net are available with \\shapes_of_net. \\probe_net allows\n"
   "    finding a net by probing a specific location.\n"
   "@/li\n"
   "@/ul\n"
   "\n"
-  "You can also use the extractor with an existing \\DeepShapeStore object "
-  "or even flat data. In this case, preparation means importing existing regions "
-  "with the \\register method.\n"
-  "If you want to use the \\LayoutToNetlist object with flat data, use the "
-  "'LayoutToNetlist(topcell, dbu)' constructor. If you want to use it with "
-  "hierarchical data and an existing DeepShapeStore object, use the "
-  "'LayoutToNetlist(dss)' constructor.\n"
-  "\n"
   "Once the extraction is done, you can persist the \\LayoutToNetlist object "
   "using \\write and restore it using \\read. You can use the query API (see below) to "
   "analyze the LayoutToNetlist database.\n"
@@ -1008,7 +1120,7 @@ Class<db::LayoutToNetlist> decl_dbLayoutToNetlist ("db", "LayoutToNetlist",
   "The query API of the \\LayoutToNetlist object consists of the following parts:\n"
   "\n"
   "@ul\n"
-  "@li Net shape retrieval: \\build_all_nets, \\build_nets, \\build_net and \\shapes_per_net @/li\n"
+  "@li Net shape retrieval: \\build_all_nets, \\build_nets, \\build_net and \\shapes_of_net @/li\n"
   "@li Layers: \\layer_by_index, \\layer_by_name, \\layer_indexes, \\layer_names, \\layer_info, \\layer_name @/li\n"
   "@li Log entries: \\each_log_entry @/li\n"
   "@li Probing (get net from position): \\probe_net @/li\n"

From be7c4538ba1c29dcc3840acb5227460bc2d9b142 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 1 Sep 2024 18:10:14 +0200
Subject: [PATCH 02/12] Fixed a crash in the properties editor

This crash happened sometimes when clicking on the
node elements (e.g. "Boxes" or "Instances") of the
object tree, the node was the second or further one
and the category clicked at was containing one
item only.
---
 src/layui/layui/layPropertiesDialog.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/layui/layui/layPropertiesDialog.cc b/src/layui/layui/layPropertiesDialog.cc
index 0e794983f..6ecffff16 100644
--- a/src/layui/layui/layPropertiesDialog.cc
+++ b/src/layui/layui/layPropertiesDialog.cc
@@ -343,9 +343,9 @@ PropertiesDialog::current_index_changed (const QModelIndex &index, const QModelI
           m_object_indexes.push_back (oi);
         }
 
-      } else {
+      } else if (mp_properties_pages [m_index]->count () > 0) {
 
-        m_object_indexes.push_back (size_t (mp_tree_model->object_index (index)));
+        m_object_indexes.push_back (0);
 
       }
 

From 1e28e30e714b52d97c5cfe9478bbd7532cfcfc8c Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sat, 7 Sep 2024 21:27:50 +0200
Subject: [PATCH 03/12] Mute noisy diagnostics with Python debug builds, avoid
 post-Finalize Python API access by pre-Finalize cleanup of modules. This
 avoids an assertion in Python debug builds and is a better style anyway.

---
 src/pya/pya/pya.cc          | 21 +++++++++++++++++++++
 src/pya/pya/pya.h           | 10 +++++++++-
 src/pya/pya/pyaCallables.cc |  3 +++
 src/pya/pya/pyaModule.cc    | 16 ++++++++++------
 src/pya/pya/pyaModule.h     |  6 ++++++
 src/pymod/pymodHelper.h     | 14 ++++++++++----
 6 files changed, 59 insertions(+), 11 deletions(-)

diff --git a/src/pya/pya/pya.cc b/src/pya/pya/pya.cc
index 5f177fa56..0de6ea55c 100644
--- a/src/pya/pya/pya.cc
+++ b/src/pya/pya/pya.cc
@@ -360,6 +360,10 @@ PythonInterpreter::PythonInterpreter (bool embedded)
 
 PythonInterpreter::~PythonInterpreter ()
 {
+  for (auto m = m_modules.begin (); m != m_modules.end (); ++m) {
+    (*m)->cleanup ();
+  }
+
   m_stdout_channel = PythonRef ();
   m_stderr_channel = PythonRef ();
   m_stdout = PythonPtr ();
@@ -370,6 +374,23 @@ PythonInterpreter::~PythonInterpreter ()
   if (m_embedded) {
     Py_Finalize ();
   }
+
+  for (auto m = m_modules.begin (); m != m_modules.end (); ++m) {
+    delete *m;
+  }
+  m_modules.clear ();
+}
+
+void
+PythonInterpreter::register_module (pya::PythonModule *module)
+{
+  for (auto m = m_modules.begin (); m != m_modules.end (); ++m) {
+    if (*m == module) {
+      return; //  already registered
+    }
+  }
+
+  m_modules.push_back (module);
 }
 
 char *
diff --git a/src/pya/pya/pya.h b/src/pya/pya/pya.h
index ee8d35381..76838f4f0 100644
--- a/src/pya/pya/pya.h
+++ b/src/pya/pya/pya.h
@@ -105,6 +105,14 @@ public:
    */
   ~PythonInterpreter ();
 
+  /**
+   *  @brief Registers a module
+   *
+   *  The registered modules are cleaned up before the interpreter shuts down. The interpreter takes
+   *  ownership of the module object.
+   */
+  void register_module (pya::PythonModule *module);
+
   /**
    *  @brief Add the given path to the search path
    */
@@ -279,7 +287,7 @@ private:
   std::map<PyObject *, size_t> m_file_id_map;
   std::wstring mp_py3_app_name;
   bool m_embedded;
-  std::unique_ptr<pya::PythonModule> m_pya_module;
+  std::vector<pya::PythonModule *> m_modules;
 };
 
 }
diff --git a/src/pya/pya/pyaCallables.cc b/src/pya/pya/pyaCallables.cc
index 5dfc3e4f1..d12ea6905 100644
--- a/src/pya/pya/pyaCallables.cc
+++ b/src/pya/pya/pyaCallables.cc
@@ -51,6 +51,9 @@ pya_object_deallocate (PyObject *self)
   //  we better work around it.
   ++self->ob_refcnt;
 
+  //  Mute Python warnings in debug case
+  PyObject_GC_UnTrack (self);
+
   PYAObjectBase *p = PYAObjectBase::from_pyobject (self);
   p->~PYAObjectBase ();
   Py_TYPE (self)->tp_free (self);
diff --git a/src/pya/pya/pyaModule.cc b/src/pya/pya/pyaModule.cc
index 30452d669..62e6bacde 100644
--- a/src/pya/pya/pyaModule.cc
+++ b/src/pya/pya/pyaModule.cc
@@ -71,12 +71,6 @@ PythonModule::PythonModule ()
 
 PythonModule::~PythonModule ()
 {
-  PYAObjectBase::clear_callbacks_cache ();
-
-  //  the Python objects were probably deleted by Python itself as it exited -
-  //  don't try to delete them again.
-  mp_module.release ();
-
   while (!m_methods_heap.empty ()) {
     delete m_methods_heap.back ();
     m_methods_heap.pop_back ();
@@ -93,6 +87,16 @@ PythonModule::~PythonModule ()
   }
 }
 
+void
+PythonModule::cleanup ()
+{
+  //  the Python objects are probably deleted by Python itself as it exits -
+  //  don't try to delete them again in the destructor.
+  mp_module.release ();
+
+  PYAObjectBase::clear_callbacks_cache ();
+}
+
 PyObject *
 PythonModule::module ()
 {
diff --git a/src/pya/pya/pyaModule.h b/src/pya/pya/pyaModule.h
index 0adcc9586..b543ae02b 100644
--- a/src/pya/pya/pyaModule.h
+++ b/src/pya/pya/pyaModule.h
@@ -65,6 +65,12 @@ public:
    */
   ~PythonModule ();
 
+  /**
+   *  @brief Clean up the module
+   *  This method is called by the interpreter before Py_Finalize
+   */
+  void cleanup ();
+
   /**
    *  @brief Initializes the module
    *  This entry point is for external use where the module has not been created yet
diff --git a/src/pymod/pymodHelper.h b/src/pymod/pymodHelper.h
index 55bc56185..6d116e234 100644
--- a/src/pymod/pymodHelper.h
+++ b/src/pymod/pymodHelper.h
@@ -34,6 +34,7 @@
 
 #include "pyaModule.h"
 #include "pyaUtils.h"
+#include "pya.h"
 
 #include "gsi.h"
 #include "gsiExpression.h"
@@ -41,7 +42,12 @@
 static PyObject *
 module_init (const char *pymod_name, const char *mod_name, const char *mod_description)
 {
-  static pya::PythonModule module;
+  if (! pya::PythonInterpreter::instance ()) {
+    return 0;
+  }
+
+  pya::PythonModule *module = new pya::PythonModule ();
+  pya::PythonInterpreter::instance ()->register_module (module);
 
   PYA_TRY
   
@@ -50,10 +56,10 @@ module_init (const char *pymod_name, const char *mod_name, const char *mod_descr
     //  required for the tiling processor for example
     gsi::initialize_expressions ();
 
-    module.init (pymod_name, mod_description);
-    module.make_classes (mod_name);
+    module->init (pymod_name, mod_description);
+    module->make_classes (mod_name);
 
-    return module.take_module ();
+    return module->take_module ();
 
   PYA_CATCH_ANYWHERE
   

From 4f72d3353f137c8f7302879d4200f8df5e9d968b Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 8 Sep 2024 00:24:56 +0200
Subject: [PATCH 04/12] Fixed new Python module handling for standalone module
 case

---
 src/pya/pya/pya.cc       |  4 +++-
 src/pya/pya/pyaModule.cc |  2 --
 src/pya/pya/pyaObject.cc | 10 +++++++++-
 src/pya/pya/pyaObject.h  |  2 +-
 src/pya/pya/pyaRefs.cc   | 33 +++++++++++++++++++++++++--------
 src/pya/pya/pyaRefs.h    |  9 +++++++++
 src/pymod/pymodHelper.h  | 16 ++++++++--------
 7 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/src/pya/pya/pya.cc b/src/pya/pya/pya.cc
index 0de6ea55c..bc544e7f7 100644
--- a/src/pya/pya/pya.cc
+++ b/src/pya/pya/pya.cc
@@ -192,7 +192,7 @@ PythonInterpreter::PythonInterpreter (bool embedded)
 
     sp_interpreter = this;
 
-    //  this monitor whether Python shuts down and deletes the interpreter's
+    //  this monitors whether Python shuts down and deletes the interpreter's
     //  instance.
     //  NOTE: this assumes, the interpreter was created with new(!)
     Py_AtExit (&reset_interpreter);
@@ -364,6 +364,8 @@ PythonInterpreter::~PythonInterpreter ()
     (*m)->cleanup ();
   }
 
+  PYAObjectBase::clear_callbacks_cache (m_embedded);
+
   m_stdout_channel = PythonRef ();
   m_stderr_channel = PythonRef ();
   m_stdout = PythonPtr ();
diff --git a/src/pya/pya/pyaModule.cc b/src/pya/pya/pyaModule.cc
index 62e6bacde..86fa88fba 100644
--- a/src/pya/pya/pyaModule.cc
+++ b/src/pya/pya/pyaModule.cc
@@ -93,8 +93,6 @@ PythonModule::cleanup ()
   //  the Python objects are probably deleted by Python itself as it exits -
   //  don't try to delete them again in the destructor.
   mp_module.release ();
-
-  PYAObjectBase::clear_callbacks_cache ();
 }
 
 PyObject *
diff --git a/src/pya/pya/pyaObject.cc b/src/pya/pya/pyaObject.cc
index 3887ed3a3..2bdc27183 100644
--- a/src/pya/pya/pyaObject.cc
+++ b/src/pya/pya/pyaObject.cc
@@ -514,8 +514,16 @@ PYAObjectBase::initialize_callbacks ()
 }
 
 void 
-PYAObjectBase::clear_callbacks_cache ()
+PYAObjectBase::clear_callbacks_cache (bool embedded)
 {
+  //  if not embedded, we cannot use the python API at this stage - do not try to
+  //  reference count the objects there.
+  if (! embedded) {
+    for (auto c = s_callbacks_cache.begin (); c != s_callbacks_cache.end (); ++c) {
+      c->first.release_const ();
+    }
+  }
+
   s_callbacks_cache.clear ();
 }
 
diff --git a/src/pya/pya/pyaObject.h b/src/pya/pya/pyaObject.h
index 96febf155..86344a328 100644
--- a/src/pya/pya/pyaObject.h
+++ b/src/pya/pya/pyaObject.h
@@ -200,7 +200,7 @@ public:
   /**
    *  @brief Clears the callbacks cache
    */
-  static void clear_callbacks_cache ();
+  static void clear_callbacks_cache (bool embedded);
 
 private:
   friend class StatusChangedListener;
diff --git a/src/pya/pya/pyaRefs.cc b/src/pya/pya/pyaRefs.cc
index 49dc7049f..420aa88ee 100644
--- a/src/pya/pya/pyaRefs.cc
+++ b/src/pya/pya/pyaRefs.cc
@@ -32,19 +32,19 @@ namespace pya
 //  PythonRef implementation
 
 PythonRef::PythonRef ()
-  : mp_obj (NULL)
+  : mp_obj (NULL), m_owns_pointer (true)
 { 
   //  .. nothing yet ..
 }
 
 PythonRef::PythonRef (const PythonPtr &ptr)
-  : mp_obj (ptr.get ())
+  : mp_obj (ptr.get ()), m_owns_pointer (true)
 { 
   Py_XINCREF (mp_obj);
 }
 
 PythonRef::PythonRef (PyObject *obj, bool new_ref)
-  : mp_obj (obj)
+  : mp_obj (obj), m_owns_pointer (true)
 { 
   if (! new_ref) {
     Py_XINCREF (mp_obj);
@@ -53,38 +53,49 @@ PythonRef::PythonRef (PyObject *obj, bool new_ref)
 
 PythonRef &PythonRef::operator= (PyObject *obj)
 {
-  Py_XDECREF (mp_obj);
+  if (m_owns_pointer) {
+    Py_XDECREF (mp_obj);
+  }
   mp_obj = obj;
+  m_owns_pointer = true;
   return *this;
 }
 
 PythonRef &PythonRef::operator= (const PythonPtr &ptr)
 {
-  Py_XDECREF (mp_obj);
+  if (m_owns_pointer) {
+    Py_XDECREF (mp_obj);
+  }
   mp_obj = ptr.get ();
   Py_XINCREF (mp_obj);
+  m_owns_pointer = true;
   return *this;
 }
 
 PythonRef &PythonRef::operator= (const PythonRef &other)
 {
   if (this != &other && mp_obj != other.mp_obj) {
-    Py_XDECREF (mp_obj);
+    if (m_owns_pointer) {
+      Py_XDECREF (mp_obj);
+    }
     mp_obj = other.mp_obj;
+    m_owns_pointer = true;
     Py_XINCREF (mp_obj);
   }
   return *this;
 }
 
 PythonRef::PythonRef (const PythonRef &other)
-  : mp_obj (other.mp_obj)
+  : mp_obj (other.mp_obj), m_owns_pointer (true)
 {
   Py_XINCREF (mp_obj);
 }
 
 PythonRef::~PythonRef ()
 {
-  Py_XDECREF (mp_obj);
+  if (m_owns_pointer) {
+    Py_XDECREF (mp_obj);
+  }
 }
 
 PythonRef::operator bool () const
@@ -109,6 +120,12 @@ PyObject *PythonRef::release ()
   return o;
 }
 
+PyObject *PythonRef::release_const () const
+{
+  m_owns_pointer = false;
+  return mp_obj;
+}
+
 // --------------------------------------------------------------------------
 //  PythonPtr implementation
 
diff --git a/src/pya/pya/pyaRefs.h b/src/pya/pya/pyaRefs.h
index 460aa7b37..c04673ae9 100644
--- a/src/pya/pya/pyaRefs.h
+++ b/src/pya/pya/pyaRefs.h
@@ -116,6 +116,14 @@ public:
    */
   PyObject *release ();
 
+  /**
+   *  @brief Takes the pointer, but does not change the value
+   *  This method will stop the reference from managing the object, but
+   *  maintains the pointer. Do not access the pointer after this
+   *  operation.
+   */
+  PyObject *release_const () const;
+
   /**
    *  @brief Comparison operator
    */
@@ -134,6 +142,7 @@ public:
 
 private:
   PyObject *mp_obj;
+  mutable bool m_owns_pointer;
 };
 
 /**
diff --git a/src/pymod/pymodHelper.h b/src/pymod/pymodHelper.h
index 6d116e234..22557ed93 100644
--- a/src/pymod/pymodHelper.h
+++ b/src/pymod/pymodHelper.h
@@ -42,12 +42,7 @@
 static PyObject *
 module_init (const char *pymod_name, const char *mod_name, const char *mod_description)
 {
-  if (! pya::PythonInterpreter::instance ()) {
-    return 0;
-  }
-
-  pya::PythonModule *module = new pya::PythonModule ();
-  pya::PythonInterpreter::instance ()->register_module (module);
+  std::unique_ptr<pya::PythonModule> module (new pya::PythonModule ());
 
   PYA_TRY
   
@@ -59,10 +54,15 @@ module_init (const char *pymod_name, const char *mod_name, const char *mod_descr
     module->init (pymod_name, mod_description);
     module->make_classes (mod_name);
 
-    return module->take_module ();
+    PyObject *mod_object = module->take_module ();
+
+    tl_assert (pya::PythonInterpreter::instance () != 0);
+    pya::PythonInterpreter::instance ()->register_module (module.release ());
+
+    return mod_object;
 
   PYA_CATCH_ANYWHERE
-  
+
   return 0;
 }
 

From d902f5d53d62dc848ef36f8e901af8c0195c59e5 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 8 Sep 2024 18:48:12 +0200
Subject: [PATCH 05/12] Apply property changes before switching tree node in
 properties editor

---
 src/layui/layui/layPropertiesDialog.cc | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/layui/layui/layPropertiesDialog.cc b/src/layui/layui/layPropertiesDialog.cc
index 6ecffff16..573d8c23d 100644
--- a/src/layui/layui/layPropertiesDialog.cc
+++ b/src/layui/layui/layPropertiesDialog.cc
@@ -308,6 +308,25 @@ PropertiesDialog::current_index_changed (const QModelIndex &index, const QModelI
 
   } else {
 
+    if (m_index >= 0 && m_index < int (mp_properties_pages.size ()) && ! mp_properties_pages [m_index]->readonly ()) {
+
+      try {
+
+        db::Transaction t (mp_manager, tl::to_string (QObject::tr ("Apply changes")), m_transaction_id);
+
+        mp_properties_pages [m_index]->apply ();
+
+        if (! t.is_empty ()) {
+          m_transaction_id = t.id ();
+        }
+
+      } catch (...) {
+      }
+
+      mp_properties_pages [m_index]->update ();
+
+    }
+
     if (mp_tree_model->parent (index).isValid ()) {
 
       m_index = mp_tree_model->page_index (index);

From 68d15e01af277c6749c793935ec5f79fff27fb79 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 8 Sep 2024 19:16:40 +0200
Subject: [PATCH 06/12] DRC: more variants for layer#insert - labels, DBU-unit
 types

---
 src/drc/drc/built-in-macros/_drc_layer.rb |  20 +++-
 src/drc/unit_tests/drcSimpleTests.cc      |   5 +
 testdata/drc/drcSimpleTests_123.drc       | 132 ++++++++++++++++++++++
 testdata/drc/drcSimpleTests_123.gds       | Bin 0 -> 1698 bytes
 testdata/drc/drcSimpleTests_au123.gds     | Bin 0 -> 2534 bytes
 5 files changed, 155 insertions(+), 2 deletions(-)
 create mode 100644 testdata/drc/drcSimpleTests_123.drc
 create mode 100644 testdata/drc/drcSimpleTests_123.gds
 create mode 100644 testdata/drc/drcSimpleTests_au123.gds

diff --git a/src/drc/drc/built-in-macros/_drc_layer.rb b/src/drc/drc/built-in-macros/_drc_layer.rb
index 0592c6e35..c384d82c6 100644
--- a/src/drc/drc/built-in-macros/_drc_layer.rb
+++ b/src/drc/drc/built-in-macros/_drc_layer.rb
@@ -42,22 +42,38 @@ module DRC
 
       @engine._context("insert") do
 
-        requires_edges_or_region
         args.each do |a|
           if a.is_a?(RBA::DBox) 
+            requires_edges_or_region
             self.data.insert(RBA::Box::from_dbox(a * (1.0 / @engine.dbu)))
           elsif a.is_a?(RBA::DPolygon) 
+            requires_edges_or_region
             self.data.insert(RBA::Polygon::from_dpoly(a * (1.0 / @engine.dbu)))
           elsif a.is_a?(RBA::DSimplePolygon) 
+            requires_edges_or_region
             self.data.insert(RBA::SimplePolygon::from_dpoly(a * (1.0 / @engine.dbu)))
           elsif a.is_a?(RBA::DPath) 
+            requires_edges_or_region
             self.data.insert(RBA::Path::from_dpath(a * (1.0 / @engine.dbu)))
+          elsif a.is_a?(RBA::Box) || a.is_a?(RBA::Polygon) || a.is_a?(RBA::SimplePolygon) || a.is_a?(RBA::Path)
+            requires_edges_or_region
+            self.data.insert(a)
           elsif a.is_a?(RBA::DEdge) 
+            requires_edges
             self.data.insert(RBA::Edge::from_dedge(a * (1.0 / @engine.dbu)))
+          elsif a.is_a?(RBA::Edge) 
+            requires_edges
+            self.data.insert(a)
+          elsif a.is_a?(RBA::DText) 
+            requires_texts
+            self.data.insert(RBA::CplxTrans::new(@engine.dbu).inverted * a)
+          elsif a.is_a?(RBA::Text) 
+            requires_texts
+            self.data.insert(a)
           elsif a.is_a?(Array)
             insert(*a)
           else
-            raise("Invalid argument type for #{a.inspect}")
+            raise("Invalid argument type of #{a.inspect}")
           end
         end
 
diff --git a/src/drc/unit_tests/drcSimpleTests.cc b/src/drc/unit_tests/drcSimpleTests.cc
index a5403178c..79122f690 100644
--- a/src/drc/unit_tests/drcSimpleTests.cc
+++ b/src/drc/unit_tests/drcSimpleTests.cc
@@ -1954,6 +1954,11 @@ TEST(122_NamedLayers)
   compare_text_files (output, au_output);
 }
 
+TEST(123_DirectInsert)
+{
+  run_test (_this, "123", false);
+}
+
 TEST(130_size_inside_outside)
 {
   run_test (_this, "130", false);
diff --git a/testdata/drc/drcSimpleTests_123.drc b/testdata/drc/drcSimpleTests_123.drc
new file mode 100644
index 000000000..1c1079cf2
--- /dev/null
+++ b/testdata/drc/drcSimpleTests_123.drc
@@ -0,0 +1,132 @@
+
+source $drc_test_source
+target $drc_test_target
+
+lp = polygons
+le = edges
+ll = labels
+
+lp.insert(RBA::Box::new(0, 0, 100, 200))
+lp.insert(RBA::DBox::new(0.200, 0, 0.300, 0.200))
+lp.insert(RBA::Polygon::new(RBA::Box::new(400, 0, 500, 200)))
+lp.insert(RBA::DPolygon::new(RBA::DBox::new(0.600, 0, 0.700, 0.200)))
+lp.insert(RBA::SimplePolygon::new(RBA::Box::new(800, 0, 900, 200)))
+lp.insert(RBA::DSimplePolygon::new(RBA::DBox::new(1.000, 0, 1.100, 0.200)))
+lp.insert(RBA::Path::new([ RBA::Point::new(1200, 100), RBA::Point::new(1300, 100) ], 200))
+lp.insert(RBA::DPath::new([ RBA::DPoint::new(1.400, 0.1), RBA::DPoint::new(1.500, 0.1) ], 0.2))
+
+begin
+  lp.insert(RBA::Edge::new(RBA::Point::new(0, 0), RBA::Point::new(100, 100)))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  lp.insert(RBA::DEdge::new(RBA::DPoint::new(0, 0), RBA::DPoint::new(0.1, 0.1)))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  lp.insert(RBA::Text::new("ABC", RBA::Trans::new(RBA::Vector::new(0, 0))))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  lp.insert(RBA::DText::new("XYZ", RBA::DTrans::new(RBA::DVector::new(0, 0.1))))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+le.insert(RBA::Box::new(0, 0, 100, 200))
+le.insert(RBA::DBox::new(0.200, 0, 0.300, 0.200))
+le.insert(RBA::Polygon::new(RBA::Box::new(400, 0, 500, 200)))
+le.insert(RBA::DPolygon::new(RBA::DBox::new(0.600, 0, 0.700, 0.200)))
+le.insert(RBA::SimplePolygon::new(RBA::Box::new(800, 0, 900, 200)))
+le.insert(RBA::DSimplePolygon::new(RBA::DBox::new(1.000, 0, 1.100, 0.200)))
+le.insert(RBA::Path::new([ RBA::Point::new(1200, 100), RBA::Point::new(1300, 100) ], 200))
+le.insert(RBA::DPath::new([ RBA::DPoint::new(1.400, 0.1), RBA::DPoint::new(1.500, 0.1) ], 0.2))
+
+le.insert(RBA::Edge::new(RBA::Point::new(0, 0), RBA::Point::new(100, 100)))
+le.insert(RBA::DEdge::new(RBA::DPoint::new(0, 0.1), RBA::DPoint::new(0.1, 0.2)))
+
+begin
+  le.insert(RBA::Text::new("ABC", RBA::Trans::new(RBA::Vector::new(0, 0))))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  le.insert(RBA::DText::new("XYZ", RBA::DTrans::new(RBA::DVector::new(0, 0.1))))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::Box::new(0, 0, 100, 200))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::DBox::new(0.200, 0, 0.300, 0.200))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::Polygon::new(RBA::Box::new(400, 0, 500, 200)))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::DPolygon::new(RBA::DBox::new(0.600, 0, 0.700, 0.200)))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::SimplePolygon::new(RBA::Box::new(800, 0, 900, 200)))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::DSimplePolygon::new(RBA::DBox::new(1.000, 0, 1.100, 0.200)))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::Path::new([ RBA::Point::new(1200, 100), RBA::Point::new(1300, 100) ], 200))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::DPath::new([ RBA::DPoint::new(1.400, 0.1), RBA::DPoint::new(1.500, 0.1) ], 0.2))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::Edge::new(RBA::Point::new(0, 0), RBA::Point::new(100, 100)))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+begin
+  ll.insert(RBA::DEdge::new(RBA::DPoint::new(0, 0.1), RBA::DPoint::new(0.1, 0.2)))
+  raise Exception::new("Error expected")
+rescue => ex
+end
+
+ll.insert(RBA::Text::new("ABC", RBA::Trans::new(RBA::Vector::new(100, 0))))
+ll.insert(RBA::DText::new("XYZ", RBA::DTrans::new(RBA::DVector::new(0, 0.1))))
+
+lp.output(1, 0)
+le.output(2, 0)
+ll.output(3, 0)
+
diff --git a/testdata/drc/drcSimpleTests_123.gds b/testdata/drc/drcSimpleTests_123.gds
new file mode 100644
index 0000000000000000000000000000000000000000..3546756f7014657a243871d48ce073152774e619
GIT binary patch
literal 1698
zcma)+PiPZC6vn^ZB%39TnrcyzEqL(YAz;9QDcE9b@DB#ml0&I_E8-ys4<ZE-0u@B?
z&_fS`mp}zU5Ihv|AmT|7Ed~r$y;zW771U5fJP6M>GyA%eq)Xkf`7Lku`{unjGYb!{
zJBh$?)2m1$ft?sc+J1)bdgYVjz(683I(F;OsTcR#!=)z&>UXCq2vYGn&+|$^(g(aP
zE<1Z$T>$%X3H}eSVII8n2vD@HWIyZD*vx$_a(6=ad@e!ssh}JUk6a;YH36Z-c}6`;
zn4KHy4=er08qwz!Md0{uqHn80qTh93G>B;4Q|}wU&9mlCN>8k;t?KzP(+2!I5v|ex
z>5cDsmD6W<o<8pVc8~ky?Qz&68>Ge;Loa`a3x|J!T93X8u<LRU^<2FkM{_I2#PT3|
z@E*AMw40_Y>mD77$8$sryiT3$QgG%qZib!hF)mGI+eMviMpw)|&Z?Ds_y?G17)LY0
zJ5rA<RDq=#doCVX@6tvU0@WcnBlA79x@0^!U)AX;)1=Pa*#oa`Xb+p(BixG}yCSon
zcR_d$aNYe}kx=5ySpC-_a}E?+S>QYGZJ|0R;kd20&H}KvO4OKC<hwz%RE&6V4k$DM
zLwQegBR_pMXQzq2@I5;Xj8ur4_k<Ex?5IcbVLfn-YfvXcvHS~oR*xugX3YG+a-HZ6
z=e^9j>%2g%vQXlhvHG<F(c2kdjeB?QH5`}M(@^4?ae8MPe@}k+U~&yjID>7W#5H5}
zc@VwheVFImFFc1+tO+Hq8LKZ9iGKctQ{r{x`kk|ZP~w_#dfm6H=S}}^-Zpxn#F?@B
Z5A#GX7Xd|8$Ha+<5@%{Xe@l}|`~wsxVif=Y

literal 0
HcmV?d00001

diff --git a/testdata/drc/drcSimpleTests_au123.gds b/testdata/drc/drcSimpleTests_au123.gds
new file mode 100644
index 0000000000000000000000000000000000000000..2fc76cdf65d6c8e3715d67f68a01ce573b4a20cf
GIT binary patch
literal 2534
zcmbW2u};H442GRca!sW|K~zYUI>26u5wRe(uplbI(4lnBL+}7BZ0sy-tc(oIs1Lyd
zWMtq0uzilWoSN3hNy&e${mFMZpA8m_S->bUoeH{8zyb7O=+B`rc6K!d7^M06<ni?S
z^Y#1e^8MuHc{ztszJarLesc?u6##a}KmvBx0Kj-J#kW6J_@l(9)<0Xe-^P<K4N`oP
zv8VO_$hPsx6FDJAO8k$f_5T}q{AN5@kvAWXXT_=5TjQrki~}q3>4W20aVqw9z7n1k
z`E2GX@y_1LXK#*YMLr)mo)xEJZ{_oQ$FmauB|Iq(u`}Ow=Wpcv?Xdv^Mc-w?U`;l}
zP#9ZsM@uy`Er#Nq@k>k5>S&OzTsA7chV!m-S{W<20Wnm)Ldo~-D-3RrHdgblX{qMT
zjG=H}vQi}#-0Bz#&rE87mTFdv7z+30*O%?*6;vB#te`e&sb(dMp>SV*-BawVqSTdY
zr@miY8}YoPW-3=0RM;4*UZMJa1=U#_i)uRVOKQ223MzjLh2i`@vHiS)uAz-Zy@vMX
zdW@E8cBB{z!~L^=QEw^Lbsn9$ce#$lE9|*mnSwnuXzg)#Bs)Mcw3F$D<j`6_@+!%1
Lakmt|4r&I!s_u*D

literal 0
HcmV?d00001


From 680874c8e3fc0a210d6a8c37dfc4749c30530a40 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 8 Sep 2024 19:45:10 +0200
Subject: [PATCH 07/12] More variants of Layout#properties_id + added
 Layout#properties_hash and Layout#properties_array. Clarification of
 documentation.

---
 src/db/db/gsiDeclDbLayout.cc    | 62 +++++++++++++++++++++++++++------
 testdata/ruby/dbLayoutTests2.rb |  7 +++-
 2 files changed, 58 insertions(+), 11 deletions(-)

diff --git a/src/db/db/gsiDeclDbLayout.cc b/src/db/db/gsiDeclDbLayout.cc
index 3edf80298..e3c633f09 100644
--- a/src/db/db/gsiDeclDbLayout.cc
+++ b/src/db/db/gsiDeclDbLayout.cc
@@ -367,9 +367,8 @@ static tl::Variant get_layout_property (const db::Layout *l, const tl::Variant &
   }
 }
 
-static tl::Variant get_layout_properties (const db::Layout *layout)
+static tl::Variant get_properties_hash (const db::Layout *layout, db::properties_id_type id)
 {
-  db::properties_id_type id = layout->prop_id ();
   if (id == 0) {
     return tl::Variant::empty_array ();
   }
@@ -382,6 +381,11 @@ static tl::Variant get_layout_properties (const db::Layout *layout)
   return res;
 }
 
+static tl::Variant get_layout_properties (const db::Layout *layout)
+{
+  return get_properties_hash (layout, layout->prop_id ());
+}
+
 static db::cell_index_type cell_by_name (db::Layout *l, const char *name)
 {
   std::pair<bool, db::cell_index_type> c = l->cell_by_name (name);
@@ -619,6 +623,18 @@ static db::properties_id_type properties_id (db::Layout *layout, const std::vect
   return layout->properties_repository ().properties_id (props);
 }
 
+static db::properties_id_type properties_id_from_hash (db::Layout *layout, const std::map<tl::Variant, tl::Variant> &properties)
+{
+  db::PropertiesRepository::properties_set props;
+
+  for (std::map<tl::Variant, tl::Variant>::const_iterator v = properties.begin (); v != properties.end (); ++v) {
+    db::property_names_id_type name_id = layout->properties_repository ().prop_name_id (v->first);
+    props.insert (std::make_pair (name_id, v->second));
+  }
+
+  return layout->properties_repository ().properties_id (props);
+}
+
 static std::vector<tl::Variant> properties (const db::Layout *layout, db::properties_id_type id)
 {
   std::vector<tl::Variant> ret;
@@ -1300,7 +1316,7 @@ Class<db::Layout> decl_Layout ("db", "Layout",
     "This method has been introduced in version 0.24."
   ) + 
   gsi::method_ext ("property", &get_layout_property, gsi::arg ("key"),
-    "@brief Gets the user property with the given key\n"
+    "@brief Gets the Layout's user property with the given key\n"
     "This method is a convenience method that gets the property with the given key. "
     "If no property with that key exists, it will return nil. Using that method is more "
     "convenient than using the properties ID to retrieve the property value. "
@@ -1308,8 +1324,8 @@ Class<db::Layout> decl_Layout ("db", "Layout",
     "This method has been introduced in version 0.24."
   ) + 
   gsi::method_ext ("properties", &get_layout_properties,
-    "@brief Gets the user properties as a hash\n"
-    "This method is a convenience method that gets all user properties as a single hash.\n"
+    "@brief Gets the Layout's user properties as a hash\n"
+    "This method is a convenience method that gets all user properties of the Layout object as a single hash.\n"
     "\n"
     "This method has been introduced in version 0.29.5."
   ) +
@@ -1324,16 +1340,42 @@ Class<db::Layout> decl_Layout ("db", "Layout",
     "@param properties The array of pairs of variants (both elements can be integer, double or string)\n"
     "@return The unique properties ID for that set"
   ) +
-  gsi::method_ext ("properties", &properties, gsi::arg ("properties_id"),
+  gsi::method_ext ("properties_id", &properties_id_from_hash, gsi::arg ("properties"),
+    "@brief Gets the properties ID for a given properties set\n"
+    "\n"
+    "This variant accepts a hash of value vs. key for the properties instead of array of key/value pairs. "
+    "Apart from this, it behaves like the other \\properties_id variant.\n"
+    "\n"
+    "@param properties A hash of property keys/values (both keys and values can be integer, double or string)\n"
+    "@return The unique properties ID for that set\n"
+    "\n"
+    "This variant has been introduced in version 0.29.7."
+  ) +
+  gsi::method_ext ("properties_array|#properties", &properties, gsi::arg ("properties_id"),
     "@brief Gets the properties set for a given properties ID\n"
     "\n"
-    "Basically performs the backward conversion of the 'properties_id' method. "
-    "Given a properties ID, returns the properties set as an array of pairs of "
-    "variants. In this array, each key and the value are stored as pairs (arrays with two elements).\n"
+    "Basically this method performs the backward conversion of the 'properties_id' method. "
+    "Given a properties ID, it returns the properties set as an array. "
+    "In this array, each key and the value is stored as a pair (an array with two elements).\n"
     "If the properties ID is not valid, an empty array is returned.\n"
+    "A version that returns a hash instead of pairs of key/values, is \\properties_hash.\n"
     "\n"
     "@param properties_id The properties ID to get the properties for\n"
-    "@return The array of variants (see \\properties_id)\n"
+    "@return An array of key/value pairs (see \\properties_id)\n"
+    "\n"
+    "The 'properties_array' alias was introduced in version 0.29.7 and the plain 'properties' alias was deprecated."
+  ) +
+  gsi::method_ext ("properties_hash", &get_properties_hash, gsi::arg ("properties_id"),
+    "@brief Gets the properties set for a given properties ID as a hash\n"
+    "\n"
+    "Returns the properties for a given properties ID as a hash.\n"
+    "It is a convenient alternative to \\properties_array, which returns "
+    "an array of key/value pairs.\n"
+    "\n"
+    "@param properties_id The properties ID to get the properties for\n"
+    "@return The hash representing the properties for the given ID (values vs. key)\n"
+    "\n"
+    "This method has been introduced in version 0.29.7."
   ) +
   gsi::method ("unique_cell_name", &db::Layout::uniquify_cell_name, gsi::arg ("name"),
     "@brief Creates a new unique cell name from the given name\n"
diff --git a/testdata/ruby/dbLayoutTests2.rb b/testdata/ruby/dbLayoutTests2.rb
index 111d0e9f8..3985738d9 100644
--- a/testdata/ruby/dbLayoutTests2.rb
+++ b/testdata/ruby/dbLayoutTests2.rb
@@ -489,7 +489,8 @@ class DBLayoutTests2_TestClass < TestBase
   def test_6_Layout_props
 
     ly = RBA::Layout::new
-    pid = ly.properties_id( { 17 => "a", "b" => [ 1, 5, 7 ] }.to_a )
+    pid = ly.properties_id({ 17 => "a", "b" => [ 1, 5, 7 ] }.to_a)
+    assert_equal(ly.properties_id({ 17 => "a", "b" => [ 1, 5, 7 ] }), pid)
 
     ci1 = ly.add_cell( "c1" )
     ci2 = ly.add_cell( "c2" )
@@ -1051,6 +1052,10 @@ class DBLayoutTests2_TestClass < TestBase
     assert_equal(ly.prop_id, 1)
     assert_equal(ly.property("x"), 1)
     assert_equal(ly.properties, {"x" => 1})
+    assert_equal(ly.properties_hash(ly.prop_id), {"x" => 1})
+    assert_equal(ly.properties_id(ly.properties_hash(ly.prop_id)), ly.prop_id)
+    assert_equal(ly.properties_array(ly.prop_id), [["x", 1]])
+    assert_equal(ly.properties_id(ly.properties_array(ly.prop_id)), ly.prop_id)
     ly.set_property("x", 17)
     assert_equal(ly.prop_id, 2)
     assert_equal(ly.property("x"), 17)

From 153d1e354133b6313ff827908a0008861c0a9f2b Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Fri, 13 Sep 2024 14:31:45 +0200
Subject: [PATCH 08/12] Trying to improve stability under PCell debugging
 conditions inside the IDE

---
 src/db/db/dbLayout.cc       |  3 +++
 src/db/db/dbLibraryProxy.cc | 19 ++++++++++---------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/db/db/dbLayout.cc b/src/db/db/dbLayout.cc
index d31cc36ac..b69ca8d73 100644
--- a/src/db/db/dbLayout.cc
+++ b/src/db/db/dbLayout.cc
@@ -2876,6 +2876,9 @@ Layout::get_context_info (cell_index_type cell_index, LayoutOrCellContextInfo &i
 
       //  one level of library indirection
       ly = &lib->layout ();
+      if (! ly->is_valid_cell_index (lib_proxy->library_cell_index ())) {
+        return any_meta; //  abort
+      }
       cptr = &ly->cell (lib_proxy->library_cell_index ());
       info.lib_name = lib->get_name ();
 
diff --git a/src/db/db/dbLibraryProxy.cc b/src/db/db/dbLibraryProxy.cc
index 1bb78a8a7..ff86a6f5e 100644
--- a/src/db/db/dbLibraryProxy.cc
+++ b/src/db/db/dbLibraryProxy.cc
@@ -132,6 +132,7 @@ LibraryProxy::get_layer_indices (db::Layout &layout, db::ImportLayerMapping *lay
 
   Library *lib = LibraryManager::instance ().lib (lib_id ());
   tl_assert (lib != 0);
+  tl_assert (lib->layout ().is_valid_cell_index (library_cell_index ()));
 
   const db::Cell &cell = lib->layout ().cell (library_cell_index ());
 
@@ -247,11 +248,11 @@ LibraryProxy::get_basic_name () const
 {
   Library *lib = LibraryManager::instance ().lib (lib_id ());
   if (lib) {
-    const db::Cell *lib_cell = &lib->layout ().cell (library_cell_index ());
-    if (! lib_cell) {
+    if (! lib->layout ().is_valid_cell_index (library_cell_index ())) {
       return "<defunct>";
     } else {
-      return lib_cell->get_basic_name ();
+      const db::Cell &lib_cell = lib->layout ().cell (library_cell_index ());
+      return lib_cell.get_basic_name ();
     }
   } else {
     return Cell::get_basic_name ();
@@ -263,11 +264,11 @@ LibraryProxy::get_display_name () const
 {
   Library *lib = LibraryManager::instance ().lib (lib_id ());
   if (lib) {
-    const db::Cell *lib_cell = &lib->layout ().cell (library_cell_index ());
-    if (! lib_cell) {
+    if (! lib->layout ().is_valid_cell_index (library_cell_index ())) {
       return lib->get_name () + "." + "<defunct>";
     } else {
-      return lib->get_name () + "." + lib_cell->get_display_name ();
+      const db::Cell &lib_cell = lib->layout ().cell (library_cell_index ());
+      return lib->get_name () + "." + lib_cell.get_display_name ();
     }
   } else {
     return Cell::get_display_name ();
@@ -279,11 +280,11 @@ LibraryProxy::get_qualified_name () const
 {
   Library *lib = LibraryManager::instance ().lib (lib_id ());
   if (lib) {
-    const db::Cell *lib_cell = &lib->layout ().cell (library_cell_index ());
-    if (! lib_cell) {
+    if (! lib->layout ().is_valid_cell_index (library_cell_index ())) {
       return lib->get_name () + "." + "<defunct>";
     } else {
-      return lib->get_name () + "." + lib_cell->get_qualified_name ();
+      const db::Cell &lib_cell = lib->layout ().cell (library_cell_index ());
+      return lib->get_name () + "." + lib_cell.get_qualified_name ();
     }
   } else {
     return Cell::get_qualified_name ();

From 56e88805a0545e547f163b6d5740614fa0bedf61 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 15 Sep 2024 17:15:21 +0200
Subject: [PATCH 09/12] Refusing to write files with empty string and to a path
 that is a directory - the latter would rename the directory and later try to
 remove it

---
 src/tl/tl/tlStream.cc              | 34 +++++++++++++++++++++---------
 src/tl/unit_tests/tlStreamTests.cc | 15 +++++++++++++
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/src/tl/tl/tlStream.cc b/src/tl/tl/tlStream.cc
index 751c4841d..f83be7bfb 100644
--- a/src/tl/tl/tlStream.cc
+++ b/src/tl/tl/tlStream.cc
@@ -1178,20 +1178,34 @@ OutputStream::seek (size_t pos)
 OutputFileBase::OutputFileBase (const std::string &p, int keep_backups)
   : m_keep_backups (keep_backups), m_path (tl::absolute_file_path (p)), m_has_error (false)
 {
+  if (p.empty ()) {
+    throw tl::Exception (tl::to_string (tr ("Path cannot be an empty string")));
+  }
+
   if (tl::file_exists (m_path)) {
-    m_backup_path = m_path + ".~backup";
-    if (tl::file_exists (m_backup_path)) {
-      if (! tl::rm_file (m_backup_path)) {
-        tl::warn << tl::sprintf (tl::to_string (tr ("Could not create backup file: unable to remove existing file '%s'")), m_backup_path);
-        m_backup_path = std::string ();
+
+    if (tl::is_dir (m_path)) {
+
+      throw tl::Exception (tl::to_string (tr ("Path exists and is a directory: '%s'")), m_path);
+
+    } else {
+
+      m_backup_path = m_path + ".~backup";
+      if (tl::file_exists (m_backup_path)) {
+        if (! tl::rm_file (m_backup_path)) {
+          tl::warn << tl::sprintf (tl::to_string (tr ("Could not create backup file: unable to remove existing file '%s'")), m_backup_path);
+          m_backup_path = std::string ();
+        }
       }
-    }
-    if (! m_backup_path.empty ()) {
-      if (! tl::rename_file (m_path, tl::filename (m_backup_path))) {
-        tl::warn << tl::sprintf (tl::to_string (tr ("Could not create backup file: unable to rename original file '%s' to backup file")), m_path, m_backup_path);
-        m_backup_path = std::string ();
+      if (! m_backup_path.empty ()) {
+        if (! tl::rename_file (m_path, tl::filename (m_backup_path))) {
+          tl::warn << tl::sprintf (tl::to_string (tr ("Could not create backup file: unable to rename original file '%s' to backup file")), m_path, m_backup_path);
+          m_backup_path = std::string ();
+        }
       }
+
     }
+
   }
 }
 
diff --git a/src/tl/unit_tests/tlStreamTests.cc b/src/tl/unit_tests/tlStreamTests.cc
index 517d71615..156f276c2 100644
--- a/src/tl/unit_tests/tlStreamTests.cc
+++ b/src/tl/unit_tests/tlStreamTests.cc
@@ -462,5 +462,20 @@ TEST(Backups)
   }
 }
 
+TEST(RefuseToWrite)
+{
+  try {
+    tl::OutputStream os ("");
+    EXPECT_EQ (1, 0);
+  } catch (tl::Exception &ex) {
+    EXPECT_EQ (ex.msg (), "Path cannot be an empty string");
+  }
 
+  try {
+    tl::OutputStream os (".");
+    EXPECT_EQ (1, 0);
+  } catch (tl::Exception &ex) {
+    EXPECT_EQ (ex.msg ().find ("Path exists and is a directory"), size_t (0));
+  }
+}
 

From 5aa67342e9edda520ad678e0dae2b04ee4d9bb2c Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 15 Sep 2024 19:11:47 +0200
Subject: [PATCH 10/12] Adding performance compare script for comparing ut runs
 by time

---
 scripts/compare_ut_performance.rb | 191 ++++++++++++++++++++++++++++++
 1 file changed, 191 insertions(+)
 create mode 100755 scripts/compare_ut_performance.rb

diff --git a/scripts/compare_ut_performance.rb b/scripts/compare_ut_performance.rb
new file mode 100755
index 000000000..aab8820f6
--- /dev/null
+++ b/scripts/compare_ut_performance.rb
@@ -0,0 +1,191 @@
+#!/bin/ruby 
+
+require "nokogiri"
+
+# Collect files from command line
+
+files = []
+time_class = :wall
+sort_key = :name
+
+ARGV.each do |arg|
+  if arg =~ /^--help|-h/
+    puts <<"END"
+#{$0} [options] <file1> <file2> ...
+
+The files are XML files produced by "ut_runner" with the -a option.
+
+Options are:
+
+  -w     Use wall time (default)
+  -u     Use user time
+  -s     Sort by average time, lowest first
+  +s     Sort by average time, largest first
+
+The script reads these files are compares performance (user and wall times)
+of the different tests.
+END
+    exit(0)
+  elsif arg == "-w"
+    time_class = :wall
+  elsif arg == "-u"
+    time_class = :user
+  elsif arg == "-s"
+    sort_key = :time_up
+  elsif arg == "+s"
+    sort_key = :time_down
+  elsif arg =~ /^-/
+    puts("*** ERROR: unknown option #{arg}. Use -h for help.")
+    exit(1)
+  else
+    files << arg
+  end
+end
+
+
+# A class representing the data from one test
+
+class TestData
+
+  def initialize(file)
+
+    @file = file
+    @data = {}
+
+    File.open(file) do |f|
+
+      doc = Nokogiri::XML(f)
+
+      doc.xpath("//testsuite").each do |testsuite|
+        ts_name = testsuite.at_xpath("@name").content
+        testsuite.xpath("testcase").each do |testcase|
+          tc_name = testcase.at_xpath("@name").content
+          times = testcase.at_xpath("x-testcase-times")
+          if times
+            wall_time = times.at_xpath("@wall").content.to_f
+            user_time = times.at_xpath("@user").content.to_f
+            @data[ [ts_name, tc_name] ] = [ wall_time, user_time ]
+          end
+        end
+      end
+
+    end
+
+  end
+
+  def file
+    @file
+  end
+
+  def keys
+    @data.keys
+  end
+
+  def times(key)
+    @data[key]
+  end
+
+end 
+
+
+# Read the tests
+
+tests = []
+files.each do |f|
+  puts("Reading test file #{f} ..")
+  tests << TestData::new(f)
+end
+
+puts "Reading done."
+puts ""
+
+
+# Build the comparison table
+
+all_tests = {}
+
+tests.each_with_index do |test,index|
+  test.keys.each do |k|
+    all_tests[k] ||= [nil] * tests.size
+    all_tests[k][index] = test.times(k)
+  end
+end
+
+
+# print the result
+
+tests.each_with_index do |test,index|
+  puts "(#{index + 1}) #{test.file}"
+end
+
+puts ""
+
+time_index = 0
+if time_class == :wall
+  puts "Wall times"
+elsif time_class == :user
+  time_index = 1
+  puts "User times"
+end
+
+puts ""
+
+l1 = all_tests.keys.collect { |k| k[0].size }.max
+l2 = all_tests.keys.collect { |k| k[1].size }.max
+
+fmt = "%-#{l1}s  %-#{l2}s  " + (["%15s"] * tests.size).join(" ") + " %15s %15s %10s"
+
+title = fmt % ([ "Testsuite", "Test", ] + tests.each_with_index.collect { |t,i| "(#{i + 1})" } + [ "Min", "Max", "Delta" ])
+puts title
+puts "-" * title.size
+
+total = [0.0] * tests.size
+
+lines = []
+
+all_tests.keys.sort { |a,b| a <=> b }.each do |k|
+
+  times = all_tests[k].collect { |t| t && t[time_index] }
+
+  min = max = delta = nil
+  if ! times.index(nil)
+    times.each_with_index do |t,i|
+      total[i] += t
+    end
+    min = times.min
+    max = times.max
+    if times.size > 1 && (max + min).abs > 1.0
+      delta = (max - min) / 0.5 / (max + min) 
+    end
+  end
+
+  line = fmt % (k + times.collect { |t| t ? ("%.6f" % t) : "" } + [ min ? "%.6f" % min : "", max ? "%.6f" % max : "", delta ? "%.2f%%" % (delta * 100) : ""])
+
+  if sort_key == :time_up
+    lines << [ min && max ? min + max : 0.0, line ]
+  elsif sort_key == :time_down
+    lines << [ min && max ? -(min + max) : 0.0, line ]
+  else
+    lines << [ k, line ]
+  end
+
+end  
+
+lines.sort { |a,b| a[0] <=> b[0] }.each do |k,line|
+  puts line
+end
+
+
+# Add total row
+
+min = total.min
+max = total.max
+delta = nil
+if total.size > 1 && (max + min).abs > 1.0
+  delta = (max - min) / 0.5 / (max + min) 
+end
+
+puts ""
+puts fmt % ([ "Total" , "" ] + total.collect { |t| t ? ("%.6f" % t) : "" } + [ min ? "%.6f" % min : "", max ? "%.6f" % max : "", delta ? "%.2f%%" % (delta * 100) : ""])
+
+

From ca9b1d779d5483a7699375b079e0bef145fe0729 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Mon, 16 Sep 2024 23:48:30 +0200
Subject: [PATCH 11/12] Print file name before warning from stream readers,
 compress warnings if necessary, add file name to errors

---
 src/db/db/dbReader.cc                         | 35 ++++++++++++++++++-
 src/db/db/dbReader.h                          | 15 ++++++++
 .../streamers/cif/db_plugin/dbCIFReader.cc    | 20 +++++++----
 .../streamers/cif/db_plugin/dbCIFReader.h     |  4 +--
 .../streamers/dxf/db_plugin/dbDXFReader.cc    | 34 +++++++++++-------
 .../streamers/dxf/db_plugin/dbDXFReader.h     |  8 ++---
 .../db_plugin/contrib/dbGDS2TextReader.cc     | 20 +++++++----
 .../gds2/db_plugin/contrib/dbGDS2TextReader.h |  4 +--
 .../streamers/gds2/db_plugin/dbGDS2Reader.cc  | 22 ++++++++----
 .../streamers/gds2/db_plugin/dbGDS2Reader.h   |  4 +--
 .../streamers/magic/db_plugin/dbMAGReader.cc  | 18 +++++++---
 .../oasis/db_plugin/dbOASISReader.cc          | 24 +++++++++----
 .../streamers/oasis/db_plugin/dbOASISReader.h |  4 +--
 13 files changed, 156 insertions(+), 56 deletions(-)

diff --git a/src/db/db/dbReader.cc b/src/db/db/dbReader.cc
index a5b10cd16..b08e14962 100644
--- a/src/db/db/dbReader.cc
+++ b/src/db/db/dbReader.cc
@@ -61,7 +61,7 @@ join_layer_names (std::string &s, const std::string &n)
 //  ReaderBase implementation
 
 ReaderBase::ReaderBase () 
-  : m_warnings_as_errors (false), m_warn_level (1)
+  : m_warnings_as_errors (false), m_warn_level (1), m_warn_count_for_same_message (0), m_first_warning (true)
 { 
 }
 
@@ -79,6 +79,39 @@ void
 ReaderBase::init (const db::LoadLayoutOptions &options)
 {
   m_warn_level = options.warn_level ();
+  m_last_warning.clear ();
+  m_warn_count_for_same_message = 0;
+  m_first_warning = true;
+}
+
+bool
+ReaderBase::first_warning ()
+{
+  bool f = m_first_warning;
+  m_first_warning = false;
+  return f;
+}
+
+int
+ReaderBase::compress_warning (const std::string &msg)
+{
+  const int max_warnings = 10;
+
+  if (! msg.empty () && msg == m_last_warning) {
+    if (m_warn_count_for_same_message < max_warnings) {
+      ++m_warn_count_for_same_message;
+      return -1;
+    } else if (m_warn_count_for_same_message == max_warnings) {
+      ++m_warn_count_for_same_message;
+      return 0;
+    } else {
+      return 1;
+    }
+  } else {
+    m_last_warning = msg;
+    m_warn_count_for_same_message = 0;
+    return -1;
+  }
 }
 
 // ---------------------------------------------------------------
diff --git a/src/db/db/dbReader.h b/src/db/db/dbReader.h
index dc278e19f..7d2b1013a 100644
--- a/src/db/db/dbReader.h
+++ b/src/db/db/dbReader.h
@@ -126,12 +126,27 @@ public:
     return m_warn_level;
   }
 
+  /**
+   *  @brief Returns true (once) if this is the first warning
+   */
+  bool first_warning ();
+
+  /**
+   *  @brief Returns a value indicating whether to compress the given warning
+   *
+   *  The return value is either -1 (do not skip), 0 (first warning not to be shown), 1 (warning not shown(.
+   */
+  int compress_warning (const std::string &msg);
+
 protected:
   virtual void init (const db::LoadLayoutOptions &options);
 
 private:
   bool m_warnings_as_errors;
   int m_warn_level;
+  std::string m_last_warning;
+  int m_warn_count_for_same_message;
+  bool m_first_warning;
 };
 
 /**
diff --git a/src/plugins/streamers/cif/db_plugin/dbCIFReader.cc b/src/plugins/streamers/cif/db_plugin/dbCIFReader.cc
index 00ce57f9a..d51a94a19 100644
--- a/src/plugins/streamers/cif/db_plugin/dbCIFReader.cc
+++ b/src/plugins/streamers/cif/db_plugin/dbCIFReader.cc
@@ -86,7 +86,7 @@ CIFReader::read (db::Layout &layout)
 void 
 CIFReader::error (const std::string &msg)
 {
-  throw CIFReaderException (msg, m_stream.line_number (), m_cellname);
+  throw CIFReaderException (msg, m_stream.line_number (), m_cellname, m_stream.source ());
 }
 
 void 
@@ -96,11 +96,19 @@ CIFReader::warn (const std::string &msg, int wl)
     return;
   }
 
-  // TODO: compress
-  tl::warn << msg 
-           << tl::to_string (tr (" (line=")) << m_stream.line_number ()
-           << tl::to_string (tr (", cell=")) << m_cellname
-           << ")";
+  if (first_warning ()) {
+    tl::warn << tl::sprintf (tl::to_string (tr ("In file %s:")), m_stream.source ());
+  }
+
+  int ws = compress_warning (msg);
+  if (ws < 0) {
+    tl::warn << msg
+             << tl::to_string (tr (" (line=")) << m_stream.line_number ()
+             << tl::to_string (tr (", cell=")) << m_cellname
+             << ")";
+  } else if (ws == 0) {
+    tl::warn << tl::to_string (tr ("... further warnings of this kind are not shown"));
+  }
 }
 
 /**
diff --git a/src/plugins/streamers/cif/db_plugin/dbCIFReader.h b/src/plugins/streamers/cif/db_plugin/dbCIFReader.h
index 556b7a0a3..a2b39e47c 100644
--- a/src/plugins/streamers/cif/db_plugin/dbCIFReader.h
+++ b/src/plugins/streamers/cif/db_plugin/dbCIFReader.h
@@ -52,8 +52,8 @@ class DB_PLUGIN_PUBLIC CIFReaderException
   : public ReaderException
 {
 public:
-  CIFReaderException (const std::string &msg, size_t l, const std::string &cell)
-    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (line=%ld, cell=%s)")), msg, l, cell))
+  CIFReaderException (const std::string &msg, size_t l, const std::string &cell, const std::string &source)
+    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (line=%ld, cell=%s), in file: %s")), msg, l, cell, source))
   { }
 };
 
diff --git a/src/plugins/streamers/dxf/db_plugin/dbDXFReader.cc b/src/plugins/streamers/dxf/db_plugin/dbDXFReader.cc
index a5c4b81ed..d93975fa4 100644
--- a/src/plugins/streamers/dxf/db_plugin/dbDXFReader.cc
+++ b/src/plugins/streamers/dxf/db_plugin/dbDXFReader.cc
@@ -367,9 +367,9 @@ void
 DXFReader::error (const std::string &msg)
 {
   if (m_ascii) {
-    throw DXFReaderException (msg, m_line_number, m_cellname);
+    throw DXFReaderException (msg, m_line_number, m_cellname, m_stream.source ());
   } else {
-    throw DXFReaderException (msg, m_stream.pos (), m_cellname);
+    throw DXFReaderException (msg, m_stream.pos (), m_cellname, m_stream.source ());
   }
 }
 
@@ -380,17 +380,25 @@ DXFReader::warn (const std::string &msg, int wl)
     return;
   }
 
-  // TODO: compress
-  if (m_ascii) {
-    tl::warn << msg 
-             << tl::to_string (tr (" (line=")) << m_line_number
-             << tl::to_string (tr (", cell=")) << m_cellname
-             << ")";
-  } else {
-    tl::warn << msg 
-             << tl::to_string (tr (" (position=")) << m_stream.pos ()
-             << tl::to_string (tr (", cell=")) << m_cellname
-             << ")";
+  if (first_warning ()) {
+    tl::warn << tl::sprintf (tl::to_string (tr ("In file %s:")), m_stream.source ());
+  }
+
+  int ws = compress_warning (msg);
+  if (ws < 0) {
+    if (m_ascii) {
+      tl::warn << msg
+               << tl::to_string (tr (" (line=")) << m_line_number
+               << tl::to_string (tr (", cell=")) << m_cellname
+               << ")";
+    } else {
+      tl::warn << msg
+               << tl::to_string (tr (" (position=")) << m_stream.pos ()
+               << tl::to_string (tr (", cell=")) << m_cellname
+               << ")";
+    }
+  } else if (ws == 0) {
+    tl::warn << tl::to_string (tr ("... further warnings of this kind are not shown"));
   }
 }
 
diff --git a/src/plugins/streamers/dxf/db_plugin/dbDXFReader.h b/src/plugins/streamers/dxf/db_plugin/dbDXFReader.h
index c2d5d4d76..4aab23cdd 100644
--- a/src/plugins/streamers/dxf/db_plugin/dbDXFReader.h
+++ b/src/plugins/streamers/dxf/db_plugin/dbDXFReader.h
@@ -53,12 +53,12 @@ class DB_PLUGIN_PUBLIC DXFReaderException
   : public ReaderException
 {
 public:
-  DXFReaderException (const std::string &msg, size_t p, const std::string &cell)
-    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (position=%ld, cell=%s)")), msg.c_str (), p, cell))
+  DXFReaderException (const std::string &msg, size_t p, const std::string &cell, const std::string &source)
+    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (position=%ld, cell=%s), in file: %s")), msg.c_str (), p, cell, source))
   { }
 
-  DXFReaderException (const std::string &msg, int line, const std::string &cell)
-    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (line=%d, cell=%s)")), msg.c_str (), line, cell))
+  DXFReaderException (const std::string &msg, int line, const std::string &cell, const std::string &source)
+    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (line=%d, cell=%s), in file: %s")), msg.c_str (), line, cell, source))
   { }
 };
 
diff --git a/src/plugins/streamers/gds2/db_plugin/contrib/dbGDS2TextReader.cc b/src/plugins/streamers/gds2/db_plugin/contrib/dbGDS2TextReader.cc
index 1fcc0605c..85bb4f8ef 100644
--- a/src/plugins/streamers/gds2/db_plugin/contrib/dbGDS2TextReader.cc
+++ b/src/plugins/streamers/gds2/db_plugin/contrib/dbGDS2TextReader.cc
@@ -298,7 +298,7 @@ GDS2ReaderText::path () const
 void 
 GDS2ReaderText::error (const std::string &msg)
 {
-  throw GDS2ReaderTextException (msg, int(sStream.line_number()), cellname().c_str ());
+  throw GDS2ReaderTextException (msg, int (sStream.line_number()), cellname ().c_str (), sStream.source ());
 }
 
 void 
@@ -308,11 +308,19 @@ GDS2ReaderText::warn (const std::string &msg, int wl)
     return;
   }
 
-  // TODO: compress
-  tl::warn << msg 
-           << tl::to_string (tr (", line number=")) << sStream.line_number()
-           << tl::to_string (tr (", cell=")) << cellname ().c_str ()
-           << ")";
+  if (first_warning ()) {
+    tl::warn << tl::sprintf (tl::to_string (tr ("In file %s:")), sStream.source ());
+  }
+
+  int ws = compress_warning (msg);
+  if (ws < 0) {
+    tl::warn << msg
+             << tl::to_string (tr (", line number=")) << sStream.line_number()
+             << tl::to_string (tr (", cell=")) << cellname ().c_str ()
+             << ")";
+  } else if (ws == 0) {
+    tl::warn << tl::to_string (tr ("... further warnings of this kind are not shown"));
+  }
 }
 
 void 
diff --git a/src/plugins/streamers/gds2/db_plugin/contrib/dbGDS2TextReader.h b/src/plugins/streamers/gds2/db_plugin/contrib/dbGDS2TextReader.h
index 6ac543101..2c40e18a2 100644
--- a/src/plugins/streamers/gds2/db_plugin/contrib/dbGDS2TextReader.h
+++ b/src/plugins/streamers/gds2/db_plugin/contrib/dbGDS2TextReader.h
@@ -39,8 +39,8 @@ class DB_PLUGIN_PUBLIC GDS2ReaderTextException
   : public ReaderException 
 {
 public:
-  GDS2ReaderTextException (const std::string &msg, size_t n, const std::string &cell)
-    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (line number=%ld, cell=%s)")).c_str (), msg.c_str (),  n, cell.c_str ()))
+  GDS2ReaderTextException (const std::string &msg, size_t n, const std::string &cell, const std::string &source)
+    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (line number=%ld, cell=%s), in file: %s")).c_str (), msg.c_str (),  n, cell.c_str (), source))
   { }
 };
 
diff --git a/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.cc b/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.cc
index 3582bf32a..bd782a465 100644
--- a/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.cc
+++ b/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.cc
@@ -282,7 +282,7 @@ GDS2Reader::path () const
 void 
 GDS2Reader::error (const std::string &msg)
 {
-  throw GDS2ReaderException (msg, m_stream.pos (), m_recnum, cellname ().c_str ());
+  throw GDS2ReaderException (msg, m_stream.pos (), m_recnum, cellname ().c_str (), m_stream.source ());
 }
 
 void 
@@ -292,12 +292,20 @@ GDS2Reader::warn (const std::string &msg, int wl)
     return;
   }
 
-  // TODO: compress
-  tl::warn << msg 
-           << tl::to_string (tr (" (position=")) << m_stream.pos ()
-           << tl::to_string (tr (", record number=")) << m_recnum
-           << tl::to_string (tr (", cell=")) << cellname ().c_str ()
-           << ")";
+  if (first_warning ()) {
+    tl::warn << tl::sprintf (tl::to_string (tr ("In file %s:")), m_stream.source ());
+  }
+
+  int ws = compress_warning (msg);
+  if (ws < 0) {
+    tl::warn << msg
+             << tl::to_string (tr (" (position=")) << m_stream.pos ()
+             << tl::to_string (tr (", record number=")) << m_recnum
+             << tl::to_string (tr (", cell=")) << cellname ().c_str ()
+             << ")";
+  } else if (ws == 0) {
+    tl::warn << tl::to_string (tr ("... further warnings of this kind are not shown"));
+  }
 }
 
 }
diff --git a/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.h b/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.h
index fe9ad2883..e7975c0be 100644
--- a/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.h
+++ b/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.h
@@ -48,8 +48,8 @@ class DB_PLUGIN_PUBLIC GDS2ReaderException
   : public ReaderException 
 {
 public:
-  GDS2ReaderException (const std::string &msg, size_t p, size_t n, const std::string &cell)
-    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (position=%ld, record number=%ld, cell=%s)")), msg, p, n, cell))
+  GDS2ReaderException (const std::string &msg, size_t p, size_t n, const std::string &cell, const std::string &source)
+    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (position=%ld, record number=%ld, cell=%s), in file: %s")), msg, p, n, cell, source))
   { }
 };
 
diff --git a/src/plugins/streamers/magic/db_plugin/dbMAGReader.cc b/src/plugins/streamers/magic/db_plugin/dbMAGReader.cc
index b56e7916e..05841c10e 100644
--- a/src/plugins/streamers/magic/db_plugin/dbMAGReader.cc
+++ b/src/plugins/streamers/magic/db_plugin/dbMAGReader.cc
@@ -143,11 +143,19 @@ MAGReader::warn (const std::string &msg, int wl)
     return;
   }
 
-  // TODO: compress
-  tl::warn << msg 
-           << tl::to_string (tr (" (line=")) << mp_current_stream->line_number ()
-           << tl::to_string (tr (", file=")) << mp_current_stream->source ()
-           << ")";
+  if (first_warning ()) {
+    tl::warn << tl::sprintf (tl::to_string (tr ("In file %s:")), mp_current_stream->source ());
+  }
+
+  int ws = compress_warning (msg);
+  if (ws < 0) {
+    tl::warn << msg
+             << tl::to_string (tr (" (line=")) << mp_current_stream->line_number ()
+             << tl::to_string (tr (", file=")) << mp_current_stream->source ()
+             << ")";
+  } else if (ws == 0) {
+    tl::warn << tl::to_string (tr ("... further warnings of this kind are not shown"));
+  }
 }
 
 db::cell_index_type
diff --git a/src/plugins/streamers/oasis/db_plugin/dbOASISReader.cc b/src/plugins/streamers/oasis/db_plugin/dbOASISReader.cc
index 04e312470..efaead612 100644
--- a/src/plugins/streamers/oasis/db_plugin/dbOASISReader.cc
+++ b/src/plugins/streamers/oasis/db_plugin/dbOASISReader.cc
@@ -482,7 +482,7 @@ OASISReader::get_gdelta (long grid)
 void
 OASISReader::error (const std::string &msg)
 {
-  throw OASISReaderException (msg, m_stream.pos (), m_cellname.c_str ());
+  throw OASISReaderException (msg, m_stream.pos (), m_cellname.c_str (), m_stream.source ());
 }
 
 void
@@ -493,13 +493,25 @@ OASISReader::warn (const std::string &msg, int wl)
   }
 
   if (warnings_as_errors ()) {
+
     error (msg);
+
   } else {
-    // TODO: compress
-    tl::warn << msg
-             << tl::to_string (tr (" (position=")) << m_stream.pos ()
-             << tl::to_string (tr (", cell=")) << m_cellname
-             << ")";
+
+    if (first_warning ()) {
+      tl::warn << tl::sprintf (tl::to_string (tr ("In file %s:")), m_stream.source ());
+    }
+
+    int ws = compress_warning (msg);
+    if (ws < 0) {
+      tl::warn << msg
+               << tl::to_string (tr (" (position=")) << m_stream.pos ()
+               << tl::to_string (tr (", cell=")) << m_cellname
+               << ")";
+    } else if (ws == 0) {
+      tl::warn << tl::to_string (tr ("... further warnings of this kind are not shown"));
+    }
+
   }
 }
 
diff --git a/src/plugins/streamers/oasis/db_plugin/dbOASISReader.h b/src/plugins/streamers/oasis/db_plugin/dbOASISReader.h
index 2f7c372e4..720da38ca 100644
--- a/src/plugins/streamers/oasis/db_plugin/dbOASISReader.h
+++ b/src/plugins/streamers/oasis/db_plugin/dbOASISReader.h
@@ -54,8 +54,8 @@ class DB_PLUGIN_PUBLIC OASISReaderException
   : public ReaderException
 {
 public:
-  OASISReaderException (const std::string &msg, size_t p, const std::string &cell)
-    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (position=%ld, cell=%s)")), msg, p, cell))
+  OASISReaderException (const std::string &msg, size_t p, const std::string &cell, const std::string &source)
+    : ReaderException (tl::sprintf (tl::to_string (tr ("%s (position=%ld, cell=%s), in file: %s")), msg, p, cell, source))
   { }
 };
 

From c6b2dba8473cb28a485b35f4bbb65ea3596a63f5 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Wed, 18 Sep 2024 23:53:51 +0200
Subject: [PATCH 12/12] Updating tests

---
 .../streamers/oasis/unit_tests/dbOASISReaderTests.cc        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc b/src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc
index a519ff1a7..a6b9359b0 100644
--- a/src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc
+++ b/src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc
@@ -127,7 +127,7 @@ run_test_error (tl::TestBase *_this, const char *test, const char *msg_au)
     error = true;
   }
   EXPECT_EQ (error, true)
-  EXPECT_EQ (msg, msg_au)
+  EXPECT_EQ (msg.find (msg_au), size_t (0));
 }
 
 TEST(1_1)
@@ -635,7 +635,7 @@ TEST(Bug_1474)
     //  Seen when private test data is not installed
     throw;
   } catch (tl::Exception &ex) {
-    EXPECT_EQ (ex.msg (), "Cell named ADDHX2 with ID 4 was already given name SEDFFTRX2 (position=763169, cell=)");
+    EXPECT_EQ (ex.msg ().find ("Cell named ADDHX2 with ID 4 was already given name SEDFFTRX2 (position=763169, cell=)"), size_t (0));
   }
 }
 
@@ -678,6 +678,6 @@ TEST(DuplicateCellname)
     //  Seen when private test data is not installed
     throw;
   } catch (tl::Exception &ex) {
-    EXPECT_EQ (ex.msg (), "Same cell name TOP, but different IDs: 3 and 0 (position=1070, cell=)");
+    EXPECT_EQ (ex.msg ().find ("Same cell name TOP, but different IDs: 3 and 0 (position=1070, cell=)"), size_t (0));
   }
 }