From df9a5e4125be20e59b1fc220e1efb76ae52003c3 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 00:13:02 +0200
Subject: [PATCH 01/22] WIP: profile feature in DRC and LVS

---
 src/drc/drc/built-in-macros/_drc_engine.rb | 97 +++++++++++++++++++++-
 1 file changed, 94 insertions(+), 3 deletions(-)

diff --git a/src/drc/drc/built-in-macros/_drc_engine.rb b/src/drc/drc/built-in-macros/_drc_engine.rb
index 17edb9e2e..73b95f4cb 100644
--- a/src/drc/drc/built-in-macros/_drc_engine.rb
+++ b/src/drc/drc/built-in-macros/_drc_engine.rb
@@ -70,6 +70,8 @@ module DRC
       dss._destroy
 
       @verbose = false
+      @profile = false
+      @profile_info = {}
 
       @in_context = nil
 
@@ -711,6 +713,18 @@ module DRC
       end
     end
 
+    # %DRC%
+    # @name profile
+    # @brief Profiles the script and provides a runtime + memory statistics
+    # @synopsis profile
+    # @synopsis profile(m)
+    # Turns profiling on or off (default). In profiling mode, the 
+    # system will collect statistics about rules executed, their execution time
+    # and memory information.
+    def profile(f = true)
+      @profile = f
+    end
+
     # %DRC%
     # @name verbose?
     # @brief Returns true, if verbose mode is enabled
@@ -2267,13 +2281,16 @@ CODE
       end
       
       t = RBA::Timer::new
+
       t.start
       self._process_events
       if @force_gc || Time::now - @time > 0.5
         GC.start    # force a garbage collection before the operation to free unused memory
         @time = Time::now
       end
+      mem_before = RBA::Timer::memory_size
       res = yield
+      mem_after = RBA::Timer::memory_size
       t.stop
 
       begin
@@ -2283,15 +2300,25 @@ CODE
           # Report result statistics
           _result_info(res, 1)
 
-          mem = RBA::Timer::memory_size
-          if mem > 0
-            info("Elapsed: #{'%.3f'%(t.sys+t.user)}s  Memory: #{'%.2f'%(mem/(1024*1024))}M", 1)
+          if mem_after > 0
+            info("Elapsed: #{'%.3f'%(t.sys+t.user)}s  Memory: #{'%.2f'%(mem_after/(1024*1024))}M", 1)
           else
             info("Elapsed: #{'%.3f'%(t.sys+t.user)}s", 1)
           end
 
         end
 
+        if @profile
+
+          # calls, sys time (in sec), user time (in sec), memory added (in bytes)
+          info = (@profile_info[desc] ||= [ 0, 0.0, 0.0, 0 ])
+          info[0] += 1
+          info[1] += t.sys
+          info[2] += t.user
+          info[3] += mem_after - mem_before
+
+        end
+
       ensure
 
         # disable progress again
@@ -2500,6 +2527,65 @@ CODE
       end
       output_cell
     end
+
+    def _dump_profile
+
+      desc_title = "Operation"
+      calls_title = "# calls"
+      time_title = "Time (s)"
+      memory_title = "Memory (k)"
+      titles = [ desc_title, calls_title, time_title, memory_title ]
+
+      max_len_desc  = [ @profile_info.keys.collect { |s| s.size }.max, desc_title.size ].max
+      max_len_calls = [ @profile_info.values.collect { |v| v[0].to_s.size }.max, calls_title.size ].max
+      max_len_time  = [ @profile_info.values.collect { |v| ("%.3f" % (v[1] + v[2])).to_s.size }.max, time_title.size ].max
+      max_len_mem   = [ @profile_info.values.collect { |v| v[3].to_s.size }.max, memory_title.size ].max
+
+      fmt = "  " +
+            "%-" + max_len_desc.to_s + "s  " +
+            "%-" + max_len_calls.to_s + "d  " +
+            "%-" + max_len_time.to_s + ".3f  " + 
+            "%-" + max_len_mem.to_s + ".0f" 
+
+      fmt_title = "  " +
+                  "%-" + max_len_desc.to_s + "s  " +
+                  "%-" + max_len_calls.to_s + "s  " +
+                  "%-" + max_len_time.to_s + "s  " + 
+                  "%-" + max_len_mem.to_s + "s" 
+
+      pi = @profile_info.keys.collect { |s| [s] + @profile_info[s] }.collect do |desc,calls,sys_time,user_time,memory|
+        [ desc, calls, sys_time + user_time, memory.to_f / 1024.0 ]
+      end
+
+      nmax = 50
+
+      self.log("")
+      self.log("Operations by execution time")
+      self.log(fmt_title % titles)
+      n = 1
+      pi.sort { |a,b| b[2] <=> a[2] }.each do |info|
+        self.log(fmt % info)
+        n += 1
+        if n > nmax
+          self.log("  ... (%d entries skipped)" % (pi.size - nmax))
+          break
+        end
+      end
+
+      self.log("")
+      self.log("Operations by memory adder")
+      self.log(fmt_title % titles)
+      n = 1
+      pi.sort { |a,b| b[3] <=> a[3] }.each do |info|
+        self.log(fmt % info)
+        n += 1
+        if n > nmax
+          self.log("  ... (%d entries skipped)" % (pi.size - nmax))
+          break
+        end
+      end
+
+    end
     
     def _start(job_description)
     
@@ -2548,6 +2634,11 @@ CODE
           info("Writing layout file: #{@output_layout_file} ..")
           @output_layout.write(@output_layout_file, gzip, opt)
         end
+
+        # dump the profile information
+        if @profile
+          _dump_profile
+        end
         
         # create the new layers as visual layers if necessary
         if view

From 2fdbd9f4ea5f28ffad89998dff92f18af982b8f1 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 00:49:12 +0200
Subject: [PATCH 02/22] WIP: profile feature

---
 src/drc/drc/built-in-macros/_drc_engine.rb | 37 ++++++++++++----------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/drc/drc/built-in-macros/_drc_engine.rb b/src/drc/drc/built-in-macros/_drc_engine.rb
index 73b95f4cb..b535f29a5 100644
--- a/src/drc/drc/built-in-macros/_drc_engine.rb
+++ b/src/drc/drc/built-in-macros/_drc_engine.rb
@@ -71,6 +71,7 @@ module DRC
 
       @verbose = false
       @profile = false
+      @profile_n = 0
       @profile_info = {}
 
       @in_context = nil
@@ -717,12 +718,20 @@ module DRC
     # @name profile
     # @brief Profiles the script and provides a runtime + memory statistics
     # @synopsis profile
-    # @synopsis profile(m)
+    # @synopsis profile(n)
     # Turns profiling on or off (default). In profiling mode, the 
     # system will collect statistics about rules executed, their execution time
-    # and memory information.
-    def profile(f = true)
-      @profile = f
+    # and memory information. The argument specifies how many operations to 
+    # print at the end of the run. Without an argument, all operations are
+    # printed. Passing "false" for the argument will disable profiling. This is the
+    # default.
+
+    def profile(n = 0)
+      if !n.is_a?(1.class) && n != nil && n != false
+        raise("Argument to 'profile' must be either an integer number or nil")
+      end
+      @profile = !!n
+      @profile_n = [n || 0, 0].max
     end
 
     # %DRC%
@@ -2541,14 +2550,12 @@ CODE
       max_len_time  = [ @profile_info.values.collect { |v| ("%.3f" % (v[1] + v[2])).to_s.size }.max, time_title.size ].max
       max_len_mem   = [ @profile_info.values.collect { |v| v[3].to_s.size }.max, memory_title.size ].max
 
-      fmt = "  " +
-            "%-" + max_len_desc.to_s + "s  " +
+      fmt = "%-" + max_len_desc.to_s + "s  " +
             "%-" + max_len_calls.to_s + "d  " +
             "%-" + max_len_time.to_s + ".3f  " + 
             "%-" + max_len_mem.to_s + ".0f" 
 
-      fmt_title = "  " +
-                  "%-" + max_len_desc.to_s + "s  " +
+      fmt_title = "%-" + max_len_desc.to_s + "s  " +
                   "%-" + max_len_calls.to_s + "s  " +
                   "%-" + max_len_time.to_s + "s  " + 
                   "%-" + max_len_mem.to_s + "s" 
@@ -2557,30 +2564,28 @@ CODE
         [ desc, calls, sys_time + user_time, memory.to_f / 1024.0 ]
       end
 
-      nmax = 50
-
       self.log("")
-      self.log("Operations by execution time")
+      self.log("Operations by execution time\n")
       self.log(fmt_title % titles)
       n = 1
       pi.sort { |a,b| b[2] <=> a[2] }.each do |info|
         self.log(fmt % info)
         n += 1
-        if n > nmax
-          self.log("  ... (%d entries skipped)" % (pi.size - nmax))
+        if @profile_n > 0 && n > @profile_n
+          self.log("... (%d entries skipped)" % (pi.size - nmax))
           break
         end
       end
 
       self.log("")
-      self.log("Operations by memory adder")
+      self.log("Operations by memory adder\n")
       self.log(fmt_title % titles)
       n = 1
       pi.sort { |a,b| b[3] <=> a[3] }.each do |info|
         self.log(fmt % info)
         n += 1
-        if n > nmax
-          self.log("  ... (%d entries skipped)" % (pi.size - nmax))
+        if @profile_n > 0 && n > @profile_n
+          self.log("... (%d entries skipped)" % (pi.size - nmax))
           break
         end
       end

From 2882fa42eea60591b600e7ec919bf5c145dea3c3 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 21:37:47 +0200
Subject: [PATCH 03/22] DRC: support multiple outputs through 'new_report',
 'new_target'

---
 src/drc/drc/built-in-macros/_drc_engine.rb | 487 +++++++++++++++------
 src/drc/drc/built-in-macros/_drc_layer.rb  |  20 +
 2 files changed, 366 insertions(+), 141 deletions(-)

diff --git a/src/drc/drc/built-in-macros/_drc_engine.rb b/src/drc/drc/built-in-macros/_drc_engine.rb
index b535f29a5..662f1e105 100644
--- a/src/drc/drc/built-in-macros/_drc_engine.rb
+++ b/src/drc/drc/built-in-macros/_drc_engine.rb
@@ -13,6 +13,125 @@ module DRC
     end
   end
 
+  class OutputChannel
+    def initialize(engine)
+      @engine = engine
+    end
+    def is_rdb?
+      false
+    end
+    def cellname=(cellname)
+      # reimplement
+    end
+    def cell
+      # reimplement
+    end
+    def write
+      # reimplement
+    end
+    def finish(final)
+      # reimplement
+    end
+    def destroy
+      # reimplement
+    end
+    def layout
+      nil
+    end
+    def rdb
+      nil
+    end
+  end
+
+  class LayoutOutputChannel < OutputChannel
+
+    def initialize(engine, layout, cell, file_name)
+      super(engine)
+      @layout = layout
+      @cell = cell
+      @file_name = file_name
+    end
+
+    def cellname=(cellname)
+      @cell = @layout.cell(cellname) || @layout.create_cell(cellname)
+    end
+
+    def is_rdb?
+      false
+    end
+
+    def finish(final, view)
+      if @file_name
+        opt = RBA::SaveLayoutOptions::new
+        gzip = opt.set_format_from_filename(@file_name)
+        @engine.info("Writing layout file: #{@file_name} ..")
+        @layout.write(@file_name, gzip, opt)
+      end
+    end
+
+    def destroy
+      @layout._destroy
+    end
+
+    def layout
+      @layout
+    end
+
+    def cell
+      @cell
+    end
+
+  end
+
+  class RDBOutputChannel < OutputChannel
+
+    def initialize(engine, rdb, rdb_index, cellname, file_name)
+      super(engine)
+      @rdb = rdb
+      @rdb_index = rdb_index
+      @cell = cellname && rdb.create_cell(cellname)
+      @file_name = file_name
+    end
+
+    def cellname=(cellname)
+      @cell = nil
+      @rdb.each_cell do |c|
+        if c.name == cellname
+          @cell = c
+        end
+      end
+      @cell ||= @rdb.create_cell(cellname)
+    end
+
+    def is_rdb?
+      true
+    end
+
+    def finish(final, view)
+      if @file_name
+        rdb_file = @engine._make_path(@file_name)
+        @engine.info("Writing report database: #{rdb_file} ..")
+        @rdb.save(rdb_file)
+      end
+      if final && view
+        view.show_rdb(@rdb_index, view.active_cellview_index)
+      end
+    end
+
+    def destroy
+      @rdb._destroy
+    end
+
+    def rdb
+      @rdb
+    end
+
+    def cell
+      @cell
+    end
+
+  end
+
   # The DRC engine
   
   # %DRC%
@@ -40,11 +159,9 @@ module DRC
       @def_source = nil
       @dbu_read = false
       use_dbu(@def_layout && @def_layout.dbu)
-      @output_layout = nil
-      @output_rdb = nil
-      @output_rdb_file = nil
-      @output_rdb_cell = nil
       @show_l2ndb = nil
+      @def_output = nil
+      @other_outputs = []
       @output_l2ndb_file = nil
       @target_netlist_file = nil
       @target_netlist_format = nil
@@ -1363,46 +1480,44 @@ module DRC
 
       self._context("report") do
 
-        @output_rdb_file = filename
+        # finish what we got so far
+        _finish(false)
 
-        name = filename && File::basename(filename)
-        name ||= "DRC"
-        
-        @output_rdb_index = nil
+        @def_output && @def_output.destroy
+        @def_output = nil
 
-        view = RBA::LayoutView::current
-        if view
-          if self._rdb_index
-            @output_rdb = RBA::ReportDatabase::new("")   # reuse existing name
-            @output_rdb_index = view.replace_rdb(self._rdb_index, @output_rdb)
-          else
-            @output_rdb = RBA::ReportDatabase::new(name)
-            @output_rdb_index = view.add_rdb(@output_rdb)
-          end
-        else
-          @output_rdb = RBA::ReportDatabase::new(name)
-        end
-        
-        @output_layout = nil
-        @output_cell = nil
-        @output_layout_file = nil
-
-        cn = nil
-        cn ||= @def_cell && @def_cell.name
-        cn ||= source && source.cell_name
-        cn ||= cellname
-
-        cn || raise("No cell name specified - either the source was not specified before 'report' or there is no default source. In the latter case, specify a cell name as the third parameter")
-
-        @output_rdb_cell = @output_rdb.create_cell(cn)
-        @output_rdb.generator = self._generator
-        @output_rdb.top_cell_name = cn
-        @output_rdb.description = description
+        @def_output = _make_report(description, filename, cellname)
 
       end
       
     end
 
+    # %DRC%
+    # @name new_report
+    # @brief Creates a new report database object for use in "output"
+    # @synopsis new_report(description [, filename [, cellname ] ])
+    # 
+    # This method creates an independent report object. This object
+    # can be used in "output" to send a layer to a different report than
+    # the default report or target.
+    #
+    # Arguments are the same than for \global#report.
+    #
+    # See \Layer#output for details about this feature.
+
+    def new_report(description, filename = nil, cellname = nil)
+
+      output = nil
+
+      self._context("new_report") do
+        output = _make_report(description, filename, cellname)
+        @other_outputs << output
+      end
+
+      output
+      
+    end
+
     # %DRC%
     # @name report_netlist
     # @brief Specifies an extracted netlist report for output
@@ -1479,25 +1594,13 @@ module DRC
         # finish what we got so far
         _flush
         
-        if @output_rdb
-        
-          cell = nil
-          @output_rdb.each_cell do |c|
-            if c.name == cellname
-              cell = c
-            end
+        if ! @def_output
+          if @def_layout
+            # establish a new default output from the default layout on this occasion
+            @def_output = LayoutOutputChannel(self, @def_layout, cellname.to_s, nil)
           end
-          
-          cell ||= @output_rdb.create_cell(cellname)
-          @output_rdb_cell = cell
-          
         else
-        
-          @output_layout ||= @def_layout
-          if @output_layout
-            @output_cell = @output_layout.cell(cellname.to_s) || @output_layout.create_cell(cellname.to_s)
-          end
-          
+          @def_output.cellname = cellname.to_s
         end
 
       end
@@ -1535,51 +1638,42 @@ module DRC
     
         # finish what we got so far
         _finish(false)
-            
-        if arg.is_a?(String)
-        
-          if arg =~ /^@(\d+|\+)/
-            view = RBA::LayoutView::current
-            view || raise("No view open")
-            if $1 == "+"
-              n = view.create_layout(true)
-              cellname ||= (@def_cell ? @def_cell.name : "TOP")
-            else
-              n = $1.to_i - 1
-            end
-            (n >= 0 && view.cellviews > n) || raise("Invalid layout index @#{n + 1}")
-            cv = view.cellview(n)
-            cv.is_valid? || raise("Invalid layout @#{n + 1}")
-            @output_layout = cv.layout
-            @output_cell = cellname ? (@output_layout.cell(cellname.to_s) || @output_layout.create_cell(cellname.to_s)) : cv.cell
-            cv.cell = @output_cell
-            @output_layout_file = nil
-          else
-            @output_layout = RBA::Layout::new
-            @output_cell = cellname && @output_layout.create_cell(cellname.to_s)
-            @output_layout_file = arg
-          end
-          
-        elsif arg.is_a?(RBA::Layout)
-        
-          @output_layout = arg
-          @output_cell = cellname && (@output_layout.cell(cellname.to_s) || @output_layout.create_cell(cellname.to_s))
-          @output_layout_file = nil
-          
-        elsif arg.is_a?(RBA::Cell)
-        
-          @output_layout = arg.layout
-          @output_cell = arg
-          @output_layout_file = nil
 
-        else
-          raise("Invalid argument '" + arg.inspect + "'")
-        end
-    
+        @def_output && @def_output.destroy
+        @def_output = nil
+
+        @def_output = _make_target(arg, cellname)
+
       end
 
     end
-    
+            
+    # %DRC%
+    # @name new_target
+    # @brief Creates a new layout target object for use in "output"
+    # @synopsis new_target(what [, cellname])
+    # 
+    # This method creates an independent target object. This object
+    # can be used in "output" to send a layer to a different layout file than
+    # the default report or target.
+    #
+    # Arguments are the same than for \global#target.
+    #
+    # See \Layer#output for details about this feature.
+
+    def new_target(arg, cellname = nil)
+
+      output = nil
+
+      self._context("new_target") do
+        output = _make_target(arg, cellname)
+        @other_outputs << output
+      end
+
+      output
+      
+    end
+
     # %DRC%
     # @name box 
     # @brief Creates a box object
@@ -2513,8 +2607,8 @@ CODE
     end
 
     def _output_layout
-      if @output_layout 
-        output = @output_layout
+      if @def_output && @def_output.layout
+        output = @def_output.layout
       else
         output = @def_layout
         output || raise("No output layout specified")
@@ -2523,15 +2617,16 @@ CODE
     end
     
     def _output_cell
-      if @output_layout 
-        if @output_cell
-          output_cell = @output_cell
+      if @def_output && @def_output.layout 
+        if @def_output.cell
+          output_cell = @def_output.cell
         elsif @def_cell
-          output_cell = @output_layout.cell(@def_cell.name) || @output_layout.create_cell(@def_cell.name)
+          output_layout = @def_output.layout
+          output_cell = output_layout.cell(@def_cell.name) || output_layout.create_cell(@def_cell.name)
         end
         output_cell || raise("No output cell specified (see 'target' instruction)")
       else
-        output_cell = @output_cell || @def_cell
+        output_cell = @def_cell
         output_cell || raise("No output cell specified")
       end
       output_cell
@@ -2619,25 +2714,15 @@ CODE
       begin
 
         _flush    
-        
+
         view = RBA::LayoutView::current
 
-        # save the report database if requested
-        if @output_rdb_file && final
-          rdb_file = _make_path(@output_rdb_file)
-          info("Writing report database: #{rdb_file} ..")
-          @output_rdb.save(rdb_file)
-        end
-        if @output_rdb && final && view
-          view.show_rdb(@output_rdb_index, view.active_cellview_index)
-        end
-      
-        # save the output file if requested
-        if @output_layout && @output_layout_file
-          opt = RBA::SaveLayoutOptions::new
-          gzip = opt.set_format_from_filename(@output_layout_file)
-          info("Writing layout file: #{@output_layout_file} ..")
-          @output_layout.write(@output_layout_file, gzip, opt)
+        @def_output && @def_output.finish(final, view)
+
+        if final
+          @other_outputs.each do |output|
+            output.finish(final, view)
+          end
         end
 
         # dump the profile information
@@ -2648,7 +2733,7 @@ CODE
         # create the new layers as visual layers if necessary
         if view
         
-          output = @output_layout || @def_layout
+          output = ( @def_output && @def_output.layout ) || @def_layout
           cv_index = nil
           view.cellviews.times do |cvi|
             view.cellview(cvi).layout == output && cv_index = cvi
@@ -2730,13 +2815,12 @@ CODE
       ensure
 
         @output_layers = []
-        @output_layout = nil
-        @output_layout_file = nil
-        @output_cell = nil
-        @output_rdb_file = nil
-        @output_rdb_cell = nil
-        @output_rdb = nil
-        @output_rdb_index = nil
+        @def_output && @def_output.destroy
+        @def_output = nil
+        if final
+          @other_outputs.each { |o| o.destroy }
+          @other_outputs = []
+        end
         @show_l2ndb = nil
         @output_l2ndb_file = nil
 
@@ -3029,21 +3113,36 @@ CODE
     
     def _output(data, *args)
 
-      if @output_rdb
+      channel = args.find { |a| a.is_a?(OutputChannel) }
+      if ! channel
+        if ! @def_output
+          @def_output = LayoutOutputChannel(self, self._output_layout, self._output_cell, nil)
+        end
+        channel = @def_output
+      end
+
+      args = args.select { |a| !a.is_a?(OutputChannel) }
+
+      if channel.rdb
         
         if args.size < 1
           raise("Invalid number of arguments - category name and optional description expected")
         end
 
-        cat = @output_rdb.create_category(args[0].to_s)
+        output_rdb = channel.rdb
+        output_cell = channel.cell
+
+        cat = output_rdb.create_category(args[0].to_s)
         args[1] && cat.description = args[1]
 
-        cat.scan_collection(@output_rdb_cell, RBA::CplxTrans::new(self.dbu), data)
+        cat.scan_collection(output_cell, RBA::CplxTrans::new(self.dbu), data)
       
-      else 
+      end
 
-        output = self._output_layout
-        output_cell = self._output_cell
+      if channel.layout
+
+        output = channel.layout
+        output_cell = channel.cell
 
         info = nil
         if args.size == 1
@@ -3073,14 +3172,16 @@ CODE
 
         begin
 
-          if !@used_output_layers[li]
-            @output_layers.push(li)
-            # Note: to avoid issues with output onto the input layer, we
-            # output to a temp layer and later swap both. The simple implementation
-            # did a clear here and the effect of that was that the data potentially
-            # got invalidated.
-            tmp = output.insert_layer(RBA::LayerInfo::new)
-            @used_output_layers[li] = true
+          if channel == @def_output
+            if !@used_output_layers[li]
+              @output_layers.push(li)
+              # Note: to avoid issues with output onto the input layer, we
+              # output to a temp layer and later swap both. The simple implementation
+              # did a clear here and the effect of that was that the data potentially
+              # got invalidated.
+              tmp = output.insert_layer(RBA::LayerInfo::new)
+              @used_output_layers[li] = true
+            end
           end
 
           # insert the data into the output layer
@@ -3116,7 +3217,111 @@ CODE
       @layout_sources[name] = src
       src
     end
+    
+    def _make_report(description, filename, cellname)
 
+      output_rdb_file = filename
+
+      name = filename && File::basename(filename)
+      name ||= "DRC"
+      
+      output_rdb_index = nil
+
+      view = RBA::LayoutView::current
+      if view
+        if self._rdb_index
+          output_rdb = RBA::ReportDatabase::new("")   # reuse existing name
+          output_rdb_index = view.replace_rdb(self._rdb_index, output_rdb)
+        else
+          output_rdb = RBA::ReportDatabase::new(name)
+          output_rdb_index = view.add_rdb(output_rdb)
+        end
+      else
+        output_rdb = RBA::ReportDatabase::new(name)
+      end
+      
+      cn = cellname && cellname.to_s
+      cn ||= @def_cell && @def_cell.name
+      cn ||= source && source.cell_name
+
+      cn || raise("No cell name specified - either the source was not specified before 'report' or there is no default source. In the latter case, specify a cell name as the third parameter")
+
+      output_rdb.generator = self._generator
+      output_rdb.top_cell_name = cn
+      output_rdb.description = description
+
+      RDBOutputChannel::new(self, output_rdb, output_rdb_index, cn, output_rdb_file)
+      
+    end
+
+    def _make_target(arg, cellname = nil)
+
+      if arg.is_a?(String)
+      
+        if arg =~ /^@(\d+|\+)/
+
+          view = RBA::LayoutView::current
+          view || raise("No view open")
+          if $1 == "+"
+            n = view.create_layout(true)
+            cellname ||= (@def_cell ? @def_cell.name : "TOP")
+          else
+            n = $1.to_i - 1
+          end
+
+          (n >= 0 && view.cellviews > n) || raise("Invalid layout index @#{n + 1}")
+
+          cv = view.cellview(n)
+          cv.is_valid? || raise("Invalid layout @#{n + 1}")
+
+          output_layout = cv.layout
+          output_cell = cellname ? (output_layout.cell(cellname.to_s) || output_layout.create_cell(cellname.to_s)) : cv.cell
+          cv.cell = output_cell
+          output_layout_file = nil
+
+        else
+
+          cn = cellname && cellname.to_s
+          cn ||= @def_cell && @def_cell.name
+          cn ||= source && source.cell_name
+
+          cn || raise("No cell name specified - either the source was not specified before 'report' or there is no default source. In the latter case, specify a cell name as the third parameter")
+
+          output_layout = RBA::Layout::new
+          output_cell = output_layout.create_cell(cn)
+          output_layout_file = arg
+
+        end
+        
+      elsif arg.is_a?(RBA::Layout)
+      
+        output_layout = arg
+
+        output_cell = cellname && (output_layout.cell(cellname.to_s) || output_layout.create_cell(cellname.to_s))
+        if ! output_cell
+          begin
+            output_cell = output_layout.top_cell
+          rescue 
+            raise("No cell name specified and the layout does not have a unique top cell - specify the name of the top cell to write the output to")
+          end
+        end
+
+        output_layout_file = nil
+        
+      elsif arg.is_a?(RBA::Cell)
+      
+        output_layout = arg.layout
+        output_cell = arg
+        output_layout_file = nil
+
+      else
+        raise("Invalid argument '" + arg.inspect + "'")
+      end
+    
+      output = LayoutOutputChannel::new(self, output_layout, output_cell, output_layout_file)
+
+    end
+    
   end
  
 end
diff --git a/src/drc/drc/built-in-macros/_drc_layer.rb b/src/drc/drc/built-in-macros/_drc_layer.rb
index e434c9d1d..30ea3a692 100644
--- a/src/drc/drc/built-in-macros/_drc_layer.rb
+++ b/src/drc/drc/built-in-macros/_drc_layer.rb
@@ -5007,6 +5007,26 @@ CODE
     #
     # See \global#report and \global#target on how to configure output to a target layout
     # or report database. 
+    # 
+    # See also \global#new_target and \global#new_report on how to create additional 
+    # targets for output. This allows saving certain layers to different files than
+    # the standard target or report. To do so, create a new target or report using one
+    # of these functions and pass that object to the corresponding "output" call as
+    # an additional argument.
+    #
+    # Example:
+    #
+    # @code
+    # check1 = ...
+    # check2 = ...
+    # check3 = ...
+    #
+    # second_report = new_report("Only for check2", "check2.lyrdb")
+    #
+    # check1.output("Check 1")
+    # check2.output("Check 2", second_report)
+    # check3.output("Check 3")
+    # @/code
     
     def output(*args)
       @engine._context("output") do

From a582eabc22b9cda2b5aaee7fad7624c782aa0b88 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 21:57:40 +0200
Subject: [PATCH 04/22] DRC: doc updates

---
 src/doc/doc/about/drc_ref_global.xml |  45 ++++++++++
 src/doc/doc/about/drc_ref_layer.xml  |  20 +++++
 src/doc/doc/manual/drc_runsets.xml   | 130 +++++++++++++++++----------
 3 files changed, 148 insertions(+), 47 deletions(-)

diff --git a/src/doc/doc/about/drc_ref_global.xml b/src/doc/doc/about/drc_ref_global.xml
index 91fa1a06e..3deac6889 100644
--- a/src/doc/doc/about/drc_ref_global.xml
+++ b/src/doc/doc/about/drc_ref_global.xml
@@ -1182,6 +1182,36 @@ See <a href="/about/drc_ref_netter.xml#netlist">Netter#netlist</a> for a descrip
 <p>
 See <a href="/about/drc_ref_netter.xml">Netter</a> for more details
 </p>
+<a name="new_report"/><h2>"new_report" - Creates a new report database object for use in "output"</h2>
+<keyword name="new_report"/>
+<p>Usage:</p>
+<ul>
+<li><tt>new_report(description [, filename [, cellname ] ])</tt></li>
+</ul>
+<p>
+This method creates an independent report object. This object
+can be used in "output" to send a layer to a different report than
+the default report or target.
+</p><p>
+Arguments are the same than for <a href="/about/drc_ref_global.xml#report">report</a>.
+</p><p>
+See <a href="/about/drc_ref_layer.xml#output">Layer#output</a> for details about this feature.
+</p>
+<a name="new_target"/><h2>"new_target" - Creates a new layout target object for use in "output"</h2>
+<keyword name="new_target"/>
+<p>Usage:</p>
+<ul>
+<li><tt>new_target(what [, cellname])</tt></li>
+</ul>
+<p>
+This method creates an independent target object. This object
+can be used in "output" to send a layer to a different layout file than
+the default report or target.
+</p><p>
+Arguments are the same than for <a href="/about/drc_ref_global.xml#target">target</a>.
+</p><p>
+See <a href="/about/drc_ref_layer.xml#output">Layer#output</a> for details about this feature.
+</p>
 <a name="no_borders"/><h2>"no_borders" - Reset the tile borders</h2>
 <keyword name="no_borders"/>
 <p>Usage:</p>
@@ -1413,6 +1443,21 @@ See <a href="/about/drc_ref_source.xml#polygons">Source#polygons</a> for a descr
 The primary input of the universal DRC function is the layer the <a href="/about/drc_ref_layer.xml#drc">Layer#drc</a> function
 is called on.
 </p>
+<a name="profile"/><h2>"profile" - Profiles the script and provides a runtime + memory statistics</h2>
+<keyword name="profile"/>
+<p>Usage:</p>
+<ul>
+<li><tt>profile</tt></li>
+<li><tt>profile(n)</tt></li>
+</ul>
+<p>
+Turns profiling on or off (default). In profiling mode, the 
+system will collect statistics about rules executed, their execution time
+and memory information. The argument specifies how many operations to 
+print at the end of the run. Without an argument, all operations are
+printed. Passing "false" for the argument will disable profiling. This is the
+default.
+</p>
 <a name="props_copy"/><h2>"props_copy" - Specifies "copy properties" on operations supporting user properties constraints</h2>
 <keyword name="props_copy"/>
 <p>
diff --git a/src/doc/doc/about/drc_ref_layer.xml b/src/doc/doc/about/drc_ref_layer.xml
index 4dbd218fa..6efff2012 100644
--- a/src/doc/doc/about/drc_ref_layer.xml
+++ b/src/doc/doc/about/drc_ref_layer.xml
@@ -2217,6 +2217,26 @@ a single <class_doc href="LayerInfo">LayerInfo</class_doc> object.
 </p><p>
 See <a href="/about/drc_ref_global.xml#report">report</a> and <a href="/about/drc_ref_global.xml#target">target</a> on how to configure output to a target layout
 or report database. 
+</p><p>
+See also <a href="/about/drc_ref_global.xml#new_target">new_target</a> and <a href="/about/drc_ref_global.xml#new_report">new_report</a> on how to create additional 
+targets for output. This allows saving certain layers to different files than
+the standard target or report. To do so, create a new target or report using one
+of these functions and pass that object to the corresponding "output" call as
+an additional argument.
+</p><p>
+Example:
+</p><p>
+<pre>
+check1 = ...
+check2 = ...
+check3 = ...
+
+second_report = new_report("Only for check2", "check2.lyrdb")
+
+check1.output("Check 1")
+check2.output("Check 2", second_report)
+check3.output("Check 3")
+</pre>
 </p>
 <a name="outside"/><h2>"outside" - Selects edges or polygons of self which are outside edges or polygons from the other layer</h2>
 <keyword name="outside"/>
diff --git a/src/doc/doc/manual/drc_runsets.xml b/src/doc/doc/manual/drc_runsets.xml
index 971bf9b79..7642c4c16 100644
--- a/src/doc/doc/manual/drc_runsets.xml
+++ b/src/doc/doc/manual/drc_runsets.xml
@@ -185,7 +185,7 @@ l.output("OUT")
 l.output(17, 0, "OUT")</pre>
 
   <p>
-  Output can be sent to other layouts using the "target" function:
+  Output can be sent to other layouts using the <a href="/about/drc_ref_global.xml#target">target</a> function:
   </p>
 
   <pre>
@@ -196,7 +196,7 @@ target("@2")
 target("out.gds", "OUT_TOP")</pre>
 
   <p>
-  Output can also be sent to a report database:
+  Output can also be sent to a report database using the <a href="/about/drc_ref_global.xml#report">report</a> function:
   </p>
 
   <pre>
@@ -234,6 +234,38 @@ l.output("check1", "The first check")</pre>
   unpredictable.
   </p>
 
+  <p>
+  It is possible to open "side" reports and targets and send layers to these
+  outputs without closing the default output. 
+  </p>
+
+  <p>
+  To open a "side report", use <a href="/about/drc_ref_global.xml#new_report">new_report</a>
+  in the same way you use "report". Instead of switching the output, this function will return a 
+  new report object that can be included in the argument list of "output" 
+  for the layer that is to be sent to that side report:
+  </p>
+
+  <pre>
+# opens a new side report
+side_report = new_report("Another report")
+...
+# Send data from layer l to new category "check1" to the side report
+l.output(side_report, "check1", "The first check")</pre>
+
+  <p>
+  In the same way, "side targets" can be opened using <a href="/about/drc_ref_global.xml#new_target">new_target</a>.
+  Such side targets open a way to write certain layers to other layout files.
+  This is very handy for debugging:
+  </p>
+
+  <pre>
+# opens a new side target for debugging
+debug_out = new_target("debug.gds")
+...
+# Send data from layer l to the debug output, layer 100/0
+l.output(debug_out, 100, 0)</pre>
+
 
   <h2>Dimension specifications</h2>
 
@@ -710,12 +742,12 @@ overlaps = layer.size(0.2).raw.merged(2)</pre>
   </p>
 
   <pre>
-  layer = input(1, 0)
-  layer.raw.sized(0.1).output(100, 0)
+layer = input(1, 0)
+layer.raw.sized(0.1).output(100, 0)
 
-  # this check will now be done on a raw layer, since the 
-  # previous raw call was putting the layer into raw mode
-  layer.width(0.2).ouput(101, 0)</pre>
+# this check will now be done on a raw layer, since the 
+# previous raw call was putting the layer into raw mode
+layer.width(0.2).ouput(101, 0)</pre>
 
   <p>
   The following two images show the effect of raw and clean mode:
@@ -792,20 +824,18 @@ overlaps = layer.size(0.2).raw.merged(2)</pre>
   </p>
 
   <pre>
-  ...
-  drc_w = input(1, 0).width(0.2)
-  ...
-  </pre>
+...
+drc_w = input(1, 0).width(0.2)
+...</pre>
 
   <p>
   can be written as:
   </p>
 
   <pre>
-  ...
-  drc_w = input(1, 0).drc(width &lt; 0.2)
-  ...
-  </pre>
+...
+drc_w = input(1, 0).drc(width &lt; 0.2)
+...</pre>
 
   <p>
   The <a href="/about/drc_ref_layer.xml#drc">drc</a> method is the "universal DRC" method.
@@ -845,12 +875,11 @@ overlaps = layer.size(0.2).raw.merged(2)</pre>
   </p>
 
   <pre>
-  ...
-  l1 = input(1, 0)
-  l2 = input(2, 0)
-  drc_sep = l1.drc(separation(l2) &lt;= 0.5)
-  ...
-  </pre>
+...
+l1 = input(1, 0)
+l2 = input(2, 0)
+drc_sep = l1.drc(separation(l2) &lt;= 0.5)
+...</pre>
 
   <p>
   Options are also specified together with the measurement and follow the same notation
@@ -858,10 +887,9 @@ overlaps = layer.size(0.2).raw.merged(2)</pre>
   </p>
 
   <pre>
-  ...
-  drc_w = input(1, 0).drc(width(projection) &lt; 0.2)
-  ...
-  </pre>
+...
+drc_w = input(1, 0).drc(width(projection) &lt; 0.2)
+...</pre>
 
   <p>
   However, the universal DRC is much more than a convenient way to write checks: 
@@ -879,10 +907,9 @@ overlaps = layer.size(0.2).raw.merged(2)</pre>
   </p>
 
   <pre>
-  ...
-  drc_ws = input(1, 0).drc((width &lt; 0.2) &amp; (space &lt; 0.3))
-  ...
-  </pre>
+...
+drc_ws = input(1, 0).drc((width &lt; 0.2) &amp; (space &lt; 0.3))
+...</pre>
 
   <p>
   The boolean AND is computed between the edges on the primary shape and returns the
@@ -891,12 +918,11 @@ overlaps = layer.size(0.2).raw.merged(2)</pre>
   </p>
 
   <pre>
-  ...
-  drc_ws1 = input(1, 0).width(0.2).edges
-  drc_ws2 = input(1, 0).space(0.3).edges
-  drc_ws = drc_ws1 &amp; drc_ws2
-  ...
-  </pre>
+...
+drc_ws1 = input(1, 0).width(0.2).edges
+drc_ws2 = input(1, 0).space(0.3).edges
+drc_ws = drc_ws1 &amp; drc_ws2
+...</pre>
 
   <p>
   The reason is that performing the boolean computation in the local loop can be 
@@ -933,11 +959,10 @@ overlaps = layer.size(0.2).raw.merged(2)</pre>
   </p>
 
   <pre>
-  ...
-  drc_w = input(1, 0).width(0.2)
-  log("Number of width violations: #{drc_w.data.size}")
-  ...
-  </pre>
+...
+drc_w = input(1, 0).width(0.2)
+log("Number of width violations: #{drc_w.data.size}")
+...</pre>
 
   <p>
   The <a href="/about/drc_ref_global.xml#error">error</a> function can be used to output error messages
@@ -946,11 +971,23 @@ overlaps = layer.size(0.2).raw.merged(2)</pre>
   </p>
 
   <pre>
-  log_file("drc_log.txt")
-  verbose(true)
-  info("This message will be sent to the log file")
-  ...
-  </pre>
+log_file("drc_log.txt")
+verbose(true)
+info("This message will be sent to the log file")
+...</pre>
+
+  <p>
+  The <a href="/about/drc_ref_global.xml#profile">profile</a> function will collect profiling 
+  information during the DRC run. At the end of the script, the operations are printed to the log
+  output, sorted by their CPU time and approximate memory footprint. "profile" can be given a 
+  numerical argument indicating the number of operations to print. Lower-ranking operations are
+  skipped in that case. By default, all operations are printed.
+  </p>
+
+  <pre>
+# enables profiling
+profile
+...</pre>
   
 
   <h2>The tiling option</h2>
@@ -1003,8 +1040,7 @@ threads(4)
 # Disable tiling
 flat
 
-... non-tiled operations ...
-</pre>
+... non-tiled operations ...</pre>
 
   <p>
   Some operations implicitly specify a tile border. If the tile border is known (see length example above), explicit borders

From 78dbabcde17c53acb1b2e5a3c2bdd408270066c9 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 23:02:16 +0200
Subject: [PATCH 05/22] DRC enhancements: bug fixes

---
 src/drc/drc/built-in-macros/_drc_engine.rb | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/src/drc/drc/built-in-macros/_drc_engine.rb b/src/drc/drc/built-in-macros/_drc_engine.rb
index 662f1e105..3c6c2f51d 100644
--- a/src/drc/drc/built-in-macros/_drc_engine.rb
+++ b/src/drc/drc/built-in-macros/_drc_engine.rb
@@ -32,9 +32,6 @@ module DRC
     def finish(final)
       # reimplement
     end
-    def destroy
-      # reimplement
-    end
     def layout
       nil
     end
@@ -69,10 +66,6 @@ module DRC
       end
     end
 
-    def destroy
-      @layout._destroy
-    end
-
     def layout
       @layout
     end
@@ -118,10 +111,6 @@ module DRC
       end
     end
 
-    def destroy
-      @rdb._destroy
-    end
-
     def rdb
       @rdb
     end
@@ -1483,9 +1472,7 @@ module DRC
         # finish what we got so far
         _finish(false)
 
-        @def_output && @def_output.destroy
         @def_output = nil
-
         @def_output = _make_report(description, filename, cellname)
 
       end
@@ -1597,7 +1584,7 @@ module DRC
         if ! @def_output
           if @def_layout
             # establish a new default output from the default layout on this occasion
-            @def_output = LayoutOutputChannel(self, @def_layout, cellname.to_s, nil)
+            @def_output = LayoutOutputChannel::new(self, @def_layout, cellname.to_s, nil)
           end
         else
           @def_output.cellname = cellname.to_s
@@ -1639,9 +1626,7 @@ module DRC
         # finish what we got so far
         _finish(false)
 
-        @def_output && @def_output.destroy
         @def_output = nil
-
         @def_output = _make_target(arg, cellname)
 
       end
@@ -2815,10 +2800,8 @@ CODE
       ensure
 
         @output_layers = []
-        @def_output && @def_output.destroy
         @def_output = nil
         if final
-          @other_outputs.each { |o| o.destroy }
           @other_outputs = []
         end
         @show_l2ndb = nil
@@ -3116,7 +3099,7 @@ CODE
       channel = args.find { |a| a.is_a?(OutputChannel) }
       if ! channel
         if ! @def_output
-          @def_output = LayoutOutputChannel(self, self._output_layout, self._output_cell, nil)
+          @def_output = LayoutOutputChannel::new(self, self._output_layout, self._output_cell, nil)
         end
         channel = @def_output
       end

From 2e33cf1442f0f9a5da1e6da190f66b587c2420c2 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 23:14:07 +0200
Subject: [PATCH 06/22] Added unit tests for new features

---
 src/drc/unit_tests/drcSimpleTests.cc      |  69 +++++++++++++++++
 testdata/drc/drcSimpleTests_14b.drc       |  24 ++++++
 testdata/drc/drcSimpleTests_14b.gds       | Bin 0 -> 362 bytes
 testdata/drc/drcSimpleTests_au14b.gds     | Bin 0 -> 298 bytes
 testdata/drc/drcSimpleTests_au14b.lyrdb   |  88 ++++++++++++++++++++++
 testdata/drc/drcSimpleTests_au14b_2.gds   | Bin 0 -> 362 bytes
 testdata/drc/drcSimpleTests_au14b_2.lyrdb |  49 ++++++++++++
 7 files changed, 230 insertions(+)
 create mode 100644 testdata/drc/drcSimpleTests_14b.drc
 create mode 100644 testdata/drc/drcSimpleTests_14b.gds
 create mode 100644 testdata/drc/drcSimpleTests_au14b.gds
 create mode 100644 testdata/drc/drcSimpleTests_au14b.lyrdb
 create mode 100644 testdata/drc/drcSimpleTests_au14b_2.gds
 create mode 100644 testdata/drc/drcSimpleTests_au14b_2.lyrdb

diff --git a/src/drc/unit_tests/drcSimpleTests.cc b/src/drc/unit_tests/drcSimpleTests.cc
index 8b24c918d..c0825e751 100644
--- a/src/drc/unit_tests/drcSimpleTests.cc
+++ b/src/drc/unit_tests/drcSimpleTests.cc
@@ -731,6 +731,75 @@ TEST(14_SwitchingTargets)
   db::compare_layouts (_this, layout2, au2, db::NoNormalization);
 }
 
+TEST(14b_SideTargetsAndReports)
+{
+  std::string rs = tl::testdata ();
+  rs += "/drc/drcSimpleTests_14b.drc";
+
+  std::string input = tl::testdata ();
+  input += "/drc/drcSimpleTests_14b.gds";
+
+  std::string au = tl::testdata ();
+  au += "/drc/drcSimpleTests_au14b.gds";
+
+  std::string au2 = tl::testdata ();
+  au2 += "/drc/drcSimpleTests_au14b_2.gds";
+
+  std::string au_report = tl::testdata ();
+  au_report += "/drc/drcSimpleTests_au14b.lyrdb";
+
+  std::string au_report2 = tl::testdata ();
+  au_report2 += "/drc/drcSimpleTests_au14b_2.lyrdb";
+
+  std::string output = this->tmp_file ("tmp.gds");
+  std::string output2 = this->tmp_file ("tmp2.gds");
+  std::string report = this->tmp_file ("tmp.lydrc");
+  std::string report2 = this->tmp_file ("tmp2.lydrc");
+
+  {
+    //  Set some variables
+    lym::Macro config;
+    config.set_text (tl::sprintf (
+        "$drc_force_gc = true\n"
+        "$drc_test_source = '%s'\n"
+        "$drc_test_target = '%s'\n"
+        "$drc_test_target2 = '%s'\n"
+        "$drc_test_report = '%s'\n"
+        "$drc_test_report2 = '%s'\n"
+      , input, output, output2, report, report2)
+    );
+    config.set_interpreter (lym::Macro::Ruby);
+    EXPECT_EQ (config.run (), 0);
+  }
+
+  lym::Macro drc;
+  drc.load_from (rs);
+  EXPECT_EQ (drc.run (), 0);
+
+  db::Layout layout;
+
+  {
+    tl::InputStream stream (output);
+    db::Reader reader (stream);
+    reader.read (layout);
+  }
+
+  db::compare_layouts (_this, layout, au, db::NoNormalization);
+
+  db::Layout layout2;
+
+  {
+    tl::InputStream stream (output2);
+    db::Reader reader (stream);
+    reader.read (layout2);
+  }
+
+  db::compare_layouts (_this, layout2, au2, db::NoNormalization);
+
+  compare_text_files (report, au_report);
+  compare_text_files (report2, au_report2);
+}
+
 TEST(15_issue548)
 {
   std::string rs = tl::testdata ();
diff --git a/testdata/drc/drcSimpleTests_14b.drc b/testdata/drc/drcSimpleTests_14b.drc
new file mode 100644
index 000000000..0d4daa56a
--- /dev/null
+++ b/testdata/drc/drcSimpleTests_14b.drc
@@ -0,0 +1,24 @@
+
+source($drc_test_source)
+
+target($drc_test_target)
+
+l1 = input(1, 0)
+l2 = input(2, 0)
+
+tcopy = new_target($drc_test_target2)
+rcopy = new_report("Report 2", $drc_test_report2)
+
+l1.output(tcopy, 101, 0)
+l2.output(tcopy, 102, 0)
+
+l1.output(1, 0)
+l1.space(1.0.um).output(100, 0)
+
+report("Report 1", $drc_test_report)
+
+l2.space(1.0.um).output("l2 space < 1µm")
+l1.width(1.0.um).output("l1 width < 1µm")
+
+l1.sep(l2, 1.0.um).output(rcopy, "l1 sep l2 < 1µm")
+
diff --git a/testdata/drc/drcSimpleTests_14b.gds b/testdata/drc/drcSimpleTests_14b.gds
new file mode 100644
index 0000000000000000000000000000000000000000..c43c2fae914f32d78e4fd74761d87f3a5a5ee24c
GIT binary patch
literal 362
zcmZQzV_;&6V31*CVt>xS${@oa!eGUqhRkN*U}E#}bYfr-VP>^+>@@d2w)}&o%MSeo
zv!g;7WLWX&V`B^P4`5(m;b353<7HxCWMJcCVqjp<5nu+ANPxM8fq_wnfdNJ%NwEmx
z(9ie<q6$QV^f4<SXpjt49$7yV!v6pN|62gnOn}f#dw_ahFfhPqm^kAE2E6)Nd?5N+
UD<Jw|G)$bu2cn-H=m!=C0Qdwm(f|Me

literal 0
HcmV?d00001

diff --git a/testdata/drc/drcSimpleTests_au14b.gds b/testdata/drc/drcSimpleTests_au14b.gds
new file mode 100644
index 0000000000000000000000000000000000000000..af3ebbcfb21c9a378554ff699e2b9799d1473940
GIT binary patch
literal 298
zcmZQzV_;&6V31*CVt>xS${@oa&cMbXgv@5(U}E#}bYfr-VP>^+>@@d2w)}&o%MSeo
zv!g;7WLWX&V`B^P4`5(m;b353<7HxCWMJcCVqjp<5nu+ANPxM8fq_wnfdNJ%NwEmx
z(9ie<q6$QV^f4<SXpjt49$9}1vi*$O|A7FgP8+DN6-a-D(J*l^4c5;NbOQ?m09_6y
A1ONa4

literal 0
HcmV?d00001

diff --git a/testdata/drc/drcSimpleTests_au14b.lyrdb b/testdata/drc/drcSimpleTests_au14b.lyrdb
new file mode 100644
index 000000000..0b6c00a9d
--- /dev/null
+++ b/testdata/drc/drcSimpleTests_au14b.lyrdb
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="utf-8"?>
+<report-database>
+ <description>Report 1</description>
+ <original-file/>
+ <generator>drc: script='.drc'</generator>
+ <top-cell>TOP</top-cell>
+ <tags>
+ </tags>
+ <categories>
+  <category>
+   <name>l2 space &lt; 1µm</name>
+   <description/>
+   <categories>
+   </categories>
+  </category>
+  <category>
+   <name>l1 width &lt; 1µm</name>
+   <description/>
+   <categories>
+   </categories>
+  </category>
+ </categories>
+ <cells>
+  <cell>
+   <name>TOP</name>
+   <variant/>
+   <references>
+   </references>
+  </cell>
+ </cells>
+ <items>
+  <item>
+   <tags/>
+   <category>'l2 space &lt; 1\302\265m'</category>
+   <cell>TOP</cell>
+   <visited>false</visited>
+   <multiplicity>1</multiplicity>
+   <image/>
+   <values>
+    <value>edge-pair: (-0.2,0.7;1,0.7)|(1,1.1;-0.2,1.1)</value>
+   </values>
+  </item>
+  <item>
+   <tags/>
+   <category>'l1 width &lt; 1\302\265m'</category>
+   <cell>TOP</cell>
+   <visited>false</visited>
+   <multiplicity>1</multiplicity>
+   <image/>
+   <values>
+    <value>edge-pair: (0,0;0,0.9)|(0.3,0.9;0.3,0)</value>
+   </values>
+  </item>
+  <item>
+   <tags/>
+   <category>'l1 width &lt; 1\302\265m'</category>
+   <cell>TOP</cell>
+   <visited>false</visited>
+   <multiplicity>1</multiplicity>
+   <image/>
+   <values>
+    <value>edge-pair: (0.3,0;0,0)|(0,0.9;0.3,0.9)</value>
+   </values>
+  </item>
+  <item>
+   <tags/>
+   <category>'l1 width &lt; 1\302\265m'</category>
+   <cell>TOP</cell>
+   <visited>false</visited>
+   <multiplicity>1</multiplicity>
+   <image/>
+   <values>
+    <value>edge-pair: (0.5,0;0.5,0.9)|(0.8,0.9;0.8,0)</value>
+   </values>
+  </item>
+  <item>
+   <tags/>
+   <category>'l1 width &lt; 1\302\265m'</category>
+   <cell>TOP</cell>
+   <visited>false</visited>
+   <multiplicity>1</multiplicity>
+   <image/>
+   <values>
+    <value>edge-pair: (0.8,0;0.5,0)|(0.5,0.9;0.8,0.9)</value>
+   </values>
+  </item>
+ </items>
+</report-database>
diff --git a/testdata/drc/drcSimpleTests_au14b_2.gds b/testdata/drc/drcSimpleTests_au14b_2.gds
new file mode 100644
index 0000000000000000000000000000000000000000..b3a5126453d1b6ef03b105c8500bf45480ee142d
GIT binary patch
literal 362
zcmZQzV_;&6V31*CVt>xS${@oa&cMbXgv@5(U}E#}bYfr-VP>^+>@@d2w)}&o%MSeo
zv!g;7WLWX&V`B^P4`5(m;b353<7Hw<WnklDVqjp<5nu+ANPxM8fq_wnfdNJ%NwEmx
z(9ie<q6$QV^f4<SXpjt49$9}H!v6pN|62gnOn}f#dw_ahFfhPqm^kAE2E6)Nd?5N+
UD<Jw|G)$bu2cn-H=m!=C0Hg>sP5=M^

literal 0
HcmV?d00001

diff --git a/testdata/drc/drcSimpleTests_au14b_2.lyrdb b/testdata/drc/drcSimpleTests_au14b_2.lyrdb
new file mode 100644
index 000000000..f3430532f
--- /dev/null
+++ b/testdata/drc/drcSimpleTests_au14b_2.lyrdb
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="utf-8"?>
+<report-database>
+ <description>Report 2</description>
+ <original-file/>
+ <generator>drc: script='.drc'</generator>
+ <top-cell>TOP</top-cell>
+ <tags>
+ </tags>
+ <categories>
+  <category>
+   <name>l1 sep l2 &lt; 1µm</name>
+   <description/>
+   <categories>
+   </categories>
+  </category>
+ </categories>
+ <cells>
+  <cell>
+   <name>TOP</name>
+   <variant/>
+   <references>
+   </references>
+  </cell>
+ </cells>
+ <items>
+  <item>
+   <tags/>
+   <category>'l1 sep l2 &lt; 1\302\265m'</category>
+   <cell>TOP</cell>
+   <visited>false</visited>
+   <multiplicity>1</multiplicity>
+   <image/>
+   <values>
+    <value>edge-pair: (0,0.9;0.3,0.9)/(1,1.1;-0.2,1.1)</value>
+   </values>
+  </item>
+  <item>
+   <tags/>
+   <category>'l1 sep l2 &lt; 1\302\265m'</category>
+   <cell>TOP</cell>
+   <visited>false</visited>
+   <multiplicity>1</multiplicity>
+   <image/>
+   <values>
+    <value>edge-pair: (0.5,0.9;0.8,0.9)/(1,1.1;-0.2,1.1)</value>
+   </values>
+  </item>
+ </items>
+</report-database>

From 7f3950f5823590784e92d888568c9c241347635d Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 23:21:14 +0200
Subject: [PATCH 07/22] Fixed DRC profile feature

---
 src/drc/drc/built-in-macros/_drc_engine.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/drc/drc/built-in-macros/_drc_engine.rb b/src/drc/drc/built-in-macros/_drc_engine.rb
index 3c6c2f51d..275e3cf2e 100644
--- a/src/drc/drc/built-in-macros/_drc_engine.rb
+++ b/src/drc/drc/built-in-macros/_drc_engine.rb
@@ -2619,6 +2619,10 @@ CODE
 
     def _dump_profile
 
+      if @profile_info.empty?
+        return
+      end
+
       desc_title = "Operation"
       calls_title = "# calls"
       time_title = "Time (s)"
@@ -2711,7 +2715,7 @@ CODE
         end
 
         # dump the profile information
-        if @profile
+        if @profile && final
           _dump_profile
         end
         

From c28fd425dbe8f14042df1609640bcdbf8b398773 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 23:33:32 +0200
Subject: [PATCH 08/22] DRC bugfix

---
 src/drc/drc/built-in-macros/_drc_engine.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/drc/drc/built-in-macros/_drc_engine.rb b/src/drc/drc/built-in-macros/_drc_engine.rb
index 275e3cf2e..8493bbf85 100644
--- a/src/drc/drc/built-in-macros/_drc_engine.rb
+++ b/src/drc/drc/built-in-macros/_drc_engine.rb
@@ -2656,7 +2656,7 @@ CODE
         self.log(fmt % info)
         n += 1
         if @profile_n > 0 && n > @profile_n
-          self.log("... (%d entries skipped)" % (pi.size - nmax))
+          self.log("... (%d entries skipped)" % (pi.size - @profile_n))
           break
         end
       end
@@ -2669,7 +2669,7 @@ CODE
         self.log(fmt % info)
         n += 1
         if @profile_n > 0 && n > @profile_n
-          self.log("... (%d entries skipped)" % (pi.size - nmax))
+          self.log("... (%d entries skipped)" % (pi.size - @profile_n))
           break
         end
       end

From de2ace4be37453b87959b12cd24f0e54a6bcbd5d Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 28 May 2023 23:34:03 +0200
Subject: [PATCH 09/22] GDS2 reader: robustness against some broken files -
 should not segfault

---
 .../streamers/gds2/db_plugin/dbGDS2Reader.cc  | 22 +++++++++++++++----
 .../streamers/gds2/db_plugin/dbGDS2Reader.h   |  2 ++
 .../gds2/db_plugin/dbGDS2ReaderBase.cc        |  4 ++++
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.cc b/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.cc
index 8167b02d5..64253e5e0 100644
--- a/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.cc
+++ b/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.cc
@@ -127,11 +127,19 @@ GDS2Reader::get_record ()
   return rec_id;
 }
 
+void
+GDS2Reader::record_underflow_error ()
+{
+  error (tl::to_string (tr ("Record too short")));
+}
+
 inline int 
 GDS2Reader::get_int ()
 {
   unsigned char *b = mp_rec_buf + m_recptr;
-  m_recptr += 4;
+  if ((m_recptr += 4) > m_reclen) {
+    record_underflow_error ();
+  }
 
   int32_t l = *((int32_t *)b);
   gds2h (l);
@@ -142,7 +150,9 @@ inline short
 GDS2Reader::get_short ()
 {
   unsigned char *b = mp_rec_buf + m_recptr;
-  m_recptr += 2;
+  if ((m_recptr += 2) > m_reclen) {
+    record_underflow_error ();
+  }
 
   int16_t s = *((int16_t *)b);
   gds2h (s);
@@ -153,7 +163,9 @@ inline unsigned short
 GDS2Reader::get_ushort ()
 {
   unsigned char *b = mp_rec_buf + m_recptr;
-  m_recptr += 2;
+  if ((m_recptr += 2) > m_reclen) {
+    record_underflow_error ();
+  }
 
   uint16_t s = *((uint16_t *)b);
   gds2h ((int16_t &) s);
@@ -164,7 +176,9 @@ inline double
 GDS2Reader::get_double ()
 {
   unsigned char *b = mp_rec_buf + m_recptr;
-  m_recptr += 8;
+  if ((m_recptr += 8) > m_reclen) {
+    record_underflow_error ();
+  }
 
   uint32_t l0 = ((uint32_t *)b) [0];
   gds2h ((int32_t &) l0);
diff --git a/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.h b/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.h
index 33cce1401..17eb2b504 100644
--- a/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.h
+++ b/src/plugins/streamers/gds2/db_plugin/dbGDS2Reader.h
@@ -106,6 +106,8 @@ private:
   virtual void get_time (unsigned int *mod_time, unsigned int *access_time);
   virtual GDS2XY *get_xy_data (unsigned int &length);
   virtual void progress_checkpoint ();
+
+  void record_underflow_error ();
 };
 
 }
diff --git a/src/plugins/streamers/gds2/db_plugin/dbGDS2ReaderBase.cc b/src/plugins/streamers/gds2/db_plugin/dbGDS2ReaderBase.cc
index 90e84d65d..ad6eed607 100644
--- a/src/plugins/streamers/gds2/db_plugin/dbGDS2ReaderBase.cc
+++ b/src/plugins/streamers/gds2/db_plugin/dbGDS2ReaderBase.cc
@@ -278,6 +278,10 @@ GDS2ReaderBase::do_read (db::Layout &layout)
 
     get_string (m_cellname);
 
+    if (m_cellname.empty ()) {
+      error (tl::to_string (tr ("Empty cell name")));
+    }
+
     //  if the first cell is the dummy cell containing the context information
     //  read this cell in a special way and store the context information separately.
     if (first_cell && m_cellname == "$$$CONTEXT_INFO$$$") {

From c3bda162cbd9f3b8250ac105597fe9a56212aa96 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Mon, 29 May 2023 01:44:57 +0200
Subject: [PATCH 10/22] DRC: Added 'write' method to output channels, enabling
 GC in profile mode to get a more realistic memory usage picture

---
 src/drc/drc/built-in-macros/_drc_engine.rb | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/drc/drc/built-in-macros/_drc_engine.rb b/src/drc/drc/built-in-macros/_drc_engine.rb
index 8493bbf85..ea2ebb3d6 100644
--- a/src/drc/drc/built-in-macros/_drc_engine.rb
+++ b/src/drc/drc/built-in-macros/_drc_engine.rb
@@ -32,6 +32,9 @@ module DRC
     def finish(final)
       # reimplement
     end
+    def write
+      # reimplement
+    end
     def layout
       nil
     end
@@ -58,6 +61,10 @@ module DRC
     end
 
     def finish(final, view)
+      write
+    end
+
+    def write
       if @file_name
         opt = RBA::SaveLayoutOptions::new
         gzip = opt.set_format_from_filename(@file_name)
@@ -101,14 +108,18 @@ module DRC
     end
 
     def finish(final, view)
+      write
+      if final && view
+        view.show_rdb(@rdb_index, view.active_cellview_index)
+      end
+    end
+
+    def write
       if @file_name
         rdb_file = @engine._make_path(@file_name)
         @engine.info("Writing report database: #{rdb_file} ..")
         @rdb.save(rdb_file)
       end
-      if final && view
-        view.show_rdb(@rdb_index, view.active_cellview_index)
-      end
     end
 
     def rdb
@@ -2372,7 +2383,7 @@ CODE
 
       t.start
       self._process_events
-      if @force_gc || Time::now - @time > 0.5
+      if @force_gc || Time::now - @time > 0.5 || @profile
         GC.start    # force a garbage collection before the operation to free unused memory
         @time = Time::now
       end

From 987bb005daa27b794218221a0c8f5976e7c875d3 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Wed, 31 May 2023 20:32:34 +0200
Subject: [PATCH 11/22] Fixed issue #1393 (GDS2 text format not supported in
 python module)

---
 setup.py                    | 2 ++
 testdata/pymod/import_db.py | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/setup.py b/setup.py
index 46e44fd58..744b46547 100644
--- a/setup.py
+++ b/setup.py
@@ -805,11 +805,13 @@ for pi in dbpi_dirs:
     mod_name = "_" + os.path.split(os.path.split(pi)[-2])[-1] + "_dbpi"
 
     pi_sources = glob.glob(os.path.join(pi, "*.cc"))
+    pi_sources += glob.glob(os.path.join(pi, "contrib", "*.cc"))
 
     pi_ext = Library(
         config.root + ".db_plugins." + mod_name,
         define_macros=config.macros() + [("MAKE_DB_PLUGIN_LIBRARY", 1)],
         include_dirs=[
+            pi,
             os.path.join("src", "plugins", "common"),
             _db_path,
             _tl_path,
diff --git a/testdata/pymod/import_db.py b/testdata/pymod/import_db.py
index 2c1881260..154bd7198 100755
--- a/testdata/pymod/import_db.py
+++ b/testdata/pymod/import_db.py
@@ -41,6 +41,12 @@ class BasicTest(unittest.TestCase):
     v.read(os.path.join(os.path.dirname(__file__), "..", "gds", "t10.gds"))
     self.assertEqual(v.top_cell().name, "RINGO")
 
+  def test_4(self):
+    # gds2_text plugin loaded? (issue #1393)
+    v = db.Layout()
+    v.read(os.path.join(os.path.dirname(__file__), "..", "gds2_txt", "read.txt"))
+    self.assertEqual(v.top_cell().name, "RINGO")
+
 # run unit tests
 if __name__ == '__main__':
   suite = unittest.TestSuite()

From f11500996f362e49dd7af26aae08fd74a4de0a54 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Wed, 31 May 2023 22:13:22 +0200
Subject: [PATCH 12/22] New methods: Layout#copy_layer, Layout#move_layer and
 Layout#clear_layer with shape selector, Shapes#clear with shape selector

---
 src/db/db/dbCell.cc                | 34 ++++++++++++++
 src/db/db/dbCell.h                 | 21 ++++++++-
 src/db/db/dbLayout.cc              | 41 +++++++++++++++--
 src/db/db/dbLayout.h               | 29 ++++++++++--
 src/db/db/dbShapes.cc              | 71 ++++++++++++++++++++++++++----
 src/db/db/dbShapes.h               | 19 +++++++-
 src/db/db/gsiDeclDbLayout.cc       | 56 ++++++++++++++++++-----
 src/db/db/gsiDeclDbShapes.cc       | 53 +++++++++++++---------
 src/db/unit_tests/dbShapesTests.cc | 45 +++++++++++++++++++
 src/lay/lay/layGSIHelpProvider.cc  |  4 ++
 testdata/ruby/dbLayoutTests1.rb    | 60 +++++++++++++++++++++++++
 testdata/ruby/dbShapesTest.rb      | 28 ++++++++++++
 12 files changed, 414 insertions(+), 47 deletions(-)

diff --git a/src/db/db/dbCell.cc b/src/db/db/dbCell.cc
index 64efb2aaf..83acc1182 100644
--- a/src/db/db/dbCell.cc
+++ b/src/db/db/dbCell.cc
@@ -187,6 +187,17 @@ Cell::clear (unsigned int index)
   }
 }
 
+void
+Cell::clear (unsigned int index, unsigned int types)
+{
+  shapes_map::iterator s = m_shapes_map.find(index);
+  if (s != m_shapes_map.end() && ! s->second.empty ()) {
+    mp_layout->invalidate_bboxes (index);  //  HINT: must come before the change is done!
+    s->second.clear (types);
+    m_bbox_needs_update = true;
+  }
+}
+
 Cell::shapes_type &
 Cell::shapes (unsigned int index) 
 {
@@ -344,6 +355,20 @@ Cell::copy (unsigned int src, unsigned int dest)
   }
 }
 
+void
+Cell::copy (unsigned int src, unsigned int dest, unsigned int types)
+{
+  if (src != dest) {
+    shapes (dest).insert (shapes (src), types);
+  } else {
+    //  When duplicating the layer, first create a copy to avoid problems with non-stable containers
+    //  Hint: using the assignment and not the copy ctor does not copy the db::Manager association.
+    db::Shapes shape_copy;
+    shape_copy.insert (shapes (src), types);
+    shapes (dest).insert (shape_copy);
+  }
+}
+
 void
 Cell::move (unsigned int src, unsigned int dest)
 {
@@ -353,6 +378,15 @@ Cell::move (unsigned int src, unsigned int dest)
   }
 }
 
+void
+Cell::move (unsigned int src, unsigned int dest, unsigned int types)
+{
+  if (src != dest) {
+    copy (src, dest, types);
+    clear (src, types);
+  }
+}
+
 void
 Cell::swap (unsigned int i1, unsigned int i2)
 {
diff --git a/src/db/db/dbCell.h b/src/db/db/dbCell.h
index 00319ee08..2d04bc0ba 100644
--- a/src/db/db/dbCell.h
+++ b/src/db/db/dbCell.h
@@ -182,6 +182,13 @@ public:
    */
   void copy (unsigned int src, unsigned int dest);
 
+  /**
+   *  @brief Copy the shapes from layer src to dest (only shapes from given classes)
+   *
+   *  The target layer is not overwritten. Instead, the shapes are added to the target layer's shapes.
+   */
+  void copy (unsigned int src, unsigned int dest, unsigned int types);
+
   /**
    *  @brief Move the shapes from layer src to dest
    *
@@ -189,6 +196,13 @@ public:
    */
   void move (unsigned int src, unsigned int dest);
 
+  /**
+   *  @brief Move the shapes from layer src to dest (only shapes from given classes)
+   *
+   *  The target layer is not overwritten. Instead, the shapes are added to the target layer's shapes.
+   */
+  void move (unsigned int src, unsigned int dest, unsigned int types);
+
   /**
    *  @brief Swap the layers given
    */
@@ -199,7 +213,12 @@ public:
    */
   void clear (unsigned int index);
 
-  /** 
+  /**
+   *  @brief Clear the shapes on the given layer (only the shapes from the given classes)
+   */
+  void clear (unsigned int index, unsigned int types);
+
+  /**
    *  @brief Erase a cell instance given by a instance proxy
    *
    *  Erasing a cell instance will destroy the sorting order and invalidate
diff --git a/src/db/db/dbLayout.cc b/src/db/db/dbLayout.cc
index 0b53b70a1..32bba3042 100644
--- a/src/db/db/dbLayout.cc
+++ b/src/db/db/dbLayout.cc
@@ -1971,7 +1971,19 @@ Layout::move_layer (unsigned int src, unsigned int dest)
   }
 }
 
-void 
+void
+Layout::move_layer (unsigned int src, unsigned int dest, unsigned int flags)
+{
+  tl_assert (m_layers.layer_state (src) != LayoutLayers::Free);
+  tl_assert (m_layers.layer_state (dest) != LayoutLayers::Free);
+
+  //  move the shapes
+  for (iterator c = begin (); c != end (); ++c) {
+    c->move (src, dest, flags);
+  }
+}
+
+void
 Layout::copy_layer (unsigned int src, unsigned int dest)
 {
   tl_assert (m_layers.layer_state (src) != LayoutLayers::Free);
@@ -1983,7 +1995,19 @@ Layout::copy_layer (unsigned int src, unsigned int dest)
   }
 }
 
-void 
+void
+Layout::copy_layer (unsigned int src, unsigned int dest, unsigned int flags)
+{
+  tl_assert (m_layers.layer_state (src) != LayoutLayers::Free);
+  tl_assert (m_layers.layer_state (dest) != LayoutLayers::Free);
+
+  //  copy the shapes
+  for (iterator c = begin (); c != end (); ++c) {
+    c->copy (src, dest, flags);
+  }
+}
+
+void
 Layout::clear_layer (unsigned int n)
 {
   tl_assert (m_layers.layer_state (n) != LayoutLayers::Free);
@@ -1994,7 +2018,18 @@ Layout::clear_layer (unsigned int n)
   }
 }
 
-void 
+void
+Layout::clear_layer (unsigned int n, unsigned int flags)
+{
+  tl_assert (m_layers.layer_state (n) != LayoutLayers::Free);
+
+  //  clear the shapes
+  for (iterator c = begin (); c != end (); ++c) {
+    c->clear (n, flags);
+  }
+}
+
+void
 Layout::delete_layer (unsigned int n)
 {
   tl_assert (m_layers.layer_state (n) != LayoutLayers::Free);
diff --git a/src/db/db/dbLayout.h b/src/db/db/dbLayout.h
index cb79d2a8a..353ad2ad9 100644
--- a/src/db/db/dbLayout.h
+++ b/src/db/db/dbLayout.h
@@ -1377,7 +1377,15 @@ public:
    */
   void move_layer (unsigned int src, unsigned int dest);
 
-  /**  
+  /**
+   *  @brief Move a layer (selected shape types only)
+   *
+   *  Move a layer from the source to the target. The target is not cleared before, so that this method
+   *  merges shapes from the source with the target layer. The source layer is empty after that operation.
+   */
+  void move_layer (unsigned int src, unsigned int dest, unsigned int flags);
+
+  /**
    *  @brief Copy a layer
    *
    *  Copy a layer from the source to the target. The target is not cleared before, so that this method 
@@ -1385,14 +1393,29 @@ public:
    */
   void copy_layer (unsigned int src, unsigned int dest);
 
-  /**  
+  /**
+   *  @brief Copy a layer (selected shape types only)
+   *
+   *  Copy a layer from the source to the target. The target is not cleared before, so that this method
+   *  merges shapes from the source with the target layer.
+   */
+  void copy_layer (unsigned int src, unsigned int dest, unsigned int flags);
+
+  /**
    *  @brief Clear a layer
    *
    *  Clears the layer: removes all shapes.
    */
   void clear_layer (unsigned int n);
 
-  /**  
+  /**
+   *  @brief Clear a layer (selected shapes only)
+   *
+   *  Clears the layer: removes the shapes of the type given the flags (ShapeIterator::shapes_type)
+   */
+  void clear_layer (unsigned int n, unsigned int flags);
+
+  /**
    *  @brief Delete a layer
    *
    *  This does free the shapes of the cells and remembers the
diff --git a/src/db/db/dbShapes.cc b/src/db/db/dbShapes.cc
index d8b439404..48736115d 100644
--- a/src/db/db/dbShapes.cc
+++ b/src/db/db/dbShapes.cc
@@ -81,6 +81,12 @@ inline bool needs_translate (object_tag<Sh> /*tag*/)
   return tl::is_equal_type<typename shape_traits<Sh>::can_deref, tl::True> () || tl::is_equal_type<typename shape_traits<Sh>::is_array, tl::True> ();
 }
 
+inline bool type_mask_applies (const db::LayerBase *layer, unsigned int flags)
+{
+  unsigned int tm = layer->type_mask ();
+  return (((flags & db::ShapeIterator::Properties) == 0 || (tm & db::ShapeIterator::Properties) != 0) && (flags & tm) != 0);
+}
+
 // ---------------------------------------------------------------------------------------
 //  layer_op implementation
 
@@ -214,7 +220,13 @@ Shapes::insert (const Shapes &d)
 }
 
 void
-Shapes::do_insert (const Shapes &d)
+Shapes::insert (const Shapes &d, unsigned int flags)
+{
+  do_insert (d, flags);
+}
+
+void
+Shapes::do_insert (const Shapes &d, unsigned int flags)
 {
   //  shortcut for "nothing to do"
   if (d.empty ()) {
@@ -228,10 +240,12 @@ Shapes::do_insert (const Shapes &d)
 
       m_layers.reserve (d.m_layers.size ());
       for (tl::vector<LayerBase *>::const_iterator l = d.m_layers.begin (); l != d.m_layers.end (); ++l) {
-        m_layers.push_back ((*l)->clone ());
-        if (manager () && manager ()->transacting ()) {
-          check_is_editable_for_undo_redo ();
-          manager ()->queue (this, new FullLayerOp (true, m_layers.back ()));
+        if (type_mask_applies (*l, flags)) {
+          m_layers.push_back ((*l)->clone ());
+          if (manager () && manager ()->transacting ()) {
+            check_is_editable_for_undo_redo ();
+            manager ()->queue (this, new FullLayerOp (true, m_layers.back ()));
+          }
         }
       }
 
@@ -239,7 +253,9 @@ Shapes::do_insert (const Shapes &d)
 
     } else {
       for (tl::vector<LayerBase *>::const_iterator l = d.m_layers.begin (); l != d.m_layers.end (); ++l) {
-        (*l)->insert_into (this);
+        if (type_mask_applies (*l, flags)) {
+          (*l)->insert_into (this);
+        }
       }
     }
 
@@ -247,14 +263,18 @@ Shapes::do_insert (const Shapes &d)
 
     //  the target is standalone - dereference
     for (tl::vector<LayerBase *>::const_iterator l = d.m_layers.begin (); l != d.m_layers.end (); ++l) {
-      (*l)->deref_into (this);
+      if (type_mask_applies (*l, flags)) {
+        (*l)->deref_into (this);
+      }
     }
 
   } else {
 
     //  both shape containers are in separate spaces - translate
     for (tl::vector<LayerBase *>::const_iterator l = d.m_layers.begin (); l != d.m_layers.end (); ++l) {
-      (*l)->translate_into (this, shape_repository (), array_repository ());
+      if (type_mask_applies (*l, flags)) {
+        (*l)->translate_into (this, shape_repository (), array_repository ());
+      }
     }
 
   }
@@ -1046,6 +1066,41 @@ Shapes::clear ()
   }
 }
 
+void
+Shapes::clear (unsigned int flags)
+{
+  if (!m_layers.empty ()) {
+
+    invalidate_state ();  //  HINT: must come before the change is done!
+
+    tl::vector<LayerBase *> new_layers;
+
+    for (tl::vector<LayerBase *>::const_iterator l = m_layers.end (); l != m_layers.begin (); ) {
+
+      //  because the undo stack will do a push, we need to remove layers from the back (this is the last undo
+      //  element to be executed)
+      --l;
+
+      if (type_mask_applies (*l, flags)) {
+
+        if (manager () && manager ()->transacting ()) {
+          check_is_editable_for_undo_redo ();
+          manager ()->queue (this, new FullLayerOp (false, (*l)));
+        } else {
+          delete *l;
+        }
+
+      } else {
+        new_layers.push_back (*l);
+      }
+
+    }
+
+    m_layers.swap (new_layers);
+
+  }
+}
+
 void Shapes::reset_bbox_dirty ()
 {
   set_dirty (false);
diff --git a/src/db/db/dbShapes.h b/src/db/db/dbShapes.h
index b8114e360..bffb08315 100644
--- a/src/db/db/dbShapes.h
+++ b/src/db/db/dbShapes.h
@@ -628,6 +628,18 @@ public:
    */
   void insert (const Shapes &d);
 
+  /**
+   *  @brief Insert all shapes from another container using the given shape types only
+   *
+   *  This method insert all shapes from the given shape container.
+   *
+   *  HINT: This method can duplicate shape containers from one layout to another.
+   *  The current implementation does not translate property Id's.
+   *  It is mainly intended for 1-to-1 copies of layouts where the whole
+   *  property repository is copied.
+   */
+  void insert (const Shapes &d, unsigned int types);
+
   /**
    *  @brief Assignment operator with transformation
    *
@@ -1219,6 +1231,11 @@ public:
    */
   void clear ();
 
+  /**
+   *  @brief Clears the collection (given shape types only)
+   */
+  void clear (unsigned int types);
+
   /**
    *  @brief Report the type mask of the objects stored herein
    *
@@ -1515,7 +1532,7 @@ private:
   db::Cell *mp_cell;  //  HINT: contains "dirty" in bit 0 and "editable" in bit 1
 
   void invalidate_state ();
-  void do_insert (const Shapes &d);
+  void do_insert (const Shapes &d, unsigned int flags = db::ShapeIterator::All);
   void check_is_editable_for_undo_redo () const;
 
   //  gets the layers array
diff --git a/src/db/db/gsiDeclDbLayout.cc b/src/db/db/gsiDeclDbLayout.cc
index 8285bc5fa..9da66889d 100644
--- a/src/db/db/gsiDeclDbLayout.cc
+++ b/src/db/db/gsiDeclDbLayout.cc
@@ -1682,29 +1682,53 @@ Class<db::Layout> decl_Layout ("db", "Layout",
     "@param a The first of the layers to swap.\n"
     "@param b The second of the layers to swap.\n"
   ) +
-  gsi::method ("move_layer", &db::Layout::move_layer, gsi::arg ("src"), gsi::arg ("dest"),
+  gsi::method ("move_layer", static_cast<void (db::Layout::*) (unsigned int, unsigned int)> (&db::Layout::move_layer), gsi::arg ("src"), gsi::arg ("dest"),
     "@brief Moves a layer\n"
     "\n"
-    "This method was introduced in version 0.19.\n"
-    "\n"
-    "Move a layer from the source to the target. The target is not cleared before, so that this method \n"
-    "merges shapes from the source with the target layer. The source layer is empty after that operation.\n"
+    "Moves a layer from the source to the destination layer. The target is not cleared before, so that this method \n"
+    "merges shapes from the source with the destination layer. The source layer is empty after that operation.\n"
     "\n"
     "@param src The layer index of the source layer.\n"
     "@param dest The layer index of the destination layer.\n"
+    "\n"
+    "This method was introduced in version 0.19.\n"
   ) +
-  gsi::method ("copy_layer", &db::Layout::copy_layer, gsi::arg ("src"), gsi::arg ("dest"),
+  gsi::method ("move_layer", static_cast<void (db::Layout::*) (unsigned int, unsigned int, unsigned int)> (&db::Layout::move_layer), gsi::arg ("src"), gsi::arg ("dest"), gsi::arg ("flags"),
+    "@brief Moves a layer (selected shape types only)\n"
+    "\n"
+    "Moves a layer from the source to the destination layer. The target is not cleared before, so that this method \n"
+    "merges shapes from the source with the destination layer. The copied shapes are removed from the source layer.\n"
+    "\n"
+    "@param src The layer index of the source layer.\n"
+    "@param dest The layer index of the destination layer.\n"
+    "@param flags A combination of the shape type flags from \\Shapes, S... constants\n"
+    "\n"
+    "This method variant has been introduced in version 0.28.9.\n"
+  ) +
+  gsi::method ("copy_layer", static_cast<void (db::Layout::*) (unsigned int, unsigned int)> (&db::Layout::copy_layer), gsi::arg ("src"), gsi::arg ("dest"),
     "@brief Copies a layer\n"
     "\n"
-    "This method was introduced in version 0.19.\n"
-    "\n"
-    "Copy a layer from the source to the target. The target is not cleared before, so that this method \n"
-    "merges shapes from the source with the target layer.\n"
+    "Copies a layer from the source to the destination layer. The destination layer is not cleared before, so that this method \n"
+    "merges shapes from the source with the destination layer.\n"
     "\n"
     "@param src The layer index of the source layer.\n"
     "@param dest The layer index of the destination layer.\n"
+    "\n"
+    "This method was introduced in version 0.19.\n"
   ) +
-  gsi::method ("clear_layer", &db::Layout::clear_layer, gsi::arg ("layer_index"),
+  gsi::method ("copy_layer", static_cast<void (db::Layout::*) (unsigned int, unsigned int, unsigned int)> (&db::Layout::copy_layer), gsi::arg ("src"), gsi::arg ("dest"), gsi::arg ("flags"),
+    "@brief Copies a layer (selected shape types only)\n"
+    "\n"
+    "Copies a layer from the source to the destination layer. The destination layer is not cleared before, so that this method \n"
+    "merges shapes from the source with the destination layer.\n"
+    "\n"
+    "@param src The layer index of the source layer.\n"
+    "@param dest The layer index of the destination layer.\n"
+    "@param flags A combination of the shape type flags from \\Shapes, S... constants\n"
+    "\n"
+    "This method variant has been introduced in version 0.28.9.\n"
+  ) +
+  gsi::method ("clear_layer", static_cast<void (db::Layout::*) (unsigned int)> (&db::Layout::clear_layer), gsi::arg ("layer_index"),
     "@brief Clears a layer\n"
     "\n"
     "Clears the layer: removes all shapes.\n"
@@ -1712,6 +1736,16 @@ Class<db::Layout> decl_Layout ("db", "Layout",
     "This method was introduced in version 0.19.\n"
     "\n"
     "@param layer_index The index of the layer to delete.\n"
+  ) +                               
+  gsi::method ("clear_layer", static_cast<void (db::Layout::*) (unsigned int, unsigned int)> (&db::Layout::clear_layer), gsi::arg ("layer_index"), gsi::arg ("flags"),
+    "@brief Clears a layer (given shape types only)\n"
+    "\n"
+    "Clears the layer: removes all shapes for the given shape types.\n"
+    "\n"
+    "This method was introduced in version 0.28.9.\n"
+    "\n"
+    "@param layer_index The index of the layer to delete.\n"
+    "@param flags The type selector for the shapes to delete (see \\Shapes class, S... constants).\n"
   ) +
   gsi::method ("delete_layer", &db::Layout::delete_layer, gsi::arg ("layer_index"),
     "@brief Deletes a layer\n"
diff --git a/src/db/db/gsiDeclDbShapes.cc b/src/db/db/gsiDeclDbShapes.cc
index e711fe0c6..33f1a90c5 100644
--- a/src/db/db/gsiDeclDbShapes.cc
+++ b/src/db/db/gsiDeclDbShapes.cc
@@ -108,47 +108,47 @@ static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin (const db
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin (flags));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_all (const db::Shapes *s)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_all (const db::Shapes *s)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin (db::ShapeIterator::All));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_overlapping (const db::Shapes *s, unsigned int flags, const db::Box &region)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_overlapping (const db::Shapes *s, unsigned int flags, const db::Box &region)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin_overlapping (region, flags));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_doverlapping (const db::Shapes *s, unsigned int flags, const db::DBox &region)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_doverlapping (const db::Shapes *s, unsigned int flags, const db::DBox &region)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin_overlapping (db::CplxTrans (shapes_dbu (s)).inverted () * region, flags));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_overlapping_all (const db::Shapes *s, const db::Box &region)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_overlapping_all (const db::Shapes *s, const db::Box &region)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin_overlapping (region, db::ShapeIterator::All));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_doverlapping_all (const db::Shapes *s, const db::DBox &region)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_doverlapping_all (const db::Shapes *s, const db::DBox &region)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin_overlapping (db::CplxTrans (shapes_dbu (s)).inverted () * region, db::ShapeIterator::All));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_touching (const db::Shapes *s, unsigned int flags, const db::Box &region)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_touching (const db::Shapes *s, unsigned int flags, const db::Box &region)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin_touching (region, flags));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_dtouching (const db::Shapes *s, unsigned int flags, const db::DBox &region)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_dtouching (const db::Shapes *s, unsigned int flags, const db::DBox &region)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin_touching (db::CplxTrans (shapes_dbu (s)).inverted () * region, flags));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_touching_all (const db::Shapes *s, const db::Box &region)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_touching_all (const db::Shapes *s, const db::Box &region)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin_touching (region, db::ShapeIterator::All));
 }
 
-static gsi::layout_locking_iterator1<db::Shapes::shape_iterator>begin_dtouching_all (const db::Shapes *s, const db::DBox &region)
+static gsi::layout_locking_iterator1<db::Shapes::shape_iterator> begin_dtouching_all (const db::Shapes *s, const db::DBox &region)
 {
   return gsi::layout_locking_iterator1<db::Shapes::shape_iterator> (s->layout (), s->begin_touching (db::CplxTrans (shapes_dbu (s)).inverted () * region, db::ShapeIterator::All));
 }
@@ -251,12 +251,7 @@ static void insert_shapes (db::Shapes *sh, const db::Shapes &s)
 
 static void insert_shapes_with_flags (db::Shapes *sh, const db::Shapes &s, unsigned int flags)
 {
-  //  NOTE: if the source (r) is from the same layout than the shapes live in, we better
-  //  lock the layout against updates while inserting
-  db::LayoutLocker locker (sh->layout ());
-  for (db::Shapes::shape_iterator i = s.begin (flags); !i.at_end(); ++i) {
-    sh->insert (*i);
-  }
+  sh->insert (s, flags);
 }
 
 static void insert_shapes_with_trans (db::Shapes *sh, const db::Shapes &s, const db::ICplxTrans &trans)
@@ -1264,9 +1259,15 @@ Class<db::Shapes> decl_Shapes ("db", "Shapes",
     "@brief Returns a value indicating whether the shapes container is empty\n"
     "This method has been introduced in version 0.20.\n"
   ) +
-  gsi::method ("clear", &db::Shapes::clear, 
+  gsi::method ("clear", static_cast<void (db::Shapes::*) ()> (&db::Shapes::clear),
     "@brief Clears the shape container\n"
-    "This method has been introduced in version 0.16. It can only be used in editable mode."
+    "This method has been introduced in version 0.16."
+  ) +
+  gsi::method ("clear", static_cast<void (db::Shapes::*) (unsigned int)> (&db::Shapes::clear), gsi::arg ("flags"),
+    "@brief Clears certain shape types from the shape container\n"
+    "Only shapes matching the shape types from 'flags' are removed. 'flags' is a combination of the S... constants.\n"
+    "\n"
+    "This method has been introduced in version 0.28.9."
   ) +
   gsi::method_ext ("size", &shapes_size,
     "@brief Gets the number of shapes in this container\n"
@@ -1296,10 +1297,19 @@ Class<db::Shapes> decl_Shapes ("db", "Shapes",
     "returned for future references."
   ) +
   gsi::method ("SAll|#s_all", &s_all,
-    "@brief Indicates that all shapes shall be retrieved"
+    "@brief Indicates that all shapes shall be retrieved\n"
+    "You can use this constant to construct 'except' classes - e.g. "
+    "to specify 'all shape types except boxes' use\n"
+    "\n"
+    "@code SAll - SBoxes @/code\n"
   ) +
   gsi::method ("SAllWithProperties|#s_all_with_properties", &s_all_with_properties,
-    "@brief Indicates that all shapes with properties shall be retrieved"
+    "@brief Indicates that all shapes with properties shall be retrieved\n"
+    "Using this selector means to retrieve only shapes with properties."
+    "You can use this constant to construct 'except' classes - e.g. "
+    "to specify 'all shape types with properties except boxes' use\n"
+    "\n"
+    "@code SAllWithProperties - SBoxes @/code\n"
   ) +
   gsi::method ("SPolygons|#s_polygons", &s_polygons,
     "@brief Indicates that polygons shall be retrieved"
@@ -1333,7 +1343,10 @@ Class<db::Shapes> decl_Shapes ("db", "Shapes",
     "@brief Indicates that user objects shall be retrieved"
   ) +
   gsi::method ("SProperties|#s_properties", &s_properties,
-    "@brief Indicates that only shapes with properties shall be retrieved"
+    "@brief Indicates that only shapes with properties shall be retrieved\n"
+    "You can or-combine this flag with the plain shape types to select a "
+    "certain shape type, but only those shapes with properties. For example to "
+    "select boxes with properties, use 'SProperties | SBoxes'."
   ) +
   gsi::method_ext ("dump_mem_statistics", &dump_mem_statistics, gsi::arg<bool> ("detailed", false),
     "@hide"
diff --git a/src/db/unit_tests/dbShapesTests.cc b/src/db/unit_tests/dbShapesTests.cc
index b2fce8945..1648f0885 100644
--- a/src/db/unit_tests/dbShapesTests.cc
+++ b/src/db/unit_tests/dbShapesTests.cc
@@ -1925,6 +1925,51 @@ TEST(7)
   }
 }
 
+//  copy, move, clear with shape types
+TEST(8)
+{
+  db::Manager m (true);
+
+  db::Layout layout (&m);
+  unsigned int lindex1 = layout.insert_layer ();
+  unsigned int lindex2 = layout.insert_layer ();
+
+  db::Cell &topcell = layout.cell (layout.add_cell ("TOP"));
+
+  topcell.shapes (lindex1).insert (db::Box (1, 2, 3, 4));
+  topcell.shapes (lindex1).insert (db::Polygon (db::Box (1, 2, 3, 4)));
+
+  {
+    db::Transaction trans (&m, "T1");
+    topcell.shapes (lindex2).insert (topcell.shapes (lindex1));
+    EXPECT_EQ (shapes_to_string (_this, topcell.shapes (lindex2)), "polygon (1,2;1,4;3,4;3,2) #0\nbox (1,2;3,4) #0\n");
+  }
+
+  m.undo ();
+  EXPECT_EQ (shapes_to_string (_this, topcell.shapes (lindex2)), "");
+
+  {
+    db::Transaction trans (&m, "T1");
+    topcell.shapes (lindex2).insert (topcell.shapes (lindex1), db::ShapeIterator::Boxes);
+    EXPECT_EQ (shapes_to_string (_this, topcell.shapes (lindex2)), "box (1,2;3,4) #0\n");
+  }
+
+  m.undo ();
+  EXPECT_EQ (shapes_to_string (_this, topcell.shapes (lindex2)), "");
+
+  topcell.shapes (lindex2).insert (topcell.shapes (lindex1));
+  EXPECT_EQ (shapes_to_string (_this, topcell.shapes (lindex2)), "polygon (1,2;1,4;3,4;3,2) #0\nbox (1,2;3,4) #0\n");
+
+  {
+    db::Transaction trans (&m, "T1");
+    topcell.shapes (lindex2).clear (db::ShapeIterator::Polygons);
+    EXPECT_EQ (shapes_to_string (_this, topcell.shapes (lindex2)), "box (1,2;3,4) #0\n");
+  }
+
+  m.undo ();
+  EXPECT_EQ (shapes_to_string (_this, topcell.shapes (lindex2)), shapes_to_string (_this, topcell.shapes (lindex1)));
+}
+
 TEST(10A)
 {
   if (db::default_editable_mode ()) { 
diff --git a/src/lay/lay/layGSIHelpProvider.cc b/src/lay/lay/layGSIHelpProvider.cc
index eda190c67..c79df181f 100644
--- a/src/lay/lay/layGSIHelpProvider.cc
+++ b/src/lay/lay/layGSIHelpProvider.cc
@@ -118,6 +118,10 @@ std::string escape_xml_with_formatting (const std::string &s, bool &in_code)
         r += "<u>";
       } else if (sc.test ("@/u")) {
         r += "</u>";
+      } else if (sc.test ("@tt")) {
+        r += "<tt>";
+      } else if (sc.test ("@/tt")) {
+        r += "</tt>";
       } else if (sc.test ("@i")) {
         r += "<i>";
       } else if (sc.test ("@/i")) {
diff --git a/testdata/ruby/dbLayoutTests1.rb b/testdata/ruby/dbLayoutTests1.rb
index dca1b4b8b..c6734efb3 100644
--- a/testdata/ruby/dbLayoutTests1.rb
+++ b/testdata/ruby/dbLayoutTests1.rb
@@ -1971,6 +1971,66 @@ class DBLayoutTests1_TestClass < TestBase
 
   end
 
+  def test_23
+
+    # layer operations with shape types
+   
+    m = RBA::Manager::new
+
+    l = RBA::Layout.new(m)
+    l1 = l.insert_layer(RBA::LayerInfo.new(1, 0))
+    l2 = l.insert_layer(RBA::LayerInfo.new(2, 0))
+    c0 = l.cell(l.add_cell("c0"))
+    c1 = l.cell(l.add_cell("c1"))
+  
+    c0.shapes(l1).insert(RBA::Box::new(1, 2, 3, 4))
+    c1.shapes(l1).insert(RBA::Polygon::new(RBA::Box::new(1, 2, 3, 4)))
+
+    str1 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l1).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str1, "c0:box (1,2;3,4)\nc1:polygon (1,2;1,4;3,4;3,2)")
+
+    m.transaction("T")
+    l.clear_layer(l1, RBA::Shapes::SPolygons)
+    m.commit
+
+    str1 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l1).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str1, "c0:box (1,2;3,4)\nc1:")
+
+    m.undo 
+
+    str1 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l1).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str1, "c0:box (1,2;3,4)\nc1:polygon (1,2;1,4;3,4;3,2)")
+
+    m.transaction("T")
+    l.move_layer(l1, l2, RBA::Shapes::SPolygons)
+    m.commit
+
+    str1 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l1).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str1, "c0:box (1,2;3,4)\nc1:")
+
+    str2 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l2).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str2, "c0:\nc1:polygon (1,2;1,4;3,4;3,2)")
+
+    m.undo 
+
+    str1 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l1).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str1, "c0:box (1,2;3,4)\nc1:polygon (1,2;1,4;3,4;3,2)")
+
+    str2 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l2).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str2, "c0:\nc1:")
+
+    m.transaction("T")
+    l.copy_layer(l1, l2, RBA::Shapes::SPolygons)
+    m.commit
+
+    str1 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l1).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str1, "c0:box (1,2;3,4)\nc1:polygon (1,2;1,4;3,4;3,2)")
+
+    str2 = l.each_cell.collect { |c| c.name + ":" + c.shapes(l2).each.collect { |sh| sh.to_s }.join(";") }.join("\n")
+    assert_equal(str2, "c0:\nc1:polygon (1,2;1,4;3,4;3,2)")
+
+  end
+
   # Iterating while flatten
   def test_issue200
 
diff --git a/testdata/ruby/dbShapesTest.rb b/testdata/ruby/dbShapesTest.rb
index 76bb1df85..a55ffa4f8 100644
--- a/testdata/ruby/dbShapesTest.rb
+++ b/testdata/ruby/dbShapesTest.rb
@@ -1595,6 +1595,34 @@ class DBShapes_TestClass < TestBase
 
   end
 
+  # Shapes with shape-type specific insert and clear
+  def test_11
+
+    s = RBA::Shapes::new
+    s.insert(RBA::Box::new(1, 2, 3, 4))
+    s.insert(RBA::Polygon::new(RBA::Box::new(1, 2, 3, 4)))
+
+    assert_equal(s.each.collect { |sh| sh.to_s }.join("; "), "polygon (1,2;1,4;3,4;3,2); box (1,2;3,4)")
+
+    s2 = RBA::Shapes::new
+    s2.insert(s)
+
+    assert_equal(s2.each.collect { |sh| sh.to_s }.join("; "), "polygon (1,2;1,4;3,4;3,2); box (1,2;3,4)")
+
+    s2.clear(RBA::Shapes::SPolygons)
+
+    assert_equal(s2.each.collect { |sh| sh.to_s }.join("; "), "box (1,2;3,4)")
+
+    s2.clear
+    
+    assert_equal(s2.each.collect { |sh| sh.to_s }.join("; "), "")
+
+    s2.insert(s, RBA::Shapes::SPolygons)
+
+    assert_equal(s2.each.collect { |sh| sh.to_s }.join("; "), "polygon (1,2;1,4;3,4;3,2)")
+
+  end
+
 end
 
 load("test_epilogue.rb")

From 2d5c67f74f8140adc1c13f4a71be8de296679c4a Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Thu, 1 Jun 2023 20:28:17 +0200
Subject: [PATCH 13/22] Allowing DRC decks with only layout inputs (no default
 input)

---
 src/drc/drc/built-in-macros/_drc_engine.rb | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/drc/drc/built-in-macros/_drc_engine.rb b/src/drc/drc/built-in-macros/_drc_engine.rb
index ea2ebb3d6..6bc5f4c72 100644
--- a/src/drc/drc/built-in-macros/_drc_engine.rb
+++ b/src/drc/drc/built-in-macros/_drc_engine.rb
@@ -2740,8 +2740,6 @@ CODE
           end
           if cv_index
           
-            view = RBA::LayoutView::current
-            
             # clear selection
             view.cancel 
       
@@ -3281,7 +3279,7 @@ CODE
 
           cn = cellname && cellname.to_s
           cn ||= @def_cell && @def_cell.name
-          cn ||= source && source.cell_name
+          cn ||= @def_source && @def_source.cell_obj && @def_source.cell_obj.name
 
           cn || raise("No cell name specified - either the source was not specified before 'report' or there is no default source. In the latter case, specify a cell name as the third parameter")
 

From 5438b9a82f2089f47e60800e884cb8508702b849 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sat, 3 Jun 2023 21:25:30 +0200
Subject: [PATCH 14/22] configuration for debugger scope override

---
 src/pya/pya/pya.cc | 11 +++++++++++
 src/rba/rba/rba.cc |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/pya/pya/pya.cc b/src/pya/pya/pya.cc
index 53d0d9496..44f31d88a 100644
--- a/src/pya/pya/pya.cc
+++ b/src/pya/pya/pya.cc
@@ -34,6 +34,7 @@
 #include "gsiDecl.h"
 #include "gsiDeclBasic.h"
 #include "tlLog.h"
+#include "tlEnv.h"
 #include "tlStream.h"
 #include "tlTimer.h"
 #include "tlFileUtils.h"
@@ -140,6 +141,16 @@ public:
 
   virtual size_t scope_index () const
   {
+    static int consider_scope = -1;
+
+    //  disable scoped debugging (e.g. DRC script lines) if $KLAYOUT_PYA_DEBUG_SCOPE is set.
+    if (consider_scope < 0) {
+      consider_scope = tl::app_flag ("pya-debug-scope") ? 0 : 1;
+    }
+    if (! consider_scope) {
+      return 0;
+    }
+
     if (! m_scope.empty ()) {
       for (size_t i = 0; i < m_stack_trace.size (); ++i) {
         if (m_stack_trace [i].file == m_scope) {
diff --git a/src/rba/rba/rba.cc b/src/rba/rba/rba.cc
index 27af10ed0..00826361c 100644
--- a/src/rba/rba/rba.cc
+++ b/src/rba/rba/rba.cc
@@ -111,7 +111,7 @@ RubyStackTraceProvider::scope_index (const std::vector<tl::BacktraceElement> &bt
 
     //  disable scoped debugging (e.g. DRC script lines) if $KLAYOUT_RBA_DEBUG_SCOPE is set.
     if (consider_scope < 0) {
-      consider_scope = tl::has_env ("KLAYOUT_RBA_DEBUG_SCOPE") ? 0 : 1;
+      consider_scope = tl::app_flag ("rba-debug-scope") ? 0 : 1;
     }
     if (! consider_scope) {
       return 0;

From 9817f1238543e674d827ccbfc917819c83f5f583 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sun, 4 Jun 2023 09:13:20 +0200
Subject: [PATCH 15/22] Speedup of hierarchy processor (context computation) in
 case if deeply hierarchical two-input cases

---
 src/db/db/dbHierProcessor.cc | 42 ++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/src/db/db/dbHierProcessor.cc b/src/db/db/dbHierProcessor.cc
index e69057b84..e65f14a3a 100644
--- a/src/db/db/dbHierProcessor.cc
+++ b/src/db/db/dbHierProcessor.cc
@@ -64,6 +64,11 @@ cronology::events::event_collection<event_compute_results, event_compute_local_c
 namespace db
 {
 
+//  Heuristic parameter to control the recursion of the cell-to-cell intruder
+//  detection: do not recurse if the intruder cell's bounding box is smaller
+//  than the overlap box times this factor.
+const double area_ratio_for_recursion = 3.0;
+
 // ---------------------------------------------------------------------------------------------
 //  Shape reference translator
 
@@ -370,6 +375,32 @@ db::Box safe_box_enlarged (const db::Box &box, db::Coord dx, db::Coord dy)
   }
 }
 
+// ---------------------------------------------------------------------------------------------
+//  Debugging utility: dump the cell contexts
+
+template <class TS, class TI, class TR>
+static void dump_cell_contexts (local_processor_contexts<TS, TI, TR> &contexts, const db::Layout *subject_layout, const db::Layout *intruder_layout)
+{
+  for (auto cc = contexts.begin (); cc != contexts.end (); ++cc) {
+    tl::info << "Cell " << subject_layout->cell_name (cc->first->cell_index ()) << ":";
+    int i = 0;
+    for (auto c = cc->second.begin (); c != cc->second.end (); ++c) {
+      tl::info << "  Context #" << ++i;
+      tl::info << "    Instances:";
+      for (auto i = c->first.first.begin (); i != c->first.first.end (); ++i) {
+        const db::CellInstArray &ci = *i;
+        tl::info << "      " << intruder_layout->cell_name (ci.object ().cell_index ()) << " @ " << ci.complex_trans (*ci.begin ()).to_string () << " (" << ci.size () << ")";
+      }
+      tl::info << "    Shapes:";
+      for (auto i = c->first.second.begin (); i != c->first.second.end (); ++i) {
+        for (auto s = i->second.begin (); s != i->second.end (); ++s) {
+          tl::info << "      " << intruder_layout->get_properties (i->first).to_string () << ": " << s->to_string ();
+        }
+      }
+    }
+  }
+}
+
 // ---------------------------------------------------------------------------------------------
 //  LocalProcessorCellContext implementation
 
@@ -1282,9 +1313,11 @@ private:
     db::ICplxTrans tni2 = tn21.inverted () * tni1;
     db::Box tbox2 = safe_box_enlarged (tni2 * cbox, -1, -1);
 
-    if (! intruder_cell.shapes (m_intruder_layer).begin_touching (tbox2, ShapeIterator::All).at_end ()) {
+    //  do not recurse further if we're overlapping with shapes from the intruder
+    //  or the intruder cell is not much bigger than the region of interest (cbox)
+    if (intruder_cell.bbox (m_intruder_layer).area () < area_ratio_for_recursion * cbox.area ()
+        || ! intruder_cell.shapes (m_intruder_layer).begin_touching (tbox2, ShapeIterator::All).at_end ()) {
 
-      //  we're overlapping with shapes from the intruder - do not recursive further
       interactions.push_back (std::make_pair (intruder_cell.cell_index (), tn21));
       return;
 
@@ -1782,6 +1815,11 @@ template <class TS, class TI, class TR>
 void
 local_processor<TS, TI, TR>::compute_results (local_processor_contexts<TS, TI, TR> &contexts, const local_operation<TS, TI, TR> *op, const std::vector<unsigned int> &output_layers) const
 {
+#if 0
+  //  debugging
+  dump_cell_contexts (contexts, mp_subject_layout, mp_intruder_layout ? mp_intruder_layout : mp_subject_layout);
+#endif
+
   tl::SelfTimer timer (tl::verbosity () > m_base_verbosity + 10, tl::to_string (tr ("Computing results for ")) + description (op));
 
   //  avoids updates while we work on the layout

From 078258bf09bb7fe96527be6a6e8064d2dd2f8080 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Tue, 6 Jun 2023 23:35:40 +0200
Subject: [PATCH 16/22] New branch for XOR performance improvement in deep mode

---
 src/db/db/dbHierProcessor.cc              |   3 ++-
 src/db/unit_tests/dbHierProcessorTests.cc |   6 ++++++
 testdata/algo/hlp18.oas                   | Bin 0 -> 482 bytes
 3 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 testdata/algo/hlp18.oas

diff --git a/src/db/db/dbHierProcessor.cc b/src/db/db/dbHierProcessor.cc
index e65f14a3a..60be2d0f9 100644
--- a/src/db/db/dbHierProcessor.cc
+++ b/src/db/db/dbHierProcessor.cc
@@ -1239,7 +1239,8 @@ private:
   void
   collect_instance_interactions (const db::CellInstArray *inst1, const db::CellInstArray *inst2)
   {
-    //  TODO: this algorithm is not in particular effective for identical arrays
+    //  TODO: this algorithm is not in particular effective for identical arrays or for arrays
+    //  vs. single instances
 
     const db::Cell &cell1 = mp_subject_layout->cell (inst1->object ().cell_index ());
     const db::Cell &cell2 = mp_intruder_layout->cell (inst2->object ().cell_index ());
diff --git a/src/db/unit_tests/dbHierProcessorTests.cc b/src/db/unit_tests/dbHierProcessorTests.cc
index 7e9fe165f..504f36c2e 100644
--- a/src/db/unit_tests/dbHierProcessorTests.cc
+++ b/src/db/unit_tests/dbHierProcessorTests.cc
@@ -1278,3 +1278,9 @@ TEST(FlatOperation)
   run_test_bool22_flat (_this, "hlp17_flat.oas", TMAndNot, 100, 101);
 }
 
+TEST(Arrays)
+{
+  //  Large arrays, NOT
+  run_test_bool2 (_this, "hlp18.oas", TMNot, 100);
+}
+
diff --git a/testdata/algo/hlp18.oas b/testdata/algo/hlp18.oas
new file mode 100644
index 0000000000000000000000000000000000000000..970739b6406487773b50cae479bef1208d6b687a
GIT binary patch
literal 482
zcmY!lcJ=kt^>+;R4CduxWH!_@V0gjKC?n3q!6L)YEF;ds&!EJR>XUoMnybM;fb~F;
z;|1o9>4KX(8~>b$4qRRSKbqrK6n{_8gJqf<tN2Z49(ZuzTK46<?Pn6=^p)Bc^=+KN
z_SR<c=GbZH`#!B^&e*#5Co?B=h<^YtgUDe<pj9G`OuUSOCm0x?u|H$Kz<+_Cmr3vd
zBQLYy4+apAo1sHpNbmvEht-TC)r<^3LSKLtDKQvmB&U7*UOZJW{lSWaw9?NdUke#G
s&X$~c^MH)9K)#Z)adAgILx*LrAOjQQHbzDUMrFogAj!}$!T<vU0Pcy)9{>OV

literal 0
HcmV?d00001


From d604003e3fef952bf4746894c345a73b5adcdab4 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Thu, 8 Jun 2023 19:03:57 +0200
Subject: [PATCH 17/22] Changing XOR implementation to mapped

The previous implementation for two layouts
was based on the twofold traversal of hierarchies
but that is less efficient than first mapping and
then doing the XOR within the single layout.
---
 src/db/db/dbDeepRegion.cc | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/db/db/dbDeepRegion.cc b/src/db/db/dbDeepRegion.cc
index 0d7d725de..bed425c3d 100644
--- a/src/db/db/dbDeepRegion.cc
+++ b/src/db/db/dbDeepRegion.cc
@@ -962,10 +962,31 @@ DeepRegion::xor_with (const Region &other, db::PropertyConstraint property_const
 
     //  Implement XOR as (A-B)+(B-A) - only this implementation
     //  is compatible with the local processor scheme
-    DeepLayer n1 (and_or_not_with (other_deep, false, property_constraint));
-    DeepLayer n2 (other_deep->and_or_not_with (this, false, property_constraint));
 
+    //  Prepare a version of "other_deep" that is mapped into the hierarchy space
+    //  of "this"
+    std::unique_ptr<DeepRegion> other_deep_mapped;
+    if (&other_deep->deep_layer ().layout () == &deep_layer ().layout ()) {
+      //  shallow copy for reconfiguration (progress etc.)
+      other_deep_mapped.reset (new DeepRegion (other_deep->deep_layer ()));
+    } else {
+      //  deep copy with mapped hierarchy
+      other_deep_mapped.reset (new DeepRegion (deep_layer ().derived ()));
+      other_deep_mapped->deep_layer ().add_from (other_deep->deep_layer ());
+    }
+
+    other_deep_mapped->set_strict_handling (strict_handling ());
+    other_deep_mapped->set_base_verbosity (base_verbosity ());
+    if (report_progress ()) {
+      other_deep_mapped->enable_progress (progress_desc () + tl::to_string (tr (" - reverse part")));
+    } else {
+      other_deep_mapped->disable_progress ();
+    }
+
+    DeepLayer n1 (and_or_not_with (other_deep_mapped.get (), false, property_constraint));
+    DeepLayer n2 (other_deep_mapped->and_or_not_with (this, false, property_constraint));
     n1.add_from (n2);
+
     return new DeepRegion (n1);
 
   }

From 6735f6c5d685c2f67722c4ac27ae4469633f01c8 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Thu, 8 Jun 2023 19:06:12 +0200
Subject: [PATCH 18/22] New test case

---
 testdata/algo/hlp18.oas | Bin 482 -> 596 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/testdata/algo/hlp18.oas b/testdata/algo/hlp18.oas
index 970739b6406487773b50cae479bef1208d6b687a..2c45f1226e2203ee2ae8a16e08b474e6f22c6c58 100644
GIT binary patch
delta 247
zcmaFFe1&B~RlTS&BLgGT1^x%I6a7RjnHZTS_)YLzD8Epimr;qq&Rp2$;DH@onWC3j
z|8f6W^h}ed$Ke{&&*ri_JmNoEU2+}n@K*`{<-1|Hp20|wN&JWXD?@|kFANO4Oo9g(
zd6|_M3KE3>9QcxG`k1xupg52G*Bz2^8%`T_Mj0C!80k)^FUaiP_)z?@Q)M8xo0`J!
z=6h{-Sc@eb_i+Dl_~-N!XyYVJ0aY`FgRGx`_U)96-ynX%xKpaZ#qNdaHK(om6WFua
u8JL;<{al$@-8@}=T$tIwlo2y?h<^YR<8CIP=ad<Xfh5DkkMEfq7#ILoKvw_&

delta 138
zcmcb@@`!mtRlUezMur#cB8^PEjDjZ^7@x5}W52+EfuEO2@Bkw(v)~T~5RaRoLtRMl
z0n>-oj3U*H3_n6&fE6h*7-%G?efwTKRWSX*iiEV%&m~_A88^<BoO$zrjIuz!lCp7e
gM?FJ_Wv?Iu6XP~UMg~S@#$q7J&;SFIvl(9k0KqsfW&i*H


From 0959676b5a17813183c1919329c6bdb23cb00620 Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Fri, 9 Jun 2023 23:26:49 +0200
Subject: [PATCH 19/22] Another shortcut for two-input box scanner, but does
 not make a large difference

---
 src/db/db/dbBoxScanner.h               | 26 ++++++++++++--
 src/db/unit_tests/dbBoxScannerTests.cc | 48 +++++++++++++++++++-------
 2 files changed, 58 insertions(+), 16 deletions(-)

diff --git a/src/db/db/dbBoxScanner.h b/src/db/db/dbBoxScanner.h
index 5b562bee2..d329316f4 100644
--- a/src/db/db/dbBoxScanner.h
+++ b/src/db/db/dbBoxScanner.h
@@ -538,7 +538,7 @@ public:
    *  @brief Default ctor
    */
   box_scanner2 (bool report_progress = false, const std::string &progress_desc = std::string ())
-    : m_fill_factor (2), m_scanner_thr (100),
+    : m_fill_factor (2), m_scanner_thr (100), m_scanner_thr1 (10),
       m_report_progress (report_progress), m_progress_desc (progress_desc)
   {
     //  .. nothing yet ..
@@ -564,6 +564,26 @@ public:
     return m_scanner_thr;
   }
 
+  /**
+   *  @brief Sets the scanner threshold per class
+   *
+   *  This value determines for how many elements in one class the implementation switches to the scanner
+   *  implementation instead of the plain element-by-element interaction test.
+   *  The default value is 10.
+   */
+  void set_scanner_threshold1 (size_t n)
+  {
+    m_scanner_thr1 = n;
+  }
+
+  /**
+   *  @brief Gets the scanner threshold per class
+   */
+  size_t scanner_threshold1 () const
+  {
+    return m_scanner_thr1;
+  }
+
   /**
    *  @brief Sets the fill factor
    *
@@ -667,7 +687,7 @@ private:
   container_type1 m_pp1;
   container_type2 m_pp2;
   double m_fill_factor;
-  size_t m_scanner_thr;
+  size_t m_scanner_thr, m_scanner_thr1;
   bool m_report_progress;
   std::string m_progress_desc;
 
@@ -732,7 +752,7 @@ private:
         rec.finish2 (i->first, i->second);
       }
 
-    } else if (m_pp1.size () + m_pp2.size () <= m_scanner_thr) {
+    } else if (m_pp1.size () + m_pp2.size () <= m_scanner_thr || m_pp2.size () <= m_scanner_thr1 || m_pp1.size () <= m_scanner_thr1) {
 
       //  below m_scanner_thr elements use the brute force approach which is faster in that case
 
diff --git a/src/db/unit_tests/dbBoxScannerTests.cc b/src/db/unit_tests/dbBoxScannerTests.cc
index 4d6de4959..9d772f340 100644
--- a/src/db/unit_tests/dbBoxScannerTests.cc
+++ b/src/db/unit_tests/dbBoxScannerTests.cc
@@ -938,6 +938,7 @@ TEST(two_1)
   db::box_convert<db::Box> bc1;
   db::box_convert<db::SimplePolygon> bc2;
   bs.set_scanner_threshold (0);
+  bs.set_scanner_threshold1 (0);
   bs.process (tr, 1, bc1, bc2);
   EXPECT_EQ (tr.str, "[i](2-12)(2-14)(4-12)(4-14)(2-15)(4-15)(5-12)(5-14)(5-15)(2-13)(4-13)(3-12)(3-14)(3-13)(3-15)(5-13)(0-10)<2><5><4><3><12><15><14><13>(0-11)(1-10)(1-11)<0><1><10><11>[f]");
 }
@@ -974,6 +975,7 @@ TEST(two_1a)
   db::box_convert<db::Box> bc1;
   db::box_convert<db::SimplePolygon> bc2;
   bs.set_scanner_threshold (0);
+  bs.set_scanner_threshold1 (0);
   bs.process (tr, 1, bc1, bc2);
   EXPECT_EQ (tr.str, "[i](2-11)(2-12)(1-11)(1-12)<1><2><11><12>(0-10)<0><10>[f]");
 }
@@ -1010,6 +1012,7 @@ TEST(two_1b)
   db::box_convert<db::Box> bc1;
   db::box_convert<db::SimplePolygon> bc2;
   bs.set_scanner_threshold (0);
+  bs.set_scanner_threshold1 (0);
   EXPECT_EQ (bs.process (tr, 1, bc1, bc2), true);
   EXPECT_EQ (tr.str, "[i](1-12)(2-12)(1-11)(2-11)<1><2><11><12>(0-10)<0><10>[f]");
 
@@ -1046,14 +1049,15 @@ TEST(two_1c)
   db::box_convert<db::Box> bc1;
   db::box_convert<db::SimplePolygon> bc2;
   bs.set_scanner_threshold (0);
+  bs.set_scanner_threshold1 (0);
   EXPECT_EQ (bs.process (tr, 1, bc1, bc2), true);
   EXPECT_EQ (tr.str, "[i]<0><10>(1-12)(2-12)(1-11)(2-11)<1><2><12><11>[f]");
 }
 
-void run_test2_two (tl::TestBase *_this, size_t n, double ff, db::Coord spread, bool touch = true)
+void run_test2_two (tl::TestBase *_this, size_t n1, size_t n2, double ff, db::Coord spread, bool touch = true, bool no_shortcut = true)
 {
   std::vector<db::Box> bb;
-  for (size_t i = 0; i < n; ++i) {
+  for (size_t i = 0; i < n1; ++i) {
     db::Coord x = rand () % spread;
     db::Coord y = rand () % spread;
     bb.push_back (db::Box (x, y, x + 100, y + 100));
@@ -1061,7 +1065,7 @@ void run_test2_two (tl::TestBase *_this, size_t n, double ff, db::Coord spread,
   }
 
   std::vector<db::SimplePolygon> bb2;
-  for (size_t i = 0; i < n; ++i) {
+  for (size_t i = 0; i < n2; ++i) {
     db::Coord x = rand () % spread;
     db::Coord y = rand () % spread;
     bb2.push_back (db::SimplePolygon (db::Box (x, y, x + 100, y + 100)));
@@ -1082,7 +1086,10 @@ void run_test2_two (tl::TestBase *_this, size_t n, double ff, db::Coord spread,
   db::box_convert<db::SimplePolygon> bc2;
   {
     tl::SelfTimer timer ("box-scanner");
-    bs.set_scanner_threshold (0);
+    if (no_shortcut) {
+      bs.set_scanner_threshold (0);
+      bs.set_scanner_threshold1 (0);
+    }
     bs.process (tr, touch ? 1 : 0, bc1, bc2);
   }
 
@@ -1118,45 +1125,60 @@ void run_test2_two (tl::TestBase *_this, size_t n, double ff, db::Coord spread,
 
 TEST(two_2a)
 {
-  run_test2_two(_this, 10, 0.0, 1000);
+  run_test2_two(_this, 10, 10, 0.0, 1000);
+  run_test2_two(_this, 10, 10, 0.0, 1000, true, false /*sub-threshold*/);
 }
 
 TEST(two_2b)
 {
-  run_test2_two(_this, 10, 0.0, 100);
+  run_test2_two(_this, 10, 10, 0.0, 100);
+  run_test2_two(_this, 10, 10, 0.0, 100, true, false /*sub-threshold*/);
 }
 
 TEST(two_2c)
 {
-  run_test2_two(_this, 10, 0.0, 10);
+  run_test2_two(_this, 10, 10, 0.0, 10);
+  run_test2_two(_this, 10, 10, 0.0, 10, true, false /*sub-threshold*/);
 }
 
 TEST(two_2d)
 {
-  run_test2_two(_this, 1000, 0.0, 1000);
+  run_test2_two(_this, 1000, 1000, 0.0, 1000);
 }
 
 TEST(two_2e)
 {
-  run_test2_two(_this, 1000, 2, 1000);
+  run_test2_two(_this, 1000, 1000, 2, 1000);
 }
 
 TEST(two_2f)
 {
-  run_test2_two(_this, 1000, 2, 1000, false);
+  run_test2_two(_this, 1000, 1000, 2, 1000, false);
 }
 
 TEST(two_2g)
 {
-  run_test2_two(_this, 1000, 2, 500);
+  run_test2_two(_this, 1000, 1000, 2, 500);
 }
 
 TEST(two_2h)
 {
-  run_test2_two(_this, 1000, 2, 100);
+  run_test2_two(_this, 1000, 1000, 2, 100);
 }
 
 TEST(two_2i)
 {
-  run_test2_two(_this, 10000, 2, 10000);
+  run_test2_two(_this, 10000, 1000, 2, 10000);
+}
+
+TEST(two_2j)
+{
+  run_test2_two(_this, 3, 1000, 0.0, 1000);
+  run_test2_two(_this, 3, 1000, 0.0, 1000, true, false /*sub-threshold*/);
+}
+
+TEST(two_2k)
+{
+  run_test2_two(_this, 1000, 3, 0.0, 1000);
+  run_test2_two(_this, 1000, 3, 0.0, 1000, true, false /*sub-threshold*/);
 }

From a0c70a2cc7fdb164bb3697378cc7c21d6294035e Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Fri, 9 Jun 2023 23:31:39 +0200
Subject: [PATCH 20/22] Another small optimization of hierarchical processor,
 makes a small difference

---
 src/db/db/dbHierProcessor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/db/db/dbHierProcessor.cc b/src/db/db/dbHierProcessor.cc
index e65f14a3a..cf8bbfcb5 100644
--- a/src/db/db/dbHierProcessor.cc
+++ b/src/db/db/dbHierProcessor.cc
@@ -1278,7 +1278,7 @@ private:
           db::Box ibox2 = tn2 * cell2.bbox (m_intruder_layer).enlarged (db::Vector (m_dist, m_dist));
 
           db::Box cbox = ibox1 & ibox2;
-          if (! cbox.empty () && cell1.has_shapes_touching (m_subject_layer, safe_box_enlarged (tni1 * cbox, -1, -1))) {
+          if (! cbox.empty () && (cbox == ibox1 || cell1.has_shapes_touching (m_subject_layer, safe_box_enlarged (tni1 * cbox, -1, -1)))) {
 
             db::ICplxTrans tn21 = tni1 * tn2;
 

From 7f8eeb3a09b099ea5145093d3e7c5c2e370dcf6a Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sat, 10 Jun 2023 09:06:56 +0200
Subject: [PATCH 21/22] Fixed issue #1281 (Layout diff should ignore shape or
 instance duplicates)

A new option in the diff tool and strmcmp has been added
(-1|--ignore-duplicates in strcmp).
In RBA/pya, the "IgnoreDuplicates" flag has been added.
---
 src/buddies/src/bd/strmcmp.cc                 |   8 +
 src/db/db/dbLayoutDiff.cc                     |  81 +++++--
 src/db/db/dbLayoutDiff.h                      |   5 +-
 src/db/unit_tests/dbLayoutDiffTests.cc        | 211 +++++++++++++++++-
 .../tools/diff/lay_plugin/DiffToolDialog.ui   | 138 ++++++------
 .../diff/lay_plugin/layDiffToolDialog.cc      |   9 +
 6 files changed, 360 insertions(+), 92 deletions(-)

diff --git a/src/buddies/src/bd/strmcmp.cc b/src/buddies/src/bd/strmcmp.cc
index 637baaab0..aefcefe6f 100644
--- a/src/buddies/src/bd/strmcmp.cc
+++ b/src/buddies/src/bd/strmcmp.cc
@@ -41,6 +41,7 @@ BD_PUBLIC int strmcmp (int argc, char *argv[])
   std::string infile_a, infile_b;
   std::string top_a, top_b;
   bool silent = false;
+  bool ignore_duplicates = false;
   bool no_text_orientation = true;
   bool no_text_details = true;
   bool no_properties = false;
@@ -106,6 +107,10 @@ BD_PUBLIC int strmcmp (int argc, char *argv[])
       << tl::arg ("--expand-arrays",           &flatten_array_insts, "Expands array instances before compare",
                   "With this option, arrays are equivalent single instances are treated identical."
                  )
+      << tl::arg ("-1|--ignore-duplicates",    &ignore_duplicates, "Ignore duplicate instances and shapes",
+                  "With this option, duplicate instances or shapes are ignored and duplication "
+                  "does not count as a difference."
+                 )
       << tl::arg ("-l|--layer-details",        &dont_summarize_missing_layers, "Prints details about differences for missing layers",
                   "With this option, missing layers are treated as \"empty\" and details about differences to "
                   "other, non-empty layers are printed. Essentially the content of the non-empty counterpart "
@@ -155,6 +160,9 @@ BD_PUBLIC int strmcmp (int argc, char *argv[])
   if (silent) {
     flags |= db::layout_diff::f_silent;
   }
+  if (ignore_duplicates) {
+    flags |= db::layout_diff::f_ignore_duplicates;
+  }
   if (no_text_orientation) {
     flags |= db::layout_diff::f_no_text_orientation;
   }
diff --git a/src/db/db/dbLayoutDiff.cc b/src/db/db/dbLayoutDiff.cc
index d52ec900e..be1e005e7 100644
--- a/src/db/db/dbLayoutDiff.cc
+++ b/src/db/db/dbLayoutDiff.cc
@@ -69,8 +69,10 @@ collect_cells (const db::Layout &l, const db::Cell *top, std::map <std::string,
 }
 
 static void
-collect_insts_of_unmapped_cells (const db::Layout & /*l*/, const db::Cell *cell, unsigned int /*flags*/, const std::map <db::cell_index_type, db::cell_index_type> &cci, std::vector <db::CellInstArrayWithProperties> &insts)
+collect_insts_of_unmapped_cells (const db::Layout & /*l*/, const db::Cell *cell, unsigned int /*flags*/, const std::map <db::cell_index_type, db::cell_index_type> &cci, std::vector <db::CellInstArrayWithProperties> &insts, bool no_duplicates)
 {
+  size_t n_before = insts.size ();
+
   for (db::Cell::const_iterator i = cell->begin (); !i.at_end (); ++i) {
 
     std::map <db::cell_index_type, db::cell_index_type>::const_iterator ccii = cci.find (i->cell_index ());
@@ -81,6 +83,13 @@ collect_insts_of_unmapped_cells (const db::Layout & /*l*/, const db::Cell *cell,
 
     }
   }
+
+  if (no_duplicates) {
+
+    std::sort (insts.begin () + n_before, insts.end ());
+    insts.erase (std::unique (insts.begin () + n_before, insts.end ()), insts.end ());
+
+  }
 }
 
 static void 
@@ -102,7 +111,7 @@ rewrite_instances_to (std::vector <db::CellInstArrayWithProperties> &insts, unsi
 }
 
 static void
-collect_insts (const db::Layout & /*l*/, const db::Cell *cell, unsigned int flags, const std::map <db::cell_index_type, db::cell_index_type> &cci, std::vector <db::CellInstArrayWithProperties> &insts, PropertyMapper &pn)
+collect_insts (const db::Layout & /*l*/, const db::Cell *cell, unsigned int flags, const std::map <db::cell_index_type, db::cell_index_type> &cci, std::vector <db::CellInstArrayWithProperties> &insts, PropertyMapper &pn, bool no_duplicates)
 {
   insts.clear ();
 
@@ -148,6 +157,10 @@ collect_insts (const db::Layout & /*l*/, const db::Cell *cell, unsigned int flag
   }
 
   std::sort (insts.begin (), insts.end ());
+
+  if (no_duplicates) {
+    insts.erase (std::unique (insts.begin (), insts.end ()), insts.end ());
+  }
 }
 
 /**
@@ -178,10 +191,10 @@ int compare_seq (I b1, I e1, I b2, I e2, Op op)
 /**
  *  @brief Reduces two vectors to the common objects as determined by the compare operator
  *  If the iterate parameter is true, the reduction is repeated until no more reduction can be 
- *  achieved. This is useful with tolerances since the sorted is not strict in that case.
+ *  achieved. This is useful with tolerances since the sorting is not strict in that case.
  */
 template <class X, class Op>
-void reduce (std::vector<X> &a, std::vector<X> &b, Op op, bool iterate)
+void reduce (std::vector<X> &a, std::vector<X> &b, Op op, bool iterate, bool no_duplicates)
 {
   do {
 
@@ -196,12 +209,29 @@ void reduce (std::vector<X> &a, std::vector<X> &b, Op op, bool iterate)
 
     while (ra != a.end () && rb != b.end ()) {
       if (op (*ra, *rb)) {
-        *wa++ = *ra++;
+        typename std::vector<X>::const_iterator r = ra++;
+        *wa = *r;
+        while (no_duplicates && ra != a.end () && !op (*ra, *r) && !op(*r, *ra)) {
+          ++ra;
+        }
+        ++wa;
       } else if (op (*rb, *ra)) {
-        *wb++ = *rb++;
+        typename std::vector<X>::const_iterator r = rb++;
+        *wb = *r;
+        while (no_duplicates && rb != b.end () && !op (*rb, *r) && !op(*r, *rb)) {
+          ++rb;
+        }
+        ++wb;
       } else {
-        ++ra;
-        ++rb;
+        typename std::vector<X>::const_iterator r;
+        r = ra++;
+        while (no_duplicates && ra != a.end () && !op (*ra, *r) && !op(*r, *ra)) {
+          ++ra;
+        }
+        r = rb++;
+        while (no_duplicates && rb != b.end () && !op (*rb, *r) && !op(*r, *rb)) {
+          ++rb;
+        }
       }
     }
 
@@ -211,14 +241,22 @@ void reduce (std::vector<X> &a, std::vector<X> &b, Op op, bool iterate)
 
     if (ra != wa) {
       while (ra != a.end ()) {
-        *wa++ = *ra++;
+        typename std::vector<X>::const_iterator r = ra++;
+        *wa++ = *r;
+        while (no_duplicates && ra != a.end () && !op (*ra, *r) && !op(*r, *ra)) {
+          ++ra;
+        }
       }
       a.erase (wa, a.end ());
     }
 
     if (rb != wb) {
       while (rb != b.end ()) {
-        *wb++ = *rb++;
+        typename std::vector<X>::const_iterator r = rb++;
+        *wb++ = *r;
+        while (no_duplicates && rb != b.end () && !op (*rb, *r) && !op(*r, *rb)) {
+          ++rb;
+        }
       }
       b.erase (wb, b.end ());
     }
@@ -405,7 +443,7 @@ struct PolygonCompareOpWithTolerance
       m_eb.push_back (*e);
     }
 
-    reduce (m_ea, m_eb, EdgeCompareOpWithTolerance (m_tolerance), m_tolerance > 0);
+    reduce (m_ea, m_eb, EdgeCompareOpWithTolerance (m_tolerance), m_tolerance > 0, false);
 
     return compare_seq (m_ea.begin (), m_ea.end (), m_eb.begin (), m_eb.end (), EdgeCompareOpWithTolerance (m_tolerance)) < 0;
   }
@@ -665,6 +703,7 @@ do_compare_layouts (const db::Layout &a, const db::Cell *top_a, const db::Layout
   }
 
   bool verbose = (flags & layout_diff::f_verbose);
+  bool no_duplicates = (flags & layout_diff::f_ignore_duplicates);
 
   db::Layout n, na, nb;
   na.properties_repository () = a.properties_repository ();
@@ -897,20 +936,20 @@ do_compare_layouts (const db::Layout &a, const db::Cell *top_a, const db::Layout
       r.bbox_differs (cell_a->bbox (), cell_b->bbox ());
     }
 
-    collect_insts (a, cell_a, flags, common_cell_indices_a, insts_a, prop_normalize_a);
-    collect_insts (b, cell_b, flags, common_cell_indices_b, insts_b, prop_normalize_b);
+    collect_insts (a, cell_a, flags, common_cell_indices_a, insts_a, prop_normalize_a, no_duplicates);
+    collect_insts (b, cell_b, flags, common_cell_indices_b, insts_b, prop_normalize_b, no_duplicates);
 
     std::vector <db::CellInstArrayWithProperties> anotb;
     std::set_difference (insts_a.begin (), insts_a.end (), insts_b.begin (), insts_b.end (), std::back_inserter (anotb));
 
     rewrite_instances_to (anotb, flags, common_cells_a, prop_remap_to_a);
-    collect_insts_of_unmapped_cells (a, cell_a, flags, common_cell_indices_a, anotb);
+    collect_insts_of_unmapped_cells (a, cell_a, flags, common_cell_indices_a, anotb, no_duplicates);
 
     std::vector <db::CellInstArrayWithProperties> bnota;
     std::set_difference (insts_b.begin (), insts_b.end (), insts_a.begin (), insts_a.end (), std::back_inserter (bnota));
 
     rewrite_instances_to (bnota, flags, common_cells_b, prop_remap_to_b);
-    collect_insts_of_unmapped_cells (b, cell_b, flags, common_cell_indices_b, bnota);
+    collect_insts_of_unmapped_cells (b, cell_b, flags, common_cell_indices_b, bnota, no_duplicates);
 
     if (! anotb.empty () || ! bnota.empty ()) {
 
@@ -979,7 +1018,7 @@ do_compare_layouts (const db::Layout &a, const db::Cell *top_a, const db::Layout
         collect_polygons (b, cell_b, layer_b, flags, polygons_b, prop_normalize_b);
       }
 
-      reduce (polygons_a, polygons_b, make_polygon_compare_func (tolerance), tolerance > 0);
+      reduce (polygons_a, polygons_b, make_polygon_compare_func (tolerance), tolerance > 0, no_duplicates);
 
       if (!polygons_a.empty () || !polygons_b.empty ()) {
         differs = true;
@@ -1007,7 +1046,7 @@ do_compare_layouts (const db::Layout &a, const db::Cell *top_a, const db::Layout
           collect_paths (b, cell_b, layer_b, flags, paths_b, prop_normalize_b);
         }
 
-        reduce (paths_a, paths_b, make_path_compare_func (tolerance), tolerance > 0);
+        reduce (paths_a, paths_b, make_path_compare_func (tolerance), tolerance > 0, no_duplicates);
 
         if (!paths_a.empty () || !paths_b.empty ()) {
           differs = true;
@@ -1034,7 +1073,7 @@ do_compare_layouts (const db::Layout &a, const db::Cell *top_a, const db::Layout
         collect_texts (b, cell_b, layer_b, flags, texts_b, prop_normalize_b);
       }
 
-      reduce (texts_a, texts_b, make_text_compare_func (tolerance), tolerance > 0);
+      reduce (texts_a, texts_b, make_text_compare_func (tolerance), tolerance > 0, no_duplicates);
 
       if (!texts_a.empty () || !texts_b.empty ()) {
         differs = true;
@@ -1061,7 +1100,7 @@ do_compare_layouts (const db::Layout &a, const db::Cell *top_a, const db::Layout
           collect_boxes (b, cell_b, layer_b, flags, boxes_b, prop_normalize_b);
         }
 
-        reduce (boxes_a, boxes_b, make_box_compare_func (tolerance), tolerance > 0);
+        reduce (boxes_a, boxes_b, make_box_compare_func (tolerance), tolerance > 0, no_duplicates);
 
         if (!boxes_a.empty () || !boxes_b.empty ()) {
           differs = true;
@@ -1088,7 +1127,7 @@ do_compare_layouts (const db::Layout &a, const db::Cell *top_a, const db::Layout
         collect_edges (b, cell_b, layer_b, flags, edges_b, prop_normalize_b);
       }
 
-      reduce (edges_a, edges_b, make_edge_compare_func (tolerance), tolerance > 0);
+      reduce (edges_a, edges_b, make_edge_compare_func (tolerance), tolerance > 0, no_duplicates);
 
       if (!edges_a.empty () || !edges_b.empty ()) {
         differs = true;
@@ -1113,7 +1152,7 @@ do_compare_layouts (const db::Layout &a, const db::Cell *top_a, const db::Layout
         collect_edge_pairs (b, cell_b, layer_b, flags, edge_pairs_b, prop_normalize_b);
       }
 
-      reduce (edge_pairs_a, edge_pairs_b, make_edge_pair_compare_func (tolerance), tolerance > 0);
+      reduce (edge_pairs_a, edge_pairs_b, make_edge_pair_compare_func (tolerance), tolerance > 0, no_duplicates);
 
       if (!edge_pairs_a.empty () || !edge_pairs_b.empty ()) {
         differs = true;
diff --git a/src/db/db/dbLayoutDiff.h b/src/db/db/dbLayoutDiff.h
index 865883af1..56f0114b6 100644
--- a/src/db/db/dbLayoutDiff.h
+++ b/src/db/db/dbLayoutDiff.h
@@ -74,12 +74,15 @@ const unsigned int f_paths_as_polygons = 0x100;
 //  Derive smart cell mapping instead of name mapping (available only if top cells are specified)
 const unsigned int f_smart_cell_mapping = 0x200;
 
-//  Don't summarize missing layers
+//  Don't summarize missing layers - print them in detail
 const unsigned int f_dont_summarize_missing_layers = 0x400;
 
 //  Ignore text details (font, size, presentation)
 const unsigned int f_no_text_details = 0x800;
 
+//  Ignore duplicate instances or shapes
+const unsigned int f_ignore_duplicates = 0x1000;
+
 }
 
 /**
diff --git a/src/db/unit_tests/dbLayoutDiffTests.cc b/src/db/unit_tests/dbLayoutDiffTests.cc
index 011d2be8d..739b9a77d 100644
--- a/src/db/unit_tests/dbLayoutDiffTests.cc
+++ b/src/db/unit_tests/dbLayoutDiffTests.cc
@@ -105,6 +105,8 @@ TestDifferenceReceiver::print_cell_inst (const db::CellInstArrayWithProperties &
   }
   if (ci.properties_id () != 0) {
     m_os << " [" << ci.properties_id () << "]" << std::endl;
+  } else {
+    m_os << "" << std::endl;
   }
 }
 
@@ -489,7 +491,9 @@ TEST(1)
     "  c4 m45 *1 -10,20\n"
     "  c4 m45 *1 -10,20\n"
     "Not in b but in a:\n"
-    "  c5x r0 *1 10,-20  c5x m45 *1 -10,20Not in a but in b:\n"
+    "  c5x r0 *1 10,-20\n"
+    "  c5x m45 *1 -10,20\n"
+    "Not in a but in b:\n"
   );
 
   g = h;
@@ -951,7 +955,7 @@ TEST(3)
   c2h.shapes (0).insert (db::Polygon (db::Box (1, 2, 1003, 1006)));
 
   r.clear ();
-  eq = db::compare_layouts (g, h, db::layout_diff::f_verbose, 0, r); 
+  eq = db::compare_layouts (g, h, db::layout_diff::f_verbose, 0, r);
 
   EXPECT_EQ (eq, false);
   EXPECT_EQ (r.text (), 
@@ -965,7 +969,7 @@ TEST(3)
   );
 
   r.clear ();
-  eq = db::compare_layouts (g, h, db::layout_diff::f_verbose, 1, r); 
+  eq = db::compare_layouts (g, h, db::layout_diff::f_verbose, 1, r);
 
   EXPECT_EQ (eq, false);
   EXPECT_EQ (r.text (), 
@@ -1499,4 +1503,205 @@ TEST(7)
   EXPECT_EQ (r.text (), "");
 }
 
+TEST(8)
+{
+  db::Layout g;
+  g.insert_layer (0);
+  g.set_properties (0, db::LayerProperties (17, 0));
+  g.insert_layer (1);
+  g.set_properties (1, db::LayerProperties (42, 1));
+
+  db::cell_index_type c1i = g.add_cell ("c1");
+  db::cell_index_type c2i = g.add_cell ("c2x");
+  db::cell_index_type c3i = g.add_cell ("c3");
+  db::cell_index_type c4i = g.add_cell ("c4");
+  db::cell_index_type c5i = g.add_cell ("c5x");
+
+  {
+
+    db::Cell &c1 (g.cell (c1i));
+    db::Cell &c2 (g.cell (c2i));
+    db::Cell &c3 (g.cell (c3i));
+    db::Cell &c4 (g.cell (c4i));
+    db::Cell &c5 (g.cell (c5i));
+    c2.shapes (0).insert (db::Box (0, 1, 2, 3));
+
+    db::FTrans f (1, true);
+    db::Vector p (-10, 20);
+    db::Trans t (f.rot (), p);
+    db::Vector pp (10, -20);
+    db::Trans tt (0, pp);
+
+    //  c4->c1 (aref)
+    c4.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1.cell_index ()), t, db::Vector(1, 1), db::Vector (0, 2), 2, 3));
+    //  c5->c1
+    c5.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1.cell_index ()), t));
+    //  c3->c5 (3x)
+    c3.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c5.cell_index ()), t));
+    c3.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c5.cell_index ()), tt));
+    c3.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c5.cell_index ()), t));
+    //  c4->c3
+    c4.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c3.cell_index ()), t));
+    //  c4->c1
+    c4.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1.cell_index ()), tt));
+    //  c2->c1 (2x)
+    c2.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1.cell_index ()), t));
+    c2.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1.cell_index ()), tt));
+    //  c2->c4 (2x)
+    c2.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c4.cell_index ()), t));
+    c2.insert (db::array <db::CellInst, db::Trans> (db::CellInst (c4.cell_index ()), t));
+
+  }
+
+  db::Layout h = g;
+
+  TestDifferenceReceiver r;
+  bool eq;
+
+  g.cell (c2i).shapes (0).insert (db::Box (1, 2, 1001, 1002));
+  g.cell (c2i).shapes (0).insert (db::Box (2, 3, 1002, 1003));
+  g.cell (c2i).shapes (0).insert (db::Box (2, 3, 1002, 1003));
+  g.cell (c2i).shapes (0).insert (db::Box (3, 4, 1003, 1004));
+  g.cell (c2i).shapes (0).insert (db::Box (3, 4, 1003, 1004));
+
+  h.cell (c2i).shapes (0).insert (db::Box (1, 2, 1001, 1002));
+  h.cell (c2i).shapes (0).insert (db::Box (1, 2, 1001, 1002));
+  h.cell (c2i).shapes (0).insert (db::Box (2, 3, 1002, 1003));
+  h.cell (c2i).shapes (0).insert (db::Box (4, 5, 1004, 1005));
+  h.cell (c2i).shapes (0).insert (db::Box (4, 5, 1004, 1005));
+
+  r.clear ();
+  eq = db::compare_layouts (g, h, db::layout_diff::f_verbose, 0, r);
+
+  EXPECT_EQ (eq, false);
+  EXPECT_EQ (r.text (),
+    "layout_diff: boxes differ for layer 17/0 in cell c2x\n"
+    "Not in b but in a:\n"
+    "  (2,3;1002,1003)\n"
+    "  (3,4;1003,1004)\n"
+    "  (3,4;1003,1004)\n"
+    "Not in a but in b:\n"
+    "  (1,2;1001,1002)\n"
+    "  (4,5;1004,1005)\n"
+    "  (4,5;1004,1005)\n"
+  );
+
+  r.clear ();
+  eq = db::compare_layouts (g, h, db::layout_diff::f_verbose + db::layout_diff::f_ignore_duplicates, 0, r);
+
+  EXPECT_EQ (eq, false);
+  EXPECT_EQ (r.text (),
+    "layout_diff: boxes differ for layer 17/0 in cell c2x\n"
+    "Not in b but in a:\n"
+    "  (3,4;1003,1004)\n"
+    "Not in a but in b:\n"
+    "  (4,5;1004,1005)\n"
+  );
+
+  //  duplicate instances
+  {
+    db::FTrans f (1, true);
+    db::Vector p (-10, 20);
+    db::Trans t (f.rot (), p);
+
+    h.cell(c4i).insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1i), t, db::Vector(1, 1), db::Vector (0, 2), 2, 3));
+    h.cell(c4i).insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1i), t));
+    h.cell(c4i).insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1i), t));
+
+    g.cell(c5i).insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1i), t));
+    g.cell(c5i).insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1i), t, db::Vector(1, 1), db::Vector (0, 2), 2, 3));
+    g.cell(c5i).insert (db::array <db::CellInst, db::Trans> (db::CellInst (c1i), t, db::Vector(1, 1), db::Vector (0, 2), 2, 3));
+
+    db::cell_index_type c6i = g.add_cell ("c6");
+    g.cell(c5i).insert (db::array <db::CellInst, db::Trans> (db::CellInst (c6i), t));
+    g.cell(c5i).insert (db::array <db::CellInst, db::Trans> (db::CellInst (c6i), t));
+
+  }
+
+  r.clear ();
+  eq = db::compare_layouts (g, h, db::layout_diff::f_verbose, 0, r);
+
+  EXPECT_EQ (eq, false);
+  EXPECT_EQ (r.text (),
+    "layout_diff: cell c6 is not present in layout b, but in a\n"
+    "layout_diff: boxes differ for layer 17/0 in cell c2x\n"
+    "Not in b but in a:\n"
+    "  (2,3;1002,1003)\n"
+    "  (3,4;1003,1004)\n"
+    "  (3,4;1003,1004)\n"
+    "Not in a but in b:\n"
+    "  (1,2;1001,1002)\n"
+    "  (4,5;1004,1005)\n"
+    "  (4,5;1004,1005)\n"
+    "layout_diff: instances differ in cell c4\n"
+    "list for a:\n"
+    "  c1 r0 *1 10,-20\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c3 m45 *1 -10,20\n"
+    "list for b:\n"
+    "  c1 r0 *1 10,-20\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c3 m45 *1 -10,20\n"
+    "Not in b but in a:\n"
+    "Not in a but in b:\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "layout_diff: instances differ in cell c5x\n"
+    "list for a:\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "list for b:\n"
+    "  c1 m45 *1 -10,20\n"
+    "Not in b but in a:\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c6 m45 *1 -10,20\n"
+    "  c6 m45 *1 -10,20\n"
+    "Not in a but in b:\n"
+  );
+
+  r.clear ();
+  eq = db::compare_layouts (g, h, db::layout_diff::f_verbose + db::layout_diff::f_ignore_duplicates, 0, r);
+
+  EXPECT_EQ (eq, false);
+  EXPECT_EQ (r.text (),
+    "layout_diff: cell c6 is not present in layout b, but in a\n"
+    "layout_diff: boxes differ for layer 17/0 in cell c2x\n"
+    "Not in b but in a:\n"
+    "  (3,4;1003,1004)\n"
+    "Not in a but in b:\n"
+    "  (4,5;1004,1005)\n"
+    "layout_diff: instances differ in cell c4\n"
+    "list for a:\n"
+    "  c1 r0 *1 10,-20\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c3 m45 *1 -10,20\n"
+    "list for b:\n"
+    "  c1 r0 *1 10,-20\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c3 m45 *1 -10,20\n"
+    "Not in b but in a:\n"
+    "Not in a but in b:\n"
+    "  c1 m45 *1 -10,20\n"
+    "layout_diff: instances differ in cell c5x\n"
+    "list for a:\n"
+    "  c1 m45 *1 -10,20\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "list for b:\n"
+    "  c1 m45 *1 -10,20\n"
+    "Not in b but in a:\n"
+    "  c1 m45 *1 -10,20[a=1,1, b=0,2, na=2, nb=3]\n"
+    "  c6 m45 *1 -10,20\n"
+    "Not in a but in b:\n"
+  );
+}
+
 
diff --git a/src/plugins/tools/diff/lay_plugin/DiffToolDialog.ui b/src/plugins/tools/diff/lay_plugin/DiffToolDialog.ui
index a14df4d1a..1995401fc 100644
--- a/src/plugins/tools/diff/lay_plugin/DiffToolDialog.ui
+++ b/src/plugins/tools/diff/lay_plugin/DiffToolDialog.ui
@@ -1,76 +1,73 @@
-<ui version="4.0" >
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
  <class>DiffToolDialog</class>
- <widget class="QDialog" name="DiffToolDialog" >
-  <property name="geometry" >
+ <widget class="QDialog" name="DiffToolDialog">
+  <property name="geometry">
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>498</width>
+    <width>503</width>
     <height>404</height>
    </rect>
   </property>
-  <property name="windowTitle" >
+  <property name="windowTitle">
    <string>Diff Tool</string>
   </property>
-  <layout class="QVBoxLayout" >
-   <property name="margin" >
-    <number>9</number>
-   </property>
-   <property name="spacing" >
+  <layout class="QVBoxLayout">
+   <property name="spacing">
     <number>6</number>
    </property>
+   <property name="margin" stdset="0">
+    <number>9</number>
+   </property>
    <item>
-    <widget class="QGroupBox" name="groupBox" >
-     <property name="title" >
+    <widget class="QGroupBox" name="groupBox">
+     <property name="title">
       <string>Input</string>
      </property>
-     <layout class="QGridLayout" >
-      <property name="margin" >
+     <layout class="QGridLayout">
+      <property name="margin" stdset="0">
        <number>9</number>
       </property>
-      <property name="spacing" >
+      <property name="spacing">
        <number>6</number>
       </property>
-      <item row="0" column="0" >
-       <widget class="QLabel" name="label" >
-        <property name="text" >
+      <item row="0" column="0">
+       <widget class="QLabel" name="label">
+        <property name="text">
          <string>Layout A   </string>
         </property>
        </widget>
       </item>
-      <item row="1" column="0" >
-       <widget class="QLabel" name="label_2" >
-        <property name="text" >
+      <item row="1" column="0">
+       <widget class="QLabel" name="label_2">
+        <property name="text">
          <string>Layout B</string>
         </property>
        </widget>
       </item>
-      <item row="0" column="1" >
-       <widget class="lay::CellViewSelectionComboBox" name="layouta" >
-        <property name="sizePolicy" >
-         <sizepolicy>
-          <hsizetype>7</hsizetype>
-          <vsizetype>5</vsizetype>
+      <item row="0" column="1">
+       <widget class="lay::CellViewSelectionComboBox" name="layouta">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Preferred">
           <horstretch>0</horstretch>
           <verstretch>0</verstretch>
          </sizepolicy>
         </property>
-        <property name="sizeAdjustPolicy" >
+        <property name="sizeAdjustPolicy">
          <enum>QComboBox::AdjustToContentsOnFirstShow</enum>
         </property>
        </widget>
       </item>
-      <item row="1" column="1" >
-       <widget class="lay::CellViewSelectionComboBox" name="layoutb" >
-        <property name="sizePolicy" >
-         <sizepolicy>
-          <hsizetype>7</hsizetype>
-          <vsizetype>5</vsizetype>
+      <item row="1" column="1">
+       <widget class="lay::CellViewSelectionComboBox" name="layoutb">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Preferred">
           <horstretch>0</horstretch>
           <verstretch>0</verstretch>
          </sizepolicy>
         </property>
-        <property name="sizeAdjustPolicy" >
+        <property name="sizeAdjustPolicy">
          <enum>QComboBox::AdjustToContentsOnFirstShow</enum>
         </property>
        </widget>
@@ -79,77 +76,84 @@
     </widget>
    </item>
    <item>
-    <widget class="QGroupBox" name="groupBox_2" >
-     <property name="title" >
+    <widget class="QGroupBox" name="groupBox_2">
+     <property name="title">
       <string>Options</string>
      </property>
-     <layout class="QVBoxLayout" >
-      <property name="margin" >
-       <number>9</number>
-      </property>
-      <property name="spacing" >
+     <layout class="QVBoxLayout">
+      <property name="spacing">
        <number>6</number>
       </property>
+      <property name="margin" stdset="0">
+       <number>9</number>
+      </property>
       <item>
-       <widget class="QCheckBox" name="smart_cbx" >
-        <property name="text" >
+       <widget class="QCheckBox" name="smart_cbx">
+        <property name="text">
          <string>Don't use names to match cells (use geometrical properties)</string>
         </property>
-        <property name="checked" >
+        <property name="checked">
          <bool>true</bool>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QCheckBox" name="xor_cbx" >
-        <property name="text" >
+       <widget class="QCheckBox" name="xor_cbx">
+        <property name="text">
          <string>Run XOR on differences</string>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QCheckBox" name="summarize_cbx" >
-        <property name="text" >
+       <widget class="QCheckBox" name="summarize_cbx">
+        <property name="text">
          <string>Summarize missing layers</string>
         </property>
-        <property name="checked" >
+        <property name="checked">
          <bool>true</bool>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QCheckBox" name="detailed_cbx" >
-        <property name="text" >
+       <widget class="QCheckBox" name="detailed_cbx">
+        <property name="text">
          <string>Detailed information</string>
         </property>
-        <property name="checked" >
+        <property name="checked">
          <bool>true</bool>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QCheckBox" name="expand_cell_arrays_cbx" >
-        <property name="text" >
+       <widget class="QCheckBox" name="expand_cell_arrays_cbx">
+        <property name="text">
          <string>Expand cell arrays (compare single instance by instance)</string>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QCheckBox" name="exact_cbx" >
-        <property name="text" >
+       <widget class="QCheckBox" name="exact_cbx">
+        <property name="text">
          <string>Exact compare (includes properties, text orientation and similar)</string>
         </property>
        </widget>
       </item>
+      <item>
+       <widget class="QCheckBox" name="ignore_duplicates_cbx">
+        <property name="text">
+         <string>Ignore duplicate instances and shapes</string>
+        </property>
+       </widget>
+      </item>
      </layout>
     </widget>
    </item>
    <item>
     <spacer>
-     <property name="orientation" >
+     <property name="orientation">
       <enum>Qt::Vertical</enum>
      </property>
-     <property name="sizeHint" >
+     <property name="sizeHint" stdset="0">
       <size>
        <width>472</width>
        <height>16</height>
@@ -158,12 +162,12 @@
     </spacer>
    </item>
    <item>
-    <widget class="QDialogButtonBox" name="buttonBox" >
-     <property name="orientation" >
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
       <enum>Qt::Horizontal</enum>
      </property>
-     <property name="standardButtons" >
-      <set>QDialogButtonBox::Cancel|QDialogButtonBox::NoButton|QDialogButtonBox::Ok</set>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
      </property>
     </widget>
    </item>
@@ -187,11 +191,11 @@
    <receiver>DiffToolDialog</receiver>
    <slot>accept()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>248</x>
      <y>254</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>157</x>
      <y>274</y>
     </hint>
@@ -203,11 +207,11 @@
    <receiver>DiffToolDialog</receiver>
    <slot>reject()</slot>
    <hints>
-    <hint type="sourcelabel" >
+    <hint type="sourcelabel">
      <x>316</x>
      <y>260</y>
     </hint>
-    <hint type="destinationlabel" >
+    <hint type="destinationlabel">
      <x>286</x>
      <y>274</y>
     </hint>
diff --git a/src/plugins/tools/diff/lay_plugin/layDiffToolDialog.cc b/src/plugins/tools/diff/lay_plugin/layDiffToolDialog.cc
index 1285f72bb..5ae9dcafb 100644
--- a/src/plugins/tools/diff/lay_plugin/layDiffToolDialog.cc
+++ b/src/plugins/tools/diff/lay_plugin/layDiffToolDialog.cc
@@ -46,6 +46,7 @@ std::string cfg_diff_smart ("diff-smart");
 std::string cfg_diff_summarize ("diff-summarize");
 std::string cfg_diff_expand_cell_arrays ("diff-expand-cell-arrays");
 std::string cfg_diff_exact ("diff-exact");
+std::string cfg_diff_ignore_duplicates ("diff-ignore-duplicates");
 
 // ------------------------------------------------------------------------------
 //  RdbDifferenceReceiver definition
@@ -650,6 +651,9 @@ DiffToolDialog::exec_dialog (lay::LayoutViewBase *view)
   if (config_root->config_get (cfg_diff_exact, f)) {
     mp_ui->exact_cbx->setChecked (f);
   }
+  if (config_root->config_get (cfg_diff_ignore_duplicates, f)) {
+    mp_ui->ignore_duplicates_cbx->setChecked (f);
+  }
 
   update ();
 
@@ -686,6 +690,7 @@ BEGIN_PROTECTED
   config_root->config_set (cfg_diff_summarize, mp_ui->summarize_cbx->isChecked ());
   config_root->config_set (cfg_diff_expand_cell_arrays, mp_ui->expand_cell_arrays_cbx->isChecked ());
   config_root->config_set (cfg_diff_exact, mp_ui->exact_cbx->isChecked ());
+  config_root->config_set (cfg_diff_ignore_duplicates, mp_ui->ignore_duplicates_cbx->isChecked ());
   config_root->config_end ();
 
   QDialog::accept ();
@@ -712,6 +717,7 @@ DiffToolDialog::run_diff ()
   bool summarize = !run_xor && mp_ui->summarize_cbx->isChecked ();
   bool expand_cell_arrays = !run_xor && mp_ui->expand_cell_arrays_cbx->isChecked ();
   bool exact = !run_xor && mp_ui->exact_cbx->isChecked ();
+  bool ignore_duplicates = mp_ui->ignore_duplicates_cbx->isChecked ();
 
   int cv_index_a = mp_ui->layouta->current_cv_index ();
   int cv_index_b = mp_ui->layoutb->current_cv_index ();
@@ -740,6 +746,9 @@ DiffToolDialog::run_diff ()
   if (smart) {
     flags |= db::layout_diff::f_smart_cell_mapping;
   }
+  if (ignore_duplicates) {
+    flags |= db::layout_diff::f_ignore_duplicates;
+  }
 
   //  TODO: make an parameter
   db::Coord tolerance = 0;

From 63f6abf7342f9138d9fa5076d1f0fccd6b538e4c Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Sat, 10 Jun 2023 09:17:00 +0200
Subject: [PATCH 22/22] Fixed a unit test failing in non-editable mode

---
 src/db/db/gsiDeclDbLayoutDiff.cc   | 11 +++++++++++
 src/db/unit_tests/dbShapesTests.cc |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/db/db/gsiDeclDbLayoutDiff.cc b/src/db/db/gsiDeclDbLayoutDiff.cc
index 1d50c6857..1ed1335ca 100644
--- a/src/db/db/gsiDeclDbLayoutDiff.cc
+++ b/src/db/db/gsiDeclDbLayoutDiff.cc
@@ -397,6 +397,10 @@ static unsigned int f_silent () {
   return db::layout_diff::f_silent;
 }
 
+static unsigned int f_ignore_duplicates () {
+  return db::layout_diff::f_ignore_duplicates;
+}
+
 static unsigned int f_no_text_orientation () {
   return db::layout_diff::f_no_text_orientation;
 }
@@ -448,6 +452,13 @@ gsi::Class<LayoutDiff> decl_LayoutDiff ("db", "LayoutDiff",
     "This constant can be used for the flags parameter of \\compare_layouts and \\compare_cells. It can be "
     "compared with other constants to form a flag set."
   ) +
+  gsi::constant ("IgnoreDuplicates", &f_ignore_duplicates,
+    "@brief Ignore duplicate instances or shapes\n"
+    "With this option present, duplicate instances or shapes are ignored and "
+    "duplication does not count as a difference.\n"
+    "\n"
+    "This option has been introduced in version 0.28.9."
+  ) +
   gsi::constant ("NoTextOrientation", &f_no_text_orientation,
     "@brief Ignore text orientation\n"
     "This constant can be used for the flags parameter of \\compare_layouts and \\compare_cells. It can be "
diff --git a/src/db/unit_tests/dbShapesTests.cc b/src/db/unit_tests/dbShapesTests.cc
index 1648f0885..b18e02bf3 100644
--- a/src/db/unit_tests/dbShapesTests.cc
+++ b/src/db/unit_tests/dbShapesTests.cc
@@ -1930,7 +1930,7 @@ TEST(8)
 {
   db::Manager m (true);
 
-  db::Layout layout (&m);
+  db::Layout layout (true, &m);
   unsigned int lindex1 = layout.insert_layer ();
   unsigned int lindex2 = layout.insert_layer ();