WIP: multi-mapping for named layer readers, bugfix for GDS/OASIS

src/db/db/dbCommonReader.cc

@@ -560,31 +560,14 @@ CommonReader::open_dl_uncached (db::Layout &layout, const LDPair &dl)
 
   } else {
 
+    for (std::set<unsigned int>::const_iterator i = li.begin (); i != li.end (); ++i) {
+      m_layer_map_out.mmap (dl, *i, layout.get_properties (*i));
+    }
+
     std::map<std::set<unsigned int>, unsigned int>::iterator mmp = m_multi_mapping_placeholders.find (li);
     if (mmp == m_multi_mapping_placeholders.end ()) {
-
+      //  create a placeholder layer
-      //  multi-mapping: create a placeholder layer if required
+      mmp = m_multi_mapping_placeholders.insert (std::make_pair (li, layout.insert_layer ())).first;
-
-      for (std::set<unsigned int>::const_iterator i = li.begin (); i != li.end (); ++i) {
-        m_layer_map_out.mmap (dl, *i, layout.get_properties (*i));
-      }
-
-      for (std::set<unsigned int>::const_iterator i = li.begin (); i != li.end (); ++i) {
-        std::set<unsigned int> sl;
-        sl.insert (*i);
-        if (m_multi_mapping_placeholders.find (sl) == m_multi_mapping_placeholders.end ()) {
-          //  a layer not used in a single-target context can be used as a placeholder layer (but only once)
-          m_multi_mapping_placeholders.insert (std::make_pair (sl, *i));
-          mmp = m_multi_mapping_placeholders.insert (std::make_pair (li, *i)).first;
-          break;
-        }
-      }
-
-      if (mmp == m_multi_mapping_placeholders.end ()) {
-        //  create a placeholder layer for later
-        mmp = m_multi_mapping_placeholders.insert (std::make_pair (li, layout.insert_layer ())).first;
-      }
-
     }
 
     return std::make_pair (true, mmp->second);

src/db/db/dbNamedLayerReader.cc

@@ -122,21 +122,32 @@ extract_ld (const char *s, int &l, int &d, std::string &n)
 
 std::pair <bool, unsigned int>
 NamedLayerReader::open_layer (db::Layout &layout, const std::string &n)
+{
+  std::map<std::string, std::pair <bool, unsigned int> >::const_iterator lc = m_layer_cache.find (n);
+  if (lc != m_layer_cache.end ()) {
+    return lc->second;
+  } else {
+    std::pair <bool, unsigned int> res = open_layer_uncached (layout, n);
+    m_layer_cache.insert (std::make_pair (n, res));
+    return res;
+  }
+}
+
+std::pair <bool, unsigned int>
+NamedLayerReader::open_layer_uncached (db::Layout &layout, const std::string &n)
 {
   int l = -1, d = -1;
   std::string on;
 
-  std::pair<bool, unsigned int> ll (false, 0);
+  std::set<unsigned int> li = m_layer_map.logical (n, layout);
-
+  if (li.empty () && ! m_keep_layer_names) {
-  ll = m_layer_map.first_logical (n, layout);
-  if (! ll.first && !m_keep_layer_names) {
 
     if (extract_plain_layer (n.c_str (), l)) {
 
       db::LayerProperties lp;
       lp.layer = l;
       lp.datatype = 0;
-      ll = m_layer_map.first_logical (lp, layout);
+      li = m_layer_map.logical (lp, layout);
 
     } else if (extract_ld (n.c_str (), l, d, on)) {
 
@@ -144,20 +155,33 @@ NamedLayerReader::open_layer (db::Layout &layout, const std::string &n)
       lp.layer = l;
       lp.datatype = d;
       lp.name = on;
-      ll = m_layer_map.first_logical (lp, layout);
+      li = m_layer_map.logical (lp, layout);
 
     }
 
   }
 
-  if (ll.first) {
+  if (! li.empty ()) {
 
-    //  create the layer if it is not part of the layout yet.
+    for (std::set<unsigned int>::const_iterator i = li.begin (); i != li.end (); ++i) {
-    if (! layout.is_valid_layer (ll.second)) {
+      m_layer_map_out.mmap (n, *i, layout.get_properties (*i));
-      layout.insert_layer (ll.second, m_layer_map.mapping (ll.second));
     }
 
-    return ll;
+    if (li.size () == 1) {
+
+      return std::make_pair (true, *li.begin ());
+
+    } else {
+
+      std::map<std::set<unsigned int>, unsigned int>::iterator mmp = m_multi_mapping_placeholders.find (li);
+      if (mmp == m_multi_mapping_placeholders.end ()) {
+        //  create a placeholder layer for later
+        mmp = m_multi_mapping_placeholders.insert (std::make_pair (li, layout.insert_layer ())).first;
+      }
+
+      return std::make_pair (true, mmp->second);
+
+    }
 
   } else if (! m_create_layers) {
 
@@ -188,21 +212,61 @@ NamedLayerReader::open_layer (db::Layout &layout, const std::string &n)
 void
 NamedLayerReader::map_layer (const std::string &name, unsigned int layer)
 {
-  m_layer_map.map (name, layer);
+  m_layer_map_out.map (name, layer);
 }
 
 void
-NamedLayerReader::prepare_layers ()
+NamedLayerReader::prepare_layers (db::Layout &layout)
 {
   m_new_layers.clear ();
   m_next_layer_index = m_layer_map.next_index ();
+
+  m_layer_map_out.clear ();
+  m_multi_mapping_placeholders.clear ();
+  m_layer_cache.clear ();
+
+  m_layer_map.prepare (layout);
+
+  //  create a pseudo-multimapping for single targets
+  for (db::LayerMap::const_iterator_layers li = m_layer_map.begin (); li != m_layer_map.end (); ++li) {
+    for (db::LayerMap::const_iterator_datatypes di = li->second.begin (); di != li->second.end (); ++di) {
+      if (di->second.size () == 1) {
+        m_multi_mapping_placeholders.insert (std::make_pair (di->second, *di->second.begin ()));
+      }
+    }
+  }
 }
 
 void
 NamedLayerReader::finish_layers (db::Layout &layout)
 {
+  //  resolve layer multi-mapping
+
+  for (std::map<std::set<unsigned int>, unsigned int>::const_iterator i = m_multi_mapping_placeholders.begin (); i != m_multi_mapping_placeholders.end (); ++i) {
+
+    if (i->first.size () > 1) {
+
+      bool discard_layer = i->first.find (i->second) == i->first.end ();
+
+      for (std::set<unsigned int>::const_iterator l = i->first.begin (); l != i->first.end (); ++l) {
+
+        //  last one? this one will get a "move"
+        std::set<unsigned int>::const_iterator ll = l;
+        if (discard_layer && ++ll == i->first.end ()) {
+          layout.move_layer (i->second, *l);
+          layout.delete_layer (i->second);
+        } else {
+          layout.copy_layer (i->second, *l);
+        }
+
+      }
+
+    }
+
+  }
+
   //  assign layer numbers to new layers
-  if (! m_new_layers.empty () && !m_keep_layer_names) {
+  if (! m_new_layers.empty () && ! m_keep_layer_names) {
 
     std::set<std::pair<int, int> > used_ld;
     for (db::Layout::layer_iterator l = layout.begin_layers (); l != layout.end_layers (); ++l) {

src/db/db/dbNamedLayerReader.h

@@ -54,6 +54,7 @@ public:
    */
   NamedLayerReader ();
 
+protected:
   /**
    *  @brief Sets a value indicating whether to create new layers
    */
@@ -75,9 +76,9 @@ public:
   /**
    *  @brief Gets the layer map
    */
-  const LayerMap &layer_map ()
+  const LayerMap &layer_map_out ()
   {
-    return m_layer_map;
+    return m_layer_map_out;
   }
 
   /**
@@ -96,7 +97,6 @@ public:
     return m_keep_layer_names;
   }
 
-protected:
   /**
    *  @brief Opens a new layer
    *  This method will create or locate a layer for a given name.
@@ -121,7 +121,7 @@ protected:
    *  @brief Prepares reading
    *  This method must be called before the reading is done.
    */
-  void prepare_layers ();
+  void prepare_layers (db::Layout &layout);
 
 private:
   bool m_create_layers;
@@ -129,6 +129,11 @@ private:
   LayerMap m_layer_map;
   unsigned int m_next_layer_index;
   std::map <std::string, unsigned int> m_new_layers;
+  db::LayerMap m_layer_map_out;
+  std::map<std::string, std::pair <bool, unsigned int> > m_layer_cache;
+  std::map<std::set<unsigned int>, unsigned int> m_multi_mapping_placeholders;
+
+  std::pair <bool, unsigned int> open_layer_uncached (db::Layout &layout, const std::string &name);
 };
 
 }

src/plugins/streamers/cif/db_plugin/dbCIFReader.cc

@@ -59,22 +59,20 @@ CIFReader::~CIFReader ()
 const LayerMap &
 CIFReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
 {
-  prepare_layers ();
-
   const db::CIFReaderOptions &specific_options = options.get_options<db::CIFReaderOptions> ();
   m_wire_mode = specific_options.wire_mode;
   m_dbu = specific_options.dbu;
 
-  db::LayerMap lm = specific_options.layer_map;
+  set_layer_map (specific_options.layer_map);
-  lm.prepare (layout);
-  set_layer_map (lm);
   set_create_layers (specific_options.create_other_layers);
   set_keep_layer_names (specific_options.keep_layer_names);
 
+  prepare_layers (layout);
+
   do_read (layout);
 
   finish_layers (layout);
-  return layer_map ();
+  return layer_map_out ();
 }
 
 const LayerMap &

src/plugins/streamers/cif/unit_tests/dbCIFReader.cc

@@ -39,13 +39,8 @@ static void run_test (tl::TestBase *_this, const std::string &base, const char *
     unsigned int ln = 0;
     tl::Extractor ex (map);
     while (! ex.at_end ()) {
-      std::string n;
+      lm.add_expr (ex, ln++);
-      int l;
-      ex.read_word_or_quoted (n);
-      ex.test (":");
-      ex.read (l);
       ex.test (",");
-      lm.map (n, ln++, db::LayerProperties (l, 0));
     }
     opt->layer_map = lm;
     opt->create_other_layers = true;
@@ -173,6 +168,11 @@ TEST(3b)
   run_test (_this, tl::testsrc_private (), "t3.cif.gz", "t3b_au.gds.gz", "CAA:43,CCA:48,CCP:47,CMF:49,CMS:51,CPG:46,CSN:45,CSP:44,CVA:50,CWN:42,XP:26", 0.00012);
 }
 
+TEST(3c)
+{
+  run_test (_this, tl::testsrc_private (), "t3.cif.gz", "t3c_au.gds.gz", "(CPG:1/0) +(CPG:1000/0) (CCP:1/0) (CMF:2/0) +(CMF:1000/0) (CVA:3/0)", 0.00012);
+}
+
 TEST(4)
 {
   run_test (_this, tl::testsrc_private (), "t4.cif.gz", "t4_au.gds.gz");

src/plugins/streamers/dxf/db_plugin/dbDXFReader.cc

@@ -318,9 +318,7 @@ DXFReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
   m_stream.reset ();
   m_initial = true;
   m_line_number = 0;
-  db::LayerMap lm = specific_options.layer_map;
+  set_layer_map (specific_options.layer_map);
-  lm.prepare (layout);
-  set_layer_map (lm);
   set_create_layers (specific_options.create_other_layers);
   set_keep_layer_names (specific_options.keep_layer_names);
 
@@ -330,7 +328,7 @@ DXFReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
   do_read (layout, top);
   cleanup (layout, top);
 
-  return layer_map ();
+  return layer_map_out ();
 }
 
 const LayerMap &
@@ -379,29 +377,24 @@ DXFReader::open_layer (db::Layout &layout, const std::string &n)
 void
 DXFReader::do_read (db::Layout &layout, db::cell_index_type top)
 {
+  prepare_layers (layout);
+
   //  create the zero layer - this is not mapped to GDS but can be specified in the layer mapping as
   //  a layer named "0".
-  std::pair<bool, unsigned int> ll = layer_map ().first_logical (zero_layer_name, layout);
+  std::pair<bool, unsigned int> ll = open_layer (layout, zero_layer_name);
   if (ll.first) {
 
-    //  create the layer if it is not part of the layout yet.
+    //  layer exists
-    if (! layout.is_valid_layer (ll.second)) {
-      layout.insert_layer (ll.second, layer_map ().mapping (ll.second));
-    }
-
     m_zero_layer = ll.second;
 
   } else {
 
     //  or explicitly create the layer:
-    m_zero_layer = layer_map ().next_index ();
+    m_zero_layer = layout.insert_layer (db::LayerProperties (0, 0, zero_layer_name));
-    layout.insert_layer (m_zero_layer, db::LayerProperties (0, 0, zero_layer_name));
     map_layer (zero_layer_name, m_zero_layer);
 
   }
 
-  prepare_layers ();
-
   // Read sections
   int g;
 

src/plugins/streamers/gds2/unit_tests/dbGDS2Reader.cc

@@ -476,6 +476,47 @@ TEST(3_MultiMapping)
   db::compare_layouts (_this, layout, fn_au, db::WriteGDS2, 1);
 }
 
+TEST(3_MultiMapping2)
+{
+  db::Manager m (false);
+  db::Layout layout (&m);
+
+  db::LoadLayoutOptions options;
+  db::LayerMap lm, lm_read;
+
+  unsigned int n = 0;
+  lm.add_expr ("(1/0:1/0)", n++);
+  lm.add_expr ("+(1/0:1000/0)", n++);
+  lm.add_expr ("(4/0:1/0)", n++);
+  lm.add_expr ("(2/0:2/0)", n++);
+  lm.add_expr ("+(2/0:1000/0)", n++);
+  lm.add_expr ("(3/0:3/0)", n++);
+  options.get_options<db::CommonReaderOptions> ().layer_map = lm;
+
+  {
+    tl::InputStream file (tl::testsrc () + "/testdata/gds/t10.gds");
+    db::Reader reader (file);
+    lm_read = reader.read (layout, options);
+  }
+
+  EXPECT_EQ (lm_read.to_string_file_format (),
+    "+1/0;4/0 : 1/0\n"
+    "+1-2/0 : 1000/0\n"
+    "+2/0 : 2/0\n"
+    "3/0 : 3/0\n"
+    "6/0 : 6/0\n"
+    "8/0 : 8/0\n"
+    "5/0 : 5/0\n"
+    "7/0 : 7/0\n"
+    "3/1 : 3/1\n"
+    "6/1 : 6/1\n"
+    "8/1 : 8/1\n"
+  );
+
+  std::string fn_au (tl::testsrc () + "/testdata/gds/alm_au3.gds");
+  db::compare_layouts (_this, layout, fn_au, db::WriteGDS2, 1);
+}
+
 TEST(4_CollectModeRename)
 {
   db::Manager m (false);

src/plugins/streamers/magic/db_plugin/dbMAGReader.cc

@@ -72,8 +72,6 @@ MAGReader::read (db::Layout &layout)
 const LayerMap &
 MAGReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
 {
-  prepare_layers ();
-
   mp_klayout_tech = 0;
   std::string klayout_tech_name = layout.meta_info_value ("technology");
   if (! klayout_tech_name.empty () && db::Technologies::instance ()->has_technology (klayout_tech_name)) {
@@ -87,9 +85,7 @@ MAGReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
   m_merge = specific_options.merge;
   mp_current_stream = 0;
 
-  db::LayerMap lm = specific_options.layer_map;
+  set_layer_map (specific_options.layer_map);
-  lm.prepare (layout);
-  set_layer_map (lm);
   set_create_layers (specific_options.create_other_layers);
   set_keep_layer_names (specific_options.keep_layer_names);
 
@@ -110,6 +106,8 @@ MAGReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
   m_dbu_trans_inv = db::CplxTrans (m_dbu).inverted ();
   m_tech.clear ();
 
+  prepare_layers (layout);
+
   {
     tl::SelfTimer timer (tl::verbosity () >= 11, "Reading MAGIC file tree");
 
@@ -129,7 +127,7 @@ MAGReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
   }
 
   finish_layers (layout);
-  return layer_map ();
+  return layer_map_out ();
 }
 
 void