Merge pull request #2084 from KLayout/feature/issue-2044

Feature/issue 2044

src/buddies/src/bd/bdConverterMain.cc

@@ -43,8 +43,13 @@ int converter_main (int argc, char *argv[], const std::string &format)
   generic_reader_options.add_options (cmd);
 
   cmd << tl::arg ("input",  &infile,  "The input file (any format, may be gzip compressed)",
-                  "You can use '+' or ',' to supply multiple files which will be read after each other into the same layout. "
-                  "This provides some cheap, but risky way of merging files. Beware of cell name conflicts.")
+                  "Multiple files can be combined using '+' or ','. '+' will combine the files in 'blending' mode. "
+                  "In this mode it is possible to combine identically named cells into one cell for example. This mode "
+                  "needs to be used with care and there some constraints - e.g. the database unit of the involved "
+                  "layouts needs to be the same. When using ',' as a separator, blending is not used, but the layouts "
+                  "are merged by first creating two layouts and then combining them into one. This mode is more robust "
+                  "but does not allow cell merging. '+' combination has higher priority than ',' - i.e. 'a+b,c' is "
+                  "understood as '(a+b),c'.")
       << tl::arg ("output", &outfile, tl::sprintf ("The output file (%s format)", format))
     ;
 

src/buddies/src/bd/bdReaderOptions.cc

@@ -22,6 +22,8 @@
 
 #include "bdReaderOptions.h"
 #include "dbLoadLayoutOptions.h"
+#include "dbLayerMapping.h"
+#include "dbCellMapping.h"
 #include "tlCommandLineParser.h"
 
 #include "tlStream.h"
@@ -831,15 +833,28 @@ static std::string::size_type find_file_sep (const std::string &s, std::string::
   }
 }
 
-static std::vector<std::string> split_file_list (const std::string &infile)
+static std::vector<std::vector<std::string> > split_file_list (const std::string &infile)
 {
-  std::vector<std::string> files;
+  std::vector<std::vector<std::string> > files;
+  files.push_back (std::vector<std::string> ());
 
   size_t p = 0;
-  for (size_t pp = 0; (pp = find_file_sep (infile, p)) != std::string::npos; p = pp + 1) {
-    files.push_back (std::string (infile, p, pp - p));
+  while (true) {
+
+    size_t sep = find_file_sep (infile, p);
+    if (sep == std::string::npos) {
+      files.back ().push_back (std::string (infile, p));
+      return files;
+    }
+
+    files.back ().push_back (std::string (infile, p, sep - p));
+    if (infile [sep] == ',') {
+      files.push_back (std::vector<std::string> ());
+    }
+
+    p = sep + 1;
+
   }
-  files.push_back (std::string (infile, p));
 
   return files;
 }
@@ -850,16 +865,73 @@ void read_files (db::Layout &layout, const std::string &infile, const db::LoadLa
   //    db::LayoutLocker locker (&layout);
   //  but there are yet unknown side effects
 
-  //  enter a LEF caching context for chaining multiple DEF with the same LEF
-  db::LoadLayoutOptions local_options (options);
-  local_options.set_option_by_name ("lefdef_config.lef_context_enabled", true);
+  std::vector<std::vector<std::string> > files = split_file_list (infile);
 
-  std::vector<std::string> files = split_file_list (infile);
+  for (auto ff = files.begin (); ff != files.end (); ++ff) {
+
+    //  enter a LEF caching context for chaining multiple DEF with the same LEF
+    db::LoadLayoutOptions local_options (options);
+    local_options.set_option_by_name ("lefdef_config.lef_context_enabled", true);
+
+    db::Layout tmp;
+    db::Layout *ly = (ff == files.begin () ? &layout : &tmp);
+
+    for (auto f = ff->begin (); f != ff->end (); ++f) {
+      tl::InputStream stream (*f);
+      db::Reader reader (stream);
+      if (f != ff->begin ()) {
+        reader.set_expected_dbu (ly->dbu ());
+      }
+      reader.read (*ly, local_options);
+    }
+
+    if (ly != &layout) {
+
+      //  Move over cells from read layout to destination ("," separated blocks).
+      //  This path does not imply limitations in terms of DBU compatibility etc.
+
+      std::vector<db::cell_index_type> cells_target;
+      std::vector<db::cell_index_type> cells_source;
+
+      for (auto c = tmp.begin_top_down (); c != tmp.end_top_cells (); ++c) {
+
+        cells_source.push_back (*c);
+
+        //  as a special rule, join ghost cells if the source top cell fits into
+        //  a ghost cell of the target.
+        auto cell_target = layout.cell_by_name (tmp.cell_name (*c));
+        if (cell_target.first && layout.cell (cell_target.second).is_ghost_cell ()) {
+          cells_target.push_back (cell_target.second);
+        } else {
+          cells_target.push_back (layout.add_cell (tmp.cell_name (*c)));
+        }
+
+      }
+
+      //  ghost cell joining also works the other way around: a top cell of destination
+      //  can match a ghost cell of the source
+      for (auto c = tmp.end_top_cells (); c != tmp.end_top_down (); ++c) {
+
+        const db::Cell &cell_source = tmp.cell (*c);
+        auto cell_target = layout.cell_by_name (tmp.cell_name (*c));
+
+        if (cell_source.is_ghost_cell () && cell_target.first) {
+          cells_source.push_back (*c);
+          cells_target.push_back (cell_target.second);
+        }
+
+      }
+
+      db::CellMapping cm;
+      cm.create_multi_mapping_full (layout, cells_target, tmp, cells_source);
+
+      db::LayerMapping lm;
+      lm.create_full (layout, tmp);
+
+      layout.move_tree_shapes (tmp, cm, lm);
+
+    }
 
-  for (std::vector<std::string>::const_iterator f = files.begin (); f != files.end (); ++f) {
-    tl::InputStream stream (*f);
-    db::Reader reader (stream);
-    reader.read (layout, local_options);
   }
 }
 

src/buddies/unit_tests/bdConverterTests.cc

@@ -483,7 +483,7 @@ TEST(10)
   std::string input;
   for (size_t i = 0; i < sizeof (def_files) / sizeof (def_files[0]); ++i) {
     if (i > 0) {
-      input += ",";
+      input += "+";
     }
     input += def_dir + "/" + def_files[i];
   }
@@ -510,3 +510,201 @@ TEST(10)
 
   db::compare_layouts (this, layout, input_au, db::WriteOAS);
 }
+
+//  Merging with +
+TEST(11_1)
+{
+  std::string input_dir = tl::testdata ();
+  input_dir += "/bd";
+
+  std::string input_au = input_dir + "/strm2oas_au_1.oas";
+  std::string input = input_dir + "/strm2oas_1.oas+" + input_dir + "/strm2oas_2.oas";
+
+  std::string output = this->tmp_file ("strm2oas_1.oas");
+  const char *argv[] = { "x",
+                         "--blend-mode=0",
+                         input.c_str (),
+                         output.c_str ()
+                       };
+
+  EXPECT_EQ (bd::converter_main (sizeof (argv) / sizeof (argv[0]), (char **) argv, bd::GenericWriterOptions::oasis_format_name), 0);
+
+  db::Layout layout;
+  {
+    tl::InputStream stream (output);
+    db::LoadLayoutOptions options;
+    db::Reader reader (stream);
+    reader.read (layout, options);
+  }
+
+  db::compare_layouts (this, layout, input_au, db::WriteOAS);
+}
+
+//  Merging with + not allowed on different DBUs
+TEST(11_2)
+{
+  std::string input_dir = tl::testdata ();
+  input_dir += "/bd";
+
+  std::string input_au = input_dir + "/strm2oas_au_1.oas";
+  std::string input = input_dir + "/strm2oas_1.oas+" + input_dir + "/strm2oas_2_10nm.oas";
+
+  std::string output = this->tmp_file ("strm2oas_1.oas");
+  const char *argv[] = { "x",
+                         "--blend-mode=0",
+                         input.c_str (),
+                         output.c_str ()
+                       };
+
+  try {
+    bd::converter_main (sizeof (argv) / sizeof (argv[0]), (char **) argv, bd::GenericWriterOptions::oasis_format_name);
+    EXPECT_EQ (1, 0);
+  } catch (tl::Exception &ex) {
+    EXPECT_EQ (ex.msg (), "Former and present database units are not compatible: 0.001 (former) vs. 0.01 (present)");
+  }
+}
+
+//  Merging with + not allowed on different DBUs
+TEST(11_3)
+{
+  std::string input_dir = tl::testdata ();
+  input_dir += "/bd";
+
+  std::string input_au = input_dir + "/strm2oas_au_3.oas";
+  std::string input = input_dir + "/strm2oas_1.oas," + input_dir + "/strm2oas_2_10nm.oas";
+
+  std::string output = this->tmp_file ("strm2oas_3.oas");
+  const char *argv[] = { "x",
+                         "--blend-mode=0",
+                         input.c_str (),
+                         output.c_str ()
+                       };
+
+  EXPECT_EQ (bd::converter_main (sizeof (argv) / sizeof (argv[0]), (char **) argv, bd::GenericWriterOptions::oasis_format_name), 0);
+
+  db::Layout layout;
+  {
+    tl::InputStream stream (output);
+    db::LoadLayoutOptions options;
+    db::Reader reader (stream);
+    reader.read (layout, options);
+  }
+
+  db::compare_layouts (this, layout, input_au, db::WriteOAS);
+}
+
+//  Merging with + and , under the presence of ghost cells: test+test,top->(test)
+TEST(12_1)
+{
+  std::string input_dir = tl::testdata ();
+  input_dir += "/bd";
+
+  std::string input_au = input_dir + "/strm2oas_au_12_1.oas";
+  std::string input = input_dir + "/strm2oas_a.oas+" + input_dir + "/strm2oas_b.oas," + input_dir + "/strm2oas_c.oas";
+
+  std::string output = this->tmp_file ("strm2oas_12_1.oas");
+  const char *argv[] = { "x",
+                         "--blend-mode=0",
+                         input.c_str (),
+                         output.c_str ()
+                       };
+
+  EXPECT_EQ (bd::converter_main (sizeof (argv) / sizeof (argv[0]), (char **) argv, bd::GenericWriterOptions::oasis_format_name), 0);
+
+  db::Layout layout;
+  {
+    tl::InputStream stream (output);
+    db::LoadLayoutOptions options;
+    db::Reader reader (stream);
+    reader.read (layout, options);
+  }
+
+  db::compare_layouts (this, layout, input_au, db::WriteOAS);
+}
+
+//  Merging with + and , under the presence of ghost cells: top->(test),test+test
+TEST(12_2)
+{
+  std::string input_dir = tl::testdata ();
+  input_dir += "/bd";
+
+  std::string input_au = input_dir + "/strm2oas_au_12_2.oas";
+  std::string input = input_dir + "/strm2oas_c.oas," + input_dir + "/strm2oas_a.oas+" + input_dir + "/strm2oas_b.oas";
+
+  std::string output = this->tmp_file ("strm2oas_12_2.oas");
+  const char *argv[] = { "x",
+                         "--blend-mode=0",
+                         input.c_str (),
+                         output.c_str ()
+                       };
+
+  EXPECT_EQ (bd::converter_main (sizeof (argv) / sizeof (argv[0]), (char **) argv, bd::GenericWriterOptions::oasis_format_name), 0);
+
+  db::Layout layout;
+  {
+    tl::InputStream stream (output);
+    db::LoadLayoutOptions options;
+    db::Reader reader (stream);
+    reader.read (layout, options);
+  }
+
+  db::compare_layouts (this, layout, input_au, db::WriteOAS);
+}
+
+//  Merging with + and , under the presence of ghost cells: test+test,toptop->top->(test)
+TEST(12_3)
+{
+  std::string input_dir = tl::testdata ();
+  input_dir += "/bd";
+
+  std::string input_au = input_dir + "/strm2oas_au_12_3.oas";
+  std::string input = input_dir + "/strm2oas_a.oas+" + input_dir + "/strm2oas_b.oas," + input_dir + "/strm2oas_cc.oas";
+
+  std::string output = this->tmp_file ("strm2oas_12_3.oas");
+  const char *argv[] = { "x",
+                         "--blend-mode=0",
+                         input.c_str (),
+                         output.c_str ()
+                       };
+
+  EXPECT_EQ (bd::converter_main (sizeof (argv) / sizeof (argv[0]), (char **) argv, bd::GenericWriterOptions::oasis_format_name), 0);
+
+  db::Layout layout;
+  {
+    tl::InputStream stream (output);
+    db::LoadLayoutOptions options;
+    db::Reader reader (stream);
+    reader.read (layout, options);
+  }
+
+  db::compare_layouts (this, layout, input_au, db::WriteOAS);
+}
+
+//  Merging with + and , under the presence of ghost cells: toptop->top->(test),test+test
+TEST(12_4)
+{
+  std::string input_dir = tl::testdata ();
+  input_dir += "/bd";
+
+  std::string input_au = input_dir + "/strm2oas_au_12_4.oas";
+  std::string input = input_dir + "/strm2oas_cc.oas," + input_dir + "/strm2oas_a.oas+" + input_dir + "/strm2oas_b.oas";
+
+  std::string output = this->tmp_file ("strm2oas_12_4.oas");
+  const char *argv[] = { "x",
+                         "--blend-mode=0",
+                         input.c_str (),
+                         output.c_str ()
+                       };
+
+  EXPECT_EQ (bd::converter_main (sizeof (argv) / sizeof (argv[0]), (char **) argv, bd::GenericWriterOptions::oasis_format_name), 0);
+
+  db::Layout layout;
+  {
+    tl::InputStream stream (output);
+    db::LoadLayoutOptions options;
+    db::Reader reader (stream);
+    reader.read (layout, options);
+  }
+
+  db::compare_layouts (this, layout, input_au, db::WriteOAS);
+}

src/db/db/dbReader.cc

@@ -61,7 +61,7 @@ join_layer_names (std::string &s, const std::string &n)
 //  ReaderBase implementation
 
 ReaderBase::ReaderBase () 
-  : m_warnings_as_errors (false), m_warn_level (1), m_warn_count_for_same_message (0), m_first_warning (true)
+  : m_warnings_as_errors (false), m_warn_level (1), m_warn_count_for_same_message (0), m_first_warning (true), m_expected_dbu (0.0)
 { 
 }
 
@@ -114,6 +114,20 @@ ReaderBase::compress_warning (const std::string &msg)
   }
 }
 
+void
+ReaderBase::set_expected_dbu (double dbu)
+{
+  m_expected_dbu = dbu;
+}
+
+void
+ReaderBase::check_dbu (double dbu) const
+{
+  if (m_expected_dbu > db::epsilon && fabs (dbu - m_expected_dbu) > db::epsilon) {
+    throw ReaderException (tl::sprintf (tl::to_string (tr ("Former and present database units are not compatible: %.12g (former) vs. %.12g (present)")), m_expected_dbu, dbu));
+  }
+}
+
 // ---------------------------------------------------------------
 //  Reader implementation
 

src/db/db/dbReader.h

@@ -138,6 +138,33 @@ public:
    */
   int compress_warning (const std::string &msg);
 
+  /**
+   *  @brief Sets the expected database unit
+   *
+   *  With this value set, the reader can check if the present database unit is
+   *  compatible with the expected one and either take actions to scale the layouts
+   *  or to reject the file.
+   *
+   *  Setting the value to 0 resets the expected DBU and will disable all checks
+   *  or scaling.
+   */
+  void set_expected_dbu (double dbu);
+
+  /**
+   *  @brief Gets the expected database unit
+   */
+  double expected_dbu () const
+  {
+    return m_expected_dbu;
+  }
+
+  /**
+   *  @brief Checks the given DBU against the expected one
+   *
+   *  This method will raise an exception if the database units do not match.
+   */
+  void check_dbu (double dbu) const;
+
 protected:
   virtual void init (const db::LoadLayoutOptions &options);
 
@@ -147,6 +174,7 @@ private:
   std::string m_last_warning;
   int m_warn_count_for_same_message;
   bool m_first_warning;
+  double m_expected_dbu;
 };
 
 /**
@@ -231,6 +259,22 @@ public:
     return mp_actual_reader->warnings_as_errors ();
   }
 
+  /**
+   *  @brief Sets the expected database unit (see ReaderBase)
+   */
+  void set_expected_dbu (double dbu)
+  {
+    return mp_actual_reader->set_expected_dbu (dbu);
+  }
+
+  /**
+   *  @brief Gets the expected database unit
+   */
+  double expected_dbu () const
+  {
+    return mp_actual_reader->expected_dbu ();
+  }
+
 private:
   ReaderBase *mp_actual_reader;
   tl::InputStream &m_stream;

src/plugins/streamers/cif/db_plugin/dbCIFReader.cc

@@ -836,6 +836,7 @@ CIFReader::do_read (db::Layout &layout)
     db::LayoutLocker locker (&layout);
   
     double sf = 0.01 / m_dbu;
+    check_dbu (m_dbu);
     layout.dbu (m_dbu);
 
     m_cellname = "{CIF top level}";

src/plugins/streamers/dxf/db_plugin/dbDXFReader.cc

@@ -350,6 +350,7 @@ DXFReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
 
   db::cell_index_type top = layout.add_cell("TOP"); // TODO: make variable ..
 
+  check_dbu (m_dbu);
   layout.dbu (m_dbu);
   do_read (layout, top);
   cleanup (layout, top);

src/plugins/streamers/gds2/db_plugin/dbGDS2ReaderBase.cc

@@ -236,6 +236,7 @@ GDS2ReaderBase::do_read (db::Layout &layout)
 
       m_dbuu = dbuu;
       m_dbu = dbum * 1e6; /*in micron*/
+      check_dbu (m_dbu);
       layout.dbu (m_dbu);
 
     } else {

src/plugins/streamers/lefdef/db_plugin/dbLEFDEFPlugin.cc

@@ -119,6 +119,7 @@ LEFDEFReader::read_lefdef (db::Layout &layout, const db::LoadLayoutOptions &opti
     effective_options = *lefdef_options;
   }
 
+  check_dbu (effective_options.dbu ());
   layout.dbu (effective_options.dbu ());
 
   std::string base_path;

src/plugins/streamers/magic/db_plugin/dbMAGReader.cc

@@ -98,6 +98,7 @@ MAGReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
     top_cell = layout.add_cell (top_cellname.c_str ());
   }
 
+  check_dbu (m_dbu);
   layout.dbu (m_dbu);
 
   m_cells_to_read.clear ();

src/plugins/streamers/maly/db_plugin/dbMALYReader.cc

@@ -51,7 +51,6 @@ namespace db
 MALYReader::MALYReader (tl::InputStream &s)
   : m_stream (s),
     m_progress (tl::to_string (tr ("Reading MALY file")), 1000),
-    m_dbu (0.001),
     m_last_record_line (0)
 {
   m_progress.set_format (tl::to_string (tr ("%.0fk lines")));
@@ -89,7 +88,10 @@ MALYReader::read (db::Layout &layout, const db::LoadLayoutOptions &options)
   init (options);
 
   const db::MALYReaderOptions &specific_options = options.get_options<db::MALYReaderOptions> ();
-  m_dbu = specific_options.dbu;
+  double dbu = specific_options.dbu;
+
+  check_dbu (dbu);
+  layout.dbu (dbu);
 
   set_layer_map (specific_options.layer_map);
   set_create_layers (specific_options.create_other_layers);

src/plugins/streamers/maly/db_plugin/dbMALYReader.h

@@ -223,7 +223,6 @@ private:
 
   tl::TextInputStream m_stream;
   tl::AbsoluteProgress m_progress;
-  double m_dbu;
   unsigned int m_last_record_line;
   std::string m_record;
   std::string m_record_returned;

src/plugins/streamers/maly/unit_tests/dbMALYReaderTests.cc

@@ -135,6 +135,7 @@ TEST(10_BasicLayout)
 {
   run_test (_this, tl::testdata (), "MALY_test10.maly", "maly_test10_au.oas");
   run_test (_this, tl::testdata (), "MALY_test10.maly", "maly_test10_lm_au.oas", "A: 10, B: 11, C: 12, D: 13");
+  run_test (_this, tl::testdata (), "MALY_test10.maly", "maly_test10_dbu10nm_au.oas", 0, 0.01);
 }
 
 TEST(11_Titles)

src/plugins/streamers/oasis/db_plugin/dbOASISReader.cc

@@ -623,7 +623,9 @@ OASISReader::do_read (db::Layout &layout)
   }
 
   //  compute database unit in pixel per meter
-  layout.dbu (1.0 / res);
+  double dbu = 1.0 / res;
+  check_dbu (dbu);
+  layout.dbu (dbu);
 
   //  read over table offsets if required
   bool table_offsets_at_end = get_uint64 () != 0;

src/plugins/streamers/pcb/db_plugin/dbGerberImporter.cc

@@ -1142,6 +1142,8 @@ public:
     db::GerberImporter importer (warn_level ());
     data.setup_importer (&importer);
 
+    check_dbu (data.dbu);
+
     importer.read (layout);
 
     std::string lyr_file = data.get_layer_properties_file ();