Feature glob expansion on LEF and GDS lists for LEF/DEF reader options.

src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.cc

@@ -38,7 +38,7 @@ namespace db
 // -----------------------------------------------------------------------------------
 //  Path resolution utility
 
-std::string correct_path (const std::string &fn_in, const db::Layout &layout, const std::string &base_path)
+std::vector<std::string> correct_path (const std::string &fn_in, const db::Layout &layout, const std::string &base_path, bool glob)
 {
   const db::Technology *tech = layout.technology ();
 
@@ -64,19 +64,28 @@ std::string correct_path (const std::string &fn_in, const db::Layout &layout, co
     if (tech && ! tech->base_path ().empty ()) {
       std::string new_fn = tl::combine_path (tech->base_path (), fn);
       if (tl::file_exists (new_fn)) {
-        return new_fn;
+        std::vector<std::string> res;
+        res.push_back (new_fn);
+        return res;
+      } else if (glob) {
+        return tl::glob_expand (new_fn);
       }
     }
 
     if (! base_path.empty ()) {
-      return tl::combine_path (base_path, fn);
-    } else {
-      return fn;
+      fn = tl::combine_path (base_path, fn);
     }
 
-  } else {
-    return fn;
   }
+
+  if (tl::file_exists (fn) || ! glob) {
+    std::vector<std::string> res;
+    res.push_back (fn);
+    return res;
+  } else {
+    return tl::glob_expand (fn);
+  }
+
 }
 
 // -----------------------------------------------------------------------------------
@@ -1059,7 +1068,7 @@ LEFDEFReaderState::read_map_file (const std::string &filename, db::Layout &layou
   std::map<std::pair<std::string, LayerDetailsKey>, std::vector<db::LayerProperties> > layer_map;
 
   for (std::vector<std::string>::const_iterator p = paths.begin (); p != paths.end (); ++p) {
-    read_single_map_file (correct_path (*p, layout, base_path), layer_map);
+    read_single_map_file (correct_path (*p, layout, base_path, false).front (), layer_map);
   }
 
   //  build an explicit layer mapping now.

src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.h

@@ -52,7 +52,7 @@ struct MacroDesc;
  *  @brief Correct a path relative to the stream and technology
  */
 DB_PLUGIN_PUBLIC
-std::string correct_path (const std::string &fn, const db::Layout &layout, const std::string &base_path);
+std::vector<std::string> correct_path (const std::string &fn, const db::Layout &layout, const std::string &base_path, bool glob);
 
 /**
  *  @brief Convers a string to a MASKSHIFT index list

src/plugins/streamers/lefdef/db_plugin/dbLEFDEFPlugin.cc

@@ -141,11 +141,12 @@ LEFDEFReader::read_lefdef (db::Layout &layout, const db::LoadLayoutOptions &opti
 
     for (std::vector<std::string>::const_iterator l = effective_options.begin_lef_files (); l != effective_options.end_lef_files (); ++l) {
 
-      std::string lp = correct_path (*l, layout, base_path);
-
-      tl::InputStream lef_stream (lp);
-      tl::log << tl::to_string (tr ("Reading")) << " " << lp;
-      importer.read (lef_stream, layout, state);
+      auto paths = correct_path (*l, layout, base_path, true);
+      for (auto lp = paths.begin (); lp != paths.end (); ++lp) {
+        tl::InputStream lef_stream (*lp);
+        tl::log << tl::to_string (tr ("Reading")) << " " << *lp;
+        importer.read (lef_stream, layout, state);
+      }
 
     }
 
@@ -164,14 +165,20 @@ LEFDEFReader::read_lefdef (db::Layout &layout, const db::LoadLayoutOptions &opti
 
     for (std::vector<std::string>::const_iterator l = effective_options.begin_lef_files (); l != effective_options.end_lef_files (); ++l) {
 
-      std::string lp = correct_path (*l, layout, base_path);
-      lef_files_read.insert (tl::normalize_path (lp));
+      auto paths = correct_path (*l, layout, base_path, true);
+      for (auto lp = paths.begin (); lp != paths.end (); ++lp) {
 
-      tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (tr ("Reading LEF file: ")) + lp);
+        if (lef_files_read.insert (tl::normalize_path (*lp)).second) {
 
-      tl::InputStream lef_stream (lp);
-      tl::log << tl::to_string (tr ("Reading")) << " " << lp;
-      importer.read_lef (lef_stream, layout, state);
+          tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (tr ("Reading LEF file: ")) + *lp);
+
+          tl::InputStream lef_stream (*lp);
+          tl::log << tl::to_string (tr ("Reading")) << " " << *lp;
+          importer.read_lef (lef_stream, layout, state);
+
+        }
+
+      }
 
     }
 
@@ -223,22 +230,25 @@ LEFDEFReader::read_lefdef (db::Layout &layout, const db::LoadLayoutOptions &opti
     tl::shared_collection<db::Layout> macro_layout_object_holder;
     for (std::vector<std::string>::const_iterator l = effective_options.begin_macro_layout_files (); l != effective_options.end_macro_layout_files (); ++l) {
 
-      std::string lp = correct_path (*l, layout, base_path);
+      auto paths = correct_path (*l, layout, base_path, true);
+      for (auto lp = paths.begin (); lp != paths.end (); ++lp) {
 
-      tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (tr ("Reading LEF macro layout file: ")) + lp);
+        tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (tr ("Reading LEF macro layout file: ")) + *lp);
 
-      tl::InputStream macro_layout_stream (lp);
-      tl::log << tl::to_string (tr ("Reading")) << " " << lp;
-      db::Layout *new_layout = new db::Layout (false);
-      macro_layout_object_holder.push_back (new_layout);
-      macro_layouts.push_back (new_layout);
+        tl::InputStream macro_layout_stream (*lp);
+        tl::log << tl::to_string (tr ("Reading")) << " " << *lp;
+        db::Layout *new_layout = new db::Layout (false);
+        macro_layout_object_holder.push_back (new_layout);
+        macro_layouts.push_back (new_layout);
 
-      db::Reader reader (macro_layout_stream);
-      reader.read (*new_layout, options);
+        db::Reader reader (macro_layout_stream);
+        reader.read (*new_layout, options);
+
+        if (fabs (new_layout->dbu () / layout.dbu () - 1.0) > db::epsilon) {
+          importer.warn (tl::sprintf (tl::to_string (tr ("DBU of macro layout file '%s' does not match reader DBU (layout DBU is %.12g, reader DBU is set to %.12g)")),
+                                      *lp, new_layout->dbu (), layout.dbu ()));
+        }
 
-      if (fabs (new_layout->dbu () / layout.dbu () - 1.0) > db::epsilon) {
-        importer.warn (tl::sprintf (tl::to_string (tr ("DBU of macro layout file '%s' does not match reader DBU (layout DBU is %.12g, reader DBU is set to %.12g)")),
-                                    lp, new_layout->dbu (), layout.dbu ()));
       }
 
     }

src/tl/tl/tlFileUtils.cc

@@ -25,6 +25,7 @@
 #include "tlLog.h"
 #include "tlInternational.h"
 #include "tlEnv.h"
+#include "tlGlobPattern.h"
 
 #include <cctype>
 #include <fstream>
@@ -430,6 +431,53 @@ std::vector<std::string> dir_entries (const std::string &s, bool with_files, boo
   return ee;
 }
 
+static void glob_partial (const std::string &where, std::vector<std::string>::const_iterator pfrom, std::vector<std::string>::const_iterator pto, std::vector<std::string> &res)
+{
+  if (pfrom == pto) {
+    if (! is_dir (where)) {
+      res.push_back (where);
+    }
+    return;
+  }
+
+  auto p = where + *pfrom;
+  if (file_exists (p)) {
+    glob_partial (p, pfrom + 1, pto, res);
+    return;
+  }
+
+  if (tl::trimmed_part (*pfrom) == "**") {
+    if (pfrom + 1 == pto) {
+      //  a glob pattern can't be "**" without anything after that
+      return;
+    }
+    auto subdirs = dir_entries (where, false, true, true);
+    for (auto s = subdirs.begin (); s != subdirs.end (); ++s) {
+      glob_partial (combine_path (where, *s), pfrom, pto, res);
+    }
+    ++pfrom;
+  }
+
+  tl::GlobPattern glob (tl::trimmed_part (*pfrom));
+  ++pfrom;
+  auto entries = dir_entries (where, true, true, true);
+  for (auto e = entries.begin (); e != entries.end (); ++e) {
+    if (glob.match (*e)) {
+      glob_partial (combine_path (where, *e), pfrom, pto, res);
+    }
+  }
+}
+
+std::vector<std::string> glob_expand (const std::string &path)
+{
+  auto apath = absolute_file_path (path);
+  auto parts = split_path (apath);
+
+  std::vector<std::string> res;
+  glob_partial (std::string (), parts.begin (), parts.end (), res);
+  return res;
+}
+
 bool mkdir (const std::string &path)
 {
 #if defined(_WIN32)

src/tl/tl/tlFileUtils.h

@@ -138,6 +138,14 @@ bool TL_PUBLIC is_dir (const std::string &s);
  */
 std::vector<std::string> TL_PUBLIC dir_entries (const std::string &s, bool with_files = true, bool with_dirs = true, bool without_dotfiles = false);
 
+/**
+ *  @brief Expands a glob pattern into a set of files
+ *
+ *  This version supports "**" for recursive directory expansion.
+ *  Apart from that the features of tl::GlobPattern are supported.
+ */
+std::vector<std::string> TL_PUBLIC glob_expand (const std::string &path);
+
 /**
  *  @brief Rename the given file
  */

src/tl/unit_tests/tlFileUtilsTests.cc

@@ -995,3 +995,94 @@ TEST (24)
   EXPECT_EQ (tl::file_exists (p), false);
 }
 
+//  glob_expand
+TEST (25)
+{
+  tl::TemporaryDirectory tmpdir ("tl_tests");
+  auto p = tmpdir.path ();
+
+  auto ad = tl::combine_path (p, "a");
+  tl::mkpath (ad);
+  auto aad = tl::combine_path (ad, "a");
+  tl::mkpath (aad);
+  auto aaad = tl::combine_path (aad, "a");
+  tl::mkpath (aaad);
+  auto bd = tl::combine_path (p, "b");
+  tl::mkpath (bd);
+
+  {
+    std::ofstream os (tl::combine_path (ad, "test.txt"));
+    os << "A test";
+    os.close ();
+  }
+
+  {
+    std::ofstream os (tl::combine_path (aad, "test.txt"));
+    os << "A test";
+    os.close ();
+  }
+
+  {
+    std::ofstream os (tl::combine_path (aaad, "test.txt"));
+    os << "A test";
+    os.close ();
+  }
+
+  {
+    std::ofstream os (tl::combine_path (aaad, "test2.txt"));
+    os << "A test";
+    os.close ();
+  }
+
+  {
+    std::ofstream os (tl::combine_path (bd, "test.txt"));
+    os << "A test";
+    os.close ();
+  }
+
+  {
+    std::ofstream os (tl::combine_path (p, "test2.txt"));
+    os << "A test";
+    os.close ();
+  }
+
+  std::vector<std::string> au;
+
+  auto res = tl::glob_expand (tl::combine_path (p, "*.txt"));
+  au.push_back (tl::combine_path (p, "test2.txt"));
+
+  std::sort (res.begin (), res.end ());
+  std::sort (au.begin (), au.end ());
+  EXPECT_EQ (res == au, true);
+
+  res = tl::glob_expand (tl::combine_path (tl::combine_path (p, "**"), "*.txt"));
+  au.clear ();
+  au.push_back (tl::combine_path (p, "test2.txt"));
+  au.push_back (tl::combine_path (ad, "test.txt"));
+  au.push_back (tl::combine_path (aad, "test.txt"));
+  au.push_back (tl::combine_path (aaad, "test.txt"));
+  au.push_back (tl::combine_path (aaad, "test2.txt"));
+  au.push_back (tl::combine_path (bd, "test.txt"));
+
+  std::sort (res.begin (), res.end ());
+  std::sort (au.begin (), au.end ());
+  EXPECT_EQ (res == au, true);
+
+  res = tl::glob_expand (tl::combine_path (tl::combine_path (p, "**"), "*2.txt"));
+  au.clear ();
+  au.push_back (tl::combine_path (p, "test2.txt"));
+  au.push_back (tl::combine_path (aaad, "test2.txt"));
+
+  std::sort (res.begin (), res.end ());
+  std::sort (au.begin (), au.end ());
+  EXPECT_EQ (res == au, true);
+
+  res = tl::glob_expand (tl::combine_path (tl::combine_path (tl::combine_path (p, "**"), "a"), "*2.txt"));
+  au.clear ();
+  au.push_back (tl::combine_path (aaad, "test2.txt"));
+
+  std::sort (res.begin (), res.end ());
+  std::sort (au.begin (), au.end ());
+  EXPECT_EQ (res == au, true);
+}
+