From 6b5268e5f768c35d35a3d6ca33db90fd32345552 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 6 Apr 2025 19:21:02 +0200 Subject: [PATCH] Feature glob expansion on LEF and GDS lists for LEF/DEF reader options. --- .../lefdef/db_plugin/dbLEFDEFImporter.cc | 25 +++-- .../lefdef/db_plugin/dbLEFDEFImporter.h | 2 +- .../lefdef/db_plugin/dbLEFDEFPlugin.cc | 56 +++++++----- src/tl/tl/tlFileUtils.cc | 48 ++++++++++ src/tl/tl/tlFileUtils.h | 8 ++ src/tl/unit_tests/tlFileUtilsTests.cc | 91 +++++++++++++++++++ 6 files changed, 198 insertions(+), 32 deletions(-) diff --git a/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.cc b/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.cc index ff229aeb9..d95144c75 100644 --- a/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.cc +++ b/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.cc @@ -38,7 +38,7 @@ namespace db // ----------------------------------------------------------------------------------- // Path resolution utility -std::string correct_path (const std::string &fn_in, const db::Layout &layout, const std::string &base_path) +std::vector correct_path (const std::string &fn_in, const db::Layout &layout, const std::string &base_path, bool glob) { const db::Technology *tech = layout.technology (); @@ -64,19 +64,28 @@ std::string correct_path (const std::string &fn_in, const db::Layout &layout, co if (tech && ! tech->base_path ().empty ()) { std::string new_fn = tl::combine_path (tech->base_path (), fn); if (tl::file_exists (new_fn)) { - return new_fn; + std::vector res; + res.push_back (new_fn); + return res; + } else if (glob) { + return tl::glob_expand (new_fn); } } if (! base_path.empty ()) { - return tl::combine_path (base_path, fn); - } else { - return fn; + fn = tl::combine_path (base_path, fn); } - } else { - return fn; } + + if (tl::file_exists (fn) || ! glob) { + std::vector res; + res.push_back (fn); + return res; + } else { + return tl::glob_expand (fn); + } + } // ----------------------------------------------------------------------------------- @@ -1059,7 +1068,7 @@ LEFDEFReaderState::read_map_file (const std::string &filename, db::Layout &layou std::map, std::vector > layer_map; for (std::vector::const_iterator p = paths.begin (); p != paths.end (); ++p) { - read_single_map_file (correct_path (*p, layout, base_path), layer_map); + read_single_map_file (correct_path (*p, layout, base_path, false).front (), layer_map); } // build an explicit layer mapping now. diff --git a/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.h b/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.h index 607410b6a..2a5736f0e 100644 --- a/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.h +++ b/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFImporter.h @@ -52,7 +52,7 @@ struct MacroDesc; * @brief Correct a path relative to the stream and technology */ DB_PLUGIN_PUBLIC -std::string correct_path (const std::string &fn, const db::Layout &layout, const std::string &base_path); +std::vector correct_path (const std::string &fn, const db::Layout &layout, const std::string &base_path, bool glob); /** * @brief Convers a string to a MASKSHIFT index list diff --git a/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFPlugin.cc b/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFPlugin.cc index 8d37fa283..c8d3d840c 100644 --- a/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFPlugin.cc +++ b/src/plugins/streamers/lefdef/db_plugin/dbLEFDEFPlugin.cc @@ -141,11 +141,12 @@ LEFDEFReader::read_lefdef (db::Layout &layout, const db::LoadLayoutOptions &opti for (std::vector::const_iterator l = effective_options.begin_lef_files (); l != effective_options.end_lef_files (); ++l) { - std::string lp = correct_path (*l, layout, base_path); - - tl::InputStream lef_stream (lp); - tl::log << tl::to_string (tr ("Reading")) << " " << lp; - importer.read (lef_stream, layout, state); + auto paths = correct_path (*l, layout, base_path, true); + for (auto lp = paths.begin (); lp != paths.end (); ++lp) { + tl::InputStream lef_stream (*lp); + tl::log << tl::to_string (tr ("Reading")) << " " << *lp; + importer.read (lef_stream, layout, state); + } } @@ -164,14 +165,20 @@ LEFDEFReader::read_lefdef (db::Layout &layout, const db::LoadLayoutOptions &opti for (std::vector::const_iterator l = effective_options.begin_lef_files (); l != effective_options.end_lef_files (); ++l) { - std::string lp = correct_path (*l, layout, base_path); - lef_files_read.insert (tl::normalize_path (lp)); + auto paths = correct_path (*l, layout, base_path, true); + for (auto lp = paths.begin (); lp != paths.end (); ++lp) { - tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (tr ("Reading LEF file: ")) + lp); + if (lef_files_read.insert (tl::normalize_path (*lp)).second) { - tl::InputStream lef_stream (lp); - tl::log << tl::to_string (tr ("Reading")) << " " << lp; - importer.read_lef (lef_stream, layout, state); + tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (tr ("Reading LEF file: ")) + *lp); + + tl::InputStream lef_stream (*lp); + tl::log << tl::to_string (tr ("Reading")) << " " << *lp; + importer.read_lef (lef_stream, layout, state); + + } + + } } @@ -223,22 +230,25 @@ LEFDEFReader::read_lefdef (db::Layout &layout, const db::LoadLayoutOptions &opti tl::shared_collection macro_layout_object_holder; for (std::vector::const_iterator l = effective_options.begin_macro_layout_files (); l != effective_options.end_macro_layout_files (); ++l) { - std::string lp = correct_path (*l, layout, base_path); + auto paths = correct_path (*l, layout, base_path, true); + for (auto lp = paths.begin (); lp != paths.end (); ++lp) { - tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (tr ("Reading LEF macro layout file: ")) + lp); + tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (tr ("Reading LEF macro layout file: ")) + *lp); - tl::InputStream macro_layout_stream (lp); - tl::log << tl::to_string (tr ("Reading")) << " " << lp; - db::Layout *new_layout = new db::Layout (false); - macro_layout_object_holder.push_back (new_layout); - macro_layouts.push_back (new_layout); + tl::InputStream macro_layout_stream (*lp); + tl::log << tl::to_string (tr ("Reading")) << " " << *lp; + db::Layout *new_layout = new db::Layout (false); + macro_layout_object_holder.push_back (new_layout); + macro_layouts.push_back (new_layout); - db::Reader reader (macro_layout_stream); - reader.read (*new_layout, options); + db::Reader reader (macro_layout_stream); + reader.read (*new_layout, options); + + if (fabs (new_layout->dbu () / layout.dbu () - 1.0) > db::epsilon) { + importer.warn (tl::sprintf (tl::to_string (tr ("DBU of macro layout file '%s' does not match reader DBU (layout DBU is %.12g, reader DBU is set to %.12g)")), + *lp, new_layout->dbu (), layout.dbu ())); + } - if (fabs (new_layout->dbu () / layout.dbu () - 1.0) > db::epsilon) { - importer.warn (tl::sprintf (tl::to_string (tr ("DBU of macro layout file '%s' does not match reader DBU (layout DBU is %.12g, reader DBU is set to %.12g)")), - lp, new_layout->dbu (), layout.dbu ())); } } diff --git a/src/tl/tl/tlFileUtils.cc b/src/tl/tl/tlFileUtils.cc index 7debc3591..a3b755db0 100644 --- a/src/tl/tl/tlFileUtils.cc +++ b/src/tl/tl/tlFileUtils.cc @@ -25,6 +25,7 @@ #include "tlLog.h" #include "tlInternational.h" #include "tlEnv.h" +#include "tlGlobPattern.h" #include #include @@ -430,6 +431,53 @@ std::vector dir_entries (const std::string &s, bool with_files, boo return ee; } +static void glob_partial (const std::string &where, std::vector::const_iterator pfrom, std::vector::const_iterator pto, std::vector &res) +{ + if (pfrom == pto) { + if (! is_dir (where)) { + res.push_back (where); + } + return; + } + + auto p = where + *pfrom; + if (file_exists (p)) { + glob_partial (p, pfrom + 1, pto, res); + return; + } + + if (tl::trimmed_part (*pfrom) == "**") { + if (pfrom + 1 == pto) { + // a glob pattern can't be "**" without anything after that + return; + } + auto subdirs = dir_entries (where, false, true, true); + for (auto s = subdirs.begin (); s != subdirs.end (); ++s) { + glob_partial (combine_path (where, *s), pfrom, pto, res); + } + ++pfrom; + } + + tl::GlobPattern glob (tl::trimmed_part (*pfrom)); + ++pfrom; + auto entries = dir_entries (where, true, true, true); + for (auto e = entries.begin (); e != entries.end (); ++e) { + if (glob.match (*e)) { + glob_partial (combine_path (where, *e), pfrom, pto, res); + } + } +} + +std::vector glob_expand (const std::string &path) +{ + auto apath = absolute_file_path (path); + auto parts = split_path (apath); + + std::vector res; + glob_partial (std::string (), parts.begin (), parts.end (), res); + return res; +} + bool mkdir (const std::string &path) { #if defined(_WIN32) diff --git a/src/tl/tl/tlFileUtils.h b/src/tl/tl/tlFileUtils.h index 3456eb6da..4401ac41b 100644 --- a/src/tl/tl/tlFileUtils.h +++ b/src/tl/tl/tlFileUtils.h @@ -138,6 +138,14 @@ bool TL_PUBLIC is_dir (const std::string &s); */ std::vector TL_PUBLIC dir_entries (const std::string &s, bool with_files = true, bool with_dirs = true, bool without_dotfiles = false); +/** + * @brief Expands a glob pattern into a set of files + * + * This version supports "**" for recursive directory expansion. + * Apart from that the features of tl::GlobPattern are supported. + */ +std::vector TL_PUBLIC glob_expand (const std::string &path); + /** * @brief Rename the given file */ diff --git a/src/tl/unit_tests/tlFileUtilsTests.cc b/src/tl/unit_tests/tlFileUtilsTests.cc index c12776608..2eafe8bc3 100644 --- a/src/tl/unit_tests/tlFileUtilsTests.cc +++ b/src/tl/unit_tests/tlFileUtilsTests.cc @@ -995,3 +995,94 @@ TEST (24) EXPECT_EQ (tl::file_exists (p), false); } +// glob_expand +TEST (25) +{ + tl::TemporaryDirectory tmpdir ("tl_tests"); + auto p = tmpdir.path (); + + auto ad = tl::combine_path (p, "a"); + tl::mkpath (ad); + auto aad = tl::combine_path (ad, "a"); + tl::mkpath (aad); + auto aaad = tl::combine_path (aad, "a"); + tl::mkpath (aaad); + auto bd = tl::combine_path (p, "b"); + tl::mkpath (bd); + + { + std::ofstream os (tl::combine_path (ad, "test.txt")); + os << "A test"; + os.close (); + } + + { + std::ofstream os (tl::combine_path (aad, "test.txt")); + os << "A test"; + os.close (); + } + + { + std::ofstream os (tl::combine_path (aaad, "test.txt")); + os << "A test"; + os.close (); + } + + { + std::ofstream os (tl::combine_path (aaad, "test2.txt")); + os << "A test"; + os.close (); + } + + { + std::ofstream os (tl::combine_path (bd, "test.txt")); + os << "A test"; + os.close (); + } + + { + std::ofstream os (tl::combine_path (p, "test2.txt")); + os << "A test"; + os.close (); + } + + std::vector au; + + auto res = tl::glob_expand (tl::combine_path (p, "*.txt")); + au.push_back (tl::combine_path (p, "test2.txt")); + + std::sort (res.begin (), res.end ()); + std::sort (au.begin (), au.end ()); + EXPECT_EQ (res == au, true); + + res = tl::glob_expand (tl::combine_path (tl::combine_path (p, "**"), "*.txt")); + au.clear (); + au.push_back (tl::combine_path (p, "test2.txt")); + au.push_back (tl::combine_path (ad, "test.txt")); + au.push_back (tl::combine_path (aad, "test.txt")); + au.push_back (tl::combine_path (aaad, "test.txt")); + au.push_back (tl::combine_path (aaad, "test2.txt")); + au.push_back (tl::combine_path (bd, "test.txt")); + + std::sort (res.begin (), res.end ()); + std::sort (au.begin (), au.end ()); + EXPECT_EQ (res == au, true); + + res = tl::glob_expand (tl::combine_path (tl::combine_path (p, "**"), "*2.txt")); + au.clear (); + au.push_back (tl::combine_path (p, "test2.txt")); + au.push_back (tl::combine_path (aaad, "test2.txt")); + + std::sort (res.begin (), res.end ()); + std::sort (au.begin (), au.end ()); + EXPECT_EQ (res == au, true); + + res = tl::glob_expand (tl::combine_path (tl::combine_path (tl::combine_path (p, "**"), "a"), "*2.txt")); + au.clear (); + au.push_back (tl::combine_path (aaad, "test2.txt")); + + std::sort (res.begin (), res.end ()); + std::sort (au.begin (), au.end ()); + EXPECT_EQ (res == au, true); +} +