diff --git a/src/lym/lym/lym.pro b/src/lym/lym/lym.pro index 24a21587b..ea352c597 100644 --- a/src/lym/lym/lym.pro +++ b/src/lym/lym/lym.pro @@ -8,11 +8,13 @@ DEFINES += MAKE_LYM_LIBRARY SOURCES = \ gsiDeclLymMacro.cc \ + lymInclude.cpp \ lymMacroInterpreter.cc \ lymMacro.cc \ HEADERS = \ lymCommon.h \ + lymInclude.h \ lymMacroInterpreter.h \ lymMacro.h \ diff --git a/src/lym/lym/lymInclude.cpp b/src/lym/lym/lymInclude.cpp new file mode 100644 index 000000000..b48de2c56 --- /dev/null +++ b/src/lym/lym/lymInclude.cpp @@ -0,0 +1,189 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2020 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "lymInclude.h" + +#include "tlAssert.h" +#include "tlString.h" +#include "tlFileUtils.h" +#include "tlStream.h" +#include "tlExpression.h" +#include "tlUri.h" + +namespace lym +{ + +static const char *valid_fn_chars = "@_:,.\\/-+"; + +IncludeExpander::IncludeExpander () +{ + // .. nothing yet .. +} + +IncludeExpander +IncludeExpander::expand (const std::string &path, std::string &expanded_text) +{ + IncludeExpander ie; + int lc = 1; + read (path, expanded_text, ie, lc); + return ie; +} + +void +IncludeExpander::read (const std::string &path, std::string &expanded_text, IncludeExpander &ie, int &line_counter) +{ + ie.m_sections [line_counter] = std::make_pair (path, 1 - line_counter); + + tl::InputStream is (path); + tl::TextInputStream text (is); + + int lnum = 0; + bool emit_section = false; + + while (! text.at_end ()) { + + std::string l = text.get_line (); + ++lnum; + + tl::Extractor ex (l.c_str ()); + if (ex.test ("#") && ex.test ("%include")) { + + std::string include_path = tl::trim (ex.skip ()); + + // allow some customization by employing expression interpolation + include_path = tl::Eval ().interpolate (include_path); + + // NOTE: by using URI's we can basically read from HTTP etc. + tl::URI current_uri (path); + tl::URI new_uri (include_path); + if (current_uri.scheme ().empty () && new_uri.scheme ().empty ()) { + if (! tl::is_absolute (include_path)) { + include_path = tl::combine_path (tl::dirname (path), include_path); + } + } else { + include_path = current_uri.resolved (new_uri).to_string (); + } + + read (include_path, expanded_text, ie, line_counter); + + emit_section = true; + + } else { + + if (emit_section) { + emit_section = false; + ie.m_sections [line_counter] = std::make_pair (path, lnum - line_counter); + } + + expanded_text += l; + expanded_text += "\n"; + ++line_counter; + + } + + } +} + +std::string +IncludeExpander::to_string () const +{ + if (m_sections.empty ()) { + + return std::string (); + + } else if (m_sections.size () == 1) { + + tl_assert (m_sections.begin ()->first == 1); + tl_assert (m_sections.begin ()->second.second == 0); + + std::string fn = m_sections.begin ()->second.first; + return tl::to_word_or_quoted_string (fn, valid_fn_chars); + + } else { + + // "@" indicates a mapping table + std::string res ("@"); + + for (std::map >::const_iterator m = m_sections.begin (); m != m_sections.end (); ++m) { + res += tl::to_string (m->first); + res += ":"; + res += tl::to_word_or_quoted_string (m->second.first, valid_fn_chars); + res += "*"; + res += tl::to_string (m->second.second); + res += ";"; + } + + return res; + + } +} + +IncludeExpander +IncludeExpander::from_string (const std::string &s) +{ + IncludeExpander ie; + + tl::Extractor ex (s.c_str ()); + + if (ex.test ("@")) { + + while (! ex.at_end ()) { + + int ln = 0; + ex.read (ln); + + std::pair &si = ie.m_sections [ln]; + + ex.expect (":"); + ex.read_word_or_quoted (si.first, valid_fn_chars); + ex.expect ("*"); + ex.read (si.second); + + ex.test (";"); + + } + + } else { + + ex.read_word_or_quoted (ie.m_sections [1].first, valid_fn_chars); + ex.expect_end (); + + } + + return ie; +} + +std::pair +IncludeExpander::translate_to_original (int line_number) +{ + std::map >::const_iterator m = m_sections.lower_bound (line_number); + if (m != m_sections.begin () && (m == m_sections.end () || m->first > line_number)) { + --m; + } + if (m == m_sections.end ()) { + return std::make_pair (std::string (), 0); + } else { + return std::make_pair (m->second.first, line_number + m->second.second); + } +} + +} diff --git a/src/lym/lym/lymInclude.h b/src/lym/lym/lymInclude.h new file mode 100644 index 000000000..5e92e70bb --- /dev/null +++ b/src/lym/lym/lymInclude.h @@ -0,0 +1,99 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2020 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_lymInclude +#define HDR_lymInclude + +#include "lymCommon.h" + +#include +#include + +namespace lym +{ + +/** + * @brief Provide the basic include expansion and file/line mapping mechanism + * + * The Expander object performs the file expansion and also stores the information + * required for translating back file names and line numbers. + * + * Include expansion happens through a pseudo-comment "# %include ..." which + * takes a file path as the argument. File paths are always resolved relative to + * the original file. + * The file path is expression-interpolated, hence can access environment variables + * through $(env("HOME")) for example. + */ +class LYM_PUBLIC IncludeExpander +{ +public: + /** + * @brief The default constructor + */ + IncludeExpander (); + + /** + * @brief Provides include expansion + * + * This method will deliver the expanded text and the include expander object. + */ + static IncludeExpander expand (const std::string &path, std::string &expanded_text); + + /** + * @brief Serializes the include expander information into a string + * + * If no include expansion happened, the serialized string will be the original file path. + * Otherwise it's a "@"-prefixed string. + */ + std::string to_string () const; + + /** + * @brief Deserializes the include expander information from a string + */ + static IncludeExpander from_string (const std::string &s); + + /** + * @brief Provides translation of the expanded text's line number to filename/line number for the original file + */ + std::pair translate_to_original (int line_number); + + /** + * @brief Provides translation of original file name/line number to included file name/line number + */ + static std::pair translate_to_original (const std::string &file, int line_number) + { + IncludeExpander ie; + ie.from_string (file); + return ie.translate_to_original (line_number); + } + +public: + std::map > m_sections; + + static void read (const std::string &path, std::string &expanded_text, IncludeExpander &ie, int &line_counter); +}; + +} + +#endif + diff --git a/src/lym/unit_tests/lymBasicTests.cc b/src/lym/unit_tests/lymBasicTests.cc deleted file mode 100644 index 171a15e0d..000000000 --- a/src/lym/unit_tests/lymBasicTests.cc +++ /dev/null @@ -1,33 +0,0 @@ - -/* - - KLayout Layout Viewer - Copyright (C) 2006-2020 Matthias Koefferlein - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - - -#include "tlUnitTest.h" - -TEST(1) -{ - EXPECT_EQ (1, 1); // avoids a compiler warning because of unreferenced _this - - // TODO: add tests for lym specific things - throw tl::CancelException (); // skip this test to indicate that there is nothing yet -} - diff --git a/src/lym/unit_tests/lymIncludeTests.cc b/src/lym/unit_tests/lymIncludeTests.cc new file mode 100644 index 000000000..8ee13fdff --- /dev/null +++ b/src/lym/unit_tests/lymIncludeTests.cc @@ -0,0 +1,104 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2020 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlUnitTest.h" +#include "tlFileUtils.h" +#include "lymInclude.h" + +TEST(1_simple) +{ + std::string fn = tl::testsrc () + "/testdata/lym/x.txt"; + + std::string et; + lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et); + EXPECT_EQ (et, "A line\nAnother line\n"); + EXPECT_EQ (ie.to_string (), fn); + EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ()); + + EXPECT_EQ (ie.translate_to_original (2).first, fn); + EXPECT_EQ (ie.translate_to_original (2).second, 2); +} + +TEST(2_single_include) +{ + std::string fn = tl::testsrc () + "/testdata/lym/x_inc1.txt"; + + std::string et; + lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et); + EXPECT_EQ (et, "A line\nincluded.1\nAnother line\n"); + + EXPECT_EQ (ie.to_string (), "@1:" + tl::testsrc () + "/testdata/lym/x_inc1.txt*0;2:" + tl::testsrc () + "/testdata/lym/inc1.txt*-1;3:" + tl::testsrc () + "/testdata/lym/x_inc1.txt*0;"); + EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ()); + + EXPECT_EQ (ie.translate_to_original (1).first, fn); + EXPECT_EQ (ie.translate_to_original (1).second, 1); + EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc1.txt"); + EXPECT_EQ (ie.translate_to_original (2).second, 1); + EXPECT_EQ (ie.translate_to_original (3).first, fn); + EXPECT_EQ (ie.translate_to_original (3).second, 3); +} + +TEST(3_multi_include) +{ + std::string fn = tl::testsrc () + "/testdata/lym/x_inc3.txt"; + + std::string et; + lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et); + EXPECT_EQ (et, "A line\ninclude.3a\nincluded.2a\nincluded.2b\ninclude.3b\nAnother line\n"); + + EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ()); + + EXPECT_EQ (ie.translate_to_original (1).first, fn); + EXPECT_EQ (ie.translate_to_original (1).second, 1); + EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc3.txt"); + EXPECT_EQ (ie.translate_to_original (2).second, 1); + EXPECT_EQ (ie.translate_to_original (3).first, tl::testsrc () + "/testdata/lym/inc2.txt"); + EXPECT_EQ (ie.translate_to_original (3).second, 1); + EXPECT_EQ (ie.translate_to_original (5).first, tl::testsrc () + "/testdata/lym/inc3.txt"); + EXPECT_EQ (ie.translate_to_original (5).second, 3); + EXPECT_EQ (ie.translate_to_original (6).first, fn); + EXPECT_EQ (ie.translate_to_original (6).second, 3); +} + +TEST(4_multi_include_interpolate) +{ + std::string fn = tl::testsrc () + "/testdata/lym/x_inc3_ip.txt"; + + std::string et; + lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et); + EXPECT_EQ (et, "A line\ninclude.3a\nincluded.2a\nincluded.2b\ninclude.3b\nAnother line\n"); + + EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ()); + + EXPECT_EQ (ie.translate_to_original (1).first, fn); + EXPECT_EQ (ie.translate_to_original (1).second, 1); + EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc3.txt"); + EXPECT_EQ (ie.translate_to_original (2).second, 1); + EXPECT_EQ (ie.translate_to_original (3).first, tl::testsrc () + "/testdata/lym/inc2.txt"); + EXPECT_EQ (ie.translate_to_original (3).second, 1); + EXPECT_EQ (ie.translate_to_original (5).first, tl::testsrc () + "/testdata/lym/inc3.txt"); + EXPECT_EQ (ie.translate_to_original (5).second, 3); + EXPECT_EQ (ie.translate_to_original (6).first, fn); + EXPECT_EQ (ie.translate_to_original (6).second, 3); +} + diff --git a/src/lym/unit_tests/unit_tests.pro b/src/lym/unit_tests/unit_tests.pro index 9c62e03a9..af122a402 100644 --- a/src/lym/unit_tests/unit_tests.pro +++ b/src/lym/unit_tests/unit_tests.pro @@ -7,7 +7,7 @@ TARGET = lym_tests include($$PWD/../../lib_ut.pri) SOURCES = \ - lymBasicTests.cc \ + lymIncludeTests.cc INCLUDEPATH += $$LYM_INC $$TL_INC $$GSI_INC DEPENDPATH += $$LYM_INC $$TL_INC $$GSI_INC diff --git a/src/tl/tl/tlStream.cc b/src/tl/tl/tlStream.cc index 83271e5d7..3d77ff3d4 100644 --- a/src/tl/tl/tlStream.cc +++ b/src/tl/tl/tlStream.cc @@ -475,6 +475,7 @@ TextInputStream::get_char () return 0; } else if (*c != '\r' && *c) { if (*c == '\n') { + peek_char (); // sets at_end if there is no more character ++m_next_line; } return *c; diff --git a/testdata/lym/.inc1.txt.swp b/testdata/lym/.inc1.txt.swp new file mode 100644 index 000000000..7854dba15 Binary files /dev/null and b/testdata/lym/.inc1.txt.swp differ diff --git a/testdata/lym/.x_inc3_ip.txt.swp b/testdata/lym/.x_inc3_ip.txt.swp new file mode 100644 index 000000000..8e615652c Binary files /dev/null and b/testdata/lym/.x_inc3_ip.txt.swp differ diff --git a/testdata/lym/inc1.txt b/testdata/lym/inc1.txt new file mode 100644 index 000000000..c090d77ad --- /dev/null +++ b/testdata/lym/inc1.txt @@ -0,0 +1 @@ +included.1 diff --git a/testdata/lym/inc2.txt b/testdata/lym/inc2.txt new file mode 100644 index 000000000..68cb7c37c --- /dev/null +++ b/testdata/lym/inc2.txt @@ -0,0 +1,2 @@ +included.2a +included.2b diff --git a/testdata/lym/inc3.txt b/testdata/lym/inc3.txt new file mode 100644 index 000000000..7d9149572 --- /dev/null +++ b/testdata/lym/inc3.txt @@ -0,0 +1,3 @@ +include.3a +# %include inc2.txt +include.3b diff --git a/testdata/lym/x.txt b/testdata/lym/x.txt new file mode 100644 index 000000000..ae88c9ef5 --- /dev/null +++ b/testdata/lym/x.txt @@ -0,0 +1,2 @@ +A line +Another line diff --git a/testdata/lym/x_inc1.txt b/testdata/lym/x_inc1.txt new file mode 100644 index 000000000..130ff22f5 --- /dev/null +++ b/testdata/lym/x_inc1.txt @@ -0,0 +1,3 @@ +A line +# %include inc1.txt +Another line diff --git a/testdata/lym/x_inc3.txt b/testdata/lym/x_inc3.txt new file mode 100644 index 000000000..659b2243d --- /dev/null +++ b/testdata/lym/x_inc3.txt @@ -0,0 +1,3 @@ +A line +# %include inc3.txt +Another line diff --git a/testdata/lym/x_inc3_ip.txt b/testdata/lym/x_inc3_ip.txt new file mode 100644 index 000000000..375357daa --- /dev/null +++ b/testdata/lym/x_inc3_ip.txt @@ -0,0 +1,3 @@ +A line +# %include inc$(1+2).txt +Another line