From 77a92532734cad05360cbf76052b52a6a2e2d5f0 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 30 Aug 2020 17:08:39 +0200 Subject: [PATCH] WIP: Provide a generic include file expansion mechanism, basic class. --- src/lym/lym/lym.pro | 2 + src/lym/lym/lymInclude.cpp | 189 ++++++++++++++++++++++++++ src/lym/lym/lymInclude.h | 99 ++++++++++++++ src/lym/unit_tests/lymBasicTests.cc | 33 ----- src/lym/unit_tests/lymIncludeTests.cc | 104 ++++++++++++++ src/lym/unit_tests/unit_tests.pro | 2 +- src/tl/tl/tlStream.cc | 1 + testdata/lym/.inc1.txt.swp | Bin 0 -> 12288 bytes testdata/lym/.x_inc3_ip.txt.swp | Bin 0 -> 12288 bytes testdata/lym/inc1.txt | 1 + testdata/lym/inc2.txt | 2 + testdata/lym/inc3.txt | 3 + testdata/lym/x.txt | 2 + testdata/lym/x_inc1.txt | 3 + testdata/lym/x_inc3.txt | 3 + testdata/lym/x_inc3_ip.txt | 3 + 16 files changed, 413 insertions(+), 34 deletions(-) create mode 100644 src/lym/lym/lymInclude.cpp create mode 100644 src/lym/lym/lymInclude.h delete mode 100644 src/lym/unit_tests/lymBasicTests.cc create mode 100644 src/lym/unit_tests/lymIncludeTests.cc create mode 100644 testdata/lym/.inc1.txt.swp create mode 100644 testdata/lym/.x_inc3_ip.txt.swp create mode 100644 testdata/lym/inc1.txt create mode 100644 testdata/lym/inc2.txt create mode 100644 testdata/lym/inc3.txt create mode 100644 testdata/lym/x.txt create mode 100644 testdata/lym/x_inc1.txt create mode 100644 testdata/lym/x_inc3.txt create mode 100644 testdata/lym/x_inc3_ip.txt diff --git a/src/lym/lym/lym.pro b/src/lym/lym/lym.pro index 24a21587b..ea352c597 100644 --- a/src/lym/lym/lym.pro +++ b/src/lym/lym/lym.pro @@ -8,11 +8,13 @@ DEFINES += MAKE_LYM_LIBRARY SOURCES = \ gsiDeclLymMacro.cc \ + lymInclude.cpp \ lymMacroInterpreter.cc \ lymMacro.cc \ HEADERS = \ lymCommon.h \ + lymInclude.h \ lymMacroInterpreter.h \ lymMacro.h \ diff --git a/src/lym/lym/lymInclude.cpp b/src/lym/lym/lymInclude.cpp new file mode 100644 index 000000000..b48de2c56 --- /dev/null +++ b/src/lym/lym/lymInclude.cpp @@ -0,0 +1,189 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2020 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "lymInclude.h" + +#include "tlAssert.h" +#include "tlString.h" +#include "tlFileUtils.h" +#include "tlStream.h" +#include "tlExpression.h" +#include "tlUri.h" + +namespace lym +{ + +static const char *valid_fn_chars = "@_:,.\\/-+"; + +IncludeExpander::IncludeExpander () +{ + // .. nothing yet .. +} + +IncludeExpander +IncludeExpander::expand (const std::string &path, std::string &expanded_text) +{ + IncludeExpander ie; + int lc = 1; + read (path, expanded_text, ie, lc); + return ie; +} + +void +IncludeExpander::read (const std::string &path, std::string &expanded_text, IncludeExpander &ie, int &line_counter) +{ + ie.m_sections [line_counter] = std::make_pair (path, 1 - line_counter); + + tl::InputStream is (path); + tl::TextInputStream text (is); + + int lnum = 0; + bool emit_section = false; + + while (! text.at_end ()) { + + std::string l = text.get_line (); + ++lnum; + + tl::Extractor ex (l.c_str ()); + if (ex.test ("#") && ex.test ("%include")) { + + std::string include_path = tl::trim (ex.skip ()); + + // allow some customization by employing expression interpolation + include_path = tl::Eval ().interpolate (include_path); + + // NOTE: by using URI's we can basically read from HTTP etc. + tl::URI current_uri (path); + tl::URI new_uri (include_path); + if (current_uri.scheme ().empty () && new_uri.scheme ().empty ()) { + if (! tl::is_absolute (include_path)) { + include_path = tl::combine_path (tl::dirname (path), include_path); + } + } else { + include_path = current_uri.resolved (new_uri).to_string (); + } + + read (include_path, expanded_text, ie, line_counter); + + emit_section = true; + + } else { + + if (emit_section) { + emit_section = false; + ie.m_sections [line_counter] = std::make_pair (path, lnum - line_counter); + } + + expanded_text += l; + expanded_text += "\n"; + ++line_counter; + + } + + } +} + +std::string +IncludeExpander::to_string () const +{ + if (m_sections.empty ()) { + + return std::string (); + + } else if (m_sections.size () == 1) { + + tl_assert (m_sections.begin ()->first == 1); + tl_assert (m_sections.begin ()->second.second == 0); + + std::string fn = m_sections.begin ()->second.first; + return tl::to_word_or_quoted_string (fn, valid_fn_chars); + + } else { + + // "@" indicates a mapping table + std::string res ("@"); + + for (std::map >::const_iterator m = m_sections.begin (); m != m_sections.end (); ++m) { + res += tl::to_string (m->first); + res += ":"; + res += tl::to_word_or_quoted_string (m->second.first, valid_fn_chars); + res += "*"; + res += tl::to_string (m->second.second); + res += ";"; + } + + return res; + + } +} + +IncludeExpander +IncludeExpander::from_string (const std::string &s) +{ + IncludeExpander ie; + + tl::Extractor ex (s.c_str ()); + + if (ex.test ("@")) { + + while (! ex.at_end ()) { + + int ln = 0; + ex.read (ln); + + std::pair &si = ie.m_sections [ln]; + + ex.expect (":"); + ex.read_word_or_quoted (si.first, valid_fn_chars); + ex.expect ("*"); + ex.read (si.second); + + ex.test (";"); + + } + + } else { + + ex.read_word_or_quoted (ie.m_sections [1].first, valid_fn_chars); + ex.expect_end (); + + } + + return ie; +} + +std::pair +IncludeExpander::translate_to_original (int line_number) +{ + std::map >::const_iterator m = m_sections.lower_bound (line_number); + if (m != m_sections.begin () && (m == m_sections.end () || m->first > line_number)) { + --m; + } + if (m == m_sections.end ()) { + return std::make_pair (std::string (), 0); + } else { + return std::make_pair (m->second.first, line_number + m->second.second); + } +} + +} diff --git a/src/lym/lym/lymInclude.h b/src/lym/lym/lymInclude.h new file mode 100644 index 000000000..5e92e70bb --- /dev/null +++ b/src/lym/lym/lymInclude.h @@ -0,0 +1,99 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2020 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_lymInclude +#define HDR_lymInclude + +#include "lymCommon.h" + +#include +#include + +namespace lym +{ + +/** + * @brief Provide the basic include expansion and file/line mapping mechanism + * + * The Expander object performs the file expansion and also stores the information + * required for translating back file names and line numbers. + * + * Include expansion happens through a pseudo-comment "# %include ..." which + * takes a file path as the argument. File paths are always resolved relative to + * the original file. + * The file path is expression-interpolated, hence can access environment variables + * through $(env("HOME")) for example. + */ +class LYM_PUBLIC IncludeExpander +{ +public: + /** + * @brief The default constructor + */ + IncludeExpander (); + + /** + * @brief Provides include expansion + * + * This method will deliver the expanded text and the include expander object. + */ + static IncludeExpander expand (const std::string &path, std::string &expanded_text); + + /** + * @brief Serializes the include expander information into a string + * + * If no include expansion happened, the serialized string will be the original file path. + * Otherwise it's a "@"-prefixed string. + */ + std::string to_string () const; + + /** + * @brief Deserializes the include expander information from a string + */ + static IncludeExpander from_string (const std::string &s); + + /** + * @brief Provides translation of the expanded text's line number to filename/line number for the original file + */ + std::pair translate_to_original (int line_number); + + /** + * @brief Provides translation of original file name/line number to included file name/line number + */ + static std::pair translate_to_original (const std::string &file, int line_number) + { + IncludeExpander ie; + ie.from_string (file); + return ie.translate_to_original (line_number); + } + +public: + std::map > m_sections; + + static void read (const std::string &path, std::string &expanded_text, IncludeExpander &ie, int &line_counter); +}; + +} + +#endif + diff --git a/src/lym/unit_tests/lymBasicTests.cc b/src/lym/unit_tests/lymBasicTests.cc deleted file mode 100644 index 171a15e0d..000000000 --- a/src/lym/unit_tests/lymBasicTests.cc +++ /dev/null @@ -1,33 +0,0 @@ - -/* - - KLayout Layout Viewer - Copyright (C) 2006-2020 Matthias Koefferlein - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - - -#include "tlUnitTest.h" - -TEST(1) -{ - EXPECT_EQ (1, 1); // avoids a compiler warning because of unreferenced _this - - // TODO: add tests for lym specific things - throw tl::CancelException (); // skip this test to indicate that there is nothing yet -} - diff --git a/src/lym/unit_tests/lymIncludeTests.cc b/src/lym/unit_tests/lymIncludeTests.cc new file mode 100644 index 000000000..8ee13fdff --- /dev/null +++ b/src/lym/unit_tests/lymIncludeTests.cc @@ -0,0 +1,104 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2020 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlUnitTest.h" +#include "tlFileUtils.h" +#include "lymInclude.h" + +TEST(1_simple) +{ + std::string fn = tl::testsrc () + "/testdata/lym/x.txt"; + + std::string et; + lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et); + EXPECT_EQ (et, "A line\nAnother line\n"); + EXPECT_EQ (ie.to_string (), fn); + EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ()); + + EXPECT_EQ (ie.translate_to_original (2).first, fn); + EXPECT_EQ (ie.translate_to_original (2).second, 2); +} + +TEST(2_single_include) +{ + std::string fn = tl::testsrc () + "/testdata/lym/x_inc1.txt"; + + std::string et; + lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et); + EXPECT_EQ (et, "A line\nincluded.1\nAnother line\n"); + + EXPECT_EQ (ie.to_string (), "@1:" + tl::testsrc () + "/testdata/lym/x_inc1.txt*0;2:" + tl::testsrc () + "/testdata/lym/inc1.txt*-1;3:" + tl::testsrc () + "/testdata/lym/x_inc1.txt*0;"); + EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ()); + + EXPECT_EQ (ie.translate_to_original (1).first, fn); + EXPECT_EQ (ie.translate_to_original (1).second, 1); + EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc1.txt"); + EXPECT_EQ (ie.translate_to_original (2).second, 1); + EXPECT_EQ (ie.translate_to_original (3).first, fn); + EXPECT_EQ (ie.translate_to_original (3).second, 3); +} + +TEST(3_multi_include) +{ + std::string fn = tl::testsrc () + "/testdata/lym/x_inc3.txt"; + + std::string et; + lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et); + EXPECT_EQ (et, "A line\ninclude.3a\nincluded.2a\nincluded.2b\ninclude.3b\nAnother line\n"); + + EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ()); + + EXPECT_EQ (ie.translate_to_original (1).first, fn); + EXPECT_EQ (ie.translate_to_original (1).second, 1); + EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc3.txt"); + EXPECT_EQ (ie.translate_to_original (2).second, 1); + EXPECT_EQ (ie.translate_to_original (3).first, tl::testsrc () + "/testdata/lym/inc2.txt"); + EXPECT_EQ (ie.translate_to_original (3).second, 1); + EXPECT_EQ (ie.translate_to_original (5).first, tl::testsrc () + "/testdata/lym/inc3.txt"); + EXPECT_EQ (ie.translate_to_original (5).second, 3); + EXPECT_EQ (ie.translate_to_original (6).first, fn); + EXPECT_EQ (ie.translate_to_original (6).second, 3); +} + +TEST(4_multi_include_interpolate) +{ + std::string fn = tl::testsrc () + "/testdata/lym/x_inc3_ip.txt"; + + std::string et; + lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et); + EXPECT_EQ (et, "A line\ninclude.3a\nincluded.2a\nincluded.2b\ninclude.3b\nAnother line\n"); + + EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ()); + + EXPECT_EQ (ie.translate_to_original (1).first, fn); + EXPECT_EQ (ie.translate_to_original (1).second, 1); + EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc3.txt"); + EXPECT_EQ (ie.translate_to_original (2).second, 1); + EXPECT_EQ (ie.translate_to_original (3).first, tl::testsrc () + "/testdata/lym/inc2.txt"); + EXPECT_EQ (ie.translate_to_original (3).second, 1); + EXPECT_EQ (ie.translate_to_original (5).first, tl::testsrc () + "/testdata/lym/inc3.txt"); + EXPECT_EQ (ie.translate_to_original (5).second, 3); + EXPECT_EQ (ie.translate_to_original (6).first, fn); + EXPECT_EQ (ie.translate_to_original (6).second, 3); +} + diff --git a/src/lym/unit_tests/unit_tests.pro b/src/lym/unit_tests/unit_tests.pro index 9c62e03a9..af122a402 100644 --- a/src/lym/unit_tests/unit_tests.pro +++ b/src/lym/unit_tests/unit_tests.pro @@ -7,7 +7,7 @@ TARGET = lym_tests include($$PWD/../../lib_ut.pri) SOURCES = \ - lymBasicTests.cc \ + lymIncludeTests.cc INCLUDEPATH += $$LYM_INC $$TL_INC $$GSI_INC DEPENDPATH += $$LYM_INC $$TL_INC $$GSI_INC diff --git a/src/tl/tl/tlStream.cc b/src/tl/tl/tlStream.cc index 83271e5d7..3d77ff3d4 100644 --- a/src/tl/tl/tlStream.cc +++ b/src/tl/tl/tlStream.cc @@ -475,6 +475,7 @@ TextInputStream::get_char () return 0; } else if (*c != '\r' && *c) { if (*c == '\n') { + peek_char (); // sets at_end if there is no more character ++m_next_line; } return *c; diff --git a/testdata/lym/.inc1.txt.swp b/testdata/lym/.inc1.txt.swp new file mode 100644 index 0000000000000000000000000000000000000000..7854dba15fdca94399bfff8facd477628bfdef60 GIT binary patch literal 12288 zcmeI&F>3-b6u|L!-Nn+b4r-?k8c!Yk3L+hKDwNo}qIjo=_YQQ?59a6S=Wx-z3DmLb zdi4JwgybcG_uE1;?M#Mm^+7un7mp&ZhtWr`_8#<)BB@oK&28>e*J75n5}WBNySB*X zc9#RqBC*?5p(eGt#%rT-u8~!1l5J|{%g?Z_8@+KDX#^0sU!c&h)}SK2ZrJggUYd>i z^V9hL{j5X)0R#|0009ILKmdW#3uqR|-c|ANTS@RoU;08R009ILKmY**5I_I{1Q0*~ zfqyJuBa!c_$k{dj{(rgu`|-)o5eOiF00IagfB*srAb>Li5IL=wy|+81aS{dWZa3I7H0AMihL zcJrd?;!r!Pd>@3z<-I$&&vx8hbuUiG?y%#FjV+Ofr?X(Nd+7|`L^7iqCuUlgmPK5) zQ&VbDE^D&2yTyXuZEEI4rCw&H8cjTnrWzVGUOLabS&-z{{U9kjI@2Fkq>KOpD+yG3 z(;jX}zvp-B{?`7UyR&`1(o~udKmY**5I_I{1Q0;re+X#l$i3alTC<(?COhA9GbAB^ z00IagfB*srAb