WIP: Provide a generic include file expansion mechanism, basic class.

src/lym/lym/lym.pro

@@ -8,11 +8,13 @@ DEFINES += MAKE_LYM_LIBRARY
 
 SOURCES = \
   gsiDeclLymMacro.cc \
+  lymInclude.cpp \
   lymMacroInterpreter.cc \
   lymMacro.cc \
 
 HEADERS = \
   lymCommon.h \
+  lymInclude.h \
   lymMacroInterpreter.h \
   lymMacro.h \
 

src/lym/lym/lymInclude.cpp

@@ -0,0 +1,189 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2020 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#include "lymInclude.h"
+
+#include "tlAssert.h"
+#include "tlString.h"
+#include "tlFileUtils.h"
+#include "tlStream.h"
+#include "tlExpression.h"
+#include "tlUri.h"
+
+namespace lym
+{
+
+static const char *valid_fn_chars = "@_:,.\\/-+";
+
+IncludeExpander::IncludeExpander ()
+{
+  //  .. nothing yet ..
+}
+
+IncludeExpander
+IncludeExpander::expand (const std::string &path, std::string &expanded_text)
+{
+  IncludeExpander ie;
+  int lc = 1;
+  read (path, expanded_text, ie, lc);
+  return ie;
+}
+
+void
+IncludeExpander::read (const std::string &path, std::string &expanded_text, IncludeExpander &ie, int &line_counter)
+{
+  ie.m_sections [line_counter] = std::make_pair (path, 1 - line_counter);
+
+  tl::InputStream is (path);
+  tl::TextInputStream text (is);
+
+  int lnum = 0;
+  bool emit_section = false;
+
+  while (! text.at_end ()) {
+
+    std::string l = text.get_line ();
+    ++lnum;
+
+    tl::Extractor ex (l.c_str ());
+    if (ex.test ("#") && ex.test ("%include")) {
+
+      std::string include_path = tl::trim (ex.skip ());
+
+      //  allow some customization by employing expression interpolation
+      include_path = tl::Eval ().interpolate (include_path);
+
+      //  NOTE: by using URI's we can basically read from HTTP etc.
+      tl::URI current_uri (path);
+      tl::URI new_uri (include_path);
+      if (current_uri.scheme ().empty () && new_uri.scheme ().empty ()) {
+        if (! tl::is_absolute (include_path)) {
+          include_path = tl::combine_path (tl::dirname (path), include_path);
+        }
+      } else {
+        include_path = current_uri.resolved (new_uri).to_string ();
+      }
+
+      read (include_path, expanded_text, ie, line_counter);
+
+      emit_section = true;
+
+    } else {
+
+      if (emit_section) {
+        emit_section = false;
+        ie.m_sections [line_counter] = std::make_pair (path, lnum - line_counter);
+      }
+
+      expanded_text += l;
+      expanded_text += "\n";
+      ++line_counter;
+
+    }
+
+  }
+}
+
+std::string
+IncludeExpander::to_string () const
+{
+  if (m_sections.empty ()) {
+
+    return std::string ();
+
+  } else if (m_sections.size () == 1) {
+
+    tl_assert (m_sections.begin ()->first == 1);
+    tl_assert (m_sections.begin ()->second.second == 0);
+
+    std::string fn = m_sections.begin ()->second.first;
+    return tl::to_word_or_quoted_string (fn, valid_fn_chars);
+
+  } else {
+
+    //  "@" indicates a mapping table
+    std::string res ("@");
+
+    for (std::map<int, std::pair<std::string, int> >::const_iterator m = m_sections.begin (); m != m_sections.end (); ++m) {
+      res += tl::to_string (m->first);
+      res += ":";
+      res += tl::to_word_or_quoted_string (m->second.first, valid_fn_chars);
+      res += "*";
+      res += tl::to_string (m->second.second);
+      res += ";";
+    }
+
+    return res;
+
+  }
+}
+
+IncludeExpander
+IncludeExpander::from_string (const std::string &s)
+{
+  IncludeExpander ie;
+
+  tl::Extractor ex (s.c_str ());
+
+  if (ex.test ("@")) {
+
+    while (! ex.at_end ()) {
+
+      int ln = 0;
+      ex.read (ln);
+
+      std::pair<std::string, int> &si = ie.m_sections [ln];
+
+      ex.expect (":");
+      ex.read_word_or_quoted (si.first, valid_fn_chars);
+      ex.expect ("*");
+      ex.read (si.second);
+
+      ex.test (";");
+
+    }
+
+  } else {
+
+    ex.read_word_or_quoted (ie.m_sections [1].first, valid_fn_chars);
+    ex.expect_end ();
+
+  }
+
+  return ie;
+}
+
+std::pair<std::string, int>
+IncludeExpander::translate_to_original (int line_number)
+{
+  std::map<int, std::pair<std::string, int> >::const_iterator m = m_sections.lower_bound (line_number);
+  if (m != m_sections.begin () && (m == m_sections.end () || m->first > line_number)) {
+    --m;
+  }
+  if (m == m_sections.end ()) {
+    return std::make_pair (std::string (), 0);
+  } else {
+    return std::make_pair (m->second.first, line_number + m->second.second);
+  }
+}
+
+}

src/lym/lym/lymInclude.h

@@ -0,0 +1,99 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2020 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+
+#ifndef HDR_lymInclude
+#define HDR_lymInclude
+
+#include "lymCommon.h"
+
+#include <string>
+#include <map>
+
+namespace lym
+{
+
+/**
+ *  @brief Provide the basic include expansion and file/line mapping mechanism
+ *
+ *  The Expander object performs the file expansion and also stores the information
+ *  required for translating back file names and line numbers.
+ *
+ *  Include expansion happens through a pseudo-comment "# %include ..." which
+ *  takes a file path as the argument. File paths are always resolved relative to
+ *  the original file.
+ *  The file path is expression-interpolated, hence can access environment variables
+ *  through $(env("HOME")) for example.
+ */
+class LYM_PUBLIC IncludeExpander
+{
+public:
+  /**
+   *  @brief The default constructor
+   */
+  IncludeExpander ();
+
+  /**
+   *  @brief Provides include expansion
+   *
+   *  This method will deliver the expanded text and the include expander object.
+   */
+  static IncludeExpander expand (const std::string &path, std::string &expanded_text);
+
+  /**
+   *  @brief Serializes the include expander information into a string
+   *
+   *  If no include expansion happened, the serialized string will be the original file path.
+   *  Otherwise it's a "@"-prefixed string.
+   */
+  std::string to_string () const;
+
+  /**
+   *  @brief Deserializes the include expander information from a string
+   */
+  static IncludeExpander from_string (const std::string &s);
+
+  /**
+   *  @brief Provides translation of the expanded text's line number to filename/line number for the original file
+   */
+  std::pair<std::string, int> translate_to_original (int line_number);
+
+  /**
+   *  @brief Provides translation of original file name/line number to included file name/line number
+   */
+  static std::pair<std::string, int> translate_to_original (const std::string &file, int line_number)
+  {
+    IncludeExpander ie;
+    ie.from_string (file);
+    return ie.translate_to_original (line_number);
+  }
+
+public:
+  std::map<int, std::pair<std::string, int> > m_sections;
+
+  static void read (const std::string &path, std::string &expanded_text, IncludeExpander &ie, int &line_counter);
+};
+
+}
+
+#endif
+

src/lym/unit_tests/lymBasicTests.cc

@@ -1,33 +0,0 @@
-
-/*
-
-  KLayout Layout Viewer
-  Copyright (C) 2006-2020 Matthias Koefferlein
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program; if not, write to the Free Software
-  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-*/
-
-
-#include "tlUnitTest.h"
-
-TEST(1) 
-{
-  EXPECT_EQ (1, 1);  //  avoids a compiler warning because of unreferenced _this
-
-  //  TODO: add tests for lym specific things
-  throw tl::CancelException ();  //  skip this test to indicate that there is nothing yet
-}
-

src/lym/unit_tests/lymIncludeTests.cc

@@ -0,0 +1,104 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2020 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+
+#include "tlUnitTest.h"
+#include "tlFileUtils.h"
+#include "lymInclude.h"
+
+TEST(1_simple)
+{
+  std::string fn = tl::testsrc () + "/testdata/lym/x.txt";
+
+  std::string et;
+  lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et);
+  EXPECT_EQ (et, "A line\nAnother line\n");
+  EXPECT_EQ (ie.to_string (), fn);
+  EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ());
+
+  EXPECT_EQ (ie.translate_to_original (2).first, fn);
+  EXPECT_EQ (ie.translate_to_original (2).second, 2);
+}
+
+TEST(2_single_include)
+{
+  std::string fn = tl::testsrc () + "/testdata/lym/x_inc1.txt";
+
+  std::string et;
+  lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et);
+  EXPECT_EQ (et, "A line\nincluded.1\nAnother line\n");
+
+  EXPECT_EQ (ie.to_string (), "@1:" + tl::testsrc () + "/testdata/lym/x_inc1.txt*0;2:" + tl::testsrc () + "/testdata/lym/inc1.txt*-1;3:" + tl::testsrc () + "/testdata/lym/x_inc1.txt*0;");
+  EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ());
+
+  EXPECT_EQ (ie.translate_to_original (1).first, fn);
+  EXPECT_EQ (ie.translate_to_original (1).second, 1);
+  EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc1.txt");
+  EXPECT_EQ (ie.translate_to_original (2).second, 1);
+  EXPECT_EQ (ie.translate_to_original (3).first, fn);
+  EXPECT_EQ (ie.translate_to_original (3).second, 3);
+}
+
+TEST(3_multi_include)
+{
+  std::string fn = tl::testsrc () + "/testdata/lym/x_inc3.txt";
+
+  std::string et;
+  lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et);
+  EXPECT_EQ (et, "A line\ninclude.3a\nincluded.2a\nincluded.2b\ninclude.3b\nAnother line\n");
+
+  EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ());
+
+  EXPECT_EQ (ie.translate_to_original (1).first, fn);
+  EXPECT_EQ (ie.translate_to_original (1).second, 1);
+  EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc3.txt");
+  EXPECT_EQ (ie.translate_to_original (2).second, 1);
+  EXPECT_EQ (ie.translate_to_original (3).first, tl::testsrc () + "/testdata/lym/inc2.txt");
+  EXPECT_EQ (ie.translate_to_original (3).second, 1);
+  EXPECT_EQ (ie.translate_to_original (5).first, tl::testsrc () + "/testdata/lym/inc3.txt");
+  EXPECT_EQ (ie.translate_to_original (5).second, 3);
+  EXPECT_EQ (ie.translate_to_original (6).first, fn);
+  EXPECT_EQ (ie.translate_to_original (6).second, 3);
+}
+
+TEST(4_multi_include_interpolate)
+{
+  std::string fn = tl::testsrc () + "/testdata/lym/x_inc3_ip.txt";
+
+  std::string et;
+  lym::IncludeExpander ie = lym::IncludeExpander::expand (fn, et);
+  EXPECT_EQ (et, "A line\ninclude.3a\nincluded.2a\nincluded.2b\ninclude.3b\nAnother line\n");
+
+  EXPECT_EQ (lym::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ());
+
+  EXPECT_EQ (ie.translate_to_original (1).first, fn);
+  EXPECT_EQ (ie.translate_to_original (1).second, 1);
+  EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/lym/inc3.txt");
+  EXPECT_EQ (ie.translate_to_original (2).second, 1);
+  EXPECT_EQ (ie.translate_to_original (3).first, tl::testsrc () + "/testdata/lym/inc2.txt");
+  EXPECT_EQ (ie.translate_to_original (3).second, 1);
+  EXPECT_EQ (ie.translate_to_original (5).first, tl::testsrc () + "/testdata/lym/inc3.txt");
+  EXPECT_EQ (ie.translate_to_original (5).second, 3);
+  EXPECT_EQ (ie.translate_to_original (6).first, fn);
+  EXPECT_EQ (ie.translate_to_original (6).second, 3);
+}
+

src/lym/unit_tests/unit_tests.pro

@@ -7,7 +7,7 @@ TARGET = lym_tests
 include($$PWD/../../lib_ut.pri)
 
 SOURCES = \
-  lymBasicTests.cc \
+  lymIncludeTests.cc
 
 INCLUDEPATH += $$LYM_INC $$TL_INC $$GSI_INC
 DEPENDPATH += $$LYM_INC $$TL_INC $$GSI_INC

src/tl/tl/tlStream.cc

@@ -475,6 +475,7 @@ TextInputStream::get_char ()
       return 0;
     } else if (*c != '\r' && *c) {
       if (*c == '\n') {
+        peek_char ();  // sets at_end if there is no more character
         ++m_next_line;
       }
       return *c;

testdata/lym/inc1.txt

@@ -0,0 +1 @@
+included.1

testdata/lym/inc2.txt

@@ -0,0 +1,2 @@
+included.2a
+included.2b

testdata/lym/inc3.txt

@@ -0,0 +1,3 @@
+include.3a
+# %include inc2.txt
+include.3b

testdata/lym/x.txt

@@ -0,0 +1,2 @@
+A line
+Another line

testdata/lym/x_inc1.txt

@@ -0,0 +1,3 @@
+A line
+# %include inc1.txt
+Another line

testdata/lym/x_inc3.txt

@@ -0,0 +1,3 @@
+A line
+# %include inc3.txt
+Another line

testdata/lym/x_inc3_ip.txt

@@ -0,0 +1,3 @@
+A line
+# %include inc$(1+2).txt
+Another line