Merge pull request #1429 from KLayout/issue-1428

Fixed #1428 (allow XML include files in macros)

src/lym/lym/lymMacro.cc

@@ -227,6 +227,7 @@ void Macro::save_to (const std::string &path)
 void Macro::load_from (const std::string &fn)
 {
   m_format = NoFormat;
+  m_interpreter = None;
 
   std::pair<bool, std::string> f = format_from_filename (fn, m_interpreter, m_dsl_interpreter, m_autorun_default, m_format);
   if (f.first) {
@@ -252,12 +253,23 @@ void Macro::load_from (const std::string &fn)
       tl::InputStream stream (path);
       tl::TextInputStream text_stream (stream);
       m_text = text_stream.read_all ();
-      sync_properties_with_text ();
+
+      if (m_format == PlainTextWithHashAnnotationsFormat) {
+        sync_properties_with_text ();
+      }
 
     }
 
   } else {
-    throw tl::Exception (tl::to_string (tr ("Unable to determine format for file from suffix or format spec ")) + fn);
+
+    if (tl::verbosity () >= 20) {
+      tl::log << "Loading macro from " << fn;
+    }
+
+    tl::InputStream stream (fn);
+    tl::TextInputStream text_stream (stream);
+    m_text = text_stream.read_all ();
+
   }
 
   m_modified = true;
@@ -268,6 +280,7 @@ void Macro::load_from (const std::string &fn)
 void Macro::load_from_string (const std::string &text, const std::string &url)
 {
   m_format = NoFormat;
+  m_interpreter = None;
 
   if (tl::verbosity () >= 20) {
     tl::log << "Loading macro from " << url;
@@ -294,7 +307,7 @@ void Macro::load_from_string (const std::string &text, const std::string &url)
     }
 
   } else {
-    throw tl::Exception (tl::to_string (tr ("Unable to determine format for file from suffix ")) + url);
+    m_text = text;
   }
 
   m_modified = true;

src/lym/lym/lymMacroInterpreter.cc

@@ -51,11 +51,35 @@ MacroInterpreter::can_run (const lym::Macro *macro)
   return false;
 }
 
+namespace
+{
+
+class MacroIncludeFileResolver
+  : public tl::IncludeFileResolver
+{
+public:
+  MacroIncludeFileResolver () { }
+
+  std::string get_text (const std::string &path) const
+  {
+    //  Use lym::Macro to resolve texts - this strips the XML envelope.
+    //  Intentionally not compatibility check is made to allow using any
+    //  type of input and specifically any extension.
+    lym::Macro macro;
+    macro.load_from (path);
+    return macro.text ();
+  }
+};
+
+}
+
 std::pair<std::string, std::string>
 MacroInterpreter::include_expansion (const lym::Macro *macro)
 {
+  MacroIncludeFileResolver include_file_resolver;
+
   std::pair<std::string, std::string> res;
-  res.first = tl::IncludeExpander::expand (macro->path (), macro->text (), res.second).to_string ();
+  res.first = tl::IncludeExpander::expand (macro->path (), macro->text (), res.second, &include_file_resolver).to_string ();
 
   if (res.first != macro->path ()) {
 

src/lym/unit_tests/lymBasicTests.cc

@@ -129,6 +129,29 @@ TEST(3_RubyInclude)
   EXPECT_EQ (np (console.text ()), np ("An error in " + tl::testsrc () + "/testdata/lym/b_inc.rb:3\n"));
 }
 
+TEST(4_RubyIncludeFromXML)
+{
+  tl_assert (rba::RubyInterpreter::instance () != 0);
+
+  lym::Macro macro;
+
+  macro.set_file_path (tl::testsrc () + "/testdata/lym/m4.rb");
+  macro.set_interpreter (lym::Macro::Ruby);
+  macro.load ();
+
+  TestCollectorConsole console;
+  rba::RubyInterpreter::instance ()->push_console (&console);
+  try {
+    EXPECT_EQ (macro.run (), 0);
+    rba::RubyInterpreter::instance ()->remove_console (&console);
+  } catch (...) {
+    rba::RubyInterpreter::instance ()->remove_console (&console);
+    throw;
+  }
+
+  EXPECT_EQ (np (console.text ()), np ("An error in " + tl::testsrc () + "/testdata/lym/b_inc.lym:3\n"));
+}
+
 TEST(11_DRCBasic)
 {
   tl_assert (rba::RubyInterpreter::instance () != 0);

src/tl/tl/tlInclude.cc

@@ -32,6 +32,9 @@
 namespace tl
 {
 
+// -----------------------------------------------------------------------------------------------------
+//  IncludeExpander implementation
+
 static const char *valid_fn_chars = "@_:,.\\/-+";
 
 IncludeExpander::IncludeExpander ()
@@ -40,30 +43,30 @@ IncludeExpander::IncludeExpander ()
 }
 
 IncludeExpander
-IncludeExpander::expand (const std::string &path, std::string &expanded_text)
+IncludeExpander::expand (const std::string &path, std::string &expanded_text, const IncludeFileResolver *resolver)
 {
   IncludeExpander ie;
   int lc = 1;
   tl::InputStream is (path);
-  ie.read (path, is, expanded_text, ie, lc);
+  ie.read (path, is, expanded_text, lc, resolver);
   return ie;
 }
 
 IncludeExpander
-IncludeExpander::expand (const std::string &path, const std::string &original_text, std::string &expanded_text)
+IncludeExpander::expand (const std::string &path, const std::string &original_text, std::string &expanded_text, const IncludeFileResolver *resolver)
 {
   IncludeExpander ie;
   int lc = 1;
   tl::InputMemoryStream ms (original_text.c_str (), original_text.size ());
   tl::InputStream is (ms);
-  ie.read (path, is, expanded_text, ie, lc);
+  ie.read (path, is, expanded_text, lc, resolver);
   return ie;
 }
 
 void
-IncludeExpander::read (const std::string &path, tl::InputStream &is, std::string &expanded_text, IncludeExpander &ie, int &line_counter)
+IncludeExpander::read (const std::string &path, tl::InputStream &is, std::string &expanded_text, int &line_counter, const IncludeFileResolver *resolver)
 {
-  ie.m_sections [line_counter] = std::make_pair (path, 1 - line_counter);
+  m_sections [line_counter] = std::make_pair (path, 1 - line_counter);
 
   tl::TextInputStream text (is);
 
@@ -100,8 +103,17 @@ IncludeExpander::read (const std::string &path, tl::InputStream &is, std::string
         include_path = current_uri.resolved (new_uri).to_abstract_path ();
       }
 
-      tl::InputStream is (include_path);
-      read (include_path, is, expanded_text, ie, line_counter);
+      std::string include_text;
+      if (resolver) {
+        include_text = resolver->get_text (include_path);
+      } else {
+        tl::InputStream iis (include_path);
+        include_text = iis.read_all ();
+      }
+
+      tl::InputMemoryStream ms (include_text.c_str (), include_text.size ());
+      tl::InputStream is (ms);
+      read (include_path, is, expanded_text, line_counter, resolver);
 
       emit_section = true;
 
@@ -109,7 +121,7 @@ IncludeExpander::read (const std::string &path, tl::InputStream &is, std::string
 
       if (emit_section) {
         emit_section = false;
-        ie.m_sections [line_counter] = std::make_pair (path, lnum - line_counter);
+        m_sections [line_counter] = std::make_pair (path, lnum - line_counter);
       }
 
       expanded_text += l;

src/tl/tl/tlInclude.h

@@ -34,6 +34,21 @@ namespace tl
 
 class InputStream;
 
+/**
+ *  @brief An interface providing the include file resolver
+ *
+ *  The task of this object is to obtain the text for an include file path.
+ *  The path already underwent variable interpolation and relative path resolution.
+ */
+class TL_PUBLIC IncludeFileResolver
+{
+public:
+  IncludeFileResolver () { }
+  virtual ~IncludeFileResolver () { }
+
+  virtual std::string get_text (const std::string &path) const = 0;
+};
+
 /**
  *  @brief Provide the basic include expansion and file/line mapping mechanism
  *
@@ -59,7 +74,7 @@ public:
    *
    *  This method will deliver the expanded text and the include expander object.
    */
-  static IncludeExpander expand (const std::string &path, std::string &expanded_text);
+  static IncludeExpander expand (const std::string &path, std::string &expanded_text, const IncludeFileResolver *resolver = 0);
 
   /**
    *  @brief Provides include expansion
@@ -67,7 +82,7 @@ public:
    *  This method will deliver the expanded text and the include expander object.
    *  This version also takes the actual text of the original file.
    */
-  static IncludeExpander expand (const std::string &path, const std::string &original_text, std::string &expanded_text);
+  static IncludeExpander expand (const std::string &path, const std::string &original_text, std::string &expanded_text, const IncludeFileResolver *resolver = 0);
 
   /**
    *  @brief Serializes the include expander information into a string
@@ -98,7 +113,7 @@ public:
 private:
   std::map<int, std::pair<std::string, int> > m_sections;
 
-  void read (const std::string &path, tl::InputStream &is, std::string &expanded_text, IncludeExpander &ie, int &line_counter);
+  void read (const std::string &path, tl::InputStream &is, std::string &expanded_text, int &line_counter, const IncludeFileResolver *mp_resolver);
 };
 
 }

testdata/lym/b_inc.lym

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<klayout-macro>
+ <description/>
+ <version/>
+ <category/>
+ <prolog/>
+ <epilog/>
+ <doc/>
+ <autorun>false</autorun>
+ <autorun-early>false</autorun-early>
+ <priority>0</priority>
+ <shortcut/>
+ <show-in-menu>false</show-in-menu>
+ <group-name/>
+ <menu-path/>
+ <interpreter>ruby</interpreter>
+ <dsl-interpreter-name/>
+ <text>
+def f
+  raise("An error")
+end
+</text>
+</klayout-macro>

testdata/lym/m4.rb

@@ -0,0 +1,12 @@
+
+# %include b_inc.lym
+
+begin
+  puts f
+rescue => ex
+  ln = ex.backtrace[0].split(":")
+  # NOTE: as the backtrace is a native Ruby feature, include file translation
+  # does not happen. We need to do this explicitly here:
+  puts ex.to_s + " in " + RBA::Macro::real_path(ln[0], ln[1].to_i) + ":" + RBA::Macro::real_line(ln[0], ln[1].to_i).to_s
+end
+