luke
/
klayout
mirror of https://github.com/KLayout/klayout.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #630 from KLayout/ruby-include 

Ruby include
This commit is contained in:
Matthias Köfferlein 2020-09-14 18:33:57 +02:00 committed by GitHub
parent 8cdb6187b8 e8c9ae4012
commit 9cef935fad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 1138 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/lay/lay/doc/manual/drc_runsets.xml
View File

@ -69,6 +69,60 @@
</p>
<h2>Including other files</h2>
<p>
The DRC script language is based on Ruby which delivers many native language
features. Basically, inside a script you can include another script through
"load". This will read a file and execute the content of this file in the
context of the script it is loaded into.
</p>
<p>
Unfortunately, "load" creates a local context for variables. Hence it's not
possible for example to use "load" to read a file that defines variables for further
use in the DRC script.
</p>
<p>
To overcome this problem, KLayout offers a specific extension which embeds
another file into the source by employing some kind of preprocessing.
This way, a file can be included into another one like it was pasted at
this place.
</p>
<p>
The notation is this:
</p>
<pre>
# %include to_include.drc
</pre>
<p>
which will include "include.drc". If no absolute path is given, this file is looked
up relative to the file it is included in.
</p>
<p>
The file name can be put in quotes as well. Expression interpolation is supported
(for the notation see <link href="/about/expressions.xml"/>). Hence it is
possible to access environment variables for example like this:
</p>
<pre>
# %include $(env("HOME"))/to_include.drc
</pre>
<p>
Because Ruby does not see the original files, some internals (e.g.
introspection) will report wrong file names and line numbers. In most
cases - for example when using "__FILE__" or "__LINE__" or when receiving stack
traces and errors - the file names and line numbers will correctly refer
to the source files before include file processing.
</p>
<h2>Input and output</h2>
<p>

73
src/lay/lay/layMacroEditorDialog.cc
View File

@ -44,6 +44,7 @@
#include "tlClassRegistry.h"
#include "tlExceptions.h"
#include "tlFileUtils.h"
#include "tlInclude.h"
#include <cstdio>
#include <limits>
@ -2781,6 +2782,9 @@ MacroEditorDialog::start_exec (gsi::Interpreter *ec)
}
m_file_to_widget.clear ();
m_include_expanders.clear ();
m_include_paths_to_ids.clear ();
m_include_file_id_cache.clear ();
m_last_process_events = tl::Clock::current ();
@ -2825,6 +2829,7 @@ MacroEditorDialog::end_exec (gsi::Interpreter *ec)
do_update_ui_to_run_mode ();
}
const size_t pseudo_file_offset = std::numeric_limits<size_t>::max () / 2;
size_t
MacroEditorDialog::id_for_path (gsi::Interpreter *, const std::string &path)
@ -2840,8 +2845,66 @@ MacroEditorDialog::id_for_path (gsi::Interpreter *, const std::string &path)
if (macro) {
m_file_to_widget.push_back (std::make_pair (macro, (MacroEditorPage *) 0));
return m_file_to_widget.size ();
} else {
return 0;
}
if (! path.empty () && path[0] == '@') {
m_include_expanders.push_back (tl::IncludeExpander::from_string (path));
return pseudo_file_offset + m_include_expanders.size () - 1;
}
return 0;
}
void
MacroEditorDialog::translate_pseudo_id (size_t &file_id, int &line)
{
if (file_id >= pseudo_file_offset) {
file_id -= pseudo_file_offset;
std::pair<size_t, int> ck (file_id, line);
std::map<std::pair<size_t, int>, std::pair<size_t, int> >::iterator ic = m_include_file_id_cache.find (ck);
if (ic != m_include_file_id_cache.end ()) {
file_id = ic->second.first;
line = ic->second.second;
} else {
if (file_id < m_include_expanders.size ()) {
std::pair<std::string, int> fp = m_include_expanders [file_id].translate_to_original (line);
line = fp.second;
std::map<std::string, size_t>::const_iterator i = m_include_paths_to_ids.find (fp.first);
if (i == m_include_paths_to_ids.end ()) {
size_t new_id = id_for_path (0, fp.first);
if (new_id < pseudo_file_offset) {
file_id = new_id;
} else {
file_id = 0;
}
m_include_paths_to_ids.insert (std::make_pair (fp.first, file_id));
} else {
file_id = i->second;
}
} else {
// give up.
file_id = 0;
line = 0;
}
m_include_file_id_cache.insert (std::make_pair (ck, std::make_pair (file_id, line)));
}
}
}
@ -2862,6 +2925,9 @@ MacroEditorDialog::exception_thrown (gsi::Interpreter *interpreter, size_t file_
return;
}
// translate the pseudo file ID and line to the real one (include file processing)
translate_pseudo_id (file_id, line);
try {
// If the exception is thrown in code that is inside a file managed by the macro collection,
@ -2958,6 +3024,9 @@ MacroEditorDialog::trace (gsi::Interpreter *interpreter, size_t file_id, int lin
m_current_stack_depth = stack_trace_provider->stack_depth ();
}
// translate the pseudo file ID and line to the real one (include file processing)
translate_pseudo_id (file_id, line);
// Note: only scripts running in the context of the execution controller (the one who called start_exec)
// can be interrupted and single-stepped, but breakpoints can make the debugger stop in other interpreters.
if (file_id > 0 && ((interpreter == mp_exec_controller && m_stop_stack_depth >= 0 && stack_trace_provider->stack_depth () <= m_stop_stack_depth) ||

5
src/lay/lay/layMacroEditorDialog.h
View File

@ -35,6 +35,7 @@
#include "tlFileSystemWatcher.h"
#include "tlDeferredExecution.h"
#include "tlScriptError.h"
#include "tlInclude.h"
#include "lymMacro.h"
#include "gsiInterpreter.h"
@ -290,6 +291,7 @@ private:
void select_trace (size_t index);
bool configure (const std::string &name, const std::string &value);
void config_finalize ();
void translate_pseudo_id (size_t &file_id, int &line);
lay::Dispatcher *mp_plugin_root;
lym::MacroCollection *mp_root;
@ -312,6 +314,9 @@ private:
QTextCharFormat m_stderr_format;
MacroEditorHighlighters m_highlighters;
std::vector<std::pair<lym::Macro *, MacroEditorPage *> > m_file_to_widget;
std::vector<tl::IncludeExpander> m_include_expanders;
std::map<std::string, size_t> m_include_paths_to_ids;
std::map<std::pair<size_t, int>, std::pair<size_t, int> > m_include_file_id_cache;
std::vector<lay::MacroEditorTree *> m_macro_trees;
bool m_in_exec, m_in_breakpoint;
gsi::Interpreter *mp_exec_controller, *mp_current_interpreter;

102
src/lym/lym/gsiDeclLymMacro.cc
View File

@ -30,6 +30,7 @@
#include "tlClassRegistry.h"
#include "tlFileUtils.h"
#include "tlInclude.h"
#include <cstdio>
@ -101,7 +102,7 @@ class MacroInterpreter
public:
MacroInterpreter ()
: lym::MacroInterpreter (),
mp_registration (0)
mp_registration (0), m_supports_include_expansion (true)
{
m_suffix = lym::MacroInterpreter::suffix ();
m_description = lym::MacroInterpreter::description ();
@ -120,6 +121,15 @@ public:
m_templates.clear ();
}
std::pair<std::string, std::string> include_expansion (const lym::Macro *macro)
{
if (m_supports_include_expansion) {
return lym::MacroInterpreter::include_expansion (macro);
} else {
return std::pair<std::string, std::string> (macro->path (), macro->text ());
}
}
void register_gsi (const char *name)
{
// makes the object owned by the C++ side
@ -139,6 +149,16 @@ public:
}
}
void set_supports_include_expansion (bool f)
{
m_supports_include_expansion = f;
}
virtual bool supports_include_expansion () const
{
return m_supports_include_expansion;
}
void set_storage_scheme (int scheme)
{
m_storage_scheme = lym::Macro::Format (scheme);
@ -231,6 +251,7 @@ private:
lym::Macro::Interpreter m_debugger_scheme;
std::string m_suffix;
std::string m_description;
bool m_supports_include_expansion;
};
int const_PlainTextFormat ()
@ -294,6 +315,14 @@ Class<gsi::MacroInterpreter> decl_MacroInterpreter ("lay", "MacroInterpreter",
"\n"
"This method must be called after \\register has called.\n"
) +
gsi::method ("supports_include_expansion=", &MacroInterpreter::set_supports_include_expansion, gsi::arg ("flag"),
"@brief Sets a value indicating whether this interpreter supports the default include file expansion scheme.\n"
"If this value is set to true (the default), lines like '# %include ...' will be substituted by the "
"content of the file following the '%include' keyword.\n"
"Set this value to false if you don't want to support this feature.\n"
"\n"
"This attribute has been introduced in version 0.27.\n"
) +
gsi::method ("syntax_scheme=", &gsi::MacroInterpreter::set_syntax_scheme, gsi::arg ("scheme"),
"@brief Sets a string indicating the syntax highlighter scheme\n"
"\n"
@ -426,6 +455,24 @@ static lym::Macro *macro_by_path (const std::string &path)
return lym::MacroCollection::root ().find_macro (path);
}
static std::string real_path (const std::string &path, int line)
{
if (! path.empty () && path[0] == '@') {
return tl::IncludeExpander::from_string (path).translate_to_original (line).first;
} else {
return path;
}
}
static int real_line (const std::string &path, int line)
{
if (! path.empty () && path[0] == '@') {
return tl::IncludeExpander::from_string (path).translate_to_original (line).second;
} else {
return line;
}
}
Class<lym::Macro> decl_Macro ("lay", "Macro",
gsi::method ("path", &lym::Macro::path,
"@brief Gets the path of the macro\n"
@ -522,6 +569,59 @@ Class<lym::Macro> decl_Macro ("lay", "Macro",
gsi::method ("menu_path=", &lym::Macro::set_menu_path, gsi::arg ("string"),
"@brief Sets the menu path\n"
"See \\menu_path for details.\n"
) +
gsi::method ("real_path", &real_path,
"@brief Gets the real path for an include-encoded path and line number\n"
"\n"
"When using KLayout's include scheme based on '# %include ...', __FILE__ and __LINE__ (Ruby) will "
"not have the proper values but encoded file names. This method allows retrieving the real file by using\n"
"\n"
"@code\n"
"# Ruby\n"
"real_file = RBA::Macro::real_path(__FILE__, __LINE__)\n"
"@/code\n"
"\n"
"This substitution is not required for top-level macros as KLayout's interpreter will automatically use this "
"function instead of __FILE__. Call this function when you need __FILE__ from files "
"included through the languages mechanisms such as 'require' or 'load' where this substitution does not happen.\n"
"\n"
"For Python there is no equivalent for __LINE__, so you always have to use:\n"
"\n"
"@code\n"
"# Python"
"import inspect\n"
"real_file = pya.Macro.real_path(__file__, inspect.currentframe().f_back.f_lineno)\n"
"@/code\n"
"\n"
"This feature has been introduced in version 0.27."
) +
gsi::method ("real_line", &real_line,
"@brief Gets the real line number for an include-encoded path and line number\n"
"\n"
"When using KLayout's include scheme based on '# %include ...', __FILE__ and __LINE__ (Ruby) will "
"not have the proper values but encoded file names. This method allows retrieving the real line number by using\n"
"\n"
"@code\n"
"# Ruby\n"
"real_line = RBA::Macro::real_line(__FILE__, __LINE__)\n"
"\n"
"# Python\n"
"real_line = pya::Macro::real_line(__file__, __line__)\n"
"@/code\n"
"\n"
"This substitution is not required for top-level macros as KLayout's interpreter will automatically use this "
"function instead of __FILE__. Call this function when you need __FILE__ from files "
"included through the languages mechanisms such as 'require' or 'load' where this substitution does not happen.\n"
"\n"
"For Python there is no equivalent for __LINE__, so you always have to use:\n"
"\n"
"@code\n"
"# Python"
"import inspect\n"
"real_line = pya.Macro.real_line(__file__, inspect.currentframe().f_back.f_lineno)\n"
"@/code\n"
"\n"
"This feature has been introduced in version 0.27."
),
"@brief A macro class\n"
"\n"

1
src/lym/lym/lym.pro
View File

@ -13,6 +13,7 @@ SOURCES = \
HEADERS = \
lymCommon.h \
lymInclude.h \
lymMacroInterpreter.h \
lymMacro.h \

12
src/lym/lym/lymMacro.cc
View File

@ -33,6 +33,7 @@
#include "tlLog.h"
#include "tlXMLParser.h"
#include "tlGlobPattern.h"
#include "tlInclude.h"
#include "rba.h"
#include "pya.h"
@ -1020,20 +1021,29 @@ int Macro::run () const
}
try {
gsi::Interpreter *ip = script_interpreter (interpreter ());
if (ip) {
static lym::MacroInterpreter def_interpreter;
if (! prolog ().empty ()) {
ip->eval_string (prolog ().c_str ());
}
ip->eval_string (text ().c_str (), path ().c_str (), 1);
std::pair<std::string, std::string> ep = def_interpreter.include_expansion (this);
ip->eval_string (ep.second.c_str (), ep.first.c_str (), 1);
if (! epilog ().empty ()) {
ip->eval_string (epilog ().c_str ());
}
} else if (interpreter () == lym::Macro::DSLInterpreter) {
lym::MacroInterpreter::execute_macro (this);
} else {
throw tl::Exception (tl::to_string (tr ("Can't run macro (no interpreter): ")) + path ());
}
} catch (tl::ExitException &ex) {
return ex.status ();
}

69
src/lym/lym/lymMacroInterpreter.cc
View File

@ -27,6 +27,7 @@
#include "tlInternational.h"
#include "tlException.h"
#include "tlClassRegistry.h"
#include "tlInclude.h"
namespace lym
{
@ -48,13 +49,79 @@ MacroInterpreter::can_run (const lym::Macro *macro)
return false;
}
std::pair<std::string, std::string>
MacroInterpreter::include_expansion (const lym::Macro *macro)
{
std::pair<std::string, std::string> res;
res.first = tl::IncludeExpander::expand (macro->path (), macro->text (), res.second).to_string ();
if (res.first != macro->path ()) {
// Fix the macro's text such that include expansion does not spoil __FILE__ or __LINE__ variables
// NOTE: this will modify the column for syntax errors. Let's hope this tiny error is acceptable.
// TODO: this substitution may be somewhat naive ...
Macro::Interpreter ip = macro->interpreter ();
if (macro->interpreter () == Macro::DSLInterpreter) {
if (syntax_scheme () == "ruby") {
ip = Macro::Ruby;
} else if (syntax_scheme () == "python") {
ip = Macro::Python;
}
}
if (ip == Macro::Ruby) {
std::string subst;
const std::string file_const ("__FILE__");
const std::string line_const ("__LINE__");
for (const char *cp = res.second.c_str (); *cp; ) {
if (strncmp (cp, file_const.c_str (), file_const.size ()) == 0 && !isalnum (cp[file_const.size ()]) && cp[file_const.size ()] != '_') {
subst += "RBA::Macro::real_path(__FILE__, __LINE__)";
cp += file_const.size ();
} else if (strncmp (cp, line_const.c_str (), line_const.size ()) == 0 && !isalnum (cp[line_const.size ()]) && cp[line_const.size ()] != '_') {
subst += "RBA::Macro::real_line(__FILE__, __LINE__)";
cp += line_const.size ();
} else {
subst += *cp++;
}
}
res.second = subst;
}
}
return res;
}
void
MacroInterpreter::execute_macro (const lym::Macro *macro)
{
for (tl::Registrar<lym::MacroInterpreter>::iterator cls = tl::Registrar<lym::MacroInterpreter>::begin (); cls != tl::Registrar<lym::MacroInterpreter>::end (); ++cls) {
if (cls.current_name () == macro->dsl_interpreter ()) {
cls->execute (macro);
std::pair<std::string, std::string> et = cls->include_expansion (macro);
if (et.first.empty () || et.first == macro->path ()) {
cls->execute (macro);
} else {
// provide a copy which takes the include-expanded version
lym::Macro tmp_macro;
tmp_macro.assign (*macro);
tmp_macro.set_text (et.second);
tmp_macro.set_file_path (et.first);
cls->execute (&tmp_macro);
}
return;
}
}

12
src/lym/lym/lymMacroInterpreter.h
View File

@ -138,6 +138,18 @@ public:
// .. nothing yet ..
}
/**
* @brief Provides generic include file expansion
*
* This method takes a given macro and substitutes include statements of the form '# %include ...' by the
* content of the respective file. Recursive include is supported.
* The return value of this method is a two-element array with two strings: the first one is a path string which
* holds the encoded information for translating back path/line number information into the original paths and
* line numbers. This first string needs to be passed to the actual script interpreter as the 'file path'. The
* second component of the returned array is the text of the macro with the include files substituted.
*/
virtual std::pair<std::string, std::string> include_expansion (const lym::Macro *macro);
/**
* @brief Runs the script for the DSL interpreter with the given name
*

203
src/lym/unit_tests/lymBasicTests.cc
View File

@ -2,7 +2,7 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2020 Matthias Koefferlein
Copyright (C) 2006-2018 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -23,11 +23,202 @@
#include "tlUnitTest.h"
TEST(1)
{
EXPECT_EQ (1, 1); // avoids a compiler warning because of unreferenced _this
#include "lymMacro.h"
#include "lymMacroInterpreter.h"
#include "gsiInterpreter.h"
#include "rba.h"
#include "pya.h"
// TODO: add tests for lym specific things
throw tl::CancelException (); // skip this test to indicate that there is nothing yet
class TestCollectorConsole
: public gsi::Console
{
public:
TestCollectorConsole () { }
~TestCollectorConsole () { }
virtual void write_str (const char *text, output_stream)
{
m_text += text;
}
virtual void flush () { }
virtual bool is_tty () { return false; }
virtual int columns () { return 80; }
virtual int rows () { return 50; }
const std::string &text () const { return m_text; }
private:
std::string m_text;
};
#if defined(HAVE_RUBY)
TEST(1_BasicRuby)
{
tl_assert (rba::RubyInterpreter::instance () != 0);
lym::Macro macro;
macro.set_file_path (tl::testsrc () + "/testdata/lym/m1.rb");
macro.set_interpreter (lym::Macro::Ruby);
macro.load ();
TestCollectorConsole console;
rba::RubyInterpreter::instance ()->push_console (&console);
try {
EXPECT_EQ (macro.run (), 0);
rba::RubyInterpreter::instance ()->remove_console (&console);
} catch (...) {
rba::RubyInterpreter::instance ()->remove_console (&console);
throw;
}
EXPECT_EQ (console.text (), "Hello, world!\n");
}
TEST(2_RubyInclude)
{
tl_assert (rba::RubyInterpreter::instance () != 0);
lym::Macro macro;
macro.set_file_path (tl::testsrc () + "/testdata/lym/m2.rb");
macro.set_interpreter (lym::Macro::Ruby);
macro.load ();
TestCollectorConsole console;
rba::RubyInterpreter::instance ()->push_console (&console);
try {
EXPECT_EQ (macro.run (), 0);
rba::RubyInterpreter::instance ()->remove_console (&console);
} catch (...) {
rba::RubyInterpreter::instance ()->remove_console (&console);
throw;
}
EXPECT_EQ (console.text (), "Stop 1: m2.rb:2\nf: a_inc.rb:3\nStop 2: m2.rb:8\n");
}
TEST(3_RubyInclude)
{
tl_assert (rba::RubyInterpreter::instance () != 0);
lym::Macro macro;
macro.set_file_path (tl::testsrc () + "/testdata/lym/m3.rb");
macro.set_interpreter (lym::Macro::Ruby);
macro.load ();
TestCollectorConsole console;
rba::RubyInterpreter::instance ()->push_console (&console);
try {
EXPECT_EQ (macro.run (), 0);
rba::RubyInterpreter::instance ()->remove_console (&console);
} catch (...) {
rba::RubyInterpreter::instance ()->remove_console (&console);
throw;
}
EXPECT_EQ (console.text (), "An error in " + tl::testsrc () + "/testdata/lym/b_inc.rb:3\n");
}
TEST(11_DRCBasic)
{
tl_assert (rba::RubyInterpreter::instance () != 0);
lym::Macro macro;
macro.set_file_path (tl::testsrc () + "/testdata/lym/m1.drc");
macro.set_interpreter (lym::Macro::DSLInterpreter);
macro.set_dsl_interpreter ("drc");
macro.load ();
TestCollectorConsole console;
rba::RubyInterpreter::instance ()->push_console (&console);
try {
EXPECT_EQ (macro.run (), 0);
rba::RubyInterpreter::instance ()->remove_console (&console);
} catch (...) {
rba::RubyInterpreter::instance ()->remove_console (&console);
throw;
}
EXPECT_EQ (console.text (), "Result: (500,500;500,2000;1000,2000;1000,500) in m1.drc:20\n");
}
TEST(12_DRCBasic)
{
tl_assert (rba::RubyInterpreter::instance () != 0);
lym::Macro macro;
macro.set_file_path (tl::testsrc () + "/testdata/lym/m2.drc");
macro.set_interpreter (lym::Macro::DSLInterpreter);
macro.set_dsl_interpreter ("drc");
macro.load ();
TestCollectorConsole console;
rba::RubyInterpreter::instance ()->push_console (&console);
try {
EXPECT_EQ (macro.run (), 0);
rba::RubyInterpreter::instance ()->remove_console (&console);
} catch (...) {
rba::RubyInterpreter::instance ()->remove_console (&console);
throw;
}
EXPECT_EQ (console.text (), "Result: (500,500;500,2000;1000,2000;1000,500) in m2.drc:14\n");
}
#endif
#if defined(HAVE_PYTHON)
TEST(101_BasicPython)
{
tl_assert (pya::PythonInterpreter::instance () != 0);
lym::Macro macro;
macro.set_file_path (tl::testsrc () + "/testdata/lym/m1.py");
macro.set_interpreter (lym::Macro::Python);
macro.load ();
TestCollectorConsole console;
pya::PythonInterpreter::instance ()->push_console (&console);
try {
EXPECT_EQ (macro.run (), 0);
pya::PythonInterpreter::instance ()->remove_console (&console);
} catch (...) {
pya::PythonInterpreter::instance ()->remove_console (&console);
throw;
}
EXPECT_EQ (console.text (), "Hello, world!\n");
}
TEST(102_PythonInclude)
{
tl_assert (pya::PythonInterpreter::instance () != 0);
lym::Macro macro;
macro.set_file_path (tl::testsrc () + "/testdata/lym/m2.py");
macro.set_interpreter (lym::Macro::Python);
macro.load ();
TestCollectorConsole console;
pya::PythonInterpreter::instance ()->push_console (&console);
try {
EXPECT_EQ (macro.run (), 0);
pya::PythonInterpreter::instance ()->remove_console (&console);
} catch (...) {
pya::PythonInterpreter::instance ()->remove_console (&console);
throw;
}
EXPECT_EQ (console.text (), "Stop 1: m2.py:8\nf: a_inc.py:5\nStop 2: m2.py:14\n");
}
#endif

8
src/lym/unit_tests/unit_tests.pro
View File

@ -7,10 +7,10 @@ TARGET = lym_tests
include($$PWD/../../lib_ut.pri)
SOURCES = \
lymBasicTests.cc \
lymBasicTests.cc
INCLUDEPATH += $$LYM_INC $$TL_INC $$GSI_INC
DEPENDPATH += $$LYM_INC $$TL_INC $$GSI_INC
INCLUDEPATH += $$RBA_INC $$PYA_INC $$LYM_INC $$TL_INC $$GSI_INC
DEPENDPATH += $$RBA_INC $$PYA_INC $$LYM_INC $$TL_INC $$GSI_INC
LIBS += -L$$DESTDIR_UT -lklayout_lym -lklayout_tl -lklayout_gsi
LIBS += -L$$DESTDIR_UT -lklayout_rba -lklayout_pya -lklayout_lym -lklayout_tl -lklayout_gsi

1
src/rba/rba/rbaUtils.cc
View File

@ -26,6 +26,7 @@
#include "rba.h"
#include "rbaUtils.h"
#include "rbaInternal.h"
#include "tlInclude.h"
#if HAVE_RUBY_VERSION_CODE >= 20200
# include <ruby/debug.h>

2
src/tl/tl/tl.pro
View File

@ -20,6 +20,7 @@ SOURCES = \
tlGlobPattern.cc \
tlHeap.cc \
tlHttpStream.cc \
tlInclude.cc \
tlInternational.cc \
tlLog.cc \
tlObject.cc \
@ -63,6 +64,7 @@ HEADERS = \
tlGlobPattern.h \
tlHeap.h \
tlHttpStream.h \
tlInclude.h \
tlInternational.h \
tlIntervalMap.h \
tlIntervalSet.h \

210
src/tl/tl/tlInclude.cc Normal file
View File

@ -0,0 +1,210 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2020 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "tlInclude.h"
#include "tlAssert.h"
#include "tlString.h"
#include "tlFileUtils.h"
#include "tlStream.h"
#include "tlExpression.h"
#include "tlUri.h"
namespace tl
{
static const char *valid_fn_chars = "@_:,.\\/-+";
IncludeExpander::IncludeExpander ()
{
// .. nothing yet ..
}
IncludeExpander
IncludeExpander::expand (const std::string &path, std::string &expanded_text)
{
IncludeExpander ie;
int lc = 1;
tl::InputStream is (path);
ie.read (path, is, expanded_text, ie, lc);
return ie;
}
IncludeExpander
IncludeExpander::expand (const std::string &path, const std::string &original_text, std::string &expanded_text)
{
IncludeExpander ie;
int lc = 1;
tl::InputMemoryStream ms (original_text.c_str (), original_text.size ());
tl::InputStream is (ms);
ie.read (path, is, expanded_text, ie, lc);
return ie;
}
void
IncludeExpander::read (const std::string &path, tl::InputStream &is, std::string &expanded_text, IncludeExpander &ie, int &line_counter)
{
ie.m_sections [line_counter] = std::make_pair (path, 1 - line_counter);
tl::TextInputStream text (is);
int lnum = 0;
bool emit_section = false;
while (! text.at_end ()) {
std::string l = text.get_line ();
++lnum;
tl::Extractor ex (l.c_str ());
if (ex.test ("#") && ex.test ("%include")) {
std::string include_path;
if (*ex.skip () == '"' || *ex.skip () == '\'') {
ex.read_quoted (include_path);
ex.expect_end ();
} else {
include_path = tl::trim (ex.skip ());
}
// allow some customization by employing expression interpolation
include_path = tl::Eval ().interpolate (include_path);
// NOTE: by using URI's we can basically read from HTTP etc.
tl::URI current_uri (path);
tl::URI new_uri (include_path);
if (current_uri.scheme ().empty () && new_uri.scheme ().empty ()) {
if (! tl::is_absolute (include_path)) {
include_path = tl::combine_path (tl::dirname (path), include_path);
}
} else {
include_path = current_uri.resolved (new_uri).to_string ();
}
tl::InputStream is (include_path);
read (include_path, is, expanded_text, ie, line_counter);
emit_section = true;
} else {
if (emit_section) {
emit_section = false;
ie.m_sections [line_counter] = std::make_pair (path, lnum - line_counter);
}
expanded_text += l;
expanded_text += "\n";
++line_counter;
}
}
}
std::string
IncludeExpander::to_string () const
{
if (m_sections.empty ()) {
return std::string ();
} else if (m_sections.size () == 1) {
tl_assert (m_sections.begin ()->first == 1);
tl_assert (m_sections.begin ()->second.second == 0);
std::string fn = m_sections.begin ()->second.first;
return tl::to_word_or_quoted_string (fn, valid_fn_chars);
} else {
// "@" indicates a mapping table
std::string res ("@");
for (std::map<int, std::pair<std::string, int> >::const_iterator m = m_sections.begin (); m != m_sections.end (); ++m) {
res += tl::to_string (m->first);
res += "*";
res += tl::to_word_or_quoted_string (m->second.first, valid_fn_chars);
res += "*";
res += tl::to_string (m->second.second);
res += ";";
}
return res;
}
}
IncludeExpander
IncludeExpander::from_string (const std::string &s)
{
IncludeExpander ie;
tl::Extractor ex (s.c_str ());
if (*ex == '"' || *ex == '\'') {
ex.read_quoted (ie.m_sections [1].first);
} else if (ex.test ("@")) {
while (! ex.at_end ()) {
int ln = 0;
ex.read (ln);
std::pair<std::string, int> &si = ie.m_sections [ln];
ex.expect ("*");
ex.read_word_or_quoted (si.first, valid_fn_chars);
ex.expect ("*");
ex.read (si.second);
ex.test (";");
}
} else {
ie.m_sections [1].first = s;
}
return ie;
}
std::pair<std::string, int>
IncludeExpander::translate_to_original (int line_number)
{
std::map<int, std::pair<std::string, int> >::const_iterator m = m_sections.lower_bound (line_number);
if (m != m_sections.begin () && (m == m_sections.end () || m->first > line_number)) {
--m;
}
if (m == m_sections.end ()) {
return std::make_pair (std::string (), 0);
} else {
return std::make_pair (m->second.first, line_number + m->second.second);
}
}
}

107
src/tl/tl/tlInclude.h Normal file
View File

@ -0,0 +1,107 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2020 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef HDR_tlInclude
#define HDR_tlInclude
#include "tlCommon.h"
#include <string>
#include <map>
namespace tl
{
class InputStream;
/**
* @brief Provide the basic include expansion and file/line mapping mechanism
*
* The Expander object performs the file expansion and also stores the information
* required for translating back file names and line numbers.
*
* Include expansion happens through a pseudo-comment "# %include ..." which
* takes a file path as the argument. File paths are always resolved relative to
* the original file.
* The file path is expression-interpolated, hence can access environment variables
* through $(env("HOME")) for example.
*/
class TL_PUBLIC IncludeExpander
{
public:
/**
* @brief The default constructor
*/
IncludeExpander ();
/**
* @brief Provides include expansion
*
* This method will deliver the expanded text and the include expander object.
*/
static IncludeExpander expand (const std::string &path, std::string &expanded_text);
/**
* @brief Provides include expansion
*
* This method will deliver the expanded text and the include expander object.
* This version also takes the actual text of the original file.
*/
static IncludeExpander expand (const std::string &path, const std::string &original_text, std::string &expanded_text);
/**
* @brief Serializes the include expander information into a string
*
* If no include expansion happened, the serialized string will be the original file path.
* Otherwise it's a "@"-prefixed string.
*/
std::string to_string () const;
/**
* @brief Deserializes the include expander information from a string
*/
static IncludeExpander from_string (const std::string &s);
/**
* @brief Provides translation of the expanded text's line number to filename/line number for the original file
*/
std::pair<std::string, int> translate_to_original (int line_number);
/**
* @brief Provides translation of original file name/line number to included file name/line number
*/
static std::pair<std::string, int> translate_to_original (const std::string &file, int line_number)
{
return IncludeExpander::from_string (file).translate_to_original (line_number);
}
private:
std::map<int, std::pair<std::string, int> > m_sections;
void read (const std::string &path, tl::InputStream &is, std::string &expanded_text, IncludeExpander &ie, int &line_counter);
};
}
#endif

57
src/tl/tl/tlScriptError.cc
View File

@ -22,6 +22,7 @@
#include "tlScriptError.h"
#include "tlString.h"
#include "tlInclude.h"
namespace tl
{
@ -32,13 +33,13 @@ namespace tl
BacktraceElement::BacktraceElement (const std::string &_file, int _line)
: file (_file), line (_line)
{
// .. nothing yet ..
translate_includes ();
}
BacktraceElement::BacktraceElement (const std::string &_file, int _line, const std::string _more_info)
: file (_file), line (_line), more_info (_more_info)
{
// .. nothing yet ..
translate_includes ();
}
BacktraceElement::BacktraceElement ()
@ -47,6 +48,20 @@ BacktraceElement::BacktraceElement ()
// .. nothing yet ..
}
void
BacktraceElement::translate_includes ()
{
if (line < 1) {
return;
}
std::pair<std::string, int> fl = tl::IncludeExpander::translate_to_original (file, line);
if (fl.second > 0) {
file = fl.first;
line = fl.second;
}
}
std::string
BacktraceElement::to_string() const
{
@ -64,20 +79,38 @@ BacktraceElement::to_string() const
// -------------------------------------------------------------------
// ScriptError implementation
ScriptError::ScriptError (const char *msg, const char *cls, const std::vector<BacktraceElement> &backtrace)
: tl::Exception (msg), m_line (-1), m_cls (cls), m_backtrace (backtrace)
{
// .. nothing yet ..
}
ScriptError::ScriptError (const char *msg, const char *sourcefile, int line, const char *cls, const std::vector<BacktraceElement> &backtrace)
: tl::Exception (msg), m_sourcefile (sourcefile), m_line (line), m_cls (cls), m_backtrace (backtrace)
{
translate_includes ();
}
ScriptError::ScriptError (const ScriptError &d)
: tl::Exception (d), m_sourcefile (d.m_sourcefile), m_line (d.m_line), m_cls (d.m_cls), m_context (d.m_context), m_backtrace (d.m_backtrace)
{
// .. nothing yet ..
}
std::string
ScriptError::basic_msg () const
{
return tl::Exception::msg ();
}
std::string
std::string
ScriptError::msg () const
{
std::string m = basic_msg ();
if (! m_context.empty ()) {
m += tl::to_string (tr (" in ")) + m_context;
}
m += tl::to_string (tr (" in ")) + m_context;
}
for (std::vector<BacktraceElement>::const_iterator bt = backtrace ().begin (); bt != backtrace ().end (); ++bt) {
m += "\n ";
@ -87,6 +120,20 @@ ScriptError::msg () const
return m;
}
void
ScriptError::translate_includes ()
{
if (m_line < 1) {
return;
}
std::pair<std::string, int> fl = tl::IncludeExpander::translate_to_original (m_sourcefile, m_line);
if (fl.second > 0) {
m_sourcefile = fl.first;
m_line = fl.second;
}
}
}

17
src/tl/tl/tlScriptError.h
View File

@ -60,6 +60,9 @@ struct TL_PUBLIC BacktraceElement
std::string file;
int line;
std::string more_info;
private:
void translate_includes ();
};
/**
@ -69,17 +72,11 @@ class TL_PUBLIC ScriptError
: public tl::Exception
{
public:
ScriptError (const char *msg, const char *cls, const std::vector <BacktraceElement> &backtrace)
: tl::Exception (msg), m_line (-1), m_cls (cls), m_backtrace (backtrace)
{ }
ScriptError (const char *msg, const char *cls, const std::vector <BacktraceElement> &backtrace);
ScriptError (const char *msg, const char *sourcefile, int line, const char *cls, const std::vector <BacktraceElement> &backtrace)
: tl::Exception (msg), m_sourcefile (sourcefile), m_line (line), m_cls (cls), m_backtrace (backtrace)
{ }
ScriptError (const char *msg, const char *sourcefile, int line, const char *cls, const std::vector <BacktraceElement> &backtrace);
ScriptError (const ScriptError &d)
: tl::Exception (d), m_sourcefile (d.m_sourcefile), m_line (d.m_line), m_cls (d.m_cls), m_context (d.m_context), m_backtrace (d.m_backtrace)
{ }
ScriptError (const ScriptError &d);
virtual ~ScriptError ()
{ }
@ -139,6 +136,8 @@ private:
std::string m_cls;
std::string m_context;
std::vector<BacktraceElement> m_backtrace;
void translate_includes ();
};
/**

13
src/tl/tl/tlStream.cc
View File

@ -449,18 +449,26 @@ TextInputStream::read_all (size_t max_count)
const std::string &
TextInputStream::get_line ()
{
m_line = m_next_line;
int line = m_next_line;
m_line_buffer.clear ();
while (! at_end ()) {
char c = get_char ();
if (c == '\n' || c == 0) {
if (c == '\n') {
// set at_end if there is nothing after this terminal LF -> this will avoid
// emitting an empty dummy line as the last one
if (peek_char () == 0) {
m_at_end = true;
}
break;
} else if (c == 0) {
break;
} else {
m_line_buffer += c;
}
}
m_line = line;
return m_line_buffer;
}
@ -489,7 +497,6 @@ TextInputStream::peek_char ()
m_line = m_next_line;
const char *c = m_stream.get (1);
if (c == 0) {
m_at_end = true;
return 0;
} else if (*c != '\r' && *c) {
char cc = *c;

105
src/tl/unit_tests/tlIncludeTests.cc Normal file
View File

@ -0,0 +1,105 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2020 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "tlUnitTest.h"
#include "tlFileUtils.h"
#include "tlStream.h"
#include "tlInclude.h"
TEST(1_simple)
{
std::string fn = tl::testsrc () + "/testdata/tl/x.txt";
std::string et;
tl::IncludeExpander ie = tl::IncludeExpander::expand (fn, et);
EXPECT_EQ (et, "A line\nAnother line\n");
EXPECT_EQ (ie.to_string (), fn);
EXPECT_EQ (tl::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ());
EXPECT_EQ (ie.translate_to_original (2).first, fn);
EXPECT_EQ (ie.translate_to_original (2).second, 2);
}
TEST(2_single_include)
{
std::string fn = tl::testsrc () + "/testdata/tl/x_inc1.txt";
std::string et;
tl::IncludeExpander ie = tl::IncludeExpander::expand (fn, tl::InputStream (fn).read_all (), et);
EXPECT_EQ (et, "A line\nincluded.1\nAnother line\n");
EXPECT_EQ (ie.to_string (), "@1*" + tl::testsrc () + "/testdata/tl/x_inc1.txt*0;2*" + tl::testsrc () + "/testdata/tl/inc1.txt*-1;3*" + tl::testsrc () + "/testdata/tl/x_inc1.txt*0;");
EXPECT_EQ (tl::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ());
EXPECT_EQ (ie.translate_to_original (1).first, fn);
EXPECT_EQ (ie.translate_to_original (1).second, 1);
EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/tl/inc1.txt");
EXPECT_EQ (ie.translate_to_original (2).second, 1);
EXPECT_EQ (ie.translate_to_original (3).first, fn);
EXPECT_EQ (ie.translate_to_original (3).second, 3);
}
TEST(3_multi_include)
{
std::string fn = tl::testsrc () + "/testdata/tl/x_inc3.txt";
std::string et;
tl::IncludeExpander ie = tl::IncludeExpander::expand (fn, et);
EXPECT_EQ (et, "A line\ninclude.3a\nincluded.2a\nincluded.2b\ninclude.3b\nAnother line\n");
EXPECT_EQ (tl::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ());
EXPECT_EQ (ie.translate_to_original (1).first, fn);
EXPECT_EQ (ie.translate_to_original (1).second, 1);
EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/tl/inc3.txt");
EXPECT_EQ (ie.translate_to_original (2).second, 1);
EXPECT_EQ (ie.translate_to_original (3).first, tl::testsrc () + "/testdata/tl/inc2.txt");
EXPECT_EQ (ie.translate_to_original (3).second, 1);
EXPECT_EQ (ie.translate_to_original (5).first, tl::testsrc () + "/testdata/tl/inc3.txt");
EXPECT_EQ (ie.translate_to_original (5).second, 3);
EXPECT_EQ (ie.translate_to_original (6).first, fn);
EXPECT_EQ (ie.translate_to_original (6).second, 3);
}
TEST(4_multi_include_interpolate)
{
std::string fn = tl::testsrc () + "/testdata/tl/x_inc3_ip.txt";
std::string et;
tl::IncludeExpander ie = tl::IncludeExpander::expand (fn, et);
EXPECT_EQ (et, "A line\ninclude.3a\nincluded.2a\nincluded.2b\ninclude.3b\nAnother line\n");
EXPECT_EQ (tl::IncludeExpander::from_string (ie.to_string ()).to_string (), ie.to_string ());
EXPECT_EQ (ie.translate_to_original (1).first, fn);
EXPECT_EQ (ie.translate_to_original (1).second, 1);
EXPECT_EQ (ie.translate_to_original (2).first, tl::testsrc () + "/testdata/tl/inc3.txt");
EXPECT_EQ (ie.translate_to_original (2).second, 1);
EXPECT_EQ (ie.translate_to_original (3).first, tl::testsrc () + "/testdata/tl/inc2.txt");
EXPECT_EQ (ie.translate_to_original (3).second, 1);
EXPECT_EQ (ie.translate_to_original (5).first, tl::testsrc () + "/testdata/tl/inc3.txt");
EXPECT_EQ (ie.translate_to_original (5).second, 3);
EXPECT_EQ (ie.translate_to_original (6).first, fn);
EXPECT_EQ (ie.translate_to_original (6).second, 3);
}

1
src/tl/unit_tests/unit_tests.pro
View File

@ -15,6 +15,7 @@ SOURCES = \
tlEvents.cc \
tlExpression.cc \
tlFileUtils.cc \
tlIncludeTests.cc \
tlIntervalMap.cc \
tlIntervalSet.cc \
tlKDTree.cc \

9
testdata/lym/a_inc.drc vendored Normal file
View File

@ -0,0 +1,9 @@
# some prep steps
ly = RBA::Layout::new
ly.create_cell("TOP")
l1 = ly.layer(1, 0)
ly.top_cell.shapes(l1).insert(RBA::Box::new(0, 0, 1000, 2000))
l2 = ly.layer(2, 0)
ly.top_cell.shapes(l2).insert(RBA::Box::new(500, 500, 1500, 2500))

6
testdata/lym/a_inc.py vendored Normal file
View File

@ -0,0 +1,6 @@
import os
def f():
return("f: " + os.path.basename(pya.Macro.real_path(__file__, lineno())) + ":" + str(pya.Macro.real_line(__file__, lineno())))

5
testdata/lym/a_inc.rb vendored Normal file
View File

@ -0,0 +1,5 @@
def f
return "f: " + File.basename(__FILE__) + ":" + __LINE__.to_s
end

5
testdata/lym/b_inc.rb vendored Normal file
View File

@ -0,0 +1,5 @@
def f
raise("An error")
end

22
testdata/lym/m1.drc vendored Normal file
View File

@ -0,0 +1,22 @@
# some prep steps
ly = RBA::Layout::new
ly.create_cell("TOP")
l1 = ly.layer(1, 0)
ly.top_cell.shapes(l1).insert(RBA::Box::new(0, 0, 1000, 2000))
l2 = ly.layer(2, 0)
ly.top_cell.shapes(l2).insert(RBA::Box::new(500, 500, 1500, 2500))
# actual "DRC"
source(ly.top_cell)
l1_drc = input(1, 0)
l2_drc = input(2, 0)
(l1_drc & l2_drc).output(100, 0)
l100 = ly.layer(100, 0)
puts "Result: " + RBA::Region::new(ly.top_cell.begin_shapes_rec(l100)).to_s + " in " + File.basename(__FILE__) + ":" + __LINE__.to_s

2
testdata/lym/m1.py vendored Normal file
View File

@ -0,0 +1,2 @@
print("Hello, world!")

2
testdata/lym/m1.rb vendored Normal file
View File

@ -0,0 +1,2 @@
puts "Hello, world!"

14
testdata/lym/m2.drc vendored Normal file
View File

@ -0,0 +1,14 @@
# %include "a_inc.drc"
# actual "DRC"
source(ly.top_cell)
l1_drc = input(1, 0)
l2_drc = input(2, 0)
(l1_drc & l2_drc).output(100, 0)
l100 = ly.layer(100, 0)
puts "Result: " + RBA::Region::new(ly.top_cell.begin_shapes_rec(l100)).to_s + " in " + File.basename(__FILE__) + ":" + __LINE__.to_s

15
testdata/lym/m2.py vendored Normal file
View File

@ -0,0 +1,15 @@
import os
import inspect
def lineno():
return inspect.currentframe().f_back.f_lineno
print("Stop 1: " + os.path.basename(pya.Macro.real_path(__file__, lineno())) + ":" + str(pya.Macro.real_line(__file__, lineno())))
# %include a_inc.py
print(f())
print("Stop 2: " + os.path.basename(pya.Macro.real_path(__file__, lineno())) + ":" + str(pya.Macro.real_line(__file__, lineno())))

9
testdata/lym/m2.rb vendored Normal file
View File

@ -0,0 +1,9 @@
puts "Stop 1: " + File.basename(__FILE__) + ":" + __LINE__.to_s
# %include a_inc.rb
puts f
puts "Stop 2: " + File.basename(__FILE__) + ":" + __LINE__.to_s

12
testdata/lym/m3.rb vendored Normal file
View File

@ -0,0 +1,12 @@
# %include b_inc.rb
begin
puts f
rescue => ex
ln = ex.backtrace[0].split(":")
# NOTE: as the backtrace is a native Ruby feature, include file translation
# does not happen. We need to do this explicitly here:
puts ex.to_s + " in " + RBA::Macro::real_path(ln[0], ln[1].to_i) + ":" + RBA::Macro::real_line(ln[0], ln[1].to_i).to_s
end

1
testdata/tl/inc1.txt vendored Normal file
View File

@ -0,0 +1 @@
included.1

2
testdata/tl/inc2.txt vendored Normal file
View File

@ -0,0 +1,2 @@
included.2a
included.2b

3
testdata/tl/inc3.txt vendored Normal file
View File

@ -0,0 +1,3 @@
include.3a
# %include inc2.txt
include.3b

2
testdata/tl/x.txt vendored Normal file
View File

@ -0,0 +1,2 @@
A line
Another line

3
testdata/tl/x_inc1.txt vendored Normal file
View File

@ -0,0 +1,3 @@
A line
# %include inc1.txt
Another line

3
testdata/tl/x_inc3.txt vendored Normal file
View File

@ -0,0 +1,3 @@
A line
# %include inc3.txt
Another line

3
testdata/tl/x_inc3_ip.txt vendored Normal file
View File

@ -0,0 +1,3 @@
A line
# %include inc$(1+2).txt
Another line