Enhanced Salt package capabilities

* Salt packages can host native Ruby and Python
  libraries now. The "ruby_lib" and "python_lib"
  templates provide such native libraries.
  This way we can basically wrap every native
  extension in such a package.
* Ability to supply binary plugins through
  Salt packages as well.

src/gsi/gsiInterpreter.cc

@@ -22,10 +22,18 @@
 
 #include "gsiInterpreter.h"
 
+namespace tl
+{
+  template<> GSI_PUBLIC tl::Registrar<gsi::Interpreter> *tl::Registrar<gsi::Interpreter>::instance = 0;
+}
+
 namespace gsi
 {
 
-Interpreter::Interpreter ()
+GSI_PUBLIC tl::Registrar<Interpreter> interpreters;
+
+Interpreter::Interpreter (int position, const char *name)
+  : tl::RegisteredClass<Interpreter> (this, position, name, false)
 {
   //  .. nothing yet ..
 }

src/gsi/gsiInterpreter.h

@@ -24,6 +24,7 @@
 #define _HDR_gsiInterpreter
 
 #include "tlScriptError.h"
+#include "tlClassRegistry.h"
 #include "gsiCommon.h"
 
 namespace gsi
@@ -169,12 +170,13 @@ public:
  *  @brief A generic interpreter interface
  */
 class GSI_PUBLIC Interpreter
+  : public tl::RegisteredClass<Interpreter>
 {
 public:
   /**
    *  @brief Constructor
    */
-  Interpreter ();
+  Interpreter (int position = 0, const char *name = "");
 
   /**
    *  @brief Destructor
@@ -301,8 +303,31 @@ public:
    *  @brief Removes the given execution handler
    */
   virtual void remove_exec_handler (ExecutionHandler *exec_handler) = 0;
+
+  /**
+   *  @brief Adds a package location to this interpreter
+   *
+   *  Interpreters may look for their packages here or in a subfolder
+   *  of this path. For example, the Python interpreter will add
+   *  <package location>/python to the sys.path search path.
+   *  If this path is already registered, the interpreter shall ignore
+   *  this request.
+   */
+  virtual void add_package_location (const std::string &package_path) = 0;
+
+  /**
+   *  @brief Removes a package location from this interpreter
+   *
+   *  This is the inverse of "add_package_location".
+   */
+  virtual void remove_package_location (const std::string &package_path) = 0;
 };
 
+/**
+ *  @brief The interpreter registry
+ */
+extern GSI_PUBLIC tl::Registrar<Interpreter> interpreters;
+
 }
 
 #endif

src/lay/gsiDeclLayApplication.cc

@@ -26,6 +26,7 @@
 #include "laySignalHandler.h"
 #include "gsiDecl.h"
 #include "gsiQtExternals.h"
+#include "tlArch.h"
 
 namespace gsi
 {
@@ -60,6 +61,11 @@ void crash_me (int reason)
   }
 }
 
+static std::string arch (lay::Application *)
+{
+  return tl::arch_string ();
+}
+
 Class<lay::Application> decl_Application (QT_EXTERNAL_BASE (QApplication) "Application",
   
   method ("instance", &lay::Application::instance, 
@@ -203,7 +209,12 @@ Class<lay::Application> decl_Application (QT_EXTERNAL_BASE (QApplication) "Appli
   ) + 
   method ("version", &lay::Application::version,
     "@brief Returns the application's version string\n"
-  ),
+  ) +
+  method_ext ("arch", &arch,
+    "@brief Returns the architecture string\n"
+    "This method has been introduced in version 0.25."
+  )
+  ,
 
   "@brief The application object\n"
   "\n"

src/lay/layApplication.cc

@@ -55,6 +55,7 @@
 #include "tlExpression.h"
 #include "tlExceptions.h"
 #include "tlInternational.h"
+#include "tlArch.h"
 
 #include <QIcon>
 #include <QDir>
@@ -291,28 +292,51 @@ Application::Application (int &argc, char **argv, bool non_ui_mode)
 
   }
 
-  //  try to locate the global plugins
+  //  Try to locate the native plugins:
+  //  Native plugins are DLL's or SO's disguised as "*.klp" files.
+  //  The are installed either
+  //    - directly in one of the KLAYOUT_PATH directories
+  //    - in a folder named by the architecture (i.e. "i686-win32-mingw" or "x86_64-linux-gcc") below
+  //      one of these folders
+  //    - in one of the Salt packages
+  //    - in one of the Salt packages, in a folder named after the architecture
+
   for (std::vector <std::string>::const_iterator p = m_klayout_path.begin (); p != m_klayout_path.end (); ++p) {
 
     std::set<std::string> modules;
 
-    QDir inst_path_dir (tl::to_qstring (*p));
+    std::vector<QString> klp_paths;
+    klp_paths.push_back (tl::to_qstring (*p));
+    klp_paths.push_back (QDir (klp_paths.back ()).filePath (tl::to_qstring (tl::arch_string ())));
 
-    QStringList name_filters;
-    name_filters << QString::fromUtf8 ("*.klp");
+    lay::Salt salt;
+    //  TODO: this is code duplicated from the SaltController. But this one
+    //  is initialized much later.
+    salt.add_location (tl::to_string (QDir (tl::to_qstring (*p)).filePath (QString::fromUtf8 ("salt"))));
+    for (lay::Salt::flat_iterator g = salt.begin_flat (); g != salt.end_flat (); ++g) {
+      klp_paths.push_back (tl::to_qstring ((*g)->path ()));
+      klp_paths.push_back (QDir (klp_paths.back ()).filePath (tl::to_qstring (tl::arch_string ())));
+    }
 
-    QStringList inst_modules = inst_path_dir.entryList (name_filters);
-    inst_modules.sort ();
+    for (std::vector<QString>::const_iterator p = klp_paths.begin (); p != klp_paths.end (); ++p) {
 
-    for (QStringList::const_iterator im = inst_modules.begin (); im != inst_modules.end (); ++im) {
-      QFileInfo klp_file (tl::to_qstring (*p), *im);
-      if (klp_file.exists () && klp_file.isReadable ()) {
-        std::string m = tl::to_string (klp_file.absoluteFilePath ());
-        if (modules.find (m) == modules.end ()) {
-          load_plugin (m);
-          modules.insert (m);
+      QStringList name_filters;
+      name_filters << QString::fromUtf8 ("*.klp");
+
+      QStringList inst_modules = QDir (*p).entryList (name_filters);
+      inst_modules.sort ();
+
+      for (QStringList::const_iterator im = inst_modules.begin (); im != inst_modules.end (); ++im) {
+        QFileInfo klp_file (*p, *im);
+        if (klp_file.exists () && klp_file.isReadable ()) {
+          std::string m = tl::to_string (klp_file.absoluteFilePath ());
+          if (modules.find (m) == modules.end ()) {
+            load_plugin (m);
+            modules.insert (m);
+          }
         }
       }
+
     }
 
   }

src/lay/layMacroController.cc

@@ -72,6 +72,10 @@ MacroController::load ()
       }
     }
 
+    for (tl::Registrar<gsi::Interpreter>::iterator i = gsi::interpreters.begin (); i != gsi::interpreters.end (); ++i) {
+      i->add_package_location (p->path);
+    }
+
   }
 }
 
@@ -296,6 +300,7 @@ MacroController::sync_implicit_macros (bool ask_before_autorun)
   }
 
   std::vector<ExternalPathDescriptor> external_paths;
+  std::vector<std::string> package_locations;
 
   //  Add additional places where the technologies define some macros
 
@@ -362,6 +367,8 @@ MacroController::sync_implicit_macros (bool ask_before_autorun)
 
       const lay::SaltGrain *g = *i;
 
+      package_locations.push_back (g->path ());
+
       for (size_t c = 0; c < macro_categories ().size (); ++c) {
 
         QDir base_dir (tl::to_qstring (g->path ()));
@@ -417,6 +424,23 @@ MacroController::sync_implicit_macros (bool ask_before_autorun)
     root->erase (*m);
   }
 
+  //  refresh the package locations by first removing the package locations and then rebuilding
+  //  TODO: maybe that is a performance bottleneck, but right now, remove_package_location doesn't do a lot.
+
+  for (std::vector<std::string>::const_iterator p = m_package_locations.begin (); p != m_package_locations.end (); ++p) {
+    for (tl::Registrar<gsi::Interpreter>::iterator i = gsi::interpreters.begin (); i != gsi::interpreters.end (); ++i) {
+      i->remove_package_location (*p);
+    }
+  }
+
+  m_package_locations = package_locations;
+
+  for (std::vector<std::string>::const_iterator p = m_package_locations.begin (); p != m_package_locations.end (); ++p) {
+    for (tl::Registrar<gsi::Interpreter>::iterator i = gsi::interpreters.begin (); i != gsi::interpreters.end (); ++i) {
+      i->add_package_location (*p);
+    }
+  }
+
   //  store new paths
   m_external_paths = external_paths;
 

src/lay/layMacroController.h

@@ -219,6 +219,7 @@ private:
   std::vector< std::pair<std::string, std::string> > m_macro_categories;
   std::vector<InternalPathDescriptor> m_internal_paths;
   std::vector<ExternalPathDescriptor> m_external_paths;
+  std::vector<std::string> m_package_locations;
   tl::FileSystemWatcher *m_file_watcher;
   tl::DeferredMethod<MacroController> dm_do_update_menu_with_macros;
   tl::DeferredMethod<MacroController> dm_do_sync_with_external_sources;

src/lay/laySaltTemplates.qrc

@@ -50,8 +50,8 @@
     <qresource prefix="/salt_templates/ruby_lib">
         <file alias="grain.xml">salt_templates/ruby_lib/grain.xml</file>
     </qresource>
-    <qresource prefix="/salt_templates/ruby_lib/macros">
-        <file alias="new_macro.lym">salt_templates/ruby_lib/macros/new_macro.lym</file>
+    <qresource prefix="/salt_templates/ruby_lib/ruby">
+        <file alias="new_object.rb">salt_templates/ruby_lib/ruby/new_object.rb</file>
     </qresource>
     <qresource prefix="/salt_templates/ruby_lib/doc">
         <file alias="readme.html">salt_templates/ruby_lib/doc/readme.html</file>
@@ -72,8 +72,8 @@
     <qresource prefix="/salt_templates/python_lib">
         <file alias="grain.xml">salt_templates/python_lib/grain.xml</file>
     </qresource>
-    <qresource prefix="/salt_templates/python_lib/pymacros">
-        <file alias="new_macro.lym">salt_templates/python_lib/pymacros/new_macro.lym</file>
+    <qresource prefix="/salt_templates/python_lib/python">
+        <file alias="new_object.py">salt_templates/python_lib/python/new_object.py</file>
     </qresource>
     <qresource prefix="/salt_templates/python_lib/doc">
         <file alias="readme.html">salt_templates/python_lib/doc/readme.html</file>

src/lay/salt_templates/python_lib/doc/readme.html

@@ -21,11 +21,10 @@ Here is what you should do:
 </ul>
 
 <p>
-Of course, the most interesting thing is how to add, edit and develop classes within
-your python library package. When the package was initialized, a "pymacros" folder with a single
-sample macro has been created. You will find this folder in the macro editor
-under the name you have given the package. The name of the sample macro is "new_macro".
-You can add more macros, Python and other files there or modify the sample macro.
+The Python library is a standalone, native library. It can be used through "import"
+from other Python code. It is not embedded into KLayout's macro framework, so you 
+have to edit it with your favorite text editor. Python will look for files in the
+"python" subfolder of the package.
 </p>
 
 <p>
@@ -43,4 +42,3 @@ Salt Mine server.
 
 </body>
 </html>
-

src/lay/salt_templates/python_lib/pymacros/new_macro.lym

@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<klayout-macro>
- <description>The New Macro</description>
- <prolog/>
- <epilog/>
- <autorun>false</autorun>
- <autorun-early>true</autorun-early>
- <shortcut/>
- <show-in-menu>false</show-in-menu>
- <group-name/>
- <menu-path/>
- <interpreter>python</interpreter>
- <dsl-interpreter-name/>
- <text># This is the new macro created with the sample Python library package
-
-class NewSampleLibraryClass(object):
-
-  def __init__(self):
-    # TODO: add your code here
-    pass
-
-# In order pull in classes from other packages, just specify these classes
-# in the dependencies of this package. Provided those packages contain macros
-# which are marked as "autorun-early", they will be loaded before this package
-# and their modules and classes will become available.
- </text>
-</klayout-macro>

src/lay/salt_templates/python_lib/python/new_object.py

@@ -0,0 +1,6 @@
+class NewSampleLibraryClass(object):
+
+  def __init__(self):
+    # TODO: add your code here
+    pass
+

src/lay/salt_templates/ruby_lib/doc/readme.html

@@ -21,11 +21,10 @@ Here is what you should do:
 </ul>
 
 <p>
-Of course, the most interesting thing is how to add, edit and develop classes within
-your python library package. When the package was initialized, a "macros" folder with a single
-sample macro has been created. You will find this folder in the macro editor
-under the name you have given the package. The name of the sample macro is "new_macro".
-You can add more macros, Ruby and other files there or modify the sample macro.
+The Ruby library is a standalone, native library. It can be used through "require"
+from other Ruby code. It is not embedded into KLayout's macro framework, so you 
+have to edit it with your favorite text editor. Ruby will look for files in the
+"ruby" subfolder of the package.
 </p>
 
 <p>

src/lay/salt_templates/ruby_lib/macros/new_macro.lym

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<klayout-macro>
- <description>The New Macro</description>
- <prolog/>
- <epilog/>
- <autorun>false</autorun>
- <autorun-early>true</autorun-early>
- <shortcut/>
- <show-in-menu>false</show-in-menu>
- <group-name/>
- <menu-path/>
- <interpreter>ruby</interpreter>
- <dsl-interpreter-name/>
- <text># This is the new macro created with the sample Ruby library package
-
-class NewSampleLibraryClass
-
-  def initialize
-    # TODO: add your code here
-  end
-
-end
-
-# In order pull in classes from other packages, just specify these classes
-# in the dependencies of this package. Provided those packages contain macros
-# which are marked as "autorun-early", they will be loaded before this package
-# and their modules and classes will become available.
- </text>
-</klayout-macro>

src/lay/salt_templates/ruby_lib/ruby/new_object.rb

@@ -0,0 +1,7 @@
+class NewSampleLibraryClass
+
+  def initialize
+    # TODO: add your code here
+  end
+
+end

src/pya/pya.cc

@@ -2975,6 +2975,22 @@ PythonInterpreter::add_path (const std::string &p)
   }
 }
 
+void
+PythonInterpreter::add_package_location (const std::string &package_path)
+{
+  std::string path = tl::to_string (QDir (tl::to_qstring (package_path)).absoluteFilePath (QString::fromUtf8 ("python")));
+  if (QDir (tl::to_qstring (path)).exists () && m_package_paths.find (path) == m_package_paths.end ()) {
+    m_package_paths.insert (path);
+    add_path (path);
+  }
+}
+
+void
+PythonInterpreter::remove_package_location (const std::string & /*package_path*/)
+{
+  //  Currently, we do not really remove the location. Python might get screwed up this way.
+}
+
 void
 PythonInterpreter::require (const std::string & /*filename*/)
 {

src/pya/pya.h

@@ -33,6 +33,7 @@
 
 #include <list>
 #include <string>
+#include <set>
 
 struct _typeobject;
 typedef _typeobject PyTypeObject;
@@ -110,6 +111,16 @@ public:
    */
   void add_path (const std::string &path);
 
+  /**
+   *  @brief Adds a package location to this interpreter
+   */
+  void add_package_location (const std::string &package_path);
+
+  /**
+   *  @brief Removes a package location from this interpreter
+   */
+  void remove_package_location (const std::string &package_path);
+
   /**
    *  @brief Requires the given module
    */
@@ -263,6 +274,7 @@ private:
   std::list<PythonRef> m_object_heap;
   std::list<std::string> m_string_heap;
   std::map<const gsi::MethodBase *, std::string> m_python_doc;
+  std::set<std::string> m_package_paths;
   std::vector<PyMethodDef *> m_methods_heap;
   std::vector<PyGetSetDef *> m_getseters_heap;
   PythonRef m_stdout_channel, m_stderr_channel;

src/pyastub/pya.cc

@@ -47,6 +47,18 @@ PythonInterpreter::add_path (const std::string &)
   // .. nothing ..
 }
 
+void
+PythonInterpreter::add_package_location (const std::string &)
+{
+  // .. nothing ..
+}
+
+void
+PythonInterpreter::remove_package_location (const std::string &)
+{
+  // .. nothing ..
+}
+
 void
 PythonInterpreter::require (const std::string &)
 {

src/pyastub/pya.h

@@ -50,6 +50,16 @@ public:
    */
   void add_path (const std::string &path);
 
+  /**
+   *  @brief Adds a package location to this interpreter
+   */
+  void add_package_location (const std::string &package_path);
+
+  /**
+   *  @brief Removes a package location from this interpreter
+   */
+  void remove_package_location (const std::string &package_path);
+
   /**
    *  @brief Requires the given module
    */

src/rba/rba.cc

@@ -49,6 +49,7 @@
 
 #include <QString>
 #include <QByteArray>
+#include <QDir>
 
 #if !defined(HAVE_RUBY_VERSION_CODE)
 #  define HAVE_RUBY_VERSION_CODE 10901
@@ -1336,6 +1337,7 @@ struct RubyInterpreterPrivateData
   std::string debugger_scope;
   std::map<const char *, size_t> file_id_map;
   std::vector<gsi::ExecutionHandler *> exec_handlers;
+  std::set<std::string> package_paths;
 };
 
 static RubyInterpreter *sp_rba_interpreter = 0;
@@ -1754,7 +1756,23 @@ RubyInterpreter::ignore_next_exception ()
   }
 }
 
-void 
+void
+RubyInterpreter::add_package_location (const std::string &package_path)
+{
+  std::string path = tl::to_string (QDir (tl::to_qstring (package_path)).absoluteFilePath (QString::fromUtf8 ("ruby")));
+  if (QDir (tl::to_qstring (path)).exists () && d->package_paths.find (path) == d->package_paths.end ()) {
+    d->package_paths.insert (path);
+    add_path (path);
+  }
+}
+
+void
+RubyInterpreter::remove_package_location (const std::string & /*package_path*/)
+{
+  //  Currently, we do not really remove the location. Ruby might get screwed up this way.
+}
+
+void
 RubyInterpreter::add_path (const std::string &path)
 {
   VALUE pv = rb_gv_get ("$:");

src/rba/rba.h

@@ -78,6 +78,16 @@ public:
    */
   void add_path (const std::string &path);
 
+  /**
+   *  @brief Adds a package location to this interpreter
+   */
+  void add_package_location (const std::string &package_path);
+
+  /**
+   *  @brief Removes a package location from this interpreter
+   */
+  void remove_package_location (const std::string &package_path);
+
   /**
    *  @brief Requires the given module (ruby "require")
    */

src/rbastub/rba.cc

@@ -40,7 +40,19 @@ RubyInterpreter::~RubyInterpreter ()
   // .. nothing ..
 }
 
-void 
+void
+RubyInterpreter::add_package_location (const std::string &)
+{
+  // .. nothing ..
+}
+
+void
+RubyInterpreter::remove_package_location (const std::string &)
+{
+  // .. nothing ..
+}
+
+void
 RubyInterpreter::add_path (const std::string &)
 {
   // .. nothing ..

src/rbastub/rba.h

@@ -47,6 +47,16 @@ public:
    */
   void add_path (const std::string &path);
 
+  /**
+   *  @brief Adds a package location to this interpreter
+   */
+  void add_package_location (const std::string &package_path);
+
+  /**
+   *  @brief Removes a package location from this interpreter
+   */
+  void remove_package_location (const std::string &package_path);
+
   /**
    *  @brief Requires the given module (ruby "require")
    */

src/tl/tl.pro

@@ -42,7 +42,8 @@ SOURCES = \
   tlXMLWriter.cc \
     tlFileSystemWatcher.cc \
     tlFileUtils.cc \
-    tlWebDAV.cc
+    tlWebDAV.cc \
+    tlArch.cc
 
 HEADERS = \
   tlAlgorithm.h \
@@ -88,7 +89,8 @@ HEADERS = \
     tlMath.h \
     tlCpp.h \
     tlFileUtils.h \
-    tlWebDAV.h
+    tlWebDAV.h \
+    tlArch.h
 
 INCLUDEPATH =
 DEPENDPATH =

src/tl/tlArch.cc

@@ -0,0 +1,65 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2017 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#include "tlArch.h"
+
+namespace tl
+{
+
+std::string
+arch_string ()
+{
+
+#if defined(_WIN32) || defined(_WIN64)
+# if defined(_WIN64)
+#   if defined(_MSC_VER)
+  return "x86_64-win32-msvc";
+#   elif defined(__MINGW32__)
+  return "x86_64-win32-mingw";
+#   endif
+# else
+#   if defined(_MSC_VER)
+  return "i686-win32-msvc";
+#   elif defined(__MINGW32__)
+  return "i686-win32-mingw";
+#   endif
+# endif
+#elif defined(__clang__)
+# if defined(__x86_64__)
+  return "x86_64-linux-clang";
+# else
+  return "i686-linux-clang";
+# endif
+#elif defined(__GNUC__)
+# if defined(__x86_64__)
+  return "x86_64-linux-gcc";
+# else
+  return "i686-linux-gcc";
+# endif
+#else
+  return "";
+#endif
+
+}
+
+}
+

src/tl/tlArch.h

@@ -0,0 +1,45 @@
+
+/*
+
+  KLayout Layout Viewer
+  Copyright (C) 2006-2017 Matthias Koefferlein
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+
+#ifndef HDR_tlArch
+#define HDR_tlArch
+
+#include "tlCommon.h"
+
+#include <string>
+
+namespace tl
+{
+
+/**
+ *  @brief Returns the architecture string
+ *
+ *  The architecture string is made from the cpu, os and compiler.
+ *  For example: i686-win32-mingw or x86_64-linux-gcc.
+ */
+TL_PUBLIC std::string arch_string ();
+
+}
+
+#endif
+

src/tl/tlClassRegistry.h

@@ -82,10 +82,10 @@ class RegisteredClass
 {
 public:
   /** 
-   *  @brief register an object Y in space X (Y must be derived from X)
+   *  @brief register an object of type X
    *
-   *  This will register the given object in X space. The Y pointer 
-   *  will become owned by the registrar.
+   *  This will register the given object. The X pointer
+   *  will become owned by the registrar if "owned" is true.
    *  The position parameter tells where to insert the class in the
    *  chain - higher positions come later.
    *  The name is an arbitrary string that is used for debugging purposes only.
@@ -99,7 +99,7 @@ public:
     mp_node = Registrar<X>::instance->insert (inst, owned, position, name);
     
     if (tl::verbosity () >= 40) {
-      tl::info << "Registered plugin '" << name << "' with priority " << position;
+      tl::info << "Registered object '" << name << "' with priority " << position;
     }
   }