Merge branch 'pymod' into net-extract

scripts/makedeb.sh

@@ -107,6 +107,10 @@ cp -pd $bininstdir/db_plugins/lib*so* makedeb-tmp/${libdir}/db_plugins
 cp -pd $bininstdir/lay_plugins/lib*so* makedeb-tmp/${libdir}/lay_plugins
 cp -pd $bininstdir/pymod/klayout/*so makedeb-tmp/${pylibdir}
 cp -pd $bininstdir/pymod/klayout/*py makedeb-tmp/${pylibdir}
+for d in db tl rdb; do
+  mkdir -p makedeb-tmp/${pylibdir}/$d
+  cp -pd $bininstdir/pymod/klayout/$d/*py makedeb-tmp/${pylibdir}/$d
+done
 
 cd makedeb-tmp
 

scripts/rpm-data/klayout.spec

@@ -130,22 +130,34 @@ strip %{_builddir}/bin.$TARGET/strm*
 
 TARGET="linux-release"
 
+# create and populate pylib
 mkdir -p %{buildroot}%{pylib}/klayout
+cp -pd %{_builddir}/bin.$TARGET/pymod/klayout/*.so %{buildroot}%{pylib}/klayout
+cp -pd %{_builddir}/bin.$TARGET/pymod/klayout/*.py %{buildroot}%{pylib}/klayout
+chmod 644 %{buildroot}%{pylib}/klayout/*
+for d in tl db rdb; do
+  mkdir -p %{buildroot}%{pylib}/klayout/$d
+  cp -pd %{_builddir}/bin.$TARGET/pymod/klayout/$d/*.py %{buildroot}%{pylib}/klayout/$d
+  chmod 644 %{buildroot}%{pylib}/klayout/$d/*
+done
+
+# create and populate libdir
 mkdir -p %{buildroot}%{_libdir}/klayout
 mkdir -p %{buildroot}%{_libdir}/klayout/db_plugins
 mkdir -p %{buildroot}%{_libdir}/klayout/lay_plugins
-mkdir -p %{buildroot}%{_bindir}
-cp -pd %{_builddir}/bin.$TARGET/pymod/klayout/*.so %{buildroot}%{pylib}/klayout
-cp -pd %{_builddir}/bin.$TARGET/pymod/klayout/*.py %{buildroot}%{pylib}/klayout
 cp -pd %{_builddir}/bin.$TARGET/lib*.so* %{buildroot}%{_libdir}/klayout
 cp -pd %{_builddir}/bin.$TARGET/db_plugins/lib*.so* %{buildroot}%{_libdir}/klayout/db_plugins
 cp -pd %{_builddir}/bin.$TARGET/lay_plugins/lib*.so* %{buildroot}%{_libdir}/klayout/lay_plugins
-chmod 644 %{buildroot}%{pylib}/klayout/*
 chmod 644 %{buildroot}%{_libdir}/klayout/*.so*
 chmod 644 %{buildroot}%{_libdir}/klayout/db_plugins/*.so*
 chmod 644 %{buildroot}%{_libdir}/klayout/lay_plugins/*.so*
+
+# create and populate bindir
+mkdir -p %{buildroot}%{_bindir}
 cp -pd %{_builddir}/bin.$TARGET/klayout %{_builddir}/bin.$TARGET/strm* %{buildroot}%{_bindir}
 chmod 755 %{buildroot}%{_bindir}/*
+
+# other files
 install -Dm644 %{_sourcedir}/etc/%{name}.desktop %{buildroot}%{_datadir}/applications/%{name}.desktop
 install -Dm644 %{_sourcedir}/etc/logo.png %{buildroot}%{_datadir}/pixmaps/%{name}.png
 

setup.py

@@ -54,13 +54,15 @@ and won't find them. So we need to take away the path with
 "-Wl,-soname" on Linux (see Config.link_args).
 """
 
-from setuptools import setup, Extension, Distribution
+from setuptools import setup, Distribution, find_packages
+from setuptools.extension import Extension, Library
 import glob
 import os
 import platform
 import distutils.sysconfig as sysconfig
 from distutils.errors import CompileError
 import distutils.command.build_ext
+import setuptools.command.build_ext
 import multiprocessing
 N_cores = multiprocessing.cpu_count()
 
@@ -104,6 +106,8 @@ if sys.version_info[0] * 10 + sys.version_info[1] > 26:
     distutils.ccompiler.CCompiler.compile = parallelCCompile
 
 
+# TODO: delete (Obsolete)
+# patch get_ext_filename
 from distutils.command.build_ext import build_ext
 _old_get_ext_filename = build_ext.get_ext_filename
 
@@ -121,6 +125,50 @@ def patched_get_ext_filename(self, ext_name):
 
 
 distutils.command.build_ext.build_ext.get_ext_filename = patched_get_ext_filename
+
+# end patch get_ext_filename
+
+
+# patch CCompiler's library_filename for _dbpi libraries (SOs)
+# Its default is to have .so for shared objects, instead of dylib,
+# hence the patch
+
+from distutils.ccompiler import CCompiler
+old_library_filename = CCompiler.library_filename
+
+
+def patched_library_filename(self, libname, lib_type='static',     # or 'shared'
+                     strip_dir=0, output_dir=''):
+    if platform.system() == "Darwin" and '_dbpi' in libname:
+        lib_type = 'dylib'
+    return old_library_filename(self, libname, lib_type=lib_type,
+        strip_dir=strip_dir, output_dir=output_dir)
+
+
+distutils.ccompiler.CCompiler.library_filename = patched_library_filename
+
+# end patch CCompiler's library_filename for _dbpi libraries (SOs)
+
+
+# despite its name, setuptools.command.build_ext.link_shared_object won't
+# link a shared object on Linux, but a static library and patches distutils
+# for this ... We're patching this back now.
+
+def always_link_shared_object(
+    self, objects, output_libname, output_dir=None, libraries=None,
+    library_dirs=None, runtime_library_dirs=None, export_symbols=None,
+    debug=0, extra_preargs=None, extra_postargs=None, build_temp=None,
+    target_lang=None):
+    self.link(
+        self.SHARED_LIBRARY, objects, output_libname,
+        output_dir, libraries, library_dirs, runtime_library_dirs,
+        export_symbols, debug, extra_preargs, extra_postargs,
+        build_temp, target_lang
+    )
+
+setuptools.command.build_ext.libtype = "shared"
+setuptools.command.build_ext.link_shared_object = always_link_shared_object
+
 # ----------------------------------------------------------------------------------------
 
 
@@ -137,36 +185,76 @@ class Config(object):
         build_cmd = Distribution().get_command_obj('build')
         build_cmd.finalize_options()
         self.build_platlib = build_cmd.build_platlib
+        self.build_temp = build_cmd.build_temp
+        if platform.system() == "Windows":
+            # Windows uses separate temp build folders for debug and release
+            if build_cmd.debug:
+                self.build_temp = os.path.join(self.build_temp, "Debug")
+            else:
+                self.build_temp = os.path.join(self.build_temp, "Release")
 
-        self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
+        build_ext_cmd = Distribution().get_command_obj('build_ext')
 
-        if self.ext_suffix is None:
+        build_ext_cmd.initialize_options()
-            self.ext_suffix = ".so"
+        build_ext_cmd.setup_shlib_compiler()
+        self.build_ext_cmd = build_ext_cmd
 
         self.root = "klayout"
 
-    def libname_of(self, mod):
+    def add_extension(self, ext):
+        self.build_ext_cmd.ext_map[ext.name] = ext
+
+    def libname_of(self, mod, is_lib=None):
         """
         Returns the library name for a given module
         The library name is usually decorated (i.e. "tl" -> "tl.cpython-35m-x86_64-linux-gnu.so").
+        This code was extracted from the source in setuptools.command.build_ext.build_ext
         """
-        ext_suffix = self.ext_suffix
+        libtype = setuptools.command.build_ext.libtype
-        if platform.system() == "Darwin" and '_dbpi' in mod:
+        full_mod = self.root + '.' + mod
-            ext_suffix = ext_suffix.replace('.so', '.dylib')
+        if is_lib is True:
-        return mod + ext_suffix
+            # Here we are guessing it is a library and that the filename
+            # is to be computed by shlib_compiler.
+            # See source code of setuptools.command.build_ext.build_ext.get_ext_filename
+            filename = self.build_ext_cmd.get_ext_filename(full_mod)
+            fn, ext = os.path.splitext(filename)
+            ext_path = self.build_ext_cmd.shlib_compiler.library_filename(fn, libtype)
+        else:
+            # This assumes that the Extension/Library object was added to the
+            # ext_map dictionary via config.add_extension.
+            assert full_mod in self.build_ext_cmd.ext_map
+            ext_path = self.build_ext_cmd.get_ext_filename(full_mod)
+        ext_filename = os.path.basename(ext_path)
 
-    def path_of(self, mod):
+        # Exception for database plugins, which will always be dylib.
+        if platform.system() == "Darwin" and '_dbpi' in mod:
+            ext_filename = ext_filename.replace('.so', '.dylib')
+        return ext_filename
+
+    def path_of(self, mod, mod_src_path):
         """
         Returns the build path of the library for a given module
         """
-        return os.path.join(self.build_platlib, self.root, self.libname_of(mod))
+        if platform.system() == "Windows":
+            # On Windows, the library to link is the import library
+            (dll_name, dll_ext) = os.path.splitext(self.libname_of(mod))
+            return os.path.join(self.build_temp, mod_src_path, dll_name + ".lib")
+        else:
+            return os.path.join(self.build_platlib, self.root, self.libname_of(mod))
 
     def compile_args(self, mod):
         """
         Gets additional compiler arguments
         """
         if platform.system() == "Windows":
-            return []
+            bits = os.getenv("KLAYOUT_BITS")
+            if bits:
+                return ["\"-I" + os.path.join(bits, "zlib", "include") + "\"",
+                        "\"-I" + os.path.join(bits, "ptw", "include") + "\"",
+                        "\"-I" + os.path.join(bits, "expat", "include") + "\"",
+                        "\"-I" + os.path.join(bits, "curl", "include") + "\""]
+            else:
+                return []
         elif platform.system() == "Darwin":
             return []
         else:
@@ -174,18 +262,37 @@ class Config(object):
                     "-std=c++11",  # because we use unordered_map/unordered_set
                     ]
 
+    def libraries(self, mod):
+        """
+        Gets the libraries to add
+        """
+        if platform.system() == "Windows":
+            if mod == "_tl":
+                return [ "libcurl", "expat", "pthreadVCE2", "zlib", "wsock32" ]
+        else:
+            if mod == "_tl":
+                return ['curl', 'expat']
+        return []
+
     def link_args(self, mod):
         """
         Gets additional linker arguments
         """
         if platform.system() == "Windows":
-            return []
+            args = ["/DLL"]
+            bits = os.getenv("KLAYOUT_BITS")
+            if bits:
+                args += ["\"/LIBPATH:" + os.path.join(bits, "zlib", "libraries") + "\"",
+                         "\"/LIBPATH:" + os.path.join(bits, "ptw", "libraries") + "\"",
+                         "\"/LIBPATH:" + os.path.join(bits, "expat", "libraries") + "\"",
+                         "\"/LIBPATH:" + os.path.join(bits, "curl", "libraries") + "\""]
+            return args
         elif platform.system() == "Darwin":
             # For the dependency modules, make sure we produce a dylib.
             # We can only link against such, but the bundles produced otherwise.
             args = []
             if mod[0] == "_":
-                args += ["-Wl,-dylib", '-Wl,-install_name,@rpath/%s' % self.libname_of(mod)]
+                args += ["-Wl,-dylib", '-Wl,-install_name,@rpath/%s' % self.libname_of(mod, is_lib=True)]
             args += ['-Wl,-rpath,@loader_path/']
             return args
         else:
@@ -195,7 +302,7 @@ class Config(object):
             # will look for the path-qualified library. But that's the
             # build path and the loader will fail.
             args = []
-            args += ['-Wl,-soname,' + self.libname_of(mod)]
+            args += ['-Wl,-soname,' + self.libname_of(mod, is_lib=True)]
             if '_dbpi' not in mod:
                 loader_path = '$ORIGIN'
             else:
@@ -213,7 +320,7 @@ class Config(object):
         """
         Gets the version string
         """
-        return "0.26.0.dev4"
+        return "0.26.0.dev7"
 
 
 config = Config()
@@ -222,7 +329,6 @@ config = Config()
 # _tl dependency library
 
 _tl_path = os.path.join("src", "tl", "tl")
-
 _tl_sources = set(glob.glob(os.path.join(_tl_path, "*.cc")))
 
 # Exclude sources which are compatible with Qt only
@@ -233,41 +339,47 @@ _tl_sources.discard(os.path.join(_tl_path, "tlHttpStreamNoQt.cc"))
 _tl_sources.discard(os.path.join(_tl_path, "tlFileSystemWatcher.cc"))
 _tl_sources.discard(os.path.join(_tl_path, "tlDeferredExecutionQt.cc"))
 
-_tl = Extension(config.root + '._tl',
+_tl = Library(config.root + '._tl',
-                define_macros=config.macros() + [('MAKE_TL_LIBRARY', 1)],
+              define_macros=config.macros() + [('MAKE_TL_LIBRARY', 1)],
-                language='c++',
+              language='c++',
-                libraries=['curl', 'expat'],
+              libraries=config.libraries('_tl'),
-                extra_link_args=config.link_args('_tl'),
+              extra_link_args=config.link_args('_tl'),
-                extra_compile_args=config.compile_args('_tl'),
+              extra_compile_args=config.compile_args('_tl'),
-                sources=list(_tl_sources))
+              sources=list(_tl_sources))
+
+config.add_extension(_tl)
 
 # ------------------------------------------------------------------
 # _gsi dependency library
 
-_gsi_sources = glob.glob("src/gsi/gsi/*.cc")
+_gsi_path = os.path.join("src", "gsi", "gsi")
+_gsi_sources = set(glob.glob(os.path.join(_gsi_path, "*.cc")))
 
-_gsi = Extension(config.root + '._gsi',
+_gsi = Library(config.root + '._gsi',
-                 define_macros=config.macros() + [('MAKE_GSI_LIBRARY', 1)],
+               define_macros=config.macros() + [('MAKE_GSI_LIBRARY', 1)],
-                 include_dirs=['src/tl/tl'],
+               include_dirs=[_tl_path],
-                 extra_objects=[config.path_of('_tl')],
+               extra_objects=[config.path_of('_tl', _tl_path)],
-                 language='c++',
+               language='c++',
-                 extra_link_args=config.link_args('_gsi'),
+               extra_link_args=config.link_args('_gsi'),
-                 extra_compile_args=config.compile_args('_gsi'),
+               extra_compile_args=config.compile_args('_gsi'),
-                 sources=_gsi_sources)
+               sources=list(_gsi_sources))
+config.add_extension(_gsi)
 
 # ------------------------------------------------------------------
 # _pya dependency library
 
-_pya_sources = glob.glob("src/pya/pya/*.cc")
+_pya_path = os.path.join("src", "pya", "pya")
+_pya_sources = set(glob.glob(os.path.join(_pya_path, "*.cc")))
 
-_pya = Extension(config.root + '._pya',
+_pya = Library(config.root + '._pya',
-                 define_macros=config.macros() + [('MAKE_PYA_LIBRARY', 1)],
+               define_macros=config.macros() + [('MAKE_PYA_LIBRARY', 1)],
-                 include_dirs=['src/tl/tl', 'src/gsi/gsi'],
+               include_dirs=[_tl_path, _gsi_path],
-                 extra_objects=[config.path_of('_tl'), config.path_of('_gsi')],
+               extra_objects=[config.path_of('_tl', _tl_path), config.path_of('_gsi', _gsi_path)],
-                 language='c++',
+               language='c++',
-                 extra_link_args=config.link_args('_pya'),
+               extra_link_args=config.link_args('_pya'),
-                 extra_compile_args=config.compile_args('_pya'),
+               extra_compile_args=config.compile_args('_pya'),
-                 sources=_pya_sources)
+               sources=list(_pya_sources))
+config.add_extension(_pya)
 
 # ------------------------------------------------------------------
 # _db dependency library
@@ -280,93 +392,98 @@ _db_sources = set(glob.glob(os.path.join(_db_path, "*.cc")))
 # not exist. So we need an error-free discard method instead of list's remove.
 _db_sources.discard(os.path.join(_db_path, "fonts.cc"))
 
-_db = Extension(config.root + '._db',
+_db = Library(config.root + '._db',
-                define_macros=config.macros() + [('MAKE_DB_LIBRARY', 1)],
+              define_macros=config.macros() + [('MAKE_DB_LIBRARY', 1)],
-                include_dirs=['src/tl/tl', 'src/gsi/gsi', 'src/db/db'],
+              include_dirs=[_tl_path, _gsi_path, _db_path],
-                extra_objects=[config.path_of('_tl'), config.path_of('_gsi')],
+              extra_objects=[config.path_of('_tl', _tl_path), config.path_of('_gsi', _gsi_path)],
-                language='c++',
+              language='c++',
-                extra_link_args=config.link_args('_db'),
+              extra_link_args=config.link_args('_db'),
-                extra_compile_args=config.compile_args('_db'),
+              extra_compile_args=config.compile_args('_db'),
-                sources=list(_db_sources))
+              sources=list(_db_sources))
+config.add_extension(_db)
 
 # ------------------------------------------------------------------
 # _rdb dependency library
 
-_rdb_sources = glob.glob("src/rdb/rdb/*.cc")
+_rdb_path = os.path.join("src", "rdb", "rdb")
+_rdb_sources = set(glob.glob(os.path.join(_rdb_path, "*.cc")))
 
-_rdb = Extension(config.root + '._rdb',
+_rdb = Library(config.root + '._rdb',
-                 define_macros=config.macros() + [('MAKE_RDB_LIBRARY', 1)],
+               define_macros=config.macros() + [('MAKE_RDB_LIBRARY', 1)],
-                 include_dirs=['src/db/db', 'src/tl/tl', 'src/gsi/gsi'],
+               include_dirs=[_db_path, _tl_path, _gsi_path],
-                 extra_objects=[config.path_of('_tl'), config.path_of(
+               extra_objects=[config.path_of('_tl', _tl_path), config.path_of('_gsi', _gsi_path), config.path_of('_db', _db_path)],
-                     '_gsi'), config.path_of('_db')],
+               language='c++',
-                 language='c++',
+               extra_link_args=config.link_args('_rdb'),
-                 extra_link_args=config.link_args('_rdb'),
+               extra_compile_args=config.compile_args('_rdb'),
-                 extra_compile_args=config.compile_args('_rdb'),
+               sources=list(_rdb_sources))
-                 sources=_rdb_sources)
+config.add_extension(_rdb)
 
 # ------------------------------------------------------------------
 # dependency libraries from db_plugins
 
 db_plugins = []
 
-for pi in glob.glob("src/plugins/*/db_plugin") + glob.glob("src/plugins/*/*/db_plugin"):
+dbpi_dirs = glob.glob(os.path.join("src", "plugins", "*", "db_plugin"))
+dbpi_dirs += glob.glob(os.path.join("src", "plugins", "*", "*", "db_plugin"))
+
+for pi in dbpi_dirs:
 
     mod_name = "_" + os.path.split(os.path.split(pi)[-2])[-1] + "_dbpi"
 
-    pi_sources = glob.glob(pi + "/*.cc")
+    pi_sources = glob.glob(os.path.join(pi, "*.cc"))
 
-    pi_ext = Extension(config.root + '.db_plugins.' + mod_name,
+    pi_ext = Library(config.root + '.db_plugins.' + mod_name,
-                       define_macros=config.macros() + [('MAKE_DB_PLUGIN_LIBRARY', 1)],
+                     define_macros=config.macros() + [('MAKE_DB_PLUGIN_LIBRARY', 1)],
-                       include_dirs=['src/plugins/common',
+                     include_dirs=[os.path.join("src", "plugins", "common"),
-                                     'src/db/db', 'src/tl/tl', 'src/gsi/gsi'],
+                                   _db_path, _tl_path, _gsi_path],
-                       extra_objects=[config.path_of('_tl'), config.path_of(
+                     extra_objects=[config.path_of('_tl', _tl_path), config.path_of('_gsi', _gsi_path), config.path_of('_db', _db_path)],
-                           '_gsi'), config.path_of('_db')],
+                     language='c++',
-                       language='c++',
+                     extra_link_args=config.link_args(mod_name),
-                       extra_link_args=config.link_args(mod_name),
+                     extra_compile_args=config.compile_args(mod_name),
-                       extra_compile_args=config.compile_args(mod_name),
+                     sources=pi_sources)
-                       sources=pi_sources)
 
     db_plugins.append(pi_ext)
+    config.add_extension(pi_ext)
 
 # ------------------------------------------------------------------
 # tl extension library
 
-tl_sources = glob.glob("src/pymod/tl/*.cc")
+tl_path = os.path.join("src", "pymod", "tl")
+tl_sources = set(glob.glob(os.path.join(tl_path, "*.cc")))
 
-tl = Extension(config.root + '.tl',
+tl = Extension(config.root + '.tlcore',
                define_macros=config.macros(),
-               include_dirs=['src/tl/tl', 'src/gsi/gsi', 'src/pya/pya'],
+               include_dirs=[_tl_path, _gsi_path, _pya_path],
-               extra_objects=[config.path_of('_tl'), config.path_of(
+               extra_objects=[config.path_of('_tl', _tl_path), config.path_of('_gsi', _gsi_path), config.path_of('_pya', _pya_path)],
-                   '_gsi'), config.path_of('_pya')],
+               extra_link_args=config.link_args('tlcore'),
-               extra_link_args=config.link_args('tl'),
+               sources=list(tl_sources))
-               sources=tl_sources)
 
 # ------------------------------------------------------------------
 # db extension library
 
-db_sources = glob.glob("src/pymod/db/*.cc")
+db_path = os.path.join("src", "pymod", "db")
+db_sources = set(glob.glob(os.path.join(db_path, "*.cc")))
 
-db = Extension(config.root + '.db',
+db = Extension(config.root + '.dbcore',
                define_macros=config.macros(),
-               include_dirs=['src/db/db', 'src/tl/tl', 'src/gsi/gsi', 'src/pya/pya'],
+               include_dirs=[_db_path, _tl_path, _gsi_path, _pya_path],
-               extra_objects=[config.path_of('_db'), config.path_of(
+               extra_objects=[config.path_of('_db', _db_path), config.path_of('_tl', _tl_path), config.path_of('_gsi', _gsi_path), config.path_of('_pya', _pya_path)],
-                   '_tl'), config.path_of('_gsi'), config.path_of('_pya')],
+               extra_link_args=config.link_args('dbcore'),
-               extra_link_args=config.link_args('db'),
+               sources=list(db_sources))
-               sources=db_sources)
 
 # ------------------------------------------------------------------
 # rdb extension library
 
-rdb_sources = glob.glob("src/pymod/rdb/*.cc")
+rdb_path = os.path.join("src", "pymod", "rdb")
+rdb_sources = set(glob.glob(os.path.join(rdb_path, "*.cc")))
 
-rdb = Extension(config.root + '.rdb',
+rdb = Extension(config.root + '.rdbcore',
                 define_macros=config.macros(),
-                include_dirs=['src/rdb/rdb', 'src/db/db',
+
-                              'src/tl/tl', 'src/gsi/gsi', 'src/pya/pya'],
+                include_dirs=[_rdb_path, _db_path, _tl_path, _gsi_path, _pya_path],
-                extra_objects=[config.path_of('_rdb'), config.path_of(
+                extra_objects=[config.path_of('_rdb', _rdb_path), config.path_of('_tl', _tl_path), config.path_of('_gsi', _gsi_path), config.path_of('_pya', _pya_path)],
-                    '_gsi'), config.path_of('_pya')],
+                extra_link_args=config.link_args('rdbcore'),
-                extra_link_args=config.link_args('rdb'),
+                sources=list(rdb_sources))
-                sources=rdb_sources)
 
 # ------------------------------------------------------------------
 # Core setup function
@@ -380,6 +497,6 @@ if __name__ == '__main__':
           author='Matthias Koefferlein',
           author_email='matthias@klayout.de',
           url='https://github.com/klayoutmatthias/klayout',
-          packages=[config.root],
+          packages=find_packages('src/pymod/distutils_src'),
-          package_dir={config.root: 'src/pymod/distutils_src'},
+          package_dir={'': 'src/pymod/distutils_src'},  # https://github.com/pypa/setuptools/issues/230
           ext_modules=[_tl, _gsi, _pya, _db, _rdb] + db_plugins + [tl, db, rdb])

src/pymod/db/db.pro

@@ -1,5 +1,6 @@
 
-TARGET = db
+TARGET = dbcore
+REALMODULE = db
 
 include($$PWD/../pymod.pri)
 

src/pymod/db/dbMain.cc

@@ -27,10 +27,10 @@
 //  to force linking of the db module
 #include "../../db/db/dbForceLink.h"
 
-static PyObject *db_module_init (const char *mod_name, const char *mod_description)
+static PyObject *db_module_init (const char *pymod_name, const char *mod_name, const char *mod_description)
 {
   db::init ();
-  return module_init (mod_name, mod_description);
+  return module_init (pymod_name, mod_name, mod_description);
 }
 
-DEFINE_PYMOD_WITH_INIT(db, "db", "KLayout core module 'db'", db_module_init)
+DEFINE_PYMOD_WITH_INIT(dbcore, "db", "KLayout core module 'db'", db_module_init)

src/pymod/distutils_src/__init__.py

@@ -1,5 +0,0 @@
-
-# klayout library definition file
-
-__all__ = [ "tl", "db", "lay", "rdb" ]
-

src/pymod/distutils_src/klayout/.gitignore

@@ -0,0 +1,9 @@
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+*.dylib

src/pymod/distutils_src/klayout/__init__.py

@@ -0,0 +1 @@
+# klayout library definition file

src/pymod/distutils_src/klayout/db/__init__.py

@@ -0,0 +1,6 @@
+import klayout.dbcore
+from klayout.dbcore import *
+
+from klayout.db.pcell_declaration_helper import PCellDeclarationHelper
+
+__all__ = klayout.dbcore.__all__ + ['PCellDeclarationHelper']

src/pymod/distutils_src/klayout/db/pcell_declaration_helper.py

@@ -0,0 +1,278 @@
+from klayout.db import Trans, PCellDeclaration, PCellParameterDeclaration
+
+
+class _PCellDeclarationHelperLayerDescriptor(object):
+    """
+    A descriptor object which translates the PCell parameters into class attributes
+    """
+
+    def __init__(self, param_index):
+        self.param_index = param_index
+
+    def __get__(self, obj, type=None):
+        return obj._layers[self.param_index]
+
+    def __set__(self, obj, value):
+        raise AttributeError("can't change layer attribute")
+
+
+class _PCellDeclarationHelperParameterDescriptor(object):
+    """
+    A descriptor object which translates the PCell parameters into class attributes
+
+    In some cases (i.e. can_convert_from_shape), these placeholders are not
+    connected to real parameters (obj._param_values is None). In this case,
+    the descriptor acts as a value holder (self.value)
+    """
+
+    def __init__(self, param_index):
+        self.param_index = param_index
+        self.value = None
+
+    def __get__(self, obj, type=None):
+        if obj._param_values:
+            return obj._param_values[self.param_index]
+        else:
+            return self.value
+
+    def __set__(self, obj, value):
+        if obj._param_values:
+            obj._param_values[self.param_index] = value
+        else:
+            self.value = value
+
+
+class _PCellDeclarationHelper(PCellDeclaration):
+    """
+    A helper class that somewhat simplifies the implementation
+    of a PCell
+    """
+
+    def __init__(self):
+        """
+        initialize this instance
+        """
+        # "private" attributes
+        self._param_decls = []
+        self._param_values = None
+        self._layer_param_index = []
+        self._layers = []
+        # public attributes
+        self.layout = None
+        self.shape = None
+        self.layer = None
+        self.cell = None
+
+    def param(self, name, value_type, description, hidden=False, readonly=False, unit=None, default=None, choices=None):
+        """
+        Defines a parameter
+          name         -> the short name of the parameter
+          type         -> the type of the parameter
+          description  -> the description text
+        named parameters
+          hidden      -> (boolean) true, if the parameter is not shown in the dialog
+          readonly    -> (boolean) true, if the parameter cannot be edited
+          unit        -> the unit string
+          default     -> the default value
+          choices     -> ([ [ d, v ], ...) choice descriptions/value for choice type
+        this method defines accessor methods for the parameters
+          {name}       -> read accessor
+          set_{name}   -> write accessor ({name}= does not work because the
+                          Ruby confuses that method with variables)
+          {name}_layer -> read accessor for the layer index for TypeLayer parameters
+        """
+
+        # create accessor methods for the parameters
+        param_index = len(self._param_decls)
+        setattr(type(self), name, _PCellDeclarationHelperParameterDescriptor(param_index))
+
+        if value_type == type(self).TypeLayer:
+            setattr(type(self), name + "_layer",
+                    _PCellDeclarationHelperLayerDescriptor(len(self._layer_param_index)))
+            self._layer_param_index.append(param_index)
+
+        # store the parameter declarations
+        pdecl = PCellParameterDeclaration(name, value_type, description)
+        self._param_decls.append(pdecl)
+
+        # set additional attributes of the parameters
+        pdecl.hidden = hidden
+        pdecl.readonly = readonly
+        if not (default is None):
+            pdecl.default = default
+        if not (unit is None):
+            pdecl.unit = unit
+        if not (choices is None):
+            if not isinstance(choices, list) and not isinstance(choices, tuple):
+                raise "choices value must be an list/tuple of two-element arrays (description, value)"
+            for c in choices:
+                if (not isinstance(choices, list) and not isinstance(choices, tuple)) or len(c) != 2:
+                    raise "choices value must be an list/tuple of two-element arrays (description, value)"
+                pdecl.add_choice(c[0], c[1])
+
+        # return the declaration object for further operations
+        return pdecl
+
+    def display_text(self, parameters):
+        """
+        implementation of display_text
+        """
+        self._param_values = parameters
+        text = self.display_text_impl()
+        self._param_values = None
+        return text
+
+    def get_parameters(self):
+        """
+        gets the parameters
+        """
+        return self._param_decls
+
+    def get_values(self):
+        """
+        gets the temporary parameter values
+        """
+        v = self._param_values
+        self._param_values = None
+        return v
+
+    def init_values(self, values=None, layers=None):
+        """
+        initializes the temporary parameter values
+        "values" are the original values. If "None" is given, the
+        default values will be used.
+        "layers" are the layer indexes corresponding to the layer
+        parameters.
+        """
+        if not values:
+            self._param_values = []
+            for pd in self._param_decls:
+                self._param_values.append(pd.default)
+        else:
+            self._param_values = values
+        self._layers = layers
+
+    def finish(self):
+        """
+        Needs to be called at the end of produce() after init_values was used
+        """
+        self._param_values = None
+        self._layers = None
+
+    def get_layers(self, parameters):
+        """
+        get the layer definitions
+        """
+        layers = []
+        for i in self._layer_param_index:
+            layers.append(parameters[i])
+        return layers
+
+    def coerce_parameters(self, layout, parameters):
+        """
+        coerce parameters (make consistent)
+        """
+        self.init_values(parameters)
+        self.layout = layout
+        self.coerce_parameters_impl()
+        self.layout = None
+        return self.get_values()
+
+    def produce(self, layout, layers, parameters, cell):
+        """
+        coerce parameters (make consistent)
+        """
+        self.init_values(parameters, layers)
+        self.cell = cell
+        self.layout = layout
+        self.produce_impl()
+        self.cell = None
+        self.layout = None
+        self.finish()
+
+    def can_create_from_shape(self, layout, shape, layer):
+        """
+        produce a helper for can_create_from_shape
+        """
+        self.layout = layout
+        self.shape = shape
+        self.layer = layer
+        ret = self.can_create_from_shape_impl()
+        self.layout = None
+        self.shape = None
+        self.layer = None
+        return ret
+
+    def transformation_from_shape(self, layout, shape, layer):
+        """
+        produce a helper for parameters_from_shape
+        """
+        self.layout = layout
+        self.shape = shape
+        self.layer = layer
+        t = self.transformation_from_shape_impl()
+        self.layout = None
+        self.shape = None
+        self.layer = None
+        return t
+
+    def parameters_from_shape(self, layout, shape, layer):
+        """
+        produce a helper for parameters_from_shape
+        with this helper, the implementation can use the parameter setters
+        """
+        self.init_values()
+        self.layout = layout
+        self.shape = shape
+        self.layer = layer
+        self.parameters_from_shape_impl()
+        param = self.get_values()
+        self.layout = None
+        self.shape = None
+        self.layer = None
+        return param
+
+    def display_text_impl(self):
+        """
+        default implementation
+        """
+        return ""
+
+    def coerce_parameters_impl(self):
+        """
+        default implementation
+        """
+        pass
+
+    def produce_impl(self):
+        """
+        default implementation
+        """
+        pass
+
+    def can_create_from_shape_impl(self):
+        """
+        default implementation
+        """
+        return False
+
+    def parameters_from_shape_impl(self):
+        """
+        default implementation
+        """
+        pass
+
+    def transformation_from_shape_impl(self):
+        """
+        default implementation
+        """
+        return Trans()
+
+
+# import the Type... constants from PCellParameterDeclaration
+for k in dir(PCellParameterDeclaration):
+    if k.startswith("Type"):
+        setattr(_PCellDeclarationHelper, k, getattr(PCellParameterDeclaration, k))
+
+# Inject the PCellDeclarationHelper into pya module for consistency:
+PCellDeclarationHelper = _PCellDeclarationHelper

src/pymod/distutils_src/klayout/rdb/__init__.py

@@ -0,0 +1,4 @@
+import klayout.rdbcore
+from klayout.rdbcore import *
+
+__all__ = klayout.rdbcore.__all__

src/pymod/distutils_src/klayout/tl/__init__.py

@@ -0,0 +1,4 @@
+import klayout.tlcore
+from klayout.tlcore import *
+
+__all__ = klayout.tlcore.__all__

src/pymod/pymod.pri

@@ -48,3 +48,25 @@ msvc {
   lib_target.extra = $(INSTALL_PROGRAM) $$DESTDIR_PYMOD/$${TARGET}$${PYTHONEXTSUFFIX} $(INSTALLROOT)$$PREFIX/pymod/klayout
 }
 INSTALLS = lib_target
+
+!equals(REALMODULE, "") {
+
+  msvc {
+    QMAKE_POST_LINK += && (if not exist $$shell_path($$DESTDIR_PYMOD/$$REALMODULE) $(MKDIR) $$shell_path($$DESTDIR_PYMOD/$$REALMODULE)) && $(COPY) $$shell_path($$PWD/distutils_src/klayout/$$REALMODULE/*.py) $$shell_path($$DESTDIR_PYMOD/$$REALMODULE)
+  } else {
+    QMAKE_POST_LINK += && $(MKDIR) $$DESTDIR_PYMOD/$$REALMODULE && $(COPY) $$PWD/distutils_src/klayout/$$REALMODULE/*.py $$DESTDIR_PYMOD/$$REALMODULE
+  }
+
+  # INSTALLS needs to be inside a lib or app templates.
+  modsrc_target.path = $$PREFIX/pymod/klayout/$$REALMODULE
+  # This would be nice:
+  #   init_target.files += $$DESTDIR_PYMOD/$$REALMODULE/*
+  # but some Qt versions need this explicitly:
+  msvc {
+    modsrc_target.extra = $(INSTALL_PROGRAM) $$shell_path($$DESTDIR_PYMOD/$$REALMODULE/*.py) $$shell_path($(INSTALLROOT)$$PREFIX/pymod/klayout/$$REALMODULE)
+  } else {
+    modsrc_target.extra = $(INSTALL_PROGRAM) $$DESTDIR_PYMOD/$$REALMODULE/*.py $(INSTALLROOT)$$PREFIX/pymod/klayout/$$REALMODULE
+  }
+  INSTALLS += modsrc_target
+
+}

src/pymod/pymodHelper.h

@@ -39,7 +39,7 @@
 #include "gsiExpression.h"
 
 static PyObject *
-module_init (const char *mod_name, const char *mod_description)
+module_init (const char *pymod_name, const char *mod_name, const char *mod_description)
 {
   static pya::PythonModule module;
 
@@ -50,7 +50,7 @@ module_init (const char *mod_name, const char *mod_description)
     //  required for the tiling processor for example
     gsi::initialize_expressions ();
 
-    module.init (mod_name, mod_description);
+    module.init (pymod_name, mod_description);
     module.make_classes (mod_name);
 
     return module.take_module ();
@@ -60,6 +60,9 @@ module_init (const char *mod_name, const char *mod_description)
   return 0;
 }
 
+#define STRINGIFY(s) _STRINGIFY(s)
+#define _STRINGIFY(s) #s
+
 #if PY_MAJOR_VERSION < 3
 
 #define DEFINE_PYMOD(__name__, __name_str__, __description__) \
@@ -67,7 +70,7 @@ module_init (const char *mod_name, const char *mod_description)
   DEF_INSIDE_PUBLIC \
   void init##__name__ () \
   { \
-    module_init (__name_str__, __description__); \
+    module_init (STRINGIFY(__name__), __name_str__, __description__); \
   } \
 
 #define DEFINE_PYMOD_WITH_INIT(__name__, __name_str__, __description__, __init__) \
@@ -75,7 +78,7 @@ module_init (const char *mod_name, const char *mod_description)
   DEF_INSIDE_PUBLIC \
   void init##__name__ () \
   { \
-    __init__ (__name_str__, __description__); \
+    __init__ (STRINGIFY(__name__), __name_str__, __description__); \
   } \
 
 #else
@@ -85,7 +88,7 @@ module_init (const char *mod_name, const char *mod_description)
   DEF_INSIDE_PUBLIC \
   PyObject *PyInit_##__name__ () \
   { \
-    return module_init (__name_str__, __description__); \
+    return module_init (STRINGIFY(__name__), __name_str__, __description__); \
   } \
 
 #define DEFINE_PYMOD_WITH_INIT(__name__, __name_str__, __description__, __init__) \
@@ -93,7 +96,7 @@ module_init (const char *mod_name, const char *mod_description)
   DEF_INSIDE_PUBLIC \
   PyObject *PyInit_##__name__ () \
   { \
-    return __init__ (__name_str__, __description__); \
+    return __init__ (STRINGIFY(__name__), __name_str__, __description__); \
   } \
 
 #endif

src/pymod/rdb/rdb.pro

@@ -1,5 +1,6 @@
 
-TARGET = rdb
+TARGET = rdbcore
+REALMODULE = rdb
 
 include($$PWD/../pymod.pri)
 

src/pymod/rdb/rdbMain.cc

@@ -25,4 +25,4 @@
 //  to force linking of the rdb module
 #include "../../rdb/rdb/rdbForceLink.h"
 
-DEFINE_PYMOD(rdb, "rdb", "KLayout core module 'rdb'")
+DEFINE_PYMOD(rdbcore, "rdb", "KLayout core module 'rdb'")

src/pymod/tl/tl.pro

@@ -1,5 +1,6 @@
 
-TARGET = tl
+TARGET = tlcore
+REALMODULE = tl
 
 include($$PWD/../pymod.pri)
 
@@ -41,3 +42,4 @@ msvc {
   init_target.extra = $(INSTALL_PROGRAM) $$DESTDIR_PYMOD/__init__.py $(INSTALLROOT)$$PREFIX/pymod/klayout
 }
 INSTALLS += init_target
+

src/pymod/tl/tlMain.cc

@@ -22,4 +22,4 @@
 
 #include "../pymodHelper.h"
 
-DEFINE_PYMOD(tl, "tl", "KLayout core module 'tl'")
+DEFINE_PYMOD(tlcore, "tl", "KLayout core module 'tl'")

src/tl/tl/tlStream.cc

@@ -883,10 +883,17 @@ InputPipe::~InputPipe ()
 void
 InputPipe::close ()
 {
+  wait ();
+}
+
+int InputPipe::wait ()
+{
+  int ret = 0;
   if (m_file != NULL) {
-    fclose (m_file);
+    ret = _pclose (m_file);
     m_file = NULL;
   }
+  return ret;
 }
 
 size_t