Merge pull request #179 from lightwave-lab/pymod

Fixed plugin detection in MacOS

.travis.yml

@@ -43,6 +43,16 @@ matrix:
         - PIP_UPDATE="1"
         - PYTHON_BUILD=true
         - BREW_BUNDLE=true
+
+    - name: "klayout python3.6.5_1 osx10.13"
+      os: osx
+      osx_image: xcode9.4  # macOS 10.13
+      env:
+        - MATRIX_EVAL="brew update; brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f2a764ef944b1080be64bd88dca9a1d80130c558/Formula/python.rb; brew switch python 3.6.5_1; shopt -s expand_aliases; alias python='python3'; alias pip='pip3';"
+        - ARCHFLAGS="-std=c++11"
+        - PIP_UPDATE="1"
+        - PYTHON_BUILD=true
+        - BREW_BUNDLE=false
     
     - name: "klayout python3 osx10.12"
       os: osx
@@ -58,11 +68,11 @@ matrix:
       os: osx
       osx_image: xcode8  # macOS 10.11
       env:
-        - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';"
+        - MATRIX_EVAL="brew update; brew config; brew upgrade python;"
         - ARCHFLAGS="-std=c++11"
         - PIP_UPDATE="1"
         - PYTHON_BUILD=true
-        - BREW_BUNDLE=true
+        - BREW_BUNDLE=false
     
     - name: "klayout python3.6 package"
       os: linux

Makefile

@@ -2,6 +2,12 @@
 
 GITCOMMIT := $(shell git rev-parse --short HEAD)
 KLAYOUT_VERSION := $(shell source version.sh && echo $$KLAYOUT_VERSION)
+ifndef PYTHON_VERSION
+    PYTHON_VERSION := B37
+endif
+ifndef MACOS_VERSION
+	MACOS_VERSION := HighSierra
+endif
 
 .ONESHELL:
 
@@ -31,6 +37,7 @@ test:
 	ln -s klayout.app/Contents/MacOS/klayout klayout; \
 	export TESTTMP=testtmp; \
 	export TESTSRC=..; \
+	export DYLD_LIBRARY_PATH=.:db_plugins/:lay_plugins/; \
 	./ut_runner -h || true; \
 	cd ..
 

macbuild/build4mac.py

@@ -581,6 +581,10 @@ def DeployBinariesForBundle():
   #                             |              +-- '*.dylib'
   #                             +-- MacOS/+
   #                             |         +-- 'klayout'
+  #                             |         +-- 'db_plugins'/+
+  #                             |                          +-- '*.dylib'
+  #                             |         +-- 'lay_plugins'/+
+  #                             |                           +-- '*.dylib'
   #                             +-- Buddy/+
   #                                       +-- 'strm2cif'
   #                                       +-- 'strm2dxf'
@@ -631,8 +635,9 @@ def DeployBinariesForBundle():
   #       :
   #-------------------------------------------------------------------------------
   os.chdir( targetDirF )
-  dynamicLinkLibs = glob.glob( AbsMacBinDir + "/*.dylib" )
+  dynamicLinkLibs = glob.glob( os.path.join( AbsMacBinDir, "*.dylib" ) )
   depDicOrdinary  = {} # inter-library dependency dictionary
+  pathDic = {} # paths to insert for each library
   for item in dynamicLinkLibs:
     if os.path.isfile(item) and not os.path.islink(item):
       #-------------------------------------------------------------------
@@ -646,13 +651,54 @@ def DeployBinariesForBundle():
       os.chmod( nameStyle3, 0o0755 )
       #-------------------------------------------------------------------
       # (B) Then get inter-library dependencies
+      #     Note that will pull all dependencies and sort them out later
+      #     dropping those which don't have a path entry
       #-------------------------------------------------------------------
-      otoolCm   = "otool -L %s | grep libklayout" % nameStyle3
+      otoolCm   = "otool -L %s | grep dylib" % nameStyle3
       otoolOut  = os.popen( otoolCm ).read()
       dependDic = DecomposeLibraryDependency(otoolOut)
       depDicOrdinary.update(dependDic)
+      #-------------------------------------------------------------------
+      # (C) This library goes into Frameworks, hence record it's path there
+      #-------------------------------------------------------------------
+      pathDic[nameStyle3] = "@executable_path/../Frameworks/" + nameStyle3
+
+  os.chdir(ProjectDir)
+  #-------------------------------------------------------------------
+  # copy the contents of the plugin directories to a place next to the application
+  # binary
+  #-------------------------------------------------------------------
+  for piDir in [ "db_plugins", "lay_plugins" ]:
+    os.makedirs( os.path.join( targetDirM, piDir ))
+    dynamicLinkLibs = glob.glob( os.path.join( MacBinDir, piDir, "*.dylib" ) )
+    for item in dynamicLinkLibs:
+      if os.path.isfile(item) and not os.path.islink(item):
+        #-------------------------------------------------------------------
+        # (A) Copy an ordinary *.dylib file here by changing the name
+        #     to style (3) and set its mode to 0755 (sanity check).
+        #-------------------------------------------------------------------
+        fullName = os.path.basename(item).split('.')
+        # e.g. [ 'libklayout_lay', '0', '25', '0', 'dylib' ]
+        nameStyle3 = fullName[0] + "." + fullName[1] + ".dylib"
+        destPath = os.path.join( targetDirM, piDir, nameStyle3 )
+        shutil.copy2( item, destPath )
+        os.chmod( destPath, 0o0755 )
+        #-------------------------------------------------------------------
+        # (B) Then get inter-library dependencies
+        #     Note that will pull all dependencies and sort them out later
+        #     dropping those which don't have a path entry
+        #-------------------------------------------------------------------
+        otoolCm   = "otool -L %s | grep 'dylib'" % destPath
+        otoolOut  = os.popen( otoolCm ).read()
+        dependDic = DecomposeLibraryDependency(otoolOut)
+        depDicOrdinary.update(dependDic)
+        #-------------------------------------------------------------------
+        # (C) This library goes into the plugin dir
+        #-------------------------------------------------------------------
+        pathDic[nameStyle3] = "@executable_path/" + piDir + "/" + nameStyle3
+
   '''
-  PrintLibraryDependencyDictionary( depDicOrdinary, "Style (3)" )
+  PrintLibraryDependencyDictionary( depDicOrdinary, pathDic, "Style (3)" )
   exit()
   '''
 
@@ -662,7 +708,8 @@ def DeployBinariesForBundle():
   #     and make the library aware of the locations of libraries
   #     on which it depends; that is, inter-library dependency
   #-------------------------------------------------------------
-  ret = SetChangeIdentificationNameOfDyLib( depDicOrdinary )
+  os.chdir( targetDirF )
+  ret = SetChangeIdentificationNameOfDyLib( depDicOrdinary, pathDic )
   if not ret == 0:
     msg = "!!! Failed to set and change to new identification names !!!"
     print(msg)
@@ -691,7 +738,6 @@ def DeployBinariesForBundle():
   shutil.copy2( sourceDir1 + "/klayout",      targetDirM )
   shutil.copy2( sourceDir2 + "/klayout.icns", targetDirR )
 
-
   os.chmod( targetDir0 + "/PkgInfo",      0o0644 )
   os.chmod( targetDir0 + "/Info.plist",   0o0644 )
   os.chmod( targetDirM + "/klayout",      0o0755 )

macbuild/build4mac_util.py

@@ -59,15 +59,18 @@ def DecomposeLibraryDependency( depstr ):
 # @param[in] depdic  dictionary
 # @param[in] namedic dictionary name
 #------------------------------------------------------------------------------
-def PrintLibraryDependencyDictionary( depdic, namedic ):
+def PrintLibraryDependencyDictionary( depdic, pathdic, namedic ):
   keys = depdic.keys()
   print("")
   print("##### Contents of <%s> #####:" % namedic )
   for key in keys:
     supporters = depdic[key]
-    print( " %s:" % key )
+    keyName = os.path.basename(key)
+    print( " %s: (%s)" % (key, pathdic[keyName]) )
     for item in supporters:
-      print( "    %s" % item )
+      itemName = os.path.basename(item)
+      if itemName != keyName and (itemName in pathdic):
+        print( "    %s (%s)" % (item, pathdic[itemName]) )
 
 #------------------------------------------------------------------------------
 ## To set and change identification name of KLayout's dylib
@@ -76,7 +79,7 @@ def PrintLibraryDependencyDictionary( depdic, namedic ):
 #
 # @return 0 on success; non-zero on failure
 #------------------------------------------------------------------------------
-def SetChangeIdentificationNameOfDyLib( libdic ):
+def SetChangeIdentificationNameOfDyLib( libdic, pathDic ):
   cmdNameId     = XcodeToolChain['nameID']
   cmdNameChg    = XcodeToolChain['nameCH']
   dependentLibs = libdic.keys()
@@ -86,7 +89,8 @@ def SetChangeIdentificationNameOfDyLib( libdic ):
     # [1] Set the identification name of each dependent library
     #-----------------------------------------------------------
     nameOld = "%s" % lib
-    nameNew = "@executable_path/../Frameworks/%s" % lib
+    libName = os.path.basename(lib)
+    nameNew = pathDic[libName]
     command = "%s %s %s" % ( cmdNameId, nameNew, nameOld )
     if subprocess.call( command, shell=True ) != 0:
       msg = "!!! Failed to set the new identification name to <%s> !!!"
@@ -98,13 +102,15 @@ def SetChangeIdentificationNameOfDyLib( libdic ):
     #-------------------------------------------------------------------------
     supporters = libdic[lib]
     for sup in supporters:
-      nameOld = "%s" % sup
+      supName = os.path.basename(sup)
-      nameNew = "@executable_path/../Frameworks/%s" % sup
+      if libName != supName and (supName in pathDic):
-      command = "%s %s %s %s" % ( cmdNameChg, nameOld, nameNew, lib )
+        nameOld = "%s" % sup
-      if subprocess.call( command, shell=True ) != 0:
+        nameNew = pathDic[supName]
-        msg = "!!! Failed to make the library aware of the new identification name <%s> of supporter <%s> !!!"
+        command = "%s %s %s %s" % ( cmdNameChg, nameOld, nameNew, lib )
-        print( msg % (nameNew, sup), file=sys.stderr )
+        if subprocess.call( command, shell=True ) != 0:
-        return 1
+          msg = "!!! Failed to make the library aware of the new identification name <%s> of supporter <%s> !!!"
+          print( msg % (nameNew, sup), file=sys.stderr )
+          return 1
   # for-lib
   return 0
 

setup.py

@@ -191,7 +191,8 @@ class Config(object):
         """
         Gets the version string
         """
-        return "0.26"
+        return "0.26.0.dev1"
+
 
 config = Config()
 
@@ -345,11 +346,14 @@ rdb = Extension(config.root + '.rdb',
 # Core setup function
 
 if __name__ == '__main__':
-    setup(name = config.root,
+    setup(name=config.root,
-          version =                 config.version(),
+          version=config.version(),
-          description =             'KLayout standalone Python package',
+          license='GNU GPLv3',
-          author =                  'Matthias Koefferlein',
+          description='KLayout standalone Python package',
-          author_email =            'matthias@klayout.de',
+          long_description='TODO',
-          packages =                [config.root],
+          author='Matthias Koefferlein',
-          package_dir =             {config.root: 'src/pymod/distutils_src'},
+          author_email='matthias@klayout.de',
-          ext_modules =             [_tl, _gsi, _pya, _db, _rdb] + db_plugins + [tl, db, rdb])
+          url='https://github.com/klayoutmatthias/klayout',
+          packages=[config.root],
+          package_dir={config.root: 'src/pymod/distutils_src'},
+          ext_modules=[_tl, _gsi, _pya, _db, _rdb] + db_plugins + [tl, db, rdb])

src/db/db/dbInit.cc

@@ -129,6 +129,8 @@ void init (const std::vector<std::string> &_paths)
 #if defined(_WIN32)
     pattern.set_case_sensitive (false);
     pattern = std::string ("*.dll");
+#elif defined(__APPLE__)
+    pattern = std::string ("*.dylib");
 #else
     pattern = std::string ("*.so");
 #endif

src/lay/lay/layInit.cc

@@ -130,6 +130,8 @@ void init (const std::vector<std::string> &_paths)
 #if defined(_WIN32)
     pattern.set_case_sensitive (false);
     pattern = std::string ("*.dll");
+#elif defined(__APPLE__)
+    pattern = std::string ("*.dylib");
 #else
     pattern = std::string ("*.so");
 #endif

src/plugins/streamers/cif/db_plugin/dbCIFFormat.h

@@ -25,6 +25,7 @@
 
 #include "dbSaveLayoutOptions.h"
 #include "dbLoadLayoutOptions.h"
+#include "dbPluginCommon.h"
 
 namespace db
 {
@@ -34,7 +35,7 @@ namespace db
  *  NOTE: this structure is non-public linkage by intention. This way it's instantiated
  *  in all compile units and the shared object does not need to be linked.
  */
-class CIFReaderOptions
+class DB_PLUGIN_PUBLIC CIFReaderOptions
   : public FormatSpecificReaderOptions
 {
 public:
@@ -118,7 +119,7 @@ public:
  *  NOTE: this structure is non-public linkage by intention. This way it's instantiated
  *  in all compile units and the shared object does not need to be linked.
  */
-class CIFWriterOptions
+class DB_PLUGIN_PUBLIC CIFWriterOptions
   : public FormatSpecificWriterOptions
 {
 public:

src/plugins/streamers/dxf/db_plugin/dbDXFFormat.h

@@ -25,6 +25,7 @@
 
 #include "dbLoadLayoutOptions.h"
 #include "dbSaveLayoutOptions.h"
+#include "dbPluginCommon.h"
 
 namespace db
 {
@@ -34,7 +35,7 @@ namespace db
  *  NOTE: this structure is non-public linkage by intention. This way it's instantiated
  *  in all compile units and the shared object does not need to be linked.
  */
-class DXFReaderOptions
+class DB_PLUGIN_PUBLIC DXFReaderOptions
   : public FormatSpecificReaderOptions
 {
 public:
@@ -192,7 +193,7 @@ public:
  *  NOTE: this structure is non-public linkage by intention. This way it's instantiated
  *  in all compile units and the shared object does not need to be linked.
  */
-class DXFWriterOptions
+class DB_PLUGIN_PUBLIC DXFWriterOptions
   : public FormatSpecificWriterOptions
 {
 public:

src/plugins/streamers/gds2/db_plugin/dbGDS2Format.h

@@ -25,6 +25,7 @@
 
 #include "dbSaveLayoutOptions.h"
 #include "dbLoadLayoutOptions.h"
+#include "dbPluginCommon.h"
 
 namespace db
 {
@@ -34,7 +35,7 @@ namespace db
  *  NOTE: this structure is non-public linkage by intention. This way it's instantiated
  *  in all compile units and the shared object does not need to be linked.
  */
-class GDS2ReaderOptions
+class DB_PLUGIN_PUBLIC GDS2ReaderOptions
   : public FormatSpecificReaderOptions
 {
 public:
@@ -97,7 +98,7 @@ public:
  *  NOTE: this structure is non-public linkage by intention. This way it's instantiated
  *  in all compile units and the shared object does not need to be linked.
  */
-class GDS2WriterOptions
+class DB_PLUGIN_PUBLIC GDS2WriterOptions
   : public FormatSpecificWriterOptions
 {
 public:

src/plugins/streamers/oasis/db_plugin/dbOASISFormat.h

@@ -25,6 +25,7 @@
 
 #include "dbSaveLayoutOptions.h"
 #include "dbLoadLayoutOptions.h"
+#include "dbPluginCommon.h"
 
 namespace db
 {
@@ -34,7 +35,7 @@ namespace db
  *  NOTE: this structure is non-public linkage by intention. This way it's instantiated
  *  in all compile units and the shared object does not need to be linked.
  */
-class OASISReaderOptions
+class DB_PLUGIN_PUBLIC OASISReaderOptions
   : public FormatSpecificReaderOptions
 {
 public:
@@ -92,7 +93,7 @@ public:
  *  NOTE: this structure is non-public linkage by intention. This way it's instantiated
  *  in all compile units and the shared object does not need to be linked.
  */
-class OASISWriterOptions
+class DB_PLUGIN_PUBLIC OASISWriterOptions
   : public FormatSpecificWriterOptions
 {
 public:

travis-build.sh

@@ -6,12 +6,16 @@ export PING_SLEEP=30s
 bash -c "while true; do echo -n '.'; sleep $PING_SLEEP; done" &
 PING_LOOP_PID=$!
 
-make build >> build.txt 2>&1
+touch build.txt
-make deploy >> build.txt 2>&1
-make test >> build.txt 2>&1 || true
-make dropbox-deploy
 
-tail -500 build.txt
+echo "build"
+make build >> build.txt 2>&1 || tail -500 build.txt
+echo "deploy"
+make deploy >> build.txt 2>&1 || tail -500 build.txt
+echo "test"
+make test >> build.txt 2>&1 || tail -500 build.txt
+echo "dropbox-deploy"
+make dropbox-deploy
 
 echo "build finished"