Merge pull request #168 from lightwave-lab/travis-ci/v0.25.4

Travis automation formulas for macOS builds

.travis.yml

@@ -0,0 +1,58 @@
+matrix:
+  include:
+    # Python 3
+    - os: osx
+      osx_image: xcode9.4 # macOS 10.13
+      env:
+        - MATRIX_EVAL=""
+        - PYTHON_VERSION=B37
+        - MACOS_VERSION=HighSierra
+    - os: osx
+      osx_image: xcode8.3 # macOS 10.12
+      env:
+        - MATRIX_EVAL=""
+        - PYTHON_VERSION=B37
+        - MACOS_VERSION=Sierra
+    - os: osx
+      osx_image: xcode8 # macOS 10.11
+      env:
+        - MATRIX_EVAL=""
+        - PYTHON_VERSION=B37
+        - MACOS_VERSION=ElCapitan
+
+    # Python 2
+    - os: osx
+      osx_image: xcode9.4 # macOS 10.13
+      env:
+        - MATRIX_EVAL=""
+        - PYTHON_VERSION=Sys
+        - MACOS_VERSION=HighSierra
+    - os: osx
+      osx_image: xcode8.3 # macOS 10.12
+      env:
+        - MATRIX_EVAL=""
+        - PYTHON_VERSION=Sys
+        - MACOS_VERSION=Sierra
+    - os: osx
+      osx_image: xcode8 # macOS 10.11
+      env:
+        - MATRIX_EVAL=""
+        - PYTHON_VERSION=Sys
+        - MACOS_VERSION=ElCapitan
+
+before_install:
+  - eval "${MATRIX_EVAL}"
+  - find "$(brew --prefix)/Caskroom/"*'/.metadata' -type f -name '*.rb' | xargs grep 'EOS.undent' --files-with-matches | xargs sed -i '' 's/EOS.undent/EOS/'
+  - brew update
+  - brew bundle
+  - env
+
+install:
+  - gem install dropbox-deployment
+  - git clone https://github.com/kristovatlas/osx-config-check
+  - cd osx-config-check ; python2.7 app.py --report-only --skip-sudo-checks ; cd ..
+
+script: ./travis-build.sh
+
+after_success:
+  - dropbox-deployment

Brewfile

@@ -0,0 +1,4 @@
+tap "homebrew/core"
+brew "python3"
+brew "python@2", link: false
+brew "qt"

Makefile

@@ -0,0 +1,51 @@
+.PHONY: help build deploy test dropbox-deploy
+
+GITCOMMIT := $(shell git rev-parse --short HEAD)
+KLAYOUT_VERSION := $(shell source version.sh && echo $$KLAYOUT_VERSION)
+
+.ONESHELL:
+
+default: help
+
+help:
+	@echo "For Mac OS only"
+	@echo "make build PYTHON_VERSION=B37"
+	@echo "make deploy PYTHON_VERSION=B37"
+	@echo "make test MACOS_VERSION=HighSierra"
+	@echo "Valid Mac OS Versions: [Yosemite, ElCapitan, Sierra, HighSierra]"
+	@echo "Valid Python Version: [nil, Sys, B37]"
+
+build:
+	@echo "Building for Mac $(GITCOMMIT)"
+	./build4mac.py -p $(PYTHON_VERSION) -q Qt5Brew -c; \
+	./build4mac.py -p $(PYTHON_VERSION) -q Qt5Brew
+
+deploy:
+	@echo "Deploying 4 Mac $(GITCOMMIT)"
+	./build4mac.py -p $(PYTHON_VERSION) -q Qt5Brew -y
+	
+test:
+	@echo "Testing 4 Mac $(GITCOMMIT)"
+	qt5.pkg.macos-$(MACOS_VERSION)-release/klayout.app/Contents/MacOS/klayout -b -r test-pylib-script.py; \
+	cd qt5.build.macos-$(MACOS_VERSION)-release; \
+	ln -s klayout.app/Contents/MacOS/klayout klayout; \
+	export TESTTMP=testtmp; \
+	export TESTSRC=..; \
+	./ut_runner -h || true; \
+	cd ..
+
+dropbox-deploy:
+	@echo "Preparing for dropbox deployment $(MACOS_VERSION) $(GITCOMMIT)"
+	mkdir deploy; \
+	pwd; \
+	ls -lah; \
+	touch build.txt; \
+	cp build.txt deploy/qt5.pkg.macos-$(MACOS_VERSION)-$(PYTHON_VERSION)-release-$(KLAYOUT_VERSION)-$(GITCOMMIT).log.txt; \
+	hdiutil convert macbuild/Resources/klayoutDMGTemplate.dmg -format UDRW -o work-KLayout.dmg; \
+	hdiutil resize -size 500m work-KLayout.dmg; \
+	hdiutil attach work-KLayout.dmg -readwrite -noverify -quiet -mountpoint tempKLayout -noautoopen; \
+	cp -a qt5.pkg.macos-$(MACOS_VERSION)-release/ tempKLayout/; \
+	hdiutil detach tempKLayout; \
+	hdiutil convert work-KLayout.dmg -format UDZO -imagekey zlib-level=9 -o deploy/qt5.pkg.macos-$(MACOS_VERSION)-$(PYTHON_VERSION)-release-$(KLAYOUT_VERSION)-$(GITCOMMIT).dmg; \
+	md5 -q deploy/qt5.pkg.macos-$(MACOS_VERSION)-$(PYTHON_VERSION)-release-$(KLAYOUT_VERSION)-$(GITCOMMIT).dmg > deploy/qt5.pkg.macos-$(MACOS_VERSION)-$(PYTHON_VERSION)-release-$(KLAYOUT_VERSION)-$(GITCOMMIT).dmg.md5; \
+	rm work-KLayout.dmg

dropbox-deployment.yml

@@ -0,0 +1,4 @@
+deploy:
+  dropbox_path: /Builds/klayout # The path to the folder on Dropbox where the files will go
+  artifacts_path: deploy # can be a single file, or a path
+  debug: true # if you want to see more logs

macbuild/ReadMe.md

@@ -31,7 +31,7 @@ Also by default, supported script languages, i.e, Ruby and Python, are those sta
 You may want to use a non-OS-standard script language such as Python 3.6 from Anaconda2 (https://www.anaconda.com/download/#macos) in combination with KLayout.
 
 Since Anaconda2 is a popular Python development environment, this is worth trying. Unfortunately, however, some dynamic linkage problems are observed as of today. 
-On the other hand, Python 3.6 provided by MacPorts or Homebrew is usable.
+On the other hand, Python 3.7 provided by MacPorts or Homebrew is usable.
 Please try this (refer to 3B below or Section #5) if you feel it's useful.
 
 # 3. Use-cases
@@ -90,20 +90,20 @@ $ ./makeDMG4mac.py -p qt5.pkg.macos-HighSierra-release -m
 
 
 # 5. Alternative building options
-### 5.1 Python 3.6 from Homebrew, Qt 5.10.1 from Homebrew
+### 5.1 Python 3.7 from Homebrew, Qt 5.10.1 from Homebrew
 
 Homebrew's installation of python3 (`brew install python3`) places a `Python.framework` in `/usr/local/opt/python/Frameworks/Python.framework/`, which you can use to build KLayout from. Qt can also be downloaded from brew with `brew install qt`.
 
 ```
 # Build step
-./build4mac.py -p B36 -q Qt5Brew
+./build4mac.py -p B37 -q Qt5Brew
 # build with log
-./build4mac.py -p B36 -q Qt5Brew 2>&1 | tee qt5.build.macos-HighSierra-release-version.log
+./build4mac.py -p B37 -q Qt5Brew 2>&1 | tee qt5.build.macos-HighSierra-release-version.log
 
 
 # Deploy step
-./build4mac.py -p B36 -q Qt5Brew -y  # normal deploy
+./build4mac.py -p B37 -q Qt5Brew -y  # normal deploy
-./build4mac.py -p B36 -q Qt5Brew -y -v 3 2>&1 | tee qt5.pkg.macos-HighSierra-release.log  # deploy with debug options
+./build4mac.py -p B37 -q Qt5Brew -y -v 3 2>&1 | tee qt5.pkg.macos-HighSierra-release.log  # deploy with debug options
 
 # Packaging step
 ./makeDMG4mac.py -p qt5.pkg.macos-HighSierra-release -m -q Qt5101

macbuild/build4mac.py

@@ -67,7 +67,7 @@ def SetGlobals():
   Usage += "                        : Refer to 'macbuild/build4mac_env.py' for details           | \n"
   Usage += "   [-q|--qt <type>]     : type=['Qt4MacPorts', 'Qt5MacPorts', 'Qt5Brew']             | qt5macports \n"
   Usage += "   [-r|--ruby <type>]   : type=['nil', 'Sys', 'Src24', 'MP24', 'B25']                | sys \n"
-  Usage += "   [-p|--python <type>] : type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B36']       | sys \n"
+  Usage += "   [-p|--python <type>] : type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B37']       | sys \n"
   Usage += "   [-n|--noqtbinding]   : don't create Qt bindings for ruby scripts                  | disabled \n"
   Usage += "   [-m|--make <option>] : option passed to 'make'                                    | -j4 \n"
   Usage += "   [-d|--debug]         : enable debug mode build                                    | disabled \n"
@@ -175,7 +175,7 @@ def ParseCommandLineArguments():
 
   p.add_option( '-p', '--python',
                 dest='type_python',
-                help="Python type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36']" )
+                help="Python type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B37']" )
 
   p.add_option( '-n', '--noqtbinding',
                 action='store_true',
@@ -294,7 +294,7 @@ def ParseCommandLineArguments():
     exit()
 
   # Determine Python type
-  candidates   = [ i.upper() for i in ['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B36'] ]
+  candidates   = [ i.upper() for i in ['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B37'] ]
   ModulePython = ""
   index        = 0
   for item in candidates:
@@ -324,7 +324,7 @@ def ParseCommandLineArguments():
         ModulePython = 'Python36MacPorts'
         NonOSStdLang = True
       elif index == 5:
-        ModulePython = 'Python36Brew'
+        ModulePython = 'Python37Brew'
         NonOSStdLang = True
     else:
       index += 1
@@ -765,12 +765,12 @@ def DeployBinariesForBundle():
 
     deploymentPython = True
     if deploymentPython and NonOSStdLang:
-      from build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
+      from build4mac_util import WalkFrameworkPaths, PerformChanges
 
       bundlePath = AbsMacPkgDir + '/klayout.app'
       # bundlePath = os.getcwd() + '/qt5.pkg.macos-HighSierra-release/klayout.app'
       bundleExecPathAbs = '%s/Contents/MacOS/' % bundlePath
-      pythonOriginalFrameworkPath = '/usr/local/opt/python3/Frameworks/Python.framework'
+      pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
       pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
 
       print(" [8.1] Deploying Python from %s ..." % pythonOriginalFrameworkPath)
@@ -1012,8 +1012,6 @@ def main():
 
     if not ret2 == 0:
       sys.exit(1)
-    else:
-      sys.exit(0)
 
 #===================================================================================
 if __name__ == "__main__":

macbuild/build4mac_env.py

@@ -126,7 +126,7 @@ RubyDictionary  = { 'nil'           : None,
 # [3] Python
 #-----------------------------------------------------
 Pythons  = [ 'nil', 'PythonYosemite', 'PythonElCapitan', 'PythonSierra', 'PythonHighSierra' ]
-Pythons += [ 'Anaconda27', 'Anaconda36', 'Python36MacPorts', 'Python36Brew' ]
+Pythons += [ 'Anaconda27', 'Anaconda36', 'Python36MacPorts', 'Python37Brew' ]
 
 #-----------------------------------------------------
 # Whereabout of different components of Python
@@ -188,11 +188,11 @@ Python36MacPorts= { 'exe': '/opt/local/Library/Frameworks/Python.framework/Versi
                     'lib': '/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/libpython3.6m.dylib'
                   }
 
-# Python 3.6 from Brew *+*+*+ EXPERIMENTAL *+*+*+
+# Python 3.7 from Brew *+*+*+ EXPERIMENTAL *+*+*+
 # [Key Type Name] = 'pybrew'
-Python36Brew= { 'exe': '/usr/local/opt/python/libexec/bin/python' ,
+Python37Brew= { 'exe': '/usr/local/opt/python/libexec/bin/python' ,
-                'inc': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/3.6/Headers',
+                'inc': '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/Headers',
-                'lib': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/3.6/Python'
+                'lib': '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/Python'
               }
 
 # Consolidated dictionary kit for Python
@@ -204,7 +204,7 @@ PythonDictionary= { 'nil'             : None,
                     'Anaconda27'      : Anaconda27,
                     'Anaconda36'      : Anaconda36,
                     'Python36MacPorts': Python36MacPorts,
-                    'Python36Brew'    : Python36Brew,
+                    'Python37Brew'    : Python37Brew,
                   }
 
 #-----------------------------------------------------

macbuild/makeDMG4mac.py

@@ -361,7 +361,7 @@ def MakeTargetDMGFile(msg=""):
   # (4) Mount the DMG
   #--------------------------------------------------------
   print( ">>> (4) Mounting <%s> to <%s>" % (WorkDMG, MountDir ) )
-  command1 = "hdiutil attach %s -readwrite -noverify -noautoopen -quiet" % WorkDMG
+  command1 = "hdiutil attach %s -readwrite -noverify -quiet -noautoopen" % WorkDMG
   os.system(command1)
 
   command2 = "hdiutil info | grep %s | grep \"/dev/\" | awk '{print $1}'" % VolumeDMG

test-pydeploy.py

@@ -1,98 +0,0 @@
-import os, subprocess
-import sys
-from macbuild.build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
-from pathlib import Path
-
-# bundlePath = AbsMacPkgDir
-bundlePath = os.getcwd() + '/qt5.pkg.macos-HighSierra-release/klayout.app'
-bundleExecPathAbs = '%s/Contents/MacOS/' % bundlePath
-pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
-pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
-
-
-print("[1] Copying Python Framework")
-shell_commands = list()
-shell_commands.append(f"rm -rf {pythonFrameworkPath}")
-shell_commands.append(f"rsync -a --safe-links {pythonOriginalFrameworkPath}/ {pythonFrameworkPath}")
-shell_commands.append(f"mkdir {pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
-shell_commands.append(f"cp -RL {pythonOriginalFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/{{pip*,pkg_resources,setuptools*,wheel*}} " +
-                      f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
-shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/lib/python3.6/test")
-shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/Resources")
-shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/bin")
-
-for command in shell_commands:
-  if subprocess.call( command, shell=True ) != 0:
-    msg = "command failed: %s"
-    print( msg % command, file=sys.stderr )
-    exit(1)
-
-print("[2] Relinking dylib dependencies inside Python.framework")
-depdict = WalkFrameworkPaths(pythonFrameworkPath)
-appPythonFrameworkPath = '@executable_path/../Frameworks/Python.framework/'
-PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
-
-usrLocalPath = '/usr/local/opt/'
-appUsrLocalPath = '@executable_path/../Frameworks/'
-depdict = WalkFrameworkPaths(pythonFrameworkPath)
-PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs, libdir=True)
-
-print("[3] Relinking dylib dependencies for klayout")
-klayoutPath = bundleExecPathAbs
-depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'klayout$')
-PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
-
-libKlayoutPath = bundleExecPathAbs + '../Frameworks'
-depdict = WalkFrameworkPaths(libKlayoutPath, filter_regex=r'libklayout')
-PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
-
-print("[4] Patching site.py, pip/, and distutils/")
-site_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site.py"
-with open(site_module, 'r') as site:
-  buf = site.readlines()
-with open(site_module, 'w') as site:
-  import re
-  for line in buf:
-    # This will fool pip into thinking it's inside a virtual environment
-    # and install new packates to the correct site-packages
-    if re.match("^PREFIXES", line) is not None:
-      line = line + "sys.real_prefix = sys.prefix\n"
-    # do not allow installation in the user folder.
-    if re.match("^ENABLE_USER_SITE", line) is not None:
-      line = "ENABLE_USER_SITE = False\n"
-    site.write(line)
-
-pip_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/pip/__init__.py"
-with open(pip_module, 'r') as pip:
-  buf = pip.readlines()
-with open(pip_module, 'w') as pip:
-  import re
-  for line in buf:
-    # this will reject user's configuration of pip, forcing the isolated mode
-    line = re.sub("return isolated$", "return isolated or True", line)
-    pip.write(line)
-
-distutilsconfig = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/distutils/distutils.cfg"
-with open(distutilsconfig, 'r') as file:
-  buf = file.readlines()
-with open(distutilsconfig, 'w') as file:
-  import re
-  for line in buf:
-    # This will cause all packages to be installed to sys.prefix
-    if re.match('prefix=', line) is not None:
-      continue
-    file.write(line)
-
-
-# pythonPath = bundleExecPathAbs + '../Frameworks/Python.framework/Versions/3.6/bin/'
-# # pythonOriginalPrefixPath = '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.6'
-# # appPythonBinPath = '@executable_path/../'
-# depdict = WalkFrameworkPaths(pythonPath, filter_regex=r'python')
-# print(depdict)
-# PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath),
-#   (Path(pythonOriginalFrameworkPath).resolve(), appPythonFrameworkPath)], bundleExecPathAbs)
-
-# usrLocalPath = '/usr/local/lib/'
-# appUsrLocalPath = '@executable_path/../Frameworks/'
-# depdict = WalkFrameworkPaths(pythonFrameworkPath)
-# PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)

test-pylib-script.py

@@ -1,14 +1,28 @@
 import site; print(site.getsitepackages())
 
-import pip
+def get_pip_main():
-installed_packages = pip.get_installed_distributions()
+    import pip
-installed_packages_list = sorted(["%s==%s" % (i.key, i.version)
+    # check if pip version is new:
-     for i in installed_packages])
+    if int(pip.__version__.split('.')[0]) > 9:
-print(installed_packages_list)
+        from pip._internal import main
+    else:
+        from pip import main
+    return main
 
+import sys
+print("Executing from: ", sys.executable)
 print("-------------")
 
-if pip.main(['install', '--upgrade', 'numpy']) > 0:
+import os
+print("Environment variables:")
+for variable, value in os.environ.items():
+    if variable == 'DROPBOX_OAUTH_BEARER':
+        continue
+    print(variable, ":", value)
+
+pipmain = get_pip_main()
+print("-------------")
+if pipmain(['install', '--upgrade', 'numpy']) > 0:
     exit(1)
 print("-------------")
 
@@ -16,14 +30,3 @@ print("-------------")
 print("Importing numpy")
 import numpy
 print("-------------")
-
-
-import sys;
-print("Executing from: ", sys.executable)
-print("-------------")
-
-import os
-print("Environment variables:")
-for variable, value in os.environ.items():
-    print(variable, ":", value)
-

travis-build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+export PING_SLEEP=30s
+
+bash -c "while true; do echo -n '.'; sleep $PING_SLEEP; done" &
+PING_LOOP_PID=$!
+
+make build >> build.txt 2>&1
+make deploy >> build.txt 2>&1
+make test >> build.txt 2>&1 || true
+make dropbox-deploy
+
+tail -500 build.txt
+
+echo "build finished"
+
+kill $PING_LOOP_PID
+
+exit 0