Merge pull request #168 from lightwave-lab/travis-ci/v0.25.4

Travis automation formulas for macOS builds
This commit is contained in:
Matthias Köfferlein 2018-09-19 21:58:34 +02:00 committed by GitHub
commit 5b34701f8f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 176 additions and 136 deletions

58
.travis.yml Normal file
View File

@ -0,0 +1,58 @@
matrix:
include:
# Python 3
- os: osx
osx_image: xcode9.4 # macOS 10.13
env:
- MATRIX_EVAL=""
- PYTHON_VERSION=B37
- MACOS_VERSION=HighSierra
- os: osx
osx_image: xcode8.3 # macOS 10.12
env:
- MATRIX_EVAL=""
- PYTHON_VERSION=B37
- MACOS_VERSION=Sierra
- os: osx
osx_image: xcode8 # macOS 10.11
env:
- MATRIX_EVAL=""
- PYTHON_VERSION=B37
- MACOS_VERSION=ElCapitan
# Python 2
- os: osx
osx_image: xcode9.4 # macOS 10.13
env:
- MATRIX_EVAL=""
- PYTHON_VERSION=Sys
- MACOS_VERSION=HighSierra
- os: osx
osx_image: xcode8.3 # macOS 10.12
env:
- MATRIX_EVAL=""
- PYTHON_VERSION=Sys
- MACOS_VERSION=Sierra
- os: osx
osx_image: xcode8 # macOS 10.11
env:
- MATRIX_EVAL=""
- PYTHON_VERSION=Sys
- MACOS_VERSION=ElCapitan
before_install:
- eval "${MATRIX_EVAL}"
- find "$(brew --prefix)/Caskroom/"*'/.metadata' -type f -name '*.rb' | xargs grep 'EOS.undent' --files-with-matches | xargs sed -i '' 's/EOS.undent/EOS/'
- brew update
- brew bundle
- env
install:
- gem install dropbox-deployment
- git clone https://github.com/kristovatlas/osx-config-check
- cd osx-config-check ; python2.7 app.py --report-only --skip-sudo-checks ; cd ..
script: ./travis-build.sh
after_success:
- dropbox-deployment

4
Brewfile Normal file
View File

@ -0,0 +1,4 @@
tap "homebrew/core"
brew "python3"
brew "python@2", link: false
brew "qt"

51
Makefile Normal file
View File

@ -0,0 +1,51 @@
.PHONY: help build deploy test dropbox-deploy
GITCOMMIT := $(shell git rev-parse --short HEAD)
KLAYOUT_VERSION := $(shell source version.sh && echo $$KLAYOUT_VERSION)
.ONESHELL:
default: help
help:
@echo "For Mac OS only"
@echo "make build PYTHON_VERSION=B37"
@echo "make deploy PYTHON_VERSION=B37"
@echo "make test MACOS_VERSION=HighSierra"
@echo "Valid Mac OS Versions: [Yosemite, ElCapitan, Sierra, HighSierra]"
@echo "Valid Python Version: [nil, Sys, B37]"
build:
@echo "Building for Mac $(GITCOMMIT)"
./build4mac.py -p $(PYTHON_VERSION) -q Qt5Brew -c; \
./build4mac.py -p $(PYTHON_VERSION) -q Qt5Brew
deploy:
@echo "Deploying 4 Mac $(GITCOMMIT)"
./build4mac.py -p $(PYTHON_VERSION) -q Qt5Brew -y
test:
@echo "Testing 4 Mac $(GITCOMMIT)"
qt5.pkg.macos-$(MACOS_VERSION)-release/klayout.app/Contents/MacOS/klayout -b -r test-pylib-script.py; \
cd qt5.build.macos-$(MACOS_VERSION)-release; \
ln -s klayout.app/Contents/MacOS/klayout klayout; \
export TESTTMP=testtmp; \
export TESTSRC=..; \
./ut_runner -h || true; \
cd ..
dropbox-deploy:
@echo "Preparing for dropbox deployment $(MACOS_VERSION) $(GITCOMMIT)"
mkdir deploy; \
pwd; \
ls -lah; \
touch build.txt; \
cp build.txt deploy/qt5.pkg.macos-$(MACOS_VERSION)-$(PYTHON_VERSION)-release-$(KLAYOUT_VERSION)-$(GITCOMMIT).log.txt; \
hdiutil convert macbuild/Resources/klayoutDMGTemplate.dmg -format UDRW -o work-KLayout.dmg; \
hdiutil resize -size 500m work-KLayout.dmg; \
hdiutil attach work-KLayout.dmg -readwrite -noverify -quiet -mountpoint tempKLayout -noautoopen; \
cp -a qt5.pkg.macos-$(MACOS_VERSION)-release/ tempKLayout/; \
hdiutil detach tempKLayout; \
hdiutil convert work-KLayout.dmg -format UDZO -imagekey zlib-level=9 -o deploy/qt5.pkg.macos-$(MACOS_VERSION)-$(PYTHON_VERSION)-release-$(KLAYOUT_VERSION)-$(GITCOMMIT).dmg; \
md5 -q deploy/qt5.pkg.macos-$(MACOS_VERSION)-$(PYTHON_VERSION)-release-$(KLAYOUT_VERSION)-$(GITCOMMIT).dmg > deploy/qt5.pkg.macos-$(MACOS_VERSION)-$(PYTHON_VERSION)-release-$(KLAYOUT_VERSION)-$(GITCOMMIT).dmg.md5; \
rm work-KLayout.dmg

4
dropbox-deployment.yml Normal file
View File

@ -0,0 +1,4 @@
deploy:
dropbox_path: /Builds/klayout # The path to the folder on Dropbox where the files will go
artifacts_path: deploy # can be a single file, or a path
debug: true # if you want to see more logs

View File

@ -31,7 +31,7 @@ Also by default, supported script languages, i.e, Ruby and Python, are those sta
You may want to use a non-OS-standard script language such as Python 3.6 from Anaconda2 (https://www.anaconda.com/download/#macos) in combination with KLayout.
Since Anaconda2 is a popular Python development environment, this is worth trying. Unfortunately, however, some dynamic linkage problems are observed as of today.
On the other hand, Python 3.6 provided by MacPorts or Homebrew is usable.
On the other hand, Python 3.7 provided by MacPorts or Homebrew is usable.
Please try this (refer to 3B below or Section #5) if you feel it's useful.
# 3. Use-cases
@ -90,20 +90,20 @@ $ ./makeDMG4mac.py -p qt5.pkg.macos-HighSierra-release -m
# 5. Alternative building options
### 5.1 Python 3.6 from Homebrew, Qt 5.10.1 from Homebrew
### 5.1 Python 3.7 from Homebrew, Qt 5.10.1 from Homebrew
Homebrew's installation of python3 (`brew install python3`) places a `Python.framework` in `/usr/local/opt/python/Frameworks/Python.framework/`, which you can use to build KLayout from. Qt can also be downloaded from brew with `brew install qt`.
```
# Build step
./build4mac.py -p B36 -q Qt5Brew
./build4mac.py -p B37 -q Qt5Brew
# build with log
./build4mac.py -p B36 -q Qt5Brew 2>&1 | tee qt5.build.macos-HighSierra-release-version.log
./build4mac.py -p B37 -q Qt5Brew 2>&1 | tee qt5.build.macos-HighSierra-release-version.log
# Deploy step
./build4mac.py -p B36 -q Qt5Brew -y # normal deploy
./build4mac.py -p B36 -q Qt5Brew -y -v 3 2>&1 | tee qt5.pkg.macos-HighSierra-release.log # deploy with debug options
./build4mac.py -p B37 -q Qt5Brew -y # normal deploy
./build4mac.py -p B37 -q Qt5Brew -y -v 3 2>&1 | tee qt5.pkg.macos-HighSierra-release.log # deploy with debug options
# Packaging step
./makeDMG4mac.py -p qt5.pkg.macos-HighSierra-release -m -q Qt5101

Binary file not shown.

View File

@ -67,7 +67,7 @@ def SetGlobals():
Usage += " : Refer to 'macbuild/build4mac_env.py' for details | \n"
Usage += " [-q|--qt <type>] : type=['Qt4MacPorts', 'Qt5MacPorts', 'Qt5Brew'] | qt5macports \n"
Usage += " [-r|--ruby <type>] : type=['nil', 'Sys', 'Src24', 'MP24', 'B25'] | sys \n"
Usage += " [-p|--python <type>] : type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B36'] | sys \n"
Usage += " [-p|--python <type>] : type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B37'] | sys \n"
Usage += " [-n|--noqtbinding] : don't create Qt bindings for ruby scripts | disabled \n"
Usage += " [-m|--make <option>] : option passed to 'make' | -j4 \n"
Usage += " [-d|--debug] : enable debug mode build | disabled \n"
@ -175,7 +175,7 @@ def ParseCommandLineArguments():
p.add_option( '-p', '--python',
dest='type_python',
help="Python type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36']" )
help="Python type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B37']" )
p.add_option( '-n', '--noqtbinding',
action='store_true',
@ -294,7 +294,7 @@ def ParseCommandLineArguments():
exit()
# Determine Python type
candidates = [ i.upper() for i in ['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B36'] ]
candidates = [ i.upper() for i in ['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B37'] ]
ModulePython = ""
index = 0
for item in candidates:
@ -324,7 +324,7 @@ def ParseCommandLineArguments():
ModulePython = 'Python36MacPorts'
NonOSStdLang = True
elif index == 5:
ModulePython = 'Python36Brew'
ModulePython = 'Python37Brew'
NonOSStdLang = True
else:
index += 1
@ -765,12 +765,12 @@ def DeployBinariesForBundle():
deploymentPython = True
if deploymentPython and NonOSStdLang:
from build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
from build4mac_util import WalkFrameworkPaths, PerformChanges
bundlePath = AbsMacPkgDir + '/klayout.app'
# bundlePath = os.getcwd() + '/qt5.pkg.macos-HighSierra-release/klayout.app'
bundleExecPathAbs = '%s/Contents/MacOS/' % bundlePath
pythonOriginalFrameworkPath = '/usr/local/opt/python3/Frameworks/Python.framework'
pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
print(" [8.1] Deploying Python from %s ..." % pythonOriginalFrameworkPath)
@ -1012,8 +1012,6 @@ def main():
if not ret2 == 0:
sys.exit(1)
else:
sys.exit(0)
#===================================================================================
if __name__ == "__main__":

View File

@ -126,7 +126,7 @@ RubyDictionary = { 'nil' : None,
# [3] Python
#-----------------------------------------------------
Pythons = [ 'nil', 'PythonYosemite', 'PythonElCapitan', 'PythonSierra', 'PythonHighSierra' ]
Pythons += [ 'Anaconda27', 'Anaconda36', 'Python36MacPorts', 'Python36Brew' ]
Pythons += [ 'Anaconda27', 'Anaconda36', 'Python36MacPorts', 'Python37Brew' ]
#-----------------------------------------------------
# Whereabout of different components of Python
@ -188,11 +188,11 @@ Python36MacPorts= { 'exe': '/opt/local/Library/Frameworks/Python.framework/Versi
'lib': '/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/libpython3.6m.dylib'
}
# Python 3.6 from Brew *+*+*+ EXPERIMENTAL *+*+*+
# Python 3.7 from Brew *+*+*+ EXPERIMENTAL *+*+*+
# [Key Type Name] = 'pybrew'
Python36Brew= { 'exe': '/usr/local/opt/python/libexec/bin/python' ,
'inc': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/3.6/Headers',
'lib': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/3.6/Python'
Python37Brew= { 'exe': '/usr/local/opt/python/libexec/bin/python' ,
'inc': '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/Headers',
'lib': '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/Python'
}
# Consolidated dictionary kit for Python
@ -204,7 +204,7 @@ PythonDictionary= { 'nil' : None,
'Anaconda27' : Anaconda27,
'Anaconda36' : Anaconda36,
'Python36MacPorts': Python36MacPorts,
'Python36Brew' : Python36Brew,
'Python37Brew' : Python37Brew,
}
#-----------------------------------------------------

View File

@ -361,7 +361,7 @@ def MakeTargetDMGFile(msg=""):
# (4) Mount the DMG
#--------------------------------------------------------
print( ">>> (4) Mounting <%s> to <%s>" % (WorkDMG, MountDir ) )
command1 = "hdiutil attach %s -readwrite -noverify -noautoopen -quiet" % WorkDMG
command1 = "hdiutil attach %s -readwrite -noverify -quiet -noautoopen" % WorkDMG
os.system(command1)
command2 = "hdiutil info | grep %s | grep \"/dev/\" | awk '{print $1}'" % VolumeDMG

View File

@ -1,98 +0,0 @@
import os, subprocess
import sys
from macbuild.build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
from pathlib import Path
# bundlePath = AbsMacPkgDir
bundlePath = os.getcwd() + '/qt5.pkg.macos-HighSierra-release/klayout.app'
bundleExecPathAbs = '%s/Contents/MacOS/' % bundlePath
pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
print("[1] Copying Python Framework")
shell_commands = list()
shell_commands.append(f"rm -rf {pythonFrameworkPath}")
shell_commands.append(f"rsync -a --safe-links {pythonOriginalFrameworkPath}/ {pythonFrameworkPath}")
shell_commands.append(f"mkdir {pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
shell_commands.append(f"cp -RL {pythonOriginalFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/{{pip*,pkg_resources,setuptools*,wheel*}} " +
f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/lib/python3.6/test")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/Resources")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/bin")
for command in shell_commands:
if subprocess.call( command, shell=True ) != 0:
msg = "command failed: %s"
print( msg % command, file=sys.stderr )
exit(1)
print("[2] Relinking dylib dependencies inside Python.framework")
depdict = WalkFrameworkPaths(pythonFrameworkPath)
appPythonFrameworkPath = '@executable_path/../Frameworks/Python.framework/'
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
usrLocalPath = '/usr/local/opt/'
appUsrLocalPath = '@executable_path/../Frameworks/'
depdict = WalkFrameworkPaths(pythonFrameworkPath)
PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs, libdir=True)
print("[3] Relinking dylib dependencies for klayout")
klayoutPath = bundleExecPathAbs
depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'klayout$')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
libKlayoutPath = bundleExecPathAbs + '../Frameworks'
depdict = WalkFrameworkPaths(libKlayoutPath, filter_regex=r'libklayout')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
print("[4] Patching site.py, pip/, and distutils/")
site_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site.py"
with open(site_module, 'r') as site:
buf = site.readlines()
with open(site_module, 'w') as site:
import re
for line in buf:
# This will fool pip into thinking it's inside a virtual environment
# and install new packates to the correct site-packages
if re.match("^PREFIXES", line) is not None:
line = line + "sys.real_prefix = sys.prefix\n"
# do not allow installation in the user folder.
if re.match("^ENABLE_USER_SITE", line) is not None:
line = "ENABLE_USER_SITE = False\n"
site.write(line)
pip_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/pip/__init__.py"
with open(pip_module, 'r') as pip:
buf = pip.readlines()
with open(pip_module, 'w') as pip:
import re
for line in buf:
# this will reject user's configuration of pip, forcing the isolated mode
line = re.sub("return isolated$", "return isolated or True", line)
pip.write(line)
distutilsconfig = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/distutils/distutils.cfg"
with open(distutilsconfig, 'r') as file:
buf = file.readlines()
with open(distutilsconfig, 'w') as file:
import re
for line in buf:
# This will cause all packages to be installed to sys.prefix
if re.match('prefix=', line) is not None:
continue
file.write(line)
# pythonPath = bundleExecPathAbs + '../Frameworks/Python.framework/Versions/3.6/bin/'
# # pythonOriginalPrefixPath = '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.6'
# # appPythonBinPath = '@executable_path/../'
# depdict = WalkFrameworkPaths(pythonPath, filter_regex=r'python')
# print(depdict)
# PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath),
# (Path(pythonOriginalFrameworkPath).resolve(), appPythonFrameworkPath)], bundleExecPathAbs)
# usrLocalPath = '/usr/local/lib/'
# appUsrLocalPath = '@executable_path/../Frameworks/'
# depdict = WalkFrameworkPaths(pythonFrameworkPath)
# PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)

View File

@ -1,14 +1,28 @@
import site; print(site.getsitepackages())
import pip
installed_packages = pip.get_installed_distributions()
installed_packages_list = sorted(["%s==%s" % (i.key, i.version)
for i in installed_packages])
print(installed_packages_list)
def get_pip_main():
import pip
# check if pip version is new:
if int(pip.__version__.split('.')[0]) > 9:
from pip._internal import main
else:
from pip import main
return main
import sys
print("Executing from: ", sys.executable)
print("-------------")
if pip.main(['install', '--upgrade', 'numpy']) > 0:
import os
print("Environment variables:")
for variable, value in os.environ.items():
if variable == 'DROPBOX_OAUTH_BEARER':
continue
print(variable, ":", value)
pipmain = get_pip_main()
print("-------------")
if pipmain(['install', '--upgrade', 'numpy']) > 0:
exit(1)
print("-------------")
@ -16,14 +30,3 @@ print("-------------")
print("Importing numpy")
import numpy
print("-------------")
import sys;
print("Executing from: ", sys.executable)
print("-------------")
import os
print("Environment variables:")
for variable, value in os.environ.items():
print(variable, ":", value)

20
travis-build.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
set -e
export PING_SLEEP=30s
bash -c "while true; do echo -n '.'; sleep $PING_SLEEP; done" &
PING_LOOP_PID=$!
make build >> build.txt 2>&1
make deploy >> build.txt 2>&1
make test >> build.txt 2>&1 || true
make dropbox-deploy
tail -500 build.txt
echo "build finished"
kill $PING_LOOP_PID
exit 0