luke
/
klayout
mirror of https://github.com/KLayout/klayout.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #138 from lightwave-lab/macos_embedding_python_pr 

Macos embedding python and Qt5.11.1 bugfix
This commit is contained in:
Matthias Köfferlein 2018-06-28 21:08:41 +02:00 committed by GitHub
parent b47c0fc3a3 896f7347f0
commit b64d14e02e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 279 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
macbuild/ReadMe.md
View File

@ -97,6 +97,9 @@ Homebrew's installation of python3 (`brew install python3`) places a `Python.fra
```
# Build step
./build4mac.py -p B36 -q Qt5Brew
# build with log
./build4mac.py -p B36 -q Qt5Brew 2>&1 | tee qt5.build.macos-HighSierra-release-version.log
# Deploy step
./build4mac.py -p B36 -q Qt5Brew -y # normal deploy

6
macbuild/Resources/klayout_console Normal file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
if [ "$1" != "" ]; then
/Applications/klayout.app/Contents/MacOS/klayout $1 -r /Applications/klayout.app/Contents/MacOS/start-console.py
else
/Applications/klayout.app/Contents/MacOS/klayout -z -r /Applications/klayout.app/Contents/MacOS/start-console.py
fi

16
macbuild/Resources/start-console.py Normal file
View File

@ -0,0 +1,16 @@
#!/Applications/klayout.app/Contents/MacOS/klayout -b -r
import readline
import code
import sys
import os
pwd = os.getcwd()
sys.path.append(pwd)
variables = globals().copy()
variables.update(locals())
shell = code.InteractiveConsole(variables)
cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
banner = "Python %s on %s\n%s\n(%s)" % (sys.version, sys.platform,
cprt, "KLayout Python Console")
exit_msg = 'now exiting %s...' % "KLayout Python Console"
shell.interact(banner, exit_msg)

136
macbuild/build4mac.py
View File

@ -1,4 +1,4 @@
#! /usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#===============================================================================
@ -538,8 +538,8 @@ def DeployBinariesForBundle():
if not DeploymentF and not DeploymentP:
return 1
if DeploymentF and NonOSStdLang:
print( "WARNING!!! You chose <-y|--deploy> while using non-OS-standard script language.", file=sys.stderr )
print( " Use <-Y|--DEPLOY> instead", file=sys.stderr )
print( " WARNING!!! You chose <-y|--deploy> while using non-OS-standard script language.", file=sys.stderr )
print( " Consider using <-Y|--DEPLOY> instead", file=sys.stderr )
#return 1
if not os.path.isfile(MacBuildLog):
print( "!!! Build log file <%s> does not present !!!" % MacBuildLog, file=sys.stderr )
@ -756,7 +756,6 @@ def DeployBinariesForBundle():
os.chdir(ProjectDir)
os.chdir(MacPkgDir)
command = "%s %s %s" % ( deploytool, app_bundle, options )
print(command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to deploy applications on OSX !!!"
print( msg, file=sys.stderr )
@ -764,43 +763,112 @@ def DeployBinariesForBundle():
os.chdir(ProjectDir)
return 1
deploymentPython = False
if deploymentPython:
# TODO Code incomplete! To be finished.
from macbuild.build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
bundlePath = 'qt5.pkg.macos-HighSierra-release/klayout.app'
bundleExecPathAbs = os.getcwd() + '/%s/Contents/MacOS/' % bundlePath
# rsync -a --safe-links /usr/local/opt/python/Frameworks/Python.framework/ qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework
# cp -RL /usr/local/opt/python/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework/Versions/3.6/lib/python3.6/
# rm -rf qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/test
pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
depdict = WalkFrameworkPaths(pythonFrameworkPath)
deploymentPython = True
if deploymentPython and NonOSStdLang:
from build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
from pathlib import Path
bundlePath = AbsMacPkgDir + '/klayout.app'
# bundlePath = os.getcwd() + '/qt5.pkg.macos-HighSierra-release/klayout.app'
bundleExecPathAbs = '%s/Contents/MacOS/' % bundlePath
pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
print(f" [8.1] Deploying Python from {pythonOriginalFrameworkPath} ..." )
print(" [1] Copying Python Framework")
shell_commands = list()
shell_commands.append(f"rm -rf {pythonFrameworkPath}")
shell_commands.append(f"rsync -a --safe-links {pythonOriginalFrameworkPath}/ {pythonFrameworkPath}")
shell_commands.append(f"mkdir {pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
shell_commands.append(f"cp -RL {pythonOriginalFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/{{pip*,pkg_resources,setuptools*,wheel*}} " +
f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/lib/python3.6/test")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/Resources")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/bin")
for command in shell_commands:
if subprocess.call( command, shell=True ) != 0:
msg = "command failed: %s"
print( msg % command, file=sys.stderr )
exit(1)
shutil.copy2( sourceDir2 + "/start-console.py", targetDirM )
shutil.copy2( sourceDir2 + "/klayout_console", targetDirM )
os.chmod( targetDirM + "/klayout_console", 0o0755 )
print(" [2] Relinking dylib dependencies inside Python.framework")
print(" [2.1] Patching Python Framework")
depdict = WalkFrameworkPaths(pythonFrameworkPath)
appPythonFrameworkPath = '@executable_path/../Frameworks/Python.framework/'
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
klayoutPath = bundleExecPathAbs
depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'klayout$')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
klayoutPath = bundleExecPathAbs + '../Frameworks'
depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'libklayout')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs)
print(" [2.2] Patching /usr/local/opt/ libs")
usrLocalPath = '/usr/local/opt/'
appUsrLocalPath = '@executable_path/../Frameworks/'
depdict = WalkFrameworkPaths(pythonFrameworkPath)
PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)
replacePairs = [(usrLocalPath, appUsrLocalPath, True)]
depdict = WalkFrameworkPaths(pythonFrameworkPath, search_path_filter=r'\t+/usr/local/(opt|Cellar)')
PerformChanges(depdict, replacePairs, bundleExecPathAbs)
# usrLocalPath = '/usr/local/lib/'
# appUsrLocalPath = '@executable_path/../Frameworks/'
# depdict = WalkFrameworkPaths(pythonFrameworkPath)
# PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)
print(" [2.3] Patching openssl, gdbm, readline, sqlite, tcl-tk, xz")
usrLocalPath = '/usr/local/opt'
appUsrLocalPath = '@executable_path/../Frameworks/'
replacePairs = [(usrLocalPath, appUsrLocalPath, True)]
replacePairs.extend([(openssl_version, '@executable_path/../Frameworks/openssl', True)
for openssl_version in list(Path('/usr/local/Cellar/openssl').glob('*'))])
depdict = WalkFrameworkPaths([pythonFrameworkPath + '/../openssl',
pythonFrameworkPath + '/../gdbm',
pythonFrameworkPath + '/../readline',
pythonFrameworkPath + '/../sqlite',
pythonFrameworkPath + '/../tcl-tk',
pythonFrameworkPath + '/../xz'], search_path_filter=r'\t+/usr/local/(opt|Cellar)')
PerformChanges(depdict, replacePairs, bundleExecPathAbs)
print(" [3] Relinking dylib dependencies for klayout")
klayoutPath = bundleExecPathAbs
depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'klayout$')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs)
libKlayoutPath = bundleExecPathAbs + '../Frameworks'
depdict = WalkFrameworkPaths(libKlayoutPath, filter_regex=r'libklayout')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs)
print(" [4] Patching site.py, pip/, and distutils/")
site_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site.py"
with open(site_module, 'r') as site:
buf = site.readlines()
with open(site_module, 'w') as site:
import re
for line in buf:
# This will fool pip into thinking it's inside a virtual environment
# and install new packates to the correct site-packages
if re.match("^PREFIXES", line) is not None:
line = line + "sys.real_prefix = sys.prefix\n"
# do not allow installation in the user folder.
if re.match("^ENABLE_USER_SITE", line) is not None:
line = "ENABLE_USER_SITE = False\n"
site.write(line)
pip_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/pip/__init__.py"
with open(pip_module, 'r') as pip:
buf = pip.readlines()
with open(pip_module, 'w') as pip:
import re
for line in buf:
# this will reject user's configuration of pip, forcing the isolated mode
line = re.sub("return isolated$", "return isolated or True", line)
pip.write(line)
distutilsconfig = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/distutils/distutils.cfg"
with open(distutilsconfig, 'r') as file:
buf = file.readlines()
with open(distutilsconfig, 'w') as file:
import re
for line in buf:
# This will cause all packages to be installed to sys.prefix
if re.match('prefix=', line) is not None:
continue
file.write(line)
else:
print( " [8] Skipped deploying Qt's Frameworks ..." )

47
macbuild/build4mac_util.py
View File

@ -200,24 +200,24 @@ if CAN_DEPLOY_PYTHON:
lib = str(Path(lib))
if lib != list(keys)[0]:
deplibs[idx] = WalkLibDependencyTree(lib, depth+1, filter_regex)
else:
return NOTHINGTODO
if depth == 0:
return deplibs
return exedepdic
else:
raise RuntimeError("Exceeded maximum recursion depth.")
def WalkFrameworkPaths(frameworkPaths, filter_regex=r'\.(so|dylib)$'):
try:
frameworkPathsIter = iter(frameworkPaths)
except TypeError:
def WalkFrameworkPaths(frameworkPaths, filter_regex=r'\.(so|dylib)$', search_path_filter=r'\t+/usr/local/opt'):
if isinstance(frameworkPaths, str):
frameworkPathsIter = [frameworkPaths]
else:
frameworkPathsIter = frameworkPaths
dependency_dict = dict()
for frameworkPath in frameworkPaths:
for frameworkPath in frameworkPathsIter:
frameworkPath = str(Path(frameworkPath))
# print("Calling:", 'find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex))
find_grep_results = os.popen('find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex)).read().split('\n')
framework_files = filter(lambda x: x != '',
map(lambda x: x.strip(),
@ -225,7 +225,7 @@ if CAN_DEPLOY_PYTHON:
dependency_dict[frameworkPath] = list()
for idx, file in enumerate(framework_files):
dict_file = {file: WalkLibDependencyTree(file)}
dict_file = {file: WalkLibDependencyTree(file, filter_regex=search_path_filter)}
dependency_dict[frameworkPath].append(dict_file)
return dependency_dict
@ -263,15 +263,12 @@ if CAN_DEPLOY_PYTHON:
relPath = path.relative_to(root_path)
return str(root_path / relPath.parts[0])
def ReplaceExecutablePath(path, executable_path):
def ResolveExecutablePath(path, executable_path):
""" Transforms @executable_path into executable_path"""
executable_path = str(executable_path)
p = Path(str(path).replace("@executable_path", "/%s/" % executable_path))
return p
def FileExists(file_path, executable_path):
p = ReplaceExecutablePath(file_path, executable_path)
return p.exists()
def DetectChanges(frameworkDependencyDict):
visited_files = list()
libNameChanges = list()
@ -282,7 +279,7 @@ if CAN_DEPLOY_PYTHON:
return libNameChanges
def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_path="/tmp/klayout", libdir=False):
def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_path="/tmp/klayout"):
libNameChanges = DetectChanges(frameworkDependencyDict)
cmdNameId = XcodeToolChain['nameID']
cmdNameChg = XcodeToolChain['nameCH']
@ -295,37 +292,41 @@ if CAN_DEPLOY_PYTHON:
dependencies = next(libNameChangeIterator)
except StopIteration:
dependencies = list()
for replaceFrom, replaceTo in replaceFromToPairs:
for replaceFrom, replaceTo, libdir in replaceFromToPairs:
replaceFrom = str(Path(replaceFrom))
replaceTo = str(Path(replaceTo))
fileName = ResolveExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
if str(fileName).startswith('/usr'):
# print(f'skipping fileName: {fileName}')
continue
if lib.find(replaceFrom) >= 0:
if libdir:
frameworkPath = FindFramework(lib, replaceFrom)
else:
frameworkPath = lib
destFrameworkPath = frameworkPath.replace(replaceFrom, replaceTo)
destFrameworkPath = ReplaceExecutablePath(destFrameworkPath, executable_path)
if not FileExists(lib.replace(replaceFrom, replaceTo), executable_path):
destFrameworkPath = ResolveExecutablePath(destFrameworkPath, executable_path)
if not fileName.exists():
print (lib.replace(replaceFrom, replaceTo), "DOES NOT EXIST")
print ("COPY", frameworkPath, " -> ", destFrameworkPath)
shutil.copytree(frameworkPath, destFrameworkPath)
fileName = ReplaceExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
nameId = lib.replace(replaceFrom, replaceTo)
command = "%s %s %s" % ( cmdNameId, nameId, fileName )
if not os.access(fileName, os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
print("\t%s" % command)
# print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
fileName = ReplaceExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
for dependency in dependencies:
if dependency.find(replaceFrom) >= 0:
print("In:", fileName)
print("\tIn:", fileName)
print("\tRENAME", dependency, " -> ", dependency.replace(replaceFrom, replaceTo))
# Try changing id first
@ -333,7 +334,7 @@ if CAN_DEPLOY_PYTHON:
command = "%s %s %s" % ( cmdNameId, nameId, fileName )
if not os.access(fileName, os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
print("\t%s" % command)
# print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
@ -346,7 +347,7 @@ if CAN_DEPLOY_PYTHON:
if not os.access(fileName, os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
print("\t%s" % command)
# print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )

2
src/lay/lay/laySaltDownloadManager.cc
View File

@ -68,7 +68,7 @@ ConfirmationDialog::add_info (const std::string &name, bool update, const std::s
item->setText (3, tl::to_qstring (url));
for (int column = 0; column < list->colorCount (); ++column) {
item->setData (column, Qt::ForegroundRole, QVariant (QBrush (update ? Qt::blue : Qt::black)));
item->setData (column, Qt::ForegroundRole, QVariant (QBrush (update ? QColor (Qt::blue) : QColor (Qt::black))));
}
}

98
test-pydeploy.py Normal file
View File

@ -0,0 +1,98 @@
import os, subprocess
import sys
from macbuild.build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
from pathlib import Path
# bundlePath = AbsMacPkgDir
bundlePath = os.getcwd() + '/qt5.pkg.macos-HighSierra-release/klayout.app'
bundleExecPathAbs = '%s/Contents/MacOS/' % bundlePath
pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
print("[1] Copying Python Framework")
shell_commands = list()
shell_commands.append(f"rm -rf {pythonFrameworkPath}")
shell_commands.append(f"rsync -a --safe-links {pythonOriginalFrameworkPath}/ {pythonFrameworkPath}")
shell_commands.append(f"mkdir {pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
shell_commands.append(f"cp -RL {pythonOriginalFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/{{pip*,pkg_resources,setuptools*,wheel*}} " +
f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/lib/python3.6/test")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/Resources")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/bin")
for command in shell_commands:
if subprocess.call( command, shell=True ) != 0:
msg = "command failed: %s"
print( msg % command, file=sys.stderr )
exit(1)
print("[2] Relinking dylib dependencies inside Python.framework")
depdict = WalkFrameworkPaths(pythonFrameworkPath)
appPythonFrameworkPath = '@executable_path/../Frameworks/Python.framework/'
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
usrLocalPath = '/usr/local/opt/'
appUsrLocalPath = '@executable_path/../Frameworks/'
depdict = WalkFrameworkPaths(pythonFrameworkPath)
PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs, libdir=True)
print("[3] Relinking dylib dependencies for klayout")
klayoutPath = bundleExecPathAbs
depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'klayout$')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
libKlayoutPath = bundleExecPathAbs + '../Frameworks'
depdict = WalkFrameworkPaths(libKlayoutPath, filter_regex=r'libklayout')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
print("[4] Patching site.py, pip/, and distutils/")
site_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site.py"
with open(site_module, 'r') as site:
buf = site.readlines()
with open(site_module, 'w') as site:
import re
for line in buf:
# This will fool pip into thinking it's inside a virtual environment
# and install new packates to the correct site-packages
if re.match("^PREFIXES", line) is not None:
line = line + "sys.real_prefix = sys.prefix\n"
# do not allow installation in the user folder.
if re.match("^ENABLE_USER_SITE", line) is not None:
line = "ENABLE_USER_SITE = False\n"
site.write(line)
pip_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/pip/__init__.py"
with open(pip_module, 'r') as pip:
buf = pip.readlines()
with open(pip_module, 'w') as pip:
import re
for line in buf:
# this will reject user's configuration of pip, forcing the isolated mode
line = re.sub("return isolated$", "return isolated or True", line)
pip.write(line)
distutilsconfig = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/distutils/distutils.cfg"
with open(distutilsconfig, 'r') as file:
buf = file.readlines()
with open(distutilsconfig, 'w') as file:
import re
for line in buf:
# This will cause all packages to be installed to sys.prefix
if re.match('prefix=', line) is not None:
continue
file.write(line)
# pythonPath = bundleExecPathAbs + '../Frameworks/Python.framework/Versions/3.6/bin/'
# # pythonOriginalPrefixPath = '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.6'
# # appPythonBinPath = '@executable_path/../'
# depdict = WalkFrameworkPaths(pythonPath, filter_regex=r'python')
# print(depdict)
# PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath),
# (Path(pythonOriginalFrameworkPath).resolve(), appPythonFrameworkPath)], bundleExecPathAbs)
# usrLocalPath = '/usr/local/lib/'
# appUsrLocalPath = '@executable_path/../Frameworks/'
# depdict = WalkFrameworkPaths(pythonFrameworkPath)
# PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)

29
test-pylib-script.py Normal file
View File

@ -0,0 +1,29 @@
import site; print(site.getsitepackages())
import pip
installed_packages = pip.get_installed_distributions()
installed_packages_list = sorted(["%s==%s" % (i.key, i.version)
for i in installed_packages])
print(installed_packages_list)
print("-------------")
if pip.main(['install', '--upgrade', 'numpy']) > 0:
exit(1)
print("-------------")
print("Importing numpy")
import numpy
print("-------------")
import sys;
print("Executing from: ", sys.executable)
print("-------------")
import os
print("Environment variables:")
for variable, value in os.environ.items():
print(variable, ":", value)