Compilation with brew dependencies successful!

Using brew's qt and python3 formulae
Bonus: I have also added a script that embeds python into bundle, recursively adding dependencies from inside /usr/local/opt. That means that /usr/local/lib folders are not going to be copied. I saw one dependency to /usr/local/lib/gcc from one of numpy's modules, instead of pointing towards /usr/local/opt/gcc/lib/gcc.

Side effects: Renaming Qt5Custom to Qt5Brew
This commit is contained in:
Thomas Ferreira de Lima 2018-03-13 04:01:36 -04:00
parent 3295b630f3
commit e90edf4434
6 changed files with 277 additions and 68 deletions

View File

@ -86,23 +86,25 @@ $ ./makeDMG4mac.py -p qt5.pkg.macos-HighSierra-release -m
By: Kazzz (January 16, 2018)
# 5. Alternative building options
### 5.1 Python 3.6 from brew, Qt 5.9.4 from offline installer
### 5.1 Python 3.6 from brew, Qt 5.10.1 from brew
Homebrew's installation of python3 places a `Python.framework` in `/usr/local/opt/python/Frameworks/Python.framework/`, which you can use to build KLayout from. Qt can be downloaded for [offline installation](https://www1.qt.io/offline-installers/). You can place it in your home folder: e.g. `/home/username/Qt5.9.4/`. Given these two dependencies, you can successfully compile KLayout with the following commands:
Homebrew's installation of python3 (`brew install python3`) places a `Python.framework` in `/usr/local/opt/python/Frameworks/Python.framework/`, which you can use to build KLayout from. Qt can also be downloaded from brew with `brew install qt`.
```
# Build step
./build4mac.py -p B36 -q Qt5Custom
./build4mac.py -p B36 -q Qt5Brew
# Deploy step
./build4mac.py -p B36 -q Qt5Custom -y # normal deploy
./build4mac.py -p B36 -q Qt5Custom -y -v 3 2>&1 | tee qt5.pkg.macos-HighSierra-release.log # deploy with debug options
./build4mac.py -p B36 -q Qt5Brew -y # normal deploy
./build4mac.py -p B36 -q Qt5Brew -y -v 3 2>&1 | tee qt5.pkg.macos-HighSierra-release.log # deploy with debug options
# Packaging step
./makeDMG4mac.py -p qt5.pkg.macos-HighSierra-release -m -q Qt594
./makeDMG4mac.py -p qt5.pkg.macos-HighSierra-release -m -q Qt5101
```
PS: If you get a syntax error in one of ruby's libraries because it is not compatible with C++11, do not fret. You only have to change one offending file.
#### Known issues
Because we link python to `/usr/local/opt/python/Frameworks/Python.framework/`, updating python from brew might break klayout if the new version is incompatible. To fix this, it is better to link python directly to `/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework`, but that might complicate release builds, because that assumes users have the exact version of python installed by brew.
[End of File]

View File

@ -65,7 +65,7 @@ def SetGlobals():
Usage += " : 'nil' = not to support the script language | \n"
Usage += " : 'Sys' = using the OS standard script language | \n"
Usage += " : Refer to 'macbuild/build4mac_env.py' for details | \n"
Usage += " [-q|--qt <type>] : type=['Qt4MacPorts', 'Qt5MacPorts', 'Qt5Custom'] | qt5macports \n"
Usage += " [-q|--qt <type>] : type=['Qt4MacPorts', 'Qt5MacPorts', 'Qt5Brew'] | qt5macports \n"
Usage += " [-r|--ruby <type>] : type=['nil', 'Sys', 'Src24', 'MP24', 'B25'] | sys \n"
Usage += " [-p|--python <type>] : type=['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B36'] | sys \n"
Usage += " [-n|--noqtbinding] : don't create Qt bindings for ruby scripts | disabled \n"
@ -94,7 +94,7 @@ def SetGlobals():
print("")
print( "!!! Sorry. Your system <%s> looks like non-Mac" % System, file=sys.stderr )
print(Usage)
quit()
exit()
release = int( Release.split(".")[0] ) # take the first of ['14', '5', '0']
if release == 14:
@ -110,13 +110,13 @@ def SetGlobals():
print("")
print( "!!! Sorry. Unsupported major OS release <%d>" % release, file=sys.stderr )
print(Usage)
quit()
exit()
if not Machine == "x86_64":
print("")
print( "!!! Sorry. Only x86_64 architecture machine is supported but found <%s>" % Machine, file=sys.stderr )
print(Usage)
quit()
exit()
# default modules
ModuleQt = "Qt5MacPorts"
@ -236,30 +236,21 @@ def ParseCommandLineArguments():
opt, args = p.parse_args()
if (opt.checkusage):
print(Usage)
quit()
exit()
# Determine Qt type
candidates = [ i.upper() for i in ['Qt4MacPorts', 'Qt5MacPorts', 'Qt5Custom'] ]
candidates = ['Qt4MacPorts', 'Qt5MacPorts', 'Qt5Brew']
candidates_upper = [ i.upper() for i in candidates ]
ModuleQt = ""
index = 0
for item in candidates:
if opt.type_qt.upper() == item:
if index == 0:
ModuleQt = 'Qt4MacPorts'
break
elif index == 1:
ModuleQt = 'Qt5MacPorts'
break
elif index == 2:
ModuleQt = 'Qt5Custom'
break
else:
index += 1
if ModuleQt == "":
if opt.type_qt.upper() in candidates_upper:
idx = candidates_upper.index(opt.type_qt.upper())
ModuleQt = candidates[idx]
else:
print("")
print( "!!! Unknown Qt type", file=sys.stderr )
print( "!!! Unknown Qt type %s. Candidates: %s" % (opt.type_qt, candidates), file=sys.stderr )
print(Usage)
quit()
exit(1)
# By default, OS-standard script languages (Ruby and Python) are used
NonOSStdLang = False
@ -300,7 +291,7 @@ def ParseCommandLineArguments():
print("")
print( "!!! Unknown Ruby type", file=sys.stderr )
print(Usage)
quit()
exit()
# Determine Python type
candidates = [ i.upper() for i in ['nil', 'Sys', 'Ana27', 'Ana36', 'MP36', 'B36'] ]
@ -341,7 +332,7 @@ def ParseCommandLineArguments():
print("")
print( "!!! Unknown Python type", file=sys.stderr )
print(Usage)
quit()
exit()
NoQtBindings = opt.no_qt_binding
MakeOptions = opt.make_option
@ -354,14 +345,14 @@ def ParseCommandLineArguments():
print("")
print( "!!! Choose either [-y|--deploy] or [-Y|--DEPLOY]", file=sys.stderr )
print(Usage)
quit()
exit()
DeployVerbose = int(opt.deploy_verbose)
if not DeployVerbose in [0, 1, 2, 3]:
print("")
print( "!!! Unsupported verbose level passed to `macdeployqt` tool", file=sys.stderr )
print(Usage)
quit()
exit()
if not DeploymentF and not DeploymentP:
target = "%s %s %s" % (Platform, Release, Machine)
@ -436,9 +427,9 @@ def RunMainBuildBash():
AbsMacBuildLog = "%s/qt5.build.macos-%s-%s.log" % (ProjectDir, Platform, mode)
parameters += " \\\n -bin %s" % MacBinDir
parameters += " \\\n -build %s" % MacBuildDir
elif ModuleQt == 'Qt5Custom':
elif ModuleQt == 'Qt5Brew':
parameters += " \\\n -qt5"
parameters += " \\\n -qmake %s" % Qt5Custom['qmake']
parameters += " \\\n -qmake %s" % Qt5Brew['qmake']
MacPkgDir = "./qt5.pkg.macos-%s-%s" % (Platform, mode)
MacBinDir = "./qt5.bin.macos-%s-%s" % (Platform, mode)
MacBuildDir = "./qt5.build.macos-%s-%s" % (Platform, mode)
@ -486,7 +477,7 @@ def RunMainBuildBash():
command += " 2>&1 | tee %s" % MacBuildLog
if CheckComOnly:
print(command)
quit()
exit()
#-----------------------------------------------------
# [3] Invoke the main Bash script; takes time:-)
@ -662,7 +653,7 @@ def DeployBinariesForBundle():
depDicOrdinary.update(dependDic)
'''
PrintLibraryDependencyDictionary( depDicOrdinary, "Style (3)" )
quit()
exit()
'''
print( " [5] Setting and changing the identification names among KLayout's libraries ..." )
@ -699,13 +690,11 @@ def DeployBinariesForBundle():
shutil.copy2( sourceDir0 + "/PkgInfo", targetDir0 ) # this file is not mandatory
shutil.copy2( sourceDir1 + "/klayout", targetDirM )
shutil.copy2( sourceDir2 + "/klayout.icns", targetDirR )
shutil.copy2( sourceDir2 + "/qt.conf", targetDirM )
os.chmod( targetDir0 + "/PkgInfo", 0o0644 )
os.chmod( targetDir0 + "/Info.plist", 0o0644 )
os.chmod( targetDirM + "/klayout", 0o0755 )
os.chmod( targetDirM + "/qt.conf", 0o0644 )
os.chmod( targetDirR + "/klayout.icns", 0o0644 )
buddies = glob.glob( sourceDir3 + "/strm*" )
@ -733,7 +722,7 @@ def DeployBinariesForBundle():
buddies = glob.glob( "klayout.app/Contents/Buddy/strm*" )
macdepQtOpt = ""
for buddy in buddies:
macdepQtOpt += "-executable=%s " % buddy
macdepQtOpt += " -executable=%s" % buddy
ret = SetChangeLibIdentificationName( buddy, "../Frameworks" )
if not ret == 0:
os.chdir(ProjectDir)
@ -755,11 +744,15 @@ def DeployBinariesForBundle():
deploytool = Qt5MacPorts['deploy']
app_bundle = "klayout.app"
options = macdepQtOpt + verbose
elif ModuleQt == 'Qt5Custom':
deploytool = Qt5Custom['deploy']
elif ModuleQt == 'Qt5Brew':
deploytool = Qt5Brew['deploy']
app_bundle = "klayout.app"
options = macdepQtOpt + verbose
# Without the following, the plugin cocoa would not be found properly.
shutil.copy2( sourceDir2 + "/qt.conf", targetDirM )
os.chmod( targetDirM + "/qt.conf", 0o0644 )
os.chdir(ProjectDir)
os.chdir(MacPkgDir)
command = "%s %s %s" % ( deploytool, app_bundle, options )
@ -770,18 +763,51 @@ def DeployBinariesForBundle():
print("")
os.chdir(ProjectDir)
return 1
else:
print( "##### Finished deploying libraries and executables for <klayout.app> #####" )
print("")
os.chdir(ProjectDir)
return 0
deploymentPython = False
if deploymentPython:
# TODO Code incomplete! To be finished.
from macbuild.build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
bundlePath = 'qt5.pkg.macos-HighSierra-release/klayout.app'
bundleExecPathAbs = os.getcwd() + '/%s/Contents/MacOS/' % bundlePath
# rsync -a --safe-links /usr/local/opt/python/Frameworks/Python.framework/ qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework
# cp -RL /usr/local/opt/python/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework/Versions/3.6/lib/python3.6/
# rm -rf qt5.pkg.macos-HighSierra-release/klayout.app/Contents/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/test
pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
depdict = WalkFrameworkPaths(pythonFrameworkPath)
pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
appPythonFrameworkPath = '@executable_path/../Frameworks/Python.framework/'
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
klayoutPath = bundleExecPathAbs
depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'klayout$')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
klayoutPath = bundleExecPathAbs + '../Frameworks'
depdict = WalkFrameworkPaths(klayoutPath, filter_regex=r'libklayout')
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath)], bundleExecPathAbs)
usrLocalPath = '/usr/local/opt/'
appUsrLocalPath = '@executable_path/../Frameworks/'
depdict = WalkFrameworkPaths(pythonFrameworkPath)
PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)
# usrLocalPath = '/usr/local/lib/'
# appUsrLocalPath = '@executable_path/../Frameworks/'
# depdict = WalkFrameworkPaths(pythonFrameworkPath)
# PerformChanges(depdict, [(usrLocalPath, appUsrLocalPath)], bundleExecPathAbs)
else:
print( " [8] Skipped deploying Qt's Frameworks ..." )
print( "##### Finished deploying libraries and executables for <klayout.app> #####" )
print("")
os.chdir(ProjectDir)
return 0
print( "##### Finished deploying libraries and executables for <klayout.app> #####" )
print("")
os.chdir(ProjectDir)
return 0
#------------------------------------------------------------------------------
## To deploy script bundles that invoke the main bundle (klayout.app) in
# editor mode or viewer mode

View File

@ -21,7 +21,7 @@ XcodeToolChain = { 'nameID': '/usr/bin/install_name_tool -id ',
#-----------------------------------------------------
# [1] Qt
#-----------------------------------------------------
Qts = [ 'Qt4MacPorts', 'Qt5MacPorts', 'Qt5Custom' ]
Qts = [ 'Qt4MacPorts', 'Qt5MacPorts', 'Qt5Brew' ]
#-----------------------------------------------------
# Whereabout of different components of Qt4
@ -41,17 +41,15 @@ Qt5MacPorts = { 'qmake' : '/opt/local/libexec/qt5/bin/qmake',
'deploy': '/opt/local/libexec/qt5/bin/macdeployqt'
}
# # Qt5 from Brew
# # [Key Type Name] = 'Qt5Brew'
# Qt5Brew = { 'qmake' : '/usr/local/Cellar/qt5/5.10.0_1/bin/qmake',
# 'deploy': '/usr/local/Cellar/qt5/5.10.0_1/bin/macdeployqt'
# }
# Qt5 from Homebrew (https://brew.sh/)
# install with 'brew install qt'
# [Key Type Name] = 'Qt5Brew'
Qt5Brew = { 'qmake' : '/usr/local/opt/qt/bin/qmake',
'deploy': '/usr/local/opt/qt/bin/macdeployqt'
}
# Qt5 Custom (https://www1.qt.io/offline-installers/)
# [Key Type Name] = 'Qt5Custom'
Qt5Custom = { 'qmake' : '~/Qt5.9.4/5.9.4/clang_64/bin/qmake',
'deploy': '~/Qt5.9.4/5.9.4/clang_64/bin/macdeployqt'
}
#-----------------------------------------------------
# [2] Ruby
@ -192,9 +190,9 @@ Python36MacPorts= { 'exe': '/opt/local/Library/Frameworks/Python.framework/Versi
# Python 3.6 from Brew *+*+*+ EXPERIMENTAL *+*+*+
# [Key Type Name] = 'pybrew'
Python36Brew= { 'exe': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/Current/bin/python3.6m' ,
'inc': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/Current/include/python3.6m',
'lib': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/Current/lib/libpython3.6m.dylib'
Python36Brew= { 'exe': '/usr/local/opt/python/libexec/bin/python' ,
'inc': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/3.6/Headers',
'lib': '/usr/local/opt/python3/Frameworks/Python.framework/Versions/3.6/Python'
}
# Consolidated dictionary kit for Python

Binary file not shown.

View File

@ -16,6 +16,15 @@ import os
import re
import string
import subprocess
import shutil
CAN_DEPLOY_PYTHON = False
try:
from pathlib import Path
Path('~').expanduser()
CAN_DEPLOY_PYTHON = True
except (ImportError,AttributeError): # python2
print("Warning: Cannot import pathlib, use python3 if you need python deployment.")
#-------------------------------------------------------------------------------
## To import global dictionaries of different modules
@ -44,10 +53,10 @@ from build4mac_env import *
def DecomposeLibraryDependency( depstr ):
alllines = depstr.split('\n')
numlines = len(alllines)
dependent = alllines[0].split(':')[0].strip(' ').strip('\t')
dependent = alllines[0].split(':')[0].strip()
supporters = []
for i in range(1, numlines):
supporter = alllines[i].split(' ')[0].strip(' ').strip('\t')
for line in alllines[1:]:
supporter = line.strip().split(' ')[0].strip()
if not supporter == '':
supporters.append(supporter)
return { dependent: supporters }
@ -107,6 +116,8 @@ def SetChangeIdentificationNameOfDyLib( libdic ):
# for-lib
return 0
#------------------------------------------------------------------------------
## To set the identification names of KLayout's libraries to an executable
# and make the application aware of the library locations
@ -170,6 +181,178 @@ def SetChangeLibIdentificationName( executable, relativedir ):
# for-lib
return 0
# TODO: undocumented
if CAN_DEPLOY_PYTHON:
def WalkLibDependencyTree( dylibPath, depth=0, filter_regex=r'\t+/usr/local/opt'):
NOTHINGTODO = [] # return empty list if nothing to do.
dylibPath = str(Path(dylibPath))
cmdNameId = XcodeToolChain['nameID']
cmdNameChg = XcodeToolChain['nameCH']
otoolCm = 'otool -L %s | grep -E "%s"' % (dylibPath, filter_regex)
otoolOut = os.popen( otoolCm ).read()
exedepdic = DecomposeLibraryDependency( dylibPath + ":\n" + otoolOut )
keys = exedepdic.keys()
deplibs = exedepdic[ list(keys)[0] ]
if depth < 5:
if len(deplibs) > 0:
for idx, lib in enumerate(deplibs):
lib = str(Path(lib))
if lib != list(keys)[0]:
deplibs[idx] = WalkLibDependencyTree(lib, depth+1, filter_regex)
else:
return NOTHINGTODO
if depth == 0:
return deplibs
return exedepdic
else:
raise RuntimeError("Exceeded maximum recursion depth.")
def WalkFrameworkPaths(frameworkPaths, filter_regex=r'\.(so|dylib)$'):
try:
frameworkPathsIter = iter(frameworkPaths)
except TypeError:
frameworkPathsIter = [frameworkPaths]
dependency_dict = dict()
for frameworkPath in frameworkPaths:
frameworkPath = str(Path(frameworkPath))
find_grep_results = os.popen('find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex)).read().split('\n')
framework_files = filter(lambda x: x != '',
map(lambda x: x.strip(),
find_grep_results))
dependency_dict[frameworkPath] = list()
for idx, file in enumerate(framework_files):
dict_file = {file: WalkLibDependencyTree(file)}
dependency_dict[frameworkPath].append(dict_file)
return dependency_dict
def WalkDictTree(dependencyDict, visited_files):
libNameChanges = list()
for lib, dependencies in dependencyDict.items():
if lib in visited_files:
continue
dependency_list = list()
if isinstance(dependencies, list):
for deplib in dependencies:
if isinstance(deplib, str):
dependency_list.append(deplib)
if deplib not in visited_files:
visited_files.append(deplib)
elif isinstance(deplib, dict):
dependency_list.append(str(next(iter(deplib))))
libNameChanges.extend(WalkDictTree(deplib, visited_files))
else:
#raise RuntimeError("Unexpected value: %s" % deplib)
pass
else:
raise RuntimeError("Unexpected value: %s" % dependencies)
if len(dependency_list) > 0:
libNameChanges.append((lib, dependency_list))
else:
libNameChanges.append((lib, ))
visited_files.append(lib)
return libNameChanges
def FindFramework(path, root_path):
path = Path(path)
root_path = Path(root_path)
relPath = path.relative_to(root_path)
return str(root_path / relPath.parts[0])
def ReplaceExecutablePath(path, executable_path):
executable_path = str(executable_path)
p = Path(str(path).replace("@executable_path", "/%s/" % executable_path))
return p
def FileExists(file_path, executable_path):
p = ReplaceExecutablePath(file_path, executable_path)
return p.exists()
def DetectChanges(frameworkDependencyDict):
visited_files = list()
libNameChanges = list()
for framework, libraries in frameworkDependencyDict.items():
for libraryDict in libraries:
libNameChanges.extend(WalkDictTree(libraryDict, visited_files))
# Changes are stored in libNameChanges in the form of ('lib.dylib', ['dep1.dylib', ...])
return libNameChanges
def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_path="/tmp/klayout", libdir=False):
libNameChanges = DetectChanges(frameworkDependencyDict)
cmdNameId = XcodeToolChain['nameID']
cmdNameChg = XcodeToolChain['nameCH']
if replaceFromToPairs is not None:
for libNameChange in libNameChanges:
libNameChangeIterator = iter(libNameChange)
lib = next(libNameChangeIterator)
try:
dependencies = next(libNameChangeIterator)
except StopIteration:
dependencies = list()
for replaceFrom, replaceTo in replaceFromToPairs:
replaceFrom = str(Path(replaceFrom))
replaceTo = str(Path(replaceTo))
if lib.find(replaceFrom) >= 0:
if libdir:
frameworkPath = FindFramework(lib, replaceFrom)
else:
frameworkPath = lib
destFrameworkPath = frameworkPath.replace(replaceFrom, replaceTo)
destFrameworkPath = ReplaceExecutablePath(destFrameworkPath, executable_path)
if not FileExists(lib.replace(replaceFrom, replaceTo), executable_path):
print (lib.replace(replaceFrom, replaceTo), "DOES NOT EXIST")
print ("COPY", frameworkPath, " -> ", destFrameworkPath)
shutil.copytree(frameworkPath, destFrameworkPath)
fileName = ReplaceExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
nameId = lib.replace(replaceFrom, replaceTo)
command = "%s %s %s" % ( cmdNameId, nameId, fileName )
if not os.access(fileName, os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
fileName = ReplaceExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
for dependency in dependencies:
if dependency.find(replaceFrom) >= 0:
print("In:", fileName)
print("\tRENAME", dependency, " -> ", dependency.replace(replaceFrom, replaceTo))
# Try changing id first
nameId = dependency.replace(replaceFrom, replaceTo)
command = "%s %s %s" % ( cmdNameId, nameId, fileName )
if not os.access(fileName, os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
# Rename dependencies
nameOld = dependency
nameNew = dependency.replace(replaceFrom, replaceTo)
command = "%s %s %s %s" % ( cmdNameChg, nameOld, nameNew, fileName )
if not os.access(fileName, os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
#------------------------------------------------------------------------------
## To get KLayout's version from a file; most likely from 'version.sh'
#

Binary file not shown.