Fix Mac OS build scripts to work with both Python 2 and Python 3

This commit is contained in:
Daniel Wang 2018-08-02 17:47:39 -04:00
parent 5efe5d20dd
commit ba3720ad4a
No known key found for this signature in database
GPG Key ID: 82968CE7F0EA634E
2 changed files with 167 additions and 173 deletions

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#===============================================================================
@ -125,7 +125,7 @@ def SetGlobals():
ModulePython = "PythonYosemite"
elif Platform == "ElCapitan":
ModuleRuby = "RubyElCapitan"
ModulePytyon = "PythonElCapitan"
ModulePython = "PythonElCapitan"
elif Platform == "Sierra":
ModuleRuby = "RubySierra"
ModulePython = "PythonSierra"
@ -766,25 +766,24 @@ def DeployBinariesForBundle():
deploymentPython = True
if deploymentPython and NonOSStdLang:
from build4mac_util import WalkFrameworkPaths, PerformChanges, DetectChanges
from pathlib import Path
bundlePath = AbsMacPkgDir + '/klayout.app'
# bundlePath = os.getcwd() + '/qt5.pkg.macos-HighSierra-release/klayout.app'
bundleExecPathAbs = '%s/Contents/MacOS/' % bundlePath
pythonOriginalFrameworkPath = '/usr/local/opt/python/Frameworks/Python.framework'
pythonOriginalFrameworkPath = '/usr/local/opt/python3/Frameworks/Python.framework'
pythonFrameworkPath = '%s/Contents/Frameworks/Python.framework' % bundlePath
print(f" [8.1] Deploying Python from {pythonOriginalFrameworkPath} ..." )
print(" [8.1] Deploying Python from %s ..." % pythonOriginalFrameworkPath)
print(" [1] Copying Python Framework")
shell_commands = list()
shell_commands.append(f"rm -rf {pythonFrameworkPath}")
shell_commands.append(f"rsync -a --safe-links {pythonOriginalFrameworkPath}/ {pythonFrameworkPath}")
shell_commands.append(f"mkdir {pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
shell_commands.append(f"cp -RL {pythonOriginalFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/{{pip*,pkg_resources,setuptools*,wheel*}} " +
f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/lib/python3.6/test")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/Resources")
shell_commands.append(f"rm -rf {pythonFrameworkPath}/Versions/3.6/bin")
shell_commands.append("rm -rf %s" % pythonFrameworkPath)
shell_commands.append("rsync -a --safe-links %s/ %s" % (pythonOriginalFrameworkPath, pythonFrameworkPath))
shell_commands.append("mkdir %s/Versions/3.7/lib/python3.7/site-packages/" % pythonFrameworkPath)
shell_commands.append("cp -RL %s/Versions/3.7/lib/python3.7/site-packages/{pip*,pkg_resources,setuptools*,wheel*} " % pythonOriginalFrameworkPath +
"%s/Versions/3.7/lib/python3.7/site-packages/" % pythonFrameworkPath)
shell_commands.append("rm -rf %s/Versions/3.7/lib/python3.7/test" % pythonFrameworkPath)
shell_commands.append("rm -rf %s/Versions/3.7/Resources" % pythonFrameworkPath)
shell_commands.append("rm -rf %s/Versions/3.7/bin" % pythonFrameworkPath)
for command in shell_commands:
if subprocess.call( command, shell=True ) != 0:
@ -834,7 +833,7 @@ def DeployBinariesForBundle():
PerformChanges(depdict, [(pythonOriginalFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs)
print(" [4] Patching site.py, pip/, and distutils/")
site_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site.py"
site_module = "%s/Versions/3.7/lib/python3.7/site.py" % pythonFrameworkPath
with open(site_module, 'r') as site:
buf = site.readlines()
with open(site_module, 'w') as site:
@ -849,7 +848,7 @@ def DeployBinariesForBundle():
line = "ENABLE_USER_SITE = False\n"
site.write(line)
pip_module = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/site-packages/pip/__init__.py"
pip_module = "%s/Versions/3.7/lib/python3.7/site-packages/pip/__init__.py" % pythonFrameworkPath
with open(pip_module, 'r') as pip:
buf = pip.readlines()
with open(pip_module, 'w') as pip:
@ -859,7 +858,7 @@ def DeployBinariesForBundle():
line = re.sub("return isolated$", "return isolated or True", line)
pip.write(line)
distutilsconfig = f"{pythonFrameworkPath}/Versions/3.6/lib/python3.6/distutils/distutils.cfg"
distutilsconfig = "%s/Versions/3.7/lib/python3.7/distutils/distutils.cfg" % pythonFrameworkPath
with open(distutilsconfig, 'r') as file:
buf = file.readlines()
with open(distutilsconfig, 'w') as file:
@ -993,9 +992,7 @@ def main():
#----------------------------------------------------------
ret = RunMainBuildBash()
if not DeploymentF and not DeploymentP:
if ret == 0:
sys.exit(0)
else:
if not ret == 0:
sys.exit(1)
else:
#----------------------------------------------------------
@ -1012,6 +1009,7 @@ def main():
# to make "KLayoutEditor.app" and "KLayoutViewer.app"
#----------------------------------------------------------
ret2 = DeployScriptBundles()
if not ret2 == 0:
sys.exit(1)
else:

View File

@ -18,13 +18,11 @@ import string
import subprocess
import shutil
CAN_DEPLOY_PYTHON = False
try:
from pathlib import Path
Path('~').expanduser()
CAN_DEPLOY_PYTHON = True
except (ImportError,AttributeError): # python2
print("Warning: Cannot import pathlib, use python3 if you need python deployment.")
from pathlib2 import Path
#-------------------------------------------------------------------------------
## To import global dictionaries of different modules
@ -181,177 +179,175 @@ def SetChangeLibIdentificationName( executable, relativedir ):
# for-lib
return 0
# TODO: undocumented
if CAN_DEPLOY_PYTHON:
def WalkLibDependencyTree( dylibPath, depth=0, filter_regex=r'\t+/usr/local/opt'):
NOTHINGTODO = [] # return empty list if nothing to do.
dylibPath = str(Path(dylibPath))
cmdNameId = XcodeToolChain['nameID']
cmdNameChg = XcodeToolChain['nameCH']
otoolCm = 'otool -L %s | grep -E "%s"' % (dylibPath, filter_regex)
otoolOut = os.popen( otoolCm ).read()
exedepdic = DecomposeLibraryDependency( dylibPath + ":\n" + otoolOut )
keys = exedepdic.keys()
deplibs = exedepdic[ list(keys)[0] ]
def WalkLibDependencyTree( dylibPath, depth=0, filter_regex=r'\t+/usr/local/opt'):
NOTHINGTODO = [] # return empty list if nothing to do.
dylibPath = str(Path(dylibPath))
cmdNameId = XcodeToolChain['nameID']
cmdNameChg = XcodeToolChain['nameCH']
otoolCm = 'otool -L %s | grep -E "%s"' % (dylibPath, filter_regex)
otoolOut = os.popen( otoolCm ).read()
exedepdic = DecomposeLibraryDependency( dylibPath + ":\n" + otoolOut )
keys = exedepdic.keys()
deplibs = exedepdic[ list(keys)[0] ]
if depth < 5:
if len(deplibs) > 0:
for idx, lib in enumerate(deplibs):
lib = str(Path(lib))
if lib != list(keys)[0]:
deplibs[idx] = WalkLibDependencyTree(lib, depth+1, filter_regex)
if depth == 0:
return deplibs
return exedepdic
if depth < 5:
if len(deplibs) > 0:
for idx, lib in enumerate(deplibs):
lib = str(Path(lib))
if lib != list(keys)[0]:
deplibs[idx] = WalkLibDependencyTree(lib, depth+1, filter_regex)
if depth == 0:
return deplibs
return exedepdic
else:
raise RuntimeError("Exceeded maximum recursion depth.")
def WalkFrameworkPaths(frameworkPaths, filter_regex=r'\.(so|dylib)$', search_path_filter=r'\t+/usr/local/opt'):
if isinstance(frameworkPaths, str):
frameworkPathsIter = [frameworkPaths]
else:
frameworkPathsIter = frameworkPaths
dependency_dict = dict()
for frameworkPath in frameworkPathsIter:
frameworkPath = str(Path(frameworkPath))
# print("Calling:", 'find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex))
find_grep_results = os.popen('find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex)).read().split('\n')
framework_files = filter(lambda x: x != '',
map(lambda x: x.strip(),
find_grep_results))
dependency_dict[frameworkPath] = list()
for idx, file in enumerate(framework_files):
dict_file = {file: WalkLibDependencyTree(file, filter_regex=search_path_filter)}
dependency_dict[frameworkPath].append(dict_file)
return dependency_dict
def WalkDictTree(dependencyDict, visited_files):
libNameChanges = list()
for lib, dependencies in dependencyDict.items():
if lib in visited_files:
continue
dependency_list = list()
if isinstance(dependencies, list):
for deplib in dependencies:
if isinstance(deplib, str):
dependency_list.append(deplib)
if deplib not in visited_files:
visited_files.append(deplib)
elif isinstance(deplib, dict):
dependency_list.append(str(next(iter(deplib))))
libNameChanges.extend(WalkDictTree(deplib, visited_files))
else:
#raise RuntimeError("Unexpected value: %s" % deplib)
pass
else:
raise RuntimeError("Exceeded maximum recursion depth.")
def WalkFrameworkPaths(frameworkPaths, filter_regex=r'\.(so|dylib)$', search_path_filter=r'\t+/usr/local/opt'):
if isinstance(frameworkPaths, str):
frameworkPathsIter = [frameworkPaths]
raise RuntimeError("Unexpected value: %s" % dependencies)
if len(dependency_list) > 0:
libNameChanges.append((lib, dependency_list))
else:
frameworkPathsIter = frameworkPaths
libNameChanges.append((lib, ))
visited_files.append(lib)
return libNameChanges
dependency_dict = dict()
def FindFramework(path, root_path):
path = Path(path)
root_path = Path(root_path)
relPath = path.relative_to(root_path)
return str(root_path / relPath.parts[0])
for frameworkPath in frameworkPathsIter:
frameworkPath = str(Path(frameworkPath))
# print("Calling:", 'find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex))
find_grep_results = os.popen('find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex)).read().split('\n')
framework_files = filter(lambda x: x != '',
map(lambda x: x.strip(),
find_grep_results))
def ResolveExecutablePath(path, executable_path):
""" Transforms @executable_path into executable_path"""
executable_path = str(executable_path)
p = Path(str(path).replace("@executable_path", "/%s/" % executable_path))
return p
dependency_dict[frameworkPath] = list()
for idx, file in enumerate(framework_files):
dict_file = {file: WalkLibDependencyTree(file, filter_regex=search_path_filter)}
dependency_dict[frameworkPath].append(dict_file)
return dependency_dict
def DetectChanges(frameworkDependencyDict):
visited_files = list()
libNameChanges = list()
for framework, libraries in frameworkDependencyDict.items():
for libraryDict in libraries:
libNameChanges.extend(WalkDictTree(libraryDict, visited_files))
# Changes are stored in libNameChanges in the form of ('lib.dylib', ['dep1.dylib', ...])
def WalkDictTree(dependencyDict, visited_files):
libNameChanges = list()
for lib, dependencies in dependencyDict.items():
if lib in visited_files:
continue
return libNameChanges
dependency_list = list()
if isinstance(dependencies, list):
for deplib in dependencies:
if isinstance(deplib, str):
dependency_list.append(deplib)
if deplib not in visited_files:
visited_files.append(deplib)
elif isinstance(deplib, dict):
dependency_list.append(str(next(iter(deplib))))
libNameChanges.extend(WalkDictTree(deplib, visited_files))
def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_path="/tmp/klayout"):
libNameChanges = DetectChanges(frameworkDependencyDict)
cmdNameId = XcodeToolChain['nameID']
cmdNameChg = XcodeToolChain['nameCH']
if replaceFromToPairs is not None:
for libNameChange in libNameChanges:
libNameChangeIterator = iter(libNameChange)
lib = next(libNameChangeIterator)
try:
dependencies = next(libNameChangeIterator)
except StopIteration:
dependencies = list()
for replaceFrom, replaceTo, libdir in replaceFromToPairs:
replaceFrom = str(Path(replaceFrom))
replaceTo = str(Path(replaceTo))
fileName = ResolveExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
if str(fileName).startswith('/usr'):
# print(f'skipping fileName: {fileName}')
continue
if lib.find(replaceFrom) >= 0:
if libdir:
frameworkPath = FindFramework(lib, replaceFrom)
else:
#raise RuntimeError("Unexpected value: %s" % deplib)
pass
else:
raise RuntimeError("Unexpected value: %s" % dependencies)
if len(dependency_list) > 0:
libNameChanges.append((lib, dependency_list))
else:
libNameChanges.append((lib, ))
visited_files.append(lib)
return libNameChanges
frameworkPath = lib
destFrameworkPath = frameworkPath.replace(replaceFrom, replaceTo)
destFrameworkPath = ResolveExecutablePath(destFrameworkPath, executable_path)
def FindFramework(path, root_path):
path = Path(path)
root_path = Path(root_path)
relPath = path.relative_to(root_path)
return str(root_path / relPath.parts[0])
if not fileName.exists():
print (lib.replace(replaceFrom, replaceTo), "DOES NOT EXIST")
print ("COPY", frameworkPath, " -> ", destFrameworkPath)
shutil.copytree(frameworkPath, str(destFrameworkPath))
def ResolveExecutablePath(path, executable_path):
""" Transforms @executable_path into executable_path"""
executable_path = str(executable_path)
p = Path(str(path).replace("@executable_path", "/%s/" % executable_path))
return p
nameId = lib.replace(replaceFrom, replaceTo)
command = "%s %s %s" % ( cmdNameId, nameId, fileName )
if not os.access(str(fileName), os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
# print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
def DetectChanges(frameworkDependencyDict):
visited_files = list()
libNameChanges = list()
for framework, libraries in frameworkDependencyDict.items():
for libraryDict in libraries:
libNameChanges.extend(WalkDictTree(libraryDict, visited_files))
# Changes are stored in libNameChanges in the form of ('lib.dylib', ['dep1.dylib', ...])
for dependency in dependencies:
if dependency.find(replaceFrom) >= 0:
print("\tIn:", fileName)
print("\tRENAME", dependency, " -> ", dependency.replace(replaceFrom, replaceTo))
return libNameChanges
def PerformChanges(frameworkDependencyDict, replaceFromToPairs=None, executable_path="/tmp/klayout"):
libNameChanges = DetectChanges(frameworkDependencyDict)
cmdNameId = XcodeToolChain['nameID']
cmdNameChg = XcodeToolChain['nameCH']
if replaceFromToPairs is not None:
for libNameChange in libNameChanges:
libNameChangeIterator = iter(libNameChange)
lib = next(libNameChangeIterator)
try:
dependencies = next(libNameChangeIterator)
except StopIteration:
dependencies = list()
for replaceFrom, replaceTo, libdir in replaceFromToPairs:
replaceFrom = str(Path(replaceFrom))
replaceTo = str(Path(replaceTo))
fileName = ResolveExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path)
if str(fileName).startswith('/usr'):
# print(f'skipping fileName: {fileName}')
continue
if lib.find(replaceFrom) >= 0:
if libdir:
frameworkPath = FindFramework(lib, replaceFrom)
else:
frameworkPath = lib
destFrameworkPath = frameworkPath.replace(replaceFrom, replaceTo)
destFrameworkPath = ResolveExecutablePath(destFrameworkPath, executable_path)
if not fileName.exists():
print (lib.replace(replaceFrom, replaceTo), "DOES NOT EXIST")
print ("COPY", frameworkPath, " -> ", destFrameworkPath)
shutil.copytree(frameworkPath, destFrameworkPath)
nameId = lib.replace(replaceFrom, replaceTo)
command = "%s %s %s" % ( cmdNameId, nameId, fileName )
if not os.access(fileName, os.W_OK):
# Try changing id first
nameId = dependency.replace(replaceFrom, replaceTo)
command = "%s %s %s" % ( cmdNameId, nameId, fileName)
if not os.access(str(fileName), os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
# print("\t%s" % command)
# print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
for dependency in dependencies:
if dependency.find(replaceFrom) >= 0:
print("\tIn:", fileName)
print("\tRENAME", dependency, " -> ", dependency.replace(replaceFrom, replaceTo))
# Rename dependencies
nameOld = dependency
nameNew = dependency.replace(replaceFrom, replaceTo)
command = "%s %s %s %s" % ( cmdNameChg, nameOld, nameNew, str(fileName) )
if not os.access(str(fileName), os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
# Try changing id first
nameId = dependency.replace(replaceFrom, replaceTo)
command = "%s %s %s" % ( cmdNameId, nameId, fileName )
if not os.access(fileName, os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
# print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
# Rename dependencies
nameOld = dependency
nameNew = dependency.replace(replaceFrom, replaceTo)
command = "%s %s %s %s" % ( cmdNameChg, nameOld, nameNew, fileName )
if not os.access(fileName, os.W_OK):
command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName)
# print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
# print("\t%s" % command)
if subprocess.call( command, shell=True ) != 0:
msg = "!!! Failed to set the new identification name to <%s> !!!"
print( msg % fileName, file=sys.stderr )
return 1
#------------------------------------------------------------------------------