#! /usr/bin/env python3 # -*- coding: utf-8 -*- import sys import os import shutil import glob import platform import optparse import subprocess #------------------------------------------------------------------------------ ## To test if the platform is a member of valid platforms # # @param[in] platforms valid platforms # # @return matching platform name on success; "" on failure #------------------------------------------------------------------------------ def Test_My_Platform( platforms=[ 'Monterey', 'Ventura', 'Sonoma' ] ): (System, Node, Release, MacVersion, Machine, Processor) = platform.uname() if not System == "Darwin": return "" release = int( Release.split(".")[0] ) # take the first of ['21', '0', '0'] if release == 23: Platform = "Sonoma" elif release == 22: Platform = "Ventura" elif release == 21: Platform = "Monterey" else: Platform = "" if Platform in platforms: return Platform else: return "" #------------------------------------------------------------------------------ ## To populate the build target dictionary # # @return a dictionary; key=integer, value=mnemonic #------------------------------------------------------------------------------ def Get_Build_Target_Dict(): buildTargetDic = dict() # buildTargetDic[0] = 'std' buildTargetDic[1] = 'ports' buildTargetDic[2] = 'brew' buildTargetDic[3] = 'brewHW' buildTargetDic[4] = 'ana3' buildTargetDic[5] = 'brewA' buildTargetDic[6] = 'brewAHW' return buildTargetDic #------------------------------------------------------------------------------ ## To get the build option dictionary # # @param[in] targetDic build target dictionary # @param[in] platform platform name # # @return (dictionary1, dictionary2)-tupple # dictionary1: key=mnemonic, value=build option list # dictionary2: key=mnemonic, value=log file name #------------------------------------------------------------------------------ def Get_Build_Options( targetDic, platform ): if QtType == 5: qtType = "Qt5" else: qtType = "Qt6" buildOp = dict() logfile = dict() for key in targetDic.keys(): target = targetDic[key] if target == "std": # use 'Qt5MacPorts' that provides Qt 5.15.2~ to run on "Big Sur", too buildOp["std"] = [ '-q', '%sMacPorts' % qtType, '-r', 'sys', '-p', 'sys' ] logfile["std"] = "%sMP.build.macos-%s-%s-%s.log" % (qtType.lower(), platform, "release", "RsysPsys") elif target == "ports": buildOp["ports"] = [ '-q', '%sMacPorts' % qtType, '-r', 'MP32', '-p', 'MP311' ] logfile["ports"] = "%sMP.build.macos-%s-%s-%s.log" % (qtType.lower(), platform, "release", "Rmp32Pmp311") elif target == "brew": buildOp["brew"] = [ '-q', '%sBrew' % qtType, '-r', 'HB32', '-p', 'HB311' ] logfile["brew"] = "%sBrew.build.macos-%s-%s-%s.log" % (qtType.lower(), platform, "release", "Rhb32Phb311") elif target == "brewHW": buildOp["brewHW"] = [ '-q', '%sBrew' % qtType, '-r', 'sys', '-p', 'HB311' ] logfile["brewHW"] = "%sBrew.build.macos-%s-%s-%s.log" % (qtType.lower(), platform, "release", "RsysPhb311") elif target == "ana3": buildOp["ana3"] = [ '-q', '%sAna3' % qtType, '-r', 'Ana3', '-p', 'Ana3' ] logfile["ana3"] = "%sAna3.build.macos-%s-%s-%s.log" % (qtType.lower(), platform, "release", "Rana3Pana3") elif target == "brewA": buildOp["brewA"] = [ '-q', '%sBrew' % qtType, '-r', 'HB32', '-p', 'HBAuto' ] logfile["brewA"] = "%sBrew.build.macos-%s-%s-%s.log" % (qtType.lower(), platform, "release", "Rhb32Phbauto") elif target == "brewAHW": buildOp["brewAHW"] = [ '-q', '%sBrew' % qtType, '-r', 'sys', '-p', 'HBAuto' ] logfile["brewAHW"] = "%sBrew.build.macos-%s-%s-%s.log" % (qtType.lower(), platform, "release", "RsysPhbauto") if WithPymod: buildOp["ports"] = buildOp["ports"] + ['--buildPymod'] buildOp["brew"] = buildOp["brew"] + ['--buildPymod'] buildOp["ana3"] = buildOp["ana3"] + ['--buildPymod'] return (buildOp, logfile) #------------------------------------------------------------------------------ ## To get the ".macQAT" dictionary for QA Test # # @param[in] targetDic build target dictionary # @param[in] platform platform name # # @return a dictionary; key=mnemonic, value=".macQAT" directory #------------------------------------------------------------------------------ def Get_QAT_Directory( targetDic, platform ): if QtType == 5: qtType = "qt5" else: qtType = "qt6" dirQAT = dict() for key in targetDic.keys(): target = targetDic[key] if target == "ports": dirQAT["ports"] = '%sMP.build.macos-%s-release-Rmp32Pmp311.macQAT' % (qtType, platform) elif target == "brew": dirQAT["brew"] = '%sBrew.build.macos-%s-release-Rhb32Phb311.macQAT' % (qtType, platform) elif target == "brewHW": dirQAT["brewHW"] = '%sBrew.build.macos-%s-release-RsysPhb311.macQAT' % (qtType, platform) elif target == "ana3": dirQAT["ana3"] = '%sAna3.build.macos-%s-release-Rana3Pana3.macQAT' % (qtType, platform) elif target == "brewA": dirQAT["brewA"] = '%sBrew.build.macos-%s-release-Rhb32Phbauto.macQAT' % (qtType, platform) elif target == "brewAHW": dirQAT["brewAHW"] = '%sBrew.build.macos-%s-release-RsysPhbauto.macQAT' % (qtType, platform) return dirQAT #------------------------------------------------------------------------------ ## To get the build option dictionary for making/cleaning DMG # # @param[in] targetDic build target dictionary # @param[in] platform platform name # @param[in] srlDMG serial number of DMG # @param[in] makeflag True to make; False to clean # # @return a dictionary; key=mnemonic, value=build option list #------------------------------------------------------------------------------ def Get_Package_Options( targetDic, platform, srlDMG, makeflag ): if QtType == 5: qtType = "qt5" else: qtType = "qt6" if makeflag: flag = '-m' else: flag = '-c' packOp = dict() for key in targetDic.keys(): target = targetDic[key] if target == "ports": packOp["ports"] = [ '-p', 'LW-%sMP.pkg.macos-%s-release-Rmp32Pmp311' % (qtType, platform), '-s', '%d' % srlDMG, '%s' % flag ] elif target == "brew": packOp["brew"] = [ '-p', 'LW-%sBrew.pkg.macos-%s-release-Rhb32Phb311' % (qtType, platform), '-s', '%d' % srlDMG, '%s' % flag ] elif target == "brewHW": packOp["brewHW"] = [ '-p', 'HW-%sBrew.pkg.macos-%s-release-RsysPhb311' % (qtType, platform), '-s', '%d' % srlDMG, '%s' % flag ] elif target == "ana3": packOp["ana3"] = [ '-p', 'LW-%sAna3.pkg.macos-%s-release-Rana3Pana3' % (qtType, platform), '-s', '%d' % srlDMG, '%s' % flag ] elif target == "brewA": packOp["brewA"] = [ '-p', 'LW-%sBrew.pkg.macos-%s-release-Rhb32Phbauto' % (qtType, platform), '-s', '%d' % srlDMG, '%s' % flag ] elif target == "brewAHW": packOp["brewAHW"] = [ '-p', 'HW-%sBrew.pkg.macos-%s-release-RsysPhbauto' % (qtType, platform), '-s', '%d' % srlDMG, '%s' % flag ] return packOp #------------------------------------------------------------------------------ ## To parse the command line arguments #------------------------------------------------------------------------------ def Parse_CommandLine_Arguments(): global Usage # usage global QtType # Qt type global Target # target list global Build # operation flag global WithPymod # operation flag global QATest # operation flag global QACheck # operation flag global MakeDMG # operation flag global CleanDMG # operation flag global Upload # operation flag global SrlDMG # DMG serial number global Dropbox # Dropbox directory global DryRun # True for dry-run platform = Test_My_Platform() if platform in [ "Sonoma", "Ventura", "Monterey" ]: targetopt = "1,2,3,4" else: targetopt = "0" Usage = "\n" Usage += "----------------------------------------------------------------------------------------------------------\n" Usage += " nightlyBuild.py [EXPERIMENTAL]\n" Usage += " << To execute the jobs for making KLayout's DMGs for\n" Usage += " macOS Monterey, Ventura, or Sonoma >>\n" Usage += "\n" Usage += "$ [python] nightlyBuild.py\n" Usage += " option & argument : comment on option if any | default value\n" Usage += " ------------------------------------------------------------------------+--------------\n" Usage += " [--qt ] : 5='qt5', 6='qt6' (migration to Qt6 is ongoing) | 5\n" Usage += " [--target ] : 1='ports', 2='brew', 3='brewHW', 4='ana3', | '%s'\n" % targetopt Usage += " 5='brewA', 6='brewAHW' |\n" Usage += " * with --qt=6, use --target='2,3' (4 is ignored) |\n" Usage += " [--build] : build and deploy | disabled\n" Usage += " [--pymod] : build and deploy Pymod, too | disabled\n" Usage += " [--test] : run the QA Test | disabled\n" Usage += " [--check] : check the QA Test results | disabled\n" Usage += " [--makedmg|--cleandmg ] : make or clean DMGs | disabled\n" Usage += " [--upload ] : upload DMGs to $HOME/Dropbox/klayout/ | disabled\n" Usage += " [--dryrun] : dry-run for --build option | disabled\n" Usage += " [-?|--?] : print this usage and exit | disabled\n" Usage += " |\n" Usage += " To use this script, make a symbolic link in the project root by: |\n" Usage += " $ ln -s ./macbuild/nightlyBuild.py . |\n" Usage += " |\n" Usage += " Regular sequence for using this script: |\n" Usage += " (1) $ ./nightlyBuild.py --build --pymod |\n" Usage += " (2) (confirm the build results) |\n" Usage += " (3) $ ./nightlyBuild.py --test |\n" Usage += " (4) $ ./nightlyBuild.py --check (confirm the QA Test results) |\n" Usage += " (5) $ ./nightlyBuild.py --makedmg 1 |\n" Usage += " (6) $ ./nightlyBuild.py --upload '0.28.12' |\n" Usage += " (7) $ ./nightlyBuild.py --cleandmg 1 |\n" Usage += "---------------------------------------------------------------------------+------------------------------\n" p = optparse.OptionParser( usage=Usage ) p.add_option( '--qt', dest='qt_type', help='Qt5 or Qt6 (5)' ) p.add_option( '--target', dest='targets', help='build target list' ) p.add_option( '--build', action='store_true', dest='build', default=False, help='build and deploy' ) p.add_option( '--pymod', action='store_true', dest='with_pymod', default=False, help='build and deploy Pymod, too ' ) p.add_option( '--test', action='store_true', dest='qa_test', default=False, help='run the QA Test' ) p.add_option( '--check', action='store_true', dest='qa_check', default=False, help='check the QA Test results' ) p.add_option( '--makedmg', dest='makedmg', help='make DMG' ) p.add_option( '--cleandmg', dest='cleandmg', help='clean DMG' ) p.add_option( '--upload', dest='upload', help='upload to Dropbox' ) p.add_option( '--dryrun', action='store_true', dest='dry_run', default=False, help='dry-run' ) p.add_option( '-?', '--??', action='store_true', dest='checkusage', default=False, help='check usage' ) p.set_defaults( qt_type = "5", targets = "%s" % targetopt, build = False, with_pymod = False, qa_test = False, qa_check = False, makedmg = "", cleandmg = "", upload = "", dry_run = False, checkusage = False ) opt, args = p.parse_args() if opt.checkusage: print(Usage) quit() myPlatform = Test_My_Platform( [ 'Monterey', 'Ventura', 'Sonoma' ] ) if myPlatform == "": print( "! Current platform is not [ 'Monterey', 'Ventura', 'Sonoma' ]" ) print(Usage) quit() QtType = int(opt.qt_type) if not QtType in [5, 6]: print( "! Invalid Qt type <%d>" % QtType ) print(Usage) quit() targetIdx = list() for target in [ int(item) for item in opt.targets.split(",") ]: if not target in targetIdx: targetIdx.append(target) # first appeared and non-duplicated index targetDic = Get_Build_Target_Dict() Target = list() for idx in targetIdx: if idx in range(1, 7): Target.append( targetDic[idx] ) Build = opt.build WithPymod = opt.with_pymod QATest = opt.qa_test QACheck = opt.qa_check MakeDMG = False CleanDMG = False Upload = False DryRun = opt.dry_run if not opt.makedmg == "": MakeDMG = True SrlDMG = int(opt.makedmg) if not opt.cleandmg == "": CleanDMG = True SrlDMG = int(opt.cleandmg) if MakeDMG and CleanDMG: print( "! --makedmg and --cleandmg cannot be used simultaneously" ) print(Usage) quit() if not opt.upload == "": Upload = True Dropbox = opt.upload if not (Build or QATest or QACheck or MakeDMG or CleanDMG or Upload): print( "! No action selected" ) print(Usage) quit() #------------------------------------------------------------------------------ ## To build and deploy #------------------------------------------------------------------------------ def Build_Deploy(): pyBuilder = "./build4mac.py" myPlatform = Test_My_Platform() buildOp, logfile = Get_Build_Options( Get_Build_Target_Dict(), myPlatform ) for key in Target: if key == "ana3" and QtType == 6: # anaconda3 does not provide Qt6 so far continue deplog = logfile[key].replace( ".log", ".dep.log" ) command1 = [ pyBuilder ] + buildOp[key] if key in [ "brewHW", "brewAHW" ] : command2 = "time" command2 += " \\\n %s" % pyBuilder for option in buildOp[key]: command2 += " \\\n %s" % option command2 += " \\\n %s" % '-y' command2 += " 2>&1 | tee %s; \\\n" % deplog command2 += "test ${PIPESTATUS[0]} -eq 0" # tee always exits with 0 else: command2 = "time" command2 += " \\\n %s" % pyBuilder for option in buildOp[key]: command2 += " \\\n %s" % option command2 += " \\\n %s" % '-Y' command2 += " 2>&1 | tee %s; \\\n" % deplog command2 += "test ${PIPESTATUS[0]} -eq 0" # tee always exits with 0 if DryRun: print( "### Target = <%s> ###" % key ) print(command1) print(command2) print( "" ) continue if subprocess.call( command1, shell=False ) != 0: print( "", file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "!!! <%s>: failed to build KLayout" % pyBuilder, file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "", file=sys.stderr ) sys.exit(1) else: print( "", file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "### <%s>: successfully built KLayout" % pyBuilder, file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "", file=sys.stderr ) if subprocess.call( command2, shell=True ) != 0: print( "", file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "!!! <%s>: failed to deploy KLayout" % pyBuilder, file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "", file=sys.stderr ) sys.exit(1) else: print( "", file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "### <%s>: successfully deployed KLayout" % pyBuilder, file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "", file=sys.stderr ) #------------------------------------------------------------------------------ ## To run the QA tests # # @param[in] exclude test to exclude such as 'pymod,pya' #------------------------------------------------------------------------------ def Run_QATest( exclude ): pyRunnerQAT = "./macQAT.py" myPlatform = Test_My_Platform() dirQAT = Get_QAT_Directory( Get_Build_Target_Dict(), myPlatform ) for key in Target: if key == 4 and QtType == 6: # anaconda3 does not provide Qt6 so far continue command1 = [ pyRunnerQAT ] + [ '--run', '--exclude', '%s' % exclude ] print( dirQAT[key], command1 ) #continue os.chdir( dirQAT[key] ) if subprocess.call( command1, shell=False ) != 0: print( "", file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "!!! <%s>: failed to run the QA Test" % pyRunnerQAT, file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "", file=sys.stderr ) sys.exit(1) else: print( "", file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "### <%s>: successfully ran the QA Test" % pyRunnerQAT, file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "", file=sys.stderr ) os.chdir( "../" ) #------------------------------------------------------------------------------ ## To check the QA test results # # @param[in] lines number of lines to dump from the tail #------------------------------------------------------------------------------ def Check_QATest_Results( lines ): tailCommand = "/usr/bin/tail" myPlatform = Test_My_Platform() dirQAT = Get_QAT_Directory( Get_Build_Target_Dict(), myPlatform ) for key in Target: if key == 4 and QtType == 6: # anaconda3 does not provide Qt6 so far continue os.chdir( dirQAT[key] ) logfile = glob.glob( "*.log" ) command1 = [ tailCommand ] + [ '-n', '%d' % lines ] + logfile print( dirQAT[key], command1 ) #continue if subprocess.call( command1, shell=False ) != 0: print( "", file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "!!! <%s>: failed to check the QA Test results" % tailCommand, file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "", file=sys.stderr ) sys.exit(1) else: print( "", file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "### <%s>: successfully checked the QA Test results" % tailCommand, file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "", file=sys.stderr ) os.chdir( "../" ) #------------------------------------------------------------------------------ ## To make DMGs # # @param[in] srlDMG DMG's serial number #------------------------------------------------------------------------------ def DMG_Make( srlDMG ): pyDMGmaker = "./makeDMG4mac.py" stashDMG = "./DMGStash" myPlatform = Test_My_Platform() packOp = Get_Package_Options( Get_Build_Target_Dict(), myPlatform, srlDMG, makeflag=True ) if os.path.isdir( stashDMG ): shutil.rmtree( stashDMG ) os.mkdir( stashDMG ) for key in Target: if key == 4 and QtType == 6: # anaconda3 does not provide Qt6 so far continue command1 = [ pyDMGmaker ] + packOp[key] print(command1) #continue if subprocess.call( command1, shell=False ) != 0: print( "", file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "!!! <%s>: failed to make KLayout DMG" % pyDMGmaker, file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "", file=sys.stderr ) sys.exit(1) else: print( "", file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "### <%s>: successfully made KLayout DMG" % pyDMGmaker, file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "", file=sys.stderr ) dmgs = glob.glob( "*.dmg*" ) for item in dmgs: shutil.move( item, stashDMG ) #------------------------------------------------------------------------------ ## To clean up DMGs # # @param[in] srlDMG DMG's serial number #------------------------------------------------------------------------------ def DMG_Clean( srlDMG ): pyDMGmaker = "./makeDMG4mac.py" stashDMG = "./DMGStash" myPlatform = Test_My_Platform() packOp = Get_Package_Options( Get_Build_Target_Dict(), myPlatform, srlDMG, makeflag=False ) if os.path.isdir( stashDMG ): shutil.rmtree( stashDMG ) for key in Target: if key == 4 and QtType == 6: # anaconda3 does not provide Qt6 so far continue command1 = [ pyDMGmaker ] + packOp[key] print(command1) #continue if subprocess.call( command1, shell=False ) != 0: print( "", file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "!!! <%s>: failed to clean KLayout DMG" % pyDMGmaker, file=sys.stderr ) print( "-----------------------------------------------------------------", file=sys.stderr ) print( "", file=sys.stderr ) sys.exit(1) else: print( "", file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "### <%s>: successfully cleaned KLayout DMG" % pyDMGmaker, file=sys.stderr ) print( "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++", file=sys.stderr ) print( "", file=sys.stderr ) #------------------------------------------------------------------------------ ## To upload DMGs to Dropbox # # @param[in] targetdir existing target directory such as "0.26.9" #------------------------------------------------------------------------------ def Upload_To_Dropbox( targetdir ): stashDMG = "./DMGStash" distDir = os.environ["HOME"] + "/Dropbox/klayout/" + targetdir if not os.path.isdir(distDir): os.makedirs(distDir) dmgs = glob.glob( "%s/*.dmg*" % stashDMG ) for item in dmgs: shutil.copy2( item, distDir ) #------------------------------------------------------------------------------ ## The main function #------------------------------------------------------------------------------ def Main(): Parse_CommandLine_Arguments() if Build: Build_Deploy() if QATest: Run_QATest( 'pymod,pya' ) if QACheck: Check_QATest_Results( 20 ) elif MakeDMG: DMG_Make( SrlDMG ) elif Upload: Upload_To_Dropbox( Dropbox ) elif CleanDMG: DMG_Clean( SrlDMG ) #=================================================================================== if __name__ == "__main__": Main() #--------------- # End of file #---------------