From 1399beb2291c042d3ecaed9535cc712c2a78fca8 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 4 Apr 2019 16:19:20 -0400 Subject: [PATCH 01/10] Using ccache for app build --- .travis.yml | 24 ++++++++++++++++++------ travis-build.sh | 12 ++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a2c94dda9..a83f16f40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -343,7 +343,9 @@ matrix: - name: "KLayout macOS 10.13 with py3.7" os: osx osx_image: xcode9.4 # macOS 10.13 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -360,7 +362,9 @@ matrix: - name: "KLayout macOS 10.12 with py3.7" os: osx osx_image: xcode8.3 # macOS 10.12 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -377,7 +381,9 @@ matrix: - name: "KLayout macOS 10.11 with py3.7" os: osx osx_image: xcode8 # macOS 10.11 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -393,7 +399,9 @@ matrix: - name: "KLayout macOS 10.13 with py2.7" os: osx osx_image: xcode9.4 # macOS 10.13 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -409,7 +417,9 @@ matrix: - name: "KLayout macOS 10.12 with py2.7" os: osx osx_image: xcode8.3 # macOS 10.12 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -425,7 +435,9 @@ matrix: - name: "KLayout macOS 10.11 with py2.7" os: osx osx_image: xcode8 # macOS 10.11 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: diff --git a/travis-build.sh b/travis-build.sh index b988d0745..bd412f30f 100755 --- a/travis-build.sh +++ b/travis-build.sh @@ -8,6 +8,14 @@ PING_LOOP_PID=$! touch build.txt +# Configure ccache +mkdir -p ccache; +export CCACHE_DIR="`pwd`/ccache" + +# Show ccache stats +echo "Cache stats:" +ccache -s + echo "build" make build >> build.txt 2>&1 || tail -500 build.txt echo "deploy" @@ -17,6 +25,10 @@ make test >> build.txt 2>&1 || tail -500 build.txt echo "dropbox-deploy" make dropbox-deploy +# Show ccache stats +echo "Cache stats:" +ccache -s + echo "build finished" kill $PING_LOOP_PID From af1289b79a2cab4d3ac88c3d965c31021bb25309 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 4 Apr 2019 16:53:10 -0400 Subject: [PATCH 02/10] increasing linux ccache size to 5G --- ci-scripts/docker/docker_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 2f6d68710..747acb867 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -36,6 +36,7 @@ elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then fi echo $PATH export CCACHE_DIR="/io/ccache" +ccache -M 5 G # set cache size to 5 G # Download proper auditwheel program git clone https://github.com/thomaslima/auditwheel.git /tmp/auditwheel @@ -45,7 +46,6 @@ AUDITWHEEL_PYTHON=$(cat `which auditwheel` | head -1 | sed -e 's/#!\(.*\)/\1/') # Install auditwheel, replacing the system's auditwheel binary $AUDITWHEEL_PYTHON -m pip install . - # Show ccache stats echo "Cache stats:" ccache -s From a40b0cb793c1b05163334ce89a9e75679c16f252 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 4 Apr 2019 17:57:31 -0400 Subject: [PATCH 03/10] Moving ccache directory for macosx wheels --- .travis.yml | 55 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index a83f16f40..329c7e11f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -201,7 +201,9 @@ matrix: - name: "cp27-cp27m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -216,7 +218,9 @@ matrix: - name: "cp27-cp27m-macosx_10_12_x86_64.whl" os: osx osx_image: xcode8.3 # macOS 10.12 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -231,7 +235,9 @@ matrix: - name: "cp27-cp27m-macosx_10_11_x86_64.whl" os: osx osx_image: xcode8 # macOS 10.11 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -248,7 +254,9 @@ matrix: - name: "cp37-cp37m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -265,7 +273,9 @@ matrix: - name: "cp36-cp36m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -280,7 +290,9 @@ matrix: - name: "cp35-cp35m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -290,27 +302,14 @@ matrix: - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - # # - name: "klayout python3.4.9 osx10.13" - # - name: "cp34-cp34m-macosx_10_13_x86_64.whl" - # os: osx - # osx_image: xcode9.4 # macOS 10.13 - # cache: ccache - # addons: - # homebrew: - # packages: - # - ccache - # env: - # - MATRIX_EVAL="brew update; brew tap sashkab/python; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';" - # - ARCHFLAGS="-std=c++11" - # - PIP_UPDATE="1" - # - PYTHON_BUILD=true # - name: "klayout python3 osx10.12" - name: "cp37-cp37m-macosx_10_12_x86_64.whl" os: osx osx_image: xcode8.3 # macOS 10.12 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -327,7 +326,9 @@ matrix: - name: "cp37-cp37m-macosx_10_11_x86_64.whl" os: osx osx_image: xcode8 # macOS 10.11 - cache: ccache + cache: + directories: + - ccache addons: homebrew: packages: @@ -462,6 +463,9 @@ before_install: - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then export PATH="/usr/local/opt/ccache/libexec:$PATH"; + mkdir -p ccache; + export CCACHE_DIR="`pwd`/ccache"; + ccache -s; fi install: @@ -506,6 +510,11 @@ script: ./travis-build.sh; fi +before_cache: + - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + ccache -s; + fi + after_success: # upload to dropbox # need DROPBOX_OAUTH_BEARER environment variable From 7cecc4507fdef92feb617ed2883f227c5734980c Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 4 Apr 2019 18:53:54 -0400 Subject: [PATCH 04/10] Adding option to load ccache in qmake command (build.sh) --- build.sh | 12 ++++++++++++ travis-build.sh | 1 + 2 files changed, 13 insertions(+) diff --git a/build.sh b/build.sh index 715ab8a7d..cec552875 100755 --- a/build.sh +++ b/build.sh @@ -231,6 +231,10 @@ while [ "$*" != "" ]; do echo " -libcurl Use libcurl instead of QtNetwork (for Qt<4.7)" echo " -libexpat Use libexpat instead of QtXml" echo "" + echo "Environment Variables:" + echo "" + echo " QMAKE_CCACHE=1 Adds CONFIG+=ccache to qmake command (only works if Qt>=5.9.2)" + echo "" exit 0 ;; *) @@ -609,6 +613,14 @@ qmake_options=( KLAYOUT_VERSION_REV="$KLAYOUT_VERSION_REV" ) +# This should speed up build time considerably +# https://ortogonal.github.io/ccache-and-qmake-qtcreator/ +if [ $QMAKE_CCACHE = 1 ]; then + qmake_options+=( + CONFIG+="ccache" + ) +fi + if [ $BUILD_EXPERT = 1 ]; then qmake_options+=( QMAKE_AR="$AR cqs" diff --git a/travis-build.sh b/travis-build.sh index bd412f30f..bab2aa080 100755 --- a/travis-build.sh +++ b/travis-build.sh @@ -11,6 +11,7 @@ touch build.txt # Configure ccache mkdir -p ccache; export CCACHE_DIR="`pwd`/ccache" +export QMAKE_CCACHE=1 # Show ccache stats echo "Cache stats:" From d54d80330184935dd9e5dfebb966f99e55375ca3 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 4 Apr 2019 20:04:28 -0400 Subject: [PATCH 06/10] Better logging for mac app build (travis) --- travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-build.sh b/travis-build.sh index bab2aa080..74765d315 100755 --- a/travis-build.sh +++ b/travis-build.sh @@ -3,7 +3,7 @@ set -e export PING_SLEEP=30s -bash -c "while true; do echo -n '.'; sleep $PING_SLEEP; done" & +bash -c "while true; do find * | wc -l; sleep $PING_SLEEP; done" & PING_LOOP_PID=$! touch build.txt From 7c8896d9dc6b70236e0734734f4827da277bc54e Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 4 Apr 2019 22:14:23 -0400 Subject: [PATCH 07/10] Travis's xcode8's homebrew is outdated --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 329c7e11f..22d10cd9c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -242,6 +242,7 @@ matrix: homebrew: packages: - ccache + update: true env: - MATRIX_EVAL="" - ARCHFLAGS="-std=c++11" @@ -333,6 +334,7 @@ matrix: homebrew: packages: - ccache + update: true env: - MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';" - ARCHFLAGS="-std=c++11" From 603e0d5e1efb0662a192c8110db13d2f6ba7d290 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 4 Apr 2019 22:21:13 -0400 Subject: [PATCH 08/10] Putting ccache installation before MATRIX_EVAL, in case brew update builds packages from source --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 22d10cd9c..3cbe3153d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -455,6 +455,12 @@ matrix: before_install: - env - gem install dropbox-deployment + - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + export PATH="/usr/local/opt/ccache/libexec:$PATH"; + mkdir -p ccache; + export CCACHE_DIR="`pwd`/ccache"; + ccache -s; + fi - eval "${MATRIX_EVAL}" - if [ "${PIP_UPDATE}" == "1" ]; then pip --version; @@ -463,12 +469,6 @@ before_install: pip install --upgrade setuptools wheel || sudo pip install --upgrade setuptools wheel; fi - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" - - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then - export PATH="/usr/local/opt/ccache/libexec:$PATH"; - mkdir -p ccache; - export CCACHE_DIR="`pwd`/ccache"; - ccache -s; - fi install: - if [ "$DOCKER_BUILD" = true ]; then From d457209665a4fefc80528a4a51aa6ce9d0e93944 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 4 Apr 2019 23:24:14 -0400 Subject: [PATCH 09/10] Forgot to install ccache in macOS 10.11 with py3.7 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3cbe3153d..b9af84bff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -391,6 +391,7 @@ matrix: homebrew: packages: - python3 + - ccache update: true env: - MATRIX_EVAL="brew update; brew config; brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/56c500b569c724b049be7ab9e12d9693f85522f9/Formula/qt.rb" # Qt 5.11.2 From e579f3a868f34c68ff3876391a124a89ba98c139 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Fri, 5 Apr 2019 12:47:06 -0400 Subject: [PATCH 10/10] minor edit in logging --- travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-build.sh b/travis-build.sh index 74765d315..9d5acabc3 100755 --- a/travis-build.sh +++ b/travis-build.sh @@ -3,7 +3,7 @@ set -e export PING_SLEEP=30s -bash -c "while true; do find * | wc -l; sleep $PING_SLEEP; done" & +bash -c "while true; do find qt* | wc -l; sleep $PING_SLEEP; done" & PING_LOOP_PID=$! touch build.txt