Merge pull request #258 from lightwave-lab/master

Accelerating Travis build (wheels < 5min, Mac app < 20min)

.travis.yml

@@ -201,7 +201,9 @@ matrix:
     - name: "cp27-cp27m-macosx_10_13_x86_64.whl"
       os: osx
       osx_image: xcode9.4  # macOS 10.13
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -216,7 +218,9 @@ matrix:
     - name: "cp27-cp27m-macosx_10_12_x86_64.whl"
       os: osx
       osx_image: xcode8.3  # macOS 10.12
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -231,11 +235,14 @@ matrix:
     - name: "cp27-cp27m-macosx_10_11_x86_64.whl"
       os: osx
       osx_image: xcode8  # macOS 10.11
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
             - ccache
+          update: true
       env:
         - MATRIX_EVAL=""
         - ARCHFLAGS="-std=c++11"
@@ -248,7 +255,9 @@ matrix:
     - name: "cp37-cp37m-macosx_10_13_x86_64.whl"
       os: osx
       osx_image: xcode9.4  # macOS 10.13
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -265,7 +274,9 @@ matrix:
     - name: "cp36-cp36m-macosx_10_13_x86_64.whl"
       os: osx
       osx_image: xcode9.4  # macOS 10.13
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -280,7 +291,9 @@ matrix:
     - name: "cp35-cp35m-macosx_10_13_x86_64.whl"
       os: osx
       osx_image: xcode9.4  # macOS 10.13
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -290,27 +303,14 @@ matrix:
         - ARCHFLAGS="-std=c++11"
         - PIP_UPDATE="1"
         - PYTHON_BUILD=true
-
-    # # - name: "klayout python3.4.9 osx10.13"
-    # - name: "cp34-cp34m-macosx_10_13_x86_64.whl"
-    #   os: osx
-    #   osx_image: xcode9.4  # macOS 10.13
-    #   cache: ccache
-    #   addons:
-    #     homebrew:
-    #       packages:
-    #         - ccache
-    #   env:
-    #     - MATRIX_EVAL="brew update; brew tap sashkab/python; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';"
-    #     - ARCHFLAGS="-std=c++11"
-    #     - PIP_UPDATE="1"
-    #     - PYTHON_BUILD=true
     
     # - name: "klayout python3 osx10.12"
     - name: "cp37-cp37m-macosx_10_12_x86_64.whl"
       os: osx
       osx_image: xcode8.3  # macOS 10.12
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -327,11 +327,14 @@ matrix:
     - name: "cp37-cp37m-macosx_10_11_x86_64.whl"
       os: osx
       osx_image: xcode8  # macOS 10.11
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
             - ccache
+          update: true
       env:
         - MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';"
         - ARCHFLAGS="-std=c++11"
@@ -343,7 +346,9 @@ matrix:
     - name: "KLayout macOS 10.13 with py3.7"
       os: osx
       osx_image: xcode9.4 # macOS 10.13
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -360,7 +365,9 @@ matrix:
     - name: "KLayout macOS 10.12 with py3.7"
       os: osx
       osx_image: xcode8.3 # macOS 10.12
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -377,11 +384,14 @@ matrix:
     - name: "KLayout macOS 10.11 with py3.7"
       os: osx
       osx_image: xcode8 # macOS 10.11
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
             - python3
+            - ccache
           update: true
       env:
         - MATRIX_EVAL="brew update; brew config; brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/56c500b569c724b049be7ab9e12d9693f85522f9/Formula/qt.rb"  # Qt 5.11.2
@@ -393,7 +403,9 @@ matrix:
     - name: "KLayout macOS 10.13 with py2.7"
       os: osx
       osx_image: xcode9.4 # macOS 10.13
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -409,7 +421,9 @@ matrix:
     - name: "KLayout macOS 10.12 with py2.7"
       os: osx
       osx_image: xcode8.3 # macOS 10.12
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -425,7 +439,9 @@ matrix:
     - name: "KLayout macOS 10.11 with py2.7"
       os: osx
       osx_image: xcode8 # macOS 10.11
-      cache: ccache
+      cache:
+        directories:
+        - ccache
       addons:
         homebrew:
           packages:
@@ -440,6 +456,12 @@ matrix:
 before_install:
   - env
   - gem install dropbox-deployment
+  - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+        export PATH="/usr/local/opt/ccache/libexec:$PATH";
+        mkdir -p ccache;
+        export CCACHE_DIR="`pwd`/ccache";
+        ccache -s;
+    fi
   - eval "${MATRIX_EVAL}"
   - if [ "${PIP_UPDATE}" == "1" ]; then
         pip --version;
@@ -448,9 +470,6 @@ before_install:
         pip install --upgrade setuptools wheel || sudo pip install --upgrade setuptools wheel;
     fi
   - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)"
-  - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
-        export PATH="/usr/local/opt/ccache/libexec:$PATH";
-    fi
 
 install:
   - if [ "$DOCKER_BUILD" = true ]; then
@@ -494,6 +513,11 @@ script:
         ./travis-build.sh;
     fi
 
+before_cache:
+  - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+        ccache -s;
+    fi
+
 after_success:
   # upload to dropbox
   # need DROPBOX_OAUTH_BEARER environment variable

build.sh

@@ -231,6 +231,10 @@ while [ "$*" != "" ]; do
     echo "  -libcurl              Use libcurl instead of QtNetwork (for Qt<4.7)"
     echo "  -libexpat             Use libexpat instead of QtXml"
     echo ""
+    echo "Environment Variables:"
+    echo ""
+    echo "  QMAKE_CCACHE=1        Adds CONFIG+=ccache to qmake command (only works if Qt>=5.9.2)"
+    echo ""
     exit 0
     ;;
   *)
@@ -609,6 +613,14 @@ qmake_options=(
   KLAYOUT_VERSION_REV="$KLAYOUT_VERSION_REV"
 )
 
+# This should speed up build time considerably
+# https://ortogonal.github.io/ccache-and-qmake-qtcreator/
+if [ $QMAKE_CCACHE = 1 ]; then
+  qmake_options+=(
+    CONFIG+="ccache"
+  )
+fi
+
 if [ $BUILD_EXPERT = 1 ]; then
   qmake_options+=(
     QMAKE_AR="$AR cqs"

ci-scripts/docker/docker_build.sh

@@ -36,6 +36,7 @@ elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then
 fi
 echo $PATH
 export CCACHE_DIR="/io/ccache"
+ccache -M 5 G  # set cache size to 5 G
 
 # Download proper auditwheel program
 git clone https://github.com/thomaslima/auditwheel.git /tmp/auditwheel
@@ -45,7 +46,6 @@ AUDITWHEEL_PYTHON=$(cat `which auditwheel` | head -1 | sed -e 's/#!\(.*\)/\1/')
 # Install auditwheel, replacing the system's auditwheel binary
 $AUDITWHEEL_PYTHON -m pip install .
 
-
 # Show ccache stats
 echo "Cache stats:"
 ccache -s

travis-build.sh

@@ -3,11 +3,20 @@ set -e
 
 export PING_SLEEP=30s
 
-bash -c "while true; do echo -n '.'; sleep $PING_SLEEP; done" &
+bash -c "while true; do find qt* | wc -l; sleep $PING_SLEEP; done" &
 PING_LOOP_PID=$!
 
 touch build.txt
 
+# Configure ccache
+mkdir -p ccache;
+export CCACHE_DIR="`pwd`/ccache"
+export QMAKE_CCACHE=1
+
+# Show ccache stats
+echo "Cache stats:"
+ccache -s
+
 echo "build"
 make build >> build.txt 2>&1 || tail -500 build.txt
 echo "deploy"
@@ -17,6 +26,10 @@ make test >> build.txt 2>&1 || tail -500 build.txt
 echo "dropbox-deploy"
 make dropbox-deploy
 
+# Show ccache stats
+echo "Cache stats:"
+ccache -s
+
 echo "build finished"
 
 kill $PING_LOOP_PID