From e5921252ea22d35a76ed3f8da9442cc0e3cd6c54 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Sun, 21 Oct 2018 14:40:32 -0400 Subject: [PATCH 01/23] prototype formula for manylinux1 --- ci-scripts/manylinux-docker.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 ci-scripts/manylinux-docker.sh diff --git a/ci-scripts/manylinux-docker.sh b/ci-scripts/manylinux-docker.sh new file mode 100644 index 000000000..4e9f6a694 --- /dev/null +++ b/ci-scripts/manylinux-docker.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# run with docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/ci-scripts/manylinux-docker.sh +# see https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml +# cache using https://github.com/travis-ci/travis-ci/issues/5358 +set -e -x + +# Install a system package required by our library +yum install -y zlib-devel +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccachec++ +ln -s /usr/bin/ccache /usr/lib64/ccachecc +ln -s /usr/bin/ccache /usr/lib64/ccachegcc +ln -s /usr/bin/ccache /usr/lib64/ccacheg++ +export PATH=/usr/lib64/ccache:$PATH + +# Compile wheels +for PYBIN in /opt/python/*/bin; do + cd /io; mkdir -p /io/wheelhouse + "${PYBIN}/python" setup.py bdist_wheel -d wheelhouse/ +done + +# Bundle external shared libraries into the wheels +for whl in wheelhouse/*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + +# Install packages and test +for PYBIN in /opt/python/*/bin/; do + "${PYBIN}/pip" install klayout --no-index -f /io/wheelhouse + "${PYBIN}" -m unittest /io/testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py +done From 55f5167a02de57267e7b48dd7cf9ada721e9ba72 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 12:11:26 -0500 Subject: [PATCH 02/23] [ci skip] first working manylinux wheel --- ci-scripts/docker/Dockerfile | 18 ++++ ci-scripts/docker/README.md | 101 ++++++++++++++++++++ ci-scripts/docker/fix_wheel.sh | 61 ++++++++++++ ci-scripts/{ => docker}/manylinux-docker.sh | 10 +- 4 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 ci-scripts/docker/Dockerfile create mode 100644 ci-scripts/docker/README.md create mode 100644 ci-scripts/docker/fix_wheel.sh rename ci-scripts/{ => docker}/manylinux-docker.sh (73%) diff --git a/ci-scripts/docker/Dockerfile b/ci-scripts/docker/Dockerfile new file mode 100644 index 000000000..df4f05dc5 --- /dev/null +++ b/ci-scripts/docker/Dockerfile @@ -0,0 +1,18 @@ +FROM quay.io/pypa/manylinux1_x86_64 +MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me) + +# Install a system package required by our library +RUN yum install -y zlib-devel +RUN yum install -y ccache +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/cc +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + +# Add ccache to PATH +ENV PATH="/usr/lib64/ccache:${PATH}" +RUN mkdir -p /persist/.ccache +ENV CCACHE_DIR="/persist/.ccache" + +# Need zip to fix wheel +RUN yum install -y zip diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md new file mode 100644 index 000000000..615255923 --- /dev/null +++ b/ci-scripts/docker/README.md @@ -0,0 +1,101 @@ +# Step 1. + +Make sure you have the quay.io/pypa/manylinux1_x86_64 image. + +```bash +$ docker images +REPOSITORY TAG IMAGE ID CREATED SIZE +quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB +hello-world latest 4ab4c602aa5e 3 months ago 1.84kB +# My image was old: +$ docker pull quay.io/pypa/manylinux1_x86_64 +Using default tag: latest +latest: Pulling from pypa/manylinux1_x86_64 +7d0d9526f38a: Already exists +3324bfadf9cb: Pull complete +20f27c7e3062: Pull complete +5bc21fc5fe97: Pull complete +Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 +Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest +``` + +# Step 2. + +This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 +Create a volume for klayout. This is necessary because docker containers don't persist data. + +```bash +$ docker volume create klayout-persist +$ docker volume inspect klayout-persist +[ + { + "CreatedAt": "2018-12-18T15:01:48Z", + "Driver": "local", + "Labels": {}, + "Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data", + "Name": "klayout-persist", + "Options": {}, + "Scope": "local" + } +] +``` + +# Step 3. + +Build image `myimage` with: + +```bash +$ docker build -t myimage:latest . +``` + +This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. + +Then I run the docker with a terminal shell and load the volume klayout-persist in /persist: + +```bash +$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage +``` + +# Step 4. + +In the shell, pull master from klayout. + +```bash +cd /persist +git clone https://github.com/lightwave-lab/klayout.git +mkdir -p wheelhouse +cd klayout +# make wheel with python 3.6 (for example) +/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/ +cd /persist +auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/ +# Need to manually fix the wheel +#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse +``` + +The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. + + +Procedure to fix the wheel: + +```bash +unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel +cd tempwheel/klayout +mv lib_* db_plugins .libs/ +cd ../klayout-0.26.0.dev8.dist-info/ +sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD +sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD +cd ../ +rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl +zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./* +cd .. +rm -rf tempwheel +``` + +Now we can install and test: +```bash +/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse +cd /persist/klayout +/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py +# Tests passed! +``` diff --git a/ci-scripts/docker/fix_wheel.sh b/ci-scripts/docker/fix_wheel.sh new file mode 100644 index 000000000..6601ed7f7 --- /dev/null +++ b/ci-scripts/docker/fix_wheel.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=`basename "$0"` + +display_usage() { + echo "This script fixes auditwheel-repaired wheels." + echo "Caution: This will delete the original wheel." + echo -e "\nUsage:\n./${SCRIPT_NAME} klayout-...-manylinux1_x86_64.whl \n" + } + +if [[ ( $1 == "--help") || $1 == "-h" ]] +then + display_usage + exit 0 +fi + +# Check number of arguments +if [ $# -eq 0 ]; then + echo -e "No filename supplied\n" + display_usage + exit 1 +elif [ ! $# -eq 1 ]; then + echo -e "Too many files supplied. Provide one filename at at time.\n" + display_usage + exit 1 +fi + +# Read wheel file from argument +WHL="$1" + +# Check WHL is a valid file +if [[ ! -f "$WHL" ]]; then + echo -e "$WHL is not a file" + exit 1 +fi + +# Convert to absolute path (linux only) +WHL=$(readlink -f $WHL) + +# Record old current directory +OLD_PWD=$PWD + +# The produced wheel from auditwheel, klayout-*-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. + +# Repair script below +echo $WHL +echo $OLD_PWD + +unzip $WHL -d /tmp/tempwheel +cd /tmp/tempwheel/klayout +mv lib_* db_plugins .libs/ +cd ../klayout-*.dist-info/ +sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD +sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD +cd ../ +rm -f $WHL +zip -rq $WHL ./* + +# Cleanup (should always execute) +cd $OLD_PWD +rm -rf /tmp/tempwheel diff --git a/ci-scripts/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh similarity index 73% rename from ci-scripts/manylinux-docker.sh rename to ci-scripts/docker/manylinux-docker.sh index 4e9f6a694..a3bd99428 100644 --- a/ci-scripts/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -7,10 +7,10 @@ set -e -x # Install a system package required by our library yum install -y zlib-devel yum install -y ccache -ln -s /usr/bin/ccache /usr/lib64/ccachec++ -ln -s /usr/bin/ccache /usr/lib64/ccachecc -ln -s /usr/bin/ccache /usr/lib64/ccachegcc -ln -s /usr/bin/ccache /usr/lib64/ccacheg++ +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ export PATH=/usr/lib64/ccache:$PATH # Compile wheels @@ -27,5 +27,5 @@ done # Install packages and test for PYBIN in /opt/python/*/bin/; do "${PYBIN}/pip" install klayout --no-index -f /io/wheelhouse - "${PYBIN}" -m unittest /io/testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py + "${PYBIN}/python" -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py done From 70d5f5d1a02a7e3d7ff1b231bf0139f50f7d273d Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 13:51:49 -0500 Subject: [PATCH 03/23] better error messages in fix_wheel.sh script. ready to go. --- ci-scripts/docker/fix_wheel.sh | 40 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/ci-scripts/docker/fix_wheel.sh b/ci-scripts/docker/fix_wheel.sh index 6601ed7f7..2892bdbeb 100644 --- a/ci-scripts/docker/fix_wheel.sh +++ b/ci-scripts/docker/fix_wheel.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash SCRIPT_NAME=`basename "$0"` +TMP_WHEEL="/tmp/klayout_tempwheel" display_usage() { echo "This script fixes auditwheel-repaired wheels." echo "Caution: This will delete the original wheel." - echo -e "\nUsage:\n./${SCRIPT_NAME} klayout-...-manylinux1_x86_64.whl \n" + echo -e "\nUsage:\n./${SCRIPT_NAME} [--help|-h] klayout-...-manylinux1_x86_64.whl \n" } if [[ ( $1 == "--help") || $1 == "-h" ]] @@ -16,11 +17,11 @@ fi # Check number of arguments if [ $# -eq 0 ]; then - echo -e "No filename supplied\n" + >&2 echo -e "ERROR: No filename supplied\n" display_usage exit 1 elif [ ! $# -eq 1 ]; then - echo -e "Too many files supplied. Provide one filename at at time.\n" + >&2 echo -e "ERROR: Too many files supplied. Provide one filename at at time.\n" display_usage exit 1 fi @@ -30,7 +31,7 @@ WHL="$1" # Check WHL is a valid file if [[ ! -f "$WHL" ]]; then - echo -e "$WHL is not a file" + >&2 echo -e "ERROR: $WHL is not a file" exit 1 fi @@ -42,20 +43,35 @@ OLD_PWD=$PWD # The produced wheel from auditwheel, klayout-*-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. -# Repair script below -echo $WHL -echo $OLD_PWD +# Checking if it was previously patched +if unzip -l $WHL | grep -q 'patched_after_auditwheel_repair'; then + echo "$(basename $WHL) is already patched. Doing nothing." + exit 0 +fi -unzip $WHL -d /tmp/tempwheel -cd /tmp/tempwheel/klayout -mv lib_* db_plugins .libs/ +# Repair script below +if [[ -d $TMP_WHEEL ]]; then + rm -rf $TMP_WHEEL +fi +echo "Unpacking $WHL into $TMP_WHEEL" +unzip -q $WHL -d $TMP_WHEEL + +cd $TMP_WHEEL/klayout +echo "Moving files: mv lib_* db_plugins .libs/" +mv lib_* db_plugins .libs/ 2>/dev/null +if [ $? -ne 0 ]; then + >&2 echo "ERROR: lib_*.so or db_plubins not found. Quitting." + exit 1 +fi cd ../klayout-*.dist-info/ +echo "Patching klayout-*.dist-info/RECORD" sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD cd ../ +touch $TMP_WHEEL/patched_after_auditwheel_repair +echo "Packing $WHL from $TMP_WHEEL" rm -f $WHL zip -rq $WHL ./* - +echo "Done. $(basename $WHL) is patched." # Cleanup (should always execute) cd $OLD_PWD -rm -rf /tmp/tempwheel From 937019e651651135a2a6aa2c37a07c6808b0d33d Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 15:17:27 -0500 Subject: [PATCH 04/23] including Dockerfile for 32-bit linux --- ci-scripts/docker/Dockerfile.i686 | 18 ++++++++++++++++++ .../docker/{Dockerfile => Dockerfile.x86_64} | 0 2 files changed, 18 insertions(+) create mode 100644 ci-scripts/docker/Dockerfile.i686 rename ci-scripts/docker/{Dockerfile => Dockerfile.x86_64} (100%) diff --git a/ci-scripts/docker/Dockerfile.i686 b/ci-scripts/docker/Dockerfile.i686 new file mode 100644 index 000000000..f73ca3ac1 --- /dev/null +++ b/ci-scripts/docker/Dockerfile.i686 @@ -0,0 +1,18 @@ +FROM quay.io/pypa/manylinux1_i686 +MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me) + +# Install a system package required by our library +RUN linux32 yum install -y zlib-devel +RUN linux32 yum install -y ccache +RUN ln -s /usr/bin/ccache /usr/lib/ccache/c++ +RUN ln -s /usr/bin/ccache /usr/lib/ccache/cc +RUN ln -s /usr/bin/ccache /usr/lib/ccache/gcc +RUN ln -s /usr/bin/ccache /usr/lib/ccache/g++ + +# Add ccache to PATH +ENV PATH="/usr/lib/ccache:${PATH}" +RUN mkdir -p /persist/.ccache +ENV CCACHE_DIR="/persist/.ccache" + +# Need zip to fix wheel +RUN linux32 yum install -y zip diff --git a/ci-scripts/docker/Dockerfile b/ci-scripts/docker/Dockerfile.x86_64 similarity index 100% rename from ci-scripts/docker/Dockerfile rename to ci-scripts/docker/Dockerfile.x86_64 From f8502adb324af278d7b7a1eba01ac5e191f18ad2 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 15:18:33 -0500 Subject: [PATCH 05/23] Tested wheel build for all manylinux python versions --- ci-scripts/docker/README.md | 26 +++++++++++++++++++++++++- ci-scripts/docker/manylinux-docker.sh | 14 +++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 615255923..054333ed7 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -45,7 +45,7 @@ $ docker volume inspect klayout-persist Build image `myimage` with: ```bash -$ docker build -t myimage:latest . +$ docker build -t myimage:latest -f Dockerfile.x86_64 . ``` This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. @@ -99,3 +99,27 @@ cd /persist/klayout /opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py # Tests passed! ``` + +Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. + +# Step 5. Iterate over all python versions. + +For that, we need something like: + +```bash +# Compile wheels +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ +done + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /persist/wheelhouse/*linux_x86_64.whl; do + auditwheel repair "$whl" -w /persist/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /persist/wheelhouse/*manylinux1_x86_64.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +``` diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh index a3bd99428..f21d40d7f 100644 --- a/ci-scripts/docker/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -15,13 +15,17 @@ export PATH=/usr/lib64/ccache:$PATH # Compile wheels for PYBIN in /opt/python/*/bin; do - cd /io; mkdir -p /io/wheelhouse - "${PYBIN}/python" setup.py bdist_wheel -d wheelhouse/ + "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ done -# Bundle external shared libraries into the wheels -for whl in wheelhouse/*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ +# Bundle external shared libraries into the wheels via auditwheel +for whl in /persist/wheelhouse/*linux_x86_64.whl; do + auditwheel repair "$whl" -w /persist/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /persist/wheelhouse/*manylinux1_x86_64.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" done # Install packages and test From 00c28d4a087d897af070339caf74b1ab6e48abae Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 15:58:33 -0500 Subject: [PATCH 06/23] tested script for linux32 as well. Also added +x permission to fix_wheel --- ci-scripts/docker/README.md | 16 ++++++++++++++-- ci-scripts/docker/fix_wheel.sh | 0 ci-scripts/docker/manylinux-docker.sh | 11 +++++++---- 3 files changed, 21 insertions(+), 6 deletions(-) mode change 100644 => 100755 ci-scripts/docker/fix_wheel.sh diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 054333ed7..5ec87ee41 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -113,13 +113,25 @@ for PYBIN in /opt/python/*/bin; do done # Bundle external shared libraries into the wheels via auditwheel -for whl in /persist/wheelhouse/*linux_x86_64.whl; do +for whl in /persist/wheelhouse/*linux_*.whl; do auditwheel repair "$whl" -w /persist/wheelhouse/ done # Fix each wheel generated by auditwheel -for whl in /persist/wheelhouse/*manylinux1_x86_64.whl; do +for whl in /persist/wheelhouse/*manylinux1_*.whl; do ./ci-scripts/docker/fix_wheel.sh "$whl" done +# Install packages and test +TEST_HOME=/persist/klayout/testdata +for PYBIN in /opt/python/*/bin/; do + "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse + "${PYBIN}/python" $TEST_HOME/pymod/import_db.py + "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py + "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py + ``` + +I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: + +`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` diff --git a/ci-scripts/docker/fix_wheel.sh b/ci-scripts/docker/fix_wheel.sh old mode 100644 new mode 100755 diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh index f21d40d7f..a4b958853 100644 --- a/ci-scripts/docker/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -19,17 +19,20 @@ for PYBIN in /opt/python/*/bin; do done # Bundle external shared libraries into the wheels via auditwheel -for whl in /persist/wheelhouse/*linux_x86_64.whl; do +for whl in /persist/wheelhouse/*linux_*.whl; do auditwheel repair "$whl" -w /persist/wheelhouse/ done # Fix each wheel generated by auditwheel -for whl in /persist/wheelhouse/*manylinux1_x86_64.whl; do +for whl in /persist/wheelhouse/*manylinux1_*.whl; do ./ci-scripts/docker/fix_wheel.sh "$whl" done # Install packages and test +TEST_HOME=/persist/klayout/testdata for PYBIN in /opt/python/*/bin/; do - "${PYBIN}/pip" install klayout --no-index -f /io/wheelhouse - "${PYBIN}/python" -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py + "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse + "${PYBIN}/python" $TEST_HOME/pymod/import_db.py + "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py + "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py done From 94b37c389c79bbd18ffe2a019139079833eb06b1 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 17:53:52 -0500 Subject: [PATCH 07/23] Continuous Integration for manylinux1 wheels (first test) --- .travis.yml | 45 +++++++++++++- ci-scripts/docker/Dockerfile.i686 | 1 - ci-scripts/docker/Dockerfile.x86_64 | 1 - ci-scripts/docker/README.md | 85 +++++++++++++++++++++++++-- ci-scripts/docker/docker_build.sh | 53 +++++++++++++++++ ci-scripts/docker/manylinux-docker.sh | 2 +- 6 files changed, 178 insertions(+), 9 deletions(-) create mode 100755 ci-scripts/docker/docker_build.sh diff --git a/.travis.yml b/.travis.yml index ee2c85e99..16cd21347 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,36 @@ matrix: include: + # python manylinux packages + + - name: "manylinux1_x86_64 cp36-cp36m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp36-cp36m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp36-cp36m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp36-cp36m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + # python 2 osx - name: "klayout python2 osx10.13" @@ -288,7 +319,19 @@ before_install: fi - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" -script: +install: + - if [ "$DOCKER_BUILD" = true ]; then + docker pull $DOCKER_IMAGE; + fi + +script: + - if [ "$DOCKER_BUILD" = true ]; then + mkdir -p ccache; + mkdir -p wheelhouse; + docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; + mkdir -p deploy/dist-pymod; + cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod; + fi - if [ "$PYTHON_BUILD" = true ]; then python setup.py build; python setup.py bdist_wheel; diff --git a/ci-scripts/docker/Dockerfile.i686 b/ci-scripts/docker/Dockerfile.i686 index f73ca3ac1..7d68054fc 100644 --- a/ci-scripts/docker/Dockerfile.i686 +++ b/ci-scripts/docker/Dockerfile.i686 @@ -10,7 +10,6 @@ RUN ln -s /usr/bin/ccache /usr/lib/ccache/gcc RUN ln -s /usr/bin/ccache /usr/lib/ccache/g++ # Add ccache to PATH -ENV PATH="/usr/lib/ccache:${PATH}" RUN mkdir -p /persist/.ccache ENV CCACHE_DIR="/persist/.ccache" diff --git a/ci-scripts/docker/Dockerfile.x86_64 b/ci-scripts/docker/Dockerfile.x86_64 index df4f05dc5..4bf03c3cd 100644 --- a/ci-scripts/docker/Dockerfile.x86_64 +++ b/ci-scripts/docker/Dockerfile.x86_64 @@ -10,7 +10,6 @@ RUN ln -s /usr/bin/ccache /usr/lib64/ccache/gcc RUN ln -s /usr/bin/ccache /usr/lib64/ccache/g++ # Add ccache to PATH -ENV PATH="/usr/lib64/ccache:${PATH}" RUN mkdir -p /persist/.ccache ENV CCACHE_DIR="/persist/.ccache" diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 5ec87ee41..48b5a4dbd 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -1,4 +1,5 @@ -# Step 1. +# Chapter 1. Testing on your own computer +## Step 1. Make sure you have the quay.io/pypa/manylinux1_x86_64 image. @@ -19,7 +20,7 @@ Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest ``` -# Step 2. +## Step 2. This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 Create a volume for klayout. This is necessary because docker containers don't persist data. @@ -40,7 +41,7 @@ $ docker volume inspect klayout-persist ] ``` -# Step 3. +## Step 3. Build image `myimage` with: @@ -56,7 +57,7 @@ Then I run the docker with a terminal shell and load the volume klayout-persist $ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage ``` -# Step 4. +## Step 4. In the shell, pull master from klayout. @@ -102,7 +103,7 @@ cd /persist/klayout Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. -# Step 5. Iterate over all python versions. +## Step 5. Iterate over all python versions. For that, we need something like: @@ -135,3 +136,77 @@ for PYBIN in /opt/python/*/bin/; do I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: `ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` + +I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. +The ccache folder after a single build has 657MB. Go figure. + +# Chapter 2. Testing CI flow with docker + +# Step 1. Testing commands in own computer + +First cloned with: +```bash +git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux +cd klayout +``` + +Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml + +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Total of 2x6 = 12 possibilities + +```bash +export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 +export PY_VERSION="cp36-cp36m" +docker pull $DOCKER_IMAGE +mkdir -p ccache +mkdir -p wheelhouse +docker run --name klayout -v `pwd` -it $DOCKER_IMAGE +``` + +Inside docker shell: +```bash +yum install -y zlib-devel +yum install -y zip + +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" + +# Compile wheel +/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/ + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +``` + +# Step 2. Automating step 1 in travis (CI). +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Build: spawn 12 travis jobs, one for each combination of word-size and python version. +Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`. +Post-build: +- cache `./ccache` +- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder) + +# Step 3. Automating deployment to PyPI: + +TBD diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh new file mode 100755 index 000000000..58dccef8b --- /dev/null +++ b/ci-scripts/docker/docker_build.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + + +if [[ -z $PY_VERSION ]]; then + echo '$PY_VERSION is not set' + exit 1 +fi + +if [[ -z $DOCKER_IMAGE ]]; then + echo '$DOCKER_IMAGE is not set' + exit 1 +fi + +echo PY_VERSION=$PY_VERSION +echo DOCKER_IMAGE=$DOCKER_IMAGE + +yum install -y zlib-devel ccache zip || exit 1 # sometimes the epel server is down. + +if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then + ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -s /usr/bin/ccache /usr/lib64/ccache/cc + ln -s /usr/bin/ccache /usr/lib64/ccache/gcc + ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then + ln -s /usr/bin/ccache /usr/lib/ccache/c++ + ln -s /usr/bin/ccache /usr/lib/ccache/cc + ln -s /usr/bin/ccache /usr/lib/ccache/gcc + ln -s /usr/bin/ccache /usr/lib/ccache/g++ +fi +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" +cd /io + +# Compile wheel +"/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ || exit 1 +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + /io/ci-scripts/docker/fix_wheel.sh "$whl" || exit 1 +done + +# Install packages and test +TEST_HOME=/io/testdata +"/opt/python/$PY_VERSION/bin/pip" install klayout --no-index -f /io/wheelhouse || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_db.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_rdb.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_tl.py || exit 1 diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh index a4b958853..7591d0181 100644 --- a/ci-scripts/docker/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -11,7 +11,7 @@ ln -s /usr/bin/ccache /usr/lib64/ccache/c++ ln -s /usr/bin/ccache /usr/lib64/ccache/cc ln -s /usr/bin/ccache /usr/lib64/ccache/gcc ln -s /usr/bin/ccache /usr/lib64/ccache/g++ -export PATH=/usr/lib64/ccache:$PATH +# export PATH=/usr/lib64/ccache:$PATH # unnecessary # Compile wheels for PYBIN in /opt/python/*/bin; do From b6d7dec289be4d161f03b884caf1bfa43c691fe0 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:05:10 -0500 Subject: [PATCH 08/23] showing ccache stats --- ci-scripts/docker/docker_build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 58dccef8b..fd3cff7a2 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -35,6 +35,9 @@ cd /io # Compile wheel "/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 +# Show ccache stats +ccache -s + # Bundle external shared libraries into the wheels via auditwheel for whl in /io/wheelhouse/*linux_*.whl; do auditwheel repair "$whl" -w /io/wheelhouse/ || exit 1 From a931db224c45c5de0d1c0ee77d3dca11bf73909f Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:26:50 -0500 Subject: [PATCH 09/23] adding other versions of python to manylinux CI --- .travis.yml | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/.travis.yml b/.travis.yml index 16cd21347..fda9b403d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,34 @@ matrix: include: # python manylinux packages + - name: "manylinux1_x86_64 cp37-cp37m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp37-cp37m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp37-cp37m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp37-cp37m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + - name: "manylinux1_x86_64 cp36-cp36m package" os: linux sudo: true @@ -30,6 +58,118 @@ matrix: directories: - ccache + - name: "manylinux1_x86_64 cp35-cp35m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp35-cp35m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp35-cp35m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp35-cp35m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_x86_64 cp34-cp34m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp34-cp34m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp34-cp34m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp34-cp34m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_x86_64 cp27-cp27mu package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp27-cp27mu" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp27-cp27mu package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp27-cp27mu" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_x86_64 cp27-cp27m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp27-cp27m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp27-cp27m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp27-cp27m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + # python 2 osx From 0848b940ced9a927368642e7336aa75d132d916a Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:27:44 -0500 Subject: [PATCH 10/23] better organization of dist-pymod folder --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fda9b403d..4fe284fb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -469,16 +469,18 @@ script: mkdir -p ccache; mkdir -p wheelhouse; docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; - mkdir -p deploy/dist-pymod; - cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod; + klayout_version=$(python -c 'import setup; print(setup.Config().version())'); + mkdir -p deploy/dist-pymod/$klayout_version; + cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod/$klayout_version; fi - if [ "$PYTHON_BUILD" = true ]; then python setup.py build; python setup.py bdist_wheel; python setup.py install; python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; - mkdir -p deploy/dist-pymod; - cp -a dist/* deploy/dist-pymod/; + klayout_version=$(python -c 'import setup; print(setup.Config().version())'); + mkdir -p deploy/dist-pymod/$klayout_version; + cp -a dist/* deploy/dist-pymod/$klayout_version; python -c 'import klayout.db as db; print(dir(db))'; python -c 'import klayout.rdb as rdb; print(dir(rdb))'; python -c 'import klayout.tl as tl; print(dir(tl))'; From f1e292373f0736be2643a88f12e3931cd18fe290 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:28:05 -0500 Subject: [PATCH 11/23] Changing version to 0.26.0.dev9 for CD tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 545126651..06f0fbeb5 100644 --- a/setup.py +++ b/setup.py @@ -322,7 +322,7 @@ class Config(object): """ Gets the version string """ - return "0.26.0.dev8" + return "0.26.0.dev9" config = Config() From e3de4030df037b42b4ae7a26aa03110882c1149e Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:38:51 -0500 Subject: [PATCH 12/23] http://mirrors.fedoraproject.org is unreliable. Retry yum 5 times. --- ci-scripts/docker/docker_build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index fd3cff7a2..235335161 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -14,7 +14,12 @@ fi echo PY_VERSION=$PY_VERSION echo DOCKER_IMAGE=$DOCKER_IMAGE -yum install -y zlib-devel ccache zip || exit 1 # sometimes the epel server is down. +# sometimes the epel server is down. retry 5 times +for i in $(seq 1 5); do + yum install -y zlib-devel ccache zip && s=0 && break || s=$? && sleep 15; +done + +[ $s -eq 0 ] || exit $s if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then ln -s /usr/bin/ccache /usr/lib64/ccache/c++ From a220e2e0e05f3925cd6b3892bef0dad7d0089b2c Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 19:28:28 -0500 Subject: [PATCH 13/23] verbose auditwheel repair --- ci-scripts/docker/docker_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 235335161..6247fbb7e 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -45,7 +45,7 @@ ccache -s # Bundle external shared libraries into the wheels via auditwheel for whl in /io/wheelhouse/*linux_*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ || exit 1 + auditwheel -v repair "$whl" -w /io/wheelhouse/ || exit 1 done # Fix each wheel generated by auditwheel From 36af902e3fcdab55d3cc52ada409d57031580523 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 19:45:11 -0500 Subject: [PATCH 14/23] bugfix: ccache not in path. --- ci-scripts/docker/docker_build.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 6247fbb7e..8acd4a16c 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -26,21 +26,28 @@ if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then ln -s /usr/bin/ccache /usr/lib64/ccache/cc ln -s /usr/bin/ccache /usr/lib64/ccache/gcc ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + export PATH="/usr/lib64/ccache/:$PATH" elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then ln -s /usr/bin/ccache /usr/lib/ccache/c++ ln -s /usr/bin/ccache /usr/lib/ccache/cc ln -s /usr/bin/ccache /usr/lib/ccache/gcc ln -s /usr/bin/ccache /usr/lib/ccache/g++ + export PATH="/usr/lib/ccache/:$PATH" fi echo $PATH -# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin export CCACHE_DIR="/io/ccache" + +# Show ccache stats +echo "Cache stats:" +ccache -s + cd /io # Compile wheel "/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 # Show ccache stats +echo "Cache stats:" ccache -s # Bundle external shared libraries into the wheels via auditwheel From a288b3ce398539b2404aa931500c99ece9de8501 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 19 Dec 2018 18:30:46 -0500 Subject: [PATCH 15/23] Patched auditwheel directly. Should fix manylinux python build. --- ci-scripts/docker/README.md | 4 ++++ ci-scripts/docker/docker_build.sh | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 48b5a4dbd..9cf7370a2 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -140,6 +140,10 @@ I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/many I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. The ccache folder after a single build has 657MB. Go figure. +I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1 + +So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default. + # Chapter 2. Testing CI flow with docker # Step 1. Testing commands in own computer diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 8acd4a16c..b511f879f 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -16,7 +16,7 @@ echo DOCKER_IMAGE=$DOCKER_IMAGE # sometimes the epel server is down. retry 5 times for i in $(seq 1 5); do - yum install -y zlib-devel ccache zip && s=0 && break || s=$? && sleep 15; + yum install -y zlib-devel ccache zip git && s=0 && break || s=$? && sleep 15; done [ $s -eq 0 ] || exit $s @@ -37,13 +37,21 @@ fi echo $PATH export CCACHE_DIR="/io/ccache" +# Download proper auditwheel program +git clone https://github.com/thomaslima/auditwheel.git /tmp/auditwheel +cd /tmp/auditwheel +git checkout 87f5306ec02cc68020afaa9933543c898b1d47c1 # patched version +AUDITWHEEL_PYTHON=$(cat `which auditwheel` | head -1 | sed -e 's/#!\(.*\)/\1/') +# Install auditwheel, replacing the system's auditwheel binary +$AUDITWHEEL_PYTHON -m pip install . + + # Show ccache stats echo "Cache stats:" ccache -s -cd /io - # Compile wheel +cd /io "/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 # Show ccache stats @@ -55,11 +63,6 @@ for whl in /io/wheelhouse/*linux_*.whl; do auditwheel -v repair "$whl" -w /io/wheelhouse/ || exit 1 done -# Fix each wheel generated by auditwheel -for whl in /io/wheelhouse/*manylinux1_*.whl; do - /io/ci-scripts/docker/fix_wheel.sh "$whl" || exit 1 -done - # Install packages and test TEST_HOME=/io/testdata "/opt/python/$PY_VERSION/bin/pip" install klayout --no-index -f /io/wheelhouse || exit 1 From 6b37e67de68d758db060a2fbe20ae0ca832c353c Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 10:59:25 -0500 Subject: [PATCH 16/23] fixing pya module. adding tests --- .travis.yml | 1 + ci-scripts/docker/docker_build.sh | 2 ++ src/pymod/__init__.py.qtless | 3 +-- src/pymod/distutils_src/pya/__init__.py | 15 ++++++--------- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fe284fb0..4dcb2f7ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -478,6 +478,7 @@ script: python setup.py bdist_wheel; python setup.py install; python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; + python testdata/pymod/pya_tests.py; klayout_version=$(python -c 'import setup; print(setup.Config().version())'); mkdir -p deploy/dist-pymod/$klayout_version; cp -a dist/* deploy/dist-pymod/$klayout_version; diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index b511f879f..2f6d68710 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -69,3 +69,5 @@ TEST_HOME=/io/testdata "/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_db.py || exit 1 "/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_rdb.py || exit 1 "/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_tl.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/pya_tests.py || exit 1 + diff --git a/src/pymod/__init__.py.qtless b/src/pymod/__init__.py.qtless index dce96f8e4..6aef40aa9 100644 --- a/src/pymod/__init__.py.qtless +++ b/src/pymod/__init__.py.qtless @@ -1,5 +1,4 @@ - # klayout library definition file -__all__ = [ "tl", "db", "lay", "rdb" ] +__all__ = [ "tl", "db", "rdb" ] diff --git a/src/pymod/distutils_src/pya/__init__.py b/src/pymod/distutils_src/pya/__init__.py index ae97d592a..70e2f49b9 100644 --- a/src/pymod/distutils_src/pya/__init__.py +++ b/src/pymod/distutils_src/pya/__init__.py @@ -1,10 +1,7 @@ +# import all packages from klayout, such as klayout.db and klayout.tl +# WARNING: doing it manually until it becomes impractical +# TODO: We need a specification document explaining what should go into pya -import klayout -import importlib - -__all__ = [] -for m in klayout.__all__: - mod = importlib.import_module("klayout." + m) - for mm in mod.__all__: - globals()[mm] = getattr(mod, mm) - +from klayout.db import * # noqa +from klayout.tl import * # noqa +from klayout.rdb import * # noqa From d20e12ef415d75afdc1bffe4858681534b27f232 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 14:08:30 -0500 Subject: [PATCH 17/23] Adding tests on host linux beyond docker. --- .travis.yml | 191 +++++++++++++++++++++++++++++----------------------- 1 file changed, 106 insertions(+), 85 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4dcb2f7ed..373f5e71e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,14 @@ matrix: - name: "manylinux1_x86_64 cp37-cp37m package" os: linux sudo: true + python: '3.7-dev' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp37-cp37m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -33,12 +35,14 @@ matrix: - name: "manylinux1_x86_64 cp36-cp36m package" os: linux sudo: true + python: '3.6' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp36-cp36m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -61,12 +65,14 @@ matrix: - name: "manylinux1_x86_64 cp35-cp35m package" os: linux sudo: true + python: '3.5' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp35-cp35m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -89,12 +95,14 @@ matrix: - name: "manylinux1_x86_64 cp34-cp34m package" os: linux sudo: true + python: '3.4' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp34-cp34m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -117,12 +125,14 @@ matrix: - name: "manylinux1_x86_64 cp27-cp27mu package" os: linux sudo: true + python: '2.7' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp27-cp27mu" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -145,12 +155,14 @@ matrix: - name: "manylinux1_x86_64 cp27-cp27m package" os: linux sudo: true + python: '2.7' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp27-cp27m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -266,96 +278,96 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.7 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.7-dev' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.7 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.7-dev' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.6 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.6' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.6 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.6' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python2.7 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '2.7' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python2.7 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '2.7' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python2.6 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '2.6' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python2.6 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '2.6' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.3 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.3' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.3 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.3' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.4 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.4' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.4 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.4' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.5 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.5' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.5 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.5' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ # KLayout builds for mac # Python 3 @@ -472,12 +484,21 @@ script: klayout_version=$(python -c 'import setup; print(setup.Config().version())'); mkdir -p deploy/dist-pymod/$klayout_version; cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod/$klayout_version; + if [ "$TEST_IN_HOST" = true ]; then + pip install klayout --no-index -f ./wheelhouse; + python testdata/pymod/import_db.py; + python testdata/pymod/import_rdb.py; + python testdata/pymod/import_tl.py; + python testdata/pymod/pya_tests.py; + fi fi - if [ "$PYTHON_BUILD" = true ]; then python setup.py build; python setup.py bdist_wheel; python setup.py install; - python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; + python testdata/pymod/import_db.py; + python testdata/pymod/import_rdb.py; + python testdata/pymod/import_tl.py; python testdata/pymod/pya_tests.py; klayout_version=$(python -c 'import setup; print(setup.Config().version())'); mkdir -p deploy/dist-pymod/$klayout_version; From 5f66df13f7ae41e56884d9b7f93bb5f730be3d08 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 14:09:56 -0500 Subject: [PATCH 18/23] adding ccache to macos builds --- .travis.yml | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.travis.yml b/.travis.yml index 373f5e71e..6dfb31bd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -188,6 +188,11 @@ matrix: - name: "klayout python2 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="" - ARCHFLAGS="-std=c++11" @@ -197,6 +202,11 @@ matrix: - name: "klayout python2 osx10.12" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew install python2 || brew link --overwrite python@2" # deficient python2 in travis's xcode8.3 (no ssl) - ARCHFLAGS="-std=c++11" @@ -206,6 +216,11 @@ matrix: - name: "klayout python2 osx10.11" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="" - ARCHFLAGS="-std=c++11" @@ -217,10 +232,12 @@ matrix: - name: "klayout python3 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - python3 + - ccache update: true env: - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" @@ -231,6 +248,11 @@ matrix: - name: "klayout python3.6.6 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python36; brew link --force --overwrite python36; shopt -s expand_aliases; alias python='/usr/local/opt/python36/bin/python3.6'; alias pip='/usr/local/opt/python36/bin/pip3.6';" - ARCHFLAGS="-std=c++11" @@ -240,6 +262,11 @@ matrix: - name: "klayout python3.5.6 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python35; brew link --force --overwrite python35; shopt -s expand_aliases; alias python='/usr/local/opt/python35/bin/python3.5'; alias pip='/usr/local/opt/python35/bin/pip3.5';" - ARCHFLAGS="-std=c++11" @@ -249,6 +276,11 @@ matrix: - name: "klayout python3.4.9 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';" - ARCHFLAGS="-std=c++11" @@ -258,10 +290,12 @@ matrix: - name: "klayout python3 osx10.12" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - python3 + - ccache update: true env: - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" @@ -272,6 +306,11 @@ matrix: - name: "klayout python3 osx10.11" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';" - ARCHFLAGS="-std=c++11" @@ -374,11 +413,13 @@ matrix: - name: "KLayout macOS 10.13 with py3.7" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -389,11 +430,13 @@ matrix: - name: "KLayout macOS 10.12 with py3.7" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -404,11 +447,13 @@ matrix: - name: "KLayout macOS 10.11 with py3.7" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8 @@ -420,10 +465,12 @@ matrix: - name: "KLayout macOS 10.13 with py2.7" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -434,10 +481,12 @@ matrix: - name: "KLayout macOS 10.12 with py2.7" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -448,10 +497,12 @@ matrix: - name: "KLayout macOS 10.11 with py2.7" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8 @@ -470,6 +521,9 @@ before_install: pip install --upgrade setuptools wheel || sudo pip install --upgrade setuptools wheel; fi - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" + - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + export PATH="/usr/local/opt/ccache/libexec:$PATH"; + fi install: - if [ "$DOCKER_BUILD" = true ]; then From 8495a18023607aa6b3592d838dab09171cef2c1d Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 13:39:25 -0500 Subject: [PATCH 19/23] changing version to 0.26.0.dev10 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 06f0fbeb5..059f4b44c 100644 --- a/setup.py +++ b/setup.py @@ -322,7 +322,7 @@ class Config(object): """ Gets the version string """ - return "0.26.0.dev9" + return "0.26.0.dev10" config = Config() From 124975d63627c8fe34f7d853fecb84b68e2df665 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 15:03:42 -0500 Subject: [PATCH 20/23] bug: forgot to add language:python in travis --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6dfb31bd9..80fe85300 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ matrix: - name: "manylinux1_x86_64 cp37-cp37m package" os: linux sudo: true + language: python python: '3.7-dev' services: - docker @@ -35,6 +36,7 @@ matrix: - name: "manylinux1_x86_64 cp36-cp36m package" os: linux sudo: true + language: python python: '3.6' services: - docker @@ -65,6 +67,7 @@ matrix: - name: "manylinux1_x86_64 cp35-cp35m package" os: linux sudo: true + language: python python: '3.5' services: - docker @@ -95,6 +98,7 @@ matrix: - name: "manylinux1_x86_64 cp34-cp34m package" os: linux sudo: true + language: python python: '3.4' services: - docker @@ -125,6 +129,7 @@ matrix: - name: "manylinux1_x86_64 cp27-cp27mu package" os: linux sudo: true + language: python python: '2.7' services: - docker @@ -155,6 +160,7 @@ matrix: - name: "manylinux1_x86_64 cp27-cp27m package" os: linux sudo: true + language: python python: '2.7' services: - docker From b3addbeeba84284dfa70a5ffdb2a056dc9bf815c Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 15:19:30 -0500 Subject: [PATCH 21/23] better organizing travis metadata. --- .travis.yml | 53 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 80fe85300..29f694ef2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ matrix: include: # python manylinux packages - - name: "manylinux1_x86_64 cp37-cp37m package" + - name: "cp37-cp37m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -19,7 +19,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp37-cp37m package" + - name: "cp37-cp37m-manylinux1_i686.whl" os: linux sudo: true services: @@ -33,7 +33,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp36-cp36m package" + - name: "cp36-cp36m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -50,7 +50,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp36-cp36m package" + - name: "cp36-cp36m-manylinux1_i686.whl" os: linux sudo: true services: @@ -64,7 +64,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp35-cp35m package" + - name: "cp35-cp35m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -81,7 +81,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp35-cp35m package" + - name: "cp35-cp35m-manylinux1_i686.whl" os: linux sudo: true services: @@ -95,7 +95,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp34-cp34m package" + - name: "cp34-cp34m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -112,7 +112,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp34-cp34m package" + - name: "cp34-cp34m-manylinux1_i686.whl" os: linux sudo: true services: @@ -126,7 +126,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp27-cp27mu package" + - name: "cp27-cp27mu-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -143,7 +143,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp27-cp27mu package" + - name: "cp27-cp27mu-manylinux1_i686.whl" os: linux sudo: true services: @@ -157,7 +157,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp27-cp27m package" + - name: "cp27-cp27m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -174,7 +174,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp27-cp27m package" + - name: "cp27-cp27m-manylinux1_i686.whl" os: linux sudo: true services: @@ -191,7 +191,8 @@ matrix: # python 2 osx - - name: "klayout python2 osx10.13" + # - name: "klayout python2 osx10.13" + - name: "cp27-cp27m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -205,7 +206,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python2 osx10.12" + # - name: "klayout python2 osx10.12" + - name: "cp27-cp27m-macosx_10_12_x86_64.whl" os: osx osx_image: xcode8.3 # macOS 10.12 cache: ccache @@ -219,7 +221,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python2 osx10.11" + # - name: "klayout python2 osx10.11" + - name: "cp27-cp27m-macosx_10_11_x86_64.whl" os: osx osx_image: xcode8 # macOS 10.11 cache: ccache @@ -235,7 +238,8 @@ matrix: # python 3 osx - - name: "klayout python3 osx10.13" + # - name: "klayout python3 osx10.13" + - name: "cp37-cp37m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -251,7 +255,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.6.6 osx10.13" + # - name: "klayout python3.6.6 osx10.13" + - name: "cp36-cp36m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -265,7 +270,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.5.6 osx10.13" + # - name: "klayout python3.5.6 osx10.13" + - name: "cp35-cp35m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -279,7 +285,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.4.9 osx10.13" + # - name: "klayout python3.4.9 osx10.13" + - name: "cp34-cp34m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -293,7 +300,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3 osx10.12" + # - name: "klayout python3 osx10.12" + - name: "cp37-cp37m-macosx_10_12_x86_64.whl" os: osx osx_image: xcode8.3 # macOS 10.12 cache: ccache @@ -309,7 +317,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3 osx10.11" + # - name: "klayout python3 osx10.11" + - name: "cp37-cp37m-macosx_10_11_x86_64.whl" os: osx osx_image: xcode8 # macOS 10.11 cache: ccache @@ -562,7 +571,7 @@ script: python testdata/pymod/pya_tests.py; klayout_version=$(python -c 'import setup; print(setup.Config().version())'); mkdir -p deploy/dist-pymod/$klayout_version; - cp -a dist/* deploy/dist-pymod/$klayout_version; + cp -a dist/*.whl deploy/dist-pymod/$klayout_version; python -c 'import klayout.db as db; print(dir(db))'; python -c 'import klayout.rdb as rdb; print(dir(rdb))'; python -c 'import klayout.tl as tl; print(dir(tl))'; From 2975d57d22a46fd4434a22aefba26c27cebd10b0 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 15:50:57 -0500 Subject: [PATCH 22/23] Not testing cp27-cp27m in travis's host (incompatible). --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29f694ef2..bdcd00179 100644 --- a/.travis.yml +++ b/.travis.yml @@ -168,7 +168,7 @@ matrix: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp27-cp27m" - DOCKER_BUILD=true - - TEST_IN_HOST=true + - TEST_IN_HOST=false # travis's python 2.7 uses ucs4 (mu), so this test fails. - MATRIX_EVAL="" cache: directories: From 1bba72e45bdfd95a28235af0e515b5917c8c9277 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 16:01:15 -0500 Subject: [PATCH 23/23] Adding documentation regarding manylinux, CI, and PyPI wheels --- ci-scripts/docker/README.md | 220 ++--------------- .../{ => development_notes}/Dockerfile.i686 | 0 .../{ => development_notes}/Dockerfile.x86_64 | 0 ci-scripts/docker/development_notes/README.md | 221 ++++++++++++++++++ .../{ => development_notes}/fix_wheel.sh | 0 .../manylinux-docker.sh | 0 ci-scripts/twine/README.md | 15 ++ 7 files changed, 249 insertions(+), 207 deletions(-) rename ci-scripts/docker/{ => development_notes}/Dockerfile.i686 (100%) rename ci-scripts/docker/{ => development_notes}/Dockerfile.x86_64 (100%) create mode 100644 ci-scripts/docker/development_notes/README.md rename ci-scripts/docker/{ => development_notes}/fix_wheel.sh (100%) rename ci-scripts/docker/{ => development_notes}/manylinux-docker.sh (100%) create mode 100644 ci-scripts/twine/README.md diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 9cf7370a2..753d2fee3 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -1,216 +1,22 @@ -# Chapter 1. Testing on your own computer -## Step 1. +Author: Thomas Ferreira de Lima +email: thomas@tlima.me -Make sure you have the quay.io/pypa/manylinux1_x86_64 image. +This folder contains scripts to be run inside docker images. See instructions on how to test this yourself in ci-scripts/docker/development_notes. + +## docker_build.sh + +We need two environment variables to get going: ```bash -$ docker images -REPOSITORY TAG IMAGE ID CREATED SIZE -quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB -hello-world latest 4ab4c602aa5e 3 months ago 1.84kB -# My image was old: -$ docker pull quay.io/pypa/manylinux1_x86_64 -Using default tag: latest -latest: Pulling from pypa/manylinux1_x86_64 -7d0d9526f38a: Already exists -3324bfadf9cb: Pull complete -20f27c7e3062: Pull complete -5bc21fc5fe97: Pull complete -Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 -Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest +DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" +PY_VERSION="cp37-cp37m" ``` -## Step 2. - -This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 -Create a volume for klayout. This is necessary because docker containers don't persist data. +The script must be run inside an image pulled from $DOCKER_IMAGE and with klayout's git repo cloned in /io. Inside the git clone folder, run: ```bash -$ docker volume create klayout-persist -$ docker volume inspect klayout-persist -[ - { - "CreatedAt": "2018-12-18T15:01:48Z", - "Driver": "local", - "Labels": {}, - "Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data", - "Name": "klayout-persist", - "Options": {}, - "Scope": "local" - } -] +docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; +# $PRE_CMD is empty for now (useless currently). ``` -## Step 3. - -Build image `myimage` with: - -```bash -$ docker build -t myimage:latest -f Dockerfile.x86_64 . -``` - -This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. - -Then I run the docker with a terminal shell and load the volume klayout-persist in /persist: - -```bash -$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage -``` - -## Step 4. - -In the shell, pull master from klayout. - -```bash -cd /persist -git clone https://github.com/lightwave-lab/klayout.git -mkdir -p wheelhouse -cd klayout -# make wheel with python 3.6 (for example) -/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/ -cd /persist -auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/ -# Need to manually fix the wheel -#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse -``` - -The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. - - -Procedure to fix the wheel: - -```bash -unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel -cd tempwheel/klayout -mv lib_* db_plugins .libs/ -cd ../klayout-0.26.0.dev8.dist-info/ -sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD -sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD -cd ../ -rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./* -cd .. -rm -rf tempwheel -``` - -Now we can install and test: -```bash -/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse -cd /persist/klayout -/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py -# Tests passed! -``` - -Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. - -## Step 5. Iterate over all python versions. - -For that, we need something like: - -```bash -# Compile wheels -for PYBIN in /opt/python/*/bin; do - "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ -done - -# Bundle external shared libraries into the wheels via auditwheel -for whl in /persist/wheelhouse/*linux_*.whl; do - auditwheel repair "$whl" -w /persist/wheelhouse/ -done - -# Fix each wheel generated by auditwheel -for whl in /persist/wheelhouse/*manylinux1_*.whl; do - ./ci-scripts/docker/fix_wheel.sh "$whl" -done - -# Install packages and test -TEST_HOME=/persist/klayout/testdata -for PYBIN in /opt/python/*/bin/; do - "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse - "${PYBIN}/python" $TEST_HOME/pymod/import_db.py - "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py - "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py - -``` - -I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: - -`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` - -I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. -The ccache folder after a single build has 657MB. Go figure. - -I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1 - -So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default. - -# Chapter 2. Testing CI flow with docker - -# Step 1. Testing commands in own computer - -First cloned with: -```bash -git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux -cd klayout -``` - -Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml - -DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 - -PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m - -Total of 2x6 = 12 possibilities - -```bash -export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 -export PY_VERSION="cp36-cp36m" -docker pull $DOCKER_IMAGE -mkdir -p ccache -mkdir -p wheelhouse -docker run --name klayout -v `pwd` -it $DOCKER_IMAGE -``` - -Inside docker shell: -```bash -yum install -y zlib-devel -yum install -y zip - -yum install -y ccache -ln -s /usr/bin/ccache /usr/lib64/ccache/c++ -ln -s /usr/bin/ccache /usr/lib64/ccache/cc -ln -s /usr/bin/ccache /usr/lib64/ccache/gcc -ln -s /usr/bin/ccache /usr/lib64/ccache/g++ -echo $PATH -# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -export CCACHE_DIR="/io/ccache" - -# Compile wheel -/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/ - -# Bundle external shared libraries into the wheels via auditwheel -for whl in /io/wheelhouse/*linux_*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ -done - -# Fix each wheel generated by auditwheel -for whl in /io/wheelhouse/*manylinux1_*.whl; do - ./ci-scripts/docker/fix_wheel.sh "$whl" -done - -``` - -# Step 2. Automating step 1 in travis (CI). -DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 - -PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m - -Build: spawn 12 travis jobs, one for each combination of word-size and python version. -Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`. -Post-build: -- cache `./ccache` -- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder) - -# Step 3. Automating deployment to PyPI: - -TBD +This command will generate a wheel and place it in `wheelhouse/klayout-*manylinux1*.whl`. This is the wheel that needs to be uploaded to PyPI via twine. See ci-scripts/twine/README.md. diff --git a/ci-scripts/docker/Dockerfile.i686 b/ci-scripts/docker/development_notes/Dockerfile.i686 similarity index 100% rename from ci-scripts/docker/Dockerfile.i686 rename to ci-scripts/docker/development_notes/Dockerfile.i686 diff --git a/ci-scripts/docker/Dockerfile.x86_64 b/ci-scripts/docker/development_notes/Dockerfile.x86_64 similarity index 100% rename from ci-scripts/docker/Dockerfile.x86_64 rename to ci-scripts/docker/development_notes/Dockerfile.x86_64 diff --git a/ci-scripts/docker/development_notes/README.md b/ci-scripts/docker/development_notes/README.md new file mode 100644 index 000000000..5a05684a5 --- /dev/null +++ b/ci-scripts/docker/development_notes/README.md @@ -0,0 +1,221 @@ +Author: Thomas Ferreira de Lima +email: thomas@tlima.me + +I wrote these notes as I was learning how to use docker and how to build python packages inside a docker image prepared by the pypa team. They require us to build there to allow wheels to have the `manylinux1` tag, meaning that these wheels would be compatible with most linux distributions around. Chapter 1 is about testing in my own computer (MacOS Mojave) and Chapter 2 is about how to automate this build using travis-ci.org. + +# Chapter 1. Testing on your own computer +## Step 1. + +Make sure you have the quay.io/pypa/manylinux1_x86_64 image. + +```bash +$ docker images +REPOSITORY TAG IMAGE ID CREATED SIZE +quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB +hello-world latest 4ab4c602aa5e 3 months ago 1.84kB +# My image was old: +$ docker pull quay.io/pypa/manylinux1_x86_64 +Using default tag: latest +latest: Pulling from pypa/manylinux1_x86_64 +7d0d9526f38a: Already exists +3324bfadf9cb: Pull complete +20f27c7e3062: Pull complete +5bc21fc5fe97: Pull complete +Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 +Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest +``` + +## Step 2. + +This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 +Create a volume for klayout. This is necessary because docker containers don't persist data. + +```bash +$ docker volume create klayout-persist +$ docker volume inspect klayout-persist +[ + { + "CreatedAt": "2018-12-18T15:01:48Z", + "Driver": "local", + "Labels": {}, + "Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data", + "Name": "klayout-persist", + "Options": {}, + "Scope": "local" + } +] +``` + +## Step 3. + +Build image `myimage` with: + +```bash +$ docker build -t myimage:latest -f Dockerfile.x86_64 . +``` + +This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. + +Then I run the docker with a terminal shell and load the volume klayout-persist in /persist: + +```bash +$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage +``` + +## Step 4. + +In the shell, pull master from klayout. + +```bash +cd /persist +git clone https://github.com/lightwave-lab/klayout.git +mkdir -p wheelhouse +cd klayout +# make wheel with python 3.6 (for example) +/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/ +cd /persist +auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/ +# Need to manually fix the wheel +#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse +``` + +The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. + + +Procedure to fix the wheel: + +```bash +unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel +cd tempwheel/klayout +mv lib_* db_plugins .libs/ +cd ../klayout-0.26.0.dev8.dist-info/ +sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD +sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD +cd ../ +rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl +zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./* +cd .. +rm -rf tempwheel +``` + +Now we can install and test: +```bash +/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse +cd /persist/klayout +/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py +# Tests passed! +``` + +Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. + +## Step 5. Iterate over all python versions. + +For that, we need something like: + +```bash +# Compile wheels +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ +done + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /persist/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /persist/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /persist/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +# Install packages and test +TEST_HOME=/persist/klayout/testdata +for PYBIN in /opt/python/*/bin/; do + "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse + "${PYBIN}/python" $TEST_HOME/pymod/import_db.py + "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py + "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py + +``` + +I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: + +`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` + +I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. +The ccache folder after a single build has 657MB. Go figure. + +I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1 + +So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default. + +# Chapter 2. Testing CI flow with docker + +# Step 1. Testing commands in own computer + +First cloned with: +```bash +git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux +cd klayout +``` + +Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml + +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Total of 2x6 = 12 possibilities + +```bash +export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 +export PY_VERSION="cp36-cp36m" +docker pull $DOCKER_IMAGE +mkdir -p ccache +mkdir -p wheelhouse +docker run --name klayout -v `pwd` -it $DOCKER_IMAGE +``` + +Inside docker shell: +```bash +yum install -y zlib-devel +yum install -y zip + +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" + +# Compile wheel +/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/ + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +``` + +# Step 2. Automating step 1 in travis (CI). +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Build: spawn 12 travis jobs, one for each combination of word-size and python version. +Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`. +Post-build: +- cache `./ccache` +- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder) + +# Step 3. Automating deployment to PyPI: + +TBD diff --git a/ci-scripts/docker/fix_wheel.sh b/ci-scripts/docker/development_notes/fix_wheel.sh similarity index 100% rename from ci-scripts/docker/fix_wheel.sh rename to ci-scripts/docker/development_notes/fix_wheel.sh diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/development_notes/manylinux-docker.sh similarity index 100% rename from ci-scripts/docker/manylinux-docker.sh rename to ci-scripts/docker/development_notes/manylinux-docker.sh diff --git a/ci-scripts/twine/README.md b/ci-scripts/twine/README.md new file mode 100644 index 000000000..3916de5d4 --- /dev/null +++ b/ci-scripts/twine/README.md @@ -0,0 +1,15 @@ +After building all the travis wheels, go to the folder where the wheels were deployed. In my case, for example, `/Users/tlima/Dropbox/Apps/travis-deploy/Builds/klayout/dist-pymod/0.26.0.dev10`. + +Then, run the command + +```bash +travis upload *.whl +``` + +, which will ask for an username and password related to your account. + +After this upload was successful, you can upload the source tarball. Go to klayout's git folder and run `python setup.py sdist`. Inside `dist/`, you'll find a tarball named, e.g., `klayout-0.26.0.dev10.tar.gz`. So just run + +```bash +travis upload klayout-0.26.0.dev10.tar.gz +```