diff --git a/.travis.yml b/.travis.yml index ee2c85e99..bdcd00179 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,235 @@ matrix: include: + # python manylinux packages + + - name: "cp37-cp37m-manylinux1_x86_64.whl" + os: linux + sudo: true + language: python + python: '3.7-dev' + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp37-cp37m" + - DOCKER_BUILD=true + - TEST_IN_HOST=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp37-cp37m-manylinux1_i686.whl" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp37-cp37m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp36-cp36m-manylinux1_x86_64.whl" + os: linux + sudo: true + language: python + python: '3.6' + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp36-cp36m" + - DOCKER_BUILD=true + - TEST_IN_HOST=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp36-cp36m-manylinux1_i686.whl" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp36-cp36m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp35-cp35m-manylinux1_x86_64.whl" + os: linux + sudo: true + language: python + python: '3.5' + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp35-cp35m" + - DOCKER_BUILD=true + - TEST_IN_HOST=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp35-cp35m-manylinux1_i686.whl" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp35-cp35m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp34-cp34m-manylinux1_x86_64.whl" + os: linux + sudo: true + language: python + python: '3.4' + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp34-cp34m" + - DOCKER_BUILD=true + - TEST_IN_HOST=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp34-cp34m-manylinux1_i686.whl" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp34-cp34m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp27-cp27mu-manylinux1_x86_64.whl" + os: linux + sudo: true + language: python + python: '2.7' + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp27-cp27mu" + - DOCKER_BUILD=true + - TEST_IN_HOST=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp27-cp27mu-manylinux1_i686.whl" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp27-cp27mu" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp27-cp27m-manylinux1_x86_64.whl" + os: linux + sudo: true + language: python + python: '2.7' + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp27-cp27m" + - DOCKER_BUILD=true + - TEST_IN_HOST=false # travis's python 2.7 uses ucs4 (mu), so this test fails. + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "cp27-cp27m-manylinux1_i686.whl" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp27-cp27m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + # python 2 osx - - name: "klayout python2 osx10.13" + # - name: "klayout python2 osx10.13" + - name: "cp27-cp27m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python2 osx10.12" + # - name: "klayout python2 osx10.12" + - name: "cp27-cp27m-macosx_10_12_x86_64.whl" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew install python2 || brew link --overwrite python@2" # deficient python2 in travis's xcode8.3 (no ssl) - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python2 osx10.11" + # - name: "klayout python2 osx10.11" + - name: "cp27-cp27m-macosx_10_11_x86_64.whl" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="" - ARCHFLAGS="-std=c++11" @@ -31,13 +238,16 @@ matrix: # python 3 osx - - name: "klayout python3 osx10.13" + # - name: "klayout python3 osx10.13" + - name: "cp37-cp37m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - python3 + - ccache update: true env: - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" @@ -45,40 +255,61 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.6.6 osx10.13" + # - name: "klayout python3.6.6 osx10.13" + - name: "cp36-cp36m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python36; brew link --force --overwrite python36; shopt -s expand_aliases; alias python='/usr/local/opt/python36/bin/python3.6'; alias pip='/usr/local/opt/python36/bin/pip3.6';" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.5.6 osx10.13" + # - name: "klayout python3.5.6 osx10.13" + - name: "cp35-cp35m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python35; brew link --force --overwrite python35; shopt -s expand_aliases; alias python='/usr/local/opt/python35/bin/python3.5'; alias pip='/usr/local/opt/python35/bin/pip3.5';" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.4.9 osx10.13" + # - name: "klayout python3.4.9 osx10.13" + - name: "cp34-cp34m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3 osx10.12" + # - name: "klayout python3 osx10.12" + - name: "cp37-cp37m-macosx_10_12_x86_64.whl" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - python3 + - ccache update: true env: - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" @@ -86,116 +317,124 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3 osx10.11" + # - name: "klayout python3 osx10.11" + - name: "cp37-cp37m-macosx_10_11_x86_64.whl" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.7 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.7-dev' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.7 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.7-dev' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.6 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.6' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.6 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.6' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python2.7 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '2.7' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python2.7 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '2.7' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python2.6 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '2.6' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python2.6 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '2.6' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.3 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.3' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.3 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.3' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.4 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.4' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.4 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.4' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.5 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.5' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.5 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.5' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ # KLayout builds for mac # Python 3 - name: "KLayout macOS 10.13 with py3.7" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -206,11 +445,13 @@ matrix: - name: "KLayout macOS 10.12 with py3.7" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -221,11 +462,13 @@ matrix: - name: "KLayout macOS 10.11 with py3.7" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8 @@ -237,10 +480,12 @@ matrix: - name: "KLayout macOS 10.13 with py2.7" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -251,10 +496,12 @@ matrix: - name: "KLayout macOS 10.12 with py2.7" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -265,10 +512,12 @@ matrix: - name: "KLayout macOS 10.11 with py2.7" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8 @@ -287,15 +536,42 @@ before_install: pip install --upgrade setuptools wheel || sudo pip install --upgrade setuptools wheel; fi - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" + - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + export PATH="/usr/local/opt/ccache/libexec:$PATH"; + fi -script: +install: + - if [ "$DOCKER_BUILD" = true ]; then + docker pull $DOCKER_IMAGE; + fi + +script: + - if [ "$DOCKER_BUILD" = true ]; then + mkdir -p ccache; + mkdir -p wheelhouse; + docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; + klayout_version=$(python -c 'import setup; print(setup.Config().version())'); + mkdir -p deploy/dist-pymod/$klayout_version; + cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod/$klayout_version; + if [ "$TEST_IN_HOST" = true ]; then + pip install klayout --no-index -f ./wheelhouse; + python testdata/pymod/import_db.py; + python testdata/pymod/import_rdb.py; + python testdata/pymod/import_tl.py; + python testdata/pymod/pya_tests.py; + fi + fi - if [ "$PYTHON_BUILD" = true ]; then python setup.py build; python setup.py bdist_wheel; python setup.py install; - python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; - mkdir -p deploy/dist-pymod; - cp -a dist/* deploy/dist-pymod/; + python testdata/pymod/import_db.py; + python testdata/pymod/import_rdb.py; + python testdata/pymod/import_tl.py; + python testdata/pymod/pya_tests.py; + klayout_version=$(python -c 'import setup; print(setup.Config().version())'); + mkdir -p deploy/dist-pymod/$klayout_version; + cp -a dist/*.whl deploy/dist-pymod/$klayout_version; python -c 'import klayout.db as db; print(dir(db))'; python -c 'import klayout.rdb as rdb; print(dir(rdb))'; python -c 'import klayout.tl as tl; print(dir(tl))'; diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md new file mode 100644 index 000000000..753d2fee3 --- /dev/null +++ b/ci-scripts/docker/README.md @@ -0,0 +1,22 @@ +Author: Thomas Ferreira de Lima +email: thomas@tlima.me + +This folder contains scripts to be run inside docker images. See instructions on how to test this yourself in ci-scripts/docker/development_notes. + +## docker_build.sh + +We need two environment variables to get going: + +```bash +DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" +PY_VERSION="cp37-cp37m" +``` + +The script must be run inside an image pulled from $DOCKER_IMAGE and with klayout's git repo cloned in /io. Inside the git clone folder, run: + +```bash +docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; +# $PRE_CMD is empty for now (useless currently). +``` + +This command will generate a wheel and place it in `wheelhouse/klayout-*manylinux1*.whl`. This is the wheel that needs to be uploaded to PyPI via twine. See ci-scripts/twine/README.md. diff --git a/ci-scripts/docker/development_notes/Dockerfile.i686 b/ci-scripts/docker/development_notes/Dockerfile.i686 new file mode 100644 index 000000000..7d68054fc --- /dev/null +++ b/ci-scripts/docker/development_notes/Dockerfile.i686 @@ -0,0 +1,17 @@ +FROM quay.io/pypa/manylinux1_i686 +MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me) + +# Install a system package required by our library +RUN linux32 yum install -y zlib-devel +RUN linux32 yum install -y ccache +RUN ln -s /usr/bin/ccache /usr/lib/ccache/c++ +RUN ln -s /usr/bin/ccache /usr/lib/ccache/cc +RUN ln -s /usr/bin/ccache /usr/lib/ccache/gcc +RUN ln -s /usr/bin/ccache /usr/lib/ccache/g++ + +# Add ccache to PATH +RUN mkdir -p /persist/.ccache +ENV CCACHE_DIR="/persist/.ccache" + +# Need zip to fix wheel +RUN linux32 yum install -y zip diff --git a/ci-scripts/docker/development_notes/Dockerfile.x86_64 b/ci-scripts/docker/development_notes/Dockerfile.x86_64 new file mode 100644 index 000000000..4bf03c3cd --- /dev/null +++ b/ci-scripts/docker/development_notes/Dockerfile.x86_64 @@ -0,0 +1,17 @@ +FROM quay.io/pypa/manylinux1_x86_64 +MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me) + +# Install a system package required by our library +RUN yum install -y zlib-devel +RUN yum install -y ccache +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/cc +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + +# Add ccache to PATH +RUN mkdir -p /persist/.ccache +ENV CCACHE_DIR="/persist/.ccache" + +# Need zip to fix wheel +RUN yum install -y zip diff --git a/ci-scripts/docker/development_notes/README.md b/ci-scripts/docker/development_notes/README.md new file mode 100644 index 000000000..5a05684a5 --- /dev/null +++ b/ci-scripts/docker/development_notes/README.md @@ -0,0 +1,221 @@ +Author: Thomas Ferreira de Lima +email: thomas@tlima.me + +I wrote these notes as I was learning how to use docker and how to build python packages inside a docker image prepared by the pypa team. They require us to build there to allow wheels to have the `manylinux1` tag, meaning that these wheels would be compatible with most linux distributions around. Chapter 1 is about testing in my own computer (MacOS Mojave) and Chapter 2 is about how to automate this build using travis-ci.org. + +# Chapter 1. Testing on your own computer +## Step 1. + +Make sure you have the quay.io/pypa/manylinux1_x86_64 image. + +```bash +$ docker images +REPOSITORY TAG IMAGE ID CREATED SIZE +quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB +hello-world latest 4ab4c602aa5e 3 months ago 1.84kB +# My image was old: +$ docker pull quay.io/pypa/manylinux1_x86_64 +Using default tag: latest +latest: Pulling from pypa/manylinux1_x86_64 +7d0d9526f38a: Already exists +3324bfadf9cb: Pull complete +20f27c7e3062: Pull complete +5bc21fc5fe97: Pull complete +Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 +Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest +``` + +## Step 2. + +This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 +Create a volume for klayout. This is necessary because docker containers don't persist data. + +```bash +$ docker volume create klayout-persist +$ docker volume inspect klayout-persist +[ + { + "CreatedAt": "2018-12-18T15:01:48Z", + "Driver": "local", + "Labels": {}, + "Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data", + "Name": "klayout-persist", + "Options": {}, + "Scope": "local" + } +] +``` + +## Step 3. + +Build image `myimage` with: + +```bash +$ docker build -t myimage:latest -f Dockerfile.x86_64 . +``` + +This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. + +Then I run the docker with a terminal shell and load the volume klayout-persist in /persist: + +```bash +$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage +``` + +## Step 4. + +In the shell, pull master from klayout. + +```bash +cd /persist +git clone https://github.com/lightwave-lab/klayout.git +mkdir -p wheelhouse +cd klayout +# make wheel with python 3.6 (for example) +/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/ +cd /persist +auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/ +# Need to manually fix the wheel +#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse +``` + +The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. + + +Procedure to fix the wheel: + +```bash +unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel +cd tempwheel/klayout +mv lib_* db_plugins .libs/ +cd ../klayout-0.26.0.dev8.dist-info/ +sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD +sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD +cd ../ +rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl +zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./* +cd .. +rm -rf tempwheel +``` + +Now we can install and test: +```bash +/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse +cd /persist/klayout +/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py +# Tests passed! +``` + +Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. + +## Step 5. Iterate over all python versions. + +For that, we need something like: + +```bash +# Compile wheels +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ +done + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /persist/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /persist/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /persist/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +# Install packages and test +TEST_HOME=/persist/klayout/testdata +for PYBIN in /opt/python/*/bin/; do + "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse + "${PYBIN}/python" $TEST_HOME/pymod/import_db.py + "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py + "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py + +``` + +I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: + +`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` + +I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. +The ccache folder after a single build has 657MB. Go figure. + +I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1 + +So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default. + +# Chapter 2. Testing CI flow with docker + +# Step 1. Testing commands in own computer + +First cloned with: +```bash +git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux +cd klayout +``` + +Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml + +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Total of 2x6 = 12 possibilities + +```bash +export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 +export PY_VERSION="cp36-cp36m" +docker pull $DOCKER_IMAGE +mkdir -p ccache +mkdir -p wheelhouse +docker run --name klayout -v `pwd` -it $DOCKER_IMAGE +``` + +Inside docker shell: +```bash +yum install -y zlib-devel +yum install -y zip + +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" + +# Compile wheel +/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/ + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +``` + +# Step 2. Automating step 1 in travis (CI). +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Build: spawn 12 travis jobs, one for each combination of word-size and python version. +Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`. +Post-build: +- cache `./ccache` +- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder) + +# Step 3. Automating deployment to PyPI: + +TBD diff --git a/ci-scripts/docker/development_notes/fix_wheel.sh b/ci-scripts/docker/development_notes/fix_wheel.sh new file mode 100755 index 000000000..2892bdbeb --- /dev/null +++ b/ci-scripts/docker/development_notes/fix_wheel.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=`basename "$0"` +TMP_WHEEL="/tmp/klayout_tempwheel" + +display_usage() { + echo "This script fixes auditwheel-repaired wheels." + echo "Caution: This will delete the original wheel." + echo -e "\nUsage:\n./${SCRIPT_NAME} [--help|-h] klayout-...-manylinux1_x86_64.whl \n" + } + +if [[ ( $1 == "--help") || $1 == "-h" ]] +then + display_usage + exit 0 +fi + +# Check number of arguments +if [ $# -eq 0 ]; then + >&2 echo -e "ERROR: No filename supplied\n" + display_usage + exit 1 +elif [ ! $# -eq 1 ]; then + >&2 echo -e "ERROR: Too many files supplied. Provide one filename at at time.\n" + display_usage + exit 1 +fi + +# Read wheel file from argument +WHL="$1" + +# Check WHL is a valid file +if [[ ! -f "$WHL" ]]; then + >&2 echo -e "ERROR: $WHL is not a file" + exit 1 +fi + +# Convert to absolute path (linux only) +WHL=$(readlink -f $WHL) + +# Record old current directory +OLD_PWD=$PWD + +# The produced wheel from auditwheel, klayout-*-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. + +# Checking if it was previously patched +if unzip -l $WHL | grep -q 'patched_after_auditwheel_repair'; then + echo "$(basename $WHL) is already patched. Doing nothing." + exit 0 +fi + +# Repair script below +if [[ -d $TMP_WHEEL ]]; then + rm -rf $TMP_WHEEL +fi +echo "Unpacking $WHL into $TMP_WHEEL" +unzip -q $WHL -d $TMP_WHEEL + +cd $TMP_WHEEL/klayout +echo "Moving files: mv lib_* db_plugins .libs/" +mv lib_* db_plugins .libs/ 2>/dev/null +if [ $? -ne 0 ]; then + >&2 echo "ERROR: lib_*.so or db_plubins not found. Quitting." + exit 1 +fi +cd ../klayout-*.dist-info/ +echo "Patching klayout-*.dist-info/RECORD" +sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD +sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD +cd ../ +touch $TMP_WHEEL/patched_after_auditwheel_repair +echo "Packing $WHL from $TMP_WHEEL" +rm -f $WHL +zip -rq $WHL ./* +echo "Done. $(basename $WHL) is patched." +# Cleanup (should always execute) +cd $OLD_PWD diff --git a/ci-scripts/docker/development_notes/manylinux-docker.sh b/ci-scripts/docker/development_notes/manylinux-docker.sh new file mode 100644 index 000000000..7591d0181 --- /dev/null +++ b/ci-scripts/docker/development_notes/manylinux-docker.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# run with docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/ci-scripts/manylinux-docker.sh +# see https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml +# cache using https://github.com/travis-ci/travis-ci/issues/5358 +set -e -x + +# Install a system package required by our library +yum install -y zlib-devel +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +# export PATH=/usr/lib64/ccache:$PATH # unnecessary + +# Compile wheels +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ +done + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /persist/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /persist/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /persist/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +# Install packages and test +TEST_HOME=/persist/klayout/testdata +for PYBIN in /opt/python/*/bin/; do + "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse + "${PYBIN}/python" $TEST_HOME/pymod/import_db.py + "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py + "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py +done diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh new file mode 100755 index 000000000..2f6d68710 --- /dev/null +++ b/ci-scripts/docker/docker_build.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + + +if [[ -z $PY_VERSION ]]; then + echo '$PY_VERSION is not set' + exit 1 +fi + +if [[ -z $DOCKER_IMAGE ]]; then + echo '$DOCKER_IMAGE is not set' + exit 1 +fi + +echo PY_VERSION=$PY_VERSION +echo DOCKER_IMAGE=$DOCKER_IMAGE + +# sometimes the epel server is down. retry 5 times +for i in $(seq 1 5); do + yum install -y zlib-devel ccache zip git && s=0 && break || s=$? && sleep 15; +done + +[ $s -eq 0 ] || exit $s + +if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then + ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -s /usr/bin/ccache /usr/lib64/ccache/cc + ln -s /usr/bin/ccache /usr/lib64/ccache/gcc + ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + export PATH="/usr/lib64/ccache/:$PATH" +elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then + ln -s /usr/bin/ccache /usr/lib/ccache/c++ + ln -s /usr/bin/ccache /usr/lib/ccache/cc + ln -s /usr/bin/ccache /usr/lib/ccache/gcc + ln -s /usr/bin/ccache /usr/lib/ccache/g++ + export PATH="/usr/lib/ccache/:$PATH" +fi +echo $PATH +export CCACHE_DIR="/io/ccache" + +# Download proper auditwheel program +git clone https://github.com/thomaslima/auditwheel.git /tmp/auditwheel +cd /tmp/auditwheel +git checkout 87f5306ec02cc68020afaa9933543c898b1d47c1 # patched version +AUDITWHEEL_PYTHON=$(cat `which auditwheel` | head -1 | sed -e 's/#!\(.*\)/\1/') +# Install auditwheel, replacing the system's auditwheel binary +$AUDITWHEEL_PYTHON -m pip install . + + +# Show ccache stats +echo "Cache stats:" +ccache -s + +# Compile wheel +cd /io +"/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 + +# Show ccache stats +echo "Cache stats:" +ccache -s + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel -v repair "$whl" -w /io/wheelhouse/ || exit 1 +done + +# Install packages and test +TEST_HOME=/io/testdata +"/opt/python/$PY_VERSION/bin/pip" install klayout --no-index -f /io/wheelhouse || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_db.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_rdb.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_tl.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/pya_tests.py || exit 1 + diff --git a/ci-scripts/twine/README.md b/ci-scripts/twine/README.md new file mode 100644 index 000000000..3916de5d4 --- /dev/null +++ b/ci-scripts/twine/README.md @@ -0,0 +1,15 @@ +After building all the travis wheels, go to the folder where the wheels were deployed. In my case, for example, `/Users/tlima/Dropbox/Apps/travis-deploy/Builds/klayout/dist-pymod/0.26.0.dev10`. + +Then, run the command + +```bash +travis upload *.whl +``` + +, which will ask for an username and password related to your account. + +After this upload was successful, you can upload the source tarball. Go to klayout's git folder and run `python setup.py sdist`. Inside `dist/`, you'll find a tarball named, e.g., `klayout-0.26.0.dev10.tar.gz`. So just run + +```bash +travis upload klayout-0.26.0.dev10.tar.gz +``` diff --git a/setup.py b/setup.py index 545126651..059f4b44c 100644 --- a/setup.py +++ b/setup.py @@ -322,7 +322,7 @@ class Config(object): """ Gets the version string """ - return "0.26.0.dev8" + return "0.26.0.dev10" config = Config() diff --git a/src/pymod/__init__.py.qtless b/src/pymod/__init__.py.qtless index dce96f8e4..6aef40aa9 100644 --- a/src/pymod/__init__.py.qtless +++ b/src/pymod/__init__.py.qtless @@ -1,5 +1,4 @@ - # klayout library definition file -__all__ = [ "tl", "db", "lay", "rdb" ] +__all__ = [ "tl", "db", "rdb" ] diff --git a/src/pymod/distutils_src/pya/__init__.py b/src/pymod/distutils_src/pya/__init__.py index ae97d592a..70e2f49b9 100644 --- a/src/pymod/distutils_src/pya/__init__.py +++ b/src/pymod/distutils_src/pya/__init__.py @@ -1,10 +1,7 @@ +# import all packages from klayout, such as klayout.db and klayout.tl +# WARNING: doing it manually until it becomes impractical +# TODO: We need a specification document explaining what should go into pya -import klayout -import importlib - -__all__ = [] -for m in klayout.__all__: - mod = importlib.import_module("klayout." + m) - for mm in mod.__all__: - globals()[mm] = getattr(mod, mm) - +from klayout.db import * # noqa +from klayout.tl import * # noqa +from klayout.rdb import * # noqa