diff --git a/.travis.yml b/.travis.yml index ee2c85e99..16cd21347 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,36 @@ matrix: include: + # python manylinux packages + + - name: "manylinux1_x86_64 cp36-cp36m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp36-cp36m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp36-cp36m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp36-cp36m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + # python 2 osx - name: "klayout python2 osx10.13" @@ -288,7 +319,19 @@ before_install: fi - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" -script: +install: + - if [ "$DOCKER_BUILD" = true ]; then + docker pull $DOCKER_IMAGE; + fi + +script: + - if [ "$DOCKER_BUILD" = true ]; then + mkdir -p ccache; + mkdir -p wheelhouse; + docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; + mkdir -p deploy/dist-pymod; + cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod; + fi - if [ "$PYTHON_BUILD" = true ]; then python setup.py build; python setup.py bdist_wheel; diff --git a/ci-scripts/docker/Dockerfile.i686 b/ci-scripts/docker/Dockerfile.i686 index f73ca3ac1..7d68054fc 100644 --- a/ci-scripts/docker/Dockerfile.i686 +++ b/ci-scripts/docker/Dockerfile.i686 @@ -10,7 +10,6 @@ RUN ln -s /usr/bin/ccache /usr/lib/ccache/gcc RUN ln -s /usr/bin/ccache /usr/lib/ccache/g++ # Add ccache to PATH -ENV PATH="/usr/lib/ccache:${PATH}" RUN mkdir -p /persist/.ccache ENV CCACHE_DIR="/persist/.ccache" diff --git a/ci-scripts/docker/Dockerfile.x86_64 b/ci-scripts/docker/Dockerfile.x86_64 index df4f05dc5..4bf03c3cd 100644 --- a/ci-scripts/docker/Dockerfile.x86_64 +++ b/ci-scripts/docker/Dockerfile.x86_64 @@ -10,7 +10,6 @@ RUN ln -s /usr/bin/ccache /usr/lib64/ccache/gcc RUN ln -s /usr/bin/ccache /usr/lib64/ccache/g++ # Add ccache to PATH -ENV PATH="/usr/lib64/ccache:${PATH}" RUN mkdir -p /persist/.ccache ENV CCACHE_DIR="/persist/.ccache" diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 5ec87ee41..48b5a4dbd 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -1,4 +1,5 @@ -# Step 1. +# Chapter 1. Testing on your own computer +## Step 1. Make sure you have the quay.io/pypa/manylinux1_x86_64 image. @@ -19,7 +20,7 @@ Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest ``` -# Step 2. +## Step 2. This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 Create a volume for klayout. This is necessary because docker containers don't persist data. @@ -40,7 +41,7 @@ $ docker volume inspect klayout-persist ] ``` -# Step 3. +## Step 3. Build image `myimage` with: @@ -56,7 +57,7 @@ Then I run the docker with a terminal shell and load the volume klayout-persist $ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage ``` -# Step 4. +## Step 4. In the shell, pull master from klayout. @@ -102,7 +103,7 @@ cd /persist/klayout Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. -# Step 5. Iterate over all python versions. +## Step 5. Iterate over all python versions. For that, we need something like: @@ -135,3 +136,77 @@ for PYBIN in /opt/python/*/bin/; do I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: `ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` + +I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. +The ccache folder after a single build has 657MB. Go figure. + +# Chapter 2. Testing CI flow with docker + +# Step 1. Testing commands in own computer + +First cloned with: +```bash +git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux +cd klayout +``` + +Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml + +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Total of 2x6 = 12 possibilities + +```bash +export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 +export PY_VERSION="cp36-cp36m" +docker pull $DOCKER_IMAGE +mkdir -p ccache +mkdir -p wheelhouse +docker run --name klayout -v `pwd` -it $DOCKER_IMAGE +``` + +Inside docker shell: +```bash +yum install -y zlib-devel +yum install -y zip + +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" + +# Compile wheel +/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/ + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +``` + +# Step 2. Automating step 1 in travis (CI). +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Build: spawn 12 travis jobs, one for each combination of word-size and python version. +Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`. +Post-build: +- cache `./ccache` +- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder) + +# Step 3. Automating deployment to PyPI: + +TBD diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh new file mode 100755 index 000000000..58dccef8b --- /dev/null +++ b/ci-scripts/docker/docker_build.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + + +if [[ -z $PY_VERSION ]]; then + echo '$PY_VERSION is not set' + exit 1 +fi + +if [[ -z $DOCKER_IMAGE ]]; then + echo '$DOCKER_IMAGE is not set' + exit 1 +fi + +echo PY_VERSION=$PY_VERSION +echo DOCKER_IMAGE=$DOCKER_IMAGE + +yum install -y zlib-devel ccache zip || exit 1 # sometimes the epel server is down. + +if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then + ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -s /usr/bin/ccache /usr/lib64/ccache/cc + ln -s /usr/bin/ccache /usr/lib64/ccache/gcc + ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then + ln -s /usr/bin/ccache /usr/lib/ccache/c++ + ln -s /usr/bin/ccache /usr/lib/ccache/cc + ln -s /usr/bin/ccache /usr/lib/ccache/gcc + ln -s /usr/bin/ccache /usr/lib/ccache/g++ +fi +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" +cd /io + +# Compile wheel +"/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ || exit 1 +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + /io/ci-scripts/docker/fix_wheel.sh "$whl" || exit 1 +done + +# Install packages and test +TEST_HOME=/io/testdata +"/opt/python/$PY_VERSION/bin/pip" install klayout --no-index -f /io/wheelhouse || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_db.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_rdb.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_tl.py || exit 1 diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh index a4b958853..7591d0181 100644 --- a/ci-scripts/docker/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -11,7 +11,7 @@ ln -s /usr/bin/ccache /usr/lib64/ccache/c++ ln -s /usr/bin/ccache /usr/lib64/ccache/cc ln -s /usr/bin/ccache /usr/lib64/ccache/gcc ln -s /usr/bin/ccache /usr/lib64/ccache/g++ -export PATH=/usr/lib64/ccache:$PATH +# export PATH=/usr/lib64/ccache:$PATH # unnecessary # Compile wheels for PYBIN in /opt/python/*/bin; do