luke
/
klayout
mirror of https://github.com/KLayout/klayout.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #213 from lightwave-lab/pymod-manylinux 

Pymod manylinux wheels built on travis
This commit is contained in:
Matthias Köfferlein 2018-12-30 22:50:18 +01:00 committed by GitHub
parent e637d0cf5f 1bba72e45b
commit c10aefe4fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 861 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

470
.travis.yml
View File

@ -1,28 +1,235 @@
matrix:
include:
# python manylinux packages
- name: "cp37-cp37m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '3.7-dev'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp37-cp37m"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp37-cp37m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp37-cp37m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp36-cp36m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '3.6'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp36-cp36m"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp36-cp36m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp36-cp36m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp35-cp35m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '3.5'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp35-cp35m"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp35-cp35m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp35-cp35m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp34-cp34m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '3.4'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp34-cp34m"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp34-cp34m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp34-cp34m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp27-cp27mu-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '2.7'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp27-cp27mu"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp27-cp27mu-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp27-cp27mu"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp27-cp27m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '2.7'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp27-cp27m"
- DOCKER_BUILD=true
- TEST_IN_HOST=false # travis's python 2.7 uses ucs4 (mu), so this test fails.
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp27-cp27m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp27-cp27m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
# python 2 osx
- name: "klayout python2 osx10.13"
# - name: "klayout python2 osx10.13"
- name: "cp27-cp27m-macosx_10_13_x86_64.whl"
os: osx
osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- ccache
env:
- MATRIX_EVAL=""
- ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- name: "klayout python2 osx10.12"
# - name: "klayout python2 osx10.12"
- name: "cp27-cp27m-macosx_10_12_x86_64.whl"
os: osx
osx_image: xcode8.3 # macOS 10.12
cache: ccache
addons:
homebrew:
packages:
- ccache
env:
- MATRIX_EVAL="brew install python2 || brew link --overwrite python@2" # deficient python2 in travis's xcode8.3 (no ssl)
- ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- name: "klayout python2 osx10.11"
# - name: "klayout python2 osx10.11"
- name: "cp27-cp27m-macosx_10_11_x86_64.whl"
os: osx
osx_image: xcode8 # macOS 10.11
cache: ccache
addons:
homebrew:
packages:
- ccache
env:
- MATRIX_EVAL=""
- ARCHFLAGS="-std=c++11"
@ -31,13 +238,16 @@ matrix:
# python 3 osx
- name: "klayout python3 osx10.13"
# - name: "klayout python3 osx10.13"
- name: "cp37-cp37m-macosx_10_13_x86_64.whl"
os: osx
osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- python3
- ccache
update: true
env:
- MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';"
@ -45,40 +255,61 @@ matrix:
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- name: "klayout python3.6.6 osx10.13"
# - name: "klayout python3.6.6 osx10.13"
- name: "cp36-cp36m-macosx_10_13_x86_64.whl"
os: osx
osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- ccache
env:
- MATRIX_EVAL="brew update; brew install sashkab/python/python36; brew link --force --overwrite python36; shopt -s expand_aliases; alias python='/usr/local/opt/python36/bin/python3.6'; alias pip='/usr/local/opt/python36/bin/pip3.6';"
- ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- name: "klayout python3.5.6 osx10.13"
# - name: "klayout python3.5.6 osx10.13"
- name: "cp35-cp35m-macosx_10_13_x86_64.whl"
os: osx
osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- ccache
env:
- MATRIX_EVAL="brew update; brew install sashkab/python/python35; brew link --force --overwrite python35; shopt -s expand_aliases; alias python='/usr/local/opt/python35/bin/python3.5'; alias pip='/usr/local/opt/python35/bin/pip3.5';"
- ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- name: "klayout python3.4.9 osx10.13"
# - name: "klayout python3.4.9 osx10.13"
- name: "cp34-cp34m-macosx_10_13_x86_64.whl"
os: osx
osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- ccache
env:
- MATRIX_EVAL="brew update; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';"
- ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- name: "klayout python3 osx10.12"
# - name: "klayout python3 osx10.12"
- name: "cp37-cp37m-macosx_10_12_x86_64.whl"
os: osx
osx_image: xcode8.3 # macOS 10.12
cache: ccache
addons:
homebrew:
packages:
- python3
- ccache
update: true
env:
- MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';"
@ -86,116 +317,124 @@ matrix:
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- name: "klayout python3 osx10.11"
# - name: "klayout python3 osx10.11"
- name: "cp37-cp37m-macosx_10_11_x86_64.whl"
os: osx
osx_image: xcode8 # macOS 10.11
cache: ccache
addons:
homebrew:
packages:
- ccache
env:
- MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';"
- ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- name: "klayout python3.7 package"
os: linux
dist: trusty # Ubuntu 14.04
sudo: false
language: python
python: '3.7-dev'
env:
- MATRIX_EVAL=""
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- CC=clang
- CXX=clang++
# - name: "klayout python3.7 package"
# os: linux
# dist: trusty # Ubuntu 14.04
# sudo: false
# language: python
# python: '3.7-dev'
# env:
# - MATRIX_EVAL=""
# - PIP_UPDATE="1"
# - PYTHON_BUILD=true
# - CC=clang
# - CXX=clang++
- name: "klayout python3.6 package"
os: linux
dist: trusty # Ubuntu 14.04
sudo: false
language: python
python: '3.6'
env:
- MATRIX_EVAL=""
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- CC=clang
- CXX=clang++
# - name: "klayout python3.6 package"
# os: linux
# dist: trusty # Ubuntu 14.04
# sudo: false
# language: python
# python: '3.6'
# env:
# - MATRIX_EVAL=""
# - PIP_UPDATE="1"
# - PYTHON_BUILD=true
# - CC=clang
# - CXX=clang++
- name: "klayout python2.7 package"
os: linux
dist: trusty # Ubuntu 14.04
sudo: false
language: python
python: '2.7'
env:
- MATRIX_EVAL=""
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- CC=clang
- CXX=clang++
# - name: "klayout python2.7 package"
# os: linux
# dist: trusty # Ubuntu 14.04
# sudo: false
# language: python
# python: '2.7'
# env:
# - MATRIX_EVAL=""
# - PIP_UPDATE="1"
# - PYTHON_BUILD=true
# - CC=clang
# - CXX=clang++
- name: "klayout python2.6 package"
os: linux
dist: trusty # Ubuntu 14.04
sudo: false
language: python
python: '2.6'
env:
- MATRIX_EVAL=""
- PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6
- PYTHON_BUILD=true
- CC=clang
- CXX=clang++
# - name: "klayout python2.6 package"
# os: linux
# dist: trusty # Ubuntu 14.04
# sudo: false
# language: python
# python: '2.6'
# env:
# - MATRIX_EVAL=""
# - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6
# - PYTHON_BUILD=true
# - CC=clang
# - CXX=clang++
- name: "klayout python3.3 package"
os: linux
dist: trusty # Ubuntu 14.04
sudo: false
language: python
python: '3.3'
env:
- MATRIX_EVAL=""
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- CC=clang
- CXX=clang++
# - name: "klayout python3.3 package"
# os: linux
# dist: trusty # Ubuntu 14.04
# sudo: false
# language: python
# python: '3.3'
# env:
# - MATRIX_EVAL=""
# - PIP_UPDATE="1"
# - PYTHON_BUILD=true
# - CC=clang
# - CXX=clang++
- name: "klayout python3.4 package"
os: linux
dist: trusty # Ubuntu 14.04
sudo: false
language: python
python: '3.4'
env:
- MATRIX_EVAL=""
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- CC=clang
- CXX=clang++
# - name: "klayout python3.4 package"
# os: linux
# dist: trusty # Ubuntu 14.04
# sudo: false
# language: python
# python: '3.4'
# env:
# - MATRIX_EVAL=""
# - PIP_UPDATE="1"
# - PYTHON_BUILD=true
# - CC=clang
# - CXX=clang++
- name: "klayout python3.5 package"
os: linux
dist: trusty # Ubuntu 14.04
sudo: false
language: python
python: '3.5'
env:
- MATRIX_EVAL=""
- PIP_UPDATE="1"
- PYTHON_BUILD=true
- CC=clang
- CXX=clang++
# - name: "klayout python3.5 package"
# os: linux
# dist: trusty # Ubuntu 14.04
# sudo: false
# language: python
# python: '3.5'
# env:
# - MATRIX_EVAL=""
# - PIP_UPDATE="1"
# - PYTHON_BUILD=true
# - CC=clang
# - CXX=clang++
# KLayout builds for mac
# Python 3
- name: "KLayout macOS 10.13 with py3.7"
os: osx
osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- python3
- qt
- ccache
update: true
env:
- MATRIX_EVAL=""
@ -206,11 +445,13 @@ matrix:
- name: "KLayout macOS 10.12 with py3.7"
os: osx
osx_image: xcode8.3 # macOS 10.12
cache: ccache
addons:
homebrew:
packages:
- python3
- qt
- ccache
update: true
env:
- MATRIX_EVAL=""
@ -221,11 +462,13 @@ matrix:
- name: "KLayout macOS 10.11 with py3.7"
os: osx
osx_image: xcode8 # macOS 10.11
cache: ccache
addons:
homebrew:
packages:
- python3
- qt
- ccache
update: true
env:
- MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8
@ -237,10 +480,12 @@ matrix:
- name: "KLayout macOS 10.13 with py2.7"
os: osx
osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- qt
- ccache
update: true
env:
- MATRIX_EVAL=""
@ -251,10 +496,12 @@ matrix:
- name: "KLayout macOS 10.12 with py2.7"
os: osx
osx_image: xcode8.3 # macOS 10.12
cache: ccache
addons:
homebrew:
packages:
- qt
- ccache
update: true
env:
- MATRIX_EVAL=""
@ -265,10 +512,12 @@ matrix:
- name: "KLayout macOS 10.11 with py2.7"
os: osx
osx_image: xcode8 # macOS 10.11
cache: ccache
addons:
homebrew:
packages:
- qt
- ccache
update: true
env:
- MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8
@ -287,15 +536,42 @@ before_install:
pip install --upgrade setuptools wheel || sudo pip install --upgrade setuptools wheel;
fi
- python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)"
- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
export PATH="/usr/local/opt/ccache/libexec:$PATH";
fi
script:
install:
- if [ "$DOCKER_BUILD" = true ]; then
docker pull $DOCKER_IMAGE;
fi
script:
- if [ "$DOCKER_BUILD" = true ]; then
mkdir -p ccache;
mkdir -p wheelhouse;
docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh";
klayout_version=$(python -c 'import setup; print(setup.Config().version())');
mkdir -p deploy/dist-pymod/$klayout_version;
cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod/$klayout_version;
if [ "$TEST_IN_HOST" = true ]; then
pip install klayout --no-index -f ./wheelhouse;
python testdata/pymod/import_db.py;
python testdata/pymod/import_rdb.py;
python testdata/pymod/import_tl.py;
python testdata/pymod/pya_tests.py;
fi
fi
- if [ "$PYTHON_BUILD" = true ]; then
python setup.py build;
python setup.py bdist_wheel;
python setup.py install;
python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py;
mkdir -p deploy/dist-pymod;
cp -a dist/* deploy/dist-pymod/;
python testdata/pymod/import_db.py;
python testdata/pymod/import_rdb.py;
python testdata/pymod/import_tl.py;
python testdata/pymod/pya_tests.py;
klayout_version=$(python -c 'import setup; print(setup.Config().version())');
mkdir -p deploy/dist-pymod/$klayout_version;
cp -a dist/*.whl deploy/dist-pymod/$klayout_version;
python -c 'import klayout.db as db; print(dir(db))';
python -c 'import klayout.rdb as rdb; print(dir(rdb))';
python -c 'import klayout.tl as tl; print(dir(tl))';

22
ci-scripts/docker/README.md Normal file
View File

@ -0,0 +1,22 @@
Author: Thomas Ferreira de Lima
email: thomas@tlima.me
This folder contains scripts to be run inside docker images. See instructions on how to test this yourself in ci-scripts/docker/development_notes.
## docker_build.sh
We need two environment variables to get going:
```bash
DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
PY_VERSION="cp37-cp37m"
```
The script must be run inside an image pulled from $DOCKER_IMAGE and with klayout's git repo cloned in /io. Inside the git clone folder, run:
```bash
docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh";
# $PRE_CMD is empty for now (useless currently).
```
This command will generate a wheel and place it in `wheelhouse/klayout-*manylinux1*.whl`. This is the wheel that needs to be uploaded to PyPI via twine. See ci-scripts/twine/README.md.

17
ci-scripts/docker/development_notes/Dockerfile.i686 Normal file
View File

@ -0,0 +1,17 @@
FROM quay.io/pypa/manylinux1_i686
MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me)
# Install a system package required by our library
RUN linux32 yum install -y zlib-devel
RUN linux32 yum install -y ccache
RUN ln -s /usr/bin/ccache /usr/lib/ccache/c++
RUN ln -s /usr/bin/ccache /usr/lib/ccache/cc
RUN ln -s /usr/bin/ccache /usr/lib/ccache/gcc
RUN ln -s /usr/bin/ccache /usr/lib/ccache/g++
# Add ccache to PATH
RUN mkdir -p /persist/.ccache
ENV CCACHE_DIR="/persist/.ccache"
# Need zip to fix wheel
RUN linux32 yum install -y zip

17
ci-scripts/docker/development_notes/Dockerfile.x86_64 Normal file
View File

@ -0,0 +1,17 @@
FROM quay.io/pypa/manylinux1_x86_64
MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me)
# Install a system package required by our library
RUN yum install -y zlib-devel
RUN yum install -y ccache
RUN ln -s /usr/bin/ccache /usr/lib64/ccache/c++
RUN ln -s /usr/bin/ccache /usr/lib64/ccache/cc
RUN ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
RUN ln -s /usr/bin/ccache /usr/lib64/ccache/g++
# Add ccache to PATH
RUN mkdir -p /persist/.ccache
ENV CCACHE_DIR="/persist/.ccache"
# Need zip to fix wheel
RUN yum install -y zip

221
ci-scripts/docker/development_notes/README.md Normal file
View File

@ -0,0 +1,221 @@
Author: Thomas Ferreira de Lima
email: thomas@tlima.me
I wrote these notes as I was learning how to use docker and how to build python packages inside a docker image prepared by the pypa team. They require us to build there to allow wheels to have the `manylinux1` tag, meaning that these wheels would be compatible with most linux distributions around. Chapter 1 is about testing in my own computer (MacOS Mojave) and Chapter 2 is about how to automate this build using travis-ci.org.
# Chapter 1. Testing on your own computer
## Step 1.
Make sure you have the quay.io/pypa/manylinux1_x86_64 image.
```bash
$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB
hello-world latest 4ab4c602aa5e 3 months ago 1.84kB
# My image was old:
$ docker pull quay.io/pypa/manylinux1_x86_64
Using default tag: latest
latest: Pulling from pypa/manylinux1_x86_64
7d0d9526f38a: Already exists
3324bfadf9cb: Pull complete
20f27c7e3062: Pull complete
5bc21fc5fe97: Pull complete
Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500
Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest
```
## Step 2.
This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72
Create a volume for klayout. This is necessary because docker containers don't persist data.
```bash
$ docker volume create klayout-persist
$ docker volume inspect klayout-persist
[
{
"CreatedAt": "2018-12-18T15:01:48Z",
"Driver": "local",
"Labels": {},
"Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data",
"Name": "klayout-persist",
"Options": {},
"Scope": "local"
}
]
```
## Step 3.
Build image `myimage` with:
```bash
$ docker build -t myimage:latest -f Dockerfile.x86_64 .
```
This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`.
Then I run the docker with a terminal shell and load the volume klayout-persist in /persist:
```bash
$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage
```
## Step 4.
In the shell, pull master from klayout.
```bash
cd /persist
git clone https://github.com/lightwave-lab/klayout.git
mkdir -p wheelhouse
cd klayout
# make wheel with python 3.6 (for example)
/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/
cd /persist
auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/
# Need to manually fix the wheel
#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse
```
The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place.
Procedure to fix the wheel:
```bash
unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel
cd tempwheel/klayout
mv lib_* db_plugins .libs/
cd ../klayout-0.26.0.dev8.dist-info/
sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD
sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD
cd ../
rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl
zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./*
cd ..
rm -rf tempwheel
```
Now we can install and test:
```bash
/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse
cd /persist/klayout
/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py
# Tests passed!
```
Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel.
## Step 5. Iterate over all python versions.
For that, we need something like:
```bash
# Compile wheels
for PYBIN in /opt/python/*/bin; do
"${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/
done
# Bundle external shared libraries into the wheels via auditwheel
for whl in /persist/wheelhouse/*linux_*.whl; do
auditwheel repair "$whl" -w /persist/wheelhouse/
done
# Fix each wheel generated by auditwheel
for whl in /persist/wheelhouse/*manylinux1_*.whl; do
./ci-scripts/docker/fix_wheel.sh "$whl"
done
# Install packages and test
TEST_HOME=/persist/klayout/testdata
for PYBIN in /opt/python/*/bin/; do
"${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse
"${PYBIN}/python" $TEST_HOME/pymod/import_db.py
"${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py
"${PYBIN}/python" $TEST_HOME/pymod/import_tl.py
```
I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error:
`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String`
I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis.
The ccache folder after a single build has 657MB. Go figure.
I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1
So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default.
# Chapter 2. Testing CI flow with docker
# Step 1. Testing commands in own computer
First cloned with:
```bash
git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux
cd klayout
```
Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml
DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686
PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m
Total of 2x6 = 12 possibilities
```bash
export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64
export PY_VERSION="cp36-cp36m"
docker pull $DOCKER_IMAGE
mkdir -p ccache
mkdir -p wheelhouse
docker run --name klayout -v `pwd` -it $DOCKER_IMAGE
```
Inside docker shell:
```bash
yum install -y zlib-devel
yum install -y zip
yum install -y ccache
ln -s /usr/bin/ccache /usr/lib64/ccache/c++
ln -s /usr/bin/ccache /usr/lib64/ccache/cc
ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
ln -s /usr/bin/ccache /usr/lib64/ccache/g++
echo $PATH
# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export CCACHE_DIR="/io/ccache"
# Compile wheel
/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/
# Bundle external shared libraries into the wheels via auditwheel
for whl in /io/wheelhouse/*linux_*.whl; do
auditwheel repair "$whl" -w /io/wheelhouse/
done
# Fix each wheel generated by auditwheel
for whl in /io/wheelhouse/*manylinux1_*.whl; do
./ci-scripts/docker/fix_wheel.sh "$whl"
done
```
# Step 2. Automating step 1 in travis (CI).
DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686
PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m
Build: spawn 12 travis jobs, one for each combination of word-size and python version.
Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`.
Post-build:
- cache `./ccache`
- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder)
# Step 3. Automating deployment to PyPI:
TBD

77
ci-scripts/docker/development_notes/fix_wheel.sh Executable file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env bash
SCRIPT_NAME=`basename "$0"`
TMP_WHEEL="/tmp/klayout_tempwheel"
display_usage() {
echo "This script fixes auditwheel-repaired wheels."
echo "Caution: This will delete the original wheel."
echo -e "\nUsage:\n./${SCRIPT_NAME} [--help|-h] klayout-...-manylinux1_x86_64.whl \n"
}
if [[ ( $1 == "--help") || $1 == "-h" ]]
then
display_usage
exit 0
fi
# Check number of arguments
if [ $# -eq 0 ]; then
>&2 echo -e "ERROR: No filename supplied\n"
display_usage
exit 1
elif [ ! $# -eq 1 ]; then
>&2 echo -e "ERROR: Too many files supplied. Provide one filename at at time.\n"
display_usage
exit 1
fi
# Read wheel file from argument
WHL="$1"
# Check WHL is a valid file
if [[ ! -f "$WHL" ]]; then
>&2 echo -e "ERROR: $WHL is not a file"
exit 1
fi
# Convert to absolute path (linux only)
WHL=$(readlink -f $WHL)
# Record old current directory
OLD_PWD=$PWD
# The produced wheel from auditwheel, klayout-*-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place.
# Checking if it was previously patched
if unzip -l $WHL | grep -q 'patched_after_auditwheel_repair'; then
echo "$(basename $WHL) is already patched. Doing nothing."
exit 0
fi
# Repair script below
if [[ -d $TMP_WHEEL ]]; then
rm -rf $TMP_WHEEL
fi
echo "Unpacking $WHL into $TMP_WHEEL"
unzip -q $WHL -d $TMP_WHEEL
cd $TMP_WHEEL/klayout
echo "Moving files: mv lib_* db_plugins .libs/"
mv lib_* db_plugins .libs/ 2>/dev/null
if [ $? -ne 0 ]; then
>&2 echo "ERROR: lib_*.so or db_plubins not found. Quitting."
exit 1
fi
cd ../klayout-*.dist-info/
echo "Patching klayout-*.dist-info/RECORD"
sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD
sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD
cd ../
touch $TMP_WHEEL/patched_after_auditwheel_repair
echo "Packing $WHL from $TMP_WHEEL"
rm -f $WHL
zip -rq $WHL ./*
echo "Done. $(basename $WHL) is patched."
# Cleanup (should always execute)
cd $OLD_PWD

38
ci-scripts/docker/development_notes/manylinux-docker.sh Normal file
View File

@ -0,0 +1,38 @@
#!/bin/bash
# run with docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/ci-scripts/manylinux-docker.sh
# see https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml
# cache using https://github.com/travis-ci/travis-ci/issues/5358
set -e -x
# Install a system package required by our library
yum install -y zlib-devel
yum install -y ccache
ln -s /usr/bin/ccache /usr/lib64/ccache/c++
ln -s /usr/bin/ccache /usr/lib64/ccache/cc
ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
ln -s /usr/bin/ccache /usr/lib64/ccache/g++
# export PATH=/usr/lib64/ccache:$PATH # unnecessary
# Compile wheels
for PYBIN in /opt/python/*/bin; do
"${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/
done
# Bundle external shared libraries into the wheels via auditwheel
for whl in /persist/wheelhouse/*linux_*.whl; do
auditwheel repair "$whl" -w /persist/wheelhouse/
done
# Fix each wheel generated by auditwheel
for whl in /persist/wheelhouse/*manylinux1_*.whl; do
./ci-scripts/docker/fix_wheel.sh "$whl"
done
# Install packages and test
TEST_HOME=/persist/klayout/testdata
for PYBIN in /opt/python/*/bin/; do
"${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse
"${PYBIN}/python" $TEST_HOME/pymod/import_db.py
"${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py
"${PYBIN}/python" $TEST_HOME/pymod/import_tl.py
done

73
ci-scripts/docker/docker_build.sh Executable file
View File

@ -0,0 +1,73 @@
#!/usr/bin/env bash
if [[ -z $PY_VERSION ]]; then
echo '$PY_VERSION is not set'
exit 1
fi
if [[ -z $DOCKER_IMAGE ]]; then
echo '$DOCKER_IMAGE is not set'
exit 1
fi
echo PY_VERSION=$PY_VERSION
echo DOCKER_IMAGE=$DOCKER_IMAGE
# sometimes the epel server is down. retry 5 times
for i in $(seq 1 5); do
yum install -y zlib-devel ccache zip git && s=0 && break || s=$? && sleep 15;
done
[ $s -eq 0 ] || exit $s
if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then
ln -s /usr/bin/ccache /usr/lib64/ccache/c++
ln -s /usr/bin/ccache /usr/lib64/ccache/cc
ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
ln -s /usr/bin/ccache /usr/lib64/ccache/g++
export PATH="/usr/lib64/ccache/:$PATH"
elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then
ln -s /usr/bin/ccache /usr/lib/ccache/c++
ln -s /usr/bin/ccache /usr/lib/ccache/cc
ln -s /usr/bin/ccache /usr/lib/ccache/gcc
ln -s /usr/bin/ccache /usr/lib/ccache/g++
export PATH="/usr/lib/ccache/:$PATH"
fi
echo $PATH
export CCACHE_DIR="/io/ccache"
# Download proper auditwheel program
git clone https://github.com/thomaslima/auditwheel.git /tmp/auditwheel
cd /tmp/auditwheel
git checkout 87f5306ec02cc68020afaa9933543c898b1d47c1 # patched version
AUDITWHEEL_PYTHON=$(cat `which auditwheel` | head -1 | sed -e 's/#!\(.*\)/\1/')
# Install auditwheel, replacing the system's auditwheel binary
$AUDITWHEEL_PYTHON -m pip install .
# Show ccache stats
echo "Cache stats:"
ccache -s
# Compile wheel
cd /io
"/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1
# Show ccache stats
echo "Cache stats:"
ccache -s
# Bundle external shared libraries into the wheels via auditwheel
for whl in /io/wheelhouse/*linux_*.whl; do
auditwheel -v repair "$whl" -w /io/wheelhouse/ || exit 1
done
# Install packages and test
TEST_HOME=/io/testdata
"/opt/python/$PY_VERSION/bin/pip" install klayout --no-index -f /io/wheelhouse || exit 1
"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_db.py || exit 1
"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_rdb.py || exit 1
"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_tl.py || exit 1
"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/pya_tests.py || exit 1

15
ci-scripts/twine/README.md Normal file
View File

@ -0,0 +1,15 @@
After building all the travis wheels, go to the folder where the wheels were deployed. In my case, for example, `/Users/tlima/Dropbox/Apps/travis-deploy/Builds/klayout/dist-pymod/0.26.0.dev10`.
Then, run the command
```bash
travis upload *.whl
```
, which will ask for an username and password related to your account.
After this upload was successful, you can upload the source tarball. Go to klayout's git folder and run `python setup.py sdist`. Inside `dist/`, you'll find a tarball named, e.g., `klayout-0.26.0.dev10.tar.gz`. So just run
```bash
travis upload klayout-0.26.0.dev10.tar.gz
```

2
setup.py
View File

@ -322,7 +322,7 @@ class Config(object):
"""
Gets the version string
"""
return "0.26.0.dev8"
return "0.26.0.dev10"
config = Config()

3
src/pymod/__init__.py.qtless
View File

@ -1,5 +1,4 @@
# klayout library definition file
__all__ = [ "tl", "db", "lay", "rdb" ]
__all__ = [ "tl", "db", "rdb" ]

15
src/pymod/distutils_src/pya/__init__.py
View File

@ -1,10 +1,7 @@
# import all packages from klayout, such as klayout.db and klayout.tl
# WARNING: doing it manually until it becomes impractical
# TODO: We need a specification document explaining what should go into pya
import klayout
import importlib
__all__ = []
for m in klayout.__all__:
mod = importlib.import_module("klayout." + m)
for mm in mod.__all__:
globals()[mm] = getattr(mod, mm)
from klayout.db import * # noqa
from klayout.tl import * # noqa
from klayout.rdb import * # noqa