luke
/
klayout
mirror of https://github.com/KLayout/klayout.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into travis-ci/fix/el-capitan

This commit is contained in:
Matthias Köfferlein 2018-12-30 22:54:52 +01:00 committed by GitHub
parent 971ad89f0b c10aefe4fb
commit 32e6879d86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 872 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

473
.travis.yml
View File

@ -1,28 +1,235 @@
matrix: matrix:
include: include:
# python manylinux packages
- name: "cp37-cp37m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '3.7-dev'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp37-cp37m"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp37-cp37m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp37-cp37m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp36-cp36m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '3.6'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp36-cp36m"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp36-cp36m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp36-cp36m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp35-cp35m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '3.5'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp35-cp35m"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp35-cp35m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp35-cp35m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp34-cp34m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '3.4'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp34-cp34m"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp34-cp34m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp34-cp34m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp27-cp27mu-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '2.7'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp27-cp27mu"
- DOCKER_BUILD=true
- TEST_IN_HOST=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp27-cp27mu-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp27-cp27mu"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp27-cp27m-manylinux1_x86_64.whl"
os: linux
sudo: true
language: python
python: '2.7'
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
- PY_VERSION="cp27-cp27m"
- DOCKER_BUILD=true
- TEST_IN_HOST=false # travis's python 2.7 uses ucs4 (mu), so this test fails.
- MATRIX_EVAL=""
cache:
directories:
- ccache
- name: "cp27-cp27m-manylinux1_i686.whl"
os: linux
sudo: true
services:
- docker
env:
- DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
- PY_VERSION="cp27-cp27m"
- DOCKER_BUILD=true
- MATRIX_EVAL=""
cache:
directories:
- ccache
# python 2 osx # python 2 osx
- name: "klayout python2 osx10.13" # - name: "klayout python2 osx10.13"
- name: "cp27-cp27m-macosx_10_13_x86_64.whl"
os: osx os: osx
osx_image: xcode9.4 # macOS 10.13 osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- ccache
env: env:
- MATRIX_EVAL="" - MATRIX_EVAL=""
- ARCHFLAGS="-std=c++11" - ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1" - PIP_UPDATE="1"
- PYTHON_BUILD=true - PYTHON_BUILD=true
- name: "klayout python2 osx10.12" # - name: "klayout python2 osx10.12"
- name: "cp27-cp27m-macosx_10_12_x86_64.whl"
os: osx os: osx
osx_image: xcode8.3 # macOS 10.12 osx_image: xcode8.3 # macOS 10.12
cache: ccache
addons:
homebrew:
packages:
- ccache
env: env:
- MATRIX_EVAL="brew install python2 || brew link --overwrite python@2" # deficient python2 in travis's xcode8.3 (no ssl) - MATRIX_EVAL="brew install python2 || brew link --overwrite python@2" # deficient python2 in travis's xcode8.3 (no ssl)
- ARCHFLAGS="-std=c++11" - ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1" - PIP_UPDATE="1"
- PYTHON_BUILD=true - PYTHON_BUILD=true
- name: "klayout python2 osx10.11" # - name: "klayout python2 osx10.11"
- name: "cp27-cp27m-macosx_10_11_x86_64.whl"
os: osx os: osx
osx_image: xcode8 # macOS 10.11 osx_image: xcode8 # macOS 10.11
cache: ccache
addons:
homebrew:
packages:
- ccache
env: env:
- MATRIX_EVAL="" - MATRIX_EVAL=""
- ARCHFLAGS="-std=c++11" - ARCHFLAGS="-std=c++11"
@ -31,13 +238,16 @@ matrix:
# python 3 osx # python 3 osx
- name: "klayout python3 osx10.13" # - name: "klayout python3 osx10.13"
- name: "cp37-cp37m-macosx_10_13_x86_64.whl"
os: osx os: osx
osx_image: xcode9.4 # macOS 10.13 osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons: addons:
homebrew: homebrew:
packages: packages:
- python3 - python3
- ccache
update: true update: true
env: env:
- MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';"
@ -45,40 +255,61 @@ matrix:
- PIP_UPDATE="1" - PIP_UPDATE="1"
- PYTHON_BUILD=true - PYTHON_BUILD=true
- name: "klayout python3.6.6 osx10.13" # - name: "klayout python3.6.6 osx10.13"
- name: "cp36-cp36m-macosx_10_13_x86_64.whl"
os: osx os: osx
osx_image: xcode9.4 # macOS 10.13 osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- ccache
env: env:
- MATRIX_EVAL="brew update; brew install sashkab/python/python36; brew link --force --overwrite python36; shopt -s expand_aliases; alias python='/usr/local/opt/python36/bin/python3.6'; alias pip='/usr/local/opt/python36/bin/pip3.6';" - MATRIX_EVAL="brew update; brew install sashkab/python/python36; brew link --force --overwrite python36; shopt -s expand_aliases; alias python='/usr/local/opt/python36/bin/python3.6'; alias pip='/usr/local/opt/python36/bin/pip3.6';"
- ARCHFLAGS="-std=c++11" - ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1" - PIP_UPDATE="1"
- PYTHON_BUILD=true - PYTHON_BUILD=true
- name: "klayout python3.5.6 osx10.13" # - name: "klayout python3.5.6 osx10.13"
- name: "cp35-cp35m-macosx_10_13_x86_64.whl"
os: osx os: osx
osx_image: xcode9.4 # macOS 10.13 osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- ccache
env: env:
- MATRIX_EVAL="brew update; brew install sashkab/python/python35; brew link --force --overwrite python35; shopt -s expand_aliases; alias python='/usr/local/opt/python35/bin/python3.5'; alias pip='/usr/local/opt/python35/bin/pip3.5';" - MATRIX_EVAL="brew update; brew install sashkab/python/python35; brew link --force --overwrite python35; shopt -s expand_aliases; alias python='/usr/local/opt/python35/bin/python3.5'; alias pip='/usr/local/opt/python35/bin/pip3.5';"
- ARCHFLAGS="-std=c++11" - ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1" - PIP_UPDATE="1"
- PYTHON_BUILD=true - PYTHON_BUILD=true
- name: "klayout python3.4.9 osx10.13" # - name: "klayout python3.4.9 osx10.13"
- name: "cp34-cp34m-macosx_10_13_x86_64.whl"
os: osx os: osx
osx_image: xcode9.4 # macOS 10.13 osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons:
homebrew:
packages:
- ccache
env: env:
- MATRIX_EVAL="brew update; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';" - MATRIX_EVAL="brew update; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';"
- ARCHFLAGS="-std=c++11" - ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1" - PIP_UPDATE="1"
- PYTHON_BUILD=true - PYTHON_BUILD=true
- name: "klayout python3 osx10.12" # - name: "klayout python3 osx10.12"
- name: "cp37-cp37m-macosx_10_12_x86_64.whl"
os: osx os: osx
osx_image: xcode8.3 # macOS 10.12 osx_image: xcode8.3 # macOS 10.12
cache: ccache
addons: addons:
homebrew: homebrew:
packages: packages:
- python3 - python3
- ccache
update: true update: true
env: env:
- MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';"
@ -86,116 +317,124 @@ matrix:
- PIP_UPDATE="1" - PIP_UPDATE="1"
- PYTHON_BUILD=true - PYTHON_BUILD=true
- name: "klayout python3 osx10.11" # - name: "klayout python3 osx10.11"
- name: "cp37-cp37m-macosx_10_11_x86_64.whl"
os: osx os: osx
osx_image: xcode8 # macOS 10.11 osx_image: xcode8 # macOS 10.11
cache: ccache
addons:
homebrew:
packages:
- ccache
env: env:
- MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';" - MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';"
- ARCHFLAGS="-std=c++11" - ARCHFLAGS="-std=c++11"
- PIP_UPDATE="1" - PIP_UPDATE="1"
- PYTHON_BUILD=true - PYTHON_BUILD=true
- name: "klayout python3.7 package" # - name: "klayout python3.7 package"
os: linux # os: linux
dist: trusty # Ubuntu 14.04 # dist: trusty # Ubuntu 14.04
sudo: false # sudo: false
language: python # language: python
python: '3.7-dev' # python: '3.7-dev'
env: # env:
- MATRIX_EVAL="" # - MATRIX_EVAL=""
- PIP_UPDATE="1" # - PIP_UPDATE="1"
- PYTHON_BUILD=true # - PYTHON_BUILD=true
- CC=clang # - CC=clang
- CXX=clang++ # - CXX=clang++
- name: "klayout python3.6 package" # - name: "klayout python3.6 package"
os: linux # os: linux
dist: trusty # Ubuntu 14.04 # dist: trusty # Ubuntu 14.04
sudo: false # sudo: false
language: python # language: python
python: '3.6' # python: '3.6'
env: # env:
- MATRIX_EVAL="" # - MATRIX_EVAL=""
- PIP_UPDATE="1" # - PIP_UPDATE="1"
- PYTHON_BUILD=true # - PYTHON_BUILD=true
- CC=clang # - CC=clang
- CXX=clang++ # - CXX=clang++
- name: "klayout python2.7 package" # - name: "klayout python2.7 package"
os: linux # os: linux
dist: trusty # Ubuntu 14.04 # dist: trusty # Ubuntu 14.04
sudo: false # sudo: false
language: python # language: python
python: '2.7' # python: '2.7'
env: # env:
- MATRIX_EVAL="" # - MATRIX_EVAL=""
- PIP_UPDATE="1" # - PIP_UPDATE="1"
- PYTHON_BUILD=true # - PYTHON_BUILD=true
- CC=clang # - CC=clang
- CXX=clang++ # - CXX=clang++
- name: "klayout python2.6 package" # - name: "klayout python2.6 package"
os: linux # os: linux
dist: trusty # Ubuntu 14.04 # dist: trusty # Ubuntu 14.04
sudo: false # sudo: false
language: python # language: python
python: '2.6' # python: '2.6'
env: # env:
- MATRIX_EVAL="" # - MATRIX_EVAL=""
- PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 # - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6
- PYTHON_BUILD=true # - PYTHON_BUILD=true
- CC=clang # - CC=clang
- CXX=clang++ # - CXX=clang++
- name: "klayout python3.3 package" # - name: "klayout python3.3 package"
os: linux # os: linux
dist: trusty # Ubuntu 14.04 # dist: trusty # Ubuntu 14.04
sudo: false # sudo: false
language: python # language: python
python: '3.3' # python: '3.3'
env: # env:
- MATRIX_EVAL="" # - MATRIX_EVAL=""
- PIP_UPDATE="1" # - PIP_UPDATE="1"
- PYTHON_BUILD=true # - PYTHON_BUILD=true
- CC=clang # - CC=clang
- CXX=clang++ # - CXX=clang++
- name: "klayout python3.4 package" # - name: "klayout python3.4 package"
os: linux # os: linux
dist: trusty # Ubuntu 14.04 # dist: trusty # Ubuntu 14.04
sudo: false # sudo: false
language: python # language: python
python: '3.4' # python: '3.4'
env: # env:
- MATRIX_EVAL="" # - MATRIX_EVAL=""
- PIP_UPDATE="1" # - PIP_UPDATE="1"
- PYTHON_BUILD=true # - PYTHON_BUILD=true
- CC=clang # - CC=clang
- CXX=clang++ # - CXX=clang++
- name: "klayout python3.5 package" # - name: "klayout python3.5 package"
os: linux # os: linux
dist: trusty # Ubuntu 14.04 # dist: trusty # Ubuntu 14.04
sudo: false # sudo: false
language: python # language: python
python: '3.5' # python: '3.5'
env: # env:
- MATRIX_EVAL="" # - MATRIX_EVAL=""
- PIP_UPDATE="1" # - PIP_UPDATE="1"
- PYTHON_BUILD=true # - PYTHON_BUILD=true
- CC=clang # - CC=clang
- CXX=clang++ # - CXX=clang++
# KLayout builds for mac # KLayout builds for mac
# Python 3 # Python 3
- name: "KLayout macOS 10.13 with py3.7" - name: "KLayout macOS 10.13 with py3.7"
os: osx os: osx
osx_image: xcode9.4 # macOS 10.13 osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons: addons:
homebrew: homebrew:
packages: packages:
- python3 - python3
- qt - qt
- ccache
update: true update: true
env: env:
- MATRIX_EVAL="" - MATRIX_EVAL=""
@ -206,11 +445,13 @@ matrix:
- name: "KLayout macOS 10.12 with py3.7" - name: "KLayout macOS 10.12 with py3.7"
os: osx os: osx
osx_image: xcode8.3 # macOS 10.12 osx_image: xcode8.3 # macOS 10.12
cache: ccache
addons: addons:
homebrew: homebrew:
packages: packages:
- python3 - python3
- qt - qt
- ccache
update: true update: true
env: env:
- MATRIX_EVAL="" - MATRIX_EVAL=""
@ -221,6 +462,7 @@ matrix:
- name: "KLayout macOS 10.11 with py3.7" - name: "KLayout macOS 10.11 with py3.7"
os: osx os: osx
osx_image: xcode8 # macOS 10.11 osx_image: xcode8 # macOS 10.11
cache: ccache
addons: addons:
homebrew: homebrew:
packages: packages:
@ -236,10 +478,12 @@ matrix:
- name: "KLayout macOS 10.13 with py2.7" - name: "KLayout macOS 10.13 with py2.7"
os: osx os: osx
osx_image: xcode9.4 # macOS 10.13 osx_image: xcode9.4 # macOS 10.13
cache: ccache
addons: addons:
homebrew: homebrew:
packages: packages:
- qt - qt
- ccache
update: true update: true
env: env:
- MATRIX_EVAL="" - MATRIX_EVAL=""
@ -250,10 +494,12 @@ matrix:
- name: "KLayout macOS 10.12 with py2.7" - name: "KLayout macOS 10.12 with py2.7"
os: osx os: osx
osx_image: xcode8.3 # macOS 10.12 osx_image: xcode8.3 # macOS 10.12
cache: ccache
addons: addons:
homebrew: homebrew:
packages: packages:
- qt - qt
- ccache
update: true update: true
env: env:
- MATRIX_EVAL="" - MATRIX_EVAL=""
@ -264,6 +510,12 @@ matrix:
- name: "KLayout macOS 10.11 with py2.7" - name: "KLayout macOS 10.11 with py2.7"
os: osx os: osx
osx_image: xcode8 # macOS 10.11 osx_image: xcode8 # macOS 10.11
cache: ccache
addons:
homebrew:
packages:
- ccache
update: true
env: env:
- MATRIX_EVAL="brew update; brew config; brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/56c500b569c724b049be7ab9e12d9693f85522f9/Formula/qt.rb" # Qt 5.11.2 - MATRIX_EVAL="brew update; brew config; brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/56c500b569c724b049be7ab9e12d9693f85522f9/Formula/qt.rb" # Qt 5.11.2
- PYTHON_VERSION=Sys - PYTHON_VERSION=Sys
@ -281,15 +533,42 @@ before_install:
pip install --upgrade setuptools wheel || sudo pip install --upgrade setuptools wheel; pip install --upgrade setuptools wheel || sudo pip install --upgrade setuptools wheel;
fi fi
- python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)"
- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
export PATH="/usr/local/opt/ccache/libexec:$PATH";
fi
script: install:
- if [ "$DOCKER_BUILD" = true ]; then
docker pull $DOCKER_IMAGE;
fi
script:
- if [ "$DOCKER_BUILD" = true ]; then
mkdir -p ccache;
mkdir -p wheelhouse;
docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh";
klayout_version=$(python -c 'import setup; print(setup.Config().version())');
mkdir -p deploy/dist-pymod/$klayout_version;
cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod/$klayout_version;
if [ "$TEST_IN_HOST" = true ]; then
pip install klayout --no-index -f ./wheelhouse;
python testdata/pymod/import_db.py;
python testdata/pymod/import_rdb.py;
python testdata/pymod/import_tl.py;
python testdata/pymod/pya_tests.py;
fi
fi
- if [ "$PYTHON_BUILD" = true ]; then - if [ "$PYTHON_BUILD" = true ]; then
python setup.py build; python setup.py build;
python setup.py bdist_wheel; python setup.py bdist_wheel;
python setup.py install; python setup.py install;
python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; python testdata/pymod/import_db.py;
mkdir -p deploy/dist-pymod; python testdata/pymod/import_rdb.py;
cp -a dist/* deploy/dist-pymod/; python testdata/pymod/import_tl.py;
python testdata/pymod/pya_tests.py;
klayout_version=$(python -c 'import setup; print(setup.Config().version())');
mkdir -p deploy/dist-pymod/$klayout_version;
cp -a dist/*.whl deploy/dist-pymod/$klayout_version;
python -c 'import klayout.db as db; print(dir(db))'; python -c 'import klayout.db as db; print(dir(db))';
python -c 'import klayout.rdb as rdb; print(dir(rdb))'; python -c 'import klayout.rdb as rdb; print(dir(rdb))';
python -c 'import klayout.tl as tl; print(dir(tl))'; python -c 'import klayout.tl as tl; print(dir(tl))';

4
Brewfile
View File

@ -1,4 +0,0 @@
tap "homebrew/core"
brew "python3"
brew "python@2", link: false
brew "qt"

22
ci-scripts/docker/README.md Normal file
View File

@ -0,0 +1,22 @@
Author: Thomas Ferreira de Lima
email: thomas@tlima.me
This folder contains scripts to be run inside docker images. See instructions on how to test this yourself in ci-scripts/docker/development_notes.
## docker_build.sh
We need two environment variables to get going:
```bash
DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
PY_VERSION="cp37-cp37m"
```
The script must be run inside an image pulled from $DOCKER_IMAGE and with klayout's git repo cloned in /io. Inside the git clone folder, run:
```bash
docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh";
# $PRE_CMD is empty for now (useless currently).
```
This command will generate a wheel and place it in `wheelhouse/klayout-*manylinux1*.whl`. This is the wheel that needs to be uploaded to PyPI via twine. See ci-scripts/twine/README.md.

17
ci-scripts/docker/development_notes/Dockerfile.i686 Normal file
View File

@ -0,0 +1,17 @@
FROM quay.io/pypa/manylinux1_i686
MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me)
# Install a system package required by our library
RUN linux32 yum install -y zlib-devel
RUN linux32 yum install -y ccache
RUN ln -s /usr/bin/ccache /usr/lib/ccache/c++
RUN ln -s /usr/bin/ccache /usr/lib/ccache/cc
RUN ln -s /usr/bin/ccache /usr/lib/ccache/gcc
RUN ln -s /usr/bin/ccache /usr/lib/ccache/g++
# Add ccache to PATH
RUN mkdir -p /persist/.ccache
ENV CCACHE_DIR="/persist/.ccache"
# Need zip to fix wheel
RUN linux32 yum install -y zip

17
ci-scripts/docker/development_notes/Dockerfile.x86_64 Normal file
View File

@ -0,0 +1,17 @@
FROM quay.io/pypa/manylinux1_x86_64
MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me)
# Install a system package required by our library
RUN yum install -y zlib-devel
RUN yum install -y ccache
RUN ln -s /usr/bin/ccache /usr/lib64/ccache/c++
RUN ln -s /usr/bin/ccache /usr/lib64/ccache/cc
RUN ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
RUN ln -s /usr/bin/ccache /usr/lib64/ccache/g++
# Add ccache to PATH
RUN mkdir -p /persist/.ccache
ENV CCACHE_DIR="/persist/.ccache"
# Need zip to fix wheel
RUN yum install -y zip

221
ci-scripts/docker/development_notes/README.md Normal file
View File

@ -0,0 +1,221 @@
Author: Thomas Ferreira de Lima
email: thomas@tlima.me
I wrote these notes as I was learning how to use docker and how to build python packages inside a docker image prepared by the pypa team. They require us to build there to allow wheels to have the `manylinux1` tag, meaning that these wheels would be compatible with most linux distributions around. Chapter 1 is about testing in my own computer (MacOS Mojave) and Chapter 2 is about how to automate this build using travis-ci.org.
# Chapter 1. Testing on your own computer
## Step 1.
Make sure you have the quay.io/pypa/manylinux1_x86_64 image.
```bash
$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB
hello-world latest 4ab4c602aa5e 3 months ago 1.84kB
# My image was old:
$ docker pull quay.io/pypa/manylinux1_x86_64
Using default tag: latest
latest: Pulling from pypa/manylinux1_x86_64
7d0d9526f38a: Already exists
3324bfadf9cb: Pull complete
20f27c7e3062: Pull complete
5bc21fc5fe97: Pull complete
Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500
Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest
```
## Step 2.
This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72
Create a volume for klayout. This is necessary because docker containers don't persist data.
```bash
$ docker volume create klayout-persist
$ docker volume inspect klayout-persist
[
{
"CreatedAt": "2018-12-18T15:01:48Z",
"Driver": "local",
"Labels": {},
"Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data",
"Name": "klayout-persist",
"Options": {},
"Scope": "local"
}
]
```
## Step 3.
Build image `myimage` with:
```bash
$ docker build -t myimage:latest -f Dockerfile.x86_64 .
```
This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`.
Then I run the docker with a terminal shell and load the volume klayout-persist in /persist:
```bash
$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage
```
## Step 4.
In the shell, pull master from klayout.
```bash
cd /persist
git clone https://github.com/lightwave-lab/klayout.git
mkdir -p wheelhouse
cd klayout
# make wheel with python 3.6 (for example)
/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/
cd /persist
auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/
# Need to manually fix the wheel
#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse
```
The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place.
Procedure to fix the wheel:
```bash
unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel
cd tempwheel/klayout
mv lib_* db_plugins .libs/
cd ../klayout-0.26.0.dev8.dist-info/
sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD
sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD
cd ../
rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl
zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./*
cd ..
rm -rf tempwheel
```
Now we can install and test:
```bash
/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse
cd /persist/klayout
/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py
# Tests passed!
```
Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel.
## Step 5. Iterate over all python versions.
For that, we need something like:
```bash
# Compile wheels
for PYBIN in /opt/python/*/bin; do
"${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/
done
# Bundle external shared libraries into the wheels via auditwheel
for whl in /persist/wheelhouse/*linux_*.whl; do
auditwheel repair "$whl" -w /persist/wheelhouse/
done
# Fix each wheel generated by auditwheel
for whl in /persist/wheelhouse/*manylinux1_*.whl; do
./ci-scripts/docker/fix_wheel.sh "$whl"
done
# Install packages and test
TEST_HOME=/persist/klayout/testdata
for PYBIN in /opt/python/*/bin/; do
"${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse
"${PYBIN}/python" $TEST_HOME/pymod/import_db.py
"${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py
"${PYBIN}/python" $TEST_HOME/pymod/import_tl.py
```
I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error:
`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String`
I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis.
The ccache folder after a single build has 657MB. Go figure.
I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1
So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default.
# Chapter 2. Testing CI flow with docker
# Step 1. Testing commands in own computer
First cloned with:
```bash
git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux
cd klayout
```
Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml
DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686
PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m
Total of 2x6 = 12 possibilities
```bash
export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64
export PY_VERSION="cp36-cp36m"
docker pull $DOCKER_IMAGE
mkdir -p ccache
mkdir -p wheelhouse
docker run --name klayout -v `pwd` -it $DOCKER_IMAGE
```
Inside docker shell:
```bash
yum install -y zlib-devel
yum install -y zip
yum install -y ccache
ln -s /usr/bin/ccache /usr/lib64/ccache/c++
ln -s /usr/bin/ccache /usr/lib64/ccache/cc
ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
ln -s /usr/bin/ccache /usr/lib64/ccache/g++
echo $PATH
# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export CCACHE_DIR="/io/ccache"
# Compile wheel
/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/
# Bundle external shared libraries into the wheels via auditwheel
for whl in /io/wheelhouse/*linux_*.whl; do
auditwheel repair "$whl" -w /io/wheelhouse/
done
# Fix each wheel generated by auditwheel
for whl in /io/wheelhouse/*manylinux1_*.whl; do
./ci-scripts/docker/fix_wheel.sh "$whl"
done
```
# Step 2. Automating step 1 in travis (CI).
DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686
PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m
Build: spawn 12 travis jobs, one for each combination of word-size and python version.
Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`.
Post-build:
- cache `./ccache`
- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder)
# Step 3. Automating deployment to PyPI:
TBD

77
ci-scripts/docker/development_notes/fix_wheel.sh Executable file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env bash
SCRIPT_NAME=`basename "$0"`
TMP_WHEEL="/tmp/klayout_tempwheel"
display_usage() {
echo "This script fixes auditwheel-repaired wheels."
echo "Caution: This will delete the original wheel."
echo -e "\nUsage:\n./${SCRIPT_NAME} [--help|-h] klayout-...-manylinux1_x86_64.whl \n"
}
if [[ ( $1 == "--help") || $1 == "-h" ]]
then
display_usage
exit 0
fi
# Check number of arguments
if [ $# -eq 0 ]; then
>&2 echo -e "ERROR: No filename supplied\n"
display_usage
exit 1
elif [ ! $# -eq 1 ]; then
>&2 echo -e "ERROR: Too many files supplied. Provide one filename at at time.\n"
display_usage
exit 1
fi
# Read wheel file from argument
WHL="$1"
# Check WHL is a valid file
if [[ ! -f "$WHL" ]]; then
>&2 echo -e "ERROR: $WHL is not a file"
exit 1
fi
# Convert to absolute path (linux only)
WHL=$(readlink -f $WHL)
# Record old current directory
OLD_PWD=$PWD
# The produced wheel from auditwheel, klayout-*-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place.
# Checking if it was previously patched
if unzip -l $WHL | grep -q 'patched_after_auditwheel_repair'; then
echo "$(basename $WHL) is already patched. Doing nothing."
exit 0
fi
# Repair script below
if [[ -d $TMP_WHEEL ]]; then
rm -rf $TMP_WHEEL
fi
echo "Unpacking $WHL into $TMP_WHEEL"
unzip -q $WHL -d $TMP_WHEEL
cd $TMP_WHEEL/klayout
echo "Moving files: mv lib_* db_plugins .libs/"
mv lib_* db_plugins .libs/ 2>/dev/null
if [ $? -ne 0 ]; then
>&2 echo "ERROR: lib_*.so or db_plubins not found. Quitting."
exit 1
fi
cd ../klayout-*.dist-info/
echo "Patching klayout-*.dist-info/RECORD"
sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD
sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD
cd ../
touch $TMP_WHEEL/patched_after_auditwheel_repair
echo "Packing $WHL from $TMP_WHEEL"
rm -f $WHL
zip -rq $WHL ./*
echo "Done. $(basename $WHL) is patched."
# Cleanup (should always execute)
cd $OLD_PWD

38
ci-scripts/docker/development_notes/manylinux-docker.sh Normal file
View File

@ -0,0 +1,38 @@
#!/bin/bash
# run with docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/ci-scripts/manylinux-docker.sh
# see https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml
# cache using https://github.com/travis-ci/travis-ci/issues/5358
set -e -x
# Install a system package required by our library
yum install -y zlib-devel
yum install -y ccache
ln -s /usr/bin/ccache /usr/lib64/ccache/c++
ln -s /usr/bin/ccache /usr/lib64/ccache/cc
ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
ln -s /usr/bin/ccache /usr/lib64/ccache/g++
# export PATH=/usr/lib64/ccache:$PATH # unnecessary
# Compile wheels
for PYBIN in /opt/python/*/bin; do
"${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/
done
# Bundle external shared libraries into the wheels via auditwheel
for whl in /persist/wheelhouse/*linux_*.whl; do
auditwheel repair "$whl" -w /persist/wheelhouse/
done
# Fix each wheel generated by auditwheel
for whl in /persist/wheelhouse/*manylinux1_*.whl; do
./ci-scripts/docker/fix_wheel.sh "$whl"
done
# Install packages and test
TEST_HOME=/persist/klayout/testdata
for PYBIN in /opt/python/*/bin/; do
"${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse
"${PYBIN}/python" $TEST_HOME/pymod/import_db.py
"${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py
"${PYBIN}/python" $TEST_HOME/pymod/import_tl.py
done

73
ci-scripts/docker/docker_build.sh Executable file
View File

@ -0,0 +1,73 @@
#!/usr/bin/env bash
if [[ -z $PY_VERSION ]]; then
echo '$PY_VERSION is not set'
exit 1
fi
if [[ -z $DOCKER_IMAGE ]]; then
echo '$DOCKER_IMAGE is not set'
exit 1
fi
echo PY_VERSION=$PY_VERSION
echo DOCKER_IMAGE=$DOCKER_IMAGE
# sometimes the epel server is down. retry 5 times
for i in $(seq 1 5); do
yum install -y zlib-devel ccache zip git && s=0 && break || s=$? && sleep 15;
done
[ $s -eq 0 ] || exit $s
if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then
ln -s /usr/bin/ccache /usr/lib64/ccache/c++
ln -s /usr/bin/ccache /usr/lib64/ccache/cc
ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
ln -s /usr/bin/ccache /usr/lib64/ccache/g++
export PATH="/usr/lib64/ccache/:$PATH"
elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then
ln -s /usr/bin/ccache /usr/lib/ccache/c++
ln -s /usr/bin/ccache /usr/lib/ccache/cc
ln -s /usr/bin/ccache /usr/lib/ccache/gcc
ln -s /usr/bin/ccache /usr/lib/ccache/g++
export PATH="/usr/lib/ccache/:$PATH"
fi
echo $PATH
export CCACHE_DIR="/io/ccache"
# Download proper auditwheel program
git clone https://github.com/thomaslima/auditwheel.git /tmp/auditwheel
cd /tmp/auditwheel
git checkout 87f5306ec02cc68020afaa9933543c898b1d47c1 # patched version
AUDITWHEEL_PYTHON=$(cat `which auditwheel` | head -1 | sed -e 's/#!\(.*\)/\1/')
# Install auditwheel, replacing the system's auditwheel binary
$AUDITWHEEL_PYTHON -m pip install .
# Show ccache stats
echo "Cache stats:"
ccache -s
# Compile wheel
cd /io
"/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1
# Show ccache stats
echo "Cache stats:"
ccache -s
# Bundle external shared libraries into the wheels via auditwheel
for whl in /io/wheelhouse/*linux_*.whl; do
auditwheel -v repair "$whl" -w /io/wheelhouse/ || exit 1
done
# Install packages and test
TEST_HOME=/io/testdata
"/opt/python/$PY_VERSION/bin/pip" install klayout --no-index -f /io/wheelhouse || exit 1
"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_db.py || exit 1
"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_rdb.py || exit 1
"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_tl.py || exit 1
"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/pya_tests.py || exit 1

15
ci-scripts/twine/README.md Normal file
View File

@ -0,0 +1,15 @@
After building all the travis wheels, go to the folder where the wheels were deployed. In my case, for example, `/Users/tlima/Dropbox/Apps/travis-deploy/Builds/klayout/dist-pymod/0.26.0.dev10`.
Then, run the command
```bash
travis upload *.whl
```
, which will ask for an username and password related to your account.
After this upload was successful, you can upload the source tarball. Go to klayout's git folder and run `python setup.py sdist`. Inside `dist/`, you'll find a tarball named, e.g., `klayout-0.26.0.dev10.tar.gz`. So just run
```bash
travis upload klayout-0.26.0.dev10.tar.gz
```

16
setup.py
View File

@ -59,13 +59,13 @@ from setuptools.extension import Extension, Library
import glob import glob
import os import os
import platform import platform
import distutils.sysconfig as sysconfig
from distutils.errors import CompileError from distutils.errors import CompileError
import distutils.command.build_ext import distutils.command.build_ext
import setuptools.command.build_ext import setuptools.command.build_ext
import multiprocessing import multiprocessing
N_cores = multiprocessing.cpu_count() N_cores = multiprocessing.cpu_count()
# monkey-patch for parallel compilation # monkey-patch for parallel compilation
# from https://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils # from https://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils
def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None):
@ -88,6 +88,7 @@ def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=N
n_tries = 2 n_tries = 2
while n_tries > 0: while n_tries > 0:
try: try:
print("Building", obj)
self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
except CompileError: except CompileError:
n_tries -= 1 n_tries -= 1
@ -95,7 +96,7 @@ def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=N
else: else:
break break
# convert to list, imap is evaluated on-demand # convert to list, imap is evaluated on-demand
list(multiprocessing.pool.ThreadPool(N).imap(_single_compile, objects)) list(multiprocessing.pool.ThreadPool(N).map(_single_compile, objects))
return objects return objects
@ -155,10 +156,10 @@ distutils.ccompiler.CCompiler.library_filename = patched_library_filename
# for this ... We're patching this back now. # for this ... We're patching this back now.
def always_link_shared_object( def always_link_shared_object(
self, objects, output_libname, output_dir=None, libraries=None, self, objects, output_libname, output_dir=None, libraries=None,
library_dirs=None, runtime_library_dirs=None, export_symbols=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None,
debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None,
target_lang=None): target_lang=None):
self.link( self.link(
self.SHARED_LIBRARY, objects, output_libname, self.SHARED_LIBRARY, objects, output_libname,
output_dir, libraries, library_dirs, runtime_library_dirs, output_dir, libraries, library_dirs, runtime_library_dirs,
@ -166,6 +167,7 @@ def always_link_shared_object(
build_temp, target_lang build_temp, target_lang
) )
setuptools.command.build_ext.libtype = "shared" setuptools.command.build_ext.libtype = "shared"
setuptools.command.build_ext.link_shared_object = always_link_shared_object setuptools.command.build_ext.link_shared_object = always_link_shared_object
@ -320,7 +322,7 @@ class Config(object):
""" """
Gets the version string Gets the version string
""" """
return "0.26.0.dev8" return "0.26.0.dev10"
config = Config() config = Config()

3
src/pymod/__init__.py.qtless
View File

@ -1,5 +1,4 @@
# klayout library definition file # klayout library definition file
__all__ = [ "tl", "db", "lay", "rdb" ] __all__ = [ "tl", "db", "rdb" ]

15
src/pymod/distutils_src/pya/__init__.py
View File

@ -1,10 +1,7 @@
# import all packages from klayout, such as klayout.db and klayout.tl
# WARNING: doing it manually until it becomes impractical
# TODO: We need a specification document explaining what should go into pya
import klayout from klayout.db import * # noqa
import importlib from klayout.tl import * # noqa
from klayout.rdb import * # noqa
__all__ = []
for m in klayout.__all__:
mod = importlib.import_module("klayout." + m)
for mm in mod.__all__:
globals()[mm] = getattr(mod, mm)