From 2bad812953920aee2550a0a7a57bd33f4c99227d Mon Sep 17 00:00:00 2001 From: "Anthony J. Bentley" Date: Fri, 7 Dec 2018 06:07:40 -0700 Subject: [PATCH 01/52] OpenBSD doesn't have sa_restorer. --- src/lay/lay/laySignalHandler.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lay/lay/laySignalHandler.cc b/src/lay/lay/laySignalHandler.cc index 5fa8bec25..12f540232 100644 --- a/src/lay/lay/laySignalHandler.cc +++ b/src/lay/lay/laySignalHandler.cc @@ -450,7 +450,7 @@ void install_signal_handlers () act.sa_sigaction = signal_handler; sigemptyset (&act.sa_mask); act.sa_flags = SA_SIGINFO; -#if !defined(__APPLE__) +#if !defined(__APPLE__) && !defined(__OpenBSD__) act.sa_restorer = 0; #endif From b75681a333b1d48b5c6f56d32f45bb9d57ec1f1f Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Tue, 11 Dec 2018 21:51:40 +0100 Subject: [PATCH 02/52] Fixed #207 (integer rounding happens on text bias and spacing values for Basic.TEXT PCell) --- src/lib/lib/libBasicText.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/lib/libBasicText.cc b/src/lib/lib/libBasicText.cc index 2ccb8c444..d1ecdc7e1 100644 --- a/src/lib/lib/libBasicText.cc +++ b/src/lib/lib/libBasicText.cc @@ -154,11 +154,11 @@ BasicText::produce (const db::Layout &layout, const std::vector &l const db::TextGenerator &font = db::TextGenerator::generators ()[f]; double m = parameters [p_magnification].to_double (); - db::Coord b = parameters [p_bias].to_double (); + double b = parameters [p_bias].to_double (); bool inv = parameters [p_inverse].to_bool (); - db::Coord dx = parameters [p_char_spacing].to_double (); - db::Coord dy = parameters [p_line_spacing].to_double (); + double dx = parameters [p_char_spacing].to_double (); + double dy = parameters [p_line_spacing].to_double (); std::string t = parameters [p_text].to_string (); From 3ed9e6348027d0b6a6ba86973b28175a0d7f31ae Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 16 Dec 2018 23:39:45 +0100 Subject: [PATCH 03/52] Fixed a typo --- src/lay/lay/doc/about/packages.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lay/lay/doc/about/packages.xml b/src/lay/lay/doc/about/packages.xml index 7f7b206f8..2b9c3daec 100644 --- a/src/lay/lay/doc/about/packages.xml +++ b/src/lay/lay/doc/about/packages.xml @@ -176,7 +176,7 @@ <version>Version</version> <title>Title of the package</title> <doc>A brief description</doc> - <doc_url>Documentation URL</doc_url> + <doc-url>Documentation URL</doc-url> <url>Download URL</url> <license>License model</license> <icon>Icon image: base64-encoded, 64x64 max, PNG preferred</icon> From db3b1d8eeee3475031cb0fdf2b77cdf472028d1d Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 16 Dec 2018 23:35:30 +0100 Subject: [PATCH 04/52] Ported #209 fix to master. --- .../laybasic/LoadLayoutOptionsDialog.ui | 51 +++++++++---------- .../laybasic/layLoadLayoutOptionsDialog.cc | 5 +- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/laybasic/laybasic/LoadLayoutOptionsDialog.ui b/src/laybasic/laybasic/LoadLayoutOptionsDialog.ui index 44bc22a6b..27860645a 100644 --- a/src/laybasic/laybasic/LoadLayoutOptionsDialog.ui +++ b/src/laybasic/laybasic/LoadLayoutOptionsDialog.ui @@ -33,17 +33,14 @@ 9 - + QFrame::NoFrame QFrame::Raised - - - 6 - + 0 @@ -56,17 +53,17 @@ 0 - - + + + + + Technology to be associated with layout - - - - + Qt::Horizontal @@ -79,25 +76,25 @@ + + + + Qt::Vertical + + + QSizePolicy::Fixed + + + + 0 + 10 + + + + - - - - Qt::Vertical - - - QSizePolicy::Fixed - - - - 0 - 10 - - - - diff --git a/src/laybasic/laybasic/layLoadLayoutOptionsDialog.cc b/src/laybasic/laybasic/layLoadLayoutOptionsDialog.cc index ead56ec08..9fc9d487f 100644 --- a/src/laybasic/laybasic/layLoadLayoutOptionsDialog.cc +++ b/src/laybasic/laybasic/layLoadLayoutOptionsDialog.cc @@ -187,7 +187,7 @@ LoadLayoutOptionsDialog::update () } const db::Technology *tech = m_tech_array [m_technology_index]; - mp_ui->options_tab->setEnabled (tech && tech->is_persisted ()); + mp_ui->options_tab->setEnabled (!tech || tech->is_persisted ()); for (std::vector< std::pair >::iterator page = m_pages.begin (); page != m_pages.end (); ++page) { if (page->first) { @@ -240,6 +240,7 @@ LoadLayoutOptionsDialog::edit_global_options (lay::PluginRoot *config_root, db:: mp_ui->tech_cbx->blockSignals (false); mp_ui->tech_cbx->show (); + mp_ui->tech_frame->show (); if (get_options_internal ()) { @@ -271,7 +272,7 @@ LoadLayoutOptionsDialog::edit_global_options (lay::PluginRoot *config_root, db:: bool LoadLayoutOptionsDialog::get_options (db::LoadLayoutOptions &options) { - mp_ui->tech_cbx->hide (); + mp_ui->tech_frame->hide (); mp_ui->always_cbx->hide (); m_opt_array.clear (); From 5d4b0dac1a7148b5f7ea93d49784f333e5721013 Mon Sep 17 00:00:00 2001 From: Stephen Date: Sun, 16 Dec 2018 18:24:22 -0800 Subject: [PATCH 05/52] use new travis homebrew addon --- .travis.yml | 70 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 318cfff21..014d8c940 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ matrix: - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - name: "klayout python2 osx10.12" os: osx @@ -20,7 +19,6 @@ matrix: - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - name: "klayout python2 osx10.11" os: osx @@ -30,19 +28,22 @@ matrix: - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false # python 3 osx - name: "klayout python3 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + addons: + homebrew: + packages: + - python3 + update: true env: - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=true - name: "klayout python3.6.6 osx10.13" os: osx @@ -52,7 +53,6 @@ matrix: - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - name: "klayout python3.5.6 osx10.13" os: osx @@ -62,7 +62,6 @@ matrix: - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - name: "klayout python3.4.9 osx10.13" os: osx @@ -72,17 +71,20 @@ matrix: - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - name: "klayout python3 osx10.12" os: osx osx_image: xcode8.3 # macOS 10.12 + addons: + homebrew: + packages: + - python3 + update: true env: - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=true - name: "klayout python3 osx10.11" os: osx @@ -92,7 +94,6 @@ matrix: - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - name: "klayout python3.7 package" os: linux @@ -104,7 +105,6 @@ matrix: - MATRIX_EVAL="" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - CC=clang - CXX=clang++ @@ -118,7 +118,6 @@ matrix: - MATRIX_EVAL="" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - CC=clang - CXX=clang++ @@ -132,7 +131,6 @@ matrix: - MATRIX_EVAL="" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - CC=clang - CXX=clang++ @@ -146,7 +144,6 @@ matrix: - MATRIX_EVAL="" - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 - PYTHON_BUILD=true - - BREW_BUNDLE=false - CC=clang - CXX=clang++ @@ -160,7 +157,6 @@ matrix: - MATRIX_EVAL="" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - CC=clang - CXX=clang++ @@ -174,7 +170,6 @@ matrix: - MATRIX_EVAL="" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - CC=clang - CXX=clang++ @@ -188,7 +183,6 @@ matrix: - MATRIX_EVAL="" - PIP_UPDATE="1" - PYTHON_BUILD=true - - BREW_BUNDLE=false - CC=clang - CXX=clang++ @@ -197,73 +191,95 @@ matrix: - name: "KLayout macOS 10.13 with py3.7" os: osx osx_image: xcode9.4 # macOS 10.13 + addons: + homebrew: + packages: + - python3 + - qt + update: true env: - MATRIX_EVAL="" - PYTHON_VERSION=B37 - MACOS_VERSION=HighSierra - KLAYOUT_BUILD=true - - BREW_BUNDLE=true - name: "KLayout macOS 10.12 with py3.7" os: osx osx_image: xcode8.3 # macOS 10.12 + addons: + homebrew: + packages: + - python3 + - qt + update: true env: - MATRIX_EVAL="" - PYTHON_VERSION=B37 - MACOS_VERSION=Sierra - KLAYOUT_BUILD=true - - BREW_BUNDLE=true - name: "KLayout macOS 10.11 with py3.7" os: osx osx_image: xcode8 # macOS 10.11 + addons: + homebrew: + packages: + - python3 + - qt + update: true env: - MATRIX_EVAL="" - PYTHON_VERSION=B37 - MACOS_VERSION=ElCapitan - KLAYOUT_BUILD=true - - BREW_BUNDLE=true # Python 2 - name: "KLayout macOS 10.13 with py2.7" os: osx osx_image: xcode9.4 # macOS 10.13 + addons: + homebrew: + packages: + - qt + update: true env: - MATRIX_EVAL="" - PYTHON_VERSION=Sys - MACOS_VERSION=HighSierra - KLAYOUT_BUILD=true - - BREW_BUNDLE=true - name: "KLayout macOS 10.12 with py2.7" os: osx osx_image: xcode8.3 # macOS 10.12 + addons: + homebrew: + packages: + - qt + update: true env: - MATRIX_EVAL="" - PYTHON_VERSION=Sys - MACOS_VERSION=Sierra - KLAYOUT_BUILD=true - - BREW_BUNDLE=true - name: "KLayout macOS 10.11 with py2.7" os: osx osx_image: xcode8 # macOS 10.11 + addons: + homebrew: + packages: + - qt + update: true env: - MATRIX_EVAL="" - PYTHON_VERSION=Sys - MACOS_VERSION=ElCapitan - KLAYOUT_BUILD=true - - BREW_BUNDLE=true before_install: - env - gem install dropbox-deployment - eval "${MATRIX_EVAL}" - - if [ "$BREW_BUNDLE" = true ]; then - find "$(brew --prefix)/Caskroom/"*'/.metadata' -type f -name '*.rb' | xargs grep 'EOS.undent' --files-with-matches | xargs sed -i '' 's/EOS.undent/EOS/'; - brew update; - brew bundle; - fi - if [ "${PIP_UPDATE}" == "1" ]; then pip --version; pip install --upgrade pip || sudo pip install --upgrade pip; From b06fe5817d2119b372d17961aab76afd8ded0b7c Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 17 Dec 2018 17:56:04 -0500 Subject: [PATCH 06/52] ThreadPool.imap stopped working in py 3.7.1 --- setup.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index b2c5e2748..545126651 100644 --- a/setup.py +++ b/setup.py @@ -59,13 +59,13 @@ from setuptools.extension import Extension, Library import glob import os import platform -import distutils.sysconfig as sysconfig from distutils.errors import CompileError import distutils.command.build_ext import setuptools.command.build_ext import multiprocessing N_cores = multiprocessing.cpu_count() + # monkey-patch for parallel compilation # from https://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): @@ -88,6 +88,7 @@ def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=N n_tries = 2 while n_tries > 0: try: + print("Building", obj) self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) except CompileError: n_tries -= 1 @@ -95,7 +96,7 @@ def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=N else: break # convert to list, imap is evaluated on-demand - list(multiprocessing.pool.ThreadPool(N).imap(_single_compile, objects)) + list(multiprocessing.pool.ThreadPool(N).map(_single_compile, objects)) return objects @@ -155,10 +156,10 @@ distutils.ccompiler.CCompiler.library_filename = patched_library_filename # for this ... We're patching this back now. def always_link_shared_object( - self, objects, output_libname, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, export_symbols=None, - debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, - target_lang=None): + self, objects, output_libname, output_dir=None, libraries=None, + library_dirs=None, runtime_library_dirs=None, export_symbols=None, + debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, + target_lang=None): self.link( self.SHARED_LIBRARY, objects, output_libname, output_dir, libraries, library_dirs, runtime_library_dirs, @@ -166,6 +167,7 @@ def always_link_shared_object( build_temp, target_lang ) + setuptools.command.build_ext.libtype = "shared" setuptools.command.build_ext.link_shared_object = always_link_shared_object From 213f2b16fa6c03584400300ef37901668473985e Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:11:56 -0500 Subject: [PATCH 07/52] xcode8 does not support homebrew addon yet --- .travis.yml | 4 ++-- Brewfile | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) delete mode 100644 Brewfile diff --git a/.travis.yml b/.travis.yml index 014d8c940..ee2c85e99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -228,7 +228,7 @@ matrix: - qt update: true env: - - MATRIX_EVAL="" + - MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8 - PYTHON_VERSION=B37 - MACOS_VERSION=ElCapitan - KLAYOUT_BUILD=true @@ -271,7 +271,7 @@ matrix: - qt update: true env: - - MATRIX_EVAL="" + - MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8 - PYTHON_VERSION=Sys - MACOS_VERSION=ElCapitan - KLAYOUT_BUILD=true diff --git a/Brewfile b/Brewfile deleted file mode 100644 index 0d5580505..000000000 --- a/Brewfile +++ /dev/null @@ -1,4 +0,0 @@ -tap "homebrew/core" -brew "python3" -brew "python@2", link: false -brew "qt" From e5921252ea22d35a76ed3f8da9442cc0e3cd6c54 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Sun, 21 Oct 2018 14:40:32 -0400 Subject: [PATCH 08/52] prototype formula for manylinux1 --- ci-scripts/manylinux-docker.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 ci-scripts/manylinux-docker.sh diff --git a/ci-scripts/manylinux-docker.sh b/ci-scripts/manylinux-docker.sh new file mode 100644 index 000000000..4e9f6a694 --- /dev/null +++ b/ci-scripts/manylinux-docker.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# run with docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/ci-scripts/manylinux-docker.sh +# see https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml +# cache using https://github.com/travis-ci/travis-ci/issues/5358 +set -e -x + +# Install a system package required by our library +yum install -y zlib-devel +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccachec++ +ln -s /usr/bin/ccache /usr/lib64/ccachecc +ln -s /usr/bin/ccache /usr/lib64/ccachegcc +ln -s /usr/bin/ccache /usr/lib64/ccacheg++ +export PATH=/usr/lib64/ccache:$PATH + +# Compile wheels +for PYBIN in /opt/python/*/bin; do + cd /io; mkdir -p /io/wheelhouse + "${PYBIN}/python" setup.py bdist_wheel -d wheelhouse/ +done + +# Bundle external shared libraries into the wheels +for whl in wheelhouse/*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + +# Install packages and test +for PYBIN in /opt/python/*/bin/; do + "${PYBIN}/pip" install klayout --no-index -f /io/wheelhouse + "${PYBIN}" -m unittest /io/testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py +done From 55f5167a02de57267e7b48dd7cf9ada721e9ba72 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 12:11:26 -0500 Subject: [PATCH 09/52] [ci skip] first working manylinux wheel --- ci-scripts/docker/Dockerfile | 18 ++++ ci-scripts/docker/README.md | 101 ++++++++++++++++++++ ci-scripts/docker/fix_wheel.sh | 61 ++++++++++++ ci-scripts/{ => docker}/manylinux-docker.sh | 10 +- 4 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 ci-scripts/docker/Dockerfile create mode 100644 ci-scripts/docker/README.md create mode 100644 ci-scripts/docker/fix_wheel.sh rename ci-scripts/{ => docker}/manylinux-docker.sh (73%) diff --git a/ci-scripts/docker/Dockerfile b/ci-scripts/docker/Dockerfile new file mode 100644 index 000000000..df4f05dc5 --- /dev/null +++ b/ci-scripts/docker/Dockerfile @@ -0,0 +1,18 @@ +FROM quay.io/pypa/manylinux1_x86_64 +MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me) + +# Install a system package required by our library +RUN yum install -y zlib-devel +RUN yum install -y ccache +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/cc +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +RUN ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + +# Add ccache to PATH +ENV PATH="/usr/lib64/ccache:${PATH}" +RUN mkdir -p /persist/.ccache +ENV CCACHE_DIR="/persist/.ccache" + +# Need zip to fix wheel +RUN yum install -y zip diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md new file mode 100644 index 000000000..615255923 --- /dev/null +++ b/ci-scripts/docker/README.md @@ -0,0 +1,101 @@ +# Step 1. + +Make sure you have the quay.io/pypa/manylinux1_x86_64 image. + +```bash +$ docker images +REPOSITORY TAG IMAGE ID CREATED SIZE +quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB +hello-world latest 4ab4c602aa5e 3 months ago 1.84kB +# My image was old: +$ docker pull quay.io/pypa/manylinux1_x86_64 +Using default tag: latest +latest: Pulling from pypa/manylinux1_x86_64 +7d0d9526f38a: Already exists +3324bfadf9cb: Pull complete +20f27c7e3062: Pull complete +5bc21fc5fe97: Pull complete +Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 +Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest +``` + +# Step 2. + +This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 +Create a volume for klayout. This is necessary because docker containers don't persist data. + +```bash +$ docker volume create klayout-persist +$ docker volume inspect klayout-persist +[ + { + "CreatedAt": "2018-12-18T15:01:48Z", + "Driver": "local", + "Labels": {}, + "Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data", + "Name": "klayout-persist", + "Options": {}, + "Scope": "local" + } +] +``` + +# Step 3. + +Build image `myimage` with: + +```bash +$ docker build -t myimage:latest . +``` + +This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. + +Then I run the docker with a terminal shell and load the volume klayout-persist in /persist: + +```bash +$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage +``` + +# Step 4. + +In the shell, pull master from klayout. + +```bash +cd /persist +git clone https://github.com/lightwave-lab/klayout.git +mkdir -p wheelhouse +cd klayout +# make wheel with python 3.6 (for example) +/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/ +cd /persist +auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/ +# Need to manually fix the wheel +#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse +``` + +The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. + + +Procedure to fix the wheel: + +```bash +unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel +cd tempwheel/klayout +mv lib_* db_plugins .libs/ +cd ../klayout-0.26.0.dev8.dist-info/ +sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD +sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD +cd ../ +rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl +zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./* +cd .. +rm -rf tempwheel +``` + +Now we can install and test: +```bash +/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse +cd /persist/klayout +/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py +# Tests passed! +``` diff --git a/ci-scripts/docker/fix_wheel.sh b/ci-scripts/docker/fix_wheel.sh new file mode 100644 index 000000000..6601ed7f7 --- /dev/null +++ b/ci-scripts/docker/fix_wheel.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=`basename "$0"` + +display_usage() { + echo "This script fixes auditwheel-repaired wheels." + echo "Caution: This will delete the original wheel." + echo -e "\nUsage:\n./${SCRIPT_NAME} klayout-...-manylinux1_x86_64.whl \n" + } + +if [[ ( $1 == "--help") || $1 == "-h" ]] +then + display_usage + exit 0 +fi + +# Check number of arguments +if [ $# -eq 0 ]; then + echo -e "No filename supplied\n" + display_usage + exit 1 +elif [ ! $# -eq 1 ]; then + echo -e "Too many files supplied. Provide one filename at at time.\n" + display_usage + exit 1 +fi + +# Read wheel file from argument +WHL="$1" + +# Check WHL is a valid file +if [[ ! -f "$WHL" ]]; then + echo -e "$WHL is not a file" + exit 1 +fi + +# Convert to absolute path (linux only) +WHL=$(readlink -f $WHL) + +# Record old current directory +OLD_PWD=$PWD + +# The produced wheel from auditwheel, klayout-*-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. + +# Repair script below +echo $WHL +echo $OLD_PWD + +unzip $WHL -d /tmp/tempwheel +cd /tmp/tempwheel/klayout +mv lib_* db_plugins .libs/ +cd ../klayout-*.dist-info/ +sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD +sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD +cd ../ +rm -f $WHL +zip -rq $WHL ./* + +# Cleanup (should always execute) +cd $OLD_PWD +rm -rf /tmp/tempwheel diff --git a/ci-scripts/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh similarity index 73% rename from ci-scripts/manylinux-docker.sh rename to ci-scripts/docker/manylinux-docker.sh index 4e9f6a694..a3bd99428 100644 --- a/ci-scripts/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -7,10 +7,10 @@ set -e -x # Install a system package required by our library yum install -y zlib-devel yum install -y ccache -ln -s /usr/bin/ccache /usr/lib64/ccachec++ -ln -s /usr/bin/ccache /usr/lib64/ccachecc -ln -s /usr/bin/ccache /usr/lib64/ccachegcc -ln -s /usr/bin/ccache /usr/lib64/ccacheg++ +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ export PATH=/usr/lib64/ccache:$PATH # Compile wheels @@ -27,5 +27,5 @@ done # Install packages and test for PYBIN in /opt/python/*/bin/; do "${PYBIN}/pip" install klayout --no-index -f /io/wheelhouse - "${PYBIN}" -m unittest /io/testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py + "${PYBIN}/python" -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py done From 70d5f5d1a02a7e3d7ff1b231bf0139f50f7d273d Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 13:51:49 -0500 Subject: [PATCH 10/52] better error messages in fix_wheel.sh script. ready to go. --- ci-scripts/docker/fix_wheel.sh | 40 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/ci-scripts/docker/fix_wheel.sh b/ci-scripts/docker/fix_wheel.sh index 6601ed7f7..2892bdbeb 100644 --- a/ci-scripts/docker/fix_wheel.sh +++ b/ci-scripts/docker/fix_wheel.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash SCRIPT_NAME=`basename "$0"` +TMP_WHEEL="/tmp/klayout_tempwheel" display_usage() { echo "This script fixes auditwheel-repaired wheels." echo "Caution: This will delete the original wheel." - echo -e "\nUsage:\n./${SCRIPT_NAME} klayout-...-manylinux1_x86_64.whl \n" + echo -e "\nUsage:\n./${SCRIPT_NAME} [--help|-h] klayout-...-manylinux1_x86_64.whl \n" } if [[ ( $1 == "--help") || $1 == "-h" ]] @@ -16,11 +17,11 @@ fi # Check number of arguments if [ $# -eq 0 ]; then - echo -e "No filename supplied\n" + >&2 echo -e "ERROR: No filename supplied\n" display_usage exit 1 elif [ ! $# -eq 1 ]; then - echo -e "Too many files supplied. Provide one filename at at time.\n" + >&2 echo -e "ERROR: Too many files supplied. Provide one filename at at time.\n" display_usage exit 1 fi @@ -30,7 +31,7 @@ WHL="$1" # Check WHL is a valid file if [[ ! -f "$WHL" ]]; then - echo -e "$WHL is not a file" + >&2 echo -e "ERROR: $WHL is not a file" exit 1 fi @@ -42,20 +43,35 @@ OLD_PWD=$PWD # The produced wheel from auditwheel, klayout-*-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. -# Repair script below -echo $WHL -echo $OLD_PWD +# Checking if it was previously patched +if unzip -l $WHL | grep -q 'patched_after_auditwheel_repair'; then + echo "$(basename $WHL) is already patched. Doing nothing." + exit 0 +fi -unzip $WHL -d /tmp/tempwheel -cd /tmp/tempwheel/klayout -mv lib_* db_plugins .libs/ +# Repair script below +if [[ -d $TMP_WHEEL ]]; then + rm -rf $TMP_WHEEL +fi +echo "Unpacking $WHL into $TMP_WHEEL" +unzip -q $WHL -d $TMP_WHEEL + +cd $TMP_WHEEL/klayout +echo "Moving files: mv lib_* db_plugins .libs/" +mv lib_* db_plugins .libs/ 2>/dev/null +if [ $? -ne 0 ]; then + >&2 echo "ERROR: lib_*.so or db_plubins not found. Quitting." + exit 1 +fi cd ../klayout-*.dist-info/ +echo "Patching klayout-*.dist-info/RECORD" sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD cd ../ +touch $TMP_WHEEL/patched_after_auditwheel_repair +echo "Packing $WHL from $TMP_WHEEL" rm -f $WHL zip -rq $WHL ./* - +echo "Done. $(basename $WHL) is patched." # Cleanup (should always execute) cd $OLD_PWD -rm -rf /tmp/tempwheel From 937019e651651135a2a6aa2c37a07c6808b0d33d Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 15:17:27 -0500 Subject: [PATCH 11/52] including Dockerfile for 32-bit linux --- ci-scripts/docker/Dockerfile.i686 | 18 ++++++++++++++++++ .../docker/{Dockerfile => Dockerfile.x86_64} | 0 2 files changed, 18 insertions(+) create mode 100644 ci-scripts/docker/Dockerfile.i686 rename ci-scripts/docker/{Dockerfile => Dockerfile.x86_64} (100%) diff --git a/ci-scripts/docker/Dockerfile.i686 b/ci-scripts/docker/Dockerfile.i686 new file mode 100644 index 000000000..f73ca3ac1 --- /dev/null +++ b/ci-scripts/docker/Dockerfile.i686 @@ -0,0 +1,18 @@ +FROM quay.io/pypa/manylinux1_i686 +MAINTAINER Thomas Ferreira de Lima (thomas@tlima.me) + +# Install a system package required by our library +RUN linux32 yum install -y zlib-devel +RUN linux32 yum install -y ccache +RUN ln -s /usr/bin/ccache /usr/lib/ccache/c++ +RUN ln -s /usr/bin/ccache /usr/lib/ccache/cc +RUN ln -s /usr/bin/ccache /usr/lib/ccache/gcc +RUN ln -s /usr/bin/ccache /usr/lib/ccache/g++ + +# Add ccache to PATH +ENV PATH="/usr/lib/ccache:${PATH}" +RUN mkdir -p /persist/.ccache +ENV CCACHE_DIR="/persist/.ccache" + +# Need zip to fix wheel +RUN linux32 yum install -y zip diff --git a/ci-scripts/docker/Dockerfile b/ci-scripts/docker/Dockerfile.x86_64 similarity index 100% rename from ci-scripts/docker/Dockerfile rename to ci-scripts/docker/Dockerfile.x86_64 From f8502adb324af278d7b7a1eba01ac5e191f18ad2 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 15:18:33 -0500 Subject: [PATCH 12/52] Tested wheel build for all manylinux python versions --- ci-scripts/docker/README.md | 26 +++++++++++++++++++++++++- ci-scripts/docker/manylinux-docker.sh | 14 +++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 615255923..054333ed7 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -45,7 +45,7 @@ $ docker volume inspect klayout-persist Build image `myimage` with: ```bash -$ docker build -t myimage:latest . +$ docker build -t myimage:latest -f Dockerfile.x86_64 . ``` This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. @@ -99,3 +99,27 @@ cd /persist/klayout /opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py # Tests passed! ``` + +Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. + +# Step 5. Iterate over all python versions. + +For that, we need something like: + +```bash +# Compile wheels +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ +done + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /persist/wheelhouse/*linux_x86_64.whl; do + auditwheel repair "$whl" -w /persist/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /persist/wheelhouse/*manylinux1_x86_64.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +``` diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh index a3bd99428..f21d40d7f 100644 --- a/ci-scripts/docker/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -15,13 +15,17 @@ export PATH=/usr/lib64/ccache:$PATH # Compile wheels for PYBIN in /opt/python/*/bin; do - cd /io; mkdir -p /io/wheelhouse - "${PYBIN}/python" setup.py bdist_wheel -d wheelhouse/ + "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ done -# Bundle external shared libraries into the wheels -for whl in wheelhouse/*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ +# Bundle external shared libraries into the wheels via auditwheel +for whl in /persist/wheelhouse/*linux_x86_64.whl; do + auditwheel repair "$whl" -w /persist/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /persist/wheelhouse/*manylinux1_x86_64.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" done # Install packages and test From 00c28d4a087d897af070339caf74b1ab6e48abae Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 15:58:33 -0500 Subject: [PATCH 13/52] tested script for linux32 as well. Also added +x permission to fix_wheel --- ci-scripts/docker/README.md | 16 ++++++++++++++-- ci-scripts/docker/fix_wheel.sh | 0 ci-scripts/docker/manylinux-docker.sh | 11 +++++++---- 3 files changed, 21 insertions(+), 6 deletions(-) mode change 100644 => 100755 ci-scripts/docker/fix_wheel.sh diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 054333ed7..5ec87ee41 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -113,13 +113,25 @@ for PYBIN in /opt/python/*/bin; do done # Bundle external shared libraries into the wheels via auditwheel -for whl in /persist/wheelhouse/*linux_x86_64.whl; do +for whl in /persist/wheelhouse/*linux_*.whl; do auditwheel repair "$whl" -w /persist/wheelhouse/ done # Fix each wheel generated by auditwheel -for whl in /persist/wheelhouse/*manylinux1_x86_64.whl; do +for whl in /persist/wheelhouse/*manylinux1_*.whl; do ./ci-scripts/docker/fix_wheel.sh "$whl" done +# Install packages and test +TEST_HOME=/persist/klayout/testdata +for PYBIN in /opt/python/*/bin/; do + "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse + "${PYBIN}/python" $TEST_HOME/pymod/import_db.py + "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py + "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py + ``` + +I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: + +`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` diff --git a/ci-scripts/docker/fix_wheel.sh b/ci-scripts/docker/fix_wheel.sh old mode 100644 new mode 100755 diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh index f21d40d7f..a4b958853 100644 --- a/ci-scripts/docker/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -19,17 +19,20 @@ for PYBIN in /opt/python/*/bin; do done # Bundle external shared libraries into the wheels via auditwheel -for whl in /persist/wheelhouse/*linux_x86_64.whl; do +for whl in /persist/wheelhouse/*linux_*.whl; do auditwheel repair "$whl" -w /persist/wheelhouse/ done # Fix each wheel generated by auditwheel -for whl in /persist/wheelhouse/*manylinux1_x86_64.whl; do +for whl in /persist/wheelhouse/*manylinux1_*.whl; do ./ci-scripts/docker/fix_wheel.sh "$whl" done # Install packages and test +TEST_HOME=/persist/klayout/testdata for PYBIN in /opt/python/*/bin/; do - "${PYBIN}/pip" install klayout --no-index -f /io/wheelhouse - "${PYBIN}/python" -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py + "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse + "${PYBIN}/python" $TEST_HOME/pymod/import_db.py + "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py + "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py done From 94b37c389c79bbd18ffe2a019139079833eb06b1 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 17:53:52 -0500 Subject: [PATCH 14/52] Continuous Integration for manylinux1 wheels (first test) --- .travis.yml | 45 +++++++++++++- ci-scripts/docker/Dockerfile.i686 | 1 - ci-scripts/docker/Dockerfile.x86_64 | 1 - ci-scripts/docker/README.md | 85 +++++++++++++++++++++++++-- ci-scripts/docker/docker_build.sh | 53 +++++++++++++++++ ci-scripts/docker/manylinux-docker.sh | 2 +- 6 files changed, 178 insertions(+), 9 deletions(-) create mode 100755 ci-scripts/docker/docker_build.sh diff --git a/.travis.yml b/.travis.yml index ee2c85e99..16cd21347 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,36 @@ matrix: include: + # python manylinux packages + + - name: "manylinux1_x86_64 cp36-cp36m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp36-cp36m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp36-cp36m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp36-cp36m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + # python 2 osx - name: "klayout python2 osx10.13" @@ -288,7 +319,19 @@ before_install: fi - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" -script: +install: + - if [ "$DOCKER_BUILD" = true ]; then + docker pull $DOCKER_IMAGE; + fi + +script: + - if [ "$DOCKER_BUILD" = true ]; then + mkdir -p ccache; + mkdir -p wheelhouse; + docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; + mkdir -p deploy/dist-pymod; + cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod; + fi - if [ "$PYTHON_BUILD" = true ]; then python setup.py build; python setup.py bdist_wheel; diff --git a/ci-scripts/docker/Dockerfile.i686 b/ci-scripts/docker/Dockerfile.i686 index f73ca3ac1..7d68054fc 100644 --- a/ci-scripts/docker/Dockerfile.i686 +++ b/ci-scripts/docker/Dockerfile.i686 @@ -10,7 +10,6 @@ RUN ln -s /usr/bin/ccache /usr/lib/ccache/gcc RUN ln -s /usr/bin/ccache /usr/lib/ccache/g++ # Add ccache to PATH -ENV PATH="/usr/lib/ccache:${PATH}" RUN mkdir -p /persist/.ccache ENV CCACHE_DIR="/persist/.ccache" diff --git a/ci-scripts/docker/Dockerfile.x86_64 b/ci-scripts/docker/Dockerfile.x86_64 index df4f05dc5..4bf03c3cd 100644 --- a/ci-scripts/docker/Dockerfile.x86_64 +++ b/ci-scripts/docker/Dockerfile.x86_64 @@ -10,7 +10,6 @@ RUN ln -s /usr/bin/ccache /usr/lib64/ccache/gcc RUN ln -s /usr/bin/ccache /usr/lib64/ccache/g++ # Add ccache to PATH -ENV PATH="/usr/lib64/ccache:${PATH}" RUN mkdir -p /persist/.ccache ENV CCACHE_DIR="/persist/.ccache" diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 5ec87ee41..48b5a4dbd 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -1,4 +1,5 @@ -# Step 1. +# Chapter 1. Testing on your own computer +## Step 1. Make sure you have the quay.io/pypa/manylinux1_x86_64 image. @@ -19,7 +20,7 @@ Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest ``` -# Step 2. +## Step 2. This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 Create a volume for klayout. This is necessary because docker containers don't persist data. @@ -40,7 +41,7 @@ $ docker volume inspect klayout-persist ] ``` -# Step 3. +## Step 3. Build image `myimage` with: @@ -56,7 +57,7 @@ Then I run the docker with a terminal shell and load the volume klayout-persist $ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage ``` -# Step 4. +## Step 4. In the shell, pull master from klayout. @@ -102,7 +103,7 @@ cd /persist/klayout Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. -# Step 5. Iterate over all python versions. +## Step 5. Iterate over all python versions. For that, we need something like: @@ -135,3 +136,77 @@ for PYBIN in /opt/python/*/bin/; do I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: `ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` + +I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. +The ccache folder after a single build has 657MB. Go figure. + +# Chapter 2. Testing CI flow with docker + +# Step 1. Testing commands in own computer + +First cloned with: +```bash +git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux +cd klayout +``` + +Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml + +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Total of 2x6 = 12 possibilities + +```bash +export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 +export PY_VERSION="cp36-cp36m" +docker pull $DOCKER_IMAGE +mkdir -p ccache +mkdir -p wheelhouse +docker run --name klayout -v `pwd` -it $DOCKER_IMAGE +``` + +Inside docker shell: +```bash +yum install -y zlib-devel +yum install -y zip + +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" + +# Compile wheel +/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/ + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +``` + +# Step 2. Automating step 1 in travis (CI). +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Build: spawn 12 travis jobs, one for each combination of word-size and python version. +Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`. +Post-build: +- cache `./ccache` +- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder) + +# Step 3. Automating deployment to PyPI: + +TBD diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh new file mode 100755 index 000000000..58dccef8b --- /dev/null +++ b/ci-scripts/docker/docker_build.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + + +if [[ -z $PY_VERSION ]]; then + echo '$PY_VERSION is not set' + exit 1 +fi + +if [[ -z $DOCKER_IMAGE ]]; then + echo '$DOCKER_IMAGE is not set' + exit 1 +fi + +echo PY_VERSION=$PY_VERSION +echo DOCKER_IMAGE=$DOCKER_IMAGE + +yum install -y zlib-devel ccache zip || exit 1 # sometimes the epel server is down. + +if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then + ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -s /usr/bin/ccache /usr/lib64/ccache/cc + ln -s /usr/bin/ccache /usr/lib64/ccache/gcc + ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then + ln -s /usr/bin/ccache /usr/lib/ccache/c++ + ln -s /usr/bin/ccache /usr/lib/ccache/cc + ln -s /usr/bin/ccache /usr/lib/ccache/gcc + ln -s /usr/bin/ccache /usr/lib/ccache/g++ +fi +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" +cd /io + +# Compile wheel +"/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ || exit 1 +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + /io/ci-scripts/docker/fix_wheel.sh "$whl" || exit 1 +done + +# Install packages and test +TEST_HOME=/io/testdata +"/opt/python/$PY_VERSION/bin/pip" install klayout --no-index -f /io/wheelhouse || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_db.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_rdb.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_tl.py || exit 1 diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/manylinux-docker.sh index a4b958853..7591d0181 100644 --- a/ci-scripts/docker/manylinux-docker.sh +++ b/ci-scripts/docker/manylinux-docker.sh @@ -11,7 +11,7 @@ ln -s /usr/bin/ccache /usr/lib64/ccache/c++ ln -s /usr/bin/ccache /usr/lib64/ccache/cc ln -s /usr/bin/ccache /usr/lib64/ccache/gcc ln -s /usr/bin/ccache /usr/lib64/ccache/g++ -export PATH=/usr/lib64/ccache:$PATH +# export PATH=/usr/lib64/ccache:$PATH # unnecessary # Compile wheels for PYBIN in /opt/python/*/bin; do From b6d7dec289be4d161f03b884caf1bfa43c691fe0 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:05:10 -0500 Subject: [PATCH 15/52] showing ccache stats --- ci-scripts/docker/docker_build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 58dccef8b..fd3cff7a2 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -35,6 +35,9 @@ cd /io # Compile wheel "/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 +# Show ccache stats +ccache -s + # Bundle external shared libraries into the wheels via auditwheel for whl in /io/wheelhouse/*linux_*.whl; do auditwheel repair "$whl" -w /io/wheelhouse/ || exit 1 From a931db224c45c5de0d1c0ee77d3dca11bf73909f Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:26:50 -0500 Subject: [PATCH 16/52] adding other versions of python to manylinux CI --- .travis.yml | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/.travis.yml b/.travis.yml index 16cd21347..fda9b403d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,34 @@ matrix: include: # python manylinux packages + - name: "manylinux1_x86_64 cp37-cp37m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp37-cp37m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp37-cp37m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp37-cp37m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + - name: "manylinux1_x86_64 cp36-cp36m package" os: linux sudo: true @@ -30,6 +58,118 @@ matrix: directories: - ccache + - name: "manylinux1_x86_64 cp35-cp35m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp35-cp35m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp35-cp35m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp35-cp35m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_x86_64 cp34-cp34m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp34-cp34m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp34-cp34m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp34-cp34m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_x86_64 cp27-cp27mu package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp27-cp27mu" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp27-cp27mu package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp27-cp27mu" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_x86_64 cp27-cp27m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + - PY_VERSION="cp27-cp27m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + + - name: "manylinux1_i686 cp27-cp27m package" + os: linux + sudo: true + services: + - docker + env: + - DOCKER_IMAGE="quay.io/pypa/manylinux1_i686" + - PY_VERSION="cp27-cp27m" + - DOCKER_BUILD=true + - MATRIX_EVAL="" + cache: + directories: + - ccache + # python 2 osx From 0848b940ced9a927368642e7336aa75d132d916a Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:27:44 -0500 Subject: [PATCH 17/52] better organization of dist-pymod folder --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fda9b403d..4fe284fb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -469,16 +469,18 @@ script: mkdir -p ccache; mkdir -p wheelhouse; docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; - mkdir -p deploy/dist-pymod; - cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod; + klayout_version=$(python -c 'import setup; print(setup.Config().version())'); + mkdir -p deploy/dist-pymod/$klayout_version; + cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod/$klayout_version; fi - if [ "$PYTHON_BUILD" = true ]; then python setup.py build; python setup.py bdist_wheel; python setup.py install; python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; - mkdir -p deploy/dist-pymod; - cp -a dist/* deploy/dist-pymod/; + klayout_version=$(python -c 'import setup; print(setup.Config().version())'); + mkdir -p deploy/dist-pymod/$klayout_version; + cp -a dist/* deploy/dist-pymod/$klayout_version; python -c 'import klayout.db as db; print(dir(db))'; python -c 'import klayout.rdb as rdb; print(dir(rdb))'; python -c 'import klayout.tl as tl; print(dir(tl))'; From f1e292373f0736be2643a88f12e3931cd18fe290 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:28:05 -0500 Subject: [PATCH 18/52] Changing version to 0.26.0.dev9 for CD tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 545126651..06f0fbeb5 100644 --- a/setup.py +++ b/setup.py @@ -322,7 +322,7 @@ class Config(object): """ Gets the version string """ - return "0.26.0.dev8" + return "0.26.0.dev9" config = Config() From e3de4030df037b42b4ae7a26aa03110882c1149e Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 18:38:51 -0500 Subject: [PATCH 19/52] http://mirrors.fedoraproject.org is unreliable. Retry yum 5 times. --- ci-scripts/docker/docker_build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index fd3cff7a2..235335161 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -14,7 +14,12 @@ fi echo PY_VERSION=$PY_VERSION echo DOCKER_IMAGE=$DOCKER_IMAGE -yum install -y zlib-devel ccache zip || exit 1 # sometimes the epel server is down. +# sometimes the epel server is down. retry 5 times +for i in $(seq 1 5); do + yum install -y zlib-devel ccache zip && s=0 && break || s=$? && sleep 15; +done + +[ $s -eq 0 ] || exit $s if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then ln -s /usr/bin/ccache /usr/lib64/ccache/c++ From a220e2e0e05f3925cd6b3892bef0dad7d0089b2c Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 19:28:28 -0500 Subject: [PATCH 20/52] verbose auditwheel repair --- ci-scripts/docker/docker_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 235335161..6247fbb7e 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -45,7 +45,7 @@ ccache -s # Bundle external shared libraries into the wheels via auditwheel for whl in /io/wheelhouse/*linux_*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ || exit 1 + auditwheel -v repair "$whl" -w /io/wheelhouse/ || exit 1 done # Fix each wheel generated by auditwheel From 36af902e3fcdab55d3cc52ada409d57031580523 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 18 Dec 2018 19:45:11 -0500 Subject: [PATCH 21/52] bugfix: ccache not in path. --- ci-scripts/docker/docker_build.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 6247fbb7e..8acd4a16c 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -26,21 +26,28 @@ if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_x86_64" ]]; then ln -s /usr/bin/ccache /usr/lib64/ccache/cc ln -s /usr/bin/ccache /usr/lib64/ccache/gcc ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + export PATH="/usr/lib64/ccache/:$PATH" elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux1_i686" ]]; then ln -s /usr/bin/ccache /usr/lib/ccache/c++ ln -s /usr/bin/ccache /usr/lib/ccache/cc ln -s /usr/bin/ccache /usr/lib/ccache/gcc ln -s /usr/bin/ccache /usr/lib/ccache/g++ + export PATH="/usr/lib/ccache/:$PATH" fi echo $PATH -# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin export CCACHE_DIR="/io/ccache" + +# Show ccache stats +echo "Cache stats:" +ccache -s + cd /io # Compile wheel "/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 # Show ccache stats +echo "Cache stats:" ccache -s # Bundle external shared libraries into the wheels via auditwheel From a288b3ce398539b2404aa931500c99ece9de8501 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 19 Dec 2018 18:30:46 -0500 Subject: [PATCH 22/52] Patched auditwheel directly. Should fix manylinux python build. --- ci-scripts/docker/README.md | 4 ++++ ci-scripts/docker/docker_build.sh | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 48b5a4dbd..9cf7370a2 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -140,6 +140,10 @@ I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/many I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. The ccache folder after a single build has 657MB. Go figure. +I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1 + +So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default. + # Chapter 2. Testing CI flow with docker # Step 1. Testing commands in own computer diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index 8acd4a16c..b511f879f 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -16,7 +16,7 @@ echo DOCKER_IMAGE=$DOCKER_IMAGE # sometimes the epel server is down. retry 5 times for i in $(seq 1 5); do - yum install -y zlib-devel ccache zip && s=0 && break || s=$? && sleep 15; + yum install -y zlib-devel ccache zip git && s=0 && break || s=$? && sleep 15; done [ $s -eq 0 ] || exit $s @@ -37,13 +37,21 @@ fi echo $PATH export CCACHE_DIR="/io/ccache" +# Download proper auditwheel program +git clone https://github.com/thomaslima/auditwheel.git /tmp/auditwheel +cd /tmp/auditwheel +git checkout 87f5306ec02cc68020afaa9933543c898b1d47c1 # patched version +AUDITWHEEL_PYTHON=$(cat `which auditwheel` | head -1 | sed -e 's/#!\(.*\)/\1/') +# Install auditwheel, replacing the system's auditwheel binary +$AUDITWHEEL_PYTHON -m pip install . + + # Show ccache stats echo "Cache stats:" ccache -s -cd /io - # Compile wheel +cd /io "/opt/python/$PY_VERSION/bin/python" setup.py bdist_wheel -d /io/wheelhouse/ || exit 1 # Show ccache stats @@ -55,11 +63,6 @@ for whl in /io/wheelhouse/*linux_*.whl; do auditwheel -v repair "$whl" -w /io/wheelhouse/ || exit 1 done -# Fix each wheel generated by auditwheel -for whl in /io/wheelhouse/*manylinux1_*.whl; do - /io/ci-scripts/docker/fix_wheel.sh "$whl" || exit 1 -done - # Install packages and test TEST_HOME=/io/testdata "/opt/python/$PY_VERSION/bin/pip" install klayout --no-index -f /io/wheelhouse || exit 1 From 6b37e67de68d758db060a2fbe20ae0ca832c353c Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 10:59:25 -0500 Subject: [PATCH 23/52] fixing pya module. adding tests --- .travis.yml | 1 + ci-scripts/docker/docker_build.sh | 2 ++ src/pymod/__init__.py.qtless | 3 +-- src/pymod/distutils_src/pya/__init__.py | 15 ++++++--------- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fe284fb0..4dcb2f7ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -478,6 +478,7 @@ script: python setup.py bdist_wheel; python setup.py install; python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; + python testdata/pymod/pya_tests.py; klayout_version=$(python -c 'import setup; print(setup.Config().version())'); mkdir -p deploy/dist-pymod/$klayout_version; cp -a dist/* deploy/dist-pymod/$klayout_version; diff --git a/ci-scripts/docker/docker_build.sh b/ci-scripts/docker/docker_build.sh index b511f879f..2f6d68710 100755 --- a/ci-scripts/docker/docker_build.sh +++ b/ci-scripts/docker/docker_build.sh @@ -69,3 +69,5 @@ TEST_HOME=/io/testdata "/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_db.py || exit 1 "/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_rdb.py || exit 1 "/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/import_tl.py || exit 1 +"/opt/python/$PY_VERSION/bin/python" $TEST_HOME/pymod/pya_tests.py || exit 1 + diff --git a/src/pymod/__init__.py.qtless b/src/pymod/__init__.py.qtless index dce96f8e4..6aef40aa9 100644 --- a/src/pymod/__init__.py.qtless +++ b/src/pymod/__init__.py.qtless @@ -1,5 +1,4 @@ - # klayout library definition file -__all__ = [ "tl", "db", "lay", "rdb" ] +__all__ = [ "tl", "db", "rdb" ] diff --git a/src/pymod/distutils_src/pya/__init__.py b/src/pymod/distutils_src/pya/__init__.py index ae97d592a..70e2f49b9 100644 --- a/src/pymod/distutils_src/pya/__init__.py +++ b/src/pymod/distutils_src/pya/__init__.py @@ -1,10 +1,7 @@ +# import all packages from klayout, such as klayout.db and klayout.tl +# WARNING: doing it manually until it becomes impractical +# TODO: We need a specification document explaining what should go into pya -import klayout -import importlib - -__all__ = [] -for m in klayout.__all__: - mod = importlib.import_module("klayout." + m) - for mm in mod.__all__: - globals()[mm] = getattr(mod, mm) - +from klayout.db import * # noqa +from klayout.tl import * # noqa +from klayout.rdb import * # noqa From d20e12ef415d75afdc1bffe4858681534b27f232 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 14:08:30 -0500 Subject: [PATCH 24/52] Adding tests on host linux beyond docker. --- .travis.yml | 191 +++++++++++++++++++++++++++++----------------------- 1 file changed, 106 insertions(+), 85 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4dcb2f7ed..373f5e71e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,14 @@ matrix: - name: "manylinux1_x86_64 cp37-cp37m package" os: linux sudo: true + python: '3.7-dev' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp37-cp37m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -33,12 +35,14 @@ matrix: - name: "manylinux1_x86_64 cp36-cp36m package" os: linux sudo: true + python: '3.6' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp36-cp36m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -61,12 +65,14 @@ matrix: - name: "manylinux1_x86_64 cp35-cp35m package" os: linux sudo: true + python: '3.5' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp35-cp35m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -89,12 +95,14 @@ matrix: - name: "manylinux1_x86_64 cp34-cp34m package" os: linux sudo: true + python: '3.4' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp34-cp34m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -117,12 +125,14 @@ matrix: - name: "manylinux1_x86_64 cp27-cp27mu package" os: linux sudo: true + python: '2.7' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp27-cp27mu" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -145,12 +155,14 @@ matrix: - name: "manylinux1_x86_64 cp27-cp27m package" os: linux sudo: true + python: '2.7' services: - docker env: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp27-cp27m" - DOCKER_BUILD=true + - TEST_IN_HOST=true - MATRIX_EVAL="" cache: directories: @@ -266,96 +278,96 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.7 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.7-dev' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.7 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.7-dev' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.6 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.6' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.6 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.6' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python2.7 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '2.7' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python2.7 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '2.7' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python2.6 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '2.6' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python2.6 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '2.6' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.3 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.3' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.3 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.3' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.4 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.4' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.4 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.4' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ - - name: "klayout python3.5 package" - os: linux - dist: trusty # Ubuntu 14.04 - sudo: false - language: python - python: '3.5' - env: - - MATRIX_EVAL="" - - PIP_UPDATE="1" - - PYTHON_BUILD=true - - CC=clang - - CXX=clang++ + # - name: "klayout python3.5 package" + # os: linux + # dist: trusty # Ubuntu 14.04 + # sudo: false + # language: python + # python: '3.5' + # env: + # - MATRIX_EVAL="" + # - PIP_UPDATE="1" + # - PYTHON_BUILD=true + # - CC=clang + # - CXX=clang++ # KLayout builds for mac # Python 3 @@ -472,12 +484,21 @@ script: klayout_version=$(python -c 'import setup; print(setup.Config().version())'); mkdir -p deploy/dist-pymod/$klayout_version; cp -a wheelhouse/klayout-*manylinux1*.whl deploy/dist-pymod/$klayout_version; + if [ "$TEST_IN_HOST" = true ]; then + pip install klayout --no-index -f ./wheelhouse; + python testdata/pymod/import_db.py; + python testdata/pymod/import_rdb.py; + python testdata/pymod/import_tl.py; + python testdata/pymod/pya_tests.py; + fi fi - if [ "$PYTHON_BUILD" = true ]; then python setup.py build; python setup.py bdist_wheel; python setup.py install; - python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; + python testdata/pymod/import_db.py; + python testdata/pymod/import_rdb.py; + python testdata/pymod/import_tl.py; python testdata/pymod/pya_tests.py; klayout_version=$(python -c 'import setup; print(setup.Config().version())'); mkdir -p deploy/dist-pymod/$klayout_version; From 5f66df13f7ae41e56884d9b7f93bb5f730be3d08 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 14:09:56 -0500 Subject: [PATCH 25/52] adding ccache to macos builds --- .travis.yml | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.travis.yml b/.travis.yml index 373f5e71e..6dfb31bd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -188,6 +188,11 @@ matrix: - name: "klayout python2 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="" - ARCHFLAGS="-std=c++11" @@ -197,6 +202,11 @@ matrix: - name: "klayout python2 osx10.12" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew install python2 || brew link --overwrite python@2" # deficient python2 in travis's xcode8.3 (no ssl) - ARCHFLAGS="-std=c++11" @@ -206,6 +216,11 @@ matrix: - name: "klayout python2 osx10.11" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="" - ARCHFLAGS="-std=c++11" @@ -217,10 +232,12 @@ matrix: - name: "klayout python3 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - python3 + - ccache update: true env: - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" @@ -231,6 +248,11 @@ matrix: - name: "klayout python3.6.6 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python36; brew link --force --overwrite python36; shopt -s expand_aliases; alias python='/usr/local/opt/python36/bin/python3.6'; alias pip='/usr/local/opt/python36/bin/pip3.6';" - ARCHFLAGS="-std=c++11" @@ -240,6 +262,11 @@ matrix: - name: "klayout python3.5.6 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python35; brew link --force --overwrite python35; shopt -s expand_aliases; alias python='/usr/local/opt/python35/bin/python3.5'; alias pip='/usr/local/opt/python35/bin/pip3.5';" - ARCHFLAGS="-std=c++11" @@ -249,6 +276,11 @@ matrix: - name: "klayout python3.4.9 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';" - ARCHFLAGS="-std=c++11" @@ -258,10 +290,12 @@ matrix: - name: "klayout python3 osx10.12" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - python3 + - ccache update: true env: - MATRIX_EVAL="shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" @@ -272,6 +306,11 @@ matrix: - name: "klayout python3 osx10.11" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache + addons: + homebrew: + packages: + - ccache env: - MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';" - ARCHFLAGS="-std=c++11" @@ -374,11 +413,13 @@ matrix: - name: "KLayout macOS 10.13 with py3.7" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -389,11 +430,13 @@ matrix: - name: "KLayout macOS 10.12 with py3.7" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -404,11 +447,13 @@ matrix: - name: "KLayout macOS 10.11 with py3.7" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache addons: homebrew: packages: - python3 - qt + - ccache update: true env: - MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8 @@ -420,10 +465,12 @@ matrix: - name: "KLayout macOS 10.13 with py2.7" os: osx osx_image: xcode9.4 # macOS 10.13 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -434,10 +481,12 @@ matrix: - name: "KLayout macOS 10.12 with py2.7" os: osx osx_image: xcode8.3 # macOS 10.12 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="" @@ -448,10 +497,12 @@ matrix: - name: "KLayout macOS 10.11 with py2.7" os: osx osx_image: xcode8 # macOS 10.11 + cache: ccache addons: homebrew: packages: - qt + - ccache update: true env: - MATRIX_EVAL="brew update; brew install qt" # homebrew addon fails for xcode8 @@ -470,6 +521,9 @@ before_install: pip install --upgrade setuptools wheel || sudo pip install --upgrade setuptools wheel; fi - python -c "import distutils.sysconfig as sysconfig; print(sysconfig.__file__)" + - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + export PATH="/usr/local/opt/ccache/libexec:$PATH"; + fi install: - if [ "$DOCKER_BUILD" = true ]; then From 8495a18023607aa6b3592d838dab09171cef2c1d Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 13:39:25 -0500 Subject: [PATCH 26/52] changing version to 0.26.0.dev10 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 06f0fbeb5..059f4b44c 100644 --- a/setup.py +++ b/setup.py @@ -322,7 +322,7 @@ class Config(object): """ Gets the version string """ - return "0.26.0.dev9" + return "0.26.0.dev10" config = Config() From 124975d63627c8fe34f7d853fecb84b68e2df665 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 15:03:42 -0500 Subject: [PATCH 27/52] bug: forgot to add language:python in travis --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6dfb31bd9..80fe85300 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ matrix: - name: "manylinux1_x86_64 cp37-cp37m package" os: linux sudo: true + language: python python: '3.7-dev' services: - docker @@ -35,6 +36,7 @@ matrix: - name: "manylinux1_x86_64 cp36-cp36m package" os: linux sudo: true + language: python python: '3.6' services: - docker @@ -65,6 +67,7 @@ matrix: - name: "manylinux1_x86_64 cp35-cp35m package" os: linux sudo: true + language: python python: '3.5' services: - docker @@ -95,6 +98,7 @@ matrix: - name: "manylinux1_x86_64 cp34-cp34m package" os: linux sudo: true + language: python python: '3.4' services: - docker @@ -125,6 +129,7 @@ matrix: - name: "manylinux1_x86_64 cp27-cp27mu package" os: linux sudo: true + language: python python: '2.7' services: - docker @@ -155,6 +160,7 @@ matrix: - name: "manylinux1_x86_64 cp27-cp27m package" os: linux sudo: true + language: python python: '2.7' services: - docker From b3addbeeba84284dfa70a5ffdb2a056dc9bf815c Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 15:19:30 -0500 Subject: [PATCH 28/52] better organizing travis metadata. --- .travis.yml | 53 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 80fe85300..29f694ef2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ matrix: include: # python manylinux packages - - name: "manylinux1_x86_64 cp37-cp37m package" + - name: "cp37-cp37m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -19,7 +19,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp37-cp37m package" + - name: "cp37-cp37m-manylinux1_i686.whl" os: linux sudo: true services: @@ -33,7 +33,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp36-cp36m package" + - name: "cp36-cp36m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -50,7 +50,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp36-cp36m package" + - name: "cp36-cp36m-manylinux1_i686.whl" os: linux sudo: true services: @@ -64,7 +64,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp35-cp35m package" + - name: "cp35-cp35m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -81,7 +81,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp35-cp35m package" + - name: "cp35-cp35m-manylinux1_i686.whl" os: linux sudo: true services: @@ -95,7 +95,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp34-cp34m package" + - name: "cp34-cp34m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -112,7 +112,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp34-cp34m package" + - name: "cp34-cp34m-manylinux1_i686.whl" os: linux sudo: true services: @@ -126,7 +126,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp27-cp27mu package" + - name: "cp27-cp27mu-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -143,7 +143,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp27-cp27mu package" + - name: "cp27-cp27mu-manylinux1_i686.whl" os: linux sudo: true services: @@ -157,7 +157,7 @@ matrix: directories: - ccache - - name: "manylinux1_x86_64 cp27-cp27m package" + - name: "cp27-cp27m-manylinux1_x86_64.whl" os: linux sudo: true language: python @@ -174,7 +174,7 @@ matrix: directories: - ccache - - name: "manylinux1_i686 cp27-cp27m package" + - name: "cp27-cp27m-manylinux1_i686.whl" os: linux sudo: true services: @@ -191,7 +191,8 @@ matrix: # python 2 osx - - name: "klayout python2 osx10.13" + # - name: "klayout python2 osx10.13" + - name: "cp27-cp27m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -205,7 +206,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python2 osx10.12" + # - name: "klayout python2 osx10.12" + - name: "cp27-cp27m-macosx_10_12_x86_64.whl" os: osx osx_image: xcode8.3 # macOS 10.12 cache: ccache @@ -219,7 +221,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python2 osx10.11" + # - name: "klayout python2 osx10.11" + - name: "cp27-cp27m-macosx_10_11_x86_64.whl" os: osx osx_image: xcode8 # macOS 10.11 cache: ccache @@ -235,7 +238,8 @@ matrix: # python 3 osx - - name: "klayout python3 osx10.13" + # - name: "klayout python3 osx10.13" + - name: "cp37-cp37m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -251,7 +255,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.6.6 osx10.13" + # - name: "klayout python3.6.6 osx10.13" + - name: "cp36-cp36m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -265,7 +270,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.5.6 osx10.13" + # - name: "klayout python3.5.6 osx10.13" + - name: "cp35-cp35m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -279,7 +285,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3.4.9 osx10.13" + # - name: "klayout python3.4.9 osx10.13" + - name: "cp34-cp34m-macosx_10_13_x86_64.whl" os: osx osx_image: xcode9.4 # macOS 10.13 cache: ccache @@ -293,7 +300,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3 osx10.12" + # - name: "klayout python3 osx10.12" + - name: "cp37-cp37m-macosx_10_12_x86_64.whl" os: osx osx_image: xcode8.3 # macOS 10.12 cache: ccache @@ -309,7 +317,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - - name: "klayout python3 osx10.11" + # - name: "klayout python3 osx10.11" + - name: "cp37-cp37m-macosx_10_11_x86_64.whl" os: osx osx_image: xcode8 # macOS 10.11 cache: ccache @@ -562,7 +571,7 @@ script: python testdata/pymod/pya_tests.py; klayout_version=$(python -c 'import setup; print(setup.Config().version())'); mkdir -p deploy/dist-pymod/$klayout_version; - cp -a dist/* deploy/dist-pymod/$klayout_version; + cp -a dist/*.whl deploy/dist-pymod/$klayout_version; python -c 'import klayout.db as db; print(dir(db))'; python -c 'import klayout.rdb as rdb; print(dir(rdb))'; python -c 'import klayout.tl as tl; print(dir(tl))'; From 2975d57d22a46fd4434a22aefba26c27cebd10b0 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 15:50:57 -0500 Subject: [PATCH 29/52] Not testing cp27-cp27m in travis's host (incompatible). --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29f694ef2..bdcd00179 100644 --- a/.travis.yml +++ b/.travis.yml @@ -168,7 +168,7 @@ matrix: - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" - PY_VERSION="cp27-cp27m" - DOCKER_BUILD=true - - TEST_IN_HOST=true + - TEST_IN_HOST=false # travis's python 2.7 uses ucs4 (mu), so this test fails. - MATRIX_EVAL="" cache: directories: From 1bba72e45bdfd95a28235af0e515b5917c8c9277 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 16:01:15 -0500 Subject: [PATCH 30/52] Adding documentation regarding manylinux, CI, and PyPI wheels --- ci-scripts/docker/README.md | 220 ++--------------- .../{ => development_notes}/Dockerfile.i686 | 0 .../{ => development_notes}/Dockerfile.x86_64 | 0 ci-scripts/docker/development_notes/README.md | 221 ++++++++++++++++++ .../{ => development_notes}/fix_wheel.sh | 0 .../manylinux-docker.sh | 0 ci-scripts/twine/README.md | 15 ++ 7 files changed, 249 insertions(+), 207 deletions(-) rename ci-scripts/docker/{ => development_notes}/Dockerfile.i686 (100%) rename ci-scripts/docker/{ => development_notes}/Dockerfile.x86_64 (100%) create mode 100644 ci-scripts/docker/development_notes/README.md rename ci-scripts/docker/{ => development_notes}/fix_wheel.sh (100%) rename ci-scripts/docker/{ => development_notes}/manylinux-docker.sh (100%) create mode 100644 ci-scripts/twine/README.md diff --git a/ci-scripts/docker/README.md b/ci-scripts/docker/README.md index 9cf7370a2..753d2fee3 100644 --- a/ci-scripts/docker/README.md +++ b/ci-scripts/docker/README.md @@ -1,216 +1,22 @@ -# Chapter 1. Testing on your own computer -## Step 1. +Author: Thomas Ferreira de Lima +email: thomas@tlima.me -Make sure you have the quay.io/pypa/manylinux1_x86_64 image. +This folder contains scripts to be run inside docker images. See instructions on how to test this yourself in ci-scripts/docker/development_notes. + +## docker_build.sh + +We need two environment variables to get going: ```bash -$ docker images -REPOSITORY TAG IMAGE ID CREATED SIZE -quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB -hello-world latest 4ab4c602aa5e 3 months ago 1.84kB -# My image was old: -$ docker pull quay.io/pypa/manylinux1_x86_64 -Using default tag: latest -latest: Pulling from pypa/manylinux1_x86_64 -7d0d9526f38a: Already exists -3324bfadf9cb: Pull complete -20f27c7e3062: Pull complete -5bc21fc5fe97: Pull complete -Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 -Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest +DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" +PY_VERSION="cp37-cp37m" ``` -## Step 2. - -This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 -Create a volume for klayout. This is necessary because docker containers don't persist data. +The script must be run inside an image pulled from $DOCKER_IMAGE and with klayout's git repo cloned in /io. Inside the git clone folder, run: ```bash -$ docker volume create klayout-persist -$ docker volume inspect klayout-persist -[ - { - "CreatedAt": "2018-12-18T15:01:48Z", - "Driver": "local", - "Labels": {}, - "Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data", - "Name": "klayout-persist", - "Options": {}, - "Scope": "local" - } -] +docker run --rm -e DOCKER_IMAGE -e PY_VERSION -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD "/io/ci-scripts/docker/docker_build.sh"; +# $PRE_CMD is empty for now (useless currently). ``` -## Step 3. - -Build image `myimage` with: - -```bash -$ docker build -t myimage:latest -f Dockerfile.x86_64 . -``` - -This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. - -Then I run the docker with a terminal shell and load the volume klayout-persist in /persist: - -```bash -$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage -``` - -## Step 4. - -In the shell, pull master from klayout. - -```bash -cd /persist -git clone https://github.com/lightwave-lab/klayout.git -mkdir -p wheelhouse -cd klayout -# make wheel with python 3.6 (for example) -/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/ -cd /persist -auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/ -# Need to manually fix the wheel -#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse -``` - -The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. - - -Procedure to fix the wheel: - -```bash -unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel -cd tempwheel/klayout -mv lib_* db_plugins .libs/ -cd ../klayout-0.26.0.dev8.dist-info/ -sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD -sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD -cd ../ -rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./* -cd .. -rm -rf tempwheel -``` - -Now we can install and test: -```bash -/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse -cd /persist/klayout -/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py -# Tests passed! -``` - -Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. - -## Step 5. Iterate over all python versions. - -For that, we need something like: - -```bash -# Compile wheels -for PYBIN in /opt/python/*/bin; do - "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ -done - -# Bundle external shared libraries into the wheels via auditwheel -for whl in /persist/wheelhouse/*linux_*.whl; do - auditwheel repair "$whl" -w /persist/wheelhouse/ -done - -# Fix each wheel generated by auditwheel -for whl in /persist/wheelhouse/*manylinux1_*.whl; do - ./ci-scripts/docker/fix_wheel.sh "$whl" -done - -# Install packages and test -TEST_HOME=/persist/klayout/testdata -for PYBIN in /opt/python/*/bin/; do - "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse - "${PYBIN}/python" $TEST_HOME/pymod/import_db.py - "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py - "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py - -``` - -I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: - -`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` - -I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. -The ccache folder after a single build has 657MB. Go figure. - -I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1 - -So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default. - -# Chapter 2. Testing CI flow with docker - -# Step 1. Testing commands in own computer - -First cloned with: -```bash -git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux -cd klayout -``` - -Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml - -DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 - -PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m - -Total of 2x6 = 12 possibilities - -```bash -export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 -export PY_VERSION="cp36-cp36m" -docker pull $DOCKER_IMAGE -mkdir -p ccache -mkdir -p wheelhouse -docker run --name klayout -v `pwd` -it $DOCKER_IMAGE -``` - -Inside docker shell: -```bash -yum install -y zlib-devel -yum install -y zip - -yum install -y ccache -ln -s /usr/bin/ccache /usr/lib64/ccache/c++ -ln -s /usr/bin/ccache /usr/lib64/ccache/cc -ln -s /usr/bin/ccache /usr/lib64/ccache/gcc -ln -s /usr/bin/ccache /usr/lib64/ccache/g++ -echo $PATH -# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -export CCACHE_DIR="/io/ccache" - -# Compile wheel -/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/ - -# Bundle external shared libraries into the wheels via auditwheel -for whl in /io/wheelhouse/*linux_*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ -done - -# Fix each wheel generated by auditwheel -for whl in /io/wheelhouse/*manylinux1_*.whl; do - ./ci-scripts/docker/fix_wheel.sh "$whl" -done - -``` - -# Step 2. Automating step 1 in travis (CI). -DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 - -PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m - -Build: spawn 12 travis jobs, one for each combination of word-size and python version. -Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`. -Post-build: -- cache `./ccache` -- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder) - -# Step 3. Automating deployment to PyPI: - -TBD +This command will generate a wheel and place it in `wheelhouse/klayout-*manylinux1*.whl`. This is the wheel that needs to be uploaded to PyPI via twine. See ci-scripts/twine/README.md. diff --git a/ci-scripts/docker/Dockerfile.i686 b/ci-scripts/docker/development_notes/Dockerfile.i686 similarity index 100% rename from ci-scripts/docker/Dockerfile.i686 rename to ci-scripts/docker/development_notes/Dockerfile.i686 diff --git a/ci-scripts/docker/Dockerfile.x86_64 b/ci-scripts/docker/development_notes/Dockerfile.x86_64 similarity index 100% rename from ci-scripts/docker/Dockerfile.x86_64 rename to ci-scripts/docker/development_notes/Dockerfile.x86_64 diff --git a/ci-scripts/docker/development_notes/README.md b/ci-scripts/docker/development_notes/README.md new file mode 100644 index 000000000..5a05684a5 --- /dev/null +++ b/ci-scripts/docker/development_notes/README.md @@ -0,0 +1,221 @@ +Author: Thomas Ferreira de Lima +email: thomas@tlima.me + +I wrote these notes as I was learning how to use docker and how to build python packages inside a docker image prepared by the pypa team. They require us to build there to allow wheels to have the `manylinux1` tag, meaning that these wheels would be compatible with most linux distributions around. Chapter 1 is about testing in my own computer (MacOS Mojave) and Chapter 2 is about how to automate this build using travis-ci.org. + +# Chapter 1. Testing on your own computer +## Step 1. + +Make sure you have the quay.io/pypa/manylinux1_x86_64 image. + +```bash +$ docker images +REPOSITORY TAG IMAGE ID CREATED SIZE +quay.io/pypa/manylinux1_x86_64 latest 1c8429c548f2 2 months ago 879MB +hello-world latest 4ab4c602aa5e 3 months ago 1.84kB +# My image was old: +$ docker pull quay.io/pypa/manylinux1_x86_64 +Using default tag: latest +latest: Pulling from pypa/manylinux1_x86_64 +7d0d9526f38a: Already exists +3324bfadf9cb: Pull complete +20f27c7e3062: Pull complete +5bc21fc5fe97: Pull complete +Digest: sha256:a13b2719fb21daebfe25c0173d80f8a85a2326dd994510d7879676e7a2193500 +Status: Downloaded newer image for quay.io/pypa/manylinux1_x86_64:latest +``` + +## Step 2. + +This step was inspired by https://dev.to/jibinliu/how-to-persist-data-in-docker-container-2m72 +Create a volume for klayout. This is necessary because docker containers don't persist data. + +```bash +$ docker volume create klayout-persist +$ docker volume inspect klayout-persist +[ + { + "CreatedAt": "2018-12-18T15:01:48Z", + "Driver": "local", + "Labels": {}, + "Mountpoint": "/var/lib/docker/volumes/klayout-persist/_data", + "Name": "klayout-persist", + "Options": {}, + "Scope": "local" + } +] +``` + +## Step 3. + +Build image `myimage` with: + +```bash +$ docker build -t myimage:latest -f Dockerfile.x86_64 . +``` + +This creates an image called `myimage` (temporary). This image will not overwrite old ones. Tip: prune old, unused images with `docker image prune`. + +Then I run the docker with a terminal shell and load the volume klayout-persist in /persist: + +```bash +$ docker run --name klayout --mount source=klayout-persist,target=/persist -it myimage +``` + +## Step 4. + +In the shell, pull master from klayout. + +```bash +cd /persist +git clone https://github.com/lightwave-lab/klayout.git +mkdir -p wheelhouse +cd klayout +# make wheel with python 3.6 (for example) +/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d /persist/wheelhouse/ +cd /persist +auditwheel repair "wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-linux_x86_64.whl" -w wheelhouse/ +# Need to manually fix the wheel +#/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /wheelhouse +``` + +The produced wheel from auditwheel, klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl, is defective in the following way: dbcore.so etc. have RPATHs reset to `$ORIGIN/.libs`, so we need to move all .so's `lib_*` into `.libs`, as well as `db_plugins`. We also need to change the dist-info/RECORD file paths. This is a bug from auditwheel, it should either have added a new RPATH, $ORIGIN/.libs, where it places libz, libcurl, libexpat, instead of renaming the existing ones, or moved the files to the right place. + + +Procedure to fix the wheel: + +```bash +unzip wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl -d tempwheel +cd tempwheel/klayout +mv lib_* db_plugins .libs/ +cd ../klayout-0.26.0.dev8.dist-info/ +sed -i 's/^klayout\/lib_/klayout\/.libs\/lib_/g' RECORD +sed -i 's/^klayout\/db_plugins/klayout\/.libs\/db_plugins/g' RECORD +cd ../ +rm -f ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl +zip -r ../wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl ./* +cd .. +rm -rf tempwheel +``` + +Now we can install and test: +```bash +/opt/python/cp36-cp36m/bin/pip install klayout --no-index -f /persist/wheelhouse +cd /persist/klayout +/opt/python/cp36-cp36m/bin/python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py +# Tests passed! +``` + +Encoded this behavior in a script called fix_wheel.sh. now you only need to run `./fix_wheel.sh wheelhouse/klayout-0.26.0.dev8-cp36-cp36m-manylinux1_x86_64.whl`, and it will overwrite the wheel. + +## Step 5. Iterate over all python versions. + +For that, we need something like: + +```bash +# Compile wheels +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/python" setup.py bdist_wheel -d /persist/wheelhouse/ +done + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /persist/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /persist/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /persist/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +# Install packages and test +TEST_HOME=/persist/klayout/testdata +for PYBIN in /opt/python/*/bin/; do + "${PYBIN}/pip" install klayout --no-index -f /persist/wheelhouse + "${PYBIN}/python" $TEST_HOME/pymod/import_db.py + "${PYBIN}/python" $TEST_HOME/pymod/import_rdb.py + "${PYBIN}/python" $TEST_HOME/pymod/import_tl.py + +``` + +I tested step 1-5 with both quay.io/pypa/manylinux1_x86_64 and quay.io/pypa/manylinux1_i686. So far the only failure was with `cp27-cp27mu` which gave this import error: + +`ImportError: /opt/python/cp27-cp27mu/lib/python2.7/site-packages/klayout/.libs/lib_pya.so: undefined symbol: PyUnicodeUCS2_AsUTF8String` + +I noticed that the ccache folder ended up with 800MB. I was hoping that the gcc compilation could reuse a lot of previously built objects but that didn't happen. I think that's because each python comes with its own header. So going forward it doesn't make sense to create a docker image for every python version. I will just cache a ccache folder via travis. +The ccache folder after a single build has 657MB. Go figure. + +I discovered that fix_wheel script was actually not properly working. So instead I looked into fixing `auditwheel` directly. Here's the commit that fixes it: https://github.com/thomaslima/auditwheel/tree/87f5306ec02cc68020afaa9933543c898b1d47c1 + +So now the plan is to change the `docker_build.sh` script so it uses the proper auditwheel, instead of their default. + +# Chapter 2. Testing CI flow with docker + +# Step 1. Testing commands in own computer + +First cloned with: +```bash +git clone git@github.com:lightwave-lab/klayout.git -b tmp/manylinux +cd klayout +``` + +Let's work with a few environment variables, like in https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml + +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Total of 2x6 = 12 possibilities + +```bash +export DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 +export PY_VERSION="cp36-cp36m" +docker pull $DOCKER_IMAGE +mkdir -p ccache +mkdir -p wheelhouse +docker run --name klayout -v `pwd` -it $DOCKER_IMAGE +``` + +Inside docker shell: +```bash +yum install -y zlib-devel +yum install -y zip + +yum install -y ccache +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +echo $PATH +# /usr/lib64/ccache:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export CCACHE_DIR="/io/ccache" + +# Compile wheel +/opt/python/$PY_VERSION/bin/python setup.py bdist_wheel -d /io/wheelhouse/ + +# Bundle external shared libraries into the wheels via auditwheel +for whl in /io/wheelhouse/*linux_*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + +# Fix each wheel generated by auditwheel +for whl in /io/wheelhouse/*manylinux1_*.whl; do + ./ci-scripts/docker/fix_wheel.sh "$whl" +done + +``` + +# Step 2. Automating step 1 in travis (CI). +DOCKER_IMAGE options: quay.io/pypa/manylinux1_x86_64 quay.io/pypa/manylinux1_i686 + +PY_VERSION options: cp27-cp27m cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m + +Build: spawn 12 travis jobs, one for each combination of word-size and python version. +Output: populated ./ccache with compiled objects and wheels inside ./wheelhouse/, one useless, `*linux_*.whl` and one useful `*manylinux1_*.whl`. +Post-build: +- cache `./ccache` +- deploy `./wheelhouse/*manylinux1_*.whl` to dropbox (./deploy folder) + +# Step 3. Automating deployment to PyPI: + +TBD diff --git a/ci-scripts/docker/fix_wheel.sh b/ci-scripts/docker/development_notes/fix_wheel.sh similarity index 100% rename from ci-scripts/docker/fix_wheel.sh rename to ci-scripts/docker/development_notes/fix_wheel.sh diff --git a/ci-scripts/docker/manylinux-docker.sh b/ci-scripts/docker/development_notes/manylinux-docker.sh similarity index 100% rename from ci-scripts/docker/manylinux-docker.sh rename to ci-scripts/docker/development_notes/manylinux-docker.sh diff --git a/ci-scripts/twine/README.md b/ci-scripts/twine/README.md new file mode 100644 index 000000000..3916de5d4 --- /dev/null +++ b/ci-scripts/twine/README.md @@ -0,0 +1,15 @@ +After building all the travis wheels, go to the folder where the wheels were deployed. In my case, for example, `/Users/tlima/Dropbox/Apps/travis-deploy/Builds/klayout/dist-pymod/0.26.0.dev10`. + +Then, run the command + +```bash +travis upload *.whl +``` + +, which will ask for an username and password related to your account. + +After this upload was successful, you can upload the source tarball. Go to klayout's git folder and run `python setup.py sdist`. Inside `dist/`, you'll find a tarball named, e.g., `klayout-0.26.0.dev10.tar.gz`. So just run + +```bash +travis upload klayout-0.26.0.dev10.tar.gz +``` From 971ad89f0b37c7d712aa033f065031349e211349 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 20 Dec 2018 22:38:01 -0500 Subject: [PATCH 31/52] brew qt 5.12 not supported in osx 10.11, reverting to qt 5.11.2 --- .travis.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 014d8c940..8232608ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -225,10 +225,9 @@ matrix: homebrew: packages: - python3 - - qt update: true env: - - MATRIX_EVAL="" + - MATRIX_EVAL="brew update; brew config; brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/56c500b569c724b049be7ab9e12d9693f85522f9/Formula/qt.rb" # Qt 5.11.2 - PYTHON_VERSION=B37 - MACOS_VERSION=ElCapitan - KLAYOUT_BUILD=true @@ -265,13 +264,8 @@ matrix: - name: "KLayout macOS 10.11 with py2.7" os: osx osx_image: xcode8 # macOS 10.11 - addons: - homebrew: - packages: - - qt - update: true env: - - MATRIX_EVAL="" + - MATRIX_EVAL="brew update; brew config; brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/56c500b569c724b049be7ab9e12d9693f85522f9/Formula/qt.rb" # Qt 5.11.2 - PYTHON_VERSION=Sys - MACOS_VERSION=ElCapitan - KLAYOUT_BUILD=true From fcc05d5b9d64e01b8f1b9490746d6d6235efa49e Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 27 Dec 2018 16:26:46 -0500 Subject: [PATCH 32/52] Bugfix: gsi method db Technology.load pointing to save --- src/db/db/gsiDeclDbTechnologies.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db/db/gsiDeclDbTechnologies.cc b/src/db/db/gsiDeclDbTechnologies.cc index e2d65e2f4..c4a54c7ee 100644 --- a/src/db/db/gsiDeclDbTechnologies.cc +++ b/src/db/db/gsiDeclDbTechnologies.cc @@ -256,7 +256,7 @@ gsi::Class technology_decl ("db", "Technology", gsi::method ("load", &db::Technology::load, gsi::arg ("file"), "@brief Loads the technology definition from a file\n" ) + - gsi::method ("load", &db::Technology::save, gsi::arg ("file"), + gsi::method ("save", &db::Technology::save, gsi::arg ("file"), "@brief Saves the technology definition to a file\n" ) + gsi::method ("technology_names", &technology_names, From 4ccbcb323b5fb4b62076d9cd628af6a678cb5b38 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 1 Jan 2019 21:59:41 -0500 Subject: [PATCH 33/52] First azure pipeline test. Downloading klayout bits --- azure-pipelines.yml | 47 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..671d7543f --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,47 @@ +# https://aka.ms/yaml +jobs: +- job: 'Test' + pool: + vmImage: 'vs2017-win2016' # other options: 'macOS-10.13', 'ubuntu-16.04' + strategy: + matrix: + Python27: + python.version: '2.7' + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + maxParallel: 3 + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + architecture: 'x64' + + # Add additional tasks to run using each Python version in the matrix above + + # PowerShell + # Run a PowerShell script on Windows, macOS, or Linux. + - task: PowerShell@2 + inputs: + targetType: 'inline' # Optional. Options: filePath, inline + #filePath: # Required when targetType == FilePath + #arguments: # Optional + script: | # Required when targetType == Inline + pwd + dir + Invoke-WebRequest -Uri "https://www.dropbox.com/s/ioax2mnic4987kn/klayout-microbits-1.0.zip?dl=1" -MaximumRedirection 2 -OutFile klayout-microbits-1.0.zip + Expand-Archive klayout-microbits-1.0.zip -DestinationPath klayout-microbits + dir klayout-microbits + #errorActionPreference: 'stop' # Optional. Options: stop, continue, silentlyContinue + #failOnStderr: false # Optional + #ignoreLASTEXITCODE: false # Optional + #pwsh: false # Optional + #workingDirectory: # Optional + displayName: 'Downloading and Extracting KLayout bits' + + - script: | + python -V + pwd + displayName: 'Diagnostic information' From 2cf52b54c1550577f24f704a96cad6278a9bbef5 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 1 Jan 2019 22:11:02 -0500 Subject: [PATCH 34/52] Building klayout python package --- azure-pipelines.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 671d7543f..255eb5fa1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,7 +41,13 @@ jobs: #workingDirectory: # Optional displayName: 'Downloading and Extracting KLayout bits' + - script: | + pip install --user --upgrade pip setuptools wheel + displayName: 'Updating pip, setuptools and wheel' + - script: | python -V - pwd - displayName: 'Diagnostic information' + set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\x64" + python setup.py install + + displayName: 'Building KLayout' From 05d9b4c851bc921c9600060721b7cac34751d62f Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 1 Jan 2019 22:40:33 -0500 Subject: [PATCH 35/52] Trying to build for py 2.7 with msvc++ 9.0 --- azure-pipelines.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 255eb5fa1..0c6f57476 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,16 +1,16 @@ # https://aka.ms/yaml jobs: -- job: 'Test' +- job: 'Build' pool: vmImage: 'vs2017-win2016' # other options: 'macOS-10.13', 'ubuntu-16.04' strategy: matrix: Python27: python.version: '2.7' - Python35: - python.version: '3.5' - Python36: - python.version: '3.6' + # Python35: + # python.version: '3.5' + # Python36: + # python.version: '3.6' maxParallel: 3 steps: @@ -26,7 +26,6 @@ jobs: - task: PowerShell@2 inputs: targetType: 'inline' # Optional. Options: filePath, inline - #filePath: # Required when targetType == FilePath #arguments: # Optional script: | # Required when targetType == Inline pwd @@ -41,6 +40,15 @@ jobs: #workingDirectory: # Optional displayName: 'Downloading and Extracting KLayout bits' + - script: | + curl https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -o VCForPython27.msi + msiexec /i VCForPython27.msi /quiet + set "VS90COMNTOOLS=C:\Users\VssAdministrator\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python\9.0\VC" + curl https://raw.githubusercontent.com/mattn/gntp-send/master/include/msinttypes/stdint.h -o "%VS90COMNTOOLS%\Include\stdint.h" + dir "%VS90COMNTOOLS%\Include" + condition: eq(variables['python.version'], '2.7') + displayName: 'Installing Microsoft Visual C++ Compiler for Python 2.7' + - script: | pip install --user --upgrade pip setuptools wheel displayName: 'Updating pip, setuptools and wheel' From 2b2d845079358ab91ac2304b0189b6b4cb9f59a5 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 1 Jan 2019 23:45:28 -0500 Subject: [PATCH 36/52] Python 2.7 not supported (old MSVC++ 9.0 fails building klayout) --- azure-pipelines.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0c6f57476..57d21f7f3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,12 +5,12 @@ jobs: vmImage: 'vs2017-win2016' # other options: 'macOS-10.13', 'ubuntu-16.04' strategy: matrix: - Python27: - python.version: '2.7' - # Python35: - # python.version: '3.5' - # Python36: - # python.version: '3.6' + # Python27: + # python.version: '2.7' + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' maxParallel: 3 steps: @@ -40,14 +40,14 @@ jobs: #workingDirectory: # Optional displayName: 'Downloading and Extracting KLayout bits' - - script: | - curl https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -o VCForPython27.msi - msiexec /i VCForPython27.msi /quiet - set "VS90COMNTOOLS=C:\Users\VssAdministrator\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python\9.0\VC" - curl https://raw.githubusercontent.com/mattn/gntp-send/master/include/msinttypes/stdint.h -o "%VS90COMNTOOLS%\Include\stdint.h" - dir "%VS90COMNTOOLS%\Include" - condition: eq(variables['python.version'], '2.7') - displayName: 'Installing Microsoft Visual C++ Compiler for Python 2.7' + # - script: | + # curl https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -o VCForPython27.msi + # msiexec /i VCForPython27.msi /quiet + # set "VS90COMNTOOLS=C:\Users\VssAdministrator\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python\9.0\VC" + # curl https://raw.githubusercontent.com/mattn/gntp-send/master/include/msinttypes/stdint.h -o "%VS90COMNTOOLS%\Include\stdint.h" + # dir "%VS90COMNTOOLS%\Include" + # condition: eq(variables['python.version'], '2.7') + # displayName: 'Installing Microsoft Visual C++ Compiler for Python 2.7' - script: | pip install --user --upgrade pip setuptools wheel From 7212d347f1db37e14539a6ef018cce142280d729 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 1 Jan 2019 23:47:47 -0500 Subject: [PATCH 37/52] Adding python 3.7 --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 57d21f7f3..34876e760 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,6 +11,8 @@ jobs: python.version: '3.5' Python36: python.version: '3.6' + Python37: + python.version: '3.7' maxParallel: 3 steps: From 8dc77ee314a76940202c04a4a4116adb11073dd1 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 1 Jan 2019 23:50:18 -0500 Subject: [PATCH 38/52] Adding x86 architecture --- azure-pipelines.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 34876e760..e549d454c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,17 +9,29 @@ jobs: # python.version: '2.7' Python35: python.version: '3.5' + python.architecture: 'x64' Python36: python.version: '3.6' + python.architecture: 'x64' Python37: python.version: '3.7' - maxParallel: 3 + python.architecture: 'x64' + Python35_x86: + python.version: '3.5' + python.architecture: 'x86' + Python36_x86: + python.version: '3.6' + python.architecture: 'x86' + Python37_x86: + python.version: '3.7' + python.architecture: 'x86' + maxParallel: 6 steps: - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' - architecture: 'x64' + architecture: '$(python.architecture)' # Add additional tasks to run using each Python version in the matrix above @@ -52,12 +64,13 @@ jobs: # displayName: 'Installing Microsoft Visual C++ Compiler for Python 2.7' - script: | - pip install --user --upgrade pip setuptools wheel + python -m pip install --upgrade pip setuptools wheel displayName: 'Updating pip, setuptools and wheel' - script: | python -V - set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\x64" + set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" + echo KLAYOUT_BITS=%KLAYOUT_BITS% python setup.py install displayName: 'Building KLayout' From b5788d8d7d4dc8ee65bf15e71e227d624b39a6b8 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 2 Jan 2019 00:09:27 -0500 Subject: [PATCH 39/52] Basic Unit testing after installation --- azure-pipelines.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e549d454c..50546bb48 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -72,5 +72,17 @@ jobs: set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" echo KLAYOUT_BITS=%KLAYOUT_BITS% python setup.py install + displayName: 'Build KLayout' + + - script: | + set PATH=%KLAYOUT_BITS%\curl\bin;%KLAYOUT_BITS%\expat\bin;%KLAYOUT_BITS%\ptw\bin;%KLAYOUT_BITS%\zlib\bin;%PATH% + python testdata/pymod/import_db.py + python testdata/pymod/import_rdb.py + python testdata/pymod/import_tl.py + python testdata/pymod/pya_tests.py + displayName: 'Test KLayout pymod' + + # - script : | + # python setup.py bdist_wheel + # displayName: 'Building KLayout wheel' - displayName: 'Building KLayout' From b72660a95e8c676e2a0c6b3d4d2cfb184f42a7f1 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 2 Jan 2019 00:34:56 -0500 Subject: [PATCH 40/52] Including external DLLs (klayout bits) in wheel --- azure-pipelines.yml | 20 ++++---- ci-scripts/windows/fix_wheel.sh | 84 +++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 ci-scripts/windows/fix_wheel.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 50546bb48..581b16998 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -52,7 +52,7 @@ jobs: #ignoreLASTEXITCODE: false # Optional #pwsh: false # Optional #workingDirectory: # Optional - displayName: 'Downloading and Extracting KLayout bits' + displayName: 'Download and Extract KLayout bits' # - script: | # curl https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -o VCForPython27.msi @@ -61,28 +61,28 @@ jobs: # curl https://raw.githubusercontent.com/mattn/gntp-send/master/include/msinttypes/stdint.h -o "%VS90COMNTOOLS%\Include\stdint.h" # dir "%VS90COMNTOOLS%\Include" # condition: eq(variables['python.version'], '2.7') - # displayName: 'Installing Microsoft Visual C++ Compiler for Python 2.7' + # displayName: 'Install Microsoft Visual C++ Compiler for Python 2.7' - script: | python -m pip install --upgrade pip setuptools wheel - displayName: 'Updating pip, setuptools and wheel' + displayName: 'Update pip, setuptools and wheel' - script: | python -V set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" echo KLAYOUT_BITS=%KLAYOUT_BITS% - python setup.py install + python setup.py bdist_wheel displayName: 'Build KLayout' + - bash: | + bash `pwd`/ci-scripts/windows/fix_wheel.sh `pwd`/dist/*.whl "`pwd`/klayout-microbits/klayout-microbits-1.0/msvc2017/$PYTHON_ARCHITECTURE" + displayName: 'Copy klayout bits dlls into wheel' + - script: | - set PATH=%KLAYOUT_BITS%\curl\bin;%KLAYOUT_BITS%\expat\bin;%KLAYOUT_BITS%\ptw\bin;%KLAYOUT_BITS%\zlib\bin;%PATH% + echo PATH=%PATH% + pip install klayout --no-index -f dist python testdata/pymod/import_db.py python testdata/pymod/import_rdb.py python testdata/pymod/import_tl.py python testdata/pymod/pya_tests.py displayName: 'Test KLayout pymod' - - # - script : | - # python setup.py bdist_wheel - # displayName: 'Building KLayout wheel' - diff --git a/ci-scripts/windows/fix_wheel.sh b/ci-scripts/windows/fix_wheel.sh new file mode 100644 index 000000000..a63a72c93 --- /dev/null +++ b/ci-scripts/windows/fix_wheel.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=`basename "$0"` +TMP_WHEEL="$PWD/tmp/klayout_tempwheel" + +display_usage() { + echo "This script includes external libraries in windows-based wheels." + echo "Caution: This will delete the original wheel." + echo -e "\nUsage:\n./${SCRIPT_NAME} [--help|-h] klayout-...-win_amd64.whl \n" + } + +if [[ ( $1 == "--help") || $1 == "-h" ]] +then + display_usage + exit 0 +fi + +# Check number of arguments +if [ $# -eq 0 ]; then + >&2 echo -e "ERROR: No filename supplied\n" + display_usage + exit 1 +elif [ ! $# -eq 2 ]; then + >&2 echo -e "ERROR: Too many files supplied. Provide one filename at at time.\n" + display_usage + exit 1 +fi + +# Read wheel file from argument +WHL="$1" +WHL=$(echo $WHL | sed 's/\\/\//g' | sed 's/://') +KLAYOUT_BITS="$2" +KLAYOUT_BITS=$(echo $KLAYOUT_BITS | sed 's/\\/\//g' | sed 's/://') + + +# Check WHL is a valid file +if [[ ! -f "$WHL" ]]; then + >&2 echo -e "ERROR: $WHL is not a file" + exit 1 +fi + +# Convert to absolute path (linux only) +WHL=$(readlink -f $WHL) + +# Record old current directory +OLD_PWD=$PWD + +# Need to include external DLLs (klayout bits) into wheel + +# # Checking if it was previously patched +# if unzip -l $WHL | grep -q 'patch_external_libraries_included'; then +# echo "$(basename $WHL) is already patched. Doing nothing." +# exit 0 +# fi + +# Repair script below +if [[ -d $TMP_WHEEL ]]; then + rm -rf $TMP_WHEEL +else + mkdir -p $TMP_WHEEL +fi +echo "Unpacking $WHL inside $TMP_WHEEL" +wheel unpack $WHL -d $TMP_WHEEL || exit 1 +TMP_WHEEL="$TMP_WHEEL/`ls $TMP_WHEEL`" + +cd $TMP_WHEEL/klayout +echo "Copying libraries: libcurl.dll, expat.dll, pthreadVCE2.dll, zlib1.dll" +echo pwd: `pwd` +cp -v $KLAYOUT_BITS/curl/bin/* . +cp -v $KLAYOUT_BITS/expat/bin/* . +cp -v $KLAYOUT_BITS/ptw/bin/* . +cp -v $KLAYOUT_BITS/zlib/bin/* . +# if [ $? -ne 0 ]; then +# >&2 echo "ERROR: lib not found. Quitting." +# exit 1 +# fi +cd $TMP_WHEEL +touch $TMP_WHEEL/patch_external_libraries_included +echo "Packing $WHL from $TMP_WHEEL" +# rm -f $WHL +wheel pack $TMP_WHEEL -d `dirname $WHL` || exit 1 +echo "Done. $(basename $WHL) is patched." +# Cleanup (should always execute) +cd $OLD_PWD From b4904f137018ec322080f51493f62876369fecef Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 2 Jan 2019 01:25:08 -0500 Subject: [PATCH 41/52] Dropping wheels as artifacts --- azure-pipelines.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 581b16998..8affbf540 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -86,3 +86,16 @@ jobs: python testdata/pymod/import_tl.py python testdata/pymod/pya_tests.py displayName: 'Test KLayout pymod' + + - task: CopyFiles@2 + condition: always() + inputs: + sourceFolder: '$(Build.SourcesDirectory)' + targetFolder: '$(Build.ArtifactStagingDirectory)' + contents: '**/?(*.whl)' + + - task: PublishBuildArtifacts@1 + condition: always() + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop From 923d083ab4bd019e84fed6ab3cc80dfe1fe73154 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 2 Jan 2019 01:53:42 -0500 Subject: [PATCH 42/52] Unique artifact name for debug --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8affbf540..8bed8fade 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -98,4 +98,4 @@ jobs: condition: always() inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: drop + artifactName: 'wheel-$(python.version).$(python.architecture)' From 2b2b38d1604bf71ec6c63253c79afebbde0bb419 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 2 Jan 2019 02:35:36 -0500 Subject: [PATCH 43/52] Combining all wheels into one folder --- azure-pipelines.yml | 65 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8bed8fade..bc1cd1019 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,28 +1,28 @@ # https://aka.ms/yaml jobs: -- job: 'Build' +- job: Build pool: vmImage: 'vs2017-win2016' # other options: 'macOS-10.13', 'ubuntu-16.04' strategy: matrix: # Python27: # python.version: '2.7' - Python35: + cp35-cp35m-win_amd64.whl: python.version: '3.5' python.architecture: 'x64' - Python36: + cp36-cp36m-win_amd64.whl: python.version: '3.6' python.architecture: 'x64' - Python37: + cp37-cp37m-win_amd64.whl: python.version: '3.7' python.architecture: 'x64' - Python35_x86: + cp35-cp35m-win32.whl: python.version: '3.5' python.architecture: 'x86' - Python36_x86: + cp36-cp36m-win32.whl: python.version: '3.6' python.architecture: 'x86' - Python37_x86: + cp37-cp37m-win32.whl: python.version: '3.7' python.architecture: 'x86' maxParallel: 6 @@ -99,3 +99,54 @@ jobs: inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: 'wheel-$(python.version).$(python.architecture)' + +- job: 'Deploy' + displayName: 'Combine Windows wheels' + dependsOn: Build + pool: + vmImage: 'vs2017-win2016' # other options: 'macOS-10.13', 'ubuntu-16.04' + steps: + - checkout: none #skip checking out the default repository resource + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts wheel-3.7.x64' + inputs: + artifactName: 'wheel-3.7.x64' + downloadPath: '$(System.DefaultWorkingDirectory)' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts wheel-3.6.x64' + inputs: + artifactName: 'wheel-3.6.x64' + downloadPath: '$(System.DefaultWorkingDirectory)' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts wheel-3.5.x64' + inputs: + artifactName: 'wheel-3.5.x64' + downloadPath: '$(System.DefaultWorkingDirectory)' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts wheel-3.7.x86' + inputs: + artifactName: 'wheel-3.7.x86' + downloadPath: '$(System.DefaultWorkingDirectory)' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts wheel-3.6.x86' + inputs: + artifactName: 'wheel-3.6.x86' + downloadPath: '$(System.DefaultWorkingDirectory)' + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts wheel-3.5.x86' + inputs: + artifactName: 'wheel-3.5.x86' + downloadPath: '$(System.DefaultWorkingDirectory)' + - task: CopyFiles@2 + condition: always() + inputs: + sourceFolder: '$(System.DefaultWorkingDirectory)' + targetFolder: '$(Build.ArtifactStagingDirectory)' + contents: '**/?(*.whl)' + flattenFolders: true + - task: PublishBuildArtifacts@1 + condition: always() + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: 'windows_wheels' + From a823d8c5df4370ca8f416319ab21f21802193161 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 2 Jan 2019 11:47:41 -0500 Subject: [PATCH 44/52] Removing unnecessary patch_external_libraries_included file from wheel --- ci-scripts/windows/fix_wheel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/windows/fix_wheel.sh b/ci-scripts/windows/fix_wheel.sh index a63a72c93..3ac9593c9 100644 --- a/ci-scripts/windows/fix_wheel.sh +++ b/ci-scripts/windows/fix_wheel.sh @@ -75,9 +75,9 @@ cp -v $KLAYOUT_BITS/zlib/bin/* . # exit 1 # fi cd $TMP_WHEEL -touch $TMP_WHEEL/patch_external_libraries_included +# touch $TMP_WHEEL/patch_external_libraries_included echo "Packing $WHL from $TMP_WHEEL" -# rm -f $WHL +rm -f $WHL wheel pack $TMP_WHEEL -d `dirname $WHL` || exit 1 echo "Done. $(basename $WHL) is patched." # Cleanup (should always execute) From 7deea2706930f0938659eb031ea5a21fd91306e0 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 2 Jan 2019 13:51:18 -0500 Subject: [PATCH 45/52] changing version to 0.26.0.dev11, release pypi wheels after merging #218 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 059f4b44c..e7455d296 100644 --- a/setup.py +++ b/setup.py @@ -322,7 +322,7 @@ class Config(object): """ Gets the version string """ - return "0.26.0.dev10" + return "0.26.0.dev11" config = Config() From e3b8d3635c30b9716b28531e9a204ff6ca163805 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 2 Jan 2019 23:18:14 +0100 Subject: [PATCH 46/52] Small bugfix: object._destroy wasn't working for directly passed objects. --- src/gsi/gsi_test/gsiTest.cc | 24 +++--- src/gsi/gsi_test/gsiTest.h | 12 +-- src/pya/pya/pyaObject.cc | 2 +- src/rba/rba/rbaInternal.cc | 2 +- testdata/python/basic.py | 128 +++++++++++++++-------------- testdata/ruby/basic_testcore.rb | 139 +++++++++++++++++--------------- 6 files changed, 163 insertions(+), 144 deletions(-) diff --git a/src/gsi/gsi_test/gsiTest.cc b/src/gsi/gsi_test/gsiTest.cc index 1f5092e68..39a4b8119 100644 --- a/src/gsi/gsi_test/gsiTest.cc +++ b/src/gsi/gsi_test/gsiTest.cc @@ -83,7 +83,7 @@ A &A::operator= (const A &a) return *this; } -int A::a0 () +int A::instance_count () { return a_count; } @@ -780,6 +780,7 @@ static gsi::QFlagsClass decl_qflags_enum ("", "Enums"); static gsi::Class decl_a ("", "A", gsi::constructor ("new_a|new", &a_ctor) + + gsi::method ("instance_count", &A::instance_count) + gsi::method ("br", &A::br) + gsi::method ("get_e", &A::get_e) + gsi::method ("get_eptr", &A::get_eptr) + @@ -814,7 +815,6 @@ static gsi::Class decl_a ("", "A", gsi::method ("af?|af", &A::af1) + gsi::method ("aa", &A::a) + gsi::method ("aa", &A::a_static) + - gsi::method ("a0", &A::a0) + gsi::method ("a1", &A::a1) + gsi::method ("a1c", &A::a1c) + gsi::method ("a2", &A::a2) + @@ -904,22 +904,22 @@ static gsi::Class decl_b ("", "B", gsi::method ("b5", &B::b5) + gsi::method ("b5a", &B::b5a) + gsi::method ("b5b", &B::b5b) + - gsi::method ("b6|make_a", &B::b6) + - gsi::method ("b8|set_an", &B::b8) + - gsi::method ("b9|an", &B::b9) + - gsi::method ("b8cref|set_an_cref", &B::b8) + - gsi::method ("b9cref|an_cref", &B::b9) + + gsi::method ("make_a", &B::make_a) + + gsi::method ("set_an", &B::set_an) + + gsi::method ("an", &B::an) + + gsi::method ("set_an_cref", &B::set_an_cref) + + gsi::method ("an_cref", &B::an_cref) + // implemented by extension below: // gsi::iterator_ext ("b10", &b10b_ext, &b10e_ext) + gsi::iterator ("b10_nc", &B::b10b_nc, &B::b10e_nc) + gsi::iterator ("b11", &B::b11b, &B::b11e) + gsi::iterator ("b12", &B::b12b, &B::b12e) + gsi::iterator ("b13", &B::b13b, &B::b13e) + - gsi::method ("b14a|amember_or_nil", &B::amember_or_nil) + - gsi::method ("b14b|amember_ptr", &B::amember_ptr) + - gsi::method ("b7|b14c|amember_cptr", &B::amember_cptr) + - gsi::method ("b7c|amember_cref", &B::amember_cref) + - gsi::method ("b7d|amember_ref", &B::amember_ref) + + gsi::method ("amember_or_nil_alt|amember_or_nil", &B::amember_or_nil) + + gsi::method ("amember_ptr_alt|amember_ptr", &B::amember_ptr) + + gsi::method ("xxx|amember_cptr", &B::amember_cptr) + + gsi::method ("yyy|amember_cref", &B::amember_cref) + + gsi::method ("zzz|amember_ref", &B::amember_ref) + gsi::method ("b15|arg_is_not_nil", &B::b15) + gsi::method ("b16a|av", &B::b16a) + gsi::method ("b16b|av_cref", &B::b16b) + diff --git a/src/gsi/gsi_test/gsiTest.h b/src/gsi/gsi_test/gsiTest.h index ad3c367aa..7e11c815d 100644 --- a/src/gsi/gsi_test/gsiTest.h +++ b/src/gsi/gsi_test/gsiTest.h @@ -104,7 +104,7 @@ struct A void set_af (bool _f) { f = _f; } bool af1 (bool /*dummy*/) { return f; } - static int a0 (); + static int instance_count (); int a1 () { return n; } @@ -465,26 +465,26 @@ struct B return m.c_str (); } - A b6 (int n) { + A make_a (int n) { return A(n); } - void b8 (int n) + void set_an (int n) { a.n = n; } - int b9 (A a) + int an (A a) { return a.n; } - void b8cref (const int &n) + void set_an_cref (const int &n) { a.n = n; } - const int &b9cref (const A &a) + const int &an_cref (const A &a) { return a.n; } diff --git a/src/pya/pya/pyaObject.cc b/src/pya/pya/pyaObject.cc index 3ed8d126d..8c1768e9b 100644 --- a/src/pya/pya/pyaObject.cc +++ b/src/pya/pya/pyaObject.cc @@ -538,7 +538,7 @@ PYAObjectBase::destroy () return; } - if (!m_can_destroy && m_obj) { + if (! (m_owned || m_can_destroy) && m_obj) { throw tl::Exception (tl::to_string (tr ("Object cannot be destroyed explicitly"))); } diff --git a/src/rba/rba/rbaInternal.cc b/src/rba/rba/rbaInternal.cc index 2b9e2dcfd..f5dc134a8 100644 --- a/src/rba/rba/rbaInternal.cc +++ b/src/rba/rba/rbaInternal.cc @@ -364,7 +364,7 @@ Proxy::destroy () return; } - if (!m_can_destroy && m_obj) { + if (! (m_owned || m_can_destroy) && m_obj) { throw tl::Exception (tl::to_string (tr ("Object cannot be destroyed explicitly"))); } diff --git a/testdata/python/basic.py b/testdata/python/basic.py index eb495d0c8..eb459411d 100644 --- a/testdata/python/basic.py +++ b/testdata/python/basic.py @@ -135,10 +135,10 @@ class BasicTest(unittest.TestCase): def test_00(self): # all references of PA are released now: - ac0 = pya.A.a0() + ic0 = pya.A.instance_count() a = pya.A.new_a(100) - self.assertEqual( pya.A.a0(), ac0 + 1 ) + self.assertEqual( pya.A.instance_count(), ic0 + 1 ) a = pya.A() self.assertEqual(a.a1(), 17) @@ -146,28 +146,28 @@ class BasicTest(unittest.TestCase): self.assertEqual(a.a1(), 110) a = None - self.assertEqual( pya.A.a0(), ac0 ) + self.assertEqual( pya.A.instance_count(), ic0 ) a = pya.A() - self.assertEqual( pya.A.a0(), ac0 ) # delayed instantiation of detached objects - A is actually created if it is used first + self.assertEqual( pya.A.instance_count(), ic0 ) # delayed instantiation of detached objects - A is actually created if it is used first a.a2() # just check, if it can be called - self.assertEqual( pya.A.a0(), ac0 + 1 ) + self.assertEqual( pya.A.instance_count(), ic0 + 1 ) # open question: with ruby 1.8, aa is not deleted if the self.assertEqual is missing. Why? # maybe the GC does not like to be called that frequently? aa = a.dup() - self.assertEqual( pya.A.a0(), ac0 + 2 ) + self.assertEqual( pya.A.instance_count(), ic0 + 2 ) aa = None - self.assertEqual( pya.A.a0(), ac0 + 1 ) + self.assertEqual( pya.A.instance_count(), ic0 + 1 ) a = None - self.assertEqual( pya.A.a0(), ac0 ) + self.assertEqual( pya.A.instance_count(), ic0 ) a = pya.A() - self.assertEqual( pya.A.a0(), ac0 ) # delayed instantiation of detached objects - A is actually created if it is used first + self.assertEqual( pya.A.instance_count(), ic0 ) # delayed instantiation of detached objects - A is actually created if it is used first a.a2() # just check, if it can be called - self.assertEqual( pya.A.a0(), ac0 + 1 ) + self.assertEqual( pya.A.instance_count(), ic0 + 1 ) # static and non-static methods can be mixed, but they will be made non-ambiguous too self.assertEqual( pya.A.aa(), "static_a" ) @@ -219,7 +219,7 @@ class BasicTest(unittest.TestCase): self.assertEqual(arr, [5, 1, -1.25]) a.destroy() - self.assertEqual( pya.A.a0(), ac0 ) + self.assertEqual( pya.A.instance_count(), ic0 ) if not leak_check: @@ -237,13 +237,13 @@ class BasicTest(unittest.TestCase): error_caught = True self.assertEqual( error_caught, True ) - self.assertEqual( pya.A.a0(), ac0 ) + self.assertEqual( pya.A.instance_count(), ic0 ) a = pya.A.new_a( 55 ) - self.assertEqual( pya.A.a0(), ac0 + 1 ) + self.assertEqual( pya.A.instance_count(), ic0 + 1 ) self.assertEqual( a.a1(), 55 ) self.assertEqual( a.a_vp1( a.a_vp2() ), "abc" ) a.destroy() - self.assertEqual( pya.A.a0(), ac0 ) + self.assertEqual( pya.A.instance_count(), ic0 ) a = pya.A.new_a(0) self.assertEqual( str(a.a9a(5)), "True" ) @@ -594,12 +594,12 @@ class BasicTest(unittest.TestCase): err_caught = False try: - b.b7().a1() # cannot call non-const method on const reference + b.amember_cptr().a1() # cannot call non-const method on const reference except: err_caught = True self.assertEqual( err_caught, True ) - b.b7().a2() + b.amember_cptr().a2() self.assertEqual( b.b1(), 5 ) self.assertEqual( b.b2(), "" ) @@ -973,50 +973,58 @@ class BasicTest(unittest.TestCase): # test copies of objects being returned b = pya.B() + b._create() # force constructor of B to allocate some A instances internally - a = b.b6( 1971 ); + a_count = pya.A.instance_count() + a = b.make_a( 1971 ); + self.assertEqual( pya.A.instance_count(), a_count + 1 ) self.assertEqual( a.a1(), 1971 ); - self.assertEqual( b.b9( a ), 1971 ); + self.assertEqual( b.an( a ), 1971 ); - aa = b.b6( -61 ); - self.assertEqual( b.b9cref( aa ), -61 ); + aa = b.make_a( -61 ); + self.assertEqual( pya.A.instance_count(), a_count + 2 ) + self.assertEqual( b.an_cref( aa ), -61 ); self.assertEqual( a.a1(), 1971 ); - self.assertEqual( b.b9( a ), 1971 ); + self.assertEqual( b.an( a ), 1971 ); self.assertEqual( aa.a1(), -61 ); - self.assertEqual( b.b9( aa ), -61 ); + self.assertEqual( b.an( aa ), -61 ); aa.a5(98); a.a5(100); self.assertEqual( a.a1(), 100 ); - self.assertEqual( b.b9( a ), 100 ); + self.assertEqual( b.an( a ), 100 ); self.assertEqual( aa.a1(), 98 ); - self.assertEqual( b.b9( aa ), 98 ); + self.assertEqual( b.an( aa ), 98 ); + + a._destroy() + aa = None + self.assertEqual( pya.A.instance_count(), a_count ) def test_18(self): - # Test references to objects (returned by b.b7) + # Test references to objects (returned by b.amember_cptr) b = pya.B() - b.b8( 77 ) - self.assertEqual( b.b7().a1c(), 77 ); + b.set_an( 77 ) + self.assertEqual( b.amember_cptr().a1c(), 77 ); - b.b8cref( 79 ) - self.assertEqual( b.b7().a1c(), 79 ); + b.set_an_cref( 79 ) + self.assertEqual( b.amember_cptr().a1c(), 79 ); - aref = b.b7() + aref = b.amember_cptr() err_caught = False if not leak_check: try: - x = aref.a1() # cannot call non-const method on const reference (as delivered by b7) + x = aref.a1() # cannot call non-const method on const reference (as delivered by amember_cptr) except: err_caught = True self.assertEqual( err_caught, True ) self.assertEqual( aref.a1c(), 79 ); - b.b8( -1 ) + b.set_an( -1 ) self.assertEqual( aref.a1c(), -1 ); def test_19(self): @@ -1068,19 +1076,19 @@ class BasicTest(unittest.TestCase): b = pya.B() - a1 = b.b14a( True ) - a2 = b.b14b() + a1 = b.amember_or_nil_alt( True ) + a2 = b.amember_ptr_alt() self.assertEqual( a1.a1(), 17 ) self.assertEqual( a2.a1(), 17 ) a1.a5( 761 ) self.assertEqual( a1.a1(), 761 ) self.assertEqual( a2.a1(), 761 ) - a1 = b.b14a( False ) + a1 = b.amember_or_nil( False ) self.assertEqual( a1, None ) - self.assertEqual( b.b15( b.b14b() ), True ) - self.assertEqual( b.b15( b.b14a( False ) ), False ) + self.assertEqual( b.b15( b.amember_ptr() ), True ) + self.assertEqual( b.b15( b.amember_or_nil( False ) ), False ) self.assertEqual( b.b15( None ), False ) def test_21(self): @@ -1094,9 +1102,9 @@ class BasicTest(unittest.TestCase): b = pya.B() - # b14b() returns a pya.A object, but it cannot be extended dynamically by a method s ... - b.b14b().s( 117 ) - self.assertEqual( b.b14b().g(), 117 ) + # amember_ptr() returns a pya.A object, but it cannot be extended dynamically by a method s ... + b.amember_ptr().s( 117 ) + self.assertEqual( b.amember_ptr().g(), 117 ) n = 0 @@ -1247,9 +1255,9 @@ class BasicTest(unittest.TestCase): self.assertEqual( b.bx("a", 15), 20.5 ) self.assertEqual( b.b32("b", 25), 20.5 ) - na = pya.A.a0() # instance count + na = pya.A.instance_count() # instance count self.assertEqual(b.bx(a, 15), "aref+i") - self.assertEqual(pya.A.a0(), na) + self.assertEqual(pya.A.instance_count(), na) err_caught = False try: # cannot cast second argument to int @@ -1258,7 +1266,7 @@ class BasicTest(unittest.TestCase): err_caught = True self.assertEqual(err_caught, True) # the exception thrown before must not leave an instance on the call stack: - self.assertEqual(pya.A.a0(), na) + self.assertEqual(pya.A.instance_count(), na) err_caught = False try: @@ -1368,52 +1376,52 @@ class BasicTest(unittest.TestCase): # destruction of an instance via c++ pya.A.a20(None) - ac0 = pya.A.a0() + ic0 = pya.A.instance_count() a = pya.A() a.create() self.assertEqual(a.destroyed(), False) - self.assertEqual(pya.A.a0(), ac0 + 1) + self.assertEqual(pya.A.instance_count(), ic0 + 1) pya.A.a20(a) # install static instance of A self.assertEqual(a.destroyed(), False) pya.A.a20(None) self.assertEqual(a.destroyed(), True) - self.assertEqual(pya.A.a0(), ac0) + self.assertEqual(pya.A.instance_count(), ic0) a = pya.A() a.create() self.assertEqual(a.destroyed(), False) - self.assertEqual(pya.A.a0(), ac0 + 1) + self.assertEqual(pya.A.instance_count(), ic0 + 1) pya.A.a20(a) # install static instance of A self.assertEqual(a.destroyed(), False) - self.assertEqual(pya.A.a0(), ac0 + 1) + self.assertEqual(pya.A.instance_count(), ic0 + 1) pya.A.a20(a) # re-install static instance of A self.assertEqual(a.destroyed(), False) - self.assertEqual(pya.A.a0(), ac0 + 1) + self.assertEqual(pya.A.instance_count(), ic0 + 1) # install another instance aa = pya.A() aa.create() self.assertEqual(aa.destroyed(), False) - self.assertEqual(pya.A.a0(), ac0 + 2) + self.assertEqual(pya.A.instance_count(), ic0 + 2) pya.A.a20(aa) # install static instance of A # original one is destroyed now, only new instance remains self.assertEqual(a.destroyed(), True) self.assertEqual(aa.destroyed(), False) - self.assertEqual(pya.A.a0(), ac0 + 1) + self.assertEqual(pya.A.instance_count(), ic0 + 1) pya.A.a20(None) # discard installed instance self.assertEqual(aa.destroyed(), True) - self.assertEqual(pya.A.a0(), ac0) + self.assertEqual(pya.A.instance_count(), ic0) # the same without create .. should work too, but not create an instance because of late # instantiation in default ctor a = pya.A() self.assertEqual(a.destroyed(), False) - self.assertEqual(pya.A.a0(), ac0) + self.assertEqual(pya.A.instance_count(), ic0) pya.A.a20(a) # install static instance of A self.assertEqual(a.destroyed(), False) pya.A.a20(None) - self.assertEqual(pya.A.a0(), ac0) + self.assertEqual(pya.A.instance_count(), ic0) self.assertEqual(a.destroyed(), True) def test_26(self): @@ -1463,36 +1471,36 @@ class BasicTest(unittest.TestCase): # destruction of an instance via c++ pya.A.a20(None) - ac0 = pya.A.a0() + ic0 = pya.A.instance_count() a = pya.A() a._create() self.assertEqual(a._destroyed(), False) - self.assertEqual(pya.A.a0(), ac0 + 1) + self.assertEqual(pya.A.instance_count(), ic0 + 1) pya.A.a20(a) self.assertEqual(pya.A.a20_get() == None, False) # release A instance -> will delete it a = None - self.assertEqual(pya.A.a0(), ac0) + self.assertEqual(pya.A.instance_count(), ic0) self.assertEqual(pya.A.a20_get() == None, True) a = pya.A() a._create() self.assertEqual(a.destroyed(), False) - self.assertEqual(pya.A.a0(), ac0 + 1) + self.assertEqual(pya.A.instance_count(), ic0 + 1) pya.A.a20(a) # install static instance of A self.assertEqual(pya.A.a20_get() == None, False) a._unmanage() # release A instance -> won't delete it since it is unmanaged a = None - self.assertEqual(pya.A.a0(), ac0 + 1) + self.assertEqual(pya.A.instance_count(), ic0 + 1) self.assertEqual(pya.A.a20_get() == None, False) a = pya.A.a20_get() a._manage() # release A instance -> will be deleted since now it's managed again a = None - self.assertEqual(pya.A.a0(), ac0) + self.assertEqual(pya.A.instance_count(), ic0) self.assertEqual(pya.A.a20_get() == None, True) def test_28(self): diff --git a/testdata/ruby/basic_testcore.rb b/testdata/ruby/basic_testcore.rb index 059d92e79..467ceaccf 100644 --- a/testdata/ruby/basic_testcore.rb +++ b/testdata/ruby/basic_testcore.rb @@ -14,11 +14,11 @@ class Basic_TestClass < TestBase GC.start # all references of A are released now: - ac0 = RBA::A::a0 - assert_equal( ac0, 0 ) + ic0 = RBA::A::instance_count + assert_equal( ic0, 0 ) a = RBA::A.new_a(100) - assert_equal( RBA::A::a0, ac0 + 1 ) + assert_equal( RBA::A::instance_count, ic0 + 1 ) a = RBA::A.new assert_equal(a.a1, 17) @@ -27,26 +27,26 @@ class Basic_TestClass < TestBase a = nil GC.start - assert_equal( RBA::A::a0, ac0 ) + assert_equal( RBA::A::instance_count, ic0 ) a = RBA::A.new - assert_equal( RBA::A::a0, ac0 ) # delayed instantiation of detached objects - A is actually created if it is used first + assert_equal( RBA::A::instance_count, ic0 ) # delayed instantiation of detached objects - A is actually created if it is used first a.a2 # just check, if it can be called - assert_equal( RBA::A::a0, ac0 + 1 ) + assert_equal( RBA::A::instance_count, ic0 + 1 ) # open question: with ruby 1.8, aa is not deleted if the assert_equal is missing. Why? # maybe the GC does not like to be called that frequently? aa = a.dup - assert_equal( RBA::A::a0, ac0 + 2 ) + assert_equal( RBA::A::instance_count, ic0 + 2 ) aa = nil GC.start if RUBY_VERSION >= "1.9.0" # GC works differently in >=1.9.x - but no leak (verified with valgrind) - ac0 = RBA::A::a0 + ic0 = RBA::A::instance_count else - assert_equal( RBA::A::a0, ac0 + 1 ) + assert_equal( RBA::A::instance_count, ic0 + 1 ) end a = nil @@ -54,15 +54,15 @@ class Basic_TestClass < TestBase if RUBY_VERSION >= "1.9.0" # GC works differently in >=1.9.x - but no leak (verified with valgrind) - ac0 = RBA::A::a0 + ic0 = RBA::A::instance_count else - assert_equal( RBA::A::a0, ac0 ) + assert_equal( RBA::A::instance_count, ic0 ) end a = RBA::A.new - assert_equal( RBA::A::a0, ac0 ) # delayed instantiation of detached objects - A is actually created if it is used first + assert_equal( RBA::A::instance_count, ic0 ) # delayed instantiation of detached objects - A is actually created if it is used first a.a2 # just check, if it can be called - assert_equal( RBA::A::a0, ac0 + 1 ) + assert_equal( RBA::A::instance_count, ic0 + 1 ) # mix of getters, setters, predicates assert_equal( a.af, false ) @@ -116,7 +116,7 @@ class Basic_TestClass < TestBase assert_equal(arr, [5, 1, -1.25]) a.destroy - assert_equal( RBA::A::a0, ac0 ) + assert_equal( RBA::A::instance_count, ic0 ) if !$leak_check @@ -138,13 +138,13 @@ class Basic_TestClass < TestBase end - assert_equal( RBA::A::a0, ac0 ) + assert_equal( RBA::A::instance_count, ic0 ) a = RBA::A::new_a( 55 ) - assert_equal( RBA::A::a0, ac0 + 1 ) + assert_equal( RBA::A::instance_count, ic0 + 1 ) assert_equal( a.a1, 55 ) assert_equal( a.a_vp1( a.a_vp2 ), "abc" ) a.destroy - assert_equal( RBA::A::a0, ac0 ) + assert_equal( RBA::A::instance_count, ic0 ) a = RBA::A::new_a(0) assert_equal( a.a9a(5).to_s, "true" ) @@ -506,7 +506,7 @@ class Basic_TestClass < TestBase err_caught = false begin - b.b7.a1 # cannot call non-const method on const reference + b.amember_cptr.a1 # cannot call non-const method on const reference rescue err_caught = true end @@ -514,7 +514,7 @@ class Basic_TestClass < TestBase end - b.b7.a2 + b.amember_cptr.a2 assert_equal( b.b1, 5 ) assert_equal( b.b2, "" ) @@ -857,46 +857,57 @@ class Basic_TestClass < TestBase # test copies of objects being returned b = RBA::B.new + b._create - a = b.b6( 1971 ); - assert_equal( a.a1, 1971 ); - assert_equal( b.b9( a ), 1971 ); + GC.start + a_count = RBA::A.instance_count + a = b.make_a( 1971 ); + assert_equal( RBA::A.instance_count, a_count + 1 ) - aa = b.b6( -61 ); - assert_equal( b.b9cref( aa ), -61 ); assert_equal( a.a1, 1971 ); - assert_equal( b.b9( a ), 1971 ); + assert_equal( b.an( a ), 1971 ); + + aa = b.make_a( -61 ); + assert_equal( RBA::A.instance_count, a_count + 2 ) + assert_equal( b.an_cref( aa ), -61 ); + assert_equal( a.a1, 1971 ); + assert_equal( b.an( a ), 1971 ); assert_equal( aa.a1, -61 ); - assert_equal( b.b9( aa ), -61 ); + assert_equal( b.an( aa ), -61 ); aa.a5 98; a.a5 100; assert_equal( a.a1, 100 ); - assert_equal( b.b9( a ), 100 ); + assert_equal( b.an( a ), 100 ); assert_equal( aa.a1, 98 ); - assert_equal( b.b9( aa ), 98 ); + assert_equal( b.an( aa ), 98 ); + + a._destroy + aa = nil + GC.start + assert_equal( RBA::A.instance_count, a_count ) end def test_18 - # Test references to objects (returned by b.b7) + # Test references to objects (returned by b.amember_cptr) b = RBA::B.new - b.b8( 77 ) - assert_equal( b.b7.a1c, 77 ); + b.set_an( 77 ) + assert_equal( b.amember_cptr.a1c, 77 ); - b.b8cref( 79 ) - assert_equal( b.b7.a1c, 79 ); + b.set_an_cref( 79 ) + assert_equal( b.amember_cptr.a1c, 79 ); - aref = b.b7 + aref = b.amember_cptr err_caught = false if !$leak_check begin - x = aref.a1 # cannot call non-const method on const reference (as delivered by b7) + x = aref.a1 # cannot call non-const method on const reference (as delivered by amember_cptr) rescue err_caught = true end @@ -905,7 +916,7 @@ class Basic_TestClass < TestBase end - b.b8( -1 ) + b.set_an( -1 ) assert_equal( aref.a1c, -1 ); end @@ -973,19 +984,19 @@ class Basic_TestClass < TestBase b = RBA::B.new - a1 = b.b14a( true ) - a2 = b.b14b + a1 = b.amember_or_nil( true ) + a2 = b.amember_ptr assert_equal( a1.a1, 17 ) assert_equal( a2.a1, 17 ) a1.a5( 761 ) assert_equal( a1.a1, 761 ) assert_equal( a2.a1, 761 ) - a1 = b.b14a( false ) + a1 = b.amember_or_nil( false ) assert_equal( a1, nil ) - assert_equal( b.b15( b.b14b ), true ) - assert_equal( b.b15( b.b14a( false ) ), false ) + assert_equal( b.b15( b.amember_ptr ), true ) + assert_equal( b.b15( b.amember_or_nil( false ) ), false ) assert_equal( b.b15( nil ), false ) end @@ -996,8 +1007,8 @@ class Basic_TestClass < TestBase b = RBA::B.new - b.b14b.s( 117 ) - assert_equal( b.b14b.g, 117 ) + b.amember_ptr.s( 117 ) + assert_equal( b.amember_ptr.g, 117 ) n = 0 b.b10_nc { |a| a.s( n ); n += 1 } @@ -1138,10 +1149,10 @@ class Basic_TestClass < TestBase assert_equal( b.b32( "b", 25 ), 20.5 ) GC.start - na = RBA::A::a0 # instance count + na = RBA::A::instance_count # instance count assert_equal( b.bx( a, 15 ), "aref+i" ) GC.start - assert_equal( RBA::A::a0, na ) + assert_equal( RBA::A::instance_count, na ) err_caught = false begin # cannot cast second argument to int @@ -1152,7 +1163,7 @@ class Basic_TestClass < TestBase assert_equal( err_caught, true ) # the exception thrown before must not leave an instance on the call stack: GC.start - assert_equal( RBA::A::a0, na ) + assert_equal( RBA::A::instance_count, na ) err_caught = false begin @@ -1254,53 +1265,53 @@ class Basic_TestClass < TestBase # destruction of an instance via c++ GC.start - ac0 = RBA::A.a0 + ic0 = RBA::A.instance_count a = RBA::A::new a.create assert_equal(a.destroyed?, false) - assert_equal(RBA::A.a0, ac0 + 1) + assert_equal(RBA::A.instance_count, ic0 + 1) RBA::A.a20(a) # install static instance of A assert_equal(a.destroyed?, false) RBA::A.a20(nil) - assert_equal(RBA::A.a0, ac0) + assert_equal(RBA::A.instance_count, ic0) assert_equal(a.destroyed?, true) a = RBA::A::new a.create assert_equal(a.destroyed?, false) - assert_equal(RBA::A.a0, ac0 + 1) + assert_equal(RBA::A.instance_count, ic0 + 1) RBA::A.a20(a) # install static instance of A assert_equal(a.destroyed?, false) - assert_equal(RBA::A.a0, ac0 + 1) + assert_equal(RBA::A.instance_count, ic0 + 1) RBA::A.a20(a) # re-install static instance of A assert_equal(a.destroyed?, false) - assert_equal(RBA::A.a0, ac0 + 1) + assert_equal(RBA::A.instance_count, ic0 + 1) # install another instance aa = RBA::A::new aa.create assert_equal(aa.destroyed?, false) - assert_equal(RBA::A.a0, ac0 + 2) + assert_equal(RBA::A.instance_count, ic0 + 2) RBA::A.a20(aa) # install static instance of A # original one is destroyed now, only new instance remains assert_equal(a.destroyed?, true) assert_equal(aa.destroyed?, false) - assert_equal(RBA::A.a0, ac0 + 1) + assert_equal(RBA::A.instance_count, ic0 + 1) RBA::A.a20(nil) # discard installed instance assert_equal(aa.destroyed, true) - assert_equal(RBA::A.a0, ac0) + assert_equal(RBA::A.instance_count, ic0) # the same without create .. should work too, but not create an instance because of late # instantiation in default ctor a = RBA::A::new assert_equal(a.destroyed?, false) - assert_equal(RBA::A.a0, ac0) + assert_equal(RBA::A.instance_count, ic0) RBA::A.a20(a) # install static instance of A assert_equal(a.destroyed?, false) RBA::A.a20(nil) - assert_equal(RBA::A.a0, ac0) + assert_equal(RBA::A.instance_count, ic0) assert_equal(a.destroyed?, true) end @@ -1309,13 +1320,13 @@ class Basic_TestClass < TestBase # destruction of an instance via c++ GC.start - ac0 = RBA::A.a0 + ic0 = RBA::A.instance_count 1.times do a = RBA::A::new a._create assert_equal(a.destroyed?, false) - assert_equal(RBA::A.a0, ac0 + 1) + assert_equal(RBA::A.instance_count, ic0 + 1) RBA::A.a20(a) # install static instance of A assert_equal(RBA::A.a20_get == nil, false) end @@ -1323,7 +1334,7 @@ class Basic_TestClass < TestBase # makes sure the objects inside the block before are deleted GC.start - assert_equal(RBA::A.a0, ac0) + assert_equal(RBA::A.instance_count, ic0) assert_equal(RBA::A.a20_get == nil, true) # "unmanage" will freeze the object and not make it destroyed by the GC @@ -1331,7 +1342,7 @@ class Basic_TestClass < TestBase a = RBA::A::new a._create assert_equal(a.destroyed?, false) - assert_equal(RBA::A.a0, ac0 + 1) + assert_equal(RBA::A.instance_count, ic0 + 1) RBA::A.a20(a) # install static instance of A assert_equal(RBA::A.a20_get == nil, false) a._unmanage @@ -1340,7 +1351,7 @@ class Basic_TestClass < TestBase # makes sure the objects inside the block before are deleted GC.start - assert_equal(RBA::A.a0, ac0 + 1) + assert_equal(RBA::A.instance_count, ic0 + 1) assert_equal(RBA::A.a20_get == nil, false) # after "manage" the object gets volatile again @@ -1352,7 +1363,7 @@ class Basic_TestClass < TestBase # makes sure the objects inside the block before are deleted GC.start - assert_equal(RBA::A.a0, ac0) + assert_equal(RBA::A.instance_count, ic0) assert_equal(RBA::A.a20_get == nil, true) end @@ -1786,7 +1797,7 @@ class Basic_TestClass < TestBase assert_equal(1, yy.size) assert_equal("RBA::Y", yy[0].class.to_s) assert_equal(true, yy[0] != nil) - yy[0].destroy + yy[0]._destroy assert_equal(true, yy[0].destroyed?) assert_equal(yc, y.vx_dyn_count) From bcac78235b2b2675f5f38c9034219f274ffcb86b Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Fri, 4 Jan 2019 01:59:52 +0100 Subject: [PATCH 47/52] Fixed a testsuite fail after last commit. --- src/gsi/unit_tests/gsiExpression.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gsi/unit_tests/gsiExpression.cc b/src/gsi/unit_tests/gsiExpression.cc index 960c30c1a..ea87f5e8d 100644 --- a/src/gsi/unit_tests/gsiExpression.cc +++ b/src/gsi/unit_tests/gsiExpression.cc @@ -38,14 +38,14 @@ TEST(1) tl::Eval e; tl::Variant v; - v = e.parse ("A.a0").execute (); + v = e.parse ("A.instance_count").execute (); int base_insts = v.to_int (); EXPECT_EQ (base_insts, 0); v = e.parse ("A.new(35).to_s").execute (); EXPECT_EQ (v.to_string (), std::string ("A: 35")); - EXPECT_EQ (e.parse ("A.a0").execute ().to_int (), 0); + EXPECT_EQ (e.parse ("A.instance_count").execute ().to_int (), 0); // mapping of to_string to to_s method v = e.parse ("A.new(35)").execute (); @@ -97,11 +97,11 @@ TEST(1) EXPECT_EQ (v.to_string (), std::string ("0.125")); v = e.parse ("a.a4([5, 1, -1.25])").execute (); EXPECT_EQ (v.to_string (), std::string ("-1.25")); - v = e.parse ("A.a0").execute (); + v = e.parse ("A.instance_count").execute (); EXPECT_EQ (v.to_int (), base_insts + 1); // one instance more - v = e.parse ("a=1; A.a0").execute (); + v = e.parse ("a=1; A.instance_count").execute (); EXPECT_EQ (v.to_int (), base_insts); // remaining instances - v = e.parse ("A.a0").execute (); + v = e.parse ("A.instance_count").execute (); EXPECT_EQ (v.to_int (), base_insts); // remaining instances v = e.parse ("var a1=A.new; a1.a5(-15); var a2=a1.dup; a2.a5(107); a1.a1").execute (); From 68490f7891c06e47e46d4f12cfba105df4887123 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Fri, 4 Jan 2019 02:59:52 -0500 Subject: [PATCH 48/52] hosting microbits in bintray --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bc1cd1019..3317369dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,7 +44,7 @@ jobs: script: | # Required when targetType == Inline pwd dir - Invoke-WebRequest -Uri "https://www.dropbox.com/s/ioax2mnic4987kn/klayout-microbits-1.0.zip?dl=1" -MaximumRedirection 2 -OutFile klayout-microbits-1.0.zip + Invoke-WebRequest -Uri "https://dl.bintray.com/lightwave-lab/klayout/klayout-microbits-1.0.beta.zip" -MaximumRedirection 2 -OutFile klayout-microbits-1.0.beta.zip Expand-Archive klayout-microbits-1.0.zip -DestinationPath klayout-microbits dir klayout-microbits #errorActionPreference: 'stop' # Optional. Options: stop, continue, silentlyContinue From a68b8a8f37a812b3eac635e5b987f18740a47f74 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Fri, 4 Jan 2019 03:09:12 -0500 Subject: [PATCH 49/52] bintray download was not successful. --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3317369dd..0c25b9206 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,8 +43,8 @@ jobs: #arguments: # Optional script: | # Required when targetType == Inline pwd + Invoke-WebRequest -Uri "https://dl.bintray.com/lightwave-lab/klayout/klayout-microbits-1.0.beta.zip" -OutFile klayout-microbits-1.0.beta.zip dir - Invoke-WebRequest -Uri "https://dl.bintray.com/lightwave-lab/klayout/klayout-microbits-1.0.beta.zip" -MaximumRedirection 2 -OutFile klayout-microbits-1.0.beta.zip Expand-Archive klayout-microbits-1.0.zip -DestinationPath klayout-microbits dir klayout-microbits #errorActionPreference: 'stop' # Optional. Options: stop, continue, silentlyContinue From 7d3ecdbe027c21a39ed93509d641e5bf35638851 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Fri, 4 Jan 2019 03:10:56 -0500 Subject: [PATCH 50/52] fix squared --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0c25b9206..efe6112d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,7 +43,7 @@ jobs: #arguments: # Optional script: | # Required when targetType == Inline pwd - Invoke-WebRequest -Uri "https://dl.bintray.com/lightwave-lab/klayout/klayout-microbits-1.0.beta.zip" -OutFile klayout-microbits-1.0.beta.zip + Invoke-WebRequest -Uri "https://dl.bintray.com/lightwave-lab/klayout/klayout-microbits-1.0.beta.zip" -OutFile klayout-microbits-1.0.zip dir Expand-Archive klayout-microbits-1.0.zip -DestinationPath klayout-microbits dir klayout-microbits From 7d2bfc9799b9feb17dc3c73fdb94b697f035fd51 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Fri, 4 Jan 2019 03:31:05 -0500 Subject: [PATCH 51/52] microbits-1.0 released --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index efe6112d4..28804a941 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,7 +43,7 @@ jobs: #arguments: # Optional script: | # Required when targetType == Inline pwd - Invoke-WebRequest -Uri "https://dl.bintray.com/lightwave-lab/klayout/klayout-microbits-1.0.beta.zip" -OutFile klayout-microbits-1.0.zip + Invoke-WebRequest -Uri "https://dl.bintray.com/lightwave-lab/klayout/klayout-microbits-1.0.zip" -OutFile klayout-microbits-1.0.zip dir Expand-Archive klayout-microbits-1.0.zip -DestinationPath klayout-microbits dir klayout-microbits From df7f3de1a160ce5b3c5630397d9693741a48badb Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Fri, 4 Jan 2019 15:45:27 -0500 Subject: [PATCH 52/52] adding classifiers to setup.py I read in the PyPI user manual it was recommended. --- setup.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/setup.py b/setup.py index e7455d296..de9790db5 100644 --- a/setup.py +++ b/setup.py @@ -498,6 +498,17 @@ if __name__ == '__main__': long_description='TODO', author='Matthias Koefferlein', author_email='matthias@klayout.de', + classifiers=[ + # Recommended classifiers + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + # Optional classifiers + "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)", + ], url='https://github.com/klayoutmatthias/klayout', packages=find_packages('src/pymod/distutils_src'), package_dir={'': 'src/pymod/distutils_src'}, # https://github.com/pypa/setuptools/issues/230