From 5203f1123f33f93db18b5828b83d818ba115a333 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Thu, 5 May 2022 13:02:11 -0700 Subject: [PATCH 1/6] add ci/cd for github actions (#1070) * add ci/cd for github actions * fix workflow and add workflow_dispatch * remove dist * fix * add gcc * sudo apt install build essential * add curl * install curl * build essential * update * test code on mac os * release on tagged version * better names * fix numbers for 3.10 * add cancel workflow for tests * fix test macOs * Adding ccache to github action * creating dependency between jobs * using cibuildwheel * ccache path bug * Testing after building wheels * misunderstood cibuildwheel * fixing error in manylinux build * fixing error in manylinux build (2) * disabling fail fast (temporary) * Don't repair macOS wheels * Not building pypy wheels * disabling 32-bit linux wheels * macos-latest only (prevent duplicate) * fixing musllinux * fixing ccache * fixing cache persistence * adding deploy job * last debug Co-authored-by: Thomas Ferreira de Lima --- .github/workflows/build.yml | 96 +++++++++++++++++++++++++++++ ci-scripts/docker/docker_prepare.sh | 50 +++++++++++++++ pyproject.toml | 27 ++++++++ 3 files changed, 173 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 ci-scripts/docker/docker_prepare.sh create mode 100644 pyproject.toml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..3d7a21dd3 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,96 @@ +--- +name: Build Python Wheels +# https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python + +on: + pull_request: + push: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + max-parallel: 12 + matrix: + include: + - os: "macos-latest" + cibuild: "*macosx*" + - os: "ubuntu-latest" + cibuild: "*manylinux*" + - os: "ubuntu-latest" + cibuild: "*musllinux*" + steps: + - name: Cancel Workflow Action + uses: styfle/cancel-workflow-action@0.9.1 + - uses: actions/checkout@v3 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ github.job }}-${{ matrix.os }}-${{ matrix.cibuild }} # Make cache specific to OS + max-size: "5G" + - name: Install dependencies + run: | + env + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> $GITHUB_PATH + HOST_CCACHE_DIR="$(ccache -k cache_dir)" + mkdir -p $HOST_CCACHE_DIR + - name: Build wheels # check https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions + uses: pypa/cibuildwheel@v2.5.0 + # to supply options, put them in 'env', like: + # env: + # CIBW_SOME_OPTION: value + env: + CIBW_BUILD: ${{ matrix.cibuild }} + - name: Download Cache from Docker (linux only) + if: ${{ runner.os == 'Linux' }} + # hack until https://github.com/pypa/cibuildwheel/issues/1030 is fixed + run: | + env + ccache -s + HOST_CCACHE_DIR="$(ccache -k cache_dir)" + rm -rf $HOST_CCACHE_DIR + mv ./wheelhouse/.ccache $HOST_CCACHE_DIR + ls -la $HOST_CCACHE_DIR + ccache -s + - uses: actions/upload-artifact@v2 + with: + path: ./wheelhouse/*.whl + +# The following was taken from https://cibuildwheel.readthedocs.io/en/stable/deliver-to-pypi/ + make_sdist: + name: Make SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Optional, use if you use setuptools_scm + submodules: true # Optional, use if you have submodules + + - name: Build SDist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + upload_all: + needs: [build, make_sdist] + runs-on: ubuntu-latest + # Uncomment for real PyPi + # if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + user: __token__ + # Test PyPI + password: ${{ secrets.test_pypi_password }} + repository_url: https://test.pypi.org/legacy/ + # Uncomment for Real Pypi + # password: ${{ secrets.pypi_password }} diff --git a/ci-scripts/docker/docker_prepare.sh b/ci-scripts/docker/docker_prepare.sh new file mode 100644 index 000000000..e380fdc45 --- /dev/null +++ b/ci-scripts/docker/docker_prepare.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -xe + +# manylinux prep +if [[ -f "/etc/centos-release" ]]; then + # sometimes the epel server is down. retry 5 times + for i in $(seq 1 5); do + yum install -y zlib-devel curl-devel expat-devel ccache && s=0 && break || s=$? && sleep 15; + done + + [ $s -eq 0 ] || exit $s + + if [[ -d "/usr/lib64/ccache" ]]; then + ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -s /usr/bin/ccache /usr/lib64/ccache/cc + ln -s /usr/bin/ccache /usr/lib64/ccache/gcc + ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + export PATH="/usr/lib64/ccache:$PATH" + elif [[ -d "/usr/lib/ccache" ]]; then + ln -s /usr/bin/ccache /usr/lib/ccache/c++ + ln -s /usr/bin/ccache /usr/lib/ccache/cc + ln -s /usr/bin/ccache /usr/lib/ccache/gcc + ln -s /usr/bin/ccache /usr/lib/ccache/g++ + export PATH="/usr/lib/ccache:$PATH" + fi + +elif [[ -f "/etc/alpine-release" ]]; then + # musllinux prep + # ccache already present + apk add curl-dev expat-dev zlib-dev ccache + export PATH="/usr/lib/ccache/bin:$PATH" +fi + +# hack until https://github.com/pypa/cibuildwheel/issues/1030 is fixed +# Place ccache folder in /outputs +HOST_CCACHE_DIR="/host${HOST_CCACHE_DIR:-/home/runner/work/klayout/klayout/.ccache}" +if [ -d $HOST_CCACHE_DIR ]; then + mkdir -p /output + cp -R $HOST_CCACHE_DIR /output/.ccache +fi + +ls -la /output/ + +ccache -o cache_dir="/output/.ccache" +# export CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" +ccache -M 5 G # set cache size to 5 G + +# Show ccache stats +echo "Cache stats:" +ccache -s diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..5e3833538 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +[tool.cibuildwheel] +build-verbosity = "3" +test-command = [ + "python {package}/testdata/pymod/import_db.py", + "python {package}/testdata/pymod/import_rdb.py", + "python {package}/testdata/pymod/import_tl.py", + "python {package}/testdata/pymod/import_lib.py", + "python {package}/testdata/pymod/pya_tests.py" +] +# Disable building PyPy wheels on all platforms +skip = "pp*" + +[tool.cibuildwheel.linux] +# beware: the before-all script does not persist environment variables! +# No point in defining $PATH or $CCACHE_DIR +before-all = "source {project}/ci-scripts/docker/docker_prepare.sh" +archs = ["auto64"] +# Append a directory to the PATH variable (this is expanded in the build environment) +environment = { PATH="/usr/lib/ccache:/usr/lib64/ccache:/usr/lib/ccache/bin:$PATH" } +before-build = "ccache -s" +# Export a variable to docker +# CCACHE_DIR="/home/runner/work/klayout/klayout/.ccache" +environment-pass = ["HOST_CCACHE_DIR"] + +[tool.cibuildwheel.macos] +# Don't repair macOS wheels +repair-wheel-command = "" \ No newline at end of file From 2b693ec39b52668cb1b1af4f306c9dfdf3b4c545 Mon Sep 17 00:00:00 2001 From: Kazunari Sekigawa Date: Fri, 6 May 2022 05:01:40 +0900 Subject: [PATCH 2/6] Updated the build system for Mac (#1073) * WIP: add "pymod" * To fix the issue reported by GitHub ticket No.1040 * WIP: To build "pymod" * Updated the build system for Mac. * Updated the build system for Mac. --- macbuild/ReadMe.md | 34 +- macbuild/Resources/script-bundle-A.zip | Bin 4440184 -> 4441255 bytes macbuild/Resources/script-bundle-B.zip | Bin 4441504 -> 4442749 bytes macbuild/Resources/script-bundle-P.zip | Bin 4463962 -> 4465202 bytes macbuild/build4mac.py | 451 ++++++++++++++++++------- 5 files changed, 348 insertions(+), 137 deletions(-) diff --git a/macbuild/ReadMe.md b/macbuild/ReadMe.md index 929985b58..2ba660933 100644 --- a/macbuild/ReadMe.md +++ b/macbuild/ReadMe.md @@ -1,9 +1,9 @@ -Relevant KLayout version: 0.27.5
+Relevant KLayout version: 0.27.9
Author: Kazzz-S
-Last modified: 2021-11-27
+Last modified: 2022-04-30
# 1. Introduction -This directory **`macbuild`** contains different files required for building KLayout (http://www.klayout.de/) version 0.27.5 or later for different 64-bit macOS, including: +This directory **`macbuild`** contains different files required for building KLayout (http://www.klayout.de/) version 0.27.9 or later for different 64-bit macOS, including: * Catalina (10.15.7) : the primary development environment * Big Sur (11.x) : experimental; Apple M1 chip is not tested since the author does not own an M1 Mac * Monterey (12.x) : -- ditto -- @@ -63,8 +63,9 @@ $ /usr/bin/python --version Python 2.7.16 ``` -Big Sur (11.x) and Monterey (12.x) still provide the Python 2.7 binaries to run various legacy applications.
+Big Sur (11.x) and Monterey (< 12.3) still provide the Python 2.7 binaries to run various legacy applications.
However, the latest Xcode 13.1 does not allow us to link the legacy Python 2.7 library with the newly compiled KLayout binaries.
+Moreover, Monterey (12.3.1) finally eliminated the Python 2.7 binaries.
Therefore, Homebrew is adopted as the default environment for Big Sur and Monterey. The build script **`build4mac.py`** provides several possible combinations of Qt[6|5], Ruy, and Python modules to accommodate such a slightly complex environment.
@@ -82,7 +83,7 @@ You need to have: ``` --------------------------------------------------------------------------------------------------------- << Usage of 'build4mac.py' >> - for building KLayout 0.27.5 or later on different Apple macOS / Mac OSX platforms. + for building KLayout 0.27.9 or later on different Apple macOS / Mac OSX platforms. $ [python] ./build4mac.py option & argument : descriptions (refer to 'macbuild/build4mac_env.py' for details)| default value @@ -109,6 +110,7 @@ $ [python] ./build4mac.py : Ana3: use Python 3.8 from Anaconda3 | : HB39: use Python 3.9 from Homebrew | : HBAuto: use the latest Python 3.x auto-detected from Homebrew | + [-j|--jump2pymod] : jump into build (developer's use only) | disabled [-n|--noqtbinding] : don't create Qt bindings for ruby scripts | disabled [-u|--noqtuitools] : don't include uitools in Qt binding | disabled [-m|--make