Merge branch 'main' into nick/fix-6209

.github/workflows/wheels.yml

@@ -55,24 +55,8 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: "3.x"
-      - name: Get FFI
-        shell: bash
-        run: |
-          mkdir -p ffi
-          curl -L https://github.com/libffi/libffi/releases/download/v3.4.8/libffi-3.4.8.tar.gz | tar --strip-components=1 -xzC ffi
-      - if: ${{ matrix.os.family == 'linux' }}
-        name: "[Linux] Bison 3.8.2"
-        shell: bash
-        run: |
-          mkdir -p bison
-          curl -L https://ftp.gnu.org/gnu/bison/bison-3.8.2.tar.gz | tar --strip-components=1 -xzC bison
       ## Software installed by default in GitHub Action Runner VMs:
       ##  https://github.com/actions/runner-images
-      - if: ${{ matrix.os.family == 'macos' }}
-        name: "[macOS] Flex/Bison"
-        run: |
-          brew install flex bison
-          echo "PATH=$(brew --prefix flex)/bin:$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV
       - if: ${{ matrix.os.family == 'windows' }}
         name: "[Windows] Flex/Bison"
         run: |
@@ -80,14 +64,16 @@ jobs:
       - name: Build wheels
         uses: pypa/[email protected]
         env:
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
           # * APIs not supported by PyPy
           # * Musllinux disabled because it increases build time from 48m to ~3h
           CIBW_SKIP: >
-            cp39*
             *musllinux*
           CIBW_ARCHS: ${{ matrix.os.archs }}
           CIBW_BUILD_VERBOSITY: "1"
           # manylinux2014 (default) does not have a modern enough C++ compiler for Yosys
+          # update: it technically does, devtoolset-11, but it's a matter of
+          #         time before it doesn't
           CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
           CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
           CIBW_BEFORE_ALL: bash ./.github/workflows/wheels/cibw_before_all.sh
@@ -95,8 +81,8 @@ jobs:
             cmake="-DCMAKE_BUILD_TYPE=Release -DYOSYS_ENABLE_HOMEBREW:BOOL=OFF"
           CIBW_ENVIRONMENT: >
             OPTFLAGS=-O3
-            PKG_CONFIG_PATH={project}/ffi/pfx/lib/pkgconfig
-            PATH="{project}/bison/src:$PATH"
+            PKG_CONFIG_PATH="$PWD/ffi/pfx/lib/pkgconfig:$PKG_CONFIG_PATH"
+            PATH="$PWD/bison/pfx/bin:$PWD/flex/pfx/bin:$PATH"
             MACOSX_DEPLOYMENT_TARGET=11
           CIBW_BEFORE_BUILD: bash ./.github/workflows/wheels/cibw_before_build.sh
           CIBW_TEST_COMMAND: python3 {project}/tests/pyosys/run_tests.py

.github/workflows/wheels/_test_cibw_local.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 UmbraLogic Technologies LLC
+# SPDX-License-Identifier: MIT
+"""
+This runs the cibuildwheel step from the wheels workflow locally.
+"""
+
+import os
+import sys
+import platform
+import subprocess
+from pathlib import Path
+
+import yaml
+from packaging.tags import sys_tags
+
+__yosys_root__ = Path(__file__).absolute().parents[3]
+
+with open(__yosys_root__ / ".github" / "workflows" / "wheels.yml") as f:
+	workflow = yaml.safe_load(f)
+
+env = os.environ.copy()
+
+steps = workflow["jobs"]["build_wheels"]["steps"]
+cibw_step = None
+for step in steps:
+	if (step.get("uses") or "").startswith("pypa/cibuildwheel"):
+		cibw_step = step
+		break
+
+env_filter = {
+	"linux": ("_WINDOWS", "_MAC"),
+	"darwin": ("_LINUX", "_WINDOWS"),
+	"win32": ("_LINUX", "_MAC"),
+}[sys.platform]
+for key, value in cibw_step["env"].items():
+	if key.endswith(env_filter):
+		continue
+	if key not in env:  # prioritize user-set keys
+		env[key] = value
+
+python_tag = next(sys_tags()).interpreter
+env["CIBW_BUILD"] = os.getenv("CIBW_BUILD", f"{python_tag}-*")
+env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS", platform.machine())
+subprocess.check_call(["cibuildwheel"], env=env)

.github/workflows/wheels/cibw_before_all.sh

@@ -1,35 +1,80 @@
 #!/usr/bin/env bash
 set -e -x
+set -o pipefail
 
 # Build-time dependencies
 ## Linux Docker Images
-if command -v yum &> /dev/null; then
-    yum install -y flex # manylinux's bison versions are hopelessly out of date
-fi
-
 if command -v apk &> /dev/null; then
     apk add flex bison
 fi
 
+## Platform-independent
+### just build bison if it's missing or out of date
+BISON_VER="3.8.2"
+BISON_SHA256SUM="9bba0214ccf7f1079c5d59210045227bcf619519840ebfa80cd3849cff5a5bf2"
+which -a bison || true
 if ! printf '%s\n' '%require "3.8"' '%%' 'start: ;' | bison -o /dev/null /dev/stdin ; then
+	PREFIX=$PWD/bison/pfx
+	rm -rf $PREFIX
+	BISON_SRC=$(mktemp -d)
 	(
 		set -e -x
-		cd bison
-		./configure
-		make clean
+		cd $BISON_SRC
+		curl --connect-timeout 5 --retry 3 -L https://ftp.gnu.org/gnu/bison/bison-${BISON_VER}.tar.xz > bison.tar.xz
+		echo "$BISON_SHA256SUM bison.tar.xz" | sha256sum -c -
+		tar --strip-components=1 -xJC . -f bison.tar.xz
+		./configure --prefix=$PREFIX
 		make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
 	)
+	rm -rf $BISON_SRC
 fi
 
-## macOS/Windows -- installed in GitHub Action itself, not container
+
+### The Xcode Command Line Tools ersion of flex has an Apple-specific
+### extension where int is substituted for size_t in some parts of the codebase.
+### This is not compatible with Yosys.
+###
+### Also, the AlmaLinux repos are slow.
+###
+### It's best to just build flex if we don't like its output.
+TMP_LEXER=$(mktemp)
+FLEX_VER="2.6.4"
+FLEX_SHA256SUM="e87aae032bf07c26f85ac0ed3250998c37621d95f8bd748b31f15b33c45ee995"
+which -a flex || true
+if ! ( printf "\n%%%%\n. { }\n%%%%\n" | flex -+ -o $TMP_LEXER && grep "yyFlexLexer::LexerOutput" $TMP_LEXER | grep -v 'size_t' ) ; then
+	PREFIX=$PWD/flex/pfx
+	rm -rf $PREFIX
+	FLEX_SRC=$(mktemp -d)
+	(
+		set -e -x
+		cd $FLEX_SRC
+		curl --connect-timeout 5 --retry 3 -L https://github.com/westes/flex/releases/download/v${FLEX_VER}/flex-${FLEX_VER}.tar.gz > flex.tar.gz
+		echo "$FLEX_SHA256SUM flex.tar.gz" | sha256sum -c -
+		tar --strip-components=1 -xzC . -f flex.tar.gz
+		./configure --prefix=$PREFIX --disable-shared --disable-silent-rules CFLAGS=-D_GNU_SOURCE
+		make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
+	)
+	rm -rf $FLEX_SRC
+fi
 
 # Runtime Dependencies
-## Build Static FFI (platform-dependent but not Python version dependent)
-(
-	set -e -x
-	cd ffi
-	## Ultimate libyosys.so will be shared, so we need fPIC for the static libraries
-	LDFLAGS=-fPIC CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx --enable-static --disable-shared
-	make clean
-	make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
-)
+## Build Static FFI (platform-dependent but not Python version dependent) if
+## missing
+LIBFFI_VER="3.4.8"
+LIBFFI_SHA256SUM="bc9842a18898bfacb0ed1252c4febcc7e78fa139fd27fdc7a3e30d9d9356119b"
+if ! echo "int main() {}" | cc -x c -l:libffi.a - ; then
+	PREFIX=$PWD/ffi/pfx
+	rm -rf $PREFIX
+	LIBFFI_SRC=$(mktemp -d)
+	(
+		set -e -x
+		cd $LIBFFI_SRC
+		curl --connect-timeout 5 --retry 3 -L "https://github.com/libffi/libffi/releases/download/v${LIBFFI_VER}/libffi-${LIBFFI_VER}.tar.gz" > libffi.tar.gz
+		echo "$LIBFFI_SHA256SUM libffi.tar.gz" | sha256sum -c -
+		tar --strip-components=1 -xzC . -f libffi.tar.gz
+		## Ultimate libyosys.so will be shared, so we need fPIC for the static libraries
+		LDFLAGS=-fPIC CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PREFIX --enable-static --disable-shared
+		make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
+	)
+	rm -rf $LIBFFI_SRC
+fi

.gitignore

@@ -61,9 +61,11 @@
 /kernel/python_wrappers.cc
 /ffi
 /bison
+/flex
 /venv
 /*.whl
 /*.egg-info
+/wheelhouse
 
 # yosysjs dependency
 /viz.js

passes/opt/dff/opt_dff.cc

@@ -37,6 +37,8 @@ OptDffWorker::OptDffWorker(const OptDffOptions &opt, Module *mod)
 
 void OptDffWorker::remove_ff_bits(Cell *cell, const pool<int> &drop)
 {
+	modwalker_ptr.reset();
+
 	FfData ff(&initvals, cell);
 	std::vector<int> keep;
 	for (int i = 0; i < ff.width; i++)

pyproject.toml

@@ -10,4 +10,7 @@ build-backend = "local_backend"
 
 [tool.ruff]
 target-version = "py310"
-lint.ignore = ["F541"]
+lint.ignore = [
+	"F541", # f-string missing placeholders
+	"W191" , # tab indentation
+]