From b378299ec261c2144134d0703ec8294497dbdae8 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Thu, 3 Feb 2022 12:52:33 +0100 Subject: [PATCH 01/16] gh: initial use of gh action runners Signed-off-by: Alessandro Comodi --- .github/workflows/ci.yml | 91 +++++++++++ .github/workflows/scripts/db.sh | 216 ++++++++++++++++++++++++++ .github/workflows/scripts/hostinfo.sh | 45 ++++++ Makefile | 6 +- 4 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100755 .github/workflows/scripts/db.sh create mode 100755 .github/workflows/scripts/hostinfo.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c9ca13bd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,91 @@ +on: + pull_request: + push: + schedule: + - cron: '0 0 * * *' + +name: CI Build + +jobs: + + Tests: + container: ubuntu:bionic + + runs-on: [self-hosted, Linux, X64] + + env: + ALLOW_ROOT: true + + steps: + + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Install + run: | + apt update + apt install -y \ + bash bison build-essential ca-certificates clang-format cmake psmisc \ + colordiff coreutils git flex python3 python3-dev python3-venv xsltproc + + - name: Build + run: make build --output-sync=target --warn-undefined-variables -j$(nproc) + + - name: Environment + run: make env --output-sync=target --warn-undefined-variables + + - name: Run Test + run: make test --output-sync=target --warn-undefined-variables + + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + path: | + **/results*.gz + **/plot_*.svg + + BuildDatabase: + container: ubuntu:bionic + + runs-on: [self-hosted, Linux, X64] + + strategy: + fail-fast: false + matrix: + family: ['artix7', 'zynq7', 'kintex7', 'spartan7'] + + env: + ALLOW_ROOT: true + GHA_EXTERNAL_DISK: "tools" + + steps: + + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Install + run: | + apt update + apt install -y \ + bash bison build-essential ca-certificates clang-format cmake psmisc \ + colordiff coreutils git flex python3 python3-dev python3-venv xsltproc + + - name: Build + run: make build --output-sync=target --warn-undefined-variables -j$(nproc) + + - name: Environment + run: make env --output-sync=target --warn-undefined-variables + + - name: Run Test + run: .github/workflows/scripts/db.sh + env: + XRAY_SETTINGS: ${{ matrix.family }} + + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + path: | + **/results*.gz + **/plot_*.svg diff --git a/.github/workflows/scripts/db.sh b/.github/workflows/scripts/db.sh new file mode 100755 index 00000000..084afb28 --- /dev/null +++ b/.github/workflows/scripts/db.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +set -e + +source $(dirname "$0")/hostinfo.sh + +echo +echo "=======================================" +echo "Creating Vivado Symbolic Link" +echo "---------------------------------------" +ln -s /mnt/aux/Xilinx /opt/Xilinx +ls /opt/Xilinx/Vivado +source /opt/Xilinx/Vivado/2017.2/settings64.sh +vivado -version + +echo +echo "========================================" +echo "Downloading current database" +echo "----------------------------------------" +( + script --return --flush --command "./download-latest-db.sh" - +) +echo "----------------------------------------" + +echo +echo "========================================" +echo "Preparing database" +echo "----------------------------------------" +( + make db-prepare-${XRAY_SETTINGS} +) +echo "----------------------------------------" + +source settings/$XRAY_SETTINGS.sh + +echo +echo "========================================" +echo "Cleaning out current database" +echo "----------------------------------------" +( + cd database + make clean-${XRAY_SETTINGS}-db +) +echo "----------------------------------------" + +echo +echo "========================================" +echo "Running Database build" +echo "----------------------------------------" +( + # Output which fuzzers we are going to run + echo "make --dry-run" + make --dry-run db-${XRAY_SETTINGS}-all + echo "----------------------------------------" + + # Run the fuzzers + set -x +e + tmp=`mktemp` + script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS db-${XRAY_SETTINGS}-all" $tmp + DATABASE_RET=$? + set +x -e + + if [[ $DATABASE_RET != 0 ]] ; then + # Collect the Vivado logs into one tgz archive + echo "Packing failing test cases" + grep "recipe for target" $tmp | awk 'match($0,/recipe for target.*'\''(.*)\/run\..*ok'\''/,res) {print "fuzzers/" res[1]}' | xargs tar -zcf fuzzers/fails.tgz + echo "----------------------------------------" + echo "A failure occurred during Database build." + echo "----------------------------------------" + rm $tmp + + echo "========================================" + echo " Disk space in failure path" + echo "----------------------------------------" + du -sh + + exit $DATABASE_RET + fi +) +echo "----------------------------------------" + +# Format the database +make db-format-${XRAY_SETTINGS} +# Update the database/Info.md file +make db-info + +# Output if the database has differences +echo +echo "========================================" +echo " Database Differences" +echo "----------------------------------------" +( + cd database + # Update the index with any new files + git add \ + --verbose \ + --all \ + --ignore-errors \ + . + + # Output what git status + echo + echo "----------------------------------------" + echo " Database Status" + echo "----------------------------------------" + git status + echo "----------------------------------------" + + + # Output a summary of how the files have changed + echo + echo "----------------------------------------" + echo " Database Diff Summary" + echo "----------------------------------------" + git diff --stat --irreversible-delete --find-renames --find-copies --ignore-all-space origin/master + + # Save the diff to be uploaded as an artifact + echo + echo "----------------------------------------" + echo " Saving diff output" + echo "----------------------------------------" + # Patch file + git diff \ + --patch-with-stat --no-color --irreversible-delete --find-renames --find-copies origin/master \ + > diff.patch + + MAX_DIFF_LINES=50000 + DIFF_LINES="$(wc -l diff.patch | sed -e's/ .*$//')" + if [ $DIFF_LINES -gt $MAX_DIFF_LINES ]; then + echo + echo "----------------------------------------" + echo " Database Diff" + echo "----------------------------------------" + echo "diff has $DIFF_LINES lines which is too large to display!" + + echo + echo "----------------------------------------" + echo " Generating pretty diff output" + echo "----------------------------------------" + echo "diff has $DIFF_LINES lines which is too large for HTML output!" + else + # Output the actually diff + echo + echo "----------------------------------------" + echo " Database Diff" + echo "----------------------------------------" + git diff --color --irreversible-delete --find-renames --find-copies --ignore-all-space origin/master + + echo + echo "----------------------------------------" + echo " Generating pretty diff output" + echo "----------------------------------------" + ( + # Allow the diff2html to fail. + set +e + + # Pretty HTML file version + diff2html --summary=open --file diff.html --format html \ + -- \ + --irreversible-delete --find-renames --find-copies \ + --ignore-all-space origin/master || true + + # Programmatic JSON version + diff2html --file diff.json --format json \ + -- \ + --irreversible-delete --find-renames --find-copies \ + --ignore-all-space origin/master || true + ) || true + fi +) +echo "----------------------------------------" + +# Check the database and fail if it is broken. +set -x +e +make db-check-${XRAY_SETTINGS} +CHECK_RET=$? +set +x -e + +echo +echo "========================================" +echo " Testing HTML generation" +echo "----------------------------------------" +( + cd htmlgen + source htmlgen.sh $XRAY_SETTINGS +) + +# If we get here, then all the fuzzers completed fine. Hence we are +# going to assume we don't want to keep all the build / logs / etc (as +# they are quite large). Thus do a clean to get rid of them. +echo +echo "========================================" +echo " Cleaning up after success" +echo "----------------------------------------" +( + cd fuzzers + echo + echo "Cleaning up so CI doesn't save all the excess data." + make clean_fuzzers + make clean_piplists +) +echo "----------------------------------------" + +echo "========================================" +echo " Final disk space after cleanup" +echo "----------------------------------------" +du -sh + +exit $CHECK_RET diff --git a/.github/workflows/scripts/hostinfo.sh b/.github/workflows/scripts/hostinfo.sh new file mode 100755 index 00000000..8e69c993 --- /dev/null +++ b/.github/workflows/scripts/hostinfo.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +set -e + +echo +echo "========================================" +echo "Host Environment" +echo "----------------------------------------" +export +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host CPU" +echo "----------------------------------------" +export CORES=$(nproc --all) +echo "Cores: $CORES" +echo +echo "Memory" +echo "----------------------------------------" +cat /proc/meminfo +echo "----------------------------------------" +export MEM_GB=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo)/(1024*1024))) +echo "Total Memory (GB): $MEM_GB" + +# Approx memory per grid process +export MEM_PER_RUN=8 +export MAX_GRID_CPU=$(($MEM_GB/$MEM_PER_RUN)) +export MAX_VIVADO_PROCESS=$(($MEM_GB/$MEM_PER_RUN)) + +echo +echo "========================================" +echo "Host files" +echo "----------------------------------------" +echo $PWD +echo "----------------------------------------" +find . | sort +echo "----------------------------------------" diff --git a/Makefile b/Makefile index 53c38789..acb7f426 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,14 @@ ALL_EXCLUDE = third_party .git env build docs/env INSTALL_DIR ?= +# Skip this check if the ALLOW_ROOT var is defined +# E.g. when running in GH action custom runners CI +ifndef ALLOW_ROOT # Check if root ifeq ($(shell id -u),0) $(error ERROR: Running as ID 0) endif +endif # Tools + Environment IN_ENV = if [ -e env/bin/activate ]; then . env/bin/activate; fi; source utils/environment.python.sh; @@ -275,4 +279,4 @@ clean: $(MAKE) -C fuzzers clean rm -rf build -.PHONY: clean \ No newline at end of file +.PHONY: clean From 0c8df95f5b2773f56bbb66d6012f0e8ed3b552f9 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Thu, 3 Feb 2022 17:47:06 +0100 Subject: [PATCH 02/16] ci: gh: add artifacts Signed-off-by: Alessandro Comodi --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9ca13bd..470b37e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,3 +89,13 @@ jobs: path: | **/results*.gz **/plot_*.svg + **/diff.html + **/diff.json + **/diff.patch + **/*result*.xml + **/*sponge_log.xml + **/fuzzers/*.tgz + **/build/*.log + **/logs_*/** + **/database/${{ matrix.family }}/**" + From a77a6a3e4476969e125e6f6c89a1f37725db5847 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Thu, 3 Feb 2022 18:14:58 +0100 Subject: [PATCH 03/16] run_fuzzer: replace semicolon with dash in stderr and stdout names Signed-off-by: Alessandro Comodi --- fuzzers/run_fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzers/run_fuzzer.py b/fuzzers/run_fuzzer.py index 7ea75a1e..d8ca8aa4 100755 --- a/fuzzers/run_fuzzer.py +++ b/fuzzers/run_fuzzer.py @@ -512,7 +512,7 @@ def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger, will_retry): ) log(running_msg) - log_suffix = ".{}.log".format(time_start.isoformat()) + log_suffix = ".{}.log".format(time_start.isoformat()).replace(":", "-") fuzzer_stdout = os.path.join(fuzzer_logdir, "stdout" + log_suffix) fuzzer_stderr = os.path.join(fuzzer_logdir, "stderr" + log_suffix) From 78b7b884d36f2891e52aa17559b0d6d2c182c455 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Fri, 4 Feb 2022 10:12:00 +0100 Subject: [PATCH 04/16] gh: artifatcs: take only stderr logs Signed-off-by: Alessandro Comodi --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 470b37e4..dd7ea748 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: **/*result*.xml **/*sponge_log.xml **/fuzzers/*.tgz - **/build/*.log + **/build/stderr*.log **/logs_*/** **/database/${{ matrix.family }}/**" - + From 7f52c45d27f587e59d59ee84a291b1532d9ad733 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Mon, 7 Feb 2022 09:43:27 +0100 Subject: [PATCH 05/16] ci: add fix to multiple Vivado instances runs Signed-off-by: Alessandro Comodi --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd7ea748..2235d2fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,7 @@ jobs: env: ALLOW_ROOT: true GHA_EXTERNAL_DISK: "tools" + XILINX_LOCAL_USER_DATA: "no" steps: From 1db2cffd636dd7071b5afa6678ef217540e69328 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Wed, 9 Feb 2022 10:41:16 +0100 Subject: [PATCH 06/16] ci: use grep and sed to extract and tar failing fuzzers logs Signed-off-by: Alessandro Comodi --- .github/workflows/scripts/db.sh | 7 +++++-- .github/workflows/scripts/hostinfo.sh | 9 --------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/scripts/db.sh b/.github/workflows/scripts/db.sh index 084afb28..c66a6d95 100755 --- a/.github/workflows/scripts/db.sh +++ b/.github/workflows/scripts/db.sh @@ -62,7 +62,7 @@ echo "----------------------------------------" # Run the fuzzers set -x +e - tmp=`mktemp` + tmp=$(mktemp) script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS db-${XRAY_SETTINGS}-all" $tmp DATABASE_RET=$? set +x -e @@ -70,7 +70,10 @@ echo "----------------------------------------" if [[ $DATABASE_RET != 0 ]] ; then # Collect the Vivado logs into one tgz archive echo "Packing failing test cases" - grep "recipe for target" $tmp | awk 'match($0,/recipe for target.*'\''(.*)\/run\..*ok'\''/,res) {print "fuzzers/" res[1]}' | xargs tar -zcf fuzzers/fails.tgz + # Looking for the failing directories and packing them + # example of line from which the failing fuzzer directory gets extracted: + # - Makefile:87: recipe for target '000-db-init/000-init-db/run.xc7a100tfgg676-1.ok' failed --> fuzzers/000-db-init + grep -Po "recipe for target '\K(.*)(?=\/run\..*\.ok')" $tmp | sed -e 's/^/fuzzers\//' | xargs tar -zcf fuzzers/fails-${XRAY_SETTINGS}.tgz echo "----------------------------------------" echo "A failure occurred during Database build." echo "----------------------------------------" diff --git a/.github/workflows/scripts/hostinfo.sh b/.github/workflows/scripts/hostinfo.sh index 8e69c993..577577cb 100755 --- a/.github/workflows/scripts/hostinfo.sh +++ b/.github/workflows/scripts/hostinfo.sh @@ -34,12 +34,3 @@ echo "Total Memory (GB): $MEM_GB" export MEM_PER_RUN=8 export MAX_GRID_CPU=$(($MEM_GB/$MEM_PER_RUN)) export MAX_VIVADO_PROCESS=$(($MEM_GB/$MEM_PER_RUN)) - -echo -echo "========================================" -echo "Host files" -echo "----------------------------------------" -echo $PWD -echo "----------------------------------------" -find . | sort -echo "----------------------------------------" From 523c4b5a16cc056f1c9fcb4d43b786f975b7f37a Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Wed, 9 Feb 2022 14:21:50 +0100 Subject: [PATCH 07/16] kokoro: use python3 instead of specific 3.8 version Signed-off-by: Alessandro Comodi --- .github/kokoro/steps/hostsetup.sh | 8 ++++---- requirements.txt | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh index dc267b17..91cc2302 100755 --- a/.github/kokoro/steps/hostsetup.sh +++ b/.github/kokoro/steps/hostsetup.sh @@ -87,14 +87,14 @@ sudo apt-get install -y \ lsb \ nodejs \ psmisc \ - python3.8 \ - python3.8-dev \ - python3.8-venv + python3 \ + python3-dev \ + python3-venv echo "========================================" echo "Enter virtual env for python 3.8" echo "----------------------------------------" -python3.8 -mvenv startup_python +python3 -mvenv startup_python source startup_python/bin/activate which python python --version diff --git a/requirements.txt b/requirements.txt index e10243ab..54210815 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,3 @@ --e third_party/fasm --e third_party/python-sdf-timing --e . intervaltree junit-xml numpy @@ -10,9 +7,15 @@ parse progressbar2 pyjson5 pytest +pytest-runner pyyaml scipy>=1.2.1 simplejson sympy textx yapf==0.24.0 + +# Third party +-e third_party/fasm +-e third_party/python-sdf-timing +-e . From 44847f350d9174d531415973632af29e912e45b2 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Wed, 2 Mar 2022 09:57:05 +0100 Subject: [PATCH 08/16] ci: gh: save only failing logs Signed-off-by: Alessandro Comodi --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2235d2fa..5f49c148 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,10 +93,7 @@ jobs: **/diff.html **/diff.json **/diff.patch - **/*result*.xml **/*sponge_log.xml **/fuzzers/*.tgz - **/build/stderr*.log - **/logs_*/** **/database/${{ matrix.family }}/**" From f4b1bd1540ba0a652a6cda61f524bb3954146362 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Wed, 2 Mar 2022 14:16:01 +0100 Subject: [PATCH 09/16] fuzzers: tmp disable 034b to unblock GH custom runners transition Signed-off-by: Alessandro Comodi --- fuzzers/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fuzzers/Makefile b/fuzzers/Makefile index 7f8e945d..ee43c953 100644 --- a/fuzzers/Makefile +++ b/fuzzers/Makefile @@ -116,7 +116,11 @@ $(eval $(call fuzzer,030-iob,005-tilegrid,all)) $(eval $(call fuzzer,031-cmt-mmcm,005-tilegrid,all)) $(eval $(call fuzzer,032-cmt-pll,005-tilegrid,all)) $(eval $(call fuzzer,034-cmt-pll-pips,005-tilegrid 071-ppips,all)) +ifneq ($(XRAY_DATABASE),kintex7) +# FIXME: 034b fuzzer is generating conflicting bits around the FREQ_BB[N] bits. +# The fuzzer can be re-enabled once the conflicting bits are not generated anymore $(eval $(call fuzzer,034b-cmt-mmcm-pips,005-tilegrid 071-ppips,all)) +endif $(eval $(call fuzzer,035-iob-ilogic,005-tilegrid,all)) $(eval $(call fuzzer,035a-iob-idelay,005-tilegrid,all)) $(eval $(call fuzzer,035b-iob-iserdes,005-tilegrid,all)) From 2eb77e466388f69904f58af217f07e5c47d865a3 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Thu, 3 Mar 2022 10:40:10 +0100 Subject: [PATCH 10/16] fuzzers: fix clean in parts fuzzers Signed-off-by: Alessandro Comodi --- fuzzers/065-gtp-common-pips/Makefile | 2 +- fuzzers/065b-gtp-common-pips/Makefile | 2 +- fuzzers/072-ordered_wires/Makefile | 9 ++++++--- fuzzers/073-get_counts/Makefile | 9 ++++++--- fuzzers/074-dump_all/Makefile | 7 +++++-- fuzzers/075-pins/Makefile | 8 +++++--- 6 files changed, 24 insertions(+), 13 deletions(-) diff --git a/fuzzers/065-gtp-common-pips/Makefile b/fuzzers/065-gtp-common-pips/Makefile index 54580674..3a7ab88c 100644 --- a/fuzzers/065-gtp-common-pips/Makefile +++ b/fuzzers/065-gtp-common-pips/Makefile @@ -54,7 +54,7 @@ $(BUILD_DIR)/cmt_regions.csv: output_cmt.tcl cd $(BUILD_DIR)/ && ${XRAY_VIVADO} -mode batch -source ${FUZDIR}/output_cmt.tcl clean: - rm -rf build_* run.*.ok + rm -rf ${BUILD_DIR} ${RUN_OK} pushdb: database ${XRAY_MERGEDB} gtp_common_mid_left $(BUILD_DIR)/segbits_gtp_common.db diff --git a/fuzzers/065b-gtp-common-pips/Makefile b/fuzzers/065b-gtp-common-pips/Makefile index 73380916..347f3cb1 100644 --- a/fuzzers/065b-gtp-common-pips/Makefile +++ b/fuzzers/065b-gtp-common-pips/Makefile @@ -53,7 +53,7 @@ $(BUILD_DIR)/cmt_regions.csv: output_cmt.tcl cd $(BUILD_DIR)/ && ${XRAY_VIVADO} -mode batch -source ${FUZDIR}/output_cmt.tcl clean: - rm -rf build_* run.*.ok + rm -rf ${BUILD_DIR} ${RUN_OK} pushdb: database ${XRAY_MERGEDB} gtp_common_mid_left $(BUILD_DIR)/segbits_gtp_common.db diff --git a/fuzzers/072-ordered_wires/Makefile b/fuzzers/072-ordered_wires/Makefile index 1905e5e4..92ab82e3 100644 --- a/fuzzers/072-ordered_wires/Makefile +++ b/fuzzers/072-ordered_wires/Makefile @@ -7,7 +7,10 @@ # SPDX-License-Identifier: ISC N := 1 + BUILD_DIR = build_${XRAY_PART} +RUN_OK = run.${XRAY_PART}.ok + SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N))) SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS)) MAX_VIVADO_PROCESS ?= 4 @@ -24,13 +27,13 @@ $(SPECIMENS_OK): touch $@ run: - rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok + rm -rf $(BUILD_DIR) $(RUN_OK) $(MAKE) database $(MAKE) pushdb - touch run.${XRAY_PART}.ok + touch $(RUN_OK) clean: - rm -rf build_* run.*.ok + rm -rf ${BUILD_DIR} ${RUN_OK} .PHONY: database pushdb run clean diff --git a/fuzzers/073-get_counts/Makefile b/fuzzers/073-get_counts/Makefile index 4cf0c021..28a9e549 100644 --- a/fuzzers/073-get_counts/Makefile +++ b/fuzzers/073-get_counts/Makefile @@ -7,7 +7,10 @@ # SPDX-License-Identifier: ISC N := 1 + BUILD_DIR = build_${XRAY_PART} +RUN_OK = run.${XRAY_PART}.ok + SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N))) SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS)) @@ -22,13 +25,13 @@ $(SPECIMENS_OK): touch $@ run: - rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok + rm -rf $(BUILD_DIR) $(RUN_OK) $(MAKE) database $(MAKE) pushdb - touch run.${XRAY_PART}.ok + touch $(RUN_OK) clean: - rm -rf build_* run.*.ok + rm -rf ${BUILD_DIR} ${RUN_OK} .PHONY: database pushdb run clean diff --git a/fuzzers/074-dump_all/Makefile b/fuzzers/074-dump_all/Makefile index d2dcabd5..2e8106dc 100644 --- a/fuzzers/074-dump_all/Makefile +++ b/fuzzers/074-dump_all/Makefile @@ -7,7 +7,10 @@ # SPDX-License-Identifier: ISC N := 1 + BUILD_DIR = build_${XRAY_PART} +RUN_OK = run.${XRAY_PART}.ok + SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N))) SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS)) MAX_VIVADO_PROCESS ?= 4 @@ -34,10 +37,10 @@ run: $(MAKE) pushdb # Clean up intermediate files after successful pushdb. find $(BUILD_DIR) -name "*.json5" -delete - touch run.${XRAY_PART}.ok + touch ${RUN_OK} clean: - rm -rf build_* run.*.ok + rm -rf ${BUILD_DIR} ${RUN_OK} .PHONY: database pushdb run clean diff --git a/fuzzers/075-pins/Makefile b/fuzzers/075-pins/Makefile index 153c8035..1b167c3e 100644 --- a/fuzzers/075-pins/Makefile +++ b/fuzzers/075-pins/Makefile @@ -7,6 +7,8 @@ # SPDX-License-Identifier: ISC N := 1 BUILD_DIR = build_${XRAY_PART} +RUN_OK = run.${XRAY_PART}.ok + SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N))) SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS)) @@ -21,12 +23,12 @@ $(SPECIMENS_OK): touch $@ run: - rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok + rm -rf $(BUILD_DIR) $(RUN_OK) $(MAKE) database $(MAKE) pushdb - touch run.$(XRAY_PART).ok + touch $(RUN_OK) clean: - rm -rf build_* run.*.ok + rm -rf ${BUILD_DIR} ${RUN_OK} .PHONY: database pushdb run clean From f3e7ab4b963f075136efdba3806281643eea2669 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Thu, 3 Mar 2022 14:27:59 +0100 Subject: [PATCH 11/16] ci: differentiate patch file per different part Signed-off-by: Alessandro Comodi --- .github/workflows/scripts/db.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/db.sh b/.github/workflows/scripts/db.sh index c66a6d95..5a6aaa73 100755 --- a/.github/workflows/scripts/db.sh +++ b/.github/workflows/scripts/db.sh @@ -129,13 +129,14 @@ echo "----------------------------------------" echo "----------------------------------------" echo " Saving diff output" echo "----------------------------------------" + PATCH_FILE=diff.${XRAY_SETTINGS}.patch # Patch file git diff \ --patch-with-stat --no-color --irreversible-delete --find-renames --find-copies origin/master \ - > diff.patch + > $PATCH_FILE MAX_DIFF_LINES=50000 - DIFF_LINES="$(wc -l diff.patch | sed -e's/ .*$//')" + DIFF_LINES="$(wc -l $PATCH_FILE | sed -e's/ .*$//')" if [ $DIFF_LINES -gt $MAX_DIFF_LINES ]; then echo echo "----------------------------------------" From f63752fe3e31343ea202fd70d4927419c38fe56c Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Thu, 3 Mar 2022 15:16:27 +0100 Subject: [PATCH 12/16] ci: gh: add artifact name Signed-off-by: Alessandro Comodi --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f49c148..61223812 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,7 @@ jobs: - uses: actions/upload-artifact@v2 if: ${{ always() }} with: + name: ${{ matrix.family }} path: | **/results*.gz **/plot_*.svg From 6c4188a52057c9fd6ec903b2ee50349aba9eeb55 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Fri, 4 Mar 2022 11:05:55 +0100 Subject: [PATCH 13/16] utils: parallelize all roi parts generation Signed-off-by: Alessandro Comodi --- utils/roi_all.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/utils/roi_all.py b/utils/roi_all.py index 660f9521..d3d3cea4 100755 --- a/utils/roi_all.py +++ b/utils/roi_all.py @@ -9,13 +9,26 @@ # # SPDX-License-Identifier: ISC import argparse -import yaml -import subprocess +import multiprocessing as mp import os import re +import subprocess +import yaml + from prjxray import util +def worker(arglist): + part, cwd = arglist + cmd = "make roi_only" + + env = os.environ.copy() + env['XRAY_PART'] = part + + print("running subprocess") + subprocess.run(cmd.split(' '), check=True, env=env, cwd=cwd) + + def main(): """Rois all parts for a family by calling "make roi_only" over all parts with the same device as XRAY_PART. @@ -36,12 +49,17 @@ def main(): if device['fabric'] == information['device']: valid_devices.append(name) + tasks = [] + for part, data in util.get_parts(db_root).items(): if data['device'] in valid_devices: - command = "make roi_only" - env['XRAY_PART'] = part cwd = os.getenv('XRAY_FUZZERS_DIR') - subprocess.run(command.split(' '), check=True, env=env, cwd=cwd) + + tasks.append((part, cwd)) + + with mp.Pool() as pool: + for _ in pool.imap_unordered(worker, tasks): + pass if __name__ == '__main__': From dcb1743764fc74828a2293b2929b6d667ad61d55 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Fri, 4 Mar 2022 12:40:22 +0100 Subject: [PATCH 14/16] add file locking system to avoid race conditions on db files Signed-off-by: Alessandro Comodi --- prjxray/tile_segbits.py | 7 +++++++ prjxray/util.py | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/prjxray/tile_segbits.py b/prjxray/tile_segbits.py index b9f940b7..8fd6b151 100644 --- a/prjxray/tile_segbits.py +++ b/prjxray/tile_segbits.py @@ -10,6 +10,7 @@ # SPDX-License-Identifier: ISC from collections import namedtuple from prjxray import bitstream +from prjxray import util from prjxray.grid_types import BlockType import enum @@ -84,15 +85,21 @@ class TileSegbits(object): if tile_db.ppips is not None: with open(tile_db.ppips) as f: + util.lock_file(f, 10) self.ppips = read_ppips(f) + util.unlock_file(f) if tile_db.segbits is not None: with open(tile_db.segbits) as f: + util.lock_file(f, 10) self.segbits[BlockType.CLB_IO_CLK] = read_segbits(f) + util.unlock_file(f) if tile_db.block_ram_segbits is not None: with open(tile_db.block_ram_segbits) as f: + util.lock_file(f, 10) self.segbits[BlockType.BLOCK_RAM] = read_segbits(f) + util.unlock_file(f) for block_type in self.segbits: for feature in self.segbits[block_type]: diff --git a/prjxray/util.py b/prjxray/util.py index 6c285795..5ce4550a 100644 --- a/prjxray/util.py +++ b/prjxray/util.py @@ -8,10 +8,12 @@ # https://opensource.org/licenses/ISC # # SPDX-License-Identifier: ISC +import fcntl import math import os import random import re +import signal import yaml from .roi import Roi @@ -254,8 +256,10 @@ def parse_db_line(line): def parse_db_lines(fn): with open(fn, "r") as f: + lock_file(f, 10) for line in f: yield line, parse_db_line(line) + unlock_file(f) def write_db_lines(fn, entries, track_origin=False): @@ -269,8 +273,10 @@ def write_db_lines(fn, entries, track_origin=False): new_lines.append(new_line) with open(fn, "w") as f: + lock_file(f, 10) for line in sorted(new_lines): print(line, file=f) + unlock_file(f) def parse_tagbit(x): @@ -402,3 +408,22 @@ def add_bool_arg(parser, yes_arg, default=False, **kwargs): yes_arg, dest=dest, action='store_true', default=default, **kwargs) parser.add_argument( '--no-' + dashed, dest=dest, action='store_false', **kwargs) + + +def timeout_handler(signum, frame): + raise Exception("ERROR: could not lock file!") + + +def lock_file(fd, timeout): + try: + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(timeout) + fcntl.flock(fd.fileno(), fcntl.LOCK_EX) + signal.alarm(0) + except Exception as e: + print(e) + exit(1) + + +def unlock_file(fd): + fcntl.flock(fd.fileno(), fcntl.LOCK_UN) From de9ce4ec28828456c84adef1c951d8576f984eb8 Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Mon, 7 Mar 2022 09:40:25 +0100 Subject: [PATCH 15/16] ci: gh: restore previous fails and patch file names Signed-off-by: Alessandro Comodi --- .github/workflows/scripts/db.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/db.sh b/.github/workflows/scripts/db.sh index 5a6aaa73..1b698073 100755 --- a/.github/workflows/scripts/db.sh +++ b/.github/workflows/scripts/db.sh @@ -73,7 +73,7 @@ echo "----------------------------------------" # Looking for the failing directories and packing them # example of line from which the failing fuzzer directory gets extracted: # - Makefile:87: recipe for target '000-db-init/000-init-db/run.xc7a100tfgg676-1.ok' failed --> fuzzers/000-db-init - grep -Po "recipe for target '\K(.*)(?=\/run\..*\.ok')" $tmp | sed -e 's/^/fuzzers\//' | xargs tar -zcf fuzzers/fails-${XRAY_SETTINGS}.tgz + grep -Po "recipe for target '\K(.*)(?=\/run\..*\.ok')" $tmp | sed -e 's/^/fuzzers\//' | xargs tar -zcf fuzzers/fails.tgz echo "----------------------------------------" echo "A failure occurred during Database build." echo "----------------------------------------" @@ -129,14 +129,13 @@ echo "----------------------------------------" echo "----------------------------------------" echo " Saving diff output" echo "----------------------------------------" - PATCH_FILE=diff.${XRAY_SETTINGS}.patch # Patch file git diff \ --patch-with-stat --no-color --irreversible-delete --find-renames --find-copies origin/master \ - > $PATCH_FILE + > diff.patch MAX_DIFF_LINES=50000 - DIFF_LINES="$(wc -l $PATCH_FILE | sed -e's/ .*$//')" + DIFF_LINES="$(wc -l diff.patch | sed -e's/ .*$//')" if [ $DIFF_LINES -gt $MAX_DIFF_LINES ]; then echo echo "----------------------------------------" From 847f97ab17532ae05907d69d4da9d083cf2d0cfb Mon Sep 17 00:00:00 2001 From: Alessandro Comodi Date: Mon, 7 Mar 2022 14:57:31 +0100 Subject: [PATCH 16/16] 065b-gtp-pips: increase iter timeout Signed-off-by: Alessandro Comodi --- fuzzers/065b-gtp-common-pips/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fuzzers/065b-gtp-common-pips/Makefile b/fuzzers/065b-gtp-common-pips/Makefile index 347f3cb1..c806e18b 100644 --- a/fuzzers/065b-gtp-common-pips/Makefile +++ b/fuzzers/065b-gtp-common-pips/Makefile @@ -5,6 +5,7 @@ # https://opensource.org/licenses/ISC # # SPDX-License-Identifier: ISC + export FUZDIR=$(shell pwd) PIP_TYPE?=gtp_common_mid_${XRAY_PART} PIP_FILE?=gtp_common_mid_ck_mux @@ -24,7 +25,7 @@ SEGMATCH_FLAGS=-c 3 SPECIMENS_DEPS=$(BUILD_DIR)/cmt_regions.csv A_PIPLIST=gtp_common_mid_ck_mux.txt -CHECK_ARGS= --zero-entries --timeout-iters 5 --todo-dir $(BUILD_DIR)/todo +CHECK_ARGS= --zero-entries --timeout-iters 10 --todo-dir $(BUILD_DIR)/todo include ../pip_loop.mk