From 5cb8d8291a217fa82f02ece5d402baf6450f8814 Mon Sep 17 00:00:00 2001 From: Geza Lore Date: Sun, 7 Jun 2026 20:50:31 +0100 Subject: [PATCH] CI: Load RTLMeter cases to run from separate file (#7730) Add a script that prints the job matrix of cases to run with RTLMeter. This enables running different sets based on parameters. It will run everything on scheduled nightly runs, but only a subset on PRs (to reduce time and variance due to short cases). Also prints the job matrix in descending duration order, which effectively makes GitHub actions schedule longest jobs first to reduce total workflow latency. --- .github/workflows/rtlmeter.yml | 102 +++++----------- ci/ci-rtlmeter-cases.py | 210 +++++++++++++++++++++++++++++++++ ci/ci-rtlmeter-report.py | 8 +- 3 files changed, 242 insertions(+), 78 deletions(-) create mode 100755 ci/ci-rtlmeter-cases.py diff --git a/.github/workflows/rtlmeter.yml b/.github/workflows/rtlmeter.yml index 8390c28af..4223f494e 100644 --- a/.github/workflows/rtlmeter.yml +++ b/.github/workflows/rtlmeter.yml @@ -26,9 +26,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name != 'schedule' }} -env: - RUN_TAGS: gcc clang gcc-hier - jobs: start: name: Start @@ -45,7 +42,11 @@ jobs: runs-on: ubuntu-24.04 outputs: old-sha: ${{ steps.start.outputs.old-sha }} + cases: ${{ steps.cases.outputs.cases }} steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Startup id: start env: @@ -57,6 +58,10 @@ jobs: OLD_SHA="$(gh api "repos/${{ github.repository }}/commits/${{ github.sha }}" --jq '.parents[0].sha')" echo "old-sha=$OLD_SHA" >> "$GITHUB_OUTPUT" + - name: Load RTLMeter cases + id: cases + run: echo "cases=$(ci/ci-rtlmeter-cases.py --event-name ${{ github.event_name }})" >> "$GITHUB_OUTPUT" + build-gcc-new: name: Build New Verilator - GCC needs: start @@ -98,6 +103,7 @@ jobs: run-gcc: name: Run GCC | ${{ matrix.cases }} needs: + - start - build-gcc-new - build-gcc-old # Run even if 'build-*-old' was skipped (no old SHA), but not if any @@ -118,39 +124,12 @@ jobs: fail-fast: false max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} matrix: - cases: - - "BlackParrot:1x1:*" - - "BlackParrot:4x4:*" - - "Caliptra:default:*" - - "NVDLA:*" - - "OpenPiton:1x1:*" - - "OpenPiton:4x4:*" - - "OpenTitan:*" - - "Servant:serv:*" - - "Servant:qerv:*" - - "VeeR-EH1:asic*" - - "VeeR-EH1:default*" - - "VeeR-EH1:hiperf*" - - "VeeR-EH2:asic*" - - "VeeR-EH2:default*" - - "VeeR-EH2:hiperf*" - - "VeeR-EL2:asic*" - - "VeeR-EL2:default*" - - "VeeR-EL2:hiperf*" - - "Vortex:mini:*" - - "Vortex:sane:*" - - "XiangShan:default-chisel3:* !*:linux" - - "XiangShan:default-chisel6:* !*:linux" - - "XiangShan:mini-chisel3:* !*:linux" - - "XiangShan:mini-chisel6:* !*:linux" - - "XuanTie-E902:*" - - "XuanTie-E906:*" - - "XuanTie-C906:*" - - "XuanTie-C910:*" + cases: ${{ fromJSON(needs.start.outputs.cases)['gcc'] }} run-clang: name: Run Clang | ${{ matrix.cases }} needs: + - start - build-clang-new - build-clang-old # Run even if 'build-*-old' was skipped (no old SHA), but not if any @@ -171,39 +150,12 @@ jobs: fail-fast: false max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} matrix: - cases: - - "BlackParrot:1x1:*" - - "BlackParrot:4x4:*" - - "Caliptra:default:*" - - "NVDLA:*" - - "OpenPiton:1x1:*" - - "OpenPiton:4x4:*" - - "OpenTitan:*" - - "Servant:serv:*" - - "Servant:qerv:*" - - "VeeR-EH1:asic*" - - "VeeR-EH1:default*" - - "VeeR-EH1:hiperf*" - - "VeeR-EH2:asic*" - - "VeeR-EH2:default*" - - "VeeR-EH2:hiperf*" - - "VeeR-EL2:asic*" - - "VeeR-EL2:default*" - - "VeeR-EL2:hiperf*" - - "Vortex:mini:*" - - "Vortex:sane:*" - - "XiangShan:default-chisel3:* !*:linux" - - "XiangShan:default-chisel6:* !*:linux" - - "XiangShan:mini-chisel3:* !*:linux" - - "XiangShan:mini-chisel6:* !*:linux" - - "XuanTie-E902:*" - - "XuanTie-E906:*" - - "XuanTie-C906:*" - - "XuanTie-C910:*" + cases: ${{ fromJSON(needs.start.outputs.cases)['clang'] }} run-gcc-hier: name: Run GCC hier | ${{ matrix.cases }} needs: + - start - build-gcc-new - build-gcc-old # Run even if 'build-*-old' was skipped (no old SHA), but not if any @@ -224,15 +176,7 @@ jobs: fail-fast: false max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} matrix: - cases: - - "BlackParrot:1x1:* !-hier" - - "BlackParrot:4x4:* !-hier" - - "NVDLA:* !-hier" - - "OpenPiton:1x1:* !-hier" - - "OpenPiton:4x4:* !-hier" - - "OpenPiton:8x8:* !-hier" - - "OpenPiton:16x16:dhry !-hier" - - "XuanTie-C910:* !-hier" + cases: ${{ fromJSON(needs.start.outputs.cases)['gcc-hier'] }} combine-results: name: Combine results @@ -246,12 +190,20 @@ jobs: || (github.event_name != 'pull_request' && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure'))) )}} runs-on: ubuntu-24.04 + outputs: + # The run tags, derived from the 'run-*' dependency job names + run-tags: ${{ steps.tags.outputs.tags }} env: GH_TOKEN: ${{ github.token }} # 'gh' resolves the repo from git otherwise, but this job has no checkout # of this repo at the workspace root GH_REPO: ${{ github.repository }} steps: + # Derive the run tags from the dependency job names ('run-') + - name: Determine run tags + id: tags + run: echo "tags=$(jq -r 'keys | map(sub("^run-"; "")) | join(" ")' <<< '${{ toJSON(needs) }}')" >> "$GITHUB_OUTPUT" + - name: Checkout RTLMeter uses: actions/checkout@v6 with: @@ -264,7 +216,7 @@ jobs: - name: Download all results run: | - for tag in $RUN_TAGS; do + for tag in ${{ steps.tags.outputs.tags }}; do mkdir artifacts all-results-$tag gh run download ${{ github.run_id }} --pattern "rtlmeter-$tag-results-*" --dir artifacts mv $(find artifacts -name "*.json") all-results-$tag/ @@ -273,7 +225,7 @@ jobs: - name: Combine results working-directory: rtlmeter run: | - for tag in $RUN_TAGS; do + for tag in ${{ steps.tags.outputs.tags }}; do ./rtlmeter collate ../all-results-$tag/*.json > ../all-results-$tag.json done - name: Upload combined results @@ -289,7 +241,7 @@ jobs: - name: Download reference results if: ${{ github.event_name == 'pull_request' }} run: | - for tag in $RUN_TAGS; do + for tag in ${{ steps.tags.outputs.tags }}; do mkdir artifacts all-reference-$tag gh run download ${{ github.run_id }} --pattern "rtlmeter-$tag-reference-*" --dir artifacts mv $(find artifacts -name "*.json") all-reference-$tag/ @@ -299,7 +251,7 @@ jobs: if: ${{ github.event_name == 'pull_request' }} working-directory: rtlmeter run: | - for tag in $RUN_TAGS; do + for tag in ${{ steps.tags.outputs.tags }}; do ./rtlmeter collate ../all-reference-$tag/*.json > ../all-reference-$tag.json done - name: Upload reference results @@ -398,7 +350,7 @@ jobs: ln -s ../rtlmeter rtlmeter gh repo set-default ${{ github.repository }} # Compare to last successful scheduled run - ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} $RUN_TAGS + ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} ${{ needs.combine-results.outputs.run-tags }} # Create the report artifact mkdir ../report-artifact mv rtlmeter-report/report ../report-artifact/ diff --git a/ci/ci-rtlmeter-cases.py b/ci/ci-rtlmeter-cases.py new file mode 100755 index 000000000..8c25cb106 --- /dev/null +++ b/ci/ci-rtlmeter-cases.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# DESCRIPTION: Verilator: RTLMeter cases to run in the rtlmeter.yml CI workflow +# +# SPDX-FileCopyrightText: 2026 Wilson Snyder +# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 + +############################################################################### +# Defines the RTLMeter cases to run in CI. This is consumed by the 'start' job +# in rtlmeter.yml, which captures the printed JSON and feeds it to the run job +# matrices. +############################################################################### + +import argparse +import json + +# Cases to run for scheduled runs, keyyed by the 'run tag' used in rtlmeter.yml +# The associated priority is roughly the number of minutes it takes to run the +# job, the emitted job matrix will try to start longest jobs first. + +# Cases to run for non-PR jobs (e.g.: nightly schedule) +# fmt: off +CASES = { + "gcc": [ + ("BlackParrot:1x1:*", 5), + ("BlackParrot:2x2:*", 14), + ("BlackParrot:4x4:*", 25), + ("Caliptra:default:*", 19), + ("NVDLA:*", 29), + ("OpenPiton:1x1:*", 4), + ("OpenPiton:2x2:*", 7), + ("OpenPiton:4x4:*", 10), + ("OpenTitan:*", 16), + ("Servant:qerv:*", 2), + ("Servant:serv:*", 2), + ("VeeR-EH1:asic:*", 4), + ("VeeR-EH1:default:*", 6), + ("VeeR-EH1:hiperf:*", 5), + ("VeeR-EH2:asic:*", 13), + ("VeeR-EH2:default:*", 6), + ("VeeR-EH2:hiperf:*", 12), + ("VeeR-EL2:asic:*", 4), + ("VeeR-EL2:default:*", 4), + ("VeeR-EL2:hiperf:*", 4), + ("Vortex:mini:*", 3), + ("Vortex:sane:*", 6), + ("XiangShan:default-chisel3:* !*:linux", 31), + ("XiangShan:default-chisel6:* !*:linux", 28), + ("XiangShan:mini-chisel3:* !*:linux", 13), + ("XiangShan:mini-chisel6:* !*:linux", 15), + ("XuanTie-C906:*", 5), + ("XuanTie-C910:*", 9), + ("XuanTie-E902:*", 5), + ("XuanTie-E906:*", 6), + ], + "clang": [ + ("BlackParrot:1x1:*", 5), + ("BlackParrot:2x2:*", 9), + ("BlackParrot:4x4:*", 12), + ("Caliptra:default:*", 16), + ("NVDLA:*", 28), + ("OpenPiton:1x1:*", 5), + ("OpenPiton:2x2:*", 6), + ("OpenPiton:4x4:*", 8), + ("OpenTitan:*", 10), + ("Servant:qerv:*", 17), + ("Servant:serv:*", 15), + ("VeeR-EH1:asic:*", 6), + ("VeeR-EH1:default:*", 8), + ("VeeR-EH1:hiperf:*", 5), + ("VeeR-EH2:asic:*", 9), + ("VeeR-EH2:default:*", 8), + ("VeeR-EH2:hiperf:*", 7), + ("VeeR-EL2:asic:*", 4), + ("VeeR-EL2:default:*", 5), + ("VeeR-EL2:hiperf:*", 5), + ("Vortex:mini:*", 3), + ("Vortex:sane:*", 5), + ("XiangShan:default-chisel3:* !*:linux", 21), + ("XiangShan:default-chisel6:* !*:linux", 19), + ("XiangShan:mini-chisel3:* !*:linux", 8), + ("XiangShan:mini-chisel6:* !*:linux", 8), + ("XuanTie-C906:*", 3), + ("XuanTie-C910:*", 7), + ("XuanTie-E902:*", 7), + ("XuanTie-E906:*", 7), + ], + "gcc-hier": [ + ("BlackParrot:1x1:*", 4), + ("BlackParrot:2x2:*", 16), + ("BlackParrot:4x4:*", 25), + ("NVDLA:*", 25), + ("OpenPiton:16x16:dhry", 16), + ("OpenPiton:1x1:*", 4), + ("OpenPiton:2x2:*", 4), + ("OpenPiton:4x4:*", 4), + ("OpenPiton:8x8:*", 6), + ("XuanTie-C910:*", 7), + ], +} +# fmt: on + +# Cases to run for PR jobs +# fmt: off +CASES_PR = { + "gcc": [ + ("BlackParrot:1x1:*", 5), + ("BlackParrot:4x4:*", 25), + ("Caliptra:default:*", 19), + ("NVDLA:*", 29), + ("OpenPiton:1x1:*", 4), + ("OpenPiton:4x4:*", 10), + ("OpenTitan:*", 16), + ("Servant:qerv:*", 2), + ("Servant:serv:*", 2), + ("VeeR-EH1:asic:*", 4), + ("VeeR-EH1:default:*", 6), + ("VeeR-EH1:hiperf:*", 5), + ("VeeR-EH2:asic:*", 13), + ("VeeR-EH2:default:*", 6), + ("VeeR-EH2:hiperf:*", 12), + ("VeeR-EL2:asic:*", 4), + ("VeeR-EL2:default:*", 4), + ("VeeR-EL2:hiperf:*", 4), + ("Vortex:mini:*", 3), + ("Vortex:sane:*", 6), + ("XiangShan:default-chisel3:* !*:linux", 31), + ("XiangShan:default-chisel6:* !*:linux", 28), + ("XiangShan:mini-chisel3:* !*:linux", 13), + ("XiangShan:mini-chisel6:* !*:linux", 15), + ("XuanTie-C906:*", 5), + ("XuanTie-C910:*", 9), + ("XuanTie-E902:*", 5), + ("XuanTie-E906:*", 6), + ], + "clang": [ + ("BlackParrot:1x1:*", 5), + ("BlackParrot:4x4:*", 12), + ("Caliptra:default:*", 16), + ("NVDLA:*", 28), + ("OpenPiton:1x1:*", 5), + ("OpenPiton:4x4:*", 8), + ("OpenTitan:*", 10), + ("Servant:qerv:*", 17), + ("Servant:serv:*", 15), + ("VeeR-EH1:asic:*", 6), + ("VeeR-EH1:default:*", 8), + ("VeeR-EH1:hiperf:*", 5), + ("VeeR-EH2:asic:*", 9), + ("VeeR-EH2:default:*", 8), + ("VeeR-EH2:hiperf:*", 7), + ("VeeR-EL2:asic:*", 4), + ("VeeR-EL2:default:*", 5), + ("VeeR-EL2:hiperf:*", 5), + ("Vortex:mini:*", 3), + ("Vortex:sane:*", 5), + ("XiangShan:default-chisel3:* !*:linux", 21), + ("XiangShan:default-chisel6:* !*:linux", 19), + ("XiangShan:mini-chisel3:* !*:linux", 8), + ("XiangShan:mini-chisel6:* !*:linux", 8), + ("XuanTie-C906:*", 3), + ("XuanTie-C910:*", 7), + ("XuanTie-E902:*", 7), + ("XuanTie-E906:*", 7), + ], + "gcc-hier": [ + ("BlackParrot:1x1:*", 4), + ("BlackParrot:4x4:*", 25), + ("NVDLA:*", 25), + ("OpenPiton:16x16:dhry", 16), + ("OpenPiton:1x1:*", 4), + ("OpenPiton:4x4:*", 4), + ("OpenPiton:8x8:*", 6), + ("XuanTie-C910:*", 7), + ], +} +# fmt: on + +parser = argparse.ArgumentParser( + description="Print the RTLMeter CI cases as JSON, keyed by run tag") +parser.add_argument("--event-name", + required=True, + help="GitHub event name that triggered the workflow") +parser.add_argument( + "--ci-testing", + action="store_true", + help="Print only the shortest (lowest priority) case per tag, to reduce CI testing time") +args = parser.parse_args() + +# Pull requests use the reduced 'CASES_PR' set, other events (e.g.: the nightly +# schedule) use the full 'CASES' set. +is_pr = args.event_name == "pull_request" +selected = CASES_PR if is_pr else CASES + +cases = {} +for tag, entries in selected.items(): + # Highest priority (longest) first + ordered = sorted(entries, key=lambda cp: cp[1], reverse=True) + # While tweaking the CI, keep only the shortest (lowest priority) case to minimise churn time + if args.ci_testing: + ordered = ordered[-1:] + # Case filters appended to every case for this tag/event + suffix = "" + # Filters out the non-hierarchical case variants for the hierarchical job + if tag == "gcc-hier": + suffix += " !-hier" + # Filter out hello tests on pull requests, these just add measurement noise + if is_pr: + suffix += " !*:hello" + cases[tag] = [case + suffix for case, _ in ordered] +print(json.dumps(cases, separators=(",", ":"))) diff --git a/ci/ci-rtlmeter-report.py b/ci/ci-rtlmeter-report.py index e743f67da..73810e715 100644 --- a/ci/ci-rtlmeter-report.py +++ b/ci/ci-rtlmeter-report.py @@ -73,6 +73,8 @@ for ref, cmp in zip(sys.argv[1::2], sys.argv[2::2]): maxGain = max(maxGain, g) meanGain *= g count += 1 + if count == 0: + continue meanGain = meanGain**(1 / count) if metric == "clocks": @@ -92,12 +94,12 @@ for ref, cmp in zip(sys.argv[1::2], sys.argv[2::2]): table.append([ runName, step, ref_json["metrics"][metric]["header"], f"{meanGain:.2f}x {status} ", - f"{minGain:.2f}x", f"{maxGain:.2f}x" + f"{minGain:.2f}x", f"{maxGain:.2f}x", f"{count}" ]) printTable( table, - headers=("Run", "Step", "Metric", "Improvement", "Min", "Max"), - colalign=("left", "left", "left", "right", "right", "right"), + headers=("Run", "Step", "Metric", "Improvement", "Min", "Max", "Samples"), + colalign=("left", "left", "left", "right", "right", "right", "right"), disable_numparse=True, )