diff --git a/.github/workflows/rtlmeter.yml b/.github/workflows/rtlmeter.yml index 8390c28af..4223f494e 100644 --- a/.github/workflows/rtlmeter.yml +++ b/.github/workflows/rtlmeter.yml @@ -26,9 +26,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name != 'schedule' }} -env: - RUN_TAGS: gcc clang gcc-hier - jobs: start: name: Start @@ -45,7 +42,11 @@ jobs: runs-on: ubuntu-24.04 outputs: old-sha: ${{ steps.start.outputs.old-sha }} + cases: ${{ steps.cases.outputs.cases }} steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Startup id: start env: @@ -57,6 +58,10 @@ jobs: OLD_SHA="$(gh api "repos/${{ github.repository }}/commits/${{ github.sha }}" --jq '.parents[0].sha')" echo "old-sha=$OLD_SHA" >> "$GITHUB_OUTPUT" + - name: Load RTLMeter cases + id: cases + run: echo "cases=$(ci/ci-rtlmeter-cases.py --event-name ${{ github.event_name }})" >> "$GITHUB_OUTPUT" + build-gcc-new: name: Build New Verilator - GCC needs: start @@ -98,6 +103,7 @@ jobs: run-gcc: name: Run GCC | ${{ matrix.cases }} needs: + - start - build-gcc-new - build-gcc-old # Run even if 'build-*-old' was skipped (no old SHA), but not if any @@ -118,39 +124,12 @@ jobs: fail-fast: false max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} matrix: - cases: - - "BlackParrot:1x1:*" - - "BlackParrot:4x4:*" - - "Caliptra:default:*" - - "NVDLA:*" - - "OpenPiton:1x1:*" - - "OpenPiton:4x4:*" - - "OpenTitan:*" - - "Servant:serv:*" - - "Servant:qerv:*" - - "VeeR-EH1:asic*" - - "VeeR-EH1:default*" - - "VeeR-EH1:hiperf*" - - "VeeR-EH2:asic*" - - "VeeR-EH2:default*" - - "VeeR-EH2:hiperf*" - - "VeeR-EL2:asic*" - - "VeeR-EL2:default*" - - "VeeR-EL2:hiperf*" - - "Vortex:mini:*" - - "Vortex:sane:*" - - "XiangShan:default-chisel3:* !*:linux" - - "XiangShan:default-chisel6:* !*:linux" - - "XiangShan:mini-chisel3:* !*:linux" - - "XiangShan:mini-chisel6:* !*:linux" - - "XuanTie-E902:*" - - "XuanTie-E906:*" - - "XuanTie-C906:*" - - "XuanTie-C910:*" + cases: ${{ fromJSON(needs.start.outputs.cases)['gcc'] }} run-clang: name: Run Clang | ${{ matrix.cases }} needs: + - start - build-clang-new - build-clang-old # Run even if 'build-*-old' was skipped (no old SHA), but not if any @@ -171,39 +150,12 @@ jobs: fail-fast: false max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} matrix: - cases: - - "BlackParrot:1x1:*" - - "BlackParrot:4x4:*" - - "Caliptra:default:*" - - "NVDLA:*" - - "OpenPiton:1x1:*" - - "OpenPiton:4x4:*" - - "OpenTitan:*" - - "Servant:serv:*" - - "Servant:qerv:*" - - "VeeR-EH1:asic*" - - "VeeR-EH1:default*" - - "VeeR-EH1:hiperf*" - - "VeeR-EH2:asic*" - - "VeeR-EH2:default*" - - "VeeR-EH2:hiperf*" - - "VeeR-EL2:asic*" - - "VeeR-EL2:default*" - - "VeeR-EL2:hiperf*" - - "Vortex:mini:*" - - "Vortex:sane:*" - - "XiangShan:default-chisel3:* !*:linux" - - "XiangShan:default-chisel6:* !*:linux" - - "XiangShan:mini-chisel3:* !*:linux" - - "XiangShan:mini-chisel6:* !*:linux" - - "XuanTie-E902:*" - - "XuanTie-E906:*" - - "XuanTie-C906:*" - - "XuanTie-C910:*" + cases: ${{ fromJSON(needs.start.outputs.cases)['clang'] }} run-gcc-hier: name: Run GCC hier | ${{ matrix.cases }} needs: + - start - build-gcc-new - build-gcc-old # Run even if 'build-*-old' was skipped (no old SHA), but not if any @@ -224,15 +176,7 @@ jobs: fail-fast: false max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} matrix: - cases: - - "BlackParrot:1x1:* !-hier" - - "BlackParrot:4x4:* !-hier" - - "NVDLA:* !-hier" - - "OpenPiton:1x1:* !-hier" - - "OpenPiton:4x4:* !-hier" - - "OpenPiton:8x8:* !-hier" - - "OpenPiton:16x16:dhry !-hier" - - "XuanTie-C910:* !-hier" + cases: ${{ fromJSON(needs.start.outputs.cases)['gcc-hier'] }} combine-results: name: Combine results @@ -246,12 +190,20 @@ jobs: || (github.event_name != 'pull_request' && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure'))) )}} runs-on: ubuntu-24.04 + outputs: + # The run tags, derived from the 'run-*' dependency job names + run-tags: ${{ steps.tags.outputs.tags }} env: GH_TOKEN: ${{ github.token }} # 'gh' resolves the repo from git otherwise, but this job has no checkout # of this repo at the workspace root GH_REPO: ${{ github.repository }} steps: + # Derive the run tags from the dependency job names ('run-') + - name: Determine run tags + id: tags + run: echo "tags=$(jq -r 'keys | map(sub("^run-"; "")) | join(" ")' <<< '${{ toJSON(needs) }}')" >> "$GITHUB_OUTPUT" + - name: Checkout RTLMeter uses: actions/checkout@v6 with: @@ -264,7 +216,7 @@ jobs: - name: Download all results run: | - for tag in $RUN_TAGS; do + for tag in ${{ steps.tags.outputs.tags }}; do mkdir artifacts all-results-$tag gh run download ${{ github.run_id }} --pattern "rtlmeter-$tag-results-*" --dir artifacts mv $(find artifacts -name "*.json") all-results-$tag/ @@ -273,7 +225,7 @@ jobs: - name: Combine results working-directory: rtlmeter run: | - for tag in $RUN_TAGS; do + for tag in ${{ steps.tags.outputs.tags }}; do ./rtlmeter collate ../all-results-$tag/*.json > ../all-results-$tag.json done - name: Upload combined results @@ -289,7 +241,7 @@ jobs: - name: Download reference results if: ${{ github.event_name == 'pull_request' }} run: | - for tag in $RUN_TAGS; do + for tag in ${{ steps.tags.outputs.tags }}; do mkdir artifacts all-reference-$tag gh run download ${{ github.run_id }} --pattern "rtlmeter-$tag-reference-*" --dir artifacts mv $(find artifacts -name "*.json") all-reference-$tag/ @@ -299,7 +251,7 @@ jobs: if: ${{ github.event_name == 'pull_request' }} working-directory: rtlmeter run: | - for tag in $RUN_TAGS; do + for tag in ${{ steps.tags.outputs.tags }}; do ./rtlmeter collate ../all-reference-$tag/*.json > ../all-reference-$tag.json done - name: Upload reference results @@ -398,7 +350,7 @@ jobs: ln -s ../rtlmeter rtlmeter gh repo set-default ${{ github.repository }} # Compare to last successful scheduled run - ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} $RUN_TAGS + ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} ${{ needs.combine-results.outputs.run-tags }} # Create the report artifact mkdir ../report-artifact mv rtlmeter-report/report ../report-artifact/ diff --git a/ci/ci-rtlmeter-cases.py b/ci/ci-rtlmeter-cases.py new file mode 100755 index 000000000..8c25cb106 --- /dev/null +++ b/ci/ci-rtlmeter-cases.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# DESCRIPTION: Verilator: RTLMeter cases to run in the rtlmeter.yml CI workflow +# +# SPDX-FileCopyrightText: 2026 Wilson Snyder +# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 + +############################################################################### +# Defines the RTLMeter cases to run in CI. This is consumed by the 'start' job +# in rtlmeter.yml, which captures the printed JSON and feeds it to the run job +# matrices. +############################################################################### + +import argparse +import json + +# Cases to run for scheduled runs, keyyed by the 'run tag' used in rtlmeter.yml +# The associated priority is roughly the number of minutes it takes to run the +# job, the emitted job matrix will try to start longest jobs first. + +# Cases to run for non-PR jobs (e.g.: nightly schedule) +# fmt: off +CASES = { + "gcc": [ + ("BlackParrot:1x1:*", 5), + ("BlackParrot:2x2:*", 14), + ("BlackParrot:4x4:*", 25), + ("Caliptra:default:*", 19), + ("NVDLA:*", 29), + ("OpenPiton:1x1:*", 4), + ("OpenPiton:2x2:*", 7), + ("OpenPiton:4x4:*", 10), + ("OpenTitan:*", 16), + ("Servant:qerv:*", 2), + ("Servant:serv:*", 2), + ("VeeR-EH1:asic:*", 4), + ("VeeR-EH1:default:*", 6), + ("VeeR-EH1:hiperf:*", 5), + ("VeeR-EH2:asic:*", 13), + ("VeeR-EH2:default:*", 6), + ("VeeR-EH2:hiperf:*", 12), + ("VeeR-EL2:asic:*", 4), + ("VeeR-EL2:default:*", 4), + ("VeeR-EL2:hiperf:*", 4), + ("Vortex:mini:*", 3), + ("Vortex:sane:*", 6), + ("XiangShan:default-chisel3:* !*:linux", 31), + ("XiangShan:default-chisel6:* !*:linux", 28), + ("XiangShan:mini-chisel3:* !*:linux", 13), + ("XiangShan:mini-chisel6:* !*:linux", 15), + ("XuanTie-C906:*", 5), + ("XuanTie-C910:*", 9), + ("XuanTie-E902:*", 5), + ("XuanTie-E906:*", 6), + ], + "clang": [ + ("BlackParrot:1x1:*", 5), + ("BlackParrot:2x2:*", 9), + ("BlackParrot:4x4:*", 12), + ("Caliptra:default:*", 16), + ("NVDLA:*", 28), + ("OpenPiton:1x1:*", 5), + ("OpenPiton:2x2:*", 6), + ("OpenPiton:4x4:*", 8), + ("OpenTitan:*", 10), + ("Servant:qerv:*", 17), + ("Servant:serv:*", 15), + ("VeeR-EH1:asic:*", 6), + ("VeeR-EH1:default:*", 8), + ("VeeR-EH1:hiperf:*", 5), + ("VeeR-EH2:asic:*", 9), + ("VeeR-EH2:default:*", 8), + ("VeeR-EH2:hiperf:*", 7), + ("VeeR-EL2:asic:*", 4), + ("VeeR-EL2:default:*", 5), + ("VeeR-EL2:hiperf:*", 5), + ("Vortex:mini:*", 3), + ("Vortex:sane:*", 5), + ("XiangShan:default-chisel3:* !*:linux", 21), + ("XiangShan:default-chisel6:* !*:linux", 19), + ("XiangShan:mini-chisel3:* !*:linux", 8), + ("XiangShan:mini-chisel6:* !*:linux", 8), + ("XuanTie-C906:*", 3), + ("XuanTie-C910:*", 7), + ("XuanTie-E902:*", 7), + ("XuanTie-E906:*", 7), + ], + "gcc-hier": [ + ("BlackParrot:1x1:*", 4), + ("BlackParrot:2x2:*", 16), + ("BlackParrot:4x4:*", 25), + ("NVDLA:*", 25), + ("OpenPiton:16x16:dhry", 16), + ("OpenPiton:1x1:*", 4), + ("OpenPiton:2x2:*", 4), + ("OpenPiton:4x4:*", 4), + ("OpenPiton:8x8:*", 6), + ("XuanTie-C910:*", 7), + ], +} +# fmt: on + +# Cases to run for PR jobs +# fmt: off +CASES_PR = { + "gcc": [ + ("BlackParrot:1x1:*", 5), + ("BlackParrot:4x4:*", 25), + ("Caliptra:default:*", 19), + ("NVDLA:*", 29), + ("OpenPiton:1x1:*", 4), + ("OpenPiton:4x4:*", 10), + ("OpenTitan:*", 16), + ("Servant:qerv:*", 2), + ("Servant:serv:*", 2), + ("VeeR-EH1:asic:*", 4), + ("VeeR-EH1:default:*", 6), + ("VeeR-EH1:hiperf:*", 5), + ("VeeR-EH2:asic:*", 13), + ("VeeR-EH2:default:*", 6), + ("VeeR-EH2:hiperf:*", 12), + ("VeeR-EL2:asic:*", 4), + ("VeeR-EL2:default:*", 4), + ("VeeR-EL2:hiperf:*", 4), + ("Vortex:mini:*", 3), + ("Vortex:sane:*", 6), + ("XiangShan:default-chisel3:* !*:linux", 31), + ("XiangShan:default-chisel6:* !*:linux", 28), + ("XiangShan:mini-chisel3:* !*:linux", 13), + ("XiangShan:mini-chisel6:* !*:linux", 15), + ("XuanTie-C906:*", 5), + ("XuanTie-C910:*", 9), + ("XuanTie-E902:*", 5), + ("XuanTie-E906:*", 6), + ], + "clang": [ + ("BlackParrot:1x1:*", 5), + ("BlackParrot:4x4:*", 12), + ("Caliptra:default:*", 16), + ("NVDLA:*", 28), + ("OpenPiton:1x1:*", 5), + ("OpenPiton:4x4:*", 8), + ("OpenTitan:*", 10), + ("Servant:qerv:*", 17), + ("Servant:serv:*", 15), + ("VeeR-EH1:asic:*", 6), + ("VeeR-EH1:default:*", 8), + ("VeeR-EH1:hiperf:*", 5), + ("VeeR-EH2:asic:*", 9), + ("VeeR-EH2:default:*", 8), + ("VeeR-EH2:hiperf:*", 7), + ("VeeR-EL2:asic:*", 4), + ("VeeR-EL2:default:*", 5), + ("VeeR-EL2:hiperf:*", 5), + ("Vortex:mini:*", 3), + ("Vortex:sane:*", 5), + ("XiangShan:default-chisel3:* !*:linux", 21), + ("XiangShan:default-chisel6:* !*:linux", 19), + ("XiangShan:mini-chisel3:* !*:linux", 8), + ("XiangShan:mini-chisel6:* !*:linux", 8), + ("XuanTie-C906:*", 3), + ("XuanTie-C910:*", 7), + ("XuanTie-E902:*", 7), + ("XuanTie-E906:*", 7), + ], + "gcc-hier": [ + ("BlackParrot:1x1:*", 4), + ("BlackParrot:4x4:*", 25), + ("NVDLA:*", 25), + ("OpenPiton:16x16:dhry", 16), + ("OpenPiton:1x1:*", 4), + ("OpenPiton:4x4:*", 4), + ("OpenPiton:8x8:*", 6), + ("XuanTie-C910:*", 7), + ], +} +# fmt: on + +parser = argparse.ArgumentParser( + description="Print the RTLMeter CI cases as JSON, keyed by run tag") +parser.add_argument("--event-name", + required=True, + help="GitHub event name that triggered the workflow") +parser.add_argument( + "--ci-testing", + action="store_true", + help="Print only the shortest (lowest priority) case per tag, to reduce CI testing time") +args = parser.parse_args() + +# Pull requests use the reduced 'CASES_PR' set, other events (e.g.: the nightly +# schedule) use the full 'CASES' set. +is_pr = args.event_name == "pull_request" +selected = CASES_PR if is_pr else CASES + +cases = {} +for tag, entries in selected.items(): + # Highest priority (longest) first + ordered = sorted(entries, key=lambda cp: cp[1], reverse=True) + # While tweaking the CI, keep only the shortest (lowest priority) case to minimise churn time + if args.ci_testing: + ordered = ordered[-1:] + # Case filters appended to every case for this tag/event + suffix = "" + # Filters out the non-hierarchical case variants for the hierarchical job + if tag == "gcc-hier": + suffix += " !-hier" + # Filter out hello tests on pull requests, these just add measurement noise + if is_pr: + suffix += " !*:hello" + cases[tag] = [case + suffix for case, _ in ordered] +print(json.dumps(cases, separators=(",", ":"))) diff --git a/ci/ci-rtlmeter-report.py b/ci/ci-rtlmeter-report.py index e743f67da..73810e715 100644 --- a/ci/ci-rtlmeter-report.py +++ b/ci/ci-rtlmeter-report.py @@ -73,6 +73,8 @@ for ref, cmp in zip(sys.argv[1::2], sys.argv[2::2]): maxGain = max(maxGain, g) meanGain *= g count += 1 + if count == 0: + continue meanGain = meanGain**(1 / count) if metric == "clocks": @@ -92,12 +94,12 @@ for ref, cmp in zip(sys.argv[1::2], sys.argv[2::2]): table.append([ runName, step, ref_json["metrics"][metric]["header"], f"{meanGain:.2f}x {status} ", - f"{minGain:.2f}x", f"{maxGain:.2f}x" + f"{minGain:.2f}x", f"{maxGain:.2f}x", f"{count}" ]) printTable( table, - headers=("Run", "Step", "Metric", "Improvement", "Min", "Max"), - colalign=("left", "left", "left", "right", "right", "right"), + headers=("Run", "Step", "Metric", "Improvement", "Min", "Max", "Samples"), + colalign=("left", "left", "left", "right", "right", "right", "right"), disable_numparse=True, )