luke
/
verilator
mirror of https://github.com/verilator/verilator.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CI: Load RTLMeter cases to run from separate file (#7730) 

Add a script that prints the job matrix of cases to run with RTLMeter.
This enables running different sets based on parameters. It will run
everything on scheduled nightly runs, but only a subset on PRs (to reduce
time and variance due to short cases).

Also prints the job matrix in descending duration order, which
effectively makes GitHub actions schedule longest jobs first to reduce
total workflow latency.
This commit is contained in:
Geza Lore 2026-06-07 20:50:31 +01:00 committed by GitHub
parent 220e46994c
commit 5cb8d8291a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 242 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
.github/workflows/rtlmeter.yml vendored
View File

@ -26,9 +26,6 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
env:
RUN_TAGS: gcc clang gcc-hier
jobs:
start:
name: Start
@ -45,7 +42,11 @@ jobs:
runs-on: ubuntu-24.04
outputs:
old-sha: ${{ steps.start.outputs.old-sha }}
cases: ${{ steps.cases.outputs.cases }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Startup
id: start
env:
@ -57,6 +58,10 @@ jobs:
OLD_SHA="$(gh api "repos/${{ github.repository }}/commits/${{ github.sha }}" --jq '.parents[0].sha')"
echo "old-sha=$OLD_SHA" >> "$GITHUB_OUTPUT"
- name: Load RTLMeter cases
id: cases
run: echo "cases=$(ci/ci-rtlmeter-cases.py --event-name ${{ github.event_name }})" >> "$GITHUB_OUTPUT"
build-gcc-new:
name: Build New Verilator - GCC
needs: start
@ -98,6 +103,7 @@ jobs:
run-gcc:
name: Run GCC | ${{ matrix.cases }}
needs:
- start
- build-gcc-new
- build-gcc-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
@ -118,39 +124,12 @@ jobs:
fail-fast: false
max-parallel: ${{ github.event == 'schedule' && 2 || 7 }}
matrix:
cases:
- "BlackParrot:1x1:*"
- "BlackParrot:4x4:*"
- "Caliptra:default:*"
- "NVDLA:*"
- "OpenPiton:1x1:*"
- "OpenPiton:4x4:*"
- "OpenTitan:*"
- "Servant:serv:*"
- "Servant:qerv:*"
- "VeeR-EH1:asic*"
- "VeeR-EH1:default*"
- "VeeR-EH1:hiperf*"
- "VeeR-EH2:asic*"
- "VeeR-EH2:default*"
- "VeeR-EH2:hiperf*"
- "VeeR-EL2:asic*"
- "VeeR-EL2:default*"
- "VeeR-EL2:hiperf*"
- "Vortex:mini:*"
- "Vortex:sane:*"
- "XiangShan:default-chisel3:* !*:linux"
- "XiangShan:default-chisel6:* !*:linux"
- "XiangShan:mini-chisel3:* !*:linux"
- "XiangShan:mini-chisel6:* !*:linux"
- "XuanTie-E902:*"
- "XuanTie-E906:*"
- "XuanTie-C906:*"
- "XuanTie-C910:*"
cases: ${{ fromJSON(needs.start.outputs.cases)['gcc'] }}
run-clang:
name: Run Clang | ${{ matrix.cases }}
needs:
- start
- build-clang-new
- build-clang-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
@ -171,39 +150,12 @@ jobs:
fail-fast: false
max-parallel: ${{ github.event == 'schedule' && 2 || 7 }}
matrix:
cases:
- "BlackParrot:1x1:*"
- "BlackParrot:4x4:*"
- "Caliptra:default:*"
- "NVDLA:*"
- "OpenPiton:1x1:*"
- "OpenPiton:4x4:*"
- "OpenTitan:*"
- "Servant:serv:*"
- "Servant:qerv:*"
- "VeeR-EH1:asic*"
- "VeeR-EH1:default*"
- "VeeR-EH1:hiperf*"
- "VeeR-EH2:asic*"
- "VeeR-EH2:default*"
- "VeeR-EH2:hiperf*"
- "VeeR-EL2:asic*"
- "VeeR-EL2:default*"
- "VeeR-EL2:hiperf*"
- "Vortex:mini:*"
- "Vortex:sane:*"
- "XiangShan:default-chisel3:* !*:linux"
- "XiangShan:default-chisel6:* !*:linux"
- "XiangShan:mini-chisel3:* !*:linux"
- "XiangShan:mini-chisel6:* !*:linux"
- "XuanTie-E902:*"
- "XuanTie-E906:*"
- "XuanTie-C906:*"
- "XuanTie-C910:*"
cases: ${{ fromJSON(needs.start.outputs.cases)['clang'] }}
run-gcc-hier:
name: Run GCC hier | ${{ matrix.cases }}
needs:
- start
- build-gcc-new
- build-gcc-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
@ -224,15 +176,7 @@ jobs:
fail-fast: false
max-parallel: ${{ github.event == 'schedule' && 2 || 7 }}
matrix:
cases:
- "BlackParrot:1x1:* !-hier"
- "BlackParrot:4x4:* !-hier"
- "NVDLA:* !-hier"
- "OpenPiton:1x1:* !-hier"
- "OpenPiton:4x4:* !-hier"
- "OpenPiton:8x8:* !-hier"
- "OpenPiton:16x16:dhry !-hier"
- "XuanTie-C910:* !-hier"
cases: ${{ fromJSON(needs.start.outputs.cases)['gcc-hier'] }}
combine-results:
name: Combine results
@ -246,12 +190,20 @@ jobs:
|| (github.event_name != 'pull_request' && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure')))
)}}
runs-on: ubuntu-24.04
outputs:
# The run tags, derived from the 'run-*' dependency job names
run-tags: ${{ steps.tags.outputs.tags }}
env:
GH_TOKEN: ${{ github.token }}
# 'gh' resolves the repo from git otherwise, but this job has no checkout
# of this repo at the workspace root
GH_REPO: ${{ github.repository }}
steps:
# Derive the run tags from the dependency job names ('run-<tag>')
- name: Determine run tags
id: tags
run: echo "tags=$(jq -r 'keys | map(sub("^run-"; "")) | join(" ")' <<< '${{ toJSON(needs) }}')" >> "$GITHUB_OUTPUT"
- name: Checkout RTLMeter
uses: actions/checkout@v6
with:
@ -264,7 +216,7 @@ jobs:
- name: Download all results
run: |
for tag in $RUN_TAGS; do
for tag in ${{ steps.tags.outputs.tags }}; do
mkdir artifacts all-results-$tag
gh run download ${{ github.run_id }} --pattern "rtlmeter-$tag-results-*" --dir artifacts
mv $(find artifacts -name "*.json") all-results-$tag/
@ -273,7 +225,7 @@ jobs:
- name: Combine results
working-directory: rtlmeter
run: |
for tag in $RUN_TAGS; do
for tag in ${{ steps.tags.outputs.tags }}; do
./rtlmeter collate ../all-results-$tag/*.json > ../all-results-$tag.json
done
- name: Upload combined results
@ -289,7 +241,7 @@ jobs:
- name: Download reference results
if: ${{ github.event_name == 'pull_request' }}
run: |
for tag in $RUN_TAGS; do
for tag in ${{ steps.tags.outputs.tags }}; do
mkdir artifacts all-reference-$tag
gh run download ${{ github.run_id }} --pattern "rtlmeter-$tag-reference-*" --dir artifacts
mv $(find artifacts -name "*.json") all-reference-$tag/
@ -299,7 +251,7 @@ jobs:
if: ${{ github.event_name == 'pull_request' }}
working-directory: rtlmeter
run: |
for tag in $RUN_TAGS; do
for tag in ${{ steps.tags.outputs.tags }}; do
./rtlmeter collate ../all-reference-$tag/*.json > ../all-reference-$tag.json
done
- name: Upload reference results
@ -398,7 +350,7 @@ jobs:
ln -s ../rtlmeter rtlmeter
gh repo set-default ${{ github.repository }}
# Compare to last successful scheduled run
ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} $RUN_TAGS
ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} ${{ needs.combine-results.outputs.run-tags }}
# Create the report artifact
mkdir ../report-artifact
mv rtlmeter-report/report ../report-artifact/

210
ci/ci-rtlmeter-cases.py Executable file
View File

@ -0,0 +1,210 @@
#!/usr/bin/env python3
# DESCRIPTION: Verilator: RTLMeter cases to run in the rtlmeter.yml CI workflow
#
# SPDX-FileCopyrightText: 2026 Wilson Snyder
# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
###############################################################################
# Defines the RTLMeter cases to run in CI. This is consumed by the 'start' job
# in rtlmeter.yml, which captures the printed JSON and feeds it to the run job
# matrices.
###############################################################################
import argparse
import json
# Cases to run for scheduled runs, keyyed by the 'run tag' used in rtlmeter.yml
# The associated priority is roughly the number of minutes it takes to run the
# job, the emitted job matrix will try to start longest jobs first.
# Cases to run for non-PR jobs (e.g.: nightly schedule)
# fmt: off
CASES = {
"gcc": [
("BlackParrot:1x1:*", 5),
("BlackParrot:2x2:*", 14),
("BlackParrot:4x4:*", 25),
("Caliptra:default:*", 19),
("NVDLA:*", 29),
("OpenPiton:1x1:*", 4),
("OpenPiton:2x2:*", 7),
("OpenPiton:4x4:*", 10),
("OpenTitan:*", 16),
("Servant:qerv:*", 2),
("Servant:serv:*", 2),
("VeeR-EH1:asic:*", 4),
("VeeR-EH1:default:*", 6),
("VeeR-EH1:hiperf:*", 5),
("VeeR-EH2:asic:*", 13),
("VeeR-EH2:default:*", 6),
("VeeR-EH2:hiperf:*", 12),
("VeeR-EL2:asic:*", 4),
("VeeR-EL2:default:*", 4),
("VeeR-EL2:hiperf:*", 4),
("Vortex:mini:*", 3),
("Vortex:sane:*", 6),
("XiangShan:default-chisel3:* !*:linux", 31),
("XiangShan:default-chisel6:* !*:linux", 28),
("XiangShan:mini-chisel3:* !*:linux", 13),
("XiangShan:mini-chisel6:* !*:linux", 15),
("XuanTie-C906:*", 5),
("XuanTie-C910:*", 9),
("XuanTie-E902:*", 5),
("XuanTie-E906:*", 6),
],
"clang": [
("BlackParrot:1x1:*", 5),
("BlackParrot:2x2:*", 9),
("BlackParrot:4x4:*", 12),
("Caliptra:default:*", 16),
("NVDLA:*", 28),
("OpenPiton:1x1:*", 5),
("OpenPiton:2x2:*", 6),
("OpenPiton:4x4:*", 8),
("OpenTitan:*", 10),
("Servant:qerv:*", 17),
("Servant:serv:*", 15),
("VeeR-EH1:asic:*", 6),
("VeeR-EH1:default:*", 8),
("VeeR-EH1:hiperf:*", 5),
("VeeR-EH2:asic:*", 9),
("VeeR-EH2:default:*", 8),
("VeeR-EH2:hiperf:*", 7),
("VeeR-EL2:asic:*", 4),
("VeeR-EL2:default:*", 5),
("VeeR-EL2:hiperf:*", 5),
("Vortex:mini:*", 3),
("Vortex:sane:*", 5),
("XiangShan:default-chisel3:* !*:linux", 21),
("XiangShan:default-chisel6:* !*:linux", 19),
("XiangShan:mini-chisel3:* !*:linux", 8),
("XiangShan:mini-chisel6:* !*:linux", 8),
("XuanTie-C906:*", 3),
("XuanTie-C910:*", 7),
("XuanTie-E902:*", 7),
("XuanTie-E906:*", 7),
],
"gcc-hier": [
("BlackParrot:1x1:*", 4),
("BlackParrot:2x2:*", 16),
("BlackParrot:4x4:*", 25),
("NVDLA:*", 25),
("OpenPiton:16x16:dhry", 16),
("OpenPiton:1x1:*", 4),
("OpenPiton:2x2:*", 4),
("OpenPiton:4x4:*", 4),
("OpenPiton:8x8:*", 6),
("XuanTie-C910:*", 7),
],
}
# fmt: on
# Cases to run for PR jobs
# fmt: off
CASES_PR = {
"gcc": [
("BlackParrot:1x1:*", 5),
("BlackParrot:4x4:*", 25),
("Caliptra:default:*", 19),
("NVDLA:*", 29),
("OpenPiton:1x1:*", 4),
("OpenPiton:4x4:*", 10),
("OpenTitan:*", 16),
("Servant:qerv:*", 2),
("Servant:serv:*", 2),
("VeeR-EH1:asic:*", 4),
("VeeR-EH1:default:*", 6),
("VeeR-EH1:hiperf:*", 5),
("VeeR-EH2:asic:*", 13),
("VeeR-EH2:default:*", 6),
("VeeR-EH2:hiperf:*", 12),
("VeeR-EL2:asic:*", 4),
("VeeR-EL2:default:*", 4),
("VeeR-EL2:hiperf:*", 4),
("Vortex:mini:*", 3),
("Vortex:sane:*", 6),
("XiangShan:default-chisel3:* !*:linux", 31),
("XiangShan:default-chisel6:* !*:linux", 28),
("XiangShan:mini-chisel3:* !*:linux", 13),
("XiangShan:mini-chisel6:* !*:linux", 15),
("XuanTie-C906:*", 5),
("XuanTie-C910:*", 9),
("XuanTie-E902:*", 5),
("XuanTie-E906:*", 6),
],
"clang": [
("BlackParrot:1x1:*", 5),
("BlackParrot:4x4:*", 12),
("Caliptra:default:*", 16),
("NVDLA:*", 28),
("OpenPiton:1x1:*", 5),
("OpenPiton:4x4:*", 8),
("OpenTitan:*", 10),
("Servant:qerv:*", 17),
("Servant:serv:*", 15),
("VeeR-EH1:asic:*", 6),
("VeeR-EH1:default:*", 8),
("VeeR-EH1:hiperf:*", 5),
("VeeR-EH2:asic:*", 9),
("VeeR-EH2:default:*", 8),
("VeeR-EH2:hiperf:*", 7),
("VeeR-EL2:asic:*", 4),
("VeeR-EL2:default:*", 5),
("VeeR-EL2:hiperf:*", 5),
("Vortex:mini:*", 3),
("Vortex:sane:*", 5),
("XiangShan:default-chisel3:* !*:linux", 21),
("XiangShan:default-chisel6:* !*:linux", 19),
("XiangShan:mini-chisel3:* !*:linux", 8),
("XiangShan:mini-chisel6:* !*:linux", 8),
("XuanTie-C906:*", 3),
("XuanTie-C910:*", 7),
("XuanTie-E902:*", 7),
("XuanTie-E906:*", 7),
],
"gcc-hier": [
("BlackParrot:1x1:*", 4),
("BlackParrot:4x4:*", 25),
("NVDLA:*", 25),
("OpenPiton:16x16:dhry", 16),
("OpenPiton:1x1:*", 4),
("OpenPiton:4x4:*", 4),
("OpenPiton:8x8:*", 6),
("XuanTie-C910:*", 7),
],
}
# fmt: on
parser = argparse.ArgumentParser(
description="Print the RTLMeter CI cases as JSON, keyed by run tag")
parser.add_argument("--event-name",
required=True,
help="GitHub event name that triggered the workflow")
parser.add_argument(
"--ci-testing",
action="store_true",
help="Print only the shortest (lowest priority) case per tag, to reduce CI testing time")
args = parser.parse_args()
# Pull requests use the reduced 'CASES_PR' set, other events (e.g.: the nightly
# schedule) use the full 'CASES' set.
is_pr = args.event_name == "pull_request"
selected = CASES_PR if is_pr else CASES
cases = {}
for tag, entries in selected.items():
# Highest priority (longest) first
ordered = sorted(entries, key=lambda cp: cp[1], reverse=True)
# While tweaking the CI, keep only the shortest (lowest priority) case to minimise churn time
if args.ci_testing:
ordered = ordered[-1:]
# Case filters appended to every case for this tag/event
suffix = ""
# Filters out the non-hierarchical case variants for the hierarchical job
if tag == "gcc-hier":
suffix += " !-hier"
# Filter out hello tests on pull requests, these just add measurement noise
if is_pr:
suffix += " !*:hello"
cases[tag] = [case + suffix for case, _ in ordered]
print(json.dumps(cases, separators=(",", ":")))

8
ci/ci-rtlmeter-report.py
View File

@ -73,6 +73,8 @@ for ref, cmp in zip(sys.argv[1::2], sys.argv[2::2]):
maxGain = max(maxGain, g)
meanGain *= g
count += 1
if count == 0:
continue
meanGain = meanGain**(1 / count)
if metric == "clocks":
@ -92,12 +94,12 @@ for ref, cmp in zip(sys.argv[1::2], sys.argv[2::2]):
table.append([
runName, step, ref_json["metrics"][metric]["header"], f"{meanGain:.2f}x {status} ",
f"{minGain:.2f}x", f"{maxGain:.2f}x"
f"{minGain:.2f}x", f"{maxGain:.2f}x", f"{count}"
])
printTable(
table,
headers=("Run", "Step", "Metric", "Improvement", "Min", "Max"),
colalign=("left", "left", "left", "right", "right", "right"),
headers=("Run", "Step", "Metric", "Improvement", "Min", "Max", "Samples"),
colalign=("left", "left", "left", "right", "right", "right", "right"),
disable_numparse=True,
)