luke/verilator
1
0
Fork 0
mirror of https://github.com/verilator/verilator.git synced 2026-09-01 18:36:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

CI: Run RTLMeter on the same machine with target commit (#7726)

Browse Source
This commit is contained in:
Geza Lore
2026-06-07 15:36:34 +01:00
committed by GitHub
parent d9cbc27902
commit 83ef69d866
4 changed files with 256 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/reusable-rtlmeter-build.yml
+25 -6
View File
@@ -15,6 +15,14 @@ on:
description: "Compiler to use: 'gcc' or 'clang'"
type: string
required: true
sha:
description: "Git SHA to build"
type: string
required: true
outputs:
archive:
description: "Name of the built installation archive artifact"
value: ${{ jobs.build.outputs.archive }}
defaults:
run:
@@ -26,8 +34,10 @@ env:
jobs:
build:
runs-on: ${{ inputs.runs-on }}
name: Build
runs-on: ${{ inputs.runs-on }}
outputs:
archive: ${{ steps.create-archive.outputs.archive }}
steps:
- name: Install dependencies
run: |
@@ -43,13 +53,17 @@ jobs:
uses: actions/cache@v5
with:
path: ccache
key: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}
key: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }}-${{ github.run_id }}
rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }}
rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}
- name: Checkout
uses: actions/checkout@v6
with:
path: repo
ref: ${{ inputs.sha }}
fetch-depth: 0 # Required for 'git describe' used for 'verilator --version'
- name: Configure
@@ -67,11 +81,16 @@ jobs:
run: make install
- name: Tar up installation
run: tar --posix -c -z -f verilator-rtlmeter.tar.gz install
id: create-archive
run: |
SHA=$(git -C repo rev-parse HEAD)
ARCHIVE=verilator-$SHA-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }}.tar.gz
tar --posix -c -z -f $ARCHIVE install
echo "archive=$ARCHIVE" >> $GITHUB_OUTPUT
- name: Upload Verilator installation archive
uses: actions/upload-artifact@v7
with:
path: verilator-rtlmeter.tar.gz
name: verilator-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }}
path: ${{ steps.create-archive.outputs.archive }}
name: ${{ steps.create-archive.outputs.archive }}
overwrite: true
.github/workflows/reusable-rtlmeter-run.yml
+100 -24
View File
@@ -19,6 +19,15 @@ on:
description: "Compiler to use: 'gcc' or 'clang'"
type: string
required: true
verilator-archive-new:
description: "Name of the installation archive artifact from reusable-rtlmeter-build, new version"
type: string
required: true
verilator-archive-old:
description: "Name of the installation archive artifact from reusable-rtlmeter-build, old version"
type: string
required: false
default: ""
# Note: The combination of 'cases' and 'run-name' must be unique for all
# invocations of this workflow within a run of the parent workflow.
# These two are used together to generate a unique results file name.
@@ -63,24 +72,6 @@ jobs:
sudo apt install ccache mold libfl-dev libjemalloc-dev libsystemc-dev || \
sudo apt install ccache mold libfl-dev libjemalloc-dev libsystemc-dev
- name: Download Verilator installation archive
uses: actions/download-artifact@v8
with:
name: verilator-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }}
- name: Unpack Verilator installation archive
run: |
tar -x -z -f verilator-rtlmeter.tar.gz
echo "${{ github.workspace }}/install/bin" >> $GITHUB_PATH
- name: Use saved ccache
if: ${{ env.CCACHE_DISABLE == 0 }}
uses: actions/cache@v5
with:
path: ${{ env.CCACHE_DIR }}
key: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}
- name: Checkout RTLMeter
uses: actions/checkout@v6
with:
@@ -91,32 +82,59 @@ jobs:
working-directory: rtlmeter
run: make venv
- name: Compile cases
- name: Use saved ccache
if: ${{ env.CCACHE_DISABLE == 0 }}
uses: actions/cache@v5
with:
path: ${{ env.CCACHE_DIR }}
key: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}
########################################################################
# Run with new Verilator
########################################################################
- name: Download Verilator installation archive - new
uses: actions/download-artifact@v8
with:
name: ${{ inputs.verilator-archive-new }}
- name: Unpack Verilator installation archive - new
run: |
tar -x -z -f ${{ inputs.verilator-archive-new }}
mv install verilator-new
- name: Compile cases - new
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' --nExecute=0
ccache -svv
- name: Execute cases
- name: Execute cases - new
working-directory: rtlmeter
continue-on-error: true # Do not fail on error, so we can at least save the successful results
run: |
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}'
- name: Collate results
- name: Collate results - new
id: results
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
# Use 'inputs.cases' and 'inputs.run-name' to generate a unique file name
hash=$(md5sum <<< '${{ inputs.cases }} ${{ inputs.run-name }}' | awk '{print $1}')
echo "hash=${hash}" >> $GITHUB_OUTPUT
./rtlmeter collate --runName "${{ inputs.run-name }}" > ../results-${hash}.json
- name: Report results
- name: Report results - new
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
./rtlmeter report --steps '*' --metrics '*' ../results-${{ steps.results.outputs.hash }}.json
- name: Upload results
- name: Upload results - new
uses: actions/upload-artifact@v7
with:
path: results-${{ steps.results.outputs.hash }}.json
@@ -124,7 +142,65 @@ jobs:
overwrite: true
retention-days: 2
- name: Report status
- name: Report status - new
working-directory: rtlmeter
run: |- # This will fail the job if any of the runs failed
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
./rtlmeter run --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}'
# Clean up for run with old version
rm -rf work
rm -rf ${{ github.workspace }}/verilator-new
########################################################################
# Run with old Verilator **on same machine for consistency**
########################################################################
- name: Download Verilator installation archive - old
if: ${{ inputs.verilator-archive-old != '' }}
uses: actions/download-artifact@v8
with:
name: ${{ inputs.verilator-archive-old }}
- name: Unpack Verilator installation archive - old
if: ${{ inputs.verilator-archive-old != '' }}
run: |
tar -x -z -f ${{ inputs.verilator-archive-old }}
mv install verilator-old
- name: Compile cases - old
if: ${{ inputs.verilator-archive-old != '' }}
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-old/bin:$PATH"
./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' --nExecute=0
ccache -svv
- name: Execute cases - old
if: ${{ inputs.verilator-archive-old != '' }}
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-old/bin:$PATH"
./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}'
- name: Collate results - old
if: ${{ inputs.verilator-archive-old != '' }}
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-old/bin:$PATH"
./rtlmeter collate --runName "${{ inputs.run-name }}" > ../reference-${{ steps.results.outputs.hash }}.json
- name: Report results - old
if: ${{ inputs.verilator-archive-old != '' }}
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-old/bin:$PATH"
./rtlmeter report --steps '*' --metrics '*' ../reference-${{ steps.results.outputs.hash }}.json
- name: Upload results - old
if: ${{ inputs.verilator-archive-old != '' }}
uses: actions/upload-artifact@v7
with:
path: reference-${{ steps.results.outputs.hash }}.json
name: rtlmeter-${{ inputs.tag }}-reference-${{ steps.results.outputs.hash }}
overwrite: true
retention-days: 2
.github/workflows/rtlmeter.yml
+95 -14
View File
@@ -40,34 +40,73 @@ jobs:
(github.event_name == 'workflow_dispatch') ||
(github.event_name == 'push')
runs-on: ubuntu-24.04
outputs:
old-sha: ${{ steps.start.outputs.old-sha }}
steps:
- name: Startup
run: echo
id: start
env:
GH_TOKEN: ${{ github.token }}
run: |
[[ "${{ github.event_name }}" == 'pull_request' ]] || exit 0
# For a pull request, 'github.sha' is the test merge commit. Its
# first parent is the target branch commit it was merged with for this run.
OLD_SHA="$(gh api "repos/${{ github.repository }}/commits/${{ github.sha }}" --jq '.parents[0].sha')"
echo "old-sha=$OLD_SHA" >> "$GITHUB_OUTPUT"
build-gcc:
name: Build GCC
build-gcc-new:
name: Build New Verilator - GCC
needs: start
uses: ./.github/workflows/reusable-rtlmeter-build.yml
with:
runs-on: ubuntu-24.04
cc: gcc
sha: ${{ github.sha }}
build-clang:
name: Build Clang
build-clang-new:
name: Build New Verilator - Clang
needs: start
uses: ./.github/workflows/reusable-rtlmeter-build.yml
with:
runs-on: ubuntu-24.04
cc: clang
sha: ${{ github.sha }}
build-gcc-old:
name: Build Old Verilator - GCC
needs: start
if: ${{ needs.start.outputs.old-sha != '' }}
uses: ./.github/workflows/reusable-rtlmeter-build.yml
with:
runs-on: ubuntu-24.04
cc: gcc
sha: ${{ needs.start.outputs.old-sha }}
build-clang-old:
name: Build Old Verilator - Clang
needs: start
if: ${{ needs.start.outputs.old-sha != '' }}
uses: ./.github/workflows/reusable-rtlmeter-build.yml
with:
runs-on: ubuntu-24.04
cc: clang
sha: ${{ needs.start.outputs.old-sha }}
run-gcc:
name: Run GCC | ${{ matrix.cases }}
needs: build-gcc
needs:
- build-gcc-new
- build-gcc-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
# dependency failed or the workflow was cancelled.
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable-rtlmeter-run.yml
with:
tag: gcc
runs-on: ubuntu-24.04
cc: gcc
verilator-archive-new: ${{ needs.build-gcc-new.outputs.archive }}
verilator-archive-old: ${{ needs.build-gcc-old.outputs.archive }}
cases: ${{ matrix.cases }}
run-name: "gcc"
compileArgs: ""
@@ -108,12 +147,19 @@ jobs:
run-clang:
name: Run Clang | ${{ matrix.cases }}
needs: build-clang
needs:
- build-clang-new
- build-clang-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
# dependency failed or the workflow was cancelled.
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable-rtlmeter-run.yml
with:
tag: clang
runs-on: ubuntu-24.04
cc: clang
verilator-archive-new: ${{ needs.build-clang-new.outputs.archive }}
verilator-archive-old: ${{ needs.build-clang-old.outputs.archive }}
cases: ${{ matrix.cases }}
run-name: "clang --threads 4"
compileArgs: "--threads 4"
@@ -154,12 +200,19 @@ jobs:
run-gcc-hier:
name: Run GCC hier | ${{ matrix.cases }}
needs: build-gcc
needs:
- build-gcc-new
- build-gcc-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
# dependency failed or the workflow was cancelled.
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable-rtlmeter-run.yml
with:
tag: gcc-hier
runs-on: ubuntu-24.04
cc: gcc
verilator-archive-new: ${{ needs.build-gcc-new.outputs.archive }}
verilator-archive-old: ${{ needs.build-gcc-old.outputs.archive }}
cases: ${{ matrix.cases }}
run-name: "gcc --hierarchical"
compileArgs: "--hierarchical"
@@ -181,9 +234,14 @@ jobs:
combine-results:
name: Combine results
needs: [run-gcc, run-clang, run-gcc-hier]
# Run if any of the dependencies have run, even if failed.
# That is: do not run if all skipped, or the workflow was cancelled.
if: ${{ (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure')) && !cancelled() }}
# Skip if cancelled.
# On PRs, run if something succeded and nothign failed.
# On non-PRs, run if anything succeded or failed (so partial results are still publided).
if: >-
${{ !cancelled() && (
(github.event_name == 'pull_request' && (contains(needs.*.result, 'success') && !contains(needs.*.result, 'failure')))
|| (github.event_name != 'pull_request' && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure')))
)}}
runs-on: ubuntu-24.04
strategy:
fail-fast: false
@@ -195,9 +253,11 @@ jobs:
with:
repository: "verilator/rtlmeter"
path: rtlmeter
- name: Setup RTLMeter venv
working-directory: rtlmeter
run: make venv
- name: Download all results
uses: actions/download-artifact@v8
with:
@@ -216,6 +276,29 @@ jobs:
overwrite: true
retention-days: 30
# The reference (old) results only exist for pull requests, where the
# 'build-*-old' jobs run and the run jobs produce '*-reference-*' artifacts.
- name: Download reference results
if: ${{ github.event_name == 'pull_request' }}
uses: actions/download-artifact@v8
with:
pattern: rtlmeter-${{ matrix.tag }}-reference-*
path: all-reference-${{ matrix.tag }}
merge-multiple: true
- name: Combine reference results
if: ${{ github.event_name == 'pull_request' }}
working-directory: rtlmeter
run: |
./rtlmeter collate ../all-reference-${{ matrix.tag }}/*.json > ../all-reference-${{ matrix.tag }}.json
- name: Upload reference results
if: ${{ github.event_name == 'pull_request' }}
uses: actions/upload-artifact@v7
with:
path: all-reference-${{ matrix.tag }}.json
name: all-reference-${{ matrix.tag }}
overwrite: true
retention-days: 30
publish-scheduled-results:
name: Publish results to verilator/verilator-rtlmeter-results
needs: combine-results
@@ -304,9 +387,7 @@ jobs:
ln -s ../rtlmeter rtlmeter
gh repo set-default ${{ github.repository }}
# Compare to last successful scheduled run
REF_ID=$(gh run list --workflow RTLMeter --event schedule --status success --limit 1 --json databaseId --jq ".[0].databaseId")
NEW_ID=${{ github.run_id }}
ci/ci-rtlmeter-report.bash $REF_ID $NEW_ID gcc clang gcc-hier
ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} gcc clang gcc-hier
# Create the report artifact
mkdir ../report-artifact
mv rtlmeter-report/report ../report-artifact/