luke
/
verilator
mirror of https://github.com/verilator/verilator.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CI: Run RTLMeter on the same machine with target commit (#7726)

This commit is contained in:
Geza Lore 2026-06-07 15:36:34 +01:00 committed by GitHub
parent d9cbc27902
commit 83ef69d866
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 256 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.github/workflows/reusable-rtlmeter-build.yml vendored
View File

@ -15,6 +15,14 @@ on:
description: "Compiler to use: 'gcc' or 'clang'"
type: string
required: true
sha:
description: "Git SHA to build"
type: string
required: true
outputs:
archive:
description: "Name of the built installation archive artifact"
value: ${{ jobs.build.outputs.archive }}
defaults:
run:
@ -26,8 +34,10 @@ env:
jobs:
build:
runs-on: ${{ inputs.runs-on }}
name: Build
runs-on: ${{ inputs.runs-on }}
outputs:
archive: ${{ steps.create-archive.outputs.archive }}
steps:
- name: Install dependencies
run: |
@ -43,13 +53,17 @@ jobs:
uses: actions/cache@v5
with:
path: ccache
key: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}
key: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }}-${{ github.run_id }}
rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }}
rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}
- name: Checkout
uses: actions/checkout@v6
with:
path: repo
ref: ${{ inputs.sha }}
fetch-depth: 0 # Required for 'git describe' used for 'verilator --version'
- name: Configure
@ -67,11 +81,16 @@ jobs:
run: make install
- name: Tar up installation
run: tar --posix -c -z -f verilator-rtlmeter.tar.gz install
id: create-archive
run: |
SHA=$(git -C repo rev-parse HEAD)
ARCHIVE=verilator-$SHA-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }}.tar.gz
tar --posix -c -z -f $ARCHIVE install
echo "archive=$ARCHIVE" >> $GITHUB_OUTPUT
- name: Upload Verilator installation archive
uses: actions/upload-artifact@v7
with:
path: verilator-rtlmeter.tar.gz
name: verilator-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }}
path: ${{ steps.create-archive.outputs.archive }}
name: ${{ steps.create-archive.outputs.archive }}
overwrite: true

124
.github/workflows/reusable-rtlmeter-run.yml vendored
View File

@ -19,6 +19,15 @@ on:
description: "Compiler to use: 'gcc' or 'clang'"
type: string
required: true
verilator-archive-new:
description: "Name of the installation archive artifact from reusable-rtlmeter-build, new version"
type: string
required: true
verilator-archive-old:
description: "Name of the installation archive artifact from reusable-rtlmeter-build, old version"
type: string
required: false
default: ""
# Note: The combination of 'cases' and 'run-name' must be unique for all
# invocations of this workflow within a run of the parent workflow.
# These two are used together to generate a unique results file name.
@ -63,24 +72,6 @@ jobs:
sudo apt install ccache mold libfl-dev libjemalloc-dev libsystemc-dev || \
sudo apt install ccache mold libfl-dev libjemalloc-dev libsystemc-dev
- name: Download Verilator installation archive
uses: actions/download-artifact@v8
with:
name: verilator-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }}
- name: Unpack Verilator installation archive
run: |
tar -x -z -f verilator-rtlmeter.tar.gz
echo "${{ github.workspace }}/install/bin" >> $GITHUB_PATH
- name: Use saved ccache
if: ${{ env.CCACHE_DISABLE == 0 }}
uses: actions/cache@v5
with:
path: ${{ env.CCACHE_DIR }}
key: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}
- name: Checkout RTLMeter
uses: actions/checkout@v6
with:
@ -91,32 +82,59 @@ jobs:
working-directory: rtlmeter
run: make venv
- name: Compile cases
- name: Use saved ccache
if: ${{ env.CCACHE_DISABLE == 0 }}
uses: actions/cache@v5
with:
path: ${{ env.CCACHE_DIR }}
key: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}
########################################################################
# Run with new Verilator
########################################################################
- name: Download Verilator installation archive - new
uses: actions/download-artifact@v8
with:
name: ${{ inputs.verilator-archive-new }}
- name: Unpack Verilator installation archive - new
run: |
tar -x -z -f ${{ inputs.verilator-archive-new }}
mv install verilator-new
- name: Compile cases - new
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' --nExecute=0
ccache -svv
- name: Execute cases
- name: Execute cases - new
working-directory: rtlmeter
continue-on-error: true # Do not fail on error, so we can at least save the successful results
run: |
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}'
- name: Collate results
- name: Collate results - new
id: results
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
# Use 'inputs.cases' and 'inputs.run-name' to generate a unique file name
hash=$(md5sum <<< '${{ inputs.cases }} ${{ inputs.run-name }}' | awk '{print $1}')
echo "hash=${hash}" >> $GITHUB_OUTPUT
./rtlmeter collate --runName "${{ inputs.run-name }}" > ../results-${hash}.json
- name: Report results
- name: Report results - new
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
./rtlmeter report --steps '*' --metrics '*' ../results-${{ steps.results.outputs.hash }}.json
- name: Upload results
- name: Upload results - new
uses: actions/upload-artifact@v7
with:
path: results-${{ steps.results.outputs.hash }}.json
@ -124,7 +142,65 @@ jobs:
overwrite: true
retention-days: 2
- name: Report status
- name: Report status - new
working-directory: rtlmeter
run: |- # This will fail the job if any of the runs failed
export PATH="${{ github.workspace }}/verilator-new/bin:$PATH"
./rtlmeter run --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}'
# Clean up for run with old version
rm -rf work
rm -rf ${{ github.workspace }}/verilator-new
########################################################################
# Run with old Verilator **on same machine for consistency**
########################################################################
- name: Download Verilator installation archive - old
if: ${{ inputs.verilator-archive-old != '' }}
uses: actions/download-artifact@v8
with:
name: ${{ inputs.verilator-archive-old }}
- name: Unpack Verilator installation archive - old
if: ${{ inputs.verilator-archive-old != '' }}
run: |
tar -x -z -f ${{ inputs.verilator-archive-old }}
mv install verilator-old
- name: Compile cases - old
if: ${{ inputs.verilator-archive-old != '' }}
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-old/bin:$PATH"
./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' --nExecute=0
ccache -svv
- name: Execute cases - old
if: ${{ inputs.verilator-archive-old != '' }}
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-old/bin:$PATH"
./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}'
- name: Collate results - old
if: ${{ inputs.verilator-archive-old != '' }}
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-old/bin:$PATH"
./rtlmeter collate --runName "${{ inputs.run-name }}" > ../reference-${{ steps.results.outputs.hash }}.json
- name: Report results - old
if: ${{ inputs.verilator-archive-old != '' }}
working-directory: rtlmeter
run: |
export PATH="${{ github.workspace }}/verilator-old/bin:$PATH"
./rtlmeter report --steps '*' --metrics '*' ../reference-${{ steps.results.outputs.hash }}.json
- name: Upload results - old
if: ${{ inputs.verilator-archive-old != '' }}
uses: actions/upload-artifact@v7
with:
path: reference-${{ steps.results.outputs.hash }}.json
name: rtlmeter-${{ inputs.tag }}-reference-${{ steps.results.outputs.hash }}
overwrite: true
retention-days: 2

109
.github/workflows/rtlmeter.yml vendored
View File

@ -40,34 +40,73 @@ jobs:
(github.event_name == 'workflow_dispatch') ||
(github.event_name == 'push')
runs-on: ubuntu-24.04
outputs:
old-sha: ${{ steps.start.outputs.old-sha }}
steps:
- name: Startup
run: echo
id: start
env:
GH_TOKEN: ${{ github.token }}
run: |
[[ "${{ github.event_name }}" == 'pull_request' ]] || exit 0
# For a pull request, 'github.sha' is the test merge commit. Its
# first parent is the target branch commit it was merged with for this run.
OLD_SHA="$(gh api "repos/${{ github.repository }}/commits/${{ github.sha }}" --jq '.parents[0].sha')"
echo "old-sha=$OLD_SHA" >> "$GITHUB_OUTPUT"
build-gcc:
name: Build GCC
build-gcc-new:
name: Build New Verilator - GCC
needs: start
uses: ./.github/workflows/reusable-rtlmeter-build.yml
with:
runs-on: ubuntu-24.04
cc: gcc
sha: ${{ github.sha }}
build-clang:
name: Build Clang
build-clang-new:
name: Build New Verilator - Clang
needs: start
uses: ./.github/workflows/reusable-rtlmeter-build.yml
with:
runs-on: ubuntu-24.04
cc: clang
sha: ${{ github.sha }}
build-gcc-old:
name: Build Old Verilator - GCC
needs: start
if: ${{ needs.start.outputs.old-sha != '' }}
uses: ./.github/workflows/reusable-rtlmeter-build.yml
with:
runs-on: ubuntu-24.04
cc: gcc
sha: ${{ needs.start.outputs.old-sha }}
build-clang-old:
name: Build Old Verilator - Clang
needs: start
if: ${{ needs.start.outputs.old-sha != '' }}
uses: ./.github/workflows/reusable-rtlmeter-build.yml
with:
runs-on: ubuntu-24.04
cc: clang
sha: ${{ needs.start.outputs.old-sha }}
run-gcc:
name: Run GCC | ${{ matrix.cases }}
needs: build-gcc
needs:
- build-gcc-new
- build-gcc-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
# dependency failed or the workflow was cancelled.
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable-rtlmeter-run.yml
with:
tag: gcc
runs-on: ubuntu-24.04
cc: gcc
verilator-archive-new: ${{ needs.build-gcc-new.outputs.archive }}
verilator-archive-old: ${{ needs.build-gcc-old.outputs.archive }}
cases: ${{ matrix.cases }}
run-name: "gcc"
compileArgs: ""
@ -108,12 +147,19 @@ jobs:
run-clang:
name: Run Clang | ${{ matrix.cases }}
needs: build-clang
needs:
- build-clang-new
- build-clang-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
# dependency failed or the workflow was cancelled.
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable-rtlmeter-run.yml
with:
tag: clang
runs-on: ubuntu-24.04
cc: clang
verilator-archive-new: ${{ needs.build-clang-new.outputs.archive }}
verilator-archive-old: ${{ needs.build-clang-old.outputs.archive }}
cases: ${{ matrix.cases }}
run-name: "clang --threads 4"
compileArgs: "--threads 4"
@ -154,12 +200,19 @@ jobs:
run-gcc-hier:
name: Run GCC hier | ${{ matrix.cases }}
needs: build-gcc
needs:
- build-gcc-new
- build-gcc-old
# Run even if 'build-*-old' was skipped (no old SHA), but not if any
# dependency failed or the workflow was cancelled.
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable-rtlmeter-run.yml
with:
tag: gcc-hier
runs-on: ubuntu-24.04
cc: gcc
verilator-archive-new: ${{ needs.build-gcc-new.outputs.archive }}
verilator-archive-old: ${{ needs.build-gcc-old.outputs.archive }}
cases: ${{ matrix.cases }}
run-name: "gcc --hierarchical"
compileArgs: "--hierarchical"
@ -181,9 +234,14 @@ jobs:
combine-results:
name: Combine results
needs: [run-gcc, run-clang, run-gcc-hier]
# Run if any of the dependencies have run, even if failed.
# That is: do not run if all skipped, or the workflow was cancelled.
if: ${{ (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure')) && !cancelled() }}
# Skip if cancelled.
# On PRs, run if something succeded and nothign failed.
# On non-PRs, run if anything succeded or failed (so partial results are still publided).
if: >-
${{ !cancelled() && (
(github.event_name == 'pull_request' && (contains(needs.*.result, 'success') && !contains(needs.*.result, 'failure')))
|| (github.event_name != 'pull_request' && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure')))
)}}
runs-on: ubuntu-24.04
strategy:
fail-fast: false
@ -195,9 +253,11 @@ jobs:
with:
repository: "verilator/rtlmeter"
path: rtlmeter
- name: Setup RTLMeter venv
working-directory: rtlmeter
run: make venv
- name: Download all results
uses: actions/download-artifact@v8
with:
@ -216,6 +276,29 @@ jobs:
overwrite: true
retention-days: 30
# The reference (old) results only exist for pull requests, where the
# 'build-*-old' jobs run and the run jobs produce '*-reference-*' artifacts.
- name: Download reference results
if: ${{ github.event_name == 'pull_request' }}
uses: actions/download-artifact@v8
with:
pattern: rtlmeter-${{ matrix.tag }}-reference-*
path: all-reference-${{ matrix.tag }}
merge-multiple: true
- name: Combine reference results
if: ${{ github.event_name == 'pull_request' }}
working-directory: rtlmeter
run: |
./rtlmeter collate ../all-reference-${{ matrix.tag }}/*.json > ../all-reference-${{ matrix.tag }}.json
- name: Upload reference results
if: ${{ github.event_name == 'pull_request' }}
uses: actions/upload-artifact@v7
with:
path: all-reference-${{ matrix.tag }}.json
name: all-reference-${{ matrix.tag }}
overwrite: true
retention-days: 30
publish-scheduled-results:
name: Publish results to verilator/verilator-rtlmeter-results
needs: combine-results
@ -304,9 +387,7 @@ jobs:
ln -s ../rtlmeter rtlmeter
gh repo set-default ${{ github.repository }}
# Compare to last successful scheduled run
REF_ID=$(gh run list --workflow RTLMeter --event schedule --status success --limit 1 --json databaseId --jq ".[0].databaseId")
NEW_ID=${{ github.run_id }}
ci/ci-rtlmeter-report.bash $REF_ID $NEW_ID gcc clang gcc-hier
ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} gcc clang gcc-hier
# Create the report artifact
mkdir ../report-artifact
mv rtlmeter-report/report ../report-artifact/

61
ci/ci-rtlmeter-report.bash
View File

@ -15,11 +15,11 @@
[ "$GITHUB_ACTIONS" != "true" ] || set -x
# Arguments:
# 1. reference run ID
# 2. new run ID
# 1. run ID
# 2. SHA of the event that triggered the run (for a PR, the test merge commit)
# rest: run tags
REF_ID=$1; shift
NEW_ID=$1; shift
RUN_ID=$1; shift
RUN_SHA=$1; shift
RUNS="$@"
# $VERILATOR_CHECKOUT/ci directory
@ -32,31 +32,30 @@ pushd rtlmeter-report &> /dev/null
TMP_DIR=$(readlink -f .)
# Artifacts to download
DOWNLOAD_ARTIFACTS=""
DOWNLOAD_NEW_ARTIFACTS=""
DOWNLOAD_REF_ARTIFACTS=""
for r in $RUNS; do
DOWNLOAD_ARTIFACTS="$DOWNLOAD_ARTIFACTS --name all-results-$r"
DOWNLOAD_NEW_ARTIFACTS="$DOWNLOAD_NEW_ARTIFACTS --name all-results-$r"
DOWNLOAD_REF_ARTIFACTS="$DOWNLOAD_REF_ARTIFACTS --name all-reference-$r"
done
# Download reference run results
# Download reference artifacts
mkdir ref
REF_DIR=$(readlink -f ref)
gh run download ${REF_ID} $DOWNLOAD_ARTIFACTS --dir $REF_DIR
gh run download ${RUN_ID} $DOWNLOAD_REF_ARTIFACTS --dir $REF_DIR
mv $REF_DIR/*/*.json $REF_DIR/
find $REF_DIR -mindepth 1 -type d -delete
# Download current run results
# Download new version artifacts
mkdir new
NEW_DIR=$(readlink -f new)
gh run download ${NEW_ID} $DOWNLOAD_ARTIFACTS --dir $NEW_DIR
gh run download ${RUN_ID} $DOWNLOAD_NEW_ARTIFACTS --dir $NEW_DIR
mv $NEW_DIR/*/*.json $NEW_DIR/
find $NEW_DIR -mindepth 1 -type d -delete
# Get Some metadata about the runs
REF_URL=$(gh run view $REF_ID --json url --jq ".url")
REF_NUM=$(gh run view $REF_ID --json number --jq ".number")
REF_DATE=$(gh run view $REF_ID --json createdAt --jq ".createdAt")
NEW_URL=$(gh run view $NEW_ID --json url --jq ".url")
NEW_NUM=$(gh run view $NEW_ID --json number --jq ".number")
RUN_URL=$(gh run view $RUN_ID --json url --jq ".url")
RUN_NUM=$(gh run view $RUN_ID --json number --jq ".number")
# Repository owner and name of the default repository, used to build the
# GitHub Pages URL of the detailed report. The owner is lowercased, as required
@ -65,6 +64,12 @@ NEW_NUM=$(gh run view $NEW_ID --json number --jq ".number")
PAGES_OWNER=$(gh repo view --json owner --jq '.owner.login' | tr '[:upper:]' '[:lower:]')
PAGES_NAME=$(gh repo view --json name --jq '.name')
# The 'old' reference was built from the target branch base commit, which is
# the first parent of the triggering merge commit (same as the 'start' job in
# rtlmeter.yml). Resolved here, while still in the default repository's git
# context (before the 'cd rtlmeter' below).
REF_SHA=$(gh api "repos/{owner}/{repo}/commits/$RUN_SHA" --jq '.parents[0].sha')
# Go back to root directory
popd &> /dev/null
# Go to RTLMeter directory
@ -75,14 +80,14 @@ SUMMARY_ARGS=()
for r in $RUNS; do
CMP_JSON=$TMP_DIR/cmp-$r.json
# Gather args for summary script
SUMMARY_ARGS+=($REF_DIR/all-results-$r.json $CMP_JSON)
SUMMARY_ARGS+=($NEW_DIR/all-results-$r.json $CMP_JSON)
# Create JSON
./rtlmeter compare --format json --steps "*" --metrics "*" \
$REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $CMP_JSON
$REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $CMP_JSON
# Also create detailed tables
./rtlmeter compare --format ascii --steps 'verilate' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/verilate-$r.txt
./rtlmeter compare --format ascii --steps 'cppbuild' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/cppbuild-$r.txt
./rtlmeter compare --format ascii --steps 'execute' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/execute-$r.txt
./rtlmeter compare --format ascii --steps 'verilate' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/verilate-$r.txt
./rtlmeter compare --format ascii --steps 'cppbuild' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/cppbuild-$r.txt
./rtlmeter compare --format ascii --steps 'execute' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/execute-$r.txt
# Chop them at new lines, into one table per file
awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-verilate-%02d.txt",prefix,NR)}' $TMP_DIR/verilate-$r.txt
awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-cppbuild-%02d.txt",prefix,NR)}' $TMP_DIR/cppbuild-$r.txt
@ -97,8 +102,10 @@ cat $TMP_DIR/summary.txt
# Create notification comment content
NOTIFICATION=$TMP_DIR/notification.txt
cat > $NOTIFICATION <<NOTIFICATION_TEMPLATE
Performance metrics for PR workflow [#$NEW_NUM]($NEW_URL) (B)
compared to scheduled run [#$REF_NUM]($REF_URL) (A) from $REF_DATE
Performance metrics for PR workflow [#$RUN_NUM]($RUN_URL), comparing:
- target branch commit [${REF_SHA:0:9}](https://github.com/${PAGES_OWNER}/${PAGES_NAME}/commit/${REF_SHA}) (A)
- with PR merge commit [${RUN_SHA:0:9}](https://github.com/${PAGES_OWNER}/${PAGES_NAME}/commit/${RUN_SHA}) (B)
<details open>
<summary>Summary of all runs</summary>
<pre>
@ -106,7 +113,11 @@ $(cat $TMP_DIR/summary.txt)
</pre>
</details>
Detailed report: [${NEW_ID}](https://${PAGES_OWNER}.github.io/${PAGES_NAME}/rtlmeter-reports/${NEW_ID}/index.html)
The reported numbers are the geometric means of the ratios of the metrics over all cases,
less than 1 is a regression, greater than 1 is an improvement.
The jobs run on variable and noisy runners, so some variance is expected between runs.
Detailed report: [${RUN_ID}](https://${PAGES_OWNER}.github.io/${PAGES_NAME}/rtlmeter-reports/${RUN_ID}/index.html)
NOTIFICATION_TEMPLATE
# Create detailed report
@ -119,7 +130,7 @@ $(cat $TMP_DIR/summary.txt)
SUMMARY_TEMPLATE
echo "<h3>Detailed results</h3>" >> $REPORT
for r in $RUNS; do
RUN_NAME=$(jq -rj ".[0].runName" $REF_DIR/all-results-$r.json)
RUN_NAME=$(jq -rj ".[0].runName" $NEW_DIR/all-results-$r.json)
echo "<h4>$RUN_NAME</h4>" >> $REPORT
for f in $(ls -1 $TMP_DIR/$r-frag-verilate-*.txt | sort) \
$(ls -1 $TMP_DIR/$r-frag-cppbuild-*.txt | sort) \
@ -141,7 +152,7 @@ cat > ${TMP_DIR}/report/index.html <<INDEX_TEMPLATE
<html>
<head>
<title>Verilator RTLMeter report #${NEW_NUM}</title>
<title>Verilator RTLMeter report #${RUN_NUM}</title>
<style>
body {
font-family: courier, serif;