' >> report.txt
- echo "" >> report.txt
- done
- echo "" >> report.txt
- done
- cat report.txt
+ ln -s ../rtlmeter rtlmeter
+ gh repo set-default ${{ github.repository }}
+ # Compare to last successful scheduled run
+ REF_ID=$(gh run list --workflow RTLMeter --event schedule --status success --limit 1 --json databaseId --jq ".[0].databaseId")
+ NEW_ID=${{ github.run_id }}
+ ci/ci-rtlmeter-pr-report.bash $REF_ID $NEW_ID gcc clang gcc-hier
+ mv rtlmeter-pr-report/report.txt ../
+ cat ../report.txt
+ # Generate notification comment content
+ mkdir -p ../notification
+ mv rtlmeter-pr-report/notification.txt ../notification/body.txt
+ echo ${{ github.event.number }} > ../notification/pr-number.txt
+
- name: Upload report
uses: actions/upload-artifact@v7
with:
path: report.txt
- name: rtlmeter-pr-results
- - name: Save PR number
- run: echo ${{ github.event.number }} > pr-number.txt
- - name: Upload PR number
+ name: rtlmeter-pr-report
+
+ - name: Upload notification
uses: actions/upload-artifact@v7
with:
- path: pr-number.txt
- name: pr-number
+ path: notification
+ name: pr-notification
# Create GitHub issue for failed scheduled jobs
# This should always be the last job (we want an issue if anything breaks)
diff --git a/Makefile.in b/Makefile.in
index 9d92056ae..c2b60d004 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -535,6 +535,7 @@ PY_PROGRAMS = \
# Python files, subject to format but not lint
PY_FILES = \
$(PY_PROGRAMS) \
+ ci/*.py \
test_regress/t/*.py \
# Python files, test_regress tests
diff --git a/ci/ci-pages-notify.bash b/ci/ci-pages-notify.bash
index 05bb92fe6..e5dd599b6 100755
--- a/ci/ci-pages-notify.bash
+++ b/ci/ci-pages-notify.bash
@@ -4,7 +4,7 @@
# SPDX-FileCopyrightText: 2025 Geza Lore
# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
-# Notify PRs via comment that their coverage reports are available
+# Notify PRs via comment that their workflow reports are available
# Get the current repo URL - might differ on a fork
readonly REPO_URL=$(gh repo view --json url --jq .url)
@@ -13,7 +13,7 @@ readonly REPO_URL=$(gh repo view --json url --jq .url)
ARTIFACTS_ROOT=artifacts
mkdir -p ${ARTIFACTS_ROOT}
-for RUN_ID in ${COVERAGE_PR_RUN_IDS//,/ }; do
+for RUN_ID in ${PR_RUN_IDS//,/ }; do
echo "@@@ Processing run ${RUN_ID}"
# Create workflow artifacts directory
@@ -21,7 +21,7 @@ for RUN_ID in ${COVERAGE_PR_RUN_IDS//,/ }; do
mkdir -p ${ARTIFACTS_DIR}
# Download artifact of this run, if exists
- gh run download ${RUN_ID} --name coverage-pr-notification --dir ${ARTIFACTS_DIR} || true
+ gh run download ${RUN_ID} --name pr-notification --dir ${ARTIFACTS_DIR} || true
ls -lsha ${ARTIFACTS_DIR}
# Move on if no notification is required
@@ -35,7 +35,7 @@ for RUN_ID in ${COVERAGE_PR_RUN_IDS//,/ }; do
gh pr comment $(cat ${ARTIFACTS_DIR}/pr-number.txt) --body-file ${ARTIFACTS_DIR}/body.txt
# Get the artifact ID
- ARTIFACT_ID=$(gh api "repos/{owner}/{repo}/actions/runs/${RUN_ID}/artifacts" --jq '.artifacts[] | select(.name == "coverage-pr-notification") | .id')
+ ARTIFACT_ID=$(gh api "repos/{owner}/{repo}/actions/runs/${RUN_ID}/artifacts" --jq '.artifacts[] | select(.name == "pr-notification") | .id')
# Delete it, so we only notify once
gh api --method DELETE "repos/{owner}/{repo}/actions/artifacts/${ARTIFACT_ID}"
diff --git a/ci/ci-pages.bash b/ci/ci-pages.bash
index ba9013621..44dc3f6d1 100755
--- a/ci/ci-pages.bash
+++ b/ci/ci-pages.bash
@@ -5,8 +5,7 @@
# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
# This scipt build the content of the GitHub Pages for the repository.
-# Currently this only hosts code coverage reports, but it would be possible to
-# add any other contents to the page in parallel here.
+# Currently this hosts code coverage reports, and PR RTLMeter detailed reports.
# Developer note: You should be able to run this script in your local checkout
# if you have GitHub CLI (command 'gh') setup, authenticated ('gh auth login'),
@@ -25,6 +24,9 @@ if [[ -z "$GITHUB_OUTPUT" ]]; then
GITHUB_OUTPUT=github-output.txt
fi
+# Run IDs of PR jobs processed
+PR_RUN_IDS=""
+
# Populates ${PAGES_ROOT}/coverage-reports
compile_coverage_reports() {
# We will process all runs up to and including this date. This is chosen to be
@@ -53,9 +55,6 @@ compile_coverage_reports() {
local CONTENTS=contents.tmp
echo > ${CONTENTS}
- # Run IDs of PR jobs processed
- local PR_RUN_IDS=""
-
# Iterate over all unique event types that triggered the workflows
for EVENT in $(jq -r 'map(.event) | sort | unique | .[]' completedRuns.json); do
echo "@@@ Processing '${EVENT}' runs"
@@ -73,11 +72,7 @@ compile_coverage_reports() {
# Record run ID of PR job
if [[ $EVENT == "pull_request" ]]; then
- if [[ -z "$PR_RUN_IDS" ]]; then
- PR_RUN_IDS="$RUN_ID"
- else
- PR_RUN_IDS="$PR_RUN_IDS,$RUN_ID"
- fi
+ PR_RUN_IDS="$PR_RUN_IDS $RUN_ID"
fi
# Create workflow artifacts directory
@@ -148,7 +143,7 @@ CONTENTS_TEMPLATE
$(cat ${CONTENTS})
-
Assembled $(date --iso-8601=minutes --utc)
+
Assembled $(date --iso-8601=minutes --utc)
@@ -156,12 +151,159 @@ INDEX_TEMPLATE
# Report size
du -shc ${COVERAGE_ROOT}/*
+}
- # Set output
- echo "coverage-pr-run-ids=${PR_RUN_IDS}" >> $GITHUB_OUTPUT
+# Populates ${PAGES_ROOT}/rtlmeter-reports
+compile_rtlmeter_reports() {
+ # We will process all runs up to and including this date. This is chosen to be
+ # slightly less than the artifact retention period for simplicity.
+ local OLDEST=$(date --date="28 days ago" --iso-8601=date)
+
+ # Gather all RTLMeter workflow runs within the time window
+ gh run list -w rtlmeter.yml --limit 1000 --created ">=${OLDEST}" --json "databaseId,event,status,conclusion,createdAt,number" > recentRuns.json
+ echo @@@ Recent runs:
+ jq "." recentRuns.json
+
+ # Select completd runs that were not cancelled or skipped, sort by descending run number
+ jq 'sort_by(-.number) | map(select(.status == "completed" and (.conclusion == "success" or .conclusion == "failure")))' recentRuns.json > completedRuns.json
+ echo @@@ Completed with success or failure:
+ jq "." completedRuns.json
+
+ # Create artifacts root directory
+ local ARTIFACTS_ROOT=artifacts
+ mkdir -p ${ARTIFACTS_ROOT}
+
+ # Create rtlmeter reports root directory
+ local RTLMETER_ROOT=${PAGES_ROOT}/rtlmeter-reports
+ mkdir -p ${RTLMETER_ROOT}
+
+ # Create index page contents fragment
+ local CONTENTS=contents.tmp
+ echo > ${CONTENTS}
+
+ # Iterate over all unique event types that triggered the workflows
+ for EVENT in $(jq -r 'map(.event) | sort | unique | .[]' completedRuns.json); do
+ echo "@@@ Processing '${EVENT}' runs"
+
+ # Emit section header if a report exists with this event type
+ EMIT_SECTION_HEADER=1
+
+ # For each worfklow run that was triggered by this event type
+ for RUN_ID in $(jq ".[] | select(.event == \"${EVENT}\") |.databaseId" completedRuns.json); do
+ echo "@@@ Processing run ${RUN_ID}"
+
+ # Extract the info of this run
+ jq ".[] | select(.databaseId == $RUN_ID)" completedRuns.json > workflow.json
+ jq "." workflow.json
+
+ # Record run ID of PR job
+ if [[ $EVENT == "pull_request" ]]; then
+ PR_RUN_IDS="$PR_RUN_IDS $RUN_ID"
+ fi
+
+ # Create workflow artifacts directory
+ local ARTIFACTS_DIR=${ARTIFACTS_ROOT}/${RUN_ID}
+ mkdir -p ${ARTIFACTS_DIR}
+
+ # Download artifacts of this run, if exists
+ gh run download ${RUN_ID} --name rtlmeter-pr-report --dir ${ARTIFACTS_DIR} || true
+ ls -lsha ${ARTIFACTS_DIR}
+
+ # Move on if no RTLMeter report is available
+ if [ ! -f ${ARTIFACTS_DIR}/report.txt ]; then
+ echo "No RTLMeter report found"
+ continue
+ fi
+ echo "RTLMeter report found"
+
+ # Emit section header
+ if [[ -n $EMIT_SECTION_HEADER ]]; then
+ unset EMIT_SECTION_HEADER
+ echo "
RTLMeter reports for '${EVENT}' runs:
" >> ${CONTENTS}
+ fi
+
+ # Extract run metadata
+ local WORKFLOW_CREATED=$(jq -r '.createdAt' workflow.json)
+ local WOFKRLOW_NUMBER=$(jq -r '.number' workflow.json)
+
+ # Wrap the report into an HTML page. The report content is already HTML
+ # (produced by ci-rtlmeter-pr-report.bash), so we just embed it in the
+ # page body.
+ cat > ${RTLMETER_ROOT}/${RUN_ID}.html <
+
+
+ Verilator RTLMeter report #${WOFKRLOW_NUMBER}
+
+
+
+
+$(cat ${ARTIFACTS_DIR}/report.txt)
+
+
+
+REPORT_TEMPLATE
+
+ # Add index page content
+ cat >> ${CONTENTS} <#${WOFKRLOW_NUMBER}
+ | GitHub: ${RUN_ID}
+ | started at: ${WORKFLOW_CREATED}
+CONTENTS_TEMPLATE
+ echo " " >> ${CONTENTS}
+ done
+
+ # Section break
+ if [[ -z "$EMIT_SECTION_HEADER" ]]; then
+ echo "" >> ${CONTENTS}
+ fi
+ done
+
+ # Write rtlmeter report index.html
+ cat > ${RTLMETER_ROOT}/index.html <
+
+
+ Verilator CI RTLMeter reports
+
+
+
+
+ $(cat ${CONTENTS})
+
Assembled $(date --iso-8601=minutes --utc)
+
+
+
+INDEX_TEMPLATE
+
+ # Report size
+ du -shc ${RTLMETER_ROOT}/*
}
# Compilie coverage reports
compile_coverage_reports;
+# Compile RTLMeter reports
+compile_rtlmeter_reports;
+
# You can build any other content here to be put under ${PAGES_ROOT}
+
+# Set output to all affected unique PR run IDs (comma separated, no trailing comma)
+IDS=$(echo ${PR_RUN_IDS} | xargs -n1 | sort -u | paste -sd,)
+echo "pr-run-ids=${IDS}" >> $GITHUB_OUTPUT
diff --git a/ci/ci-rtlmeter-pr-report.bash b/ci/ci-rtlmeter-pr-report.bash
new file mode 100755
index 000000000..f44f074e2
--- /dev/null
+++ b/ci/ci-rtlmeter-pr-report.bash
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+# DESCRIPTION: Verilator: CI script for 'rtlmeter.yml' PR results
+#
+# SPDX-FileCopyrightText: 2026 Wilson Snyder
+# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
+
+# This scipt builds the content of the response comment posten on PRs
+# at the end of RTLMeter runs.
+
+# Developer note: You should be able to run this script in your local checkout
+# if you have GitHub CLI (command 'gh') setup, authenticated ('gh auth login'),
+# and have set a default repository ('gh repo set-default').
+
+# Trace when running in the CI
+[ "$GITHUB_ACTIONS" != "true" ] || set -x
+
+# Arguments:
+# 1. reference run ID
+# 2. new run ID
+# rest: run tags
+REF_ID=$1; shift
+NEW_ID=$1; shift
+RUNS="$@"
+
+# $VERILATOR_CHECKOUT/ci directory
+SCRIPT_DIR=$(readlink -f $(dirname ${BASH_SOURCE[0]}))
+
+# Move into a temporary directory
+rm -rf rtlmeter-pr-report
+mkdir rtlmeter-pr-report
+pushd rtlmeter-pr-report &> /dev/null
+TMP_DIR=$(readlink -f .)
+
+# Artifacts to download
+DOWNLOAD_ARTIFACTS=""
+for r in $RUNS; do
+ DOWNLOAD_ARTIFACTS="$DOWNLOAD_ARTIFACTS --name all-results-$r"
+done
+
+# Download reference run results
+mkdir ref
+REF_DIR=$(readlink -f ref)
+gh run download ${REF_ID} $DOWNLOAD_ARTIFACTS --dir $REF_DIR
+mv $REF_DIR/*/*.json $REF_DIR/
+find $REF_DIR -mindepth 1 -type d -delete
+
+# Download current run results
+mkdir new
+NEW_DIR=$(readlink -f new)
+gh run download ${NEW_ID} $DOWNLOAD_ARTIFACTS --dir $NEW_DIR
+mv $NEW_DIR/*/*.json $NEW_DIR/
+find $NEW_DIR -mindepth 1 -type d -delete
+
+# Get Some metadata about the runs
+REF_URL=$(gh run view $REF_ID --json url --jq ".url")
+REF_NUM=$(gh run view $REF_ID --json number --jq ".number")
+REF_DATE=$(gh run view $REF_ID --json createdAt --jq ".createdAt")
+NEW_URL=$(gh run view $NEW_ID --json url --jq ".url")
+NEW_NUM=$(gh run view $NEW_ID --json number --jq ".number")
+
+# Go back to root directory
+popd &> /dev/null
+# Go to RTLMeter directory
+cd rtlmeter
+
+# Compare results
+SUMMARY_ARGS=()
+for r in $RUNS; do
+ CMP_JSON=$TMP_DIR/cmp-$r.json
+ # Gather args for summary script
+ SUMMARY_ARGS+=($REF_DIR/all-results-$r.json $CMP_JSON)
+ # Create JSON
+ ./rtlmeter compare --format json --steps "*" --metrics "*" \
+ $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $CMP_JSON
+ # Also create detailed tables
+ ./rtlmeter compare --format ascii --steps 'verilate' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/verilate-$r.txt
+ ./rtlmeter compare --format ascii --steps 'cppbuild' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/cppbuild-$r.txt
+ ./rtlmeter compare --format ascii --steps 'execute' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/execute-$r.txt
+ # Chop them at new lines, into one table per file
+ awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-verilate-%02d.txt",prefix,NR)}' $TMP_DIR/verilate-$r.txt
+ awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-cppbuild-%02d.txt",prefix,NR)}' $TMP_DIR/cppbuild-$r.txt
+ awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-execute-%02d.txt" ,prefix,NR)}' $TMP_DIR/execute-$r.txt
+done
+
+# Create summary
+venv/bin/python3 $SCRIPT_DIR/ci-rtlmeter-pr-report.py ${SUMMARY_ARGS[@]} > $TMP_DIR/summary.txt
+# Print it
+cat $TMP_DIR/summary.txt
+
+# Create notification comment content
+NOTIFICATION=$TMP_DIR/notification.txt
+cat > $NOTIFICATION <
+ Summary of all runs
+
+$(cat $TMP_DIR/summary.txt)
+
+
+Blah Blah Blah
+NOTIFICATION_TEMPLATE
+
+# Create detailed report
+REPORT=$TMP_DIR/report.txt
+cat > $REPORT <
+ Summary of all runs
+