173 lines
6.0 KiB
Bash
Executable File
173 lines
6.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# DESCRIPTION: Verilator: CI script for 'rtlmeter.yml' PR results
|
|
#
|
|
# SPDX-FileCopyrightText: 2026 Wilson Snyder
|
|
# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
|
|
|
# This scipt builds the content of the response comment posten on PRs
|
|
# at the end of RTLMeter runs.
|
|
|
|
# Developer note: You should be able to run this script in your local checkout
|
|
# if you have GitHub CLI (command 'gh') setup, authenticated ('gh auth login'),
|
|
# and have set a default repository ('gh repo set-default').
|
|
|
|
# Trace when running in the CI
|
|
[ "$GITHUB_ACTIONS" != "true" ] || set -x
|
|
|
|
# Arguments:
|
|
# 1. run ID
|
|
# 2. SHA of the event that triggered the run (for a PR, the test merge commit)
|
|
# rest: run tags
|
|
RUN_ID=$1; shift
|
|
RUN_SHA=$1; shift
|
|
RUNS="$@"
|
|
|
|
# $VERILATOR_CHECKOUT/ci directory
|
|
SCRIPT_DIR=$(readlink -f $(dirname ${BASH_SOURCE[0]}))
|
|
|
|
# Move into a temporary directory
|
|
rm -rf rtlmeter-report
|
|
mkdir rtlmeter-report
|
|
pushd rtlmeter-report &> /dev/null
|
|
TMP_DIR=$(readlink -f .)
|
|
|
|
# Artifacts to download
|
|
DOWNLOAD_NEW_ARTIFACTS=""
|
|
DOWNLOAD_REF_ARTIFACTS=""
|
|
for r in $RUNS; do
|
|
DOWNLOAD_NEW_ARTIFACTS="$DOWNLOAD_NEW_ARTIFACTS --name all-results-$r"
|
|
DOWNLOAD_REF_ARTIFACTS="$DOWNLOAD_REF_ARTIFACTS --name all-reference-$r"
|
|
done
|
|
|
|
# Download reference artifacts
|
|
mkdir ref
|
|
REF_DIR=$(readlink -f ref)
|
|
gh run download ${RUN_ID} $DOWNLOAD_REF_ARTIFACTS --dir $REF_DIR
|
|
mv $REF_DIR/*/*.json $REF_DIR/
|
|
find $REF_DIR -mindepth 1 -type d -delete
|
|
|
|
# Download new version artifacts
|
|
mkdir new
|
|
NEW_DIR=$(readlink -f new)
|
|
gh run download ${RUN_ID} $DOWNLOAD_NEW_ARTIFACTS --dir $NEW_DIR
|
|
mv $NEW_DIR/*/*.json $NEW_DIR/
|
|
find $NEW_DIR -mindepth 1 -type d -delete
|
|
|
|
# Get Some metadata about the runs
|
|
RUN_URL=$(gh run view $RUN_ID --json url --jq ".url")
|
|
RUN_NUM=$(gh run view $RUN_ID --json number --jq ".number")
|
|
|
|
# Repository owner and name of the default repository, used to build the
|
|
# GitHub Pages URL of the detailed report. The owner is lowercased, as required
|
|
# for the '<owner>.github.io' Pages domain. Resolved here, while still in the
|
|
# default repository's git context (before the 'cd rtlmeter' below).
|
|
PAGES_OWNER=$(gh repo view --json owner --jq '.owner.login' | tr '[:upper:]' '[:lower:]')
|
|
PAGES_NAME=$(gh repo view --json name --jq '.name')
|
|
|
|
# The 'old' reference was built from the target branch base commit, which is
|
|
# the first parent of the triggering merge commit (same as the 'start' job in
|
|
# rtlmeter.yml). Resolved here, while still in the default repository's git
|
|
# context (before the 'cd rtlmeter' below).
|
|
REF_SHA=$(gh api "repos/{owner}/{repo}/commits/$RUN_SHA" --jq '.parents[0].sha')
|
|
|
|
# Go back to root directory
|
|
popd &> /dev/null
|
|
# Go to RTLMeter directory
|
|
cd rtlmeter
|
|
|
|
# Compare results
|
|
SUMMARY_ARGS=()
|
|
for r in $RUNS; do
|
|
CMP_JSON=$TMP_DIR/cmp-$r.json
|
|
# Gather args for summary script
|
|
SUMMARY_ARGS+=($NEW_DIR/all-results-$r.json $CMP_JSON)
|
|
# Create JSON
|
|
./rtlmeter compare --format json --steps "*" --metrics "*" \
|
|
$REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $CMP_JSON
|
|
# Also create detailed tables
|
|
./rtlmeter compare --format ascii --steps 'verilate' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/verilate-$r.txt
|
|
./rtlmeter compare --format ascii --steps 'cppbuild' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/cppbuild-$r.txt
|
|
./rtlmeter compare --format ascii --steps 'execute' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/execute-$r.txt
|
|
# Chop them at new lines, into one table per file
|
|
awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-verilate-%02d.txt",prefix,NR)}' $TMP_DIR/verilate-$r.txt
|
|
awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-cppbuild-%02d.txt",prefix,NR)}' $TMP_DIR/cppbuild-$r.txt
|
|
awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-execute-%02d.txt" ,prefix,NR)}' $TMP_DIR/execute-$r.txt
|
|
done
|
|
|
|
# Create summary
|
|
venv/bin/python3 $SCRIPT_DIR/ci-rtlmeter-report.py ${SUMMARY_ARGS[@]} > $TMP_DIR/summary.txt
|
|
# Print it
|
|
cat $TMP_DIR/summary.txt
|
|
|
|
# Create notification comment content
|
|
NOTIFICATION=$TMP_DIR/notification.txt
|
|
cat > $NOTIFICATION <<NOTIFICATION_TEMPLATE
|
|
Performance metrics for PR workflow [#$RUN_NUM]($RUN_URL), comparing:
|
|
- target branch commit [${REF_SHA:0:9}](https://github.com/${PAGES_OWNER}/${PAGES_NAME}/commit/${REF_SHA}) (A)
|
|
- with PR merge commit [${RUN_SHA:0:9}](https://github.com/${PAGES_OWNER}/${PAGES_NAME}/commit/${RUN_SHA}) (B)
|
|
|
|
<details open>
|
|
<summary>Summary of all runs</summary>
|
|
<pre>
|
|
$(cat $TMP_DIR/summary.txt)
|
|
</pre>
|
|
</details>
|
|
|
|
The reported numbers are the geometric means of the ratios of the metrics over all cases,
|
|
less than 1 is a regression, greater than 1 is an improvement.
|
|
The jobs run on variable and noisy runners, so some variance is expected between runs.
|
|
|
|
Detailed report: [${RUN_ID}](https://${PAGES_OWNER}.github.io/${PAGES_NAME}/rtlmeter-reports/${RUN_ID}/index.html)
|
|
NOTIFICATION_TEMPLATE
|
|
|
|
# Create detailed report
|
|
REPORT=$TMP_DIR/body.html
|
|
cat > $REPORT <<SUMMARY_TEMPLATE
|
|
<h3>Summary of all runs</h3>
|
|
<pre>
|
|
$(cat $TMP_DIR/summary.txt)
|
|
</pre>
|
|
SUMMARY_TEMPLATE
|
|
echo "<h3>Detailed results</h3>" >> $REPORT
|
|
for r in $RUNS; do
|
|
RUN_NAME=$(jq -rj ".[0].runName" $NEW_DIR/all-results-$r.json)
|
|
echo "<h4>$RUN_NAME</h4>" >> $REPORT
|
|
for f in $(ls -1 $TMP_DIR/$r-frag-verilate-*.txt | sort) \
|
|
$(ls -1 $TMP_DIR/$r-frag-cppbuild-*.txt | sort) \
|
|
$(ls -1 $TMP_DIR/$r-frag-execute-*.txt | sort); do
|
|
cat >> $REPORT <<DETAIL_TAMPLATE
|
|
<details>
|
|
<summary>$(head -n 1 $f | tr -d '\n')</summary>
|
|
<pre>
|
|
$(tail -n +2 $f)
|
|
</pre>
|
|
</details>
|
|
DETAIL_TAMPLATE
|
|
done
|
|
done
|
|
|
|
# Turn the report into a proper HTML page
|
|
mkdir -p ${TMP_DIR}/report
|
|
cat > ${TMP_DIR}/report/index.html <<INDEX_TEMPLATE
|
|
<html>
|
|
|
|
<head>
|
|
<title>Verilator RTLMeter report #${RUN_NUM}</title>
|
|
<style>
|
|
body {
|
|
font-family: courier, serif;
|
|
background-color: #f3f3f3;
|
|
a {
|
|
color: #008fd7;
|
|
}
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<body>
|
|
$(cat ${TMP_DIR}/body.html)
|
|
</body>
|
|
|
|
</html>
|
|
INDEX_TEMPLATE
|