From 83ef69d86621d3ea42896b4ea1bf205bbbb806d8 Mon Sep 17 00:00:00 2001 From: Geza Lore Date: Sun, 7 Jun 2026 15:36:34 +0100 Subject: [PATCH] CI: Run RTLMeter on the same machine with target commit (#7726) --- .github/workflows/reusable-rtlmeter-build.yml | 31 ++++- .github/workflows/reusable-rtlmeter-run.yml | 124 ++++++++++++++---- .github/workflows/rtlmeter.yml | 109 +++++++++++++-- ci/ci-rtlmeter-report.bash | 61 +++++---- 4 files changed, 256 insertions(+), 69 deletions(-) diff --git a/.github/workflows/reusable-rtlmeter-build.yml b/.github/workflows/reusable-rtlmeter-build.yml index 85deb9706..321033a72 100644 --- a/.github/workflows/reusable-rtlmeter-build.yml +++ b/.github/workflows/reusable-rtlmeter-build.yml @@ -15,6 +15,14 @@ on: description: "Compiler to use: 'gcc' or 'clang'" type: string required: true + sha: + description: "Git SHA to build" + type: string + required: true + outputs: + archive: + description: "Name of the built installation archive artifact" + value: ${{ jobs.build.outputs.archive }} defaults: run: @@ -26,8 +34,10 @@ env: jobs: build: - runs-on: ${{ inputs.runs-on }} name: Build + runs-on: ${{ inputs.runs-on }} + outputs: + archive: ${{ steps.create-archive.outputs.archive }} steps: - name: Install dependencies run: | @@ -43,13 +53,17 @@ jobs: uses: actions/cache@v5 with: path: ccache - key: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }} + key: rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }}-${{ github.run_id }} + rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.sha }} + rtlmeter-build-ccache-${{ inputs.runs-on }}-${{ inputs.cc }} - name: Checkout uses: actions/checkout@v6 with: path: repo + ref: ${{ inputs.sha }} fetch-depth: 0 # Required for 'git describe' used for 'verilator --version' - name: Configure @@ -67,11 +81,16 @@ jobs: run: make install - name: Tar up installation - run: tar --posix -c -z -f verilator-rtlmeter.tar.gz install + id: create-archive + run: | + SHA=$(git -C repo rev-parse HEAD) + ARCHIVE=verilator-$SHA-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }}.tar.gz + tar --posix -c -z -f $ARCHIVE install + echo "archive=$ARCHIVE" >> $GITHUB_OUTPUT - name: Upload Verilator installation archive uses: actions/upload-artifact@v7 with: - path: verilator-rtlmeter.tar.gz - name: verilator-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }} + path: ${{ steps.create-archive.outputs.archive }} + name: ${{ steps.create-archive.outputs.archive }} overwrite: true diff --git a/.github/workflows/reusable-rtlmeter-run.yml b/.github/workflows/reusable-rtlmeter-run.yml index 5e163f555..ebbaffef8 100644 --- a/.github/workflows/reusable-rtlmeter-run.yml +++ b/.github/workflows/reusable-rtlmeter-run.yml @@ -19,6 +19,15 @@ on: description: "Compiler to use: 'gcc' or 'clang'" type: string required: true + verilator-archive-new: + description: "Name of the installation archive artifact from reusable-rtlmeter-build, new version" + type: string + required: true + verilator-archive-old: + description: "Name of the installation archive artifact from reusable-rtlmeter-build, old version" + type: string + required: false + default: "" # Note: The combination of 'cases' and 'run-name' must be unique for all # invocations of this workflow within a run of the parent workflow. # These two are used together to generate a unique results file name. @@ -63,24 +72,6 @@ jobs: sudo apt install ccache mold libfl-dev libjemalloc-dev libsystemc-dev || \ sudo apt install ccache mold libfl-dev libjemalloc-dev libsystemc-dev - - name: Download Verilator installation archive - uses: actions/download-artifact@v8 - with: - name: verilator-rtlmeter-${{ inputs.runs-on }}-${{ inputs.cc }} - - - name: Unpack Verilator installation archive - run: | - tar -x -z -f verilator-rtlmeter.tar.gz - echo "${{ github.workspace }}/install/bin" >> $GITHUB_PATH - - - name: Use saved ccache - if: ${{ env.CCACHE_DISABLE == 0 }} - uses: actions/cache@v5 - with: - path: ${{ env.CCACHE_DIR }} - key: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }} - - name: Checkout RTLMeter uses: actions/checkout@v6 with: @@ -91,32 +82,59 @@ jobs: working-directory: rtlmeter run: make venv - - name: Compile cases + - name: Use saved ccache + if: ${{ env.CCACHE_DISABLE == 0 }} + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: rtlmeter-run-ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.cases }}-${{ inputs.compileArgs }} + + ######################################################################## + # Run with new Verilator + ######################################################################## + + - name: Download Verilator installation archive - new + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.verilator-archive-new }} + + - name: Unpack Verilator installation archive - new + run: | + tar -x -z -f ${{ inputs.verilator-archive-new }} + mv install verilator-new + + - name: Compile cases - new working-directory: rtlmeter run: | + export PATH="${{ github.workspace }}/verilator-new/bin:$PATH" ./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' --nExecute=0 + ccache -svv - - name: Execute cases + - name: Execute cases - new working-directory: rtlmeter continue-on-error: true # Do not fail on error, so we can at least save the successful results run: | + export PATH="${{ github.workspace }}/verilator-new/bin:$PATH" ./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' - - name: Collate results + - name: Collate results - new id: results working-directory: rtlmeter run: | + export PATH="${{ github.workspace }}/verilator-new/bin:$PATH" # Use 'inputs.cases' and 'inputs.run-name' to generate a unique file name hash=$(md5sum <<< '${{ inputs.cases }} ${{ inputs.run-name }}' | awk '{print $1}') echo "hash=${hash}" >> $GITHUB_OUTPUT ./rtlmeter collate --runName "${{ inputs.run-name }}" > ../results-${hash}.json - - name: Report results + - name: Report results - new working-directory: rtlmeter run: | + export PATH="${{ github.workspace }}/verilator-new/bin:$PATH" ./rtlmeter report --steps '*' --metrics '*' ../results-${{ steps.results.outputs.hash }}.json - - name: Upload results + - name: Upload results - new uses: actions/upload-artifact@v7 with: path: results-${{ steps.results.outputs.hash }}.json @@ -124,7 +142,65 @@ jobs: overwrite: true retention-days: 2 - - name: Report status + - name: Report status - new working-directory: rtlmeter run: |- # This will fail the job if any of the runs failed + export PATH="${{ github.workspace }}/verilator-new/bin:$PATH" ./rtlmeter run --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' + # Clean up for run with old version + rm -rf work + rm -rf ${{ github.workspace }}/verilator-new + + ######################################################################## + # Run with old Verilator **on same machine for consistency** + ######################################################################## + + - name: Download Verilator installation archive - old + if: ${{ inputs.verilator-archive-old != '' }} + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.verilator-archive-old }} + + - name: Unpack Verilator installation archive - old + if: ${{ inputs.verilator-archive-old != '' }} + run: | + tar -x -z -f ${{ inputs.verilator-archive-old }} + mv install verilator-old + + - name: Compile cases - old + if: ${{ inputs.verilator-archive-old != '' }} + working-directory: rtlmeter + run: | + export PATH="${{ github.workspace }}/verilator-old/bin:$PATH" + ./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' --nExecute=0 + ccache -svv + + - name: Execute cases - old + if: ${{ inputs.verilator-archive-old != '' }} + working-directory: rtlmeter + run: | + export PATH="${{ github.workspace }}/verilator-old/bin:$PATH" + ./rtlmeter run --timeout 60 --verbose --cases='${{inputs.cases}}' --compileArgs='${{inputs.compileArgs}}' --executeArgs='${{inputs.executeArgs}}' + + - name: Collate results - old + if: ${{ inputs.verilator-archive-old != '' }} + working-directory: rtlmeter + run: | + export PATH="${{ github.workspace }}/verilator-old/bin:$PATH" + ./rtlmeter collate --runName "${{ inputs.run-name }}" > ../reference-${{ steps.results.outputs.hash }}.json + + - name: Report results - old + if: ${{ inputs.verilator-archive-old != '' }} + working-directory: rtlmeter + run: | + export PATH="${{ github.workspace }}/verilator-old/bin:$PATH" + ./rtlmeter report --steps '*' --metrics '*' ../reference-${{ steps.results.outputs.hash }}.json + + - name: Upload results - old + if: ${{ inputs.verilator-archive-old != '' }} + uses: actions/upload-artifact@v7 + with: + path: reference-${{ steps.results.outputs.hash }}.json + name: rtlmeter-${{ inputs.tag }}-reference-${{ steps.results.outputs.hash }} + overwrite: true + retention-days: 2 diff --git a/.github/workflows/rtlmeter.yml b/.github/workflows/rtlmeter.yml index 3c633d9f0..e5d632b9f 100644 --- a/.github/workflows/rtlmeter.yml +++ b/.github/workflows/rtlmeter.yml @@ -40,34 +40,73 @@ jobs: (github.event_name == 'workflow_dispatch') || (github.event_name == 'push') runs-on: ubuntu-24.04 + outputs: + old-sha: ${{ steps.start.outputs.old-sha }} steps: - name: Startup - run: echo + id: start + env: + GH_TOKEN: ${{ github.token }} + run: | + [[ "${{ github.event_name }}" == 'pull_request' ]] || exit 0 + # For a pull request, 'github.sha' is the test merge commit. Its + # first parent is the target branch commit it was merged with for this run. + OLD_SHA="$(gh api "repos/${{ github.repository }}/commits/${{ github.sha }}" --jq '.parents[0].sha')" + echo "old-sha=$OLD_SHA" >> "$GITHUB_OUTPUT" - build-gcc: - name: Build GCC + build-gcc-new: + name: Build New Verilator - GCC needs: start uses: ./.github/workflows/reusable-rtlmeter-build.yml with: runs-on: ubuntu-24.04 cc: gcc + sha: ${{ github.sha }} - build-clang: - name: Build Clang + build-clang-new: + name: Build New Verilator - Clang needs: start uses: ./.github/workflows/reusable-rtlmeter-build.yml with: runs-on: ubuntu-24.04 cc: clang + sha: ${{ github.sha }} + + build-gcc-old: + name: Build Old Verilator - GCC + needs: start + if: ${{ needs.start.outputs.old-sha != '' }} + uses: ./.github/workflows/reusable-rtlmeter-build.yml + with: + runs-on: ubuntu-24.04 + cc: gcc + sha: ${{ needs.start.outputs.old-sha }} + + build-clang-old: + name: Build Old Verilator - Clang + needs: start + if: ${{ needs.start.outputs.old-sha != '' }} + uses: ./.github/workflows/reusable-rtlmeter-build.yml + with: + runs-on: ubuntu-24.04 + cc: clang + sha: ${{ needs.start.outputs.old-sha }} run-gcc: name: Run GCC | ${{ matrix.cases }} - needs: build-gcc + needs: + - build-gcc-new + - build-gcc-old + # Run even if 'build-*-old' was skipped (no old SHA), but not if any + # dependency failed or the workflow was cancelled. + if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable-rtlmeter-run.yml with: tag: gcc runs-on: ubuntu-24.04 cc: gcc + verilator-archive-new: ${{ needs.build-gcc-new.outputs.archive }} + verilator-archive-old: ${{ needs.build-gcc-old.outputs.archive }} cases: ${{ matrix.cases }} run-name: "gcc" compileArgs: "" @@ -108,12 +147,19 @@ jobs: run-clang: name: Run Clang | ${{ matrix.cases }} - needs: build-clang + needs: + - build-clang-new + - build-clang-old + # Run even if 'build-*-old' was skipped (no old SHA), but not if any + # dependency failed or the workflow was cancelled. + if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable-rtlmeter-run.yml with: tag: clang runs-on: ubuntu-24.04 cc: clang + verilator-archive-new: ${{ needs.build-clang-new.outputs.archive }} + verilator-archive-old: ${{ needs.build-clang-old.outputs.archive }} cases: ${{ matrix.cases }} run-name: "clang --threads 4" compileArgs: "--threads 4" @@ -154,12 +200,19 @@ jobs: run-gcc-hier: name: Run GCC hier | ${{ matrix.cases }} - needs: build-gcc + needs: + - build-gcc-new + - build-gcc-old + # Run even if 'build-*-old' was skipped (no old SHA), but not if any + # dependency failed or the workflow was cancelled. + if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable-rtlmeter-run.yml with: tag: gcc-hier runs-on: ubuntu-24.04 cc: gcc + verilator-archive-new: ${{ needs.build-gcc-new.outputs.archive }} + verilator-archive-old: ${{ needs.build-gcc-old.outputs.archive }} cases: ${{ matrix.cases }} run-name: "gcc --hierarchical" compileArgs: "--hierarchical" @@ -181,9 +234,14 @@ jobs: combine-results: name: Combine results needs: [run-gcc, run-clang, run-gcc-hier] - # Run if any of the dependencies have run, even if failed. - # That is: do not run if all skipped, or the workflow was cancelled. - if: ${{ (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure')) && !cancelled() }} + # Skip if cancelled. + # On PRs, run if something succeded and nothign failed. + # On non-PRs, run if anything succeded or failed (so partial results are still publided). + if: >- + ${{ !cancelled() && ( + (github.event_name == 'pull_request' && (contains(needs.*.result, 'success') && !contains(needs.*.result, 'failure'))) + || (github.event_name != 'pull_request' && (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure'))) + )}} runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -195,9 +253,11 @@ jobs: with: repository: "verilator/rtlmeter" path: rtlmeter + - name: Setup RTLMeter venv working-directory: rtlmeter run: make venv + - name: Download all results uses: actions/download-artifact@v8 with: @@ -216,6 +276,29 @@ jobs: overwrite: true retention-days: 30 + # The reference (old) results only exist for pull requests, where the + # 'build-*-old' jobs run and the run jobs produce '*-reference-*' artifacts. + - name: Download reference results + if: ${{ github.event_name == 'pull_request' }} + uses: actions/download-artifact@v8 + with: + pattern: rtlmeter-${{ matrix.tag }}-reference-* + path: all-reference-${{ matrix.tag }} + merge-multiple: true + - name: Combine reference results + if: ${{ github.event_name == 'pull_request' }} + working-directory: rtlmeter + run: | + ./rtlmeter collate ../all-reference-${{ matrix.tag }}/*.json > ../all-reference-${{ matrix.tag }}.json + - name: Upload reference results + if: ${{ github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v7 + with: + path: all-reference-${{ matrix.tag }}.json + name: all-reference-${{ matrix.tag }} + overwrite: true + retention-days: 30 + publish-scheduled-results: name: Publish results to verilator/verilator-rtlmeter-results needs: combine-results @@ -304,9 +387,7 @@ jobs: ln -s ../rtlmeter rtlmeter gh repo set-default ${{ github.repository }} # Compare to last successful scheduled run - REF_ID=$(gh run list --workflow RTLMeter --event schedule --status success --limit 1 --json databaseId --jq ".[0].databaseId") - NEW_ID=${{ github.run_id }} - ci/ci-rtlmeter-report.bash $REF_ID $NEW_ID gcc clang gcc-hier + ci/ci-rtlmeter-report.bash ${{ github.run_id }} ${{ github.sha }} gcc clang gcc-hier # Create the report artifact mkdir ../report-artifact mv rtlmeter-report/report ../report-artifact/ diff --git a/ci/ci-rtlmeter-report.bash b/ci/ci-rtlmeter-report.bash index 8fadd8416..29380d472 100755 --- a/ci/ci-rtlmeter-report.bash +++ b/ci/ci-rtlmeter-report.bash @@ -15,11 +15,11 @@ [ "$GITHUB_ACTIONS" != "true" ] || set -x # Arguments: -# 1. reference run ID -# 2. new run ID +# 1. run ID +# 2. SHA of the event that triggered the run (for a PR, the test merge commit) # rest: run tags -REF_ID=$1; shift -NEW_ID=$1; shift +RUN_ID=$1; shift +RUN_SHA=$1; shift RUNS="$@" # $VERILATOR_CHECKOUT/ci directory @@ -32,31 +32,30 @@ pushd rtlmeter-report &> /dev/null TMP_DIR=$(readlink -f .) # Artifacts to download -DOWNLOAD_ARTIFACTS="" +DOWNLOAD_NEW_ARTIFACTS="" +DOWNLOAD_REF_ARTIFACTS="" for r in $RUNS; do - DOWNLOAD_ARTIFACTS="$DOWNLOAD_ARTIFACTS --name all-results-$r" + DOWNLOAD_NEW_ARTIFACTS="$DOWNLOAD_NEW_ARTIFACTS --name all-results-$r" + DOWNLOAD_REF_ARTIFACTS="$DOWNLOAD_REF_ARTIFACTS --name all-reference-$r" done -# Download reference run results +# Download reference artifacts mkdir ref REF_DIR=$(readlink -f ref) -gh run download ${REF_ID} $DOWNLOAD_ARTIFACTS --dir $REF_DIR +gh run download ${RUN_ID} $DOWNLOAD_REF_ARTIFACTS --dir $REF_DIR mv $REF_DIR/*/*.json $REF_DIR/ find $REF_DIR -mindepth 1 -type d -delete -# Download current run results +# Download new version artifacts mkdir new NEW_DIR=$(readlink -f new) -gh run download ${NEW_ID} $DOWNLOAD_ARTIFACTS --dir $NEW_DIR +gh run download ${RUN_ID} $DOWNLOAD_NEW_ARTIFACTS --dir $NEW_DIR mv $NEW_DIR/*/*.json $NEW_DIR/ find $NEW_DIR -mindepth 1 -type d -delete # Get Some metadata about the runs -REF_URL=$(gh run view $REF_ID --json url --jq ".url") -REF_NUM=$(gh run view $REF_ID --json number --jq ".number") -REF_DATE=$(gh run view $REF_ID --json createdAt --jq ".createdAt") -NEW_URL=$(gh run view $NEW_ID --json url --jq ".url") -NEW_NUM=$(gh run view $NEW_ID --json number --jq ".number") +RUN_URL=$(gh run view $RUN_ID --json url --jq ".url") +RUN_NUM=$(gh run view $RUN_ID --json number --jq ".number") # Repository owner and name of the default repository, used to build the # GitHub Pages URL of the detailed report. The owner is lowercased, as required @@ -65,6 +64,12 @@ NEW_NUM=$(gh run view $NEW_ID --json number --jq ".number") PAGES_OWNER=$(gh repo view --json owner --jq '.owner.login' | tr '[:upper:]' '[:lower:]') PAGES_NAME=$(gh repo view --json name --jq '.name') +# The 'old' reference was built from the target branch base commit, which is +# the first parent of the triggering merge commit (same as the 'start' job in +# rtlmeter.yml). Resolved here, while still in the default repository's git +# context (before the 'cd rtlmeter' below). +REF_SHA=$(gh api "repos/{owner}/{repo}/commits/$RUN_SHA" --jq '.parents[0].sha') + # Go back to root directory popd &> /dev/null # Go to RTLMeter directory @@ -75,14 +80,14 @@ SUMMARY_ARGS=() for r in $RUNS; do CMP_JSON=$TMP_DIR/cmp-$r.json # Gather args for summary script - SUMMARY_ARGS+=($REF_DIR/all-results-$r.json $CMP_JSON) + SUMMARY_ARGS+=($NEW_DIR/all-results-$r.json $CMP_JSON) # Create JSON ./rtlmeter compare --format json --steps "*" --metrics "*" \ - $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $CMP_JSON + $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $CMP_JSON # Also create detailed tables - ./rtlmeter compare --format ascii --steps 'verilate' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/verilate-$r.txt - ./rtlmeter compare --format ascii --steps 'cppbuild' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/cppbuild-$r.txt - ./rtlmeter compare --format ascii --steps 'execute' --metrics '* !system !user' $REF_DIR/all-results-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/execute-$r.txt + ./rtlmeter compare --format ascii --steps 'verilate' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/verilate-$r.txt + ./rtlmeter compare --format ascii --steps 'cppbuild' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/cppbuild-$r.txt + ./rtlmeter compare --format ascii --steps 'execute' --metrics '*' $REF_DIR/all-reference-$r.json $NEW_DIR/all-results-$r.json > $TMP_DIR/execute-$r.txt # Chop them at new lines, into one table per file awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-verilate-%02d.txt",prefix,NR)}' $TMP_DIR/verilate-$r.txt awk -v RS= -v prefix=$TMP_DIR/$r-frag '{print > sprintf("%s-cppbuild-%02d.txt",prefix,NR)}' $TMP_DIR/cppbuild-$r.txt @@ -97,8 +102,10 @@ cat $TMP_DIR/summary.txt # Create notification comment content NOTIFICATION=$TMP_DIR/notification.txt cat > $NOTIFICATION < Summary of all runs 
@@ -106,7 +113,11 @@ $(cat $TMP_DIR/summary.txt)
-Detailed report: [${NEW_ID}](https://${PAGES_OWNER}.github.io/${PAGES_NAME}/rtlmeter-reports/${NEW_ID}/index.html) +The reported numbers are the geometric means of the ratios of the metrics over all cases, +less than 1 is a regression, greater than 1 is an improvement. +The jobs run on variable and noisy runners, so some variance is expected between runs. + +Detailed report: [${RUN_ID}](https://${PAGES_OWNER}.github.io/${PAGES_NAME}/rtlmeter-reports/${RUN_ID}/index.html) NOTIFICATION_TEMPLATE # Create detailed report @@ -119,7 +130,7 @@ $(cat $TMP_DIR/summary.txt) SUMMARY_TEMPLATE echo "

Detailed results

" >> $REPORT for r in $RUNS; do - RUN_NAME=$(jq -rj ".[0].runName" $REF_DIR/all-results-$r.json) + RUN_NAME=$(jq -rj ".[0].runName" $NEW_DIR/all-results-$r.json) echo "

$RUN_NAME

" >> $REPORT for f in $(ls -1 $TMP_DIR/$r-frag-verilate-*.txt | sort) \ $(ls -1 $TMP_DIR/$r-frag-cppbuild-*.txt | sort) \ @@ -141,7 +152,7 @@ cat > ${TMP_DIR}/report/index.html < - Verilator RTLMeter report #${NEW_NUM} + Verilator RTLMeter report #${RUN_NUM}