CI: Run coverage job on 'pr: dev-coverage' label in PRs (#6527)

.github/workflows/coverage.yml

@@ -8,19 +8,40 @@ on:
   workflow_dispatch:
   schedule:
     - cron: '0 0 * * 0'  # weekly
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
 
 permissions:
   contents: read
 
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  # At most 1 job per branch. Auto cancel all but scheduled jobs
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
 jobs:
 
   build:
     name: Build
     # Only run scheduled jobs if explicitly enabled for that repo (e.g.: not on forks)
-    if: ${{ github.event_name != 'schedule' || vars.ENABLE_SCHEDULED_JOBS == 'true' }}
+    # Only run pull request jobs if labelled as needing an coverage run
+    # Always run workflow dispatch jobs
+    if: |
+      (github.event_name == 'schedule'
+        && vars.ENABLE_SCHEDULED_JOBS == 'true') ||
+      (github.event_name == 'pull_request'
+        && contains(github.event.pull_request.labels.*.name, 'pr: dev-coverage')) ||
+      (github.event_name == 'workflow_dispatch')
     uses: ./.github/workflows/reusable-build.yml
     with:
-      sha: ${{ github.sha }}
+      # For pull requests, build the head of the pull request branch, not the
+      # merge commit, otherwise patch coverage would include the changes
+      # between the root of the pull request and the target branch
+      sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       os: ubuntu-24.04
       os-name: linux
       cc: gcc
@@ -82,10 +103,14 @@ jobs:
   prepare-report:
     name: Prepare HTML report
     needs: [build, test]
+    if: ${{ contains(needs.*.result, 'success') && !cancelled() }}
     runs-on: ubuntu-24.04
     steps:
       - name: Install dependencies
-        run: sudo apt install lcov
+        run: |
+          echo 'set man-db/auto-update false' | sudo debconf-communicate >/dev/null
+          sudo dpkg-reconfigure man-db
+          sudo apt install lcov
 
       - name: Download repository archive
         uses: actions/download-artifact@v5
@@ -107,12 +132,41 @@ jobs:
 
       - name: Create report
         working-directory: repo
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
           ls -lsha obj_coverage
-          make coverage-combine # Just to create dependency files, overwrite below
+          # Combine reports from test jobs
           nodist/fastcov.py -C obj_coverage/verilator-*.info --lcov -o obj_coverage/verilator.info
-          make coverage-report
+          # For a PR, report patch coverage against the merge-base between the head of the PR and the target branch
-          find obj_coverage -mindepth 1 -maxdepth 1 -not -name report | xargs rm -rf
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            COVERAGE_BASE=$(git rev-parse --short $(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}))
+            make coverage-report COVERAGE_BASE=${COVERAGE_BASE} |& tee ${{ github.workspace }}/make-coverage-report.log
+          else
+            make coverage-report
+          fi
+          # Remove data files
+          rm -f obj_coverage/verilator*.info
+          # Some extra work for PRs only
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            # Save PR number in report
+            echo ${{ github.event.number }} > obj_coverage/pr-number.txt
+            # Generate notification comment content
+            mkdir -p notification
+            echo ${{ github.event.number }} > notification/pr-number.txt
+            NUM=$(gh run view ${{ github.run_id }} --json number --jq ".number")
+            URL=$(gh run view ${{ github.run_id }} --json url    --jq ".url")
+            echo "Patch coverage from PR workflow [#$NUM]($URL) (code coverage of lines changed relative to ${COVERAGE_BASE}):" > notification/body.txt
+            if [[ ! -f obj_coverage/empty-patch ]]; then
+              echo "<pre>" >> notification/body.txt
+              grep -E "(lines|branches)\.*:" ${{ github.workspace }}/make-coverage-report.log | sed "s/\.*:/:/" >> notification/body.txt || true
+              echo "</pre>" >> notification/body.txt
+              echo "Report: [${{ github.run_id }}](https://${{ github.repository_owner }}.github.io/verilator/coverage-reports/${{ github.run_id }}/index.html)" >> notification/body.txt
+            else
+              echo "Patch contains no code changes" >> notification/body.txt
+            fi
+            cat notification/body.txt
+          fi
 
       - name: Upload report
         uses: actions/upload-artifact@v4
@@ -120,6 +174,13 @@ jobs:
           path: repo/obj_coverage
           name: coverage-report
 
+      - name: Upload notification
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: repo/notification
+          name: coverage-pr-notification
+
   # Create GitHub issue for failed scheduled jobs
   # This should always be the last job (we want an issue if anything breaks)
   create-issue:

Makefile.in

@@ -607,7 +607,6 @@ ifeq ($(CFG_WITH_DEV_GCOV),yes)
 COVERAGE_REF_BASE := $(if $(COVERAGE_BASE),$(shell git rev-parse --short $(COVERAGE_BASE)))
 COVERAGE_REF_HEAD := $(shell git rev-parse --short HEAD)
 override undefine COVERAGE_BASE # Use the above variabels instead
-COVERAGE_PATCH_FILE := $(COVERAGE_DIR)/$(if $(COVERAGE_REF_BASE),$(COVERAGE_REF_BASE).patch,.none.pach)
 
 # 'fastcov' setup
 FASTCOV := nodist/fastcov.py
@@ -649,7 +648,6 @@ FASTCOV_OPT += BROKEN_BASE_RTN
 FASTCOV_OPT += SELF_CHECK
 FASTCOV_OPT += 'if (VL_UNCOVERABLE'
 FASTCOV_OPT += '} else if (VL_UNCOVERABLE'
-FASTCOV_OPT += $(if $(COVERAGE_REF_BASE),--diff-filter $(COVERAGE_PATCH_FILE))
 
 # 'genhtml' setup
 GENHTML := genhtml
@@ -662,7 +660,7 @@ GENHTML_OPT += --ignore-errors negative
 ifeq ($(COVERAGE_REF_BASE),)
 GENHTML_OPT += --header-title "Code coverage for Verilator $(shell git describe --dirty)"
 else
-GENHTML_OPT += --header-title "Patch coverage for Verilator $(COVERAGE_REF_BASE)..$(COVERAGE_REF_HEAD)$(if $(git status --porcelain),,-dirty)"
+GENHTML_OPT += --header-title "Patch coverage for Verilator $(COVERAGE_REF_BASE)..$(COVERAGE_REF_HEAD)$(if $(shell git status --porcelain),-dirty)"
 endif
 GENHTML_OPT += --flat
 GENHTML_OPT += --precision 2
@@ -674,30 +672,35 @@ GENHTML_OPT += --filter brace,blank,range
 GCNO_FILES = $(shell find . -name '*.gcno')
 GCDA_FILES = $(shell find . -name '*.gcda')
 
-# Patch file to filter coverage with - unused if doing full coverage
-$(COVERAGE_PATCH_FILE):
-	@mkdir -p $(COVERAGE_DIR)
-	@touch $@
-	$(if $(COVERAGE_REF_BASE), git diff $(COVERAGE_REF_BASE) > $(COVERAGE_PATCH_FILE))
-
 # Combine all .gcda coverage date files into lcov .info file
-$(COVERAGE_DIR)/verilator.info: $(COVERAGE_PATCH_FILE) $(GCNO_FILES) $(GCDA_FILES)
+$(COVERAGE_DIR)/verilator.info: $(GCNO_FILES) $(GCDA_FILES)
 	@echo "####################################################################"
 	@echo "# fastcov: combining all .gcda files into lcov .info"
 	@echo "####################################################################"
+	@mkdir -p $(COVERAGE_DIR)
 	/usr/bin/time -f "That took %E" \
     $(FASTCOV) $(FASTCOV_OPT) --output $@
-	@# Uncommitted changes not tracked, force rebuild on next run if patch coverage
+
-	@$(if $(COVERAGE_REF_BASE),rm $(COVERAGE_PATCH_FILE))
+# Filter combined .info file for patch coverage
+$(COVERAGE_DIR)/verilator-patch.info: $(COVERAGE_DIR)/verilator.info
+	@echo "####################################################################"
+	@echo "# fastcov: Filtering for patch coverage"
+	@echo "####################################################################"
+	rm -f $(COVERAGE_DIR)/empty-patch
+	git diff $(COVERAGE_REF_BASE) -- include src > $(COVERAGE_DIR)/filter.patch
+	[ -s $(COVERAGE_DIR)/filter.patch ]] || touch $(COVERAGE_DIR)/empty-patch
+	$(FASTCOV) -C $^ --lcov -o $@ --diff-filter $(COVERAGE_DIR)/filter.patch
 
 # Build coverage report
-$(COVERAGE_DIR)/report/index.html: $(COVERAGE_DIR)/verilator.info
+$(COVERAGE_DIR)/report/index.html: $(COVERAGE_DIR)/verilator$(if $(COVERAGE_REF_BASE),-patch).info
 	@echo "####################################################################"
 	@echo "# genhtml: Generating coverage report"
 	@echo "####################################################################"
-	@rm -rf $(COVERAGE_DIR)/reprot
+	@rm -rf $(COVERAGE_DIR)/report
-	/usr/bin/time -f "That took %E" \
+	[ -f $(COVERAGE_DIR)/empty-patch ]] || /usr/bin/time -f "That took %E" \
-    $(GENHTML) $(GENHTML_OPT) --output-directory $(COVERAGE_DIR)/report $^
+    $(GENHTML) $(GENHTML_OPT) --output-directory $(COVERAGE_DIR)/report $^ || true
+	@# Uncommitted changes not tracked, force rebuild on next run if patch coverage
+	@$(if $(COVERAGE_REF_BASE),mv $(COVERAGE_DIR)/verilator-patch.info $(COVERAGE_DIR)/verilator-patch-last.info)
 
 # Convenience targets
 .PHONY: coverage-combine
@@ -706,16 +709,22 @@ coverage-combine: $(COVERAGE_DIR)/verilator.info
 # Via recursive make, so the message is always printed
 .PHONY: coverage-report
 coverage-report:
-	@$(MAKE) --no-print-directory $(COVERAGE_DIR)/report/index.html
+	@$(MAKE) --no-print-directory $(COVERAGE_DIR)/report/index.html || true
 	@echo "####################################################################"
-	@echo "# Coverage report is at: $(COVERAGE_DIR)/report/index.html"
+	@if [ -f $(COVERAGE_DIR)/report/index.html ]; then \
-	@echo "# Use 'make coverage-view' to open it in your default browser"
+    echo "# Coverage report is at: $(COVERAGE_DIR)/report/index.html"; \
+    echo "# Use 'make coverage-view' to open it in your default browser"; \
+  elif [ -f $(COVERAGE_DIR)/empty-patch ]; then\
+    echo "# Patch is empty"; \
+  else \
+    echo "# Failed to create coverage report. Maybe there is no data?"; \
+  fi
 	@echo "####################################################################"
 
 # Open covarage report in default web browser
 .PHONY: coverage-view
-coverage-view: $(COVERAGE_DIR)/report/index.html
+coverage-view: coverage-report
-	open $<
+	@test -f $(COVERAGE_DIR)/report/index.html && open $(COVERAGE_DIR)/report/index.html || true
 
 # Deletes all coverage data files (.gcda)
 .PHONY: coverage-zero

ci/ci-pages.bash

@@ -159,8 +159,7 @@ CONTENTS_TEMPLATE
 INDEX_TEMPLATE
 
   # Report size
-  ${COVERAGE_ROOT}
+  du -shc ${COVERAGE_ROOT}/*
-  du -shc *
 
   # Set output
   echo "coverage-pr-run-ids=${PR_RUN_IDS}" >> $GITHUB_OUTPUT