CI: Factor out artifact based caching as a reusable action (#7883)
This commit is contained in:
Geza Lore
GitHub
parent
bf4c72f93b
commit
fcc7acbfc6
|
|
@ -0,0 +1,165 @@
|
|||
---
|
||||
# DESCRIPTION: Github actions composite action
|
||||
# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
|
||||
|
||||
name: Artifact cache
|
||||
description: >-
|
||||
Cache a directory across workflow runs using a GitHub Actions artifact instead
|
||||
of actions/cache (useful where artifacts are preferable, e.g. no 10 GB limit
|
||||
on public repos). The base 'key' is scoped automatically: per pull request
|
||||
(<key>-pr-<N>) or per branch (<key>-branch-<name>), and on a PR restore falls
|
||||
back to the target branch's cache. Call once with mode=restore before the work
|
||||
and once with mode=save after, passing the same 'key' and 'path'.
|
||||
|
||||
Branch-scoped restores are provenance-checked (the artifact must come from a
|
||||
same-repo push to that branch) - artifact names are a global, unauthenticated
|
||||
namespace, so otherwise a fork PR could forge one.
|
||||
|
||||
mode=restore calls the GitHub API via 'gh', so the caller's job must grant
|
||||
'actions: read'.
|
||||
|
||||
inputs:
|
||||
mode:
|
||||
description: "'restore' or 'save'"
|
||||
required: true
|
||||
path:
|
||||
description: "Directory to cache (archived on save, extracted into on restore)"
|
||||
required: true
|
||||
key:
|
||||
description: "Base artifact name; scoped per PR/branch automatically"
|
||||
required: true
|
||||
retention-days:
|
||||
description: "Artifact retention in days (mode=save)"
|
||||
required: false
|
||||
default: '3'
|
||||
token:
|
||||
description: "Token for the GitHub API (mode=restore); defaults to the job's GITHUB_TOKEN"
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
outputs:
|
||||
key:
|
||||
description: "Echoes the input 'key', so a later save can reuse it without repeating the expression"
|
||||
value: ${{ inputs.key }}
|
||||
cache-hit:
|
||||
description: "'true' if an artifact was restored (mode=restore)"
|
||||
value: ${{ steps.restore.outputs.cache-hit }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Check mode
|
||||
shell: bash
|
||||
env:
|
||||
MODE: ${{ inputs.mode }}
|
||||
run: |
|
||||
case "$MODE" in
|
||||
restore|save) ;;
|
||||
*) echo "::error::artifact-cache: 'mode' must be 'restore' or 'save' (got '$MODE')"; exit 1 ;;
|
||||
esac
|
||||
|
||||
# Scope the base key: <key>-pr-<N> for pull requests (with the target branch
|
||||
# as fallback), else <key>-branch-<name>. Refs come via env, not ${{ }}
|
||||
# interpolation, so a branch name with shell metacharacters can't inject.
|
||||
- name: Compute artifact name
|
||||
id: name
|
||||
shell: bash
|
||||
env:
|
||||
BASE_KEY: ${{ inputs.key }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sanitize() { printf '%s' "$1" | tr '/' '-'; } # artifact names cannot contain '/'
|
||||
if [ "$EVENT_NAME" = pull_request ]; then
|
||||
{
|
||||
echo "primary=${BASE_KEY}-pr-${PR_NUMBER}"
|
||||
echo "primary-branch="
|
||||
echo "fallback=${BASE_KEY}-branch-$(sanitize "$BASE_REF")"
|
||||
echo "fallback-branch=$BASE_REF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
{
|
||||
echo "primary=${BASE_KEY}-branch-$(sanitize "$REF_NAME")"
|
||||
echo "primary-branch=$REF_NAME"
|
||||
echo "fallback="
|
||||
echo "fallback-branch="
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Restore artifact cache
|
||||
id: restore
|
||||
if: ${{ inputs.mode == 'restore' }}
|
||||
continue-on-error: true # a cold start is fine; a restore failure must not fail the caller
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ inputs.token || github.token }}
|
||||
CACHE_PATH: ${{ inputs.path }}
|
||||
PRIMARY: ${{ steps.name.outputs.primary }}
|
||||
PRIMARY_BRANCH: ${{ steps.name.outputs.primary-branch }}
|
||||
FALLBACK: ${{ steps.name.outputs.fallback }}
|
||||
FALLBACK_BRANCH: ${{ steps.name.outputs.fallback-branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$CACHE_PATH"
|
||||
# Newest non-expired artifact named $1 (empty if none). With a non-empty
|
||||
# $2 (branch), require provenance: a same-repo run (head_repository_id ==
|
||||
# repository_id) whose head branch is $2, so a fork cannot forge it.
|
||||
newest_run_id() {
|
||||
want="$1" br="$2" gh api \
|
||||
"/repos/$GITHUB_REPOSITORY/actions/artifacts?name=$1&per_page=100" \
|
||||
--jq '.artifacts[]
|
||||
| select(.expired == false and .name == env.want)
|
||||
| select(env.br == ""
|
||||
or (.workflow_run.head_branch == env.br
|
||||
and .workflow_run.head_repository_id == .workflow_run.repository_id))
|
||||
| [.created_at, (.workflow_run.id | tostring)] | @tsv' \
|
||||
| sort | tail -n1 | cut -f2
|
||||
}
|
||||
restore() { # $1=name $2=branch(empty ok); returns 0 on a successful restore
|
||||
local name="$1" br="$2" rid tarball
|
||||
[ -n "$name" ] || return 1
|
||||
rid="$(newest_run_id "$name" "$br")" || return 1
|
||||
[ -n "$rid" ] || return 1
|
||||
echo "Restoring '$CACHE_PATH' from artifact '$name' (run $rid)"
|
||||
gh run download "$rid" -R "$GITHUB_REPOSITORY" --name "$name" --dir "$RUNNER_TEMP/artifact-cache-dl" || return 1
|
||||
# Each artifact holds a single tarball; match by extension so the inner
|
||||
# filename is not part of the contract.
|
||||
tarball="$(find "$RUNNER_TEMP/artifact-cache-dl" -name '*.tar.zst' -print -quit)"
|
||||
[ -n "$tarball" ] || return 1
|
||||
tar -I zstd -x -f "$tarball" -C "$CACHE_PATH"
|
||||
}
|
||||
if restore "$PRIMARY" "$PRIMARY_BRANCH"; then
|
||||
echo "Restored from primary key"
|
||||
echo "cache-hit=true" >> "$GITHUB_OUTPUT"
|
||||
elif restore "$FALLBACK" "$FALLBACK_BRANCH"; then
|
||||
echo "Restored from fallback key"
|
||||
echo "cache-hit=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No matching artifact found; starting cold"
|
||||
echo "cache-hit=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Pack artifact cache
|
||||
id: pack
|
||||
if: ${{ inputs.mode == 'save' }}
|
||||
shell: bash
|
||||
env:
|
||||
CACHE_PATH: ${{ inputs.path }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
packdir="$(mktemp -d)" # unique dir so concurrent saves cannot collide
|
||||
tar -I 'zstd -T0' -cf "$packdir/cache.tar.zst" -C "$CACHE_PATH" .
|
||||
echo "tarball=$packdir/cache.tar.zst" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload artifact cache
|
||||
if: ${{ inputs.mode == 'save' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: ${{ steps.name.outputs.primary }}
|
||||
path: ${{ steps.pack.outputs.tarball }}
|
||||
retention-days: ${{ inputs.retention-days }}
|
||||
overwrite: true
|
||||
compression-level: 0 # tarball is already zstd-compressed
|
||||
|
|
@ -42,7 +42,6 @@ env:
|
|||
CCACHE_COMPILERCHECK: content
|
||||
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
||||
CXX: ${{ inputs.cc == 'clang' && 'clang++' || 'g++' }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
|
|
@ -69,72 +68,13 @@ jobs:
|
|||
ls -lsha
|
||||
|
||||
# Test-job ccache is stored as an artifact, not actions/cache
|
||||
# - pull_request: restore this PR's own previous run; if none, fall back to the PR's target branch.
|
||||
# - branch push: restore that branch's own cache
|
||||
# Artifact names are a global, unauthenticated namespace, so a PR could
|
||||
# upload a "-branch-<x>" artifact impersonating a branch. Branch-scoped
|
||||
# restores are therefore provenance-checked below (must come from a same-repo
|
||||
# push to that branch); the pr-<N> scope is not (a PR can only affect its own
|
||||
# unmerged cache).
|
||||
- name: Compute ccache artifact name
|
||||
# Pass github refs via env, not ${{ }} interpolation, so a branch name
|
||||
# containing shell metacharacters can't break or inject into the script.
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
base="ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.suite }}"
|
||||
sanitize() { printf '%s' "$1" | tr '/' '_'; } # artifact names cannot contain '/'
|
||||
if [ "${{ github.event_name }}" = pull_request ]; then
|
||||
echo "CCACHE_ARTIFACT=${base}-pr-${{ github.event.pull_request.number }}" >> "$GITHUB_ENV"
|
||||
echo "CCACHE_ARTIFACT_BRANCH=" >> "$GITHUB_ENV"
|
||||
echo "CCACHE_FALLBACK=${base}-branch-$(sanitize "$BASE_REF")" >> "$GITHUB_ENV"
|
||||
echo "CCACHE_FALLBACK_BRANCH=${BASE_REF}" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "CCACHE_ARTIFACT=${base}-branch-$(sanitize "$REF_NAME")" >> "$GITHUB_ENV"
|
||||
echo "CCACHE_ARTIFACT_BRANCH=${REF_NAME}" >> "$GITHUB_ENV"
|
||||
fi
|
||||
|
||||
- name: Restore ccache from artifact
|
||||
continue-on-error: true # a cold start is fine; never fail the job here
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$CCACHE_DIR"
|
||||
# Return newest non-expired artifact named $1 (empty if none). When
|
||||
# $2 (branch) is non-empty, require provenance: the artifact must come from
|
||||
# a same-repo run (head_repository_id == repository_id) whose head branch
|
||||
# is $2, so a fork PR cannot forge a branch-scoped artifact.
|
||||
newest_run_id() {
|
||||
want="$1" br="$2" gh api \
|
||||
"/repos/$GITHUB_REPOSITORY/actions/artifacts?name=$1&per_page=100" \
|
||||
--jq '.artifacts[]
|
||||
| select(.expired == false and .name == env.want)
|
||||
| select(env.br == ""
|
||||
or (.workflow_run.head_branch == env.br
|
||||
and .workflow_run.head_repository_id == .workflow_run.repository_id))
|
||||
| [.created_at, (.workflow_run.id | tostring)] | @tsv' \
|
||||
| sort | tail -n1 | cut -f2
|
||||
}
|
||||
restore() { # $1=name $2=branch(empty ok); returns 0 on a successful restore
|
||||
local name="$1" br="$2" rid tarball
|
||||
[ -n "$name" ] || return 1
|
||||
rid="$(newest_run_id "$name" "$br")" || return 1
|
||||
[ -n "$rid" ] || return 1
|
||||
echo "Restoring ccache from '$name' (run $rid)"
|
||||
gh run download "$rid" --name "$name" --dir "$RUNNER_TEMP/ccache-dl" || return 1
|
||||
tarball="$(find "$RUNNER_TEMP/ccache-dl" -name ccache.tar.zst -print -quit)"
|
||||
[ -n "$tarball" ] || return 1
|
||||
tar -I zstd -x -f "$tarball" -C "$CCACHE_DIR"
|
||||
}
|
||||
if restore "${CCACHE_ARTIFACT:-}" "${CCACHE_ARTIFACT_BRANCH:-}"; then
|
||||
echo "Restored primary ccache"
|
||||
exit 0
|
||||
fi
|
||||
if restore "${CCACHE_FALLBACK:-}" "${CCACHE_FALLBACK_BRANCH:-}"; then
|
||||
echo "Restored fallback ccache"
|
||||
exit 0
|
||||
fi
|
||||
echo "No ccache artifact found; starting cold"
|
||||
- name: Restore ccache
|
||||
id: ccache
|
||||
uses: ./repo/.github/actions/artifact-cache
|
||||
with:
|
||||
mode: restore
|
||||
path: ${{ env.CCACHE_DIR }}
|
||||
key: ccache-${{ inputs.runs-on }}-${{ inputs.cc }}-${{ inputs.suite }}
|
||||
|
||||
- name: Install test dependencies
|
||||
run: |
|
||||
|
|
@ -168,19 +108,15 @@ jobs:
|
|||
path: ${{ github.workspace }}/repo/obj_coverage/verilator-${{ inputs.suite }}.info
|
||||
name: code-coverage-${{ inputs.suite }}
|
||||
|
||||
- name: Pack ccache
|
||||
if: ${{ !cancelled() && env.CCACHE_ARTIFACT != '' }}
|
||||
run: tar -I 'zstd -T0' -cf "$GITHUB_WORKSPACE/ccache.tar.zst" -C "$CCACHE_DIR" .
|
||||
|
||||
- name: Save ccache artifact
|
||||
if: ${{ !cancelled() && env.CCACHE_ARTIFACT != '' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
- name: Save ccache
|
||||
if: ${{ !cancelled() }}
|
||||
uses: ./repo/.github/actions/artifact-cache
|
||||
with:
|
||||
name: ${{ env.CCACHE_ARTIFACT }}
|
||||
path: ${{ github.workspace }}/ccache.tar.zst
|
||||
retention-days: 3
|
||||
overwrite: true
|
||||
compression-level: 0
|
||||
mode: save
|
||||
path: ${{ env.CCACHE_DIR }}
|
||||
key: ${{ steps.ccache.outputs.key }}
|
||||
# Keep master's cache around longer; PR/branch caches churn faster
|
||||
retention-days: ${{ github.ref == 'refs/heads/master' && 7 || 3 }}
|
||||
|
||||
- name: Fail job if a test failed
|
||||
if: ${{ steps.run-test.outcome == 'failure' && !cancelled() }}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
# Executed in the 'test' stage.
|
||||
################################################################################
|
||||
|
||||
# Destructive to the checkout (runs 'find . -delete'); never run locally
|
||||
# Destructive to the checkout (wipes most of it); never run locally
|
||||
if [ "$GITHUB_ACTIONS" != "true" ]; then
|
||||
echo "ERROR: $(basename "$0") must only be run in GitHub Actions CI" >&2
|
||||
exit 1
|
||||
|
|
@ -66,8 +66,8 @@ if [ -n "$OPT_RELOC" ]; then
|
|||
mv test_regress "$TEST_REGRESS"
|
||||
NODIST="$OPT_RELOC/nodist"
|
||||
mv nodist "$NODIST"
|
||||
# Delete everything else
|
||||
find . -delete
|
||||
# Delete everything else, but keep the CI infrastructure
|
||||
find . -mindepth 1 -maxdepth 1 ! -name .github ! -name ci -exec rm -rf {} +
|
||||
ls -la .
|
||||
fi
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue