CI: Add timeout to apt install steps (#8154)

These tend to hang for hours on occasion. Install all apt dependencies
via ci-install.bash, and add a hard timeout and retry for each attempt.

Also removed unnecessary man-db config (GitHub has this off by default
now) and ineffective install path exclusions.

.github/workflows/coverage.yml

@@ -104,13 +104,6 @@ jobs:
     if: ${{ contains(needs.*.result, 'success') && !cancelled() }}
     runs-on: ubuntu-24.04
     steps:
-      - name: Install dependencies
-        run: |
-          echo 'set man-db/auto-update false' | sudo debconf-communicate >/dev/null
-          sudo dpkg-reconfigure man-db
-          sudo apt install lcov || \
-          sudo apt install lcov
-
       - name: Download repository archive
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8
         with:
@@ -122,6 +115,10 @@ jobs:
           tar --zstd -x -f ${{ needs.build.outputs.archive }}
           ls -lsha
 
+      - name: Install dependencies
+        working-directory: repo
+        run: ./ci/ci-install.bash coverage
+
       - name: Download code coverage data
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8
         with:

.github/workflows/format.yml

@@ -27,9 +27,10 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install packages for build
+        run: ./ci/ci-install.bash format
+
+      - name: Configure git identity
         run: |
-          sudo apt install clang-format-18 || \
-          sudo apt install clang-format-18
           git config --global user.email "action@example.com"
           git config --global user.name "github action"
 

.github/workflows/reusable-rtlmeter-run.yml

@@ -62,15 +62,15 @@ jobs:
     runs-on: ${{ inputs.runs-on }}
     name: Run
     steps:
+      - name: Checkout Verilator CI scripts
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
+        with:
+          path: repo
+          sparse-checkout: ci
+
       - name: Install dependencies
-        run: |
-          echo "path-exclude /usr/share/doc/*"  | sudo tee -a /etc/dpkg/dpkg.cfg.d/01_nodoc
-          echo "path-exclude /usr/share/man/*"  | sudo tee -a /etc/dpkg/dpkg.cfg.d/01_nodoc
-          echo "path-exclude /usr/share/info/*" | sudo tee -a /etc/dpkg/dpkg.cfg.d/01_nodoc
-          sudo apt update || \
-          sudo apt update
-          sudo apt install ccache mold libfl-dev libjemalloc-dev libsystemc-dev || \
-          sudo apt install ccache mold libfl-dev libjemalloc-dev libsystemc-dev
+        working-directory: repo
+        run: ./ci/ci-install.bash rtlmeter-run
 
       - name: Checkout RTLMeter
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7

ci/ci-install.bash

@@ -26,28 +26,40 @@ source "$(dirname "$0")/ci-common.bash"
 ################################################################################
 # Parse arguments
 
-# Which stage to install dependencies for: 'build' or 'test'
+# Which stage to install dependencies for
 STAGE="$1"
 
 ################################################################################
 # Install dependencies
 
-# Avoid occasional cpan failures "Issued certificate has expired."
-export PERL_LWP_SSL_VERIFY_HOSTNAME=0
-echo "check_certificate = off" >> ~/.wgetrc
-
-if [ "$HOST_OS" = "linux" ]; then
-  # Avoid slow "processing triggers for man db"
-  echo "path-exclude /usr/share/doc/*"  | sudo tee -a /etc/dpkg/dpkg.cfg.d/01_nodoc
-  echo "path-exclude /usr/share/man/*"  | sudo tee -a /etc/dpkg/dpkg.cfg.d/01_nodoc
-  echo "path-exclude /usr/share/info/*" | sudo tee -a /etc/dpkg/dpkg.cfg.d/01_nodoc
-elif [ "$HOST_OS" = "macOS" ]; then
+if [ "$HOST_OS" = "macOS" ]; then
   # The macos runner image ships an untrusted third-party tap we don't use;
   # untap it so brew stops emitting a tap-trust warning. Force + '|| true' since
   # untap fails if a formula was installed from it, which is harmless here.
   brew untap --force aws/tap || true
 fi
 
+# Run 'sudo apt-get', with timeout, retrying if failed
+apt-get-retry() {
+  local _attempt
+  for _attempt in 1 2 3; do
+    [ "$_attempt" -eq 1 ] || sleep $((15 * _attempt))
+    sudo timeout --kill-after=30s 5m env DEBIAN_FRONTEND=noninteractive \
+      apt-get -o DPkg::Lock::Timeout=60 "$@" && return 0
+  done
+  fatal "'apt-get $*' failed on all attempts"
+}
+
+# Run 'brew', retrying if failed
+brew-retry() {
+  local _attempt
+  for _attempt in 1 2 3; do
+    [ "$_attempt" -eq 1 ] || sleep $((15 * _attempt))
+    brew "$@" && return 0
+  done
+  fatal "'brew $*' failed on all attempts"
+}
+
 install-wavediff() {
   source ci/docker/buildenv/wavetools.conf
   local _base_url="https://github.com/hudson-trading/wavetools/releases/download/${WAVETOOLS_VERSION}"
@@ -88,10 +100,8 @@ if [ "$STAGE" = "build" ]; then
       if [ "$DISTRO_VERSION" != "22.04" ]; then
         PACKAGES+=(libjemalloc-dev)
       fi
-      sudo apt-get update ||
-      sudo apt-get update
-      sudo apt-get install --yes "${PACKAGES[@]}" ||
-      sudo apt-get install --yes "${PACKAGES[@]}"
+      apt-get-retry update
+      apt-get-retry install --yes "${PACKAGES[@]}"
     fi
   elif [ "$HOST_OS" = "macOS" ]; then
     PACKAGES=(
@@ -103,10 +113,8 @@ if [ "$STAGE" = "build" ]; then
       help2man
       perl
     )
-    brew update ||
-    brew update
-    brew install "${PACKAGES[@]}" ||
-    brew install "${PACKAGES[@]}"
+    brew-retry update
+    brew-retry install "${PACKAGES[@]}"
   else
     fatal "Unknown HOST_OS: '$HOST_OS'"
   fi
@@ -128,10 +136,8 @@ elif [ "$STAGE" = "test" ]; then
         python3-clang
         z3
       )
-      sudo apt-get update ||
-      sudo apt-get update
-      sudo apt-get install --yes "${PACKAGES[@]}" ||
-      sudo apt-get install --yes "${PACKAGES[@]}"
+      apt-get-retry update
+      apt-get-retry install --yes "${PACKAGES[@]}"
     fi
   elif [ "$HOST_OS" = "macOS" ]; then
     PACKAGES=(
@@ -140,10 +146,8 @@ elif [ "$STAGE" = "test" ]; then
       perl
       z3
     )
-    brew update ||
-    brew update
-    brew install "${PACKAGES[@]}" ||
-    brew install "${PACKAGES[@]}"
+    brew-retry update
+    brew-retry install "${PACKAGES[@]}"
   else
     fatal "Unknown HOST_OS: '$HOST_OS'"
   fi
@@ -152,20 +156,59 @@ elif [ "$STAGE" = "test" ]; then
   # Workaround -fsanitize=address crash
   sudo sysctl -w vm.mmap_rnd_bits=28
 elif [ "$STAGE" = "lint-py" ]; then
-  # nodist/clang_check_attributes.
+  ##############################################################################
+  # Dependencies for nodist/clang_check_attributes
+
   if [ "$HOST_OS" = "linux" ] && [ "$DISTRO_ID" = "ubuntu" ]; then
     PACKAGES=(
       python3-clang  # Not run, but importers are linted
     )
-    sudo apt-get update ||
-    sudo apt-get update
-    sudo apt-get install --yes "${PACKAGES[@]}" ||
-    sudo apt-get install --yes "${PACKAGES[@]}"
+    apt-get-retry update
+    apt-get-retry install --yes "${PACKAGES[@]}"
+  fi
+elif [ "$STAGE" = "format" ]; then
+  ##############################################################################
+  # Dependencies of 'make format'
+
+  if [ "$HOST_OS" = "linux" ] && [ "$DISTRO_ID" = "ubuntu" ]; then
+    PACKAGES=(
+      clang-format-18  # Version pinned, so all of CI formats alike
+    )
+    apt-get-retry update
+    apt-get-retry install --yes "${PACKAGES[@]}"
+  fi
+elif [ "$STAGE" = "coverage" ]; then
+  ##############################################################################
+  # Dependencies of the coverage report job, which only collates what the
+  # 'test' stage jobs measured
+
+  if [ "$HOST_OS" = "linux" ] && [ "$DISTRO_ID" = "ubuntu" ]; then
+    PACKAGES=(
+      lcov
+    )
+    apt-get-retry update
+    apt-get-retry install --yes "${PACKAGES[@]}"
+  fi
+elif [ "$STAGE" = "rtlmeter-run" ]; then
+  ##############################################################################
+  # Dependencies of the RTLMeter jobs, which compile and run designs with an
+  # already built Verilator, so they need its runtime prerequisites only
+
+  if [ "$HOST_OS" = "linux" ] && [ "$DISTRO_ID" = "ubuntu" ]; then
+    PACKAGES=(
+      ccache
+      libfl-dev
+      libjemalloc-dev
+      libsystemc-dev
+      mold
+    )
+    apt-get-retry update
+    apt-get-retry install --yes "${PACKAGES[@]}"
   fi
 else
   ##############################################################################
   # Unknown build stage
-  fatal "Unknown stage '$STAGE' (expected 'build', 'test' or 'lint-py')"
+  fatal "Unknown stage '$STAGE'"
 fi
 
 # Report where the tools we may have installed live (ok if some are missing)