Chore: CI apt caching for flakiness (#14647)

---------

Co-authored-by: Codex <267193182+codex@users.noreply.github.com>

.github/actions/apt-install/action.yml

@@ -1,53 +1,53 @@
 name: Install apt packages
 description: >
-  Installs apt packages with a per-attempt timeout and retries. The hosted runner's Azure apt mirror intermittently serves archives at a crawl, which apt's own timeout does not catch. A stuck attempt is killed and retried, which usually lands on a healthy mirror node.
+  Installs apt packages with bounded download retries and Ubuntu mirror fallback. Caches downloaded packages as an optimization; apt resolves dependencies on every run.
 
 inputs:
   packages:
-    description: Space separated list of packages to install
+    description: Space separated list of package names to install
     required: true
   attempts:
-    description: Maximum number of attempts
+    description: Maximum number of download attempts
     default: "3"
   attempt-timeout:
-    description: Seconds before a single attempt is killed
+    description: Seconds before a single download attempt is killed
     default: "180"
 runs:
   using: composite
   steps:
+    - name: Identify apt cache
+      id: cache-key
+      shell: bash
+      env:
+        PACKAGES: ${{ inputs.packages }}
+      run: |
+        . /etc/os-release
+        packages_hash=$(printf '%s' "$PACKAGES" | sha256sum | cut -d ' ' -f 1)
+        # Cached downloads are optional: apt fetches any missing/newer packages.
+        # Refresh weekly, including inside job containers without ImageVersion.
+        echo "key=apt-v2-${ID}-${VERSION_ID}-$(dpkg --print-architecture)-$(date -u +%G-%V)-${packages_hash}" >> "$GITHUB_OUTPUT"
+    - name: Restore apt packages
+      id: cache
+      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ${{ runner.temp }}/apt-packages/*.deb
+          ${{ runner.temp }}/apt-packages/.keep
+        key: ${{ steps.cache-key.outputs.key }}
     - name: Install ${{ inputs.packages }}
       shell: bash
       env:
         PACKAGES: ${{ inputs.packages }}
         ATTEMPTS: ${{ inputs.attempts }}
         ATTEMPT_TIMEOUT: ${{ inputs.attempt-timeout }}
-      run: |
-        SUDO=""
-        if [ "$(id -u)" -ne 0 ]; then
-          SUDO="sudo"
-        fi
-
-        attempt_install() {
-          $SUDO apt-get update -q
-          # Intentionally unquoted, this is a list of package names
-          $SUDO apt-get install -q --yes --no-install-recommends $PACKAGES
-        }
-        export -f attempt_install
-        export SUDO PACKAGES
-
-        for attempt in $(seq 1 "${ATTEMPTS}"); do
-          echo "::group::apt install attempt ${attempt}/${ATTEMPTS}"
-          start=$(date +%s)
-          if timeout --kill-after=10 "${ATTEMPT_TIMEOUT}" bash -c attempt_install; then
-            echo "::endgroup::"
-            echo "Installed in $(( $(date +%s) - start ))s"
-            exit 0
-          fi
-          echo "::endgroup::"
-          echo "::warning::apt install attempt ${attempt}/${ATTEMPTS} failed or exceeded ${ATTEMPT_TIMEOUT}s"
-          # A killed attempt can leave dpkg half configured
-          $SUDO dpkg --configure -a || true
-        done
-
-        echo "::error::apt install failed after ${ATTEMPTS} attempts"
-        exit 1
+        CACHE_DIR: ${{ runner.temp }}/apt-packages
+      run: bash "$GITHUB_ACTION_PATH/install.sh"
+    # Save immediately: a later test failure must not discard downloaded packages.
+    - name: Save apt packages
+      if: steps.cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ${{ runner.temp }}/apt-packages/*.deb
+          ${{ runner.temp }}/apt-packages/.keep
+        key: ${{ steps.cache-key.outputs.key }}

.github/actions/apt-install/install.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run as root in the timed child process. Only downloads may be interrupted.
+if [ "${1:-}" = download ]; then
+	download_archive_dir=$2
+	shift 2
+	apt-get update -q
+	apt-get install -q --yes --no-install-recommends --download-only \
+		-o "Dir::Cache::archives=$download_archive_dir" "$@"
+	exit 0
+fi
+
+SUDO=()
+if [ "$(id -u)" -ne 0 ]; then
+	SUDO=(sudo)
+fi
+read -r -a requested_packages <<< "$PACKAGES"
+mkdir -p "$CACHE_DIR"
+touch "$CACHE_DIR/.keep"
+
+use_ubuntu_mirrors() {
+	# Hosted runners use a mirror list; job containers can point directly at Azure.
+	# Handle both layouts, leaving other repositories and signature checks intact.
+	if [ -f /etc/apt/apt-mirrors.txt ]; then
+		"${SUDO[@]}" sed -i '\|azure\.archive\.ubuntu\.com|d' /etc/apt/apt-mirrors.txt
+	fi
+	for source in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
+		if [ -f "$source" ]; then
+			"${SUDO[@]}" sed -i 's|azure\.archive\.ubuntu\.com|archive.ubuntu.com|g' "$source"
+		fi
+	done
+}
+
+for attempt in $(seq 1 "$ATTEMPTS"); do
+	echo "::group::apt download attempt ${attempt}/${ATTEMPTS}"
+	if timeout --kill-after=10 "$ATTEMPT_TIMEOUT" "${SUDO[@]}" bash "$0" download "$CACHE_DIR" "${requested_packages[@]}"; then
+		echo "::endgroup::"
+		# Apt handles aliases (e.g. libmagic1 -> libmagic1t64) and dependencies.
+		"${SUDO[@]}" apt-get install -q --yes --no-install-recommends --no-download \
+			-o "Dir::Cache::archives=$CACHE_DIR" "${requested_packages[@]}"
+		# Allow the unprivileged cache action to read archives downloaded as root.
+		find "$CACHE_DIR" -maxdepth 1 -name '*.deb' -exec "${SUDO[@]}" chmod a+r {} +
+		exit 0
+	fi
+	echo "::endgroup::"
+	echo "::warning::apt download attempt ${attempt}/${ATTEMPTS} failed or exceeded ${ATTEMPT_TIMEOUT}s"
+	if [ "$attempt" -eq 1 ] && [ "$ATTEMPTS" -gt 1 ]; then
+		echo "Switching from Azure to Ubuntu archive mirrors for remaining attempts"
+		use_ubuntu_mirrors
+	fi
+done
+
+echo "::error::apt download failed after ${ATTEMPTS} attempts"
+exit 1

.github/workflows/ci-backend.yml

@@ -77,6 +77,7 @@ jobs:
               - 'uv.lock'
               - 'docker/compose/docker-compose.ci-test.yml'
               - '.github/workflows/ci-backend.yml'
+              - '.github/actions/apt-install/**'
   test:
     needs: changes
     if: needs.changes.outputs.backend_changed == 'true'

.github/workflows/ci-frontend.yml

@@ -69,6 +69,7 @@ jobs:
             frontend:
               - 'src-ui/**'
               - '.github/workflows/ci-frontend.yml'
+              - '.github/actions/apt-install/**'
   install-dependencies:
     needs: changes
     if: needs.changes.outputs.frontend_changed == 'true'