From f9989847d13f88126cce664b343846c97e4fe579 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:12:42 -0700 Subject: [PATCH] Fix: simplify CI apt caching and mirror handling Let apt resolve package aliases and missing dependencies while reusing cached downloads. Support Azure mirror replacement in both hosted runners and job containers, and keep timeouts limited to downloads. Co-Authored-By: Codex <267193182+codex@users.noreply.github.com> --- .github/actions/apt-install/action.yml | 29 ++++------- .github/actions/apt-install/install.sh | 66 ++++++++++---------------- 2 files changed, 34 insertions(+), 61 deletions(-) diff --git a/.github/actions/apt-install/action.yml b/.github/actions/apt-install/action.yml index e73ca221c..5915e7316 100644 --- a/.github/actions/apt-install/action.yml +++ b/.github/actions/apt-install/action.yml @@ -1,6 +1,6 @@ name: Install apt packages description: > - Caches apt packages and their dependencies for an exact hosted runner image. Cache hits install offline; cache misses download from Ubuntu mirrors with bounded retries. + Installs apt packages using Ubuntu mirrors and bounded download retries. Caches downloaded packages as an optimization; apt resolves dependencies on every run. inputs: packages: @@ -21,26 +21,18 @@ runs: env: PACKAGES: ${{ inputs.packages }} run: | - # The bundle contains only dependencies missing from this runner. Never - # restore it onto a different image or initial package installation. - # Without an image version (e.g. self-hosted), download normally. - if [ -n "${ImageVersion:-}" ]; then - fingerprint=$({ - printf '%s\n' "$PACKAGES" - dpkg --print-architecture - dpkg-query -W -f='${binary:Package}=${Version} ${db:Status-Abbrev}\n' - sha256sum "$GITHUB_ACTION_PATH/action.yml" "$GITHUB_ACTION_PATH/install.sh" | cut -d ' ' -f 1 - } | sha256sum | cut -d ' ' -f 1) - echo "key=apt-v1-${ImageOS}-${ImageVersion}-${fingerprint}" >> "$GITHUB_OUTPUT" - fi + . /etc/os-release + packages_hash=$(printf '%s' "$PACKAGES" | sha256sum | cut -d ' ' -f 1) + # Cached downloads are optional: apt fetches any missing/newer packages. + # Refresh weekly, including inside job containers without ImageVersion. + echo "key=apt-v2-${ID}-${VERSION_ID}-$(dpkg --print-architecture)-$(date -u +%G-%V)-${packages_hash}" >> "$GITHUB_OUTPUT" - name: Restore apt packages id: cache - if: steps.cache-key.outputs.key != '' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | ${{ runner.temp }}/apt-packages/*.deb - ${{ runner.temp }}/apt-packages/complete + ${{ runner.temp }}/apt-packages/.keep key: ${{ steps.cache-key.outputs.key }} - name: Install ${{ inputs.packages }} shell: bash @@ -49,14 +41,13 @@ runs: ATTEMPTS: ${{ inputs.attempts }} ATTEMPT_TIMEOUT: ${{ inputs.attempt-timeout }} CACHE_DIR: ${{ runner.temp }}/apt-packages - CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} run: bash "$GITHUB_ACTION_PATH/install.sh" - # Save immediately: a later test failure must not discard a good apt bundle. + # Save immediately: a later test failure must not discard downloaded packages. - name: Save apt packages - if: steps.cache-key.outputs.key != '' && steps.cache.outputs.cache-hit != 'true' + if: steps.cache.outputs.cache-hit != 'true' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | ${{ runner.temp }}/apt-packages/*.deb - ${{ runner.temp }}/apt-packages/complete + ${{ runner.temp }}/apt-packages/.keep key: ${{ steps.cache-key.outputs.key }} diff --git a/.github/actions/apt-install/install.sh b/.github/actions/apt-install/install.sh index cb9161183..ae908f412 100644 --- a/.github/actions/apt-install/install.sh +++ b/.github/actions/apt-install/install.sh @@ -1,62 +1,44 @@ #!/usr/bin/env bash set -euo pipefail +# Run as root in the timed child process. Only downloads may be interrupted. +if [ "${1:-}" = download ]; then + download_archive_dir=$2 + shift 2 + apt-get update -q + apt-get install -q --yes --no-install-recommends --download-only \ + -o "Dir::Cache::archives=$download_archive_dir" "$@" + exit 0 +fi + SUDO=() if [ "$(id -u)" -ne 0 ]; then SUDO=(sudo) fi - -# Inputs are package names, separated by whitespace (not shell expressions). read -r -a requested_packages <<< "$PACKAGES" mkdir -p "$CACHE_DIR" +touch "$CACHE_DIR/.keep" -install_local() { - local archives=("$CACHE_DIR"/*.deb) - if [ -e "${archives[0]}" ]; then - # Local .deb arguments pin the downloaded versions. No package lists or - # network access are needed, and apt still runs all package maintainer scripts. - "${SUDO[@]}" chmod a+r "${archives[@]}" - "${SUDO[@]}" apt-get install -q --yes --no-install-recommends \ - --no-download -o "Dir::Cache::archives=$CACHE_DIR" "${archives[@]}" - fi - # An empty bundle is valid when the runner already includes every package. - local package - for package in "${requested_packages[@]}"; do - [ "$(dpkg-query -W -f='${Status}' "$package")" = "install ok installed" ] - done -} - -if [ "$CACHE_HIT" = true ]; then - echo "Installing cached apt packages without contacting package repositories" - install_local - exit 0 -fi - -# The hosted runner gives Azure priority 1. A slow transfer never triggers its -# fallback mirrors, so retrying alone keeps selecting the same unhealthy mirror. +# Hosted runners use a mirror list; job containers can point directly at Azure. +# Handle both layouts, leaving other repositories and signature checks intact. if [ -f /etc/apt/apt-mirrors.txt ]; then "${SUDO[@]}" sed -i '\|azure\.archive\.ubuntu\.com|d' /etc/apt/apt-mirrors.txt fi - -# Download first so the bundle contains every missing transitive dependency. -# Keep downloads outside apt's default cache, which Docker cleanup hooks may wipe. -download_packages() { - "${SUDO[@]}" apt-get update -q --error-on=any - "${SUDO[@]}" apt-get install -q --yes --no-install-recommends \ - --download-only -o "Dir::Cache::archives=$CACHE_DIR" "${requested_packages[@]}" -} -# Recreate the sudo and package arrays in the timed child process. -if [ "${1:-}" = download ]; then - download_packages - exit 0 -fi +for source in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do + if [ -f "$source" ]; then + "${SUDO[@]}" sed -i 's|azure\.archive\.ubuntu\.com|archive.ubuntu.com|g' "$source" + fi +done for attempt in $(seq 1 "$ATTEMPTS"); do echo "::group::apt download attempt ${attempt}/${ATTEMPTS}" - if timeout --kill-after=10 "$ATTEMPT_TIMEOUT" bash "$0" download; then + if timeout --kill-after=10 "$ATTEMPT_TIMEOUT" "${SUDO[@]}" bash "$0" download "$CACHE_DIR" "${requested_packages[@]}"; then echo "::endgroup::" - install_local - touch "$CACHE_DIR/complete" + # Apt handles aliases (e.g. libmagic1 -> libmagic1t64) and dependencies. + "${SUDO[@]}" apt-get install -q --yes --no-install-recommends --no-download \ + -o "Dir::Cache::archives=$CACHE_DIR" "${requested_packages[@]}" + # Allow the unprivileged cache action to read archives downloaded as root. + find "$CACHE_DIR" -maxdepth 1 -name '*.deb' -exec "${SUDO[@]}" chmod a+r {} + exit 0 fi echo "::endgroup::"