diff --git a/.github/workflows/build_all_targets.yml b/.github/workflows/build_all_targets.yml index 94ae292b6c2..8be856b24a7 100644 --- a/.github/workflows/build_all_targets.yml +++ b/.github/workflows/build_all_targets.yml @@ -66,13 +66,16 @@ jobs: group_targets: name: Scan for Board Targets # runs-on: ubuntu-latest - runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache] outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} seeders: ${{ steps.set-seeders.outputs.seeders }} + cold_seeders: ${{ steps.filter-seeders.outputs.cold_seeders }} + has_cold: ${{ steps.filter-seeders.outputs.has_cold }} timestamp: ${{ steps.set-timestamp.outputs.timestamp }} branchname: ${{ steps.set-branch.outputs.branchname }} steps: + - uses: runs-on/action@v2 - uses: actions/checkout@v6 - name: Cache Python pip @@ -109,6 +112,15 @@ jobs: echo "EOF" } >> "$GITHUB_OUTPUT" + # Seed only chip families whose ccache namespace has no warm cache. + # Fail-open semantics live in the script: any probe problem emits + # the full matrix, reproducing unconditional seeding. + - id: filter-seeders + name: Probe Family Caches + env: + ALL_SEEDERS: ${{ steps.set-seeders.outputs.seeders }} + run: ./Tools/ci/filter_cold_seeders.py <<< "$ALL_SEEDERS" + - id: set-timestamp name: Save Current Timestamp run: | @@ -133,20 +145,25 @@ jobs: echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)" # =========================================================================== - # CACHE SEEDER JOBS + # CACHE SEEDER JOBS (conditional) # =========================================================================== - # Build one representative target per chip family to warm the ccache. - # Matrix jobs fall back to these caches via restore-keys when no - # group-specific cache exists yet. If any seeder fails, the build matrix - # does not start, catching common build errors early. + # Build one representative target per chip family to warm the ccache, + # but only for families whose cache namespace came up cold in the + # group_targets probe. Measured steady state is zero seeder restores + # (group jobs live on their own rolling caches), so on warm runs the + # whole stage skips and the build matrix starts immediately. When a + # family IS cold (matrix regrouping, cache expiry, new chip family), + # its seeder runs and still gates the matrix, which is the only case + # where the warmup is worth waiting for. # =========================================================================== seed: name: Seed [${{ matrix.chip_family }}] needs: group_targets + if: needs.group_targets.outputs.has_cold == 'true' runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false,extras=s3-cache] strategy: - matrix: ${{ fromJson(needs.group_targets.outputs.seeders) }} + matrix: ${{ fromJson(needs.group_targets.outputs.cold_seeders) }} fail-fast: false container: image: ${{ matrix.container }} @@ -163,7 +180,7 @@ jobs: - uses: ./.github/actions/setup-ccache id: ccache with: - cache-key-prefix: ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-seeder + cache-key-prefix: ${{ matrix.cache_prefix }}-${{ matrix.group }} max-size: 400M - name: Build seed target run: make ${{ matrix.target }} @@ -199,12 +216,12 @@ jobs: uses: actions/cache/restore@v5 with: path: ~/.ccache - key: ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-${{ matrix.group }}-${{ github.ref_name }}-${{ github.sha }} + key: ${{ matrix.cache_prefix }}-${{ matrix.group }}-${{ github.ref_name }}-${{ github.sha }} restore-keys: | - ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-${{ matrix.group }}-${{ github.ref_name }}- - ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-${{ matrix.group }}-${{ github.base_ref || 'main' }}- - ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-${{ matrix.group }}- - ccache-${{ matrix.chip_family }}-${{ matrix.runner }}- + ${{ matrix.cache_prefix }}-${{ matrix.group }}-${{ github.ref_name }}- + ${{ matrix.cache_prefix }}-${{ matrix.group }}-${{ github.base_ref || 'main' }}- + ${{ matrix.cache_prefix }}-${{ matrix.group }}- + ${{ matrix.cache_prefix }}- - name: Cache - Configure ccache run: | diff --git a/Tools/ci/filter_cold_seeders.py b/Tools/ci/filter_cold_seeders.py new file mode 100755 index 00000000000..b9d6ad776f3 --- /dev/null +++ b/Tools/ci/filter_cold_seeders.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Emit the seeder matrix filtered to chip families with no warm ccache. + +Reads the full seeder matrix JSON on stdin and writes two GitHub Actions +outputs (to $GITHUB_OUTPUT, or stdout when unset for local runs): + + cold_seeders matrix containing only the families with no cache entry + under their ccache namespace in the RunsOn magic-cache + S3 bucket + has_cold "true"/"false", whether cold_seeders has any entries + +Fails open, never non-zero: on any probe problem (missing bucket env, +aws cli error, unexpected input) the full input matrix is emitted with +has_cold=true, reproducing unconditional seeding. The probe can only +skip work, never leave a truly cold family unseeded. +""" +import json +import os +import subprocess +import sys + + +def scope_key_basenames(bucket, scope): + """Cache-key basenames in one ref scope of the magic-cache bucket. + + The magic cache stores GitHub cache entries as + cache/v1/{org}/{repo}/{ref}/{version-hash}/{cache-key}; the version + hash is the actions/cache path+compression digest and not knowable + here, so list the whole ref scope (auto-paginated) and match on the + final path component. + """ + prefix = f"cache/v1/{os.environ['GITHUB_REPOSITORY']}/{scope}/" + result = subprocess.run( + ["aws", "s3", "ls", f"s3://{bucket}/{prefix}", "--recursive"], + capture_output=True, text=True, timeout=120) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip()) + return [line.split()[-1].rsplit("/", 1)[-1] + for line in result.stdout.splitlines() if line.strip()] + + +def filter_cold(matrix): + bucket = os.environ["RUNS_ON_S3_BUCKET_CACHE"] + # Default-branch caches are visible to every ref, so main-scope + # warmth is what the build jobs' last restore-keys fallback can + # always reach regardless of the branch being built. + scope = os.environ.get("SEEDER_PROBE_SCOPE", "refs/heads/main") + basenames = scope_key_basenames(bucket, scope) + cold = [] + for entry in matrix["include"]: + # The namespace root is minted by generate_board_targets_json.py + # and arrives on every matrix entry; no key format knowledge here. + prefix = entry["cache_prefix"] + "-" + warm = any(name.startswith(prefix) for name in basenames) + print(f"::notice title=seeder probe::{entry['chip_family']}/" + f"{entry['runner']}: {'warm' if warm else 'COLD'}", file=sys.stderr) + if not warm: + cold.append(entry) + return {"include": cold} + + +def write_outputs(cold_json, has_cold): + payload = (f"cold_seeders<