ci(build-all): MCU-based groups, cache seeders, build infra overhaul (#27050)

Signed-off-by: Ramon Roche <mrpollo@gmail.com>

.github/workflows/build_all_targets.yml

@@ -69,6 +69,7 @@ jobs:
     runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
+      seeders: ${{ steps.set-seeders.outputs.seeders }}
       timestamp: ${{ steps.set-timestamp.outputs.timestamp }}
       branchname: ${{ steps.set-branch.outputs.branchname }}
     steps:
@@ -82,19 +83,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
 
-      - name: Update python packaging to avoid canonicalize_version() error
-        run: |
-            pip3 install -U packaging
-
       - name: Install Python Dependencies
-        uses: py-actions/py-dependency-install@v4
-        with:
-          path: "./Tools/setup/requirements.txt"
+        run: pip3 install -U packaging -r ./Tools/setup/requirements.txt
 
       - id: set-matrix
         name: Generate Build Matrix
         run: echo "matrix=$(./Tools/ci/generate_board_targets_json.py --group)" >> $GITHUB_OUTPUT
 
+      - id: set-seeders
+        name: Generate Seeder Matrix
+        run: echo "seeders=$(./Tools/ci/generate_board_targets_json.py --group --seeders)" >> $GITHUB_OUTPUT
+
       - id: set-timestamp
         name: Save Current Timestamp
         run: echo "timestamp=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT
@@ -116,11 +115,52 @@ jobs:
           echo "${{ steps.set-branch.outputs.branchname }}"
           echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)"
 
+  # ===========================================================================
+  # CACHE SEEDER JOBS
+  # ===========================================================================
+  # Build one representative target per chip family to warm the ccache.
+  # Matrix jobs fall back to these caches via restore-keys when no
+  # group-specific cache exists yet. If any seeder fails, the build matrix
+  # does not start, catching common build errors early.
+  # ===========================================================================
+
+  seed:
+    name: Seed [${{ matrix.chip_family }}]
+    needs: group_targets
+    runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    strategy:
+      matrix: ${{ fromJson(needs.group_targets.outputs.seeders) }}
+      fail-fast: false
+    container:
+      image: ${{ matrix.container }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: runs-on/action@v2
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+      - name: Configure Git Safe Directory
+        run: git config --system --add safe.directory '*'
+      - uses: ./.github/actions/setup-ccache
+        id: ccache
+        with:
+          cache-key-prefix: ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-seeder
+          max-size: 400M
+      - name: Build seed target
+        run: make ${{ matrix.target }}
+      - uses: ./.github/actions/save-ccache
+        if: always()
+        with:
+          cache-primary-key: ${{ steps.ccache.outputs.cache-primary-key }}
+
   setup:
     name: Build [${{ matrix.runner }}][${{ matrix.group }}]
     # runs-on: ubuntu-latest
-    runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false]
-    needs: group_targets
+    runs-on: [runs-on,"runner=4cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    needs: [group_targets, seed]
+    if: "!failure() && !cancelled()"
     strategy:
       matrix: ${{ fromJson(needs.group_targets.outputs.matrix) }}
       fail-fast: false
@@ -137,34 +177,29 @@ jobs:
       - name: Configure Git Safe Directory
         run: git config --system --add safe.directory '*'
 
-      # ccache key breakdown:
-      # ccache-<system os>-<system arch>-<builder group>-
-      # ccache-<linux>-<arm64>-<aarch64-0>-
-      # ccache-<linux>-<x64>-<nuttx-0>-
-      - name: Cache Restore from Key
-        id: cc_restore
+      - name: Cache - Restore ccache
+        id: ccache-restore
         uses: actions/cache/restore@v5
         with:
           path: ~/.ccache
-          key: ${{ format('ccache-{0}-{1}-{2}', runner.os, matrix.runner, matrix.group) }}
+          key: ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-${{ matrix.group }}-${{ github.ref_name }}-${{ github.sha }}
           restore-keys: |
-            ccache-${{ runner.os }}-${{ matrix.runner }}-${{ matrix.group }}-
-            ccache-${{ runner.os }}-${{ matrix.runner }}-
-            ccache-${{ runner.os }}-${{ matrix.runner }}-
-            ccache-${{ runner.os }}-
-            ccache-
+            ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-${{ matrix.group }}-${{ github.ref_name }}-
+            ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-${{ matrix.group }}-${{ github.base_ref || 'main' }}-
+            ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-${{ matrix.group }}-
+            ccache-${{ matrix.chip_family }}-${{ matrix.runner }}-
 
-      - name: Cache Config and Stats
+      - name: Cache - Configure ccache
         run: |
-            mkdir -p ~/.ccache
-            echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-            echo "compression = true" >> ~/.ccache/ccache.conf
-            echo "compression_level = 6" >> ~/.ccache/ccache.conf
-            echo "max_size = 120M" >> ~/.ccache/ccache.conf
-            echo "hash_dir = false" >> ~/.ccache/ccache.conf
-            echo "compiler_check = content" >> ~/.ccache/ccache.conf
-            ccache -s
-            ccache -z
+          mkdir -p ~/.ccache
+          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
+          echo "compression = true" >> ~/.ccache/ccache.conf
+          echo "compression_level = 6" >> ~/.ccache/ccache.conf
+          echo "max_size = ${{ matrix.cache_size }}" >> ~/.ccache/ccache.conf
+          echo "hash_dir = false" >> ~/.ccache/ccache.conf
+          echo "compiler_check = content" >> ~/.ccache/ccache.conf
+          ccache -s
+          ccache -z
 
       - name: Building Artifacts for [${{ matrix.targets }}]
         run: |
@@ -180,18 +215,10 @@ jobs:
           name: px4_${{matrix.group}}_build_artifacts
           path: artifacts/
 
-      - name: Cache Post Build Stats
-        if: always()
-        run: |
-            ccache -s
-            ccache -z
-
-      - name: Cache Save
-        if: always()
-        uses: actions/cache/save@v5
+      - uses: ./.github/actions/save-ccache
+        if: success()
         with:
-          path: ~/.ccache
-          key: ${{ steps.cc_restore.outputs.cache-primary-key }}
+          cache-primary-key: ${{ steps.ccache-restore.outputs.cache-primary-key }}
 
   # ===========================================================================
   # ARTIFACT UPLOAD JOB

Tools/ci/build_all_config.yml

@@ -0,0 +1,71 @@
+# Build All Targets CI Configuration
+#
+# Controls board grouping, cache sizes, runner specs, and seeder targets
+# for the build_all_targets workflow. Forks can customize this file to
+# adjust for their infrastructure (e.g., lower cache sizes for GitHub's
+# 10GB cache limit, fewer CPU cores for smaller runners).
+
+# Container images
+containers:
+  default: "ghcr.io/px4/px4-dev:v1.17.0-rc2"
+  voxl2: "ghcr.io/px4/px4-dev-voxl2:v1.7"
+
+# Runner specs
+runners:
+  seeder_cpu: 8
+  matrix_cpu: 4
+
+# Default ccache max-size for build groups
+cache:
+  default_size: "400M"
+  # Per-chip overrides for groups with many diverse boards
+  chip_sizes:
+    stm32h7: "800M"
+    stm32f4: "800M"
+    stm32f7: "800M"
+    imxrt: "800M"
+
+# Board grouping limits
+grouping:
+  # Max targets per group, tuned for ~10 min wall-clock with warm cache
+  chip_split_limits:
+    stm32h7: 10
+    stm32f7: 12
+    stm32f4: 20
+    stm32f1: 39
+    imxrt: 12
+    kinetis: 14
+    s32k: 17
+    rp2040: 10
+    special: 10
+    native: 17
+  default_split_limit: 12
+  # Minimum targets for a manufacturer to get a named group
+  lower_limit: 3
+  # If last chunk has fewer targets than this, merge into previous chunk
+  merge_back_threshold: 5
+
+# Labels that isolate builds into the "special" group
+special_labels:
+  - lto
+  - protected
+
+# NXP chip families are pooled under "nxp-{chip}" regardless of board directory
+nxp_chip_families:
+  - imxrt
+  - kinetis
+  - s32k
+
+# Seeder targets: one representative build per chip family
+seeders:
+  stm32h7: "px4_fmu-v6x_default"
+  stm32f7: "px4_fmu-v5_default"
+  stm32f4: "px4_fmu-v4_default"
+  stm32f1: "px4_io-v2_default"
+  imxrt: "nxp_mr-tropic_default"
+  kinetis: "nxp_fmuk66-v3_default"
+  s32k: "nxp_mr-canhubk3_default"
+  rp2040: "raspberrypi_pico_default"
+  special: "px4_fmu-v6x_default"
+  native: "px4_sitl_default"
+  voxl2: "modalai_voxl2_default"

Tools/ci/build_all_runner.sh

@@ -3,7 +3,6 @@
 # Please only modify if you know what you are doing
 set -e
 
-echo "### :clock1: Build Times" >> $GITHUB_STEP_SUMMARY
 targets=$1
 for target in ${targets//,/ }
 do
@@ -14,6 +13,5 @@ do
     diff=$(($stop-$start))
     build_time="$(($diff /60/60))h $(($diff /60))m $(($diff % 60))s elapsed"
     echo -e "\033[0;32mBuild Time: [$build_time]"
-    echo "* **$target** - $build_time" >> $GITHUB_STEP_SUMMARY
     echo "::endgroup::"
 done