ci: replace build workflows (#23550)

.editorconfig

@@ -9,6 +9,6 @@ tab_width = 8
 # Not in the official standard, but supported by many editors
 max_line_length = 120
 
-[*.yaml]
+[*.yaml, *.yml]
 indent_style = space
 indent_size = 2

.github/workflows/build_all_targets.yml

@@ -0,0 +1,83 @@
+name: Build all targets
+
+on:
+  push:
+    branches:
+    - 'main'
+    - 'stable'
+    - 'beta'
+    - 'release/*'
+  pull_request:
+    branches:
+    - '*'
+
+jobs:
+  group_targets:
+    name: Scan for Board Targets
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      timestamp: ${{ steps.set-timestamp.outputs.timestamp }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Python Dependencies
+        uses: py-actions/py-dependency-install@v4
+        with:
+          path: "./Tools/setup/requirements.txt"
+
+      - id: set-matrix
+        run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py --group)"
+
+      - id: set-timestamp
+        run: echo "::set-output name=timestamp::$(date +"%Y%m%d%H%M%S")"
+
+  setup:
+    name: ${{ matrix.group }}
+    runs-on: ubuntu-latest
+    needs: group_targets
+    strategy:
+      matrix: ${{ fromJson(needs.group_targets.outputs.matrix) }}
+    container:
+      image: ${{ matrix.container }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ownership workaround
+        run: git config --system --add safe.directory '*'
+
+      - name: ccache setup keys
+        uses: actions/cache@v4
+        with:
+          path: ~/.ccache
+          key: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }}
+          restore-keys: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }}
+
+      - name: setup ccache
+        run: |
+            mkdir -p ~/.ccache
+            echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
+            echo "compression = true" >> ~/.ccache/ccache.conf
+            echo "compression_level = 6" >> ~/.ccache/ccache.conf
+            echo "max_size = 120M" >> ~/.ccache/ccache.conf
+            echo "hash_dir = false" >> ~/.ccache/ccache.conf
+            ccache -s
+            ccache -z
+
+      - name: build target group
+        run: |
+            ./Tools/ci_build_all_runner.sh ${{matrix.targets}}
+
+      - name: Upload px4 package
+        uses: actions/upload-artifact@v4
+        with:
+          name: px4_${{matrix.group}}_build_artifacts
+          path: |
+            build/**/*.px4
+            build/**/*.bin
+          compression-level: 0
+
+      - name: ccache post-run
+        run: ccache -s

.github/workflows/compile_linux.yml

@@ -1,58 +0,0 @@
-name: Compile Linux Targets
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'stable'
-    - 'beta'
-    - 'release/*'
-  pull_request:
-    branches:
-    - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-armhf:2023-06-26
-    strategy:
-      matrix:
-        config: [
-          beaglebone_blue_default,
-          emlid_navio2_default,
-          px4_raspberrypi_default,
-          scumaker_pilotpi_default,
-          ]
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-    - name: ownership workaround
-      run: git config --system --add safe.directory '*'
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        message("::set-output name=timestamp::${current_date}")
-    - name: ccache cache files
-      uses: actions/cache@v2
-      with:
-        path: ~/.ccache
-        key: ${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: ${{matrix.config}}-ccache-
-    - name: setup ccache
-      run: |
-          mkdir -p ~/.ccache
-          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-          echo "compression = true" >> ~/.ccache/ccache.conf
-          echo "compression_level = 6" >> ~/.ccache/ccache.conf
-          echo "max_size = 100M" >> ~/.ccache/ccache.conf
-          echo "hash_dir = false" >> ~/.ccache/ccache.conf
-          ccache -s
-          ccache -z
-
-    - name: make ${{matrix.config}}
-      run: make ${{matrix.config}}
-    - name: ccache post-run
-      run: ccache -s

.github/workflows/compile_linux_arm64.yml

@@ -1,54 +0,0 @@
-name: Compile Linux ARM64 Targets
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'stable'
-    - 'beta'
-    - 'release/*'
-  pull_request:
-    branches:
-    - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-aarch64:2022-08-12
-    strategy:
-      matrix:
-        config: [
-          scumaker_pilotpi_arm64,
-          ]
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        message("::set-output name=timestamp::${current_date}")
-    - name: ccache cache files
-      uses: actions/cache@v2
-      with:
-        path: ~/.ccache
-        key: ${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: ${{matrix.config}}-ccache-
-    - name: setup ccache
-      run: |
-          mkdir -p ~/.ccache
-          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-          echo "compression = true" >> ~/.ccache/ccache.conf
-          echo "compression_level = 6" >> ~/.ccache/ccache.conf
-          echo "max_size = 100M" >> ~/.ccache/ccache.conf
-          echo "hash_dir = false" >> ~/.ccache/ccache.conf
-          ccache -s
-          ccache -z
-
-    - name: make ${{matrix.config}}
-      run: make ${{matrix.config}}
-    - name: ccache post-run
-      run: ccache -s

.github/workflows/compile_nuttx.yml

@@ -1,137 +0,0 @@
-name: Compile Nuttx Targets
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'stable'
-    - 'beta'
-    - 'release/*'
-  pull_request:
-    branches:
-    - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-nuttx-focal:2022-08-12
-    strategy:
-      fail-fast: false
-      matrix:
-        config: [
-          3dr_ctrl-zero-h7-oem-revg,
-          airmind_mindpx-v2,
-          ark_can-flow,
-          ark_can-gps,
-          ark_can-rtk-gps,
-          ark_cannode,
-          ark_fmu-v6x,
-          ark_pi6x,
-          ark_septentrio-gps,
-          atl_mantis-edu,
-          av_x-v1,
-          bitcraze_crazyflie,
-          bitcraze_crazyflie21,
-          cuav_can-gps-v1,
-          cuav_nora,
-          cuav_x7pro,
-          cubepilot_cubeorange,
-          cubepilot_cubeorangeplus,
-          cubepilot_cubeyellow,
-          diatone_mamba-f405-mk2,
-          freefly_can-rtk-gps,
-          holybro_can-gps-v1,
-          holybro_durandal-v1,
-          holybro_kakutef7,
-          holybro_kakuteh7,
-          holybro_pix32v5,
-          matek_gnss-m9n-f4,
-          matek_h743,
-          matek_h743-mini,
-          matek_h743-slim,
-          modalai_fc-v1,
-          modalai_fc-v2,
-          mro_ctrl-zero-classic,
-          mro_ctrl-zero-f7,
-          mro_ctrl-zero-f7-oem,
-          mro_ctrl-zero-h7,
-          mro_ctrl-zero-h7-oem,
-          mro_pixracerpro,
-          mro_x21,
-          mro_x21-777,
-          nxp_fmuk66-e,
-          nxp_fmuk66-v3,
-          nxp_mr-canhubk3,
-          nxp_ucans32k146,
-          omnibus_f4sd,
-          px4_fmu-v2,
-          px4_fmu-v3,
-          px4_fmu-v4,
-          px4_fmu-v4pro,
-          px4_fmu-v5,
-          px4_fmu-v5x,
-          px4_fmu-v6c,
-          px4_fmu-v6u,
-          px4_fmu-v6x,
-          px4_fmu-v6xrt,
-          raspberrypi_pico,
-          sky-drones_smartap-airlink,
-          spracing_h7extreme,
-          uvify_core,
-          siyi_n7
-          ]
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        message("::set-output name=timestamp::${current_date}")
-    - name: ccache cache files
-      uses: actions/cache@v2
-      with:
-        path: ~/.ccache
-        key: ${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: ${{matrix.config}}-ccache-
-    - name: setup ccache
-      run: |
-          mkdir -p ~/.ccache
-          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-          echo "compression = true" >> ~/.ccache/ccache.conf
-          echo "compression_level = 6" >> ~/.ccache/ccache.conf
-          echo "max_size = 120M" >> ~/.ccache/ccache.conf
-          echo "hash_dir = false" >> ~/.ccache/ccache.conf
-          ccache -s
-          ccache -z
-
-    - name: make all_variants_${{matrix.config}}
-      run: make all_variants_${{matrix.config}}
-      timeout-minutes: 45
-    - name: make ${{matrix.config}} bloaty_compileunits
-      run: make ${{matrix.config}} bloaty_compileunits || true
-    - name: make ${{matrix.config}} bloaty_inlines
-      run: make ${{matrix.config}} bloaty_inlines || true
-    - name: make ${{matrix.config}} bloaty_segments
-      run: make ${{matrix.config}} bloaty_segments || true
-    - name: make ${{matrix.config}} bloaty_symbols
-      run: make ${{matrix.config}} bloaty_symbols || true
-    - name: make ${{matrix.config}} bloaty_templates
-      run: make ${{matrix.config}} bloaty_templates || true
-    - name: make ${{matrix.config}} bloaty_ram
-      run: make ${{matrix.config}} bloaty_ram || true
-    - name: make ${{matrix.config}} bloaty_compare_master
-      run: make ${{matrix.config}} bloaty_compare_master || true
-    - name: ccache post-run
-      run: ccache -s
-
-    - name: Upload px4 package
-      uses: actions/upload-artifact@v2
-      with:
-        name: px4_package_${{matrix.config}}
-        path: |
-          build/**/*.px4
-          build/**/*.bin

Tools/ci_build_all_runner.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# This script is meant to be used by the build_all.yml workflow in a github runner
+# Please only modify if you know what you are doing
+set -e
+
+echo "### :clock1: Build Times" >> $GITHUB_STEP_SUMMARY
+targets=$1
+for target in ${targets//,/ }
+do
+    echo "::group::Building: [${target}]"
+    start=$(date +%s)
+    make $target
+    stop=$(date +%s)
+    diff=$(($stop-$start))
+    build_time="$(($diff /60/60))h $(($diff /60))m $(($diff % 60))s elapsed"
+    echo -e "\033[0;32mBuild Time: [$build_time]"
+    echo "* **$target** - $build_time" >> $GITHUB_STEP_SUMMARY
+    echo "::endgroup::"
+done

Tools/generate_board_targets_json.py

@@ -23,11 +23,14 @@ parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                     help='Verbose Output')
 parser.add_argument('-p', '--pretty', dest='pretty', action='store_true',
                     help='Pretty output instead of a single line')
+parser.add_argument('-g', '--groups', dest='group', action='store_true',
+                    help='Groups targets')
 
 args = parser.parse_args()
 verbose = args.verbose
 
 build_configs = []
+grouped_targets = {}
 excluded_boards = ['modalai_voxl2', 'px4_ros2']  # TODO: fix and enable
 excluded_manufacturers = ['atlflight']
 excluded_platforms = ['qurt']
@@ -37,10 +40,27 @@ excluded_labels = [
     'uavcanv1', # TODO: fix and enable
     ]
 
+github_action_config = { 'include': build_configs }
+extra_args = {}
+if args.pretty:
+    extra_args['indent'] = 2
+
+def chunks(arr, size):
+    # splits array into parts
+    for i in range(0, len(arr), size):
+        yield arr[i:i + size]
+
+def comma_targets(targets):
+    # turns array of targets into a comma split string
+    return ",".join(targets)
+
 def process_target(px4board_file, target_name):
+    # reads through the board file and grabs
+    # useful information for building
     ret = None
     platform = None
     toolchain = None
+    group = None
 
     if px4board_file.endswith("default.px4board") or \
         px4board_file.endswith("recovery.px4board") or \
@@ -63,22 +83,34 @@ def process_target(px4board_file, target_name):
         # get the container based on the platform and toolchain
         if platform == 'posix':
             container = 'px4io/px4-dev-base-focal:2021-09-08'
+            group = 'base'
             if toolchain:
                 if toolchain.startswith('aarch64'):
                     container = 'px4io/px4-dev-aarch64:2022-08-12'
+                    group = 'aarch64'
                 elif toolchain == 'arm-linux-gnueabihf':
                     container = 'px4io/px4-dev-armhf:2023-06-26'
+                    group = 'armhf'
                 else:
                     if verbose: print(f'unmatched toolchain: {toolchain}')
         elif platform == 'nuttx':
             container = 'px4io/px4-dev-nuttx-focal:2022-08-12'
+            group = 'nuttx'
         else:
             if verbose: print(f'unmatched platform: {platform}')
 
         ret = {'target': target_name, 'container': container}
+        if(args.group):
+            ret['arch'] = group
 
     return ret
 
+# Look for board targets in the ./boards directory
+if(verbose):
+    print("=======================")
+    print("= scanning for boards =")
+    print("=======================")
+
 for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
     if not manufacturer.is_dir():
         continue
@@ -105,12 +137,140 @@ for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
                     if verbose: print(f'excluding label {label} ({target_name})')
                     continue
                 target = process_target(files.path, target_name)
+                if (args.group and target is not None):
+                    if (target['arch'] not in grouped_targets):
+                        grouped_targets[target['arch']] = {}
+                        grouped_targets[target['arch']]['container'] = target['container']
+                        grouped_targets[target['arch']]['manufacturers'] = {}
+                    if(manufacturer.name not in grouped_targets[target['arch']]['manufacturers']):
+                        grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = {}
+                        grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = []
+                    grouped_targets[target['arch']]['manufacturers'][manufacturer.name].append(target_name)
                 if target is not None:
                     build_configs.append(target)
 
+if(verbose):
+    import pprint
+    print("============================")
+    print("= Boards found in ./boards =")
+    print("============================")
+    pprint.pp(grouped_targets)
 
-github_action_config = { 'include': build_configs }
+if (args.group):
-extra_args = {}
+    # if we are using this script for grouping builds
-if args.pretty:
+    # we loop trough the manufacturers list and split their targets
-    extra_args['indent'] = 2
+    # if a manufacturer has more than a LIMIT of boards then we split that
-print(json.dumps(github_action_config, **extra_args))
+    # into sub groups such as "arch-manufacturer name-index"
+    # example:
+    #   nuttx-px4-0
+    #   nuttx-px4-1
+    #   nuttx-px4-2
+    #   nuttx-ark-0
+    #   nuttx-ark-1
+    # if the manufacturer doesn't have more targets than LIMIT then we add
+    # them to a generic group with the following structure "arch-index"
+    # example:
+    #   nuttx-0
+    #   nuttx-1
+    final_groups = []
+    temp_group = []
+    group_number = {}
+    last_man = ''
+    last_arch = ''
+    SPLIT_LIMIT = 10
+    LOWER_LIMIT = 5
+    for arch in grouped_targets:
+        if(last_arch == ''):
+            last_arch = arch
+        if(arch not in group_number):
+                group_number[arch] = 0
+
+        if(last_arch != arch and len(temp_group) > 0):
+            group_name = last_arch + "-" + str(group_number[last_arch])
+            group_number[last_arch] += 1
+            targets = comma_targets(temp_group)
+            final_groups.append({
+                "container": grouped_targets[last_arch]['container'],
+                "targets": targets,
+                "arch": last_arch,
+                "group": group_name,
+                "len": len(temp_group)
+            })
+            last_arch = arch
+            temp_group = []
+        for man in grouped_targets[arch]['manufacturers']:
+            for tar in grouped_targets[arch]['manufacturers'][man]:
+                if(last_man != man):
+                    man_len = len(grouped_targets[arch]['manufacturers'][man])
+                    if(man_len > LOWER_LIMIT and man_len < (SPLIT_LIMIT + 1)):
+                        # Manufacturers can have their own group
+                        group_name = arch + "-" + man
+                        targets = comma_targets(grouped_targets[arch]['manufacturers'][man])
+                        last_man = man
+                        final_groups.append({
+                            "container": grouped_targets[arch]['container'],
+                            "targets": targets,
+                            "arch": arch,
+                            "group": group_name,
+                            "len": len(grouped_targets[arch]['manufacturers'][man])
+                        })
+                    elif(man_len >= (SPLIT_LIMIT + 1)):
+                        # Split big man groups into subgroups
+                        # example: Pixhawk
+                        chunk_limit = SPLIT_LIMIT
+                        chunk_counter = 0
+                        for chunk in chunks(grouped_targets[arch]['manufacturers'][man], chunk_limit):
+                            group_name = arch + "-" + man + "-" + str(chunk_counter)
+                            targets = comma_targets(chunk)
+                            last_man = man
+                            final_groups.append({
+                                "container": grouped_targets[arch]['container'],
+                                "targets": targets,
+                                "arch": arch,
+                                "group": group_name,
+                                "len": len(chunk),
+                            })
+                            chunk_counter += 1
+                    else:
+                        temp_group.append(tar)
+
+            if(last_arch != arch and len(temp_group) > 0):
+                group_name = last_arch + "-" + str(group_number[last_arch])
+                group_number[last_arch] += 1
+                targets = comma_targets(temp_group)
+                final_groups.append({
+                    "container": grouped_targets[last_arch]['container'],
+                    "targets": targets,
+                    "arch": last_arch,
+                    "group": group_name,
+                    "len": len(temp_group)
+                })
+                last_arch = arch
+                temp_group = []
+            if(len(temp_group) > (LOWER_LIMIT - 1)):
+                group_name = arch + "-" + str(group_number[arch])
+                last_arch = arch
+                group_number[arch] += 1
+                targets = comma_targets(temp_group)
+                final_groups.append({
+                    "container": grouped_targets[arch]['container'],
+                    "targets": targets,
+                    "arch": arch,
+                    "group": group_name,
+                    "len": len(temp_group)
+                })
+                temp_group = []
+    if(verbose):
+        import pprint
+        print("================")
+        print("= final_groups =")
+        print("================")
+        pprint.pp(final_groups)
+
+        print("===============")
+        print("= JSON output =")
+        print("===============")
+
+    print(json.dumps({ "include": final_groups }, **extra_args))
+else:
+    print(json.dumps(github_action_config, **extra_args))