ci: Upload Artifacts to S3

- replaces metadata and deploy workflows
- move ci scripts under Tools/ci

.github/workflows/build_all_targets.yml

@@ -32,6 +32,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       timestamp: ${{ steps.set-timestamp.outputs.timestamp }}
       tagname: ${{ steps.set-tag.outputs.tagname }}
+      branchname: ${{ steps.set-branch.outputs.branchname }}
     steps:
       - uses: actions/checkout@v4
 
@@ -41,7 +42,7 @@ jobs:
           path: "./Tools/setup/requirements.txt"
 
       - id: set-matrix
-        run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py --group)"
+        run: echo "::set-output name=matrix::$(./Tools/ci/generate_board_targets_json.py --group)"
 
       - id: set-timestamp
         run: echo "::set-output name=timestamp::$(date +"%Y%m%d%H%M%S")"
@@ -60,6 +61,13 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
         run: echo "::set-output name=tagname::${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}"
 
+      - id: set-branch
+        run: echo "::set-output name=branchname::${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+
+      - name: Debug Matrix Output
+        if: runner.debug == '1'
+        run: echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)"
+
   setup:
     name: Build Group [${{ matrix.group }}]
     # runs-on: ubuntu-latest
@@ -74,17 +82,17 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: ownership workaround
+      - name: Git ownership workaround
         run: git config --system --add safe.directory '*'
 
-      - name: ccache setup keys
+      - name: Setup ccache
         uses: actions/cache@v4
         with:
           path: ~/.ccache
           key: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }}
           restore-keys: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }}
 
-      - name: setup ccache
+      - name: Configure ccache
         run: |
             mkdir -p ~/.ccache
             echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
@@ -95,24 +103,54 @@ jobs:
             ccache -s
             ccache -z
 
-      - name: building [${{ matrix.group }}]
+      - name: Building [${{ matrix.group }}]
         run: |
-            ./Tools/ci_build_all_runner.sh ${{matrix.targets}}
+            ./Tools/ci/build_all_runner.sh ${{matrix.targets}}
 
-      - name: Upload px4 package
+      - name: Arrange Build Artifacts
+        run: |
+          ./Tools/ci/package_build_artifacts.sh
+
+      - name: Upload Build Artifacts
         uses: actions/upload-artifact@v4
         with:
           name: px4_${{matrix.group}}_build_artifacts
-          path: |
-            build/**/*.px4
-            build/**/*.bin
-          compression-level: 0
+          path: artifacts/
 
-      - name: ccache post-run
+      - name: Cache Save
         run: ccache -s
 
+  artifacts:
+    name: Upload Artifacts to S3
+    # runs-on: ubuntu-latest
+    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}"]
+    needs: [setup, group_targets]
+    if: contains(fromJSON('["main", "stable", "beta"]'), needs.group_targets.outputs.branchname)
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts/
+          merge-multiple: true
+
+      - name: Branch Name
+        run: |
+          echo "${{ needs.group_targets.outputs.branchname }}"
+
+      - name: Uploading Artifacts to S3 [${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}]
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --acl public-read
+        env:
+          AWS_S3_BUCKET: 'px4-travis'
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: 'us-west-1'
+          SOURCE_DIR: artifacts/
+          DEST_DIR: Firmware/${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}/
+
   release:
-    name: Release and Upload
+    name: Create Release and Upload Artifacts
     # runs-on: ubuntu-latest
     runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}"]
     needs: [setup, group_targets]
@@ -126,7 +164,7 @@ jobs:
           mkdir artifacts
           cp **/**/*.px4 artifacts/
 
-      - name: Upload Binaries
+      - name: Upload Binaries to Release
         uses: softprops/action-gh-release@v2
         with:
           name: ${{ needs.group_targets.outputs.tagname }}

.github/workflows/deploy_all.yml

@@ -1,56 +0,0 @@
-name: Deploy metadata for all targets
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'release/*'
-    - 'pr-metadata-test'
-
-jobs:
-  enumerate_targets:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-    - id: set-matrix
-      run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py)"
-  build:
-    runs-on: ubuntu-latest
-    needs: enumerate_targets
-    strategy:
-      matrix: ${{fromJson(needs.enumerate_targets.outputs.matrix)}}
-    container: ${{ matrix.container }}
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-
-    - name: ownership workaround
-      run: git config --system --add safe.directory '*'
-
-    - name: make ${{matrix.target}}
-      run: make ${{matrix.target}}
-
-    - name: parameter & events metadata
-      run: |
-        make ${{matrix.target}} ver_gen events_json actuators_json
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/${{ matrix.target }} >> $GITHUB_ENV
-        cd build/${{ matrix.target }}
-        mkdir _metadata || true
-        cp parameters.* events/*.xz actuators.json* _metadata
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/${{ matrix.target }}/_metadata/'
-        DEST_DIR: 'Firmware/${{ env.version }}/${{ matrix.target }}/'

.github/workflows/metadata.yml

@@ -1,133 +0,0 @@
-name: Metadata
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'release/*'
-    - 'pr-metadata-test'
-
-jobs:
-
-  airframe:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: airframe metadata
-      run: |
-        make airframe_metadata
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV
-        cd build/px4_sitl_default/docs
-      # TODO: deploy to userguide gitbook
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/px4_sitl_default/docs/'
-        DEST_DIR: 'Firmware/${{ env.version }}/_general/'
-
-  module:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: module documentation
-      run: |
-        make module_documentation
-        cd build/px4_sitl_default/docs
-        ls -ls *
-      # TODO: deploy to userguide gitbook and s3
-
-  parameter:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: parameter metadata
-      run: |
-        make parameters_metadata
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/px4_sitl_default/docs/'
-        DEST_DIR: 'Firmware/${{ env.version }}/_general/'
-
-  events:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: events metadata
-      run: |
-        make extract_events
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV
-        cd build/px4_sitl_default
-        mkdir _events_full || true
-        cp events/all_events_full.json.xz _events_full/all_events.json.xz
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/px4_sitl_default/_events_full/'
-        DEST_DIR: 'Firmware/${{ env.version }}/_general/'
-
-  uorb_graph:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-nuttx-focal:2022-08-12
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: uORB graph
-      run: |
-        make uorb_graphs
-        cd Tools/uorb_graph
-        ls -ls *
-      # TODO: deploy graph_px4_sitl.json to S3 px4-travis:Firmware/master/
-
-  ROS2_msgs:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: PX4 ROS2 msgs
-      run: |
-        git clone https://github.com/PX4/px4_msgs.git
-        rm px4_msgs/msg/*.msg
-        rm px4_msgs/srv/*.srv
-        cp msg/*.msg px4_msgs/msg/
-        cp srv/*.srv px4_msgs/srv/

Tools/ci/generate_board_targets_json.py

@@ -111,7 +111,19 @@ if(verbose):
     print("= scanning for boards =")
     print("=======================")
 
-for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
+# We also need to build metadata
+# includes:
+# - Airframe
+# - Parameters
+# - Events
+metadata_targets = ['airframe_metadata', 'parameters_metadata', 'extract_events']
+grouped_targets['base'] = {}
+grouped_targets['base']['container'] = 'px4io/px4-dev-base-focal:2021-09-08'
+grouped_targets['base']['manufacturers'] = {}
+grouped_targets['base']['manufacturers']['px4'] = []
+grouped_targets['base']['manufacturers']['px4'] += metadata_targets
+
+for manufacturer in os.scandir(os.path.join(source_dir, '../boards')):
     if not manufacturer.is_dir():
         continue
     if manufacturer.name in excluded_manufacturers:
@@ -143,7 +155,6 @@ for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
                         grouped_targets[target['arch']]['container'] = target['container']
                         grouped_targets[target['arch']]['manufacturers'] = {}
                     if(manufacturer.name not in grouped_targets[target['arch']]['manufacturers']):
-                        grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = {}
                         grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = []
                     grouped_targets[target['arch']]['manufacturers'][manufacturer.name].append(target_name)
                 if target is not None:
@@ -156,6 +167,11 @@ if(verbose):
     print("============================")
     pprint.pp(grouped_targets)
 
+if(verbose):
+    print("===================")
+    print("= Generating JSON =")
+    print("===================")
+
 if (args.group):
     # if we are using this script for grouping builds
     # we loop trough the manufacturers list and split their targets
@@ -179,16 +195,24 @@ if (args.group):
     last_arch = ''
     SPLIT_LIMIT = 10
     LOWER_LIMIT = 5
+    if(verbose):
+        print(f'=:Architectures: [{grouped_targets.keys()}]')
     for arch in grouped_targets:
+        if(verbose):
+            print(f'=:Processing: [{arch}] Last: [{last_arch}]')
+
         if(last_arch == ''):
             last_arch = arch
         if(arch not in group_number):
                 group_number[arch] = 0
 
         if(last_arch != arch and len(temp_group) > 0):
+
             group_name = last_arch + "-" + str(group_number[last_arch])
             group_number[last_arch] += 1
             targets = comma_targets(temp_group)
+            if(verbose):
+                print(f'=:Orphan: [{arch}][{last_arch}][{targets}]')
             final_groups.append({
                 "container": grouped_targets[last_arch]['container'],
                 "targets": targets,
@@ -198,12 +222,22 @@ if (args.group):
             })
             last_arch = arch
             temp_group = []
+
         for man in grouped_targets[arch]['manufacturers']:
+            if(verbose):
+                print(f'=:Processing: [{arch}][{man}]')
             for tar in grouped_targets[arch]['manufacturers'][man]:
+                man_len = len(grouped_targets[arch]['manufacturers'][man])
+                if(verbose):
+                    print(f'=:Processing: [{arch}][{man}][{man_len}][{tar}]')
                 if(last_man != man):
-                    man_len = len(grouped_targets[arch]['manufacturers'][man])
+                    # if(verbose):
+                    #     print(f'=:Processing: [{arch}][{man}][{tar}][{man_len}]')
                     if(man_len > LOWER_LIMIT and man_len < (SPLIT_LIMIT + 1)):
                         # Manufacturers can have their own group
+                        if(verbose):
+                            print(f'=:Processing: ==Manufacturers can have their own group')
+                            print(f'=:Processing: Limits[{LOWER_LIMIT}][{SPLIT_LIMIT}]')
                         group_name = arch + "-" + man
                         targets = comma_targets(grouped_targets[arch]['manufacturers'][man])
                         last_man = man
@@ -234,19 +268,6 @@ if (args.group):
                     else:
                         temp_group.append(tar)
 
-            if(last_arch != arch and len(temp_group) > 0):
-                group_name = last_arch + "-" + str(group_number[last_arch])
-                group_number[last_arch] += 1
-                targets = comma_targets(temp_group)
-                final_groups.append({
-                    "container": grouped_targets[last_arch]['container'],
-                    "targets": targets,
-                    "arch": last_arch,
-                    "group": group_name,
-                    "len": len(temp_group)
-                })
-                last_arch = arch
-                temp_group = []
             if(len(temp_group) > (LOWER_LIMIT - 1)):
                 group_name = arch + "-" + str(group_number[arch])
                 last_arch = arch

Tools/ci/package_build_artifacts.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+mkdir artifacts
+cp **/**/*.px4 artifacts/
+cp **/**/*.elf artifacts/
+for build_dir_path in build/*/ ; do
+  build_dir=${build_dir_path#*/}
+  build_dir=${build_dir::${#build_dir}-1}
+  mkdir artifacts/$build_dir
+  find artifacts/ -maxdepth 1 -type f -name "*$build_dir*"
+  # Airframe
+  cp $build_dir_path/airframes.xml artifacts/$build_dir/
+  # Parameters
+  cp $build_dir_path/parameters.xml artifacts/$build_dir/
+  cp $build_dir_path/parameters.json artifacts/$build_dir/
+  cp $build_dir_path/parameters.json.xz artifacts/$build_dir/
+  # Actuators
+  cp $build_dir_path/actuators.json artifacts/$build_dir/
+  cp $build_dir_path/actuators.json.xz artifacts/$build_dir/
+  # Events
+  cp $build_dir_path/events/all_events.json.xz artifacts/$build_dir/
+  # ROS 2 msgs
+  cp $build_dir_path/events/all_events.json.xz artifacts/$build_dir/
+  # Module Docs
+  ls -la artifacts/$build_dir
+  echo "----------"
+done
+
+# general metadata
+mkdir artifacts/_general/
+cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
+# Airframe
+cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
+# Parameters
+cp artifacts/px4_sitl_default/parameters.xml artifacts/_general/
+cp artifacts/px4_sitl_default/parameters.json artifacts/_general/
+cp artifacts/px4_sitl_default/parameters.json.xz artifacts/_general/
+# Actuators
+cp artifacts/px4_sitl_default/actuators.json artifacts/_general/
+cp artifacts/px4_sitl_default/actuators.json.xz artifacts/_general/
+# Events
+cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
+# ROS 2 msgs
+cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
+# Module Docs
+ls -la artifacts/_general/