ci: build every target on a single step

Switching to use an all_variants approach to reduce the number of jobs
we are creating per workflow since are starting to get throttled by
GitHub, and we have limited resources.

TODO: Build artifact uploadeds to S3, and GH Releases

.github/workflows/compile_nuttx.yml

@@ -1,120 +0,0 @@
-name: Nuttx Targets
-
-on:
-  push:
-    branches:
-    - 'master'
-  pull_request:
-    branches:
-    - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-nuttx-focal:2021-09-08
-    strategy:
-      fail-fast: false
-      matrix:
-        config: [
-          airmind_mindpx-v2,
-          ark_can-flow,
-          ark_can-gps,
-          ark_can-rtk-gps,
-          atl_mantis-edu,
-          av_x-v1,
-          bitcraze_crazyflie,
-          bitcraze_crazyflie21,
-          cuav_can-gps-v1,
-          cuav_nora,
-          cuav_x7pro,
-          cubepilot_cubeorange,
-          cubepilot_cubeyellow,
-          freefly_can-rtk-gps,
-          holybro_can-gps-v1,
-          holybro_durandal-v1,
-          holybro_kakutef7,
-          holybro_kakuteh7,
-          holybro_pix32v5,
-          matek_h743-slim,
-          matek_gnss-m9n-f4,
-          modalai_fc-v1,
-          modalai_fc-v2,
-          mro_ctrl-zero-f7,
-          mro_ctrl-zero-f7-oem,
-          mro_ctrl-zero-h7,
-          mro_ctrl-zero-h7-oem,
-          mro_pixracerpro,
-          mro_x21,
-          mro_x21-777,
-          nxp_fmuk66-e,
-          nxp_fmuk66-v3,
-          nxp_fmurt1062-v1,
-          nxp_ucans32k146,
-          omnibus_f4sd,
-          raspberrypi_pico,
-          px4_fmu-v2,
-          px4_fmu-v3,
-          px4_fmu-v4,
-          px4_fmu-v4pro,
-          px4_fmu-v5,
-          px4_fmu-v5x,
-          px4_fmu-v6u,
-          px4_fmu-v6x,
-          spracing_h7extreme,
-          uvify_core
-          ]
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        message("::set-output name=timestamp::${current_date}")
-    - name: ccache cache files
-      uses: actions/cache@v2
-      with:
-        path: ~/.ccache
-        key: ${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: ${{matrix.config}}-ccache-
-    - name: setup ccache
-      run: |
-          mkdir -p ~/.ccache
-          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-          echo "compression = true" >> ~/.ccache/ccache.conf
-          echo "compression_level = 6" >> ~/.ccache/ccache.conf
-          echo "max_size = 120M" >> ~/.ccache/ccache.conf
-          echo "hash_dir = false" >> ~/.ccache/ccache.conf
-          ccache -s
-          ccache -z
-
-    - name: make all_variants_${{matrix.config}}
-      run: make all_variants_${{matrix.config}}
-      timeout-minutes: 45
-    - name: make ${{matrix.config}} bloaty_compileunits
-      run: make ${{matrix.config}} bloaty_compileunits || true
-    - name: make ${{matrix.config}} bloaty_inlines
-      run: make ${{matrix.config}} bloaty_inlines || true
-    - name: make ${{matrix.config}} bloaty_segments
-      run: make ${{matrix.config}} bloaty_segments || true
-    - name: make ${{matrix.config}} bloaty_symbols
-      run: make ${{matrix.config}} bloaty_symbols || true
-    - name: make ${{matrix.config}} bloaty_templates
-      run: make ${{matrix.config}} bloaty_templates || true
-    - name: make ${{matrix.config}} bloaty_ram
-      run: make ${{matrix.config}} bloaty_ram || true
-    - name: make ${{matrix.config}} bloaty_compare_master
-      run: make ${{matrix.config}} bloaty_compare_master || true
-    - name: ccache post-run
-      run: ccache -s
-
-    - name: Upload px4 package
-      uses: actions/upload-artifact@v2
-      with:
-        name: px4_package_${{matrix.config}}
-        path: |
-          build/**/*.px4
-          build/**/*.bin

.github/workflows/deploy_all.yml

@@ -1,11 +1,17 @@
-name: Deploy metadata for all targets
+name: Build and deploy all targets with metadata
 
 on:
+  release:
+    types: [published]
+  create:
+    tags:
+      - 'v*'
   push:
     branches:
     - 'master'
     - 'release/*'
     - 'pr-metadata-test'
+    - 'build_and_deploy_mix'
 
 jobs:
   enumerate_targets:
@@ -18,8 +24,8 @@ jobs:
       with:
         token: ${{secrets.ACCESS_TOKEN}}
     - id: set-matrix
-      run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py)"
+      run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py -a)"
-  build:
+  build_all:
     runs-on: ubuntu-latest
     needs: enumerate_targets
     strategy:
@@ -34,21 +40,4 @@ jobs:
       run: make ${{matrix.target}}
 
     - name: parameter & events metadata
-      run: |
+      run: make ${{matrix.target}} ver_gen events_json actuators_json
-        make ${{matrix.target}} ver_gen events_json actuators_json
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/${{ matrix.target }} >> $GITHUB_ENV
-        cd build/${{ matrix.target }}
-        mkdir _metadata || true
-        cp parameters.* events/*.xz actuators.json* _metadata
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/${{ matrix.target }}/_metadata/'
-        DEST_DIR: 'Firmware/${{ env.version }}/${{ matrix.target }}/'
-

Tools/generate_board_targets_json.py

@@ -23,6 +23,10 @@ parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                     help='Verbose Output')
 parser.add_argument('-p', '--pretty', dest='pretty', action='store_true',
                     help='Pretty output instead of a single line')
+parser.add_argument('-a', '--all_variants', dest='all_variants', action='store_true',
+                    help='Prints targets into a single all_variants_* target')
+parser.add_argument('-b', '--bloaty', dest='bloaty', action='store_true',
+                    help='Includes all bloaty targets')
 
 args = parser.parse_args()
 verbose = args.verbose
@@ -35,6 +39,15 @@ excluded_labels = [
     'nolockstep', 'replay', 'test',
     'uavcanv1' # TODO: fix and enable
     ]
+bloaty_helpers = [
+    'bloaty_compileunits',
+    'bloaty_inlines',
+    'bloaty_segments',
+    'bloaty_symbols',
+    'bloaty_templates',
+    'bloaty_ram',
+    'bloaty_compare_master',
+    ]
 
 def process_target(px4board_file, target_name):
     ret = None
@@ -77,6 +90,15 @@ def process_target(px4board_file, target_name):
 
     return ret
 
+def process_bloaty(target_path, target_name):
+    response = []
+    for bloat in bloaty_helpers:
+        bloaty_name = target_name + ' ' + bloat + ' || true'
+        processed_target = process_target(target_path, bloaty_name)
+        response.append(processed_target)
+    return response
+
+# Look up boards from each manufacturer
 for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
     if not manufacturer.is_dir():
         continue
@@ -85,18 +107,34 @@ for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
         continue
 
     for board in os.scandir(manufacturer.path):
+        # Only boards are directories don't proceed if otherwise
         if not board.is_dir():
             continue
-        for files in os.scandir(board.path):
+
-            if files.is_file() and files.name.endswith('.px4board'):
+        if args.all_variants:
-                label = files.name[:-9]
+            # The all_variants target makes all targets for a board
-                target_name = manufacturer.name + '_' + board.name + '_' + label
+            target_name = 'all_variants_' + manufacturer.name + '_' + board.name
-                if label in excluded_labels:
+            default_target_path = f'{board.path}/default.px4board'
-                    if verbose: print(f'excluding label {label} ({target_name})')
+            target = process_target(default_target_path, target_name)
-                    continue
+            if target is not None:
-                target = process_target(files.path, target_name)
+                build_configs.append(target)
-                if target is not None:
+            if args.bloaty and target is not None:
-                    build_configs.append(target)
+                # bloaty targets
+                bloat_target_name = manufacturer.name + '_' + board.name
+                bloaty_targets = process_bloaty(default_target_path, bloat_target_name)
+                build_configs += bloaty_targets
+        else:
+            # Each board can have multiple variant targets
+            for files in os.scandir(board.path):
+                if files.is_file() and files.name.endswith('.px4board'):
+                    label = files.name[:-9]
+                    target_name = manufacturer.name + '_' + board.name + '_' + label
+                    if label in excluded_labels:
+                        if verbose: print(f'excluding label {label} ({target_name})')
+                        continue
+                    target = process_target(files.path, target_name)
+                    if target is not None:
+                        build_configs.append(target)
 
 
 github_action_config = { 'include': build_configs }