diff --git a/.github/runs-on.yml b/.github/runs-on.yml new file mode 100644 index 0000000000..f68d406c8a --- /dev/null +++ b/.github/runs-on.yml @@ -0,0 +1,24 @@ +runners: + x86-small-runner: + cpu: [1, 2] + ram: [1, 4] + disk: default + spot: price-capacity-optimized + image: ubuntu24-full-x64 + extras: s3-cache + x86-firmware-builder: + cpu: [4, 8] + ram: [8, 16] + disk: default + family: ["c7i", "m7i", "r7i"] + spot: price-capacity-optimized + image: ubuntu24-full-x64 + extras: s3-cache + arm64-firmware-builder: + cpu: [4, 8] + ram: [8, 16] + disk: default + family: ["c7g", "m7g", "r7g"] + spot: price-capacity-optimized + image: ubuntu24-full-arm64 + extras: s3-cache diff --git a/.github/workflows/build_all_targets.yml b/.github/workflows/build_all_targets.yml index ef3b4d1470..e2646be336 100644 --- a/.github/workflows/build_all_targets.yml +++ b/.github/workflows/build_all_targets.yml @@ -26,6 +26,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + contents: write + actions: read + jobs: group_targets: name: Scan for Board Targets @@ -48,12 +52,15 @@ jobs: path: "./Tools/setup/requirements.txt" - id: set-matrix - run: echo "::set-output name=matrix::$(./Tools/ci/generate_board_targets_json.py --group)" + name: Generate Build Matrix + run: echo "matrix=$(./Tools/ci/generate_board_targets_json.py --group)" >> $GITHUB_OUTPUT - id: set-timestamp - run: echo "::set-output name=timestamp::$(date +"%Y%m%d%H%M%S")" + name: Save Current Timestamp + run: echo "timestamp=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT - id: set-branch + name: Save Current Branch Name run: | echo "branchname=${{ github.event_name == 'pull_request' && @@ -70,7 +77,7 @@ jobs: echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)" setup: - name: Build Group [${{ matrix.group }}][${{ matrix.arch == 'nuttx' && 'x86' || 'arm64' }}] + name: Build [${{ matrix.runner }}][${{ matrix.group }}] # runs-on: ubuntu-latest runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false] needs: group_targets @@ -80,6 +87,7 @@ jobs: container: image: ${{ matrix.container }} steps: + - uses: runs-on/action@v2 - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -87,14 +95,24 @@ jobs: - name: Git ownership workaround run: git config --system --add safe.directory '*' - - name: Setup ccache - uses: actions/cache@v4 + # ccache key breakdown: + # ccache---- + # ccache---- + # ccache---- + - name: Restore ccache from key + id: cc_restore + uses: actions/cache/restore@v4 with: path: ~/.ccache - key: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }} - restore-keys: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }} + key: ${{ format('ccache-{0}-{1}-{2}', runner.os, matrix.runner, matrix.group) }} + restore-keys: | + ccache-${{ runner.os }}-${{ matrix.runner }}-${{ matrix.group }}- + ccache-${{ runner.os }}-${{ matrix.runner }}- + ccache-${{ runner.os }}-${{ matrix.runner }}- + ccache-${{ runner.os }}- + ccache- - - name: Configure ccache + - name: Set ccache defaults and show stats run: | mkdir -p ~/.ccache echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf @@ -102,10 +120,11 @@ jobs: echo "compression_level = 6" >> ~/.ccache/ccache.conf echo "max_size = 120M" >> ~/.ccache/ccache.conf echo "hash_dir = false" >> ~/.ccache/ccache.conf + echo "compiler_check = content" >> ~/.ccache/ccache.conf ccache -s ccache -z - - name: Building [${{ matrix.group }}] + - name: Building [${{ matrix.targets }}] run: | ./Tools/ci/build_all_runner.sh ${{matrix.targets}} ${{matrix.arch}} @@ -119,15 +138,27 @@ jobs: name: px4_${{matrix.group}}_build_artifacts path: artifacts/ + - name: Post Build Cache Stats + if: always() + run: | + ccache -s + ccache -z + - name: Cache Save - run: ccache -s + if: always() + uses: actions/cache/save@v4 + with: + path: ~/.ccache + key: ${{ steps.cc_restore.outputs.cache-primary-key }} artifacts: - name: Upload Artifacts to S3 + name: Upload Artifacts # runs-on: ubuntu-latest runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] needs: [setup, group_targets] - if: contains(fromJSON('["main", "stable", "beta"]'), needs.group_targets.outputs.branchname) + if: startsWith(github.ref, 'refs/tags/v') || contains(fromJSON('["main","stable","beta"]'), needs.group_targets.outputs.branchname) + outputs: + uploadlocation: ${{ steps.upload-location.outputs.uploadlocation }} steps: - name: Download Artifacts uses: actions/download-artifact@v4 @@ -135,11 +166,36 @@ jobs: path: artifacts/ merge-multiple: true - - name: Branch Name + - name: Upload Location + id: upload-location run: | - echo "${{ needs.group_targets.outputs.branchname }}" + # Determine upload location based on branch or tag with the following considerations: + # Destination: AWS S3 bucket px4-travis in folder Firmware/ + # - If branch is main -> upload to master/ + # - Older versions of QGC are hardocded to look for master/ + # - If branch is stable or beta -> upload to stable/ or beta/ + # - If a tag vX.Y.Z -> upload to vX.Y.Z/ + # - Also update stable/ to point to the same version + #. - Older versions of QGC are hardocded to look for stable/ + # - If a pull request -> do not upload + set -euo pipefail - - name: Uploading Artifacts to S3 [${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}] + ref="${GITHUB_REF}" + branch=${{ needs.group_targets.outputs.branchname }} + location="$branch" + + if [[ "$branch" == "main" ]]; then + location="master" + fi + + if [[ "$ref" == refs/tags/v[0-9]* ]]; then + tag="${ref#refs/tags/}" + location="$tag" + fi + + echo "uploadlocation=$location" >> $GITHUB_OUTPUT + + - name: Uploading Artifacts to S3 [${{ steps.upload-location.outputs.uploadlocation }}] uses: jakejarvis/s3-sync-action@master with: args: --acl public-read @@ -149,25 +205,30 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: 'us-west-1' SOURCE_DIR: artifacts/ - DEST_DIR: Firmware/${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}/ + DEST_DIR: Firmware/${{ steps.upload-location.outputs.uploadlocation }}/ - release: - name: Create Release and Upload Artifacts - permissions: - contents: write - # runs-on: ubuntu-latest - runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] - needs: [setup, group_targets] - if: startsWith(github.ref, 'refs/tags/') - steps: - - name: Download Artifacts - uses: actions/download-artifact@v4 + # if we are uploading artifacts to a versioned folder + # we should also update the stable folder in the s3 bucket + - name: Uploading Artifacts to S3 [stable] + uses: jakejarvis/s3-sync-action@master + if: startsWith(github.ref, 'refs/tags/v') with: - path: artifacts/ - merge-multiple: true + args: --acl public-read + env: + AWS_S3_BUCKET: 'px4-travis' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'us-west-1' + SOURCE_DIR: artifacts/ + DEST_DIR: Firmware/stable/ + # if build is a release triggered by a versioned tag then create a github release + # and upload the build artifacts. A draft release is created so that the release + # can be reviewed before publishing - name: Upload Binaries to Release uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/v') with: draft: true files: artifacts/*.px4 + name: ${{ steps.upload-location.outputs.uploadlocation }} diff --git a/.github/workflows/compile_ubuntu.yml b/.github/workflows/compile_ubuntu.yml index b213a5beaa..019081b9f6 100644 --- a/.github/workflows/compile_ubuntu.yml +++ b/.github/workflows/compile_ubuntu.yml @@ -29,7 +29,7 @@ jobs: fail-fast: false matrix: version: ['ubuntu:22.04', 'ubuntu:24.04'] - runs-on: [runs-on,runner=8cpu-linux-x64,"image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=4cpu-linux-x64,"image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false] container: image: ${{ matrix.version }} volumes: diff --git a/.github/workflows/dev_container.yml b/.github/workflows/dev_container.yml index a8dd32388e..ee61d80c3c 100644 --- a/.github/workflows/dev_container.yml +++ b/.github/workflows/dev_container.yml @@ -39,7 +39,7 @@ jobs: name: Set Tags and Variables permissions: contents: read - runs-on: [runs-on,"runner=1cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false,extras=s3-cache] + runs-on: [runs-on,"runner=1cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",extras=s3-cache,spot=false] outputs: px4_version: ${{ steps.px4_version.outputs.px4_version }} meta_tags: ${{ steps.meta.outputs.tags }} @@ -87,7 +87,7 @@ jobs: - platform: linux/amd64 arch: amd64 runner: x64 - runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false,extras=s3-cache] + runs-on: [runs-on,"runner=4cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",extras=s3-cache,spot=false] steps: - uses: runs-on/action@v1 - uses: actions/checkout@v4 @@ -138,7 +138,7 @@ jobs: permissions: contents: read packages: write - runs-on: [runs-on,"runner=8cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false,extras=s3-cache] + runs-on: [runs-on,"runner=4cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",extras=s3-cache,spot=false] needs: [build, setup] if: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_to_registry) }} steps: diff --git a/.github/workflows/docs_deploy.yml b/.github/workflows/docs_deploy.yml index b8991e38c9..bdf729b2b3 100644 --- a/.github/workflows/docs_deploy.yml +++ b/.github/workflows/docs_deploy.yml @@ -20,7 +20,7 @@ env: jobs: build: - runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache] + runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",extras=s3-cache,spot=false] steps: - uses: runs-on/action@v1 - name: Checkout @@ -55,7 +55,7 @@ jobs: deploy: if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged) || github.event_name == 'workflow_dispatch' }} needs: build - runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] steps: - name: Download Artifact diff --git a/.github/workflows/docs_deploy_aws.yml b/.github/workflows/docs_deploy_aws.yml index 304100fd81..788a303d5e 100644 --- a/.github/workflows/docs_deploy_aws.yml +++ b/.github/workflows/docs_deploy_aws.yml @@ -30,7 +30,7 @@ concurrency: jobs: build: - runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache] + runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache] outputs: branchname: ${{ steps.set-branch.outputs.branchname }} releaseversion: ${{ steps.set-version.outputs.releaseversion }} diff --git a/.github/workflows/flash_analysis.yml b/.github/workflows/flash_analysis.yml index e909e24822..143d54790c 100644 --- a/.github/workflows/flash_analysis.yml +++ b/.github/workflows/flash_analysis.yml @@ -24,7 +24,7 @@ env: jobs: analyze_flash: name: Analyzing ${{ matrix.target }} - runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] container: image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556 strategy: @@ -97,7 +97,7 @@ jobs: # Track this issue https://github.com/PX4/PX4-Autopilot/issues/24408 post_pr_comment: name: Publish Results - runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}"] needs: [analyze_flash] env: V5X-SUMMARY-MAP-ABS: ${{ fromJSON(fromJSON(needs.analyze_flash.outputs.px4_fmu-v5x-bloaty-summary-map).vm-absolute) }} diff --git a/.github/workflows/itcm_check.yml b/.github/workflows/itcm_check.yml index f6d5c4fd8c..5a1b0ecddc 100644 --- a/.github/workflows/itcm_check.yml +++ b/.github/workflows/itcm_check.yml @@ -22,7 +22,7 @@ concurrency: jobs: check_itcm: name: Checking ${{ matrix.target }} - runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] container: image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556 strategy: diff --git a/.github/workflows/ros_integration_tests.yml b/.github/workflows/ros_integration_tests.yml index e15aa081e7..022b46fb10 100644 --- a/.github/workflows/ros_integration_tests.yml +++ b/.github/workflows/ros_integration_tests.yml @@ -23,7 +23,7 @@ concurrency: jobs: build: - runs-on: [runs-on,runner=16cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false] container: image: px4io/px4-dev-ros2-galactic:2021-09-08 options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined diff --git a/.github/workflows/ros_translation_node.yml b/.github/workflows/ros_translation_node.yml index 9b161dae90..64bae13f83 100644 --- a/.github/workflows/ros_translation_node.yml +++ b/.github/workflows/ros_translation_node.yml @@ -21,7 +21,7 @@ concurrency: jobs: build_and_test: name: Build and test - runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] strategy: fail-fast: false matrix: diff --git a/.github/workflows/sitl_tests.yml b/.github/workflows/sitl_tests.yml index 6dc6eccd2d..68e16bed71 100644 --- a/.github/workflows/sitl_tests.yml +++ b/.github/workflows/sitl_tests.yml @@ -24,7 +24,7 @@ concurrency: jobs: build: name: Testing PX4 ${{ matrix.config.model }} - runs-on: [runs-on,runner=16cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false] container: image: px4io/px4-dev-simulation-focal:2021-09-08 options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined