diff --git a/.github/workflows/docs-orchestrator.yml b/.github/workflows/docs-orchestrator.yml index 99a86ae5dd..7b1ba45980 100644 --- a/.github/workflows/docs-orchestrator.yml +++ b/.github/workflows/docs-orchestrator.yml @@ -16,6 +16,7 @@ permissions: contents: write actions: read id-token: write + pull-requests: write concurrency: group: docs-orchestrator-${{ github.ref }} @@ -203,6 +204,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 - name: Download metadata artifact if: needs.pr-metadata-regen.result == 'success' @@ -211,15 +213,47 @@ jobs: name: pr-metadata path: docs/ + - name: Get changed doc files + id: changed-files + uses: tj-actions/changed-files@v46.0.5 + with: + json: true + base_sha: ${{ github.event.pull_request.base.sha }} + sha: ${{ github.event.pull_request.head.sha }} + files: | + docs/en/**/*.md + + - name: Save changed files list + run: | + mkdir -p logs + echo '${{ steps.changed-files.outputs.all_changed_files }}' | jq '.' > ./logs/prFiles.json + echo "Changed files:" + cat ./logs/prFiles.json + - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 20 - - name: Run link checker + - name: Run filtered link checker (changed files) run: | npm -g install markdown_link_checker_sc@0.0.138 - mkdir -p logs + if [ "$(jq length ./logs/prFiles.json)" -gt 0 ]; then + markdown_link_checker_sc \ + -r "$GITHUB_WORKSPACE" \ + -d docs \ + -e en \ + -f ./logs/prFiles.json \ + -i assets \ + -u docs.px4.io/main/ \ + > ./logs/filtered-link-check-results.md || true + else + echo "No changed doc files found" > ./logs/filtered-link-check-results.md + fi + cat ./logs/filtered-link-check-results.md + + - name: Run full link checker + run: | markdown_link_checker_sc \ -r "$GITHUB_WORKSPACE" \ -d docs \ @@ -229,11 +263,17 @@ jobs: > ./logs/link-check-results.md || true cat ./logs/link-check-results.md + - name: Post PR comment with link check results + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: flaws + path: ./logs/filtered-link-check-results.md + - name: Upload link check results uses: actions/upload-artifact@v4 with: name: link-check-results - path: logs/link-check-results.md + path: logs/ retention-days: 7 # ============================================================================= @@ -241,8 +281,11 @@ jobs: # ============================================================================= build-site: name: "Build Site" - needs: [detect-changes, pr-metadata-regen, metadata-regen] - if: always() && (needs.metadata-regen.result == 'success' || needs.metadata-regen.result == 'skipped') + needs: [detect-changes, pr-metadata-regen, metadata-regen, link-check] + if: >- + always() && + (needs.metadata-regen.result == 'success' || needs.metadata-regen.result == 'skipped') && + (needs.link-check.result == 'success' || needs.link-check.result == 'skipped') runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache] outputs: branchname: ${{ steps.set-branch.outputs.branchname }} diff --git a/.github/workflows/docs_deploy_aws.yml b/.github/workflows/docs_deploy_aws.yml deleted file mode 100644 index 788a303d5e..0000000000 --- a/.github/workflows/docs_deploy_aws.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Docs - Deploy PX4 User Guide to AWS - -on: - push: - branches: - - "main" - - "release/**" - paths: - - "docs/en/**" - - "docs/zh/**" - - "docs/uk/**" - - "docs/ko/**" - pull_request: - paths: - - "docs/en/**" - - "docs/zh/**" - - "docs/uk/**" - - "docs/ko/**" - - workflow_dispatch: - -permissions: - contents: read - actions: read - id-token: write # for AWS OIDC - -concurrency: - group: docs-deploy - cancel-in-progress: false - -jobs: - build: - runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache] - outputs: - branchname: ${{ steps.set-branch.outputs.branchname }} - releaseversion: ${{ steps.set-version.outputs.releaseversion }} - steps: - - uses: runs-on/action@v1 - - - name: Checkout - uses: actions/checkout@v4 - - - id: set-branch - run: echo "branchname=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - - - id: set-version - run: | - branch="${{ steps.set-branch.outputs.branchname }}" - if [[ "$branch" == "main" ]]; then - version="main" - else - version="v${branch#release/}" - fi - echo "releaseversion=$version" >> $GITHUB_OUTPUT - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - cache-dependency-path: ./docs/yarn.lock - - - name: Install dependencies - run: yarn install --frozen-lockfile --cwd ./docs - - - name: Build with VitePress - working-directory: ./docs - env: - BRANCH_NAME: ${{ steps.set-version.outputs.releaseversion }} - run: | - npm run docs:build_ubuntu - touch .vitepress/dist/.nojekyll - npm run docs:sitemap - - - name: Upload artifact - if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged) || github.event_name == 'workflow_dispatch' }} - uses: actions/upload-artifact@v4 - with: - name: px4_docs_build - path: docs/.vitepress/dist/ - retention-days: 1 - - deploy: - if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged) || github.event_name == 'workflow_dispatch' }} - needs: build - runs-on: ubuntu-latest - - steps: - - name: Download Artifact - uses: actions/download-artifact@v4 - with: - name: px4_docs_build - path: ~/_book - - - name: Configure AWS from OIDC - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: us-west-2 - - - name: Sanity check AWS credentials - run: aws sts get-caller-identity - - - name: Upload HTML with short cache - run: | - aws s3 sync ~/_book/ s3://px4-docs/${{ needs.build.outputs.releaseversion }}/ \ - --delete \ - --exclude "*" --include "*.html" \ - --cache-control "public, max-age=60" - - - name: Upload assets with long cache - run: | - aws s3 sync ~/_book/ s3://px4-docs/${{ needs.build.outputs.releaseversion }}/ \ - --delete \ - --exclude "*.html" \ - --cache-control "public, max-age=86400, immutable" diff --git a/.github/workflows/docs_flaw_checker.yml b/.github/workflows/docs_flaw_checker.yml deleted file mode 100644 index de1dd68292..0000000000 --- a/.github/workflows/docs_flaw_checker.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Docs - Check for flaws in PX4 Guide Source -# So far: - # Modifications of translations files - # Broken internal links - -on: - pull_request_target: - types: [opened, edited, synchronize] - paths: - - 'docs/en/**' - -jobs: - check_flaws: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - - name: Install Node.js - uses: actions/setup-node@v3 - with: - node-version: '18' - - - name: Create logs directory - run: | - mkdir logs - - - name: Get changed english doc files - id: get_changed_markdown_english - uses: tj-actions/changed-files@v46.0.5 - with: - json: true - base_sha: "${{ github.event.pull_request.base.sha }}" - sha: "${{ github.event.pull_request.head.sha }}" - # Below are used to output files to a directory. May use in flaw checker. - # write_output_files: true - # output_dir: "./logs" - files: | - docs/en/**/*.md - - - name: Save JSON file containing files to link check - run: | - echo "$ALL_CHANGED_FILES" - # echo "$ALL_CHANGED_FILES" > ./logs/prFiles.json - echo "$ALL_CHANGED_FILES" | sed 's/\\//g' | jq '.' > ./logs/prFiles.json - env: - ALL_CHANGED_FILES: ${{ steps.get_changed_markdown_english.outputs.all_changed_files }} - - - name: Run link checker - id: link-check - run: | - npm -g install markdown_link_checker_sc@0.0.138 - markdown_link_checker_sc \ - -r "$GITHUB_WORKSPACE" \ - -d docs \ - -e en \ - -f ./logs/prFiles.json \ - -i assets \ - -u docs.px4.io/main/ \ - > ./logs/errorsFilteredByPrPages.md - mkdir -p ./pr - cp ./logs/errorsFilteredByPrPages.md ./pr/errorsFilteredByPrPages.md - - - name: Read errorsFilteredByPrPages.md file - id: read-errors-by-page - uses: juliangruber/read-file-action@v1 - with: - path: ./logs/errorsFilteredByPrPages.md - - - name: Echo Errors by Page - run: echo "$ERRORS" - env: - ERRORS: ${{ steps.read-errors-by-page.outputs.content }} - - - name: Save PR number - run: echo "$PR_NUMBER" > ./pr/pr_number - env: - PR_NUMBER: ${{ github.event.number }} - - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr/ diff --git a/.github/workflows/docs_metadata_check.yml b/.github/workflows/docs_metadata_check.yml deleted file mode 100644 index b9a0ad3572..0000000000 --- a/.github/workflows/docs_metadata_check.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Docs Metadata Checks - -permissions: - contents: write - -on: - pull_request: {} - push: - branches: - - main - - 'release/**' - -jobs: - metadata-check: - name: ${{ matrix.name }} metadata - runs-on: [runs-on,runner=2cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=true] - container: - image: ghcr.io/px4/px4-dev:v1.16.0-ondemand - strategy: - fail-fast: false - matrix: - include: - - name: uORB Graphs - script: Tools/ci/metadata_uorb_graph.sh - # - name: Failsafe Web - # script: Tools/ci/metadata_failsafe_web.sh - - name: uORB Messages - script: Tools/ci/metadata_msg_docs.sh - - name: Parameter Reference - script: Tools/ci/metadata_parameters.sh - - name: Airframe Reference - script: Tools/ci/metadata_airframe.sh - - name: Module Reference - script: Tools/ci/metadata_modules.sh - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - persist-credentials: true - - - name: Mark all directories safe for Git - run: git config --system --add safe.directory '*' - - - name: Run ${{ matrix.name }} metadata check - run: ${{ matrix.script }} --test-only - - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/docs_pr_comment.yml b/.github/workflows/docs_pr_comment.yml deleted file mode 100644 index 41e443a786..0000000000 --- a/.github/workflows/docs_pr_comment.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Docs - Comment Workflow -on: - workflow_run: - workflows: ["Docs - Check for flaws in PX4 Guide Source"] - types: - - completed - -jobs: - comment: - permissions: - pull-requests: write # for marocchino/sticky-pull-request-comment - name: Comments - runs-on: ubuntu-latest - steps: - - name: 'Download PR artifact' - uses: actions/github-script@v6 - with: - script: | - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "pr_number" - })[0]; - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); - - - name: 'Unzip artifact' - run: unzip pr_number.zip - - # Doesn't work across workflows - #- name: Get artifacts from flaw checker workflow - # uses: actions/download-artifact@v3 - # with: - # name: logs_and_errors - # #path: ./logs - - - name: Read errorsFilteredByPrPages.md file - id: read-errors-by-page - uses: juliangruber/read-file-action@v1 - with: - path: ./errorsFilteredByPrPages.md - - - name: Read PR number - id: read-error-pr-number - uses: juliangruber/read-file-action@v1 - with: - path: ./pr_number - - - name: File detail info - run: | - echo "$ERRORS" - echo "$PRNUM" - env: - ERRORS: ${{ steps.read-errors-by-page.outputs.content }} - PRNUM: ${{ steps.read-error-pr-number.outputs.content }} - - - name: Create or update comment - id: comment_to_pr - uses: marocchino/sticky-pull-request-comment@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - recreate: true - number: ${{ steps.read-error-pr-number.outputs.content }} - header: flaws - message: ${{ steps.read-errors-by-page.outputs.content || 'No flaws found' }} - - #- name: Dump GitHub context - # env: - # GITHUB_CONTEXT: ${{ toJSON(github) }} - # run: echo "$GITHUB_CONTEXT" - - # Would like to do this, but it doesn't work (for me). - # Moving to time-based, or triggering on workflow - #- name: Wait for artifacts upload to succeed - # uses: lewagon/wait-on-check-action@v1.3.1 - # with: - # ref: ${{ github.ref }} - # check-name: 'Archive production artifacts' - # repo-token: ${{ secrets.GITHUB_TOKEN }} - # wait-interval: 80 - - # Not needed for now - trying to trigger off the workflow - #- name: Sleep for 80 seconds - # run: sleep 80s - # shell: bash - #- name: Find Comment - # uses: peter-evans/find-comment@v2 - # id: fc - # with: - # issue-number: ${{ steps.read-error-pr-number.outputs.content }} - # comment-author: 'github-actions[bot]' - # body-includes: Flaws (may be none) - - #- name: Create or update comment - # uses: peter-evans/create-or-update-comment@v3 - # with: - # comment-id: ${{ steps.fc.outputs.comment-id }} - # issue-number: ${{ steps.read-error-pr-number.outputs.content }} - # body: | - # Flaws (may be none) - # ${{ steps.read-errors-by-page.outputs.content }} - # edit-mode: replace diff --git a/docs/en/test_and_ci/continous_integration.md b/docs/en/test_and_ci/continous_integration.md index 0cbae2fca9..087deda8f0 100644 --- a/docs/en/test_and_ci/continous_integration.md +++ b/docs/en/test_and_ci/continous_integration.md @@ -1,3 +1,187 @@ # PX4 Continuous Integration -PX4 builds and testing are performed using GitHub actions and a Jenkins server. +PX4 uses GitHub Actions for continuous integration, with different workflows handling code builds, testing, and documentation. + +## Documentation CI + +The documentation pipeline handles building, deploying, and translating the PX4 User Guide. +All documentation CI is consolidated into a single orchestrator workflow organized in tiers. + +### Docs Orchestrator + +**Workflow file:** [`docs-orchestrator.yml`](https://github.com/PX4/PX4-Autopilot/blob/main/.github/workflows/docs-orchestrator.yml) + +This is the main documentation workflow. It runs on pull requests and pushes to `main` and `release/**` branches, performing different jobs depending on the trigger event. +Jobs are organized in tiers, where each tier depends on the previous one completing successfully. + +#### Tier Structure + +| Tier | Job | PR | Push | Description | +|------|-----|----|------|-------------| +| 0 | Detect Changed Paths | Yes | — | Checks if source code files changed (triggers metadata regen) | +| 1 | Metadata Generation | Yes (conditional) | Yes | Builds PX4 SITL and regenerates all auto-generated docs | +| 2 | Link Validation | Yes | — | Checks for broken links in changed files, posts PR comment | +| 3 | Build Site | Yes (gated on Tier 2) | Yes (after Tier 1) | Builds the VitePress documentation site | +| 4 | Deploy + Crowdin Upload | — | Yes | Deploys to AWS S3 and uploads sources to Crowdin | + +#### Pull Request Flow + +When a PR modifies files in `docs/**`, the workflow validates the changes: + +``` +PR Event + │ + ▼ +┌─────────────────────────────────────┐ +│ Tier 0: Detect Changed Paths │ +│ • Checks if src/msg/ROMFS changed │ +└─────────────────┬───────────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ Tier 1: PR Metadata Regen │ +│ (only if source files changed) │ +│ • Builds px4_sitl_default │ +│ • Generates parameter/airframe/ │ +│ module documentation │ +│ • Builds failsafe web simulator │ +└─────────────────┬───────────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ Tier 2: Link Validation (~30 sec) │ +│ • Detects changed docs/en/*.md │ +│ • Runs filtered link check │ +│ • Posts sticky PR comment (flaws) │ +│ • Runs full link check (artifact) │ +└─────────────────┬───────────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ Tier 3: Build Site (~7-10 min) │ +│ (only if link check passed) │ +│ • Builds VitePress site │ +│ • Verifies no build errors │ +└─────────────────┬───────────────────┘ + │ + ▼ + DONE +``` + +| Job | Duration | Description | +|-----|----------|-------------| +| **Detect Changed Paths** | ~10s | Determines if metadata regeneration is needed | +| **PR Metadata Regen** | ~10-15m | Rebuilds PX4 SITL and regenerates all metadata (conditional) | +| **Link Validation** | ~30s | Checks for broken links in changed markdown files and posts a sticky comment to the PR | +| **Build Site** | ~7-10m | Builds the VitePress site to verify there are no build errors. Gated on link check passing. | + +#### Push Flow (main/release branches) + +When changes are pushed to `main` or `release/**` branches, the workflow regenerates metadata, builds, and deploys: + +``` +Push Event + │ + ▼ +┌─────────────────────────────────────┐ +│ Tier 1: Regenerate Metadata │ +│ (~10-15 min) │ +│ • Builds px4_sitl_default │ +│ • Generates parameter/airframe/ │ +│ module documentation │ +│ • Builds failsafe web simulator │ +│ • Formats with Prettier │ +│ • Auto-commits if changes detected │ +└─────────────────┬───────────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ Tier 3: Build Site (~7-10 min) │ +│ • Builds VitePress site │ +│ • Uploads build artifact │ +└─────────────────┬───────────────────┘ + │ + ┌───────┴───────┐ + ▼ ▼ +┌──────────────┐ ┌──────────────────┐ +│ Tier 4: │ │ Tier 4: │ +│ Deploy to │ │ Crowdin Upload │ +│ AWS (~3 min) │ │ (~4 min) │ +│ • S3 sync │ │ • Upload sources │ +└──────────────┘ └──────────────────┘ +``` + +| Job | Duration | Description | +|-----|----------|-------------| +| **Regenerate Metadata** | ~10-15m | Rebuilds PX4 SITL, regenerates all metadata, formats with Prettier, auto-commits | +| **Build Site** | ~7-10m | Builds the VitePress documentation site | +| **Deploy to AWS** | ~3m | Syncs built site to AWS S3 (HTML: 60s cache, assets: 24h cache) | +| **Crowdin Upload** | ~4m | Uploads English source files to Crowdin for translation (main branch only) | + +#### Generated Metadata + +The metadata regeneration job creates the following auto-generated documentation: + +| Type | Output | Description | +|------|--------|-------------| +| Parameters | `docs/en/advanced_config/parameter_reference.md` | Complete parameter reference | +| Airframes | `docs/en/airframes/airframe_reference.md` | Airframe configurations | +| Modules | `docs/en/modules/*.md` | Module documentation | +| Messages | `docs/en/msg_docs/*.md` | uORB message documentation | +| uORB Graphs | `docs/public/middleware/*.json` | Topic dependency graphs | +| Failsafe Simulator | `docs/public/config/failsafe/*` | Interactive failsafe simulator (WebAssembly) | + +::: warning +Do not manually edit the auto-generated files listed above. They are overwritten on every push to main. +::: + +#### Path Triggers + +The workflow triggers on changes to these paths: + +| Path | Reason | +|------|--------| +| `docs/**` | Documentation source files | +| `src/**` | Source code changes affect metadata | +| `msg/**` | Message definitions affect metadata | +| `ROMFS/**` | ROMFS files affect metadata | +| `Tools/module_config/**` | Module configuration affects metadata | + +### Crowdin Download Workflow + +**Workflow file:** [`docs_crowdin_download.yml`](https://github.com/PX4/PX4-Autopilot/blob/main/.github/workflows/docs_crowdin_download.yml) + +This scheduled workflow downloads completed translations from Crowdin and creates pull requests. + +| Setting | Value | +|---------|-------| +| **Schedule** | Every Sunday at 00:00 UTC | +| **Target Languages** | Korean (ko), Ukrainian (uk), Chinese Simplified (zh-CN) | + +**Process:** + +1. Downloads translations for each target language from Crowdin +2. Creates a separate PR for each language with new translations +3. PRs are labeled "Documentation" and assigned for review + +### Caching Strategy + +The workflows use caching to speed up builds: + +| Cache | Size | Purpose | +|-------|------|---------| +| ccache | 1GB | C++ compilation cache for SITL builds | +| node_modules | ~26MB | Node.js dependencies for VitePress | + +### Infrastructure + +Jobs run on [runs-on](https://runs-on.com/) self-hosted runners with S3 cache: + +| Job | Runner | +|-----|--------| +| Detect Changed Paths | ubuntu-latest | +| Link Validation | ubuntu-latest | +| Metadata Regen | 4 CPU (with px4-dev container) | +| Build Site | 4 CPU | +| Deploy to AWS | ubuntu-latest | +| Crowdin Upload | ubuntu-latest |