From 380ae3047da935183eb99901ca359b86b40f3773 Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Thu, 15 Jan 2026 13:01:10 -0800 Subject: [PATCH] tools: merge all scripts into a single one Signed-off-by: Ramon Roche --- Tools/ci/metadata_airframe.sh | 68 ----- Tools/ci/metadata_failsafe_web.sh | 112 -------- Tools/ci/metadata_modules.sh | 99 ------- Tools/ci/metadata_msg_docs.sh | 84 ------ Tools/ci/metadata_parameters.sh | 71 ----- Tools/ci/metadata_sync.sh | 447 ++++++++++++++++++++++++++++++ Tools/ci/metadata_uorb_graph.sh | 108 -------- Tools/ci/test_metadata_sync.sh | 163 +++++++++++ 8 files changed, 610 insertions(+), 542 deletions(-) delete mode 100755 Tools/ci/metadata_airframe.sh delete mode 100755 Tools/ci/metadata_failsafe_web.sh delete mode 100755 Tools/ci/metadata_modules.sh delete mode 100755 Tools/ci/metadata_msg_docs.sh delete mode 100755 Tools/ci/metadata_parameters.sh create mode 100755 Tools/ci/metadata_sync.sh delete mode 100755 Tools/ci/metadata_uorb_graph.sh create mode 100755 Tools/ci/test_metadata_sync.sh diff --git a/Tools/ci/metadata_airframe.sh b/Tools/ci/metadata_airframe.sh deleted file mode 100755 index 42793527ad..0000000000 --- a/Tools/ci/metadata_airframe.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# metadata_airframe.sh โ€” generate and sync PX4 airframe reference documentation -# -# Usage: -# Tools/ci/metadata_airframe.sh [--test-only] [--debug] -# -# Options: -# --test-only Run make target and comparison; exit 1 if diffs found, without copying file -# --debug Show full make output and debug info for comparison -# -set -euo pipefail -shopt -s nullglob - -# Parse flags -test_only=false -debug=false -while [[ $# -gt 0 ]]; do - case "$1" in - --test-only) test_only=true; shift ;; - --debug) debug=true; shift ;; - *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; - esac -done - -# Paths and make target -make_target="airframe_metadata" -src_file="build/px4_sitl_default/docs/airframes.md" -dest_file="docs/en/airframes/airframe_reference.md" - -# Run make target -if [ "$debug" = true ]; then - echo "๐Ÿ”ง Running 'make $make_target' (verbose)" - make $make_target -else - echo "๐Ÿ”ง Running 'make $make_target'" - make $make_target > /dev/null 2>&1 -fi - -# Verify build output -if [[ ! -f "$src_file" ]]; then - echo "โŒ Generated file not found: $src_file" - exit 1 -fi - -echo "๐Ÿ” Comparing airframe reference docs" - -# Compare files -if cmp -s "$src_file" "$dest_file"; then - echo "โœ… Airframe reference is up to date." - exit 0 -else - if [ "$debug" = true ]; then - echo "DEBUG: cmp -s '$src_file' '$dest_file'; echo \$?" - fi - echo "โš ๏ธ Airframe reference needs updating." - if [ "$test_only" = true ]; then - exit 1 - fi - # Copy over updated file - echo "๐Ÿ“‚ Copying updated airframe_reference.md" - cp -v "$src_file" "$dest_file" - echo "๐Ÿšจ Airframe docs updated; commit the change:" - echo " git status -s $dest_file" - echo " git add $dest_file" - echo " git commit -m 'docs: update airframe reference metadata'" - exit 1 -fi diff --git a/Tools/ci/metadata_failsafe_web.sh b/Tools/ci/metadata_failsafe_web.sh deleted file mode 100755 index 0cb4a5bca5..0000000000 --- a/Tools/ci/metadata_failsafe_web.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash -# -# metadata_failsafe_web.sh โ€” build and sync failsafe webapp metadata files -# -# Usage: -# Tools/ci/metadata_failsafe_web.sh [--test-only] [--debug] -# -# Options: -# --test-only Run build and comparison; exit 1 if diffs found, without copying files -# --debug Show full build output and debug info for file comparisons and EMSDK install -# -set -euo pipefail -shopt -s nullglob - -# Parse flags -test_only=false -debug=false -while [[ $# -gt 0 ]]; do - case "$1" in - --test-only) test_only=true; shift ;; - --debug) debug=true; shift ;; - *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; - esac -done - -# Paths and commands -build_cmd="make failsafe_web" -src_dir="build/px4_sitl_default_failsafe_web" -dest_dir="docs/public/config/failsafe" - -# Ensure Emscripten SDK is available -if ! command -v emcc >/dev/null 2>&1; then - echo "๐Ÿ”ง Emscripten not found. Ensuring EMSDK is installed." - # Clone SDK only if not already present - if [ ! -d "_emscripten_sdk" ]; then - if [ "$debug" = true ]; then - git clone https://github.com/emscripten-core/emsdk.git _emscripten_sdk - else - git clone https://github.com/emscripten-core/emsdk.git _emscripten_sdk > /dev/null 2>&1 - fi - fi - pushd _emscripten_sdk >/dev/null - if [ "$debug" = true ]; then - ./emsdk install latest - ./emsdk activate latest - else - ./emsdk install latest > /dev/null 2>&1 - ./emsdk activate latest > /dev/null 2>&1 - fi - popd >/dev/null - # Load environment into current shell - if [ "$debug" = true ]; then - # shellcheck source=/dev/null - . ./_emscripten_sdk/emsdk_env.sh - else - # shellcheck source=/dev/null - . ./_emscripten_sdk/emsdk_env.sh > /dev/null 2>&1 - fi -fi - -# Build step -if [ "$debug" = true ]; then - echo "๐Ÿ”ง Running build: $build_cmd" - $build_cmd -else - echo "๐Ÿ”ง Running build" - $build_cmd > /dev/null 2>&1 -fi - -# Gather built files -src_files=("$src_dir"/*.{js,wasm,json}) -if [ ${#src_files[@]} -eq 0 ]; then - echo "โŒ No generated files found in $src_dir. Build failed or path wrong." - exit 1 -fi - -# Prepare destination -echo "๐Ÿ” Checking failsafe web metadata" -mkdir -p "$dest_dir" - -changed=() -for src in "${src_files[@]}"; do - name=$(basename "$src") - dst="$dest_dir/$name" - - if [[ ! -f "$dst" ]]; then - [ "$debug" = true ] && echo "DEBUG: missing $dst" - changed+=("$name") - elif ! cmp -s "$src" "$dst"; then - [ "$debug" = true ] && echo "DEBUG: cmp -s '$src' '$dst'; echo \$?" - changed+=("$name") - fi -done - -if [ ${#changed[@]} -eq 0 ]; then - echo "โœ… All failsafe web metadata files are in sync." - exit 0 -fi - -echo "โš ๏ธ Detected updates in:" -for f in "${changed[@]}"; do echo " - $f"; done - -if [ "$test_only" = true ]; then - echo "๐Ÿšจ Failsafe web metadata needs update; rerun without --test-only to apply." - exit 1 -fi - -echo "๐Ÿ“‚ Copying updated files" -for f in "${changed[@]}"; do cp -v "$src_dir/$f" "$dest_dir/$f"; done - -echo "๐Ÿšจ Failsafe web metadata updated; please commit changes." -exit 1 diff --git a/Tools/ci/metadata_modules.sh b/Tools/ci/metadata_modules.sh deleted file mode 100755 index 9c83cd324c..0000000000 --- a/Tools/ci/metadata_modules.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# -# metadata_modules.sh - generate and sync PX4 module reference documentation -# -# Usage: -# Tools/ci/metadata_modules.sh [--test-only] [--debug] -# -# Options: -# --test-only Run make target and comparison; exit 1 if diffs found, without copying files -# --debug Show full make output and debug info for file comparisons -# -set -euo pipefail -shopt -s nullglob - -# Parse flags -test_only=false -debug=false -while [[ $# -gt 0 ]]; do - case "$1" in - --test-only) test_only=true; shift ;; - --debug) debug=true; shift ;; - *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; - esac -done - -# Paths and make target -make_target="module_documentation" -src_dir="build/px4_sitl_default/docs/modules" -dest_dir="docs/en/modules" - -# Run make target -if [ "$debug" = true ]; then - echo "๐Ÿ”ง Running 'make $make_target' (verbose)" - make $make_target -else - echo "๐Ÿ”ง Running 'make $make_target'" - make $make_target > /dev/null 2>&1 -fi - -# Verify build output -src_files=("$src_dir"/*) -if [ ${#src_files[@]} -eq 0 ]; then - echo "โŒ No generated module docs found in $src_dir. Build failed or path wrong." - exit 1 -fi - -# โ”€โ”€โ”€ Strip trailing whitespace from all generated module docs (unless test-only) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -if [ "$test_only" = false ]; then - echo "โœ‚๏ธ Removing trailing whitespace from generated module docs" - for src in "${src_files[@]}"; do - sed -i 's/[[:space:]]\+$//' "$src" - done -else - [ "$debug" = true ] && echo "๐Ÿงช Test-only mode: skipping whitespace removal" -fi - -echo "๐Ÿ” Checking module reference docs in $dest_dir" -mkdir -p "$dest_dir" - -changed=() -for src in "${src_files[@]}"; do - name=$(basename "$src") - dst="$dest_dir/$name" - - if [[ ! -e "$dst" ]]; then - [ "$debug" = true ] && echo "DEBUG: missing $dst" - changed+=("$name") - else - # Use diff -q -b (ignore whitespace changes) and --strip-trailing-cr (ignore CRLF vs LF) - if ! diff -q -b --strip-trailing-cr "$src" "$dst" > /dev/null; then - [ "$debug" = true ] && echo "DEBUG: diff -q -b --strip-trailing-cr '$src' '$dst' (exit_code=$?)" - changed+=("$name") - fi - fi -done - -if [ ${#changed[@]} -eq 0 ]; then - echo "โœ… All module reference docs are up to date." - exit 0 -fi - -echo "โš ๏ธ Detected updates in module docs:" -for f in "${changed[@]}"; do echo " - $f"; done - -if [ "$test_only" = true ]; then - echo "๐Ÿšจ Module reference docs need updating; rerun without --test-only to apply." - exit 1 -fi - -echo "๐Ÿ“‚ Copying updated module docs to $dest_dir" -for f in "${changed[@]}"; do - cp -rv "$src_dir/$f" "$dest_dir/$f" -done - -echo "๐Ÿšจ Module reference docs updated; please commit changes:" -echo " git status -s $dest_dir" -echo " git add $dest_dir/*" -echo " git commit -m 'docs: update module reference metadata'" -exit 1 diff --git a/Tools/ci/metadata_msg_docs.sh b/Tools/ci/metadata_msg_docs.sh deleted file mode 100755 index bc1c7e42ad..0000000000 --- a/Tools/ci/metadata_msg_docs.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -# -# metadata_msg_docs.sh โ€” generate and sync uORB message reference documentation -# -# Usage: -# Tools/ci/metadata_msg_docs.sh [--test-only] [--debug] -# -# Options: -# --test-only Run make target and comparison; exit 1 if diffs found, without copying files -# --debug Show full make output and debug info for file comparisons -# -set -euo pipefail -shopt -s nullglob - -# Parse flags -test_only=false -debug=false -while [[ $# -gt 0 ]]; do - case "$1" in - --test-only) test_only=true; shift ;; - --debug) debug=true; shift ;; - *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; - esac -done - -# Paths and make target -make_target="msg_docs" -src_dir="build/msg_docs" -dest_dir="docs/en/msg_docs" - -# Run make target -if [ "$debug" = true ]; then - echo "๐Ÿ”ง Running 'make $make_target' (verbose)" - make $make_target -else - echo "๐Ÿ”ง Running 'make $make_target'" - make $make_target > /dev/null 2>&1 -fi - -# Verify build output -src_files=("$src_dir"/*) -if [ ${#src_files[@]} -eq 0 ]; then - echo "โŒ No files found in $src_dir. Build target '$make_target' failed or path is wrong." - exit 1 -fi - -echo "๐Ÿ” Checking uORB message docs in $dest_dir" -mkdir -p "$dest_dir" - -changed=() -for src in "${src_files[@]}"; do - name=$(basename "$src") - dst="$dest_dir/$name" - - if [[ ! -f "$dst" ]]; then - [ "$debug" = true ] && echo "DEBUG: missing $dst" - changed+=("$name") - elif ! cmp -s "$src" "$dst"; then - [ "$debug" = true ] && echo "DEBUG: cmp -s '$src' '$dst'; echo \$?" - changed+=("$name") - fi -done - -if [ ${#changed[@]} -eq 0 ]; then - echo "โœ… All uORB message docs are up to date." - exit 0 -fi - -echo "โš ๏ธ Detected updates in the following docs:" -for f in "${changed[@]}"; do echo " - $f"; done - -if [ "$test_only" = true ]; then - echo "๐Ÿšจ uORB message docs need updating! Rerun without --test-only to apply changes." - exit 1 -fi - -echo "๐Ÿ“‚ Copying updated doc files to $dest_dir" -for f in "${changed[@]}"; do cp -v "$src_dir/$f" "$dest_dir/$f"; done - -echo "๐Ÿšจ uORB message docs updated; please commit changes:" -echo " git status -s $dest_dir" -echo " git add $dest_dir/*" -echo " git commit -m 'docs: update uORB message reference docs'" -exit 1 diff --git a/Tools/ci/metadata_parameters.sh b/Tools/ci/metadata_parameters.sh deleted file mode 100755 index 35c2f98851..0000000000 --- a/Tools/ci/metadata_parameters.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# -# metadata_parameters.sh โ€” generate and sync PX4 parameter reference documentation -# -# Usage: -# Tools/ci/metadata_parameters.sh [--test-only] [--debug] -# -# Options: -# --test-only Run make target and comparison; exit 1 if diffs found, without copying file -# --debug Show full make output and debug info for comparison -# -set -euo pipefail - -# Parse flags -test_only=false -debug=false -while [[ $# -gt 0 ]]; do - case "$1" in - --test-only) test_only=true; shift ;; - --debug) debug=true; shift ;; - *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; - esac -done - -# Paths and make target -make_target="parameters_metadata" -src_file="build/px4_sitl_default/docs/parameters.md" -dest_file="docs/en/advanced_config/parameter_reference.md" - -# Run make target -if [ "$debug" = true ]; then - echo "๐Ÿ”ง Running 'make $make_target' (verbose)" - make $make_target -else - echo "๐Ÿ”ง Running 'make $make_target'" - make $make_target > /dev/null 2>&1 -fi - -# โ”€โ”€โ”€ Strip trailing whitespace from the newly generated Markdown โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -echo "โœ‚๏ธ Removing trailing whitespace from $src_file" -sed -i 's/[[:space:]]\+$//' "$src_file" - -# Verify build output -if [[ ! -f "$src_file" ]]; then - echo "โŒ Generated file not found: $src_file" - exit 1 -fi - -echo "๐Ÿ” Comparing parameter docs" - -# Compare files -if cmp -s "$src_file" "$dest_file"; then - echo "โœ… Parameter reference is up to date." - exit 0 -else - if [ "$debug" = true ]; then - echo "DEBUG: cmp -s '$src_file' '$dest_file'; echo \$?" - fi - echo "โš ๏ธ Parameter reference needs updating." - if [ "$test_only" = true ]; then - exit 1 - fi - # Copy over updated file - echo "๐Ÿ“‚ Copying updated parameter_reference.md" - cp -v "$src_file" "$dest_file" - echo "๐Ÿšจ Parameter docs updated; commit the change:" - echo " git status -s $dest_file" - echo " git add $dest_file" - echo " git commit -m 'docs: update parameter reference metadata'" - exit 1 -fi diff --git a/Tools/ci/metadata_sync.sh b/Tools/ci/metadata_sync.sh new file mode 100755 index 0000000000..ef5cedb1f2 --- /dev/null +++ b/Tools/ci/metadata_sync.sh @@ -0,0 +1,447 @@ +#!/usr/bin/env bash +# +# metadata_sync.sh - Unified metadata generation and synchronization for PX4 docs +# +# Usage: +# Tools/ci/metadata_sync.sh [OPTIONS] [TYPES...] +# +# Types: +# parameters - Parameter reference (docs/en/advanced_config/parameter_reference.md) +# airframes - Airframe reference (docs/en/airframes/airframe_reference.md) +# modules - Module documentation (docs/en/modules/*.md) +# msg_docs - uORB message docs (docs/en/msg_docs/*.md + docs/en/middleware/dds_topics.md) +# uorb_graphs - uORB graph JSONs (docs/public/middleware/*.json) +# failsafe_web - Failsafe simulator (docs/public/config/failsafe/*.{js,wasm,json}) +# all - All of the above (default) +# +# Options: +# --generate Build the make targets to generate fresh metadata +# --sync Copy generated files to docs/ +# --verbose Show detailed output +# --help Show this help +# +# Exit codes: +# 0 - Success (files synced or already up-to-date) +# 1 - Error (build failed, missing files, etc.) +# +# Examples: +# # Full regeneration and sync (orchestrator use case) +# Tools/ci/metadata_sync.sh --generate --sync all +# +# # Just sync specific type (assumes already built) +# Tools/ci/metadata_sync.sh --sync parameters +# +# # Generate only, don't copy +# Tools/ci/metadata_sync.sh --generate uorb_graphs +# +set -euo pipefail +shopt -s nullglob + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Configuration +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +EMSCRIPTEN_VERSION="3.1.64" +EMSDK_DIR="${EMSDK_DIR:-_emscripten_sdk}" + +# All available metadata types +ALL_TYPES=(parameters airframes modules msg_docs uorb_graphs failsafe_web) + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Logging +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +VERBOSE=false + +log() { + echo "[metadata_sync] $*" +} + +log_verbose() { + if [[ "$VERBOSE" == "true" ]]; then + echo "[metadata_sync] $*" + fi +} + +die() { + echo "[metadata_sync] ERROR: $*" >&2 + exit 1 +} + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Help +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +show_help() { + head -n 35 "$0" | tail -n +2 | sed 's/^# \?//' + exit 0 +} + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Emscripten Setup +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +ensure_emscripten() { + if command -v emcc >/dev/null 2>&1; then + log_verbose "Emscripten already available: $(emcc --version | head -1)" + return 0 + fi + + log "Setting up Emscripten ${EMSCRIPTEN_VERSION}..." + + if [[ ! -d "$EMSDK_DIR" ]]; then + log_verbose "Cloning emsdk to $EMSDK_DIR" + if [[ "$VERBOSE" == "true" ]]; then + git clone https://github.com/emscripten-core/emsdk.git "$EMSDK_DIR" + else + git clone https://github.com/emscripten-core/emsdk.git "$EMSDK_DIR" >/dev/null 2>&1 + fi + fi + + pushd "$EMSDK_DIR" >/dev/null + if [[ "$VERBOSE" == "true" ]]; then + ./emsdk install "$EMSCRIPTEN_VERSION" + ./emsdk activate "$EMSCRIPTEN_VERSION" + else + ./emsdk install "$EMSCRIPTEN_VERSION" >/dev/null 2>&1 + ./emsdk activate "$EMSCRIPTEN_VERSION" >/dev/null 2>&1 + fi + popd >/dev/null + + # shellcheck source=/dev/null + source "${EMSDK_DIR}/emsdk_env.sh" >/dev/null 2>&1 + + log_verbose "Emscripten ready: $(emcc --version | head -1)" +} + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Whitespace Normalization +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +normalize_whitespace() { + local file="$1" + if [[ -f "$file" ]]; then + # Remove trailing whitespace from each line + sed -i 's/[[:space:]]*$//' "$file" + fi +} + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Generation Functions +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +generate_parameters() { + log "Generating parameters metadata..." + if [[ "$VERBOSE" == "true" ]]; then + make parameters_metadata + else + make parameters_metadata >/dev/null 2>&1 + fi +} + +generate_airframes() { + log "Generating airframes metadata..." + if [[ "$VERBOSE" == "true" ]]; then + make airframe_metadata + else + make airframe_metadata >/dev/null 2>&1 + fi +} + +generate_modules() { + log "Generating modules documentation..." + if [[ "$VERBOSE" == "true" ]]; then + make module_documentation + else + make module_documentation >/dev/null 2>&1 + fi +} + +generate_msg_docs() { + log "Generating message documentation..." + if [[ "$VERBOSE" == "true" ]]; then + make msg_docs + else + make msg_docs >/dev/null 2>&1 + fi +} + +generate_uorb_graphs() { + log "Generating uORB graphs..." + if [[ "$VERBOSE" == "true" ]]; then + make uorb_graphs + else + make uorb_graphs >/dev/null 2>&1 + fi +} + +generate_failsafe_web() { + ensure_emscripten + log "Generating failsafe web..." + if [[ "$VERBOSE" == "true" ]]; then + make failsafe_web + else + make failsafe_web >/dev/null 2>&1 + fi +} + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Sync Functions +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +sync_parameters() { + local src="build/px4_sitl_default/docs/parameters.md" + local dest="docs/en/advanced_config/parameter_reference.md" + + log "Syncing parameters..." + + if [[ ! -f "$src" ]]; then + die "Source file not found: $src (did you run --generate?)" + fi + + normalize_whitespace "$src" + mkdir -p "$(dirname "$dest")" + cp "$src" "$dest" + log_verbose " $src -> $dest" +} + +sync_airframes() { + local src="build/px4_sitl_default/docs/airframes.md" + local dest="docs/en/airframes/airframe_reference.md" + + log "Syncing airframes..." + + if [[ ! -f "$src" ]]; then + die "Source file not found: $src (did you run --generate?)" + fi + + normalize_whitespace "$src" + mkdir -p "$(dirname "$dest")" + cp "$src" "$dest" + log_verbose " $src -> $dest" +} + +sync_modules() { + local src_dir="build/px4_sitl_default/docs/modules" + local dest_dir="docs/en/modules" + + log "Syncing modules..." + + if [[ ! -d "$src_dir" ]]; then + die "Source directory not found: $src_dir (did you run --generate?)" + fi + + local src_files=("$src_dir"/*.md) + if [[ ${#src_files[@]} -eq 0 ]]; then + die "No .md files found in $src_dir" + fi + + mkdir -p "$dest_dir" + + for src in "${src_files[@]}"; do + normalize_whitespace "$src" + local name + name=$(basename "$src") + cp "$src" "$dest_dir/$name" + log_verbose " $src -> $dest_dir/$name" + done +} + +sync_msg_docs() { + local src_dir="build/px4_sitl_default/msg_docs" + local dest_dir="docs/en/msg_docs" + local middleware_dir="docs/en/middleware" + + log "Syncing message docs..." + + if [[ ! -d "$src_dir" ]]; then + die "Source directory not found: $src_dir (did you run --generate?)" + fi + + local src_files=("$src_dir"/*.md) + if [[ ${#src_files[@]} -eq 0 ]]; then + die "No .md files found in $src_dir" + fi + + mkdir -p "$dest_dir" + mkdir -p "$middleware_dir" + + for src in "${src_files[@]}"; do + normalize_whitespace "$src" + local name + name=$(basename "$src") + + # dds_topics.md goes to middleware dir + if [[ "$name" == "dds_topics.md" ]]; then + cp "$src" "$middleware_dir/$name" + log_verbose " $src -> $middleware_dir/$name" + else + cp "$src" "$dest_dir/$name" + log_verbose " $src -> $dest_dir/$name" + fi + done +} + +sync_uorb_graphs() { + local src_dir="Tools/uorb_graph" + local dest_dir="docs/public/middleware" + + log "Syncing uORB graphs..." + + local src_files=("$src_dir"/*.json) + if [[ ${#src_files[@]} -eq 0 ]]; then + die "No .json files found in $src_dir (did you run --generate?)" + fi + + mkdir -p "$dest_dir" + + for src in "${src_files[@]}"; do + local name + name=$(basename "$src") + cp "$src" "$dest_dir/$name" + log_verbose " $src -> $dest_dir/$name" + done +} + +sync_failsafe_web() { + local src_dir="build/px4_sitl_default_failsafe_web" + local dest_dir="docs/public/config/failsafe" + + log "Syncing failsafe web..." + + if [[ ! -d "$src_dir" ]]; then + die "Source directory not found: $src_dir (did you run --generate?)" + fi + + # Gather js, wasm, json files + local src_files=() + for ext in js wasm json; do + src_files+=("$src_dir"/*."$ext") + done + + if [[ ${#src_files[@]} -eq 0 ]]; then + die "No .js/.wasm/.json files found in $src_dir" + fi + + mkdir -p "$dest_dir" + + for src in "${src_files[@]}"; do + local name + name=$(basename "$src") + cp "$src" "$dest_dir/$name" + log_verbose " $src -> $dest_dir/$name" + done +} + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Main Logic +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +DO_GENERATE=false +DO_SYNC=false +SELECTED_TYPES=() + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --generate) + DO_GENERATE=true + shift + ;; + --sync) + DO_SYNC=true + shift + ;; + --verbose) + VERBOSE=true + shift + ;; + --help|-h) + show_help + ;; + -*) + die "Unknown option: $1" + ;; + *) + # It's a type + SELECTED_TYPES+=("$1") + shift + ;; + esac + done + + # Default to all types if none specified + if [[ ${#SELECTED_TYPES[@]} -eq 0 ]]; then + SELECTED_TYPES=("all") + fi + + # Expand "all" to all types + local expanded_types=() + for t in "${SELECTED_TYPES[@]}"; do + if [[ "$t" == "all" ]]; then + expanded_types+=("${ALL_TYPES[@]}") + else + expanded_types+=("$t") + fi + done + SELECTED_TYPES=("${expanded_types[@]}") + + # Validate types + for t in "${SELECTED_TYPES[@]}"; do + local valid=false + for valid_type in "${ALL_TYPES[@]}"; do + if [[ "$t" == "$valid_type" ]]; then + valid=true + break + fi + done + if [[ "$valid" == "false" ]]; then + die "Unknown type: $t (valid: ${ALL_TYPES[*]})" + fi + done + + # Must specify at least one action + if [[ "$DO_GENERATE" == "false" && "$DO_SYNC" == "false" ]]; then + die "Must specify at least one of: --generate, --sync" + fi +} + +main() { + parse_args "$@" + + log "Selected types: ${SELECTED_TYPES[*]}" + [[ "$DO_GENERATE" == "true" ]] && log "Actions: generate" + [[ "$DO_SYNC" == "true" ]] && log "Actions: sync" + + # Remove duplicates from SELECTED_TYPES + local -A seen + local unique_types=() + for t in "${SELECTED_TYPES[@]}"; do + if [[ -z "${seen[$t]:-}" ]]; then + seen[$t]=1 + unique_types+=("$t") + fi + done + SELECTED_TYPES=("${unique_types[@]}") + + # Generate phase + if [[ "$DO_GENERATE" == "true" ]]; then + log "=== Generation Phase ===" + for t in "${SELECTED_TYPES[@]}"; do + "generate_$t" + done + fi + + # Sync phase + if [[ "$DO_SYNC" == "true" ]]; then + log "=== Sync Phase ===" + for t in "${SELECTED_TYPES[@]}"; do + "sync_$t" + done + fi + + log "Done." + exit 0 +} + +main "$@" diff --git a/Tools/ci/metadata_uorb_graph.sh b/Tools/ci/metadata_uorb_graph.sh deleted file mode 100755 index 336b22bd20..0000000000 --- a/Tools/ci/metadata_uorb_graph.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash -# -# update_uorb_graphs.sh โ€” generate, compare, and sync uORB graph JSONs -# -# Usage: -# ./scripts/update_uorb_graphs.sh [--test-only] [--debug] -# -# Options: -# --test-only Run generation and comparison only; exit 1 if diffs found, without copying files -# --debug Echo debug info for missing or differing files and show full make output -# -# Examples: -# # CI mode: fail if docs need updates -# ./scripts/update_uorb_graphs.sh --test-only -# -# # Developer mode: regenerate and sync JSONs -# ./scripts/update_uorb_graphs.sh -# -set -euo pipefail - -shopt -s nullglob - -# Parse flags -test_only=false -debug=false -while [[ $# -gt 0 ]]; do - case "$1" in - --test-only) - test_only=true - shift - ;; - --debug) - debug=true - shift - ;; - *) - echo "Usage: $0 [--test-only] [--debug]" - exit 2 - ;; - esac -done - -# Paths -graph_dir="Tools/uorb_graph" -dest_dir="docs/public/middleware" - -# Generate uORB graphs (conditionally silent) -if [ "$debug" = true ]; then - echo "๐Ÿ”ง Generating uORB message graphs (verbose output)" - make uorb_graphs -else - echo "๐Ÿ”ง Generating uORB message graphs" - make uorb_graphs > /dev/null 2>&1 -fi - -# Verify generation -src_files=("$graph_dir"/*.json) -if [ ${#src_files[@]} -eq 0 ]; then - echo "โŒ No JSON files found in $graph_dir. Generation failed or path is wrong." - exit 1 -fi - -echo "๐Ÿ” Checking for updated uORB graph JSONs" -mkdir -p "$dest_dir" - -changed=() -for src in "${src_files[@]}"; do - name=$(basename "$src") - dst="$dest_dir/$name" - - if [[ ! -f "$dst" ]]; then - [ "$debug" = true ] && echo "DEBUG: $dst missing" - changed+=("$name") - - elif ! cmp -s "$src" "$dst"; then - [ "$debug" = true ] && echo "DEBUG: cmp -s '$src' '$dst'; echo \$?" - changed+=("$name") - fi -done - -if [ ${#changed[@]} -eq 0 ]; then - echo "โœ… All uORB graph JSONs are already in sync." - exit 0 -fi - -echo "โš ๏ธ Detected updates in the following files:" -for name in "${changed[@]}"; do - echo " - $name" -done - -if [ "$test_only" = true ]; then - echo - echo "๐Ÿšจ uORB graph docs need updating! Rerun without --test-only to apply changes." - exit 1 -fi - -echo -echo "๐Ÿ“‚ Copying updated files over" -for name in "${changed[@]}"; do - cp -v "$graph_dir/$name" "$dest_dir/$name" -done - -echo -echo "๐Ÿšจ uORB graph docs need updating! Review above, then run:" -echo " git status -s $dest_dir/" -echo " git add $dest_dir/*.json" -echo " git commit -m 'docs: metadata: update uORB graph JSONs'" -exit 1 diff --git a/Tools/ci/test_metadata_sync.sh b/Tools/ci/test_metadata_sync.sh new file mode 100755 index 0000000000..55dbc64a72 --- /dev/null +++ b/Tools/ci/test_metadata_sync.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# +# test_metadata_sync.sh - Test metadata_sync.sh locally using Docker +# +# Usage: +# Tools/ci/test_metadata_sync.sh [OPTIONS] [TYPES...] +# +# Options: +# --shell Drop into interactive shell instead of running sync +# --verbose Pass --verbose to metadata_sync.sh +# --skip-build Skip SITL build (use existing build artifacts) +# --help Show this help +# +# Types: +# Same as metadata_sync.sh: parameters, airframes, modules, msg_docs, uorb_graphs, failsafe_web, all +# +# Examples: +# # Test full regeneration +# Tools/ci/test_metadata_sync.sh all +# +# # Test just parameters (faster) +# Tools/ci/test_metadata_sync.sh parameters +# +# # Drop into shell for debugging +# Tools/ci/test_metadata_sync.sh --shell +# +# # Skip build if you already have artifacts +# Tools/ci/test_metadata_sync.sh --skip-build --verbose all +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +DOCKER_IMAGE="px4io/px4-dev:v1.17.0-alpha1" +CONTAINER_NAME="px4-metadata-test-$$" + +SHELL_MODE=false +VERBOSE="" +SKIP_BUILD=false +TYPES=() + +show_help() { + head -n 28 "$0" | tail -n +2 | sed 's/^# \?//' + exit 0 +} + +cleanup() { + echo "[test] Cleaning up container..." + docker rm -f "$CONTAINER_NAME" 2>/dev/null || true +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --shell) + SHELL_MODE=true + shift + ;; + --verbose) + VERBOSE="--verbose" + shift + ;; + --skip-build) + SKIP_BUILD=true + shift + ;; + --help|-h) + show_help + ;; + -*) + echo "Unknown option: $1" >&2 + exit 1 + ;; + *) + TYPES+=("$1") + shift + ;; + esac + done + + # Default to all types + if [[ ${#TYPES[@]} -eq 0 ]]; then + TYPES=("all") + fi +} + +main() { + parse_args "$@" + + cd "$REPO_ROOT" + + echo "[test] Using Docker image: $DOCKER_IMAGE" + echo "[test] Repository root: $REPO_ROOT" + + # Pull image if not present + if ! docker image inspect "$DOCKER_IMAGE" >/dev/null 2>&1; then + echo "[test] Pulling Docker image..." + docker pull "$DOCKER_IMAGE" + fi + + trap cleanup EXIT + + # Handle git worktrees: the .git file points to the main repo's .git directory + # We need to mount that directory too so git works inside the container + local git_mounts=() + if [[ -f "$REPO_ROOT/.git" ]]; then + # It's a worktree - read the gitdir path and mount it + local gitdir + gitdir=$(grep '^gitdir:' "$REPO_ROOT/.git" | cut -d' ' -f2) + if [[ -n "$gitdir" ]]; then + # Mount the gitdir at the same path so the .git file reference works + git_mounts+=("-v" "$gitdir:$gitdir:ro") + # Also need the main .git directory (parent of worktrees/) + local main_git_dir + main_git_dir=$(dirname "$(dirname "$gitdir")") + git_mounts+=("-v" "$main_git_dir:$main_git_dir:ro") + echo "[test] Detected git worktree, mounting git directories" + fi + fi + + if [[ "$SHELL_MODE" == "true" ]]; then + echo "[test] Starting interactive shell..." + echo "[test] Run: Tools/ci/metadata_sync.sh --generate --sync all" + docker run -it --rm \ + --name "$CONTAINER_NAME" \ + -v "$REPO_ROOT:/src" \ + "${git_mounts[@]}" \ + -w /src \ + "$DOCKER_IMAGE" \ + /bin/bash + else + echo "[test] Running metadata sync for: ${TYPES[*]}" + + # Build the command + local cmd="" + + if [[ "$SKIP_BUILD" == "false" ]]; then + cmd="Tools/ci/metadata_sync.sh --generate --sync $VERBOSE ${TYPES[*]}" + else + cmd="Tools/ci/metadata_sync.sh --sync $VERBOSE ${TYPES[*]}" + fi + + echo "[test] Command: $cmd" + + docker run --rm \ + --name "$CONTAINER_NAME" \ + -v "$REPO_ROOT:/src" \ + "${git_mounts[@]}" \ + -w /src \ + "$DOCKER_IMAGE" \ + /bin/bash -c "$cmd" + + echo "" + echo "[test] Done! Check git status for changes:" + echo " git status -s docs/" + echo "" + echo "[test] To see what changed:" + echo " git diff docs/" + fi +} + +main "$@"