diff --git a/.github/workflows/build_all_targets.yml b/.github/workflows/build_all_targets.yml index ebcc10c447d..605348619a5 100644 --- a/.github/workflows/build_all_targets.yml +++ b/.github/workflows/build_all_targets.yml @@ -90,6 +90,10 @@ jobs: - name: Install Python Dependencies run: pip3 install pyyaml kconfiglib 'empy>=3.3,<4' pyros-genmsg + - name: Test Build Matrix and Slot Assignment + working-directory: Tools/ci + run: python3 -m unittest -v test_generate_board_targets_json test_build_all_runner + # The generator runs as a standalone assignment so a non-zero exit # fails the step; embedded in `echo "x=$(...)"` it would be masked # and the empty output silently skips the whole build matrix. @@ -236,9 +240,14 @@ jobs: ccache -s ccache -z - - name: Building Artifacts for [${{ matrix.targets }}] - run: | - ./Tools/ci/build_all_runner.sh ${{matrix.targets}} ${{matrix.arch}} + - name: Building Artifacts + run: ./Tools/ci/build_all_runner.py --slots ${{ matrix.slots }} ${{ join(matrix.targets, ' ') }} + + # Full per-target compiler output, one collapsed group per target, kept + # out of the build step so a running job shows progress, not logs. + - name: Build Logs + if: ${{ !cancelled() }} + run: ./Tools/ci/build_all_runner.py --print-logs - name: Arrange Build Artifacts run: | diff --git a/.gitignore b/.gitignore index 3af4ce0808f..e495f700cfb 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ Testing/ Packages/* build/* build_*/ +/.build_slots/ /docker-context/ cscope.out cscope.in.out diff --git a/Tools/ci/build_all_config.yml b/Tools/ci/build_all_config.yml index b86fedc6c43..2a4f9aa4e5e 100644 --- a/Tools/ci/build_all_config.yml +++ b/Tools/ci/build_all_config.yml @@ -14,6 +14,11 @@ containers: runners: seeder_cpu: 8 matrix_cpu: 4 + # Targets each build job compiles concurrently, each in its own git + # worktree (NuttX builds inside its source tree, so one per checkout). + # 2 keeps a 4-vCPU runner ~80% busy; more slots or larger runners cost more + # than they save (PR #28870). 1 = sequential. + build_slots: 2 # Default ccache max-size for build groups cache: diff --git a/Tools/ci/build_all_runner.py b/Tools/ci/build_all_runner.py new file mode 100755 index 00000000000..0a0e50590c2 --- /dev/null +++ b/Tools/ci/build_all_runner.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""Build a group of PX4 targets for the build_all_targets workflow. + +Usage: build_all_runner.py [--slots N] TARGET [TARGET ...] + build_all_runner.py --print-logs + +With --slots 1 the targets build one after another in the checkout, exactly +like running `make TARGET` for each. With --slots N the targets are split +across N build slots that run concurrently: slot 0 is the checkout itself, +slots 1..N-1 are git worktrees of the same commit under .build_slots/. +NuttX compiles inside platforms/nuttx/NuttX/{nuttx,apps} rather than in +build//, so two builds can only run at once in separate checkouts. +Worktrees build the committed HEAD, not uncommitted changes in the checkout. + +All submodules are fetched into the checkout before any build starts, and +each slot clones them from there (hardlinked, no network). The builds run +with GIT_SUBMODULES_ARE_EVIL=1 so CMake does not fetch or sync submodules +itself, which concurrent builds would do at the same time on the shared +.git/config. + +Build output streams live with every line prefixed by its slot, target and +seconds since that target started (the mavsdk_tests runner convention), so +interleaved lines from concurrent builds stay attributable; compiler errors +and warnings are colored. Each target's unprefixed output is also kept in +.build_slots/logs/, and --print-logs prints those afterwards as one +collapsed group per target titled with its result. Every target is built even if an earlier +one fails, and the script exits non-zero if any target failed. Finally, the +slot build directories are moved into build/ so package_build_artifacts.sh +finds every target in one place. +""" +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +SLOTS_DIR = '.build_slots' +LOGS_DIR = f'{SLOTS_DIR}/logs' +RESULTS_FILE = f'{LOGS_DIR}/results.json' +FAILURE_EXCERPT_LINES = 60 +FAILURE_TAIL_LINES = 5 + +ANSI_ESCAPE = re.compile(r'\x1b\[[0-9;]*[A-Za-z]') +# ninja progress lines ("[12/990] Building CXX object ..."), noise in a failure excerpt +NINJA_PROGRESS = re.compile(r'^\[\d+/\d+\] ') +# linker memory report rows, e.g. " FLASH_AXIM: 1405156 B 1952 KB 70.30%" +MEMORY_REGION = re.compile(r'^\s*(\S+):\s+\d+\s*[KMG]?B\s+\d+\s*[KMG]?B\s+([\d.]+)%') + +# Makefile targets that build inside another target's build directory +# (Makefile metadata rules all run `make px4_sitl_default ...`). +METADATA_TARGETS = {'airframe_metadata', 'parameters_metadata', + 'module_documentation', 'extract_events'} + +print_lock = threading.Lock() + +# Submodules are fetched before building; stop CMake's per-configure +# check_submodules.sh from syncing and updating them concurrently. +BUILD_ENV = {**os.environ, 'GIT_SUBMODULES_ARE_EVIL': '1'} + +# ANSI colors render in the Actions log viewer; NO_COLOR turns them off. +USE_COLOR = 'NO_COLOR' not in os.environ +GRAY, RED, YELLOW, GREEN, RESET = '\033[90m', '\033[91m', '\033[93m', '\033[92m', '\033[0m' + + +def colorize(text, color): + return f'{color}{text}{RESET}' if USE_COLOR else text + + +def highlight(line): + """Color compiler and build-system errors red and warnings yellow.""" + lower = line.lower() + if 'error:' in lower or 'failed:' in lower or ('make' in lower and 'error ' in lower): + return colorize(line, RED) + if 'warning:' in lower: + return colorize(line, YELLOW) + return line + + +def build_dir_name(target): + """Directory under build/ that `make target` writes to.""" + if target in METADATA_TARGETS: + return 'px4_sitl_default' + if target.endswith('_deb'): + # Makefile deb rule builds _deb into build/_default + return target[:-len('_deb')] + '_default' + return target + + +def assign_slots(targets, slots): + """Split targets into at most `slots` contiguous, count-balanced lists. + + Targets that write to the same build directory always share a slot, + otherwise two slots would produce the same build/ directory and one + would overwrite the other. Contiguous splitting keeps variants of one + board (alphabetically adjacent) together, and the assignment is + deterministic, so a target lands in the same slot path on every run. + """ + by_dir = {} + for target in targets: + by_dir.setdefault(build_dir_name(target), []).append(target) + units = list(by_dir.values()) + + slots = max(1, min(slots, len(units))) + assigned = [[] for _ in range(slots)] + position = 0 + for unit in units: + assigned[position * slots // len(targets)].extend(unit) + position += len(unit) + return [a for a in assigned if a] + + +def check_plan(plan): + """Refuse a plan where two slots would write the same build/ directory. + + collect_build_dirs() would otherwise replace one slot's output with the + other's without any error. + """ + owner = {} + for slot, targets in enumerate(plan): + for name in {build_dir_name(t) for t in targets}: + if owner.setdefault(name, slot) != slot: + sys.exit(f'refusing to build: build/{name} would be written by ' + f'slot {owner[name]} and slot {slot}') + + +def git(*args, cwd): + return subprocess.run(['git', *args], cwd=cwd, check=True, + capture_output=True, text=True).stdout.strip() + + +def fetch_submodules(root): + """Fetch every submodule, recursively, into the checkout.""" + start = time.monotonic() + git('submodule', 'update', '--init', '--recursive', '--jobs', '8', cwd=root) + say(f'fetched all submodules in {time.monotonic() - start:.0f}s') + + +def prepare_slot(root, index): + """Return the source directory for slot `index`, creating it if needed. + + A slot is a worktree of the checkout's commit whose submodules are + cloned from the checkout's own (url..insteadOf), so no network is + used. An existing slot (from an earlier local run) is moved to the + checkout's current commit instead of being recreated. + """ + if index == 0: + return root + path = root / SLOTS_DIR / f's{index}' + head = git('rev-parse', 'HEAD', cwd=root) + if path.exists(): + git('checkout', '--quiet', '--detach', head, cwd=path) + else: + git('worktree', 'add', '--quiet', '--detach', str(path), head, cwd=root) + local = [] + for line in git('submodule', 'status', '--recursive', cwd=root).splitlines(): + sub = line.split()[1] + url = git('remote', 'get-url', 'origin', cwd=root / sub) + local += ['-c', f'url.{root / sub}.insteadOf={url}'] + # recorded submodule commits are often detached, not on a branch + git('-c', 'protocol.file.allow=always', '-c', 'uploadpack.allowAnySHA1InWant=true', *local, + 'submodule', 'update', '--init', '--recursive', '--jobs', '8', cwd=path) + return path + + +def say(*lines): + with print_lock: + for line in lines: + print(line) + sys.stdout.flush() + + +def memory_report(lines): + """The linker's 'Memory region' table from a build log, or [] if absent.""" + for i, line in enumerate(lines): + if line.lstrip().startswith('Memory region'): + rows = [line] + for row in lines[i + 1:]: + if not MEMORY_REGION.match(row): + break + rows.append(row) + return rows + return [] + + +def flash_usage(report): + """Highest '%age Used' among the FLASH regions of a memory report.""" + used = [float(m.group(2)) for m in map(MEMORY_REGION.match, report) + if m and 'FLASH' in m.group(1).upper()] + return max(used) if used else None + + +def is_error_line(line): + plain = ANSI_ESCAPE.sub('', line) + return 'error:' in plain.lower() or plain.startswith('FAILED:') + + +def failure_excerpt(lines): + """The part of a failed build log that explains the failure. + + Starts at the first compiler/linker error or ninja FAILED line (ninja + keeps printing other jobs after it, so a plain tail can miss it), skips + the progress lines of those other jobs, and always ends with the last + lines of the log, where make reports the exit. + """ + start = next((i for i, line in enumerate(lines) if is_error_line(line)), None) + if start is None: + return lines[-FAILURE_EXCERPT_LINES:] + diagnostics = [i for i in range(start, len(lines)) + if not NINJA_PROGRESS.match(ANSI_ESCAPE.sub('', lines[i]))] + shown = diagnostics[:FAILURE_EXCERPT_LINES] + tail = [i for i in range(len(lines) - FAILURE_TAIL_LINES, len(lines)) if i > shown[-1]] + return [lines[i] for i in shown] + (['...'] if tail else []) + [lines[i] for i in tail] + + +def annotation(text): + """Escape text for a workflow command message.""" + return text.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A') + + +def result_line(result): + status = '✅' if result['returncode'] == 0 else f'❌ exit {result["returncode"]}' + flash = f', flash {result["flash"]:.1f}%' if result.get('flash') is not None else '' + return f'{status} {result["target"]} (slot {result["slot"]}, {result["seconds"]:.0f}s{flash})' + + +def build(source, target, slot, logs, width): + """Build one target, streaming its output with a slot/target/time prefix.""" + start = time.monotonic() + + def prefix(): + tag = f'[{time.monotonic() - start:7.1f}|s{slot} {target.ljust(width)}]' + return colorize(tag, GRAY) + + say(f'{prefix()} ▶ make {target}') + log = logs / f'{target}.log' + with open(log, 'w') as out: + process = subprocess.Popen(['make', target], cwd=source, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors='replace', + env=BUILD_ENV) + assert process.stdout is not None + for line in process.stdout: + out.write(line) + say(f'{prefix()} {highlight(line.rstrip())}') + returncode = process.wait() + + lines = log.read_text(errors='replace').splitlines() + report = memory_report(lines) + result = {'target': target, 'slot': slot, 'returncode': returncode, + 'seconds': time.monotonic() - start, 'flash': flash_usage(report)} + # each block is printed under one lock so the other slot cannot interleave + if returncode == 0: + say(f'{prefix()} {colorize(result_line(result), GREEN)}', + *(f' {row}' for row in report)) + else: + excerpt = failure_excerpt(lines) + plain = [ANSI_ESCAPE.sub('', l).strip() for l in excerpt] + # prefer the compiler/linker message over ninja's FAILED: line + first_error = next((l for l in plain if 'error:' in l.lower()), + next((l for l in plain if l.startswith('FAILED:')), + f'make exited with {returncode}')) + say(f'{prefix()} {colorize(result_line(result), RED)}', + f'--- {target}: failure ---', *(highlight(l) for l in excerpt), + f'--- end of {target} failure (full log in the Build Logs step) ---', + f'::error title={target} failed::{annotation(first_error)}') + return result + + +def build_slot(source, targets, slot, logs, width): + return [build(source, target, slot, logs, width) for target in targets] + + +def collect_build_dirs(root, source, targets): + """Move a slot's build directories into the checkout's build/.""" + for name in {build_dir_name(t) for t in targets}: + src = source / 'build' / name + if not src.exists(): + continue + dst = root / 'build' / name + if dst.exists(): + shutil.rmtree(dst) + dst.parent.mkdir(exist_ok=True) + shutil.move(src, dst) + + +def write_summary(results): + """Results table for the terminal and, in Actions, the job summary page.""" + def flash(r): + return f'{r["flash"]:.1f}%' if r.get('flash') is not None else '-' + + say('', f'{"target":<50} {"slot":>4} {"time":>8} {"flash":>7} result') + for r in results: + say(f'{r["target"]:<50} {r["slot"]:>4} {r["seconds"]:7.0f}s {flash(r):>7} ' + f'{"ok" if r["returncode"] == 0 else "FAILED"}') + summary = os.environ.get('GITHUB_STEP_SUMMARY') + if summary: + with open(summary, 'a') as f: + f.write('| target | slot | time | flash | result |\n|---|---|---|---|---|\n') + for r in results: + f.write(f'| `{r["target"]}` | {r["slot"]} | {r["seconds"]:.0f}s | {flash(r)} | ' + f'{"✅" if r["returncode"] == 0 else "❌"} |\n') + + +def print_logs(root): + """Print each target's full build log as one collapsed group.""" + results_file = root / RESULTS_FILE + if not results_file.exists(): + # build step died before recording results: print whatever logs exist + for log in sorted((root / LOGS_DIR).glob('*.log')): + print(f'::group::{log.stem} (no result recorded)') + print(log.read_text(errors='replace'), end='') + print('::endgroup::') + return 0 + for result in json.loads(results_file.read_text()): + log = root / LOGS_DIR / f'{result["target"]}.log' + print(f'::group::{result_line(result)}') + print(log.read_text(errors='replace') if log.exists() else '(no log)', end='') + print('::endgroup::') + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description='Build a group of PX4 targets, optionally in concurrent slots.') + parser.add_argument('--slots', type=int, default=1, + help='number of concurrent builds (default: 1)') + parser.add_argument('--print-logs', action='store_true', + help='print the build logs of the last run and exit') + parser.add_argument('targets', nargs='*', help='make targets to build') + args = parser.parse_args() + + root = Path(git('rev-parse', '--show-toplevel', cwd=None)) + if args.print_logs: + return print_logs(root) + if not args.targets: + parser.error('no targets given') + + plan = assign_slots(args.targets, args.slots) + check_plan(plan) + for slot, targets in enumerate(plan): + say(f'slot {slot}: {" ".join(targets)}') + say('') + + logs = root / LOGS_DIR + logs.mkdir(parents=True, exist_ok=True) + fetch_submodules(root) + sources = [prepare_slot(root, slot) for slot in range(len(plan))] + + width = max(len(t) for t in args.targets) + with ThreadPoolExecutor(max_workers=len(plan)) as pool: + futures = [pool.submit(build_slot, sources[slot], targets, slot, logs, width) + for slot, targets in enumerate(plan)] + results = [r for f in futures for r in f.result()] + (root / RESULTS_FILE).write_text(json.dumps(results)) + + for slot, targets in enumerate(plan[1:], start=1): + collect_build_dirs(root, sources[slot], targets) + + write_summary(results) + failed = [r['target'] for r in results if r['returncode'] != 0] + if failed: + say('', f'{len(failed)} of {len(results)} targets failed: {" ".join(failed)}') + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/Tools/ci/build_all_runner.sh b/Tools/ci/build_all_runner.sh deleted file mode 100755 index 132b634ebdf..00000000000 --- a/Tools/ci/build_all_runner.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# This script is meant to be used by the build_all.yml workflow in a github runner -# Please only modify if you know what you are doing -set -e - -# NuttX .o/.a live in the shared submodule trees, not per-board build dirs. -# Wipe them between boards so stale objects from a prior board's defconfig -# don't get linked into the next board's elf. -clean_nuttx_state() { - git -C platforms/nuttx/NuttX/nuttx clean -dXf -q - git -C platforms/nuttx/NuttX/apps clean -dXf -q -} - -targets=$1 -for target in ${targets//,/ } -do - echo "::group::Building: [${target}]" - start=$(date +%s) - make $target - stop=$(date +%s) - diff=$(($stop-$start)) - build_time="$(($diff /60/60))h $(($diff /60))m $(($diff % 60))s elapsed" - echo -e "\033[0;32mBuild Time: [$build_time]" - echo "::endgroup::" - clean_nuttx_state -done diff --git a/Tools/ci/generate_board_targets_json.py b/Tools/ci/generate_board_targets_json.py index 0b61aa51bb9..0599239fdce 100755 --- a/Tools/ci/generate_board_targets_json.py +++ b/Tools/ci/generate_board_targets_json.py @@ -88,10 +88,6 @@ def chunks_merged(arr, size): result.pop() return result -def comma_targets(targets): - # turns array of targets into a comma split string - return ",".join(targets) - def process_target(px4board_file, target_name, manufacturer_name=None, board_dir_name=None, label=None): # reads through the board file and grabs # useful information for building @@ -299,6 +295,8 @@ if (args.group): DEFAULT_CACHE_SIZE = cache_config.get('default_size', '400M') CHIP_CACHE_SIZES = cache_config.get('chip_sizes', {}) + BUILD_SLOTS = ci_config.get('runners', {}).get('build_slots', 1) + if(verbose): print(f'=:Architectures: [{grouped_targets.keys()}]') @@ -376,7 +374,7 @@ if (args.group): print(f'=: [{man}][{man_len}] -> {group_name}') final_groups.append({ "container": grouped_targets[arch]['container'], - "targets": comma_targets(man_targets[man]), + "targets": man_targets[man], "arch": arch, "chip_family": chip, "runner": runner, @@ -391,7 +389,7 @@ if (args.group): print(f'=: [{man}][{man_len}] -> {group_name} ({len(chunk)})') final_groups.append({ "container": grouped_targets[arch]['container'], - "targets": comma_targets(chunk), + "targets": chunk, "arch": arch, "chip_family": chip, "runner": runner, @@ -418,7 +416,7 @@ if (args.group): print(f'=: [misc][{len(chunk)}] -> {group_name}') final_groups.append({ "container": grouped_targets[arch]['container'], - "targets": comma_targets(chunk), + "targets": chunk, "arch": arch, "chip_family": chip, "runner": runner, @@ -444,7 +442,7 @@ if (args.group): print(f'=: [special][{len(chunk)}] -> {group_name}') final_groups.append({ "container": grouped_targets[arch]['container'], - "targets": comma_targets(chunk), + "targets": chunk, "arch": arch, "chip_family": "special", "runner": runner, @@ -461,7 +459,7 @@ if (args.group): if all_targets: final_groups.append({ "container": grouped_targets[arch]['container'], - "targets": comma_targets(all_targets), + "targets": all_targets, "arch": arch, "chip_family": "voxl2", "runner": runner, @@ -484,7 +482,7 @@ if (args.group): group_name = f"{arch}-{chunk_counter}" final_groups.append({ "container": grouped_targets[arch]['container'], - "targets": comma_targets(chunk), + "targets": chunk, "arch": arch, "chip_family": "native", "runner": runner, @@ -501,6 +499,7 @@ if (args.group): for g in final_groups: g['cache_size'] = CHIP_CACHE_SIZES.get(g['chip_family'], DEFAULT_CACHE_SIZE) g['cache_prefix'] = f"ccache-{g['chip_family']}-{g['runner']}" + g['slots'] = BUILD_SLOTS if(verbose): import pprint diff --git a/Tools/ci/test_build_all_runner.py b/Tools/ci/test_build_all_runner.py new file mode 100644 index 00000000000..41ca80cd091 --- /dev/null +++ b/Tools/ci/test_build_all_runner.py @@ -0,0 +1,119 @@ +"""Check build_all_runner.py's slot split and what it reports per build. + +A wrong split fails silently: two slots writing the same build/ directory +overwrite each other's firmware, and an unbalanced split throws away the +time the slots are meant to save. With two builds sharing one log stream, +the failure excerpt and memory report printed per target are what people +read, so they must contain the actual error and the full memory table. +""" + +import unittest + +from build_all_runner import (assign_slots, check_plan, failure_excerpt, + flash_usage, memory_report) + +# ninja keeps printing the jobs already running after the failing one +NINJA_FAILURE = ( + ['[1/990] Building C object src/lib/crc/crc.c.obj'] + + ['FAILED: src/modules/ekf2/CMakeFiles/ekf2.dir/EKF2.cpp.obj', + '/usr/bin/arm-none-eabi-g++ -c ../../src/modules/ekf2/EKF2.cpp', + '../../src/modules/ekf2/EKF2.cpp:42:5: \x1b[01;31m\x1b[Kerror: \x1b[m\x1b[K' + "'foo' was not declared in this scope"] + + [f'[{n}/990] Building CXX object src/modules/other{n}.cpp.obj' for n in range(2, 80)] + + ['ninja: build stopped: subcommand failed.', + 'make: *** [Makefile:232: atl_mantis-edu_default] Error 1'] +) + +MEMORY_TAIL = [ + '[988/990] Linking CXX executable atl_mantis-edu_default.elf', + 'Memory region Used Size Region Size %age Used', + ' FLASH_ITCM: 0 B 1952 KB 0.00%', + ' FLASH_AXIM: 1405156 B 1952 KB 70.30%', + ' SRAM1: 48272 B 368 KB 12.81%', + '[989/990] Generating ../../atl_mantis-edu_default.bin', +] + + +class FailureExcerptTest(unittest.TestCase): + def test_starts_at_the_error_not_the_tail(self): + excerpt = failure_excerpt(NINJA_FAILURE) + self.assertTrue(excerpt[0].startswith('FAILED:')) + self.assertTrue(any("'foo' was not declared" in line for line in excerpt)) + + def test_skips_other_jobs_progress_lines(self): + excerpt = failure_excerpt(NINJA_FAILURE) + self.assertFalse(any('Building CXX object src/modules/other' in l for l in excerpt)) + + def test_keeps_the_make_exit_line(self): + self.assertEqual(failure_excerpt(NINJA_FAILURE)[-1], NINJA_FAILURE[-1]) + + def test_no_error_line_falls_back_to_tail(self): + lines = [f'line {n}' for n in range(100)] + self.assertEqual(failure_excerpt(lines)[-1], 'line 99') + + +class MemoryReportTest(unittest.TestCase): + def test_extracts_whole_table(self): + report = memory_report(MEMORY_TAIL) + self.assertEqual(report, MEMORY_TAIL[1:5]) + self.assertEqual(flash_usage(report), 70.30) + + def test_posix_build_has_no_report(self): + self.assertEqual(memory_report(['[1/2] Linking CXX executable px4']), []) + self.assertIsNone(flash_usage([])) + +MISC_STM32F7 = [ + 'atl_mantis-edu_default', 'av_x-v1_default', 'corvon_v5_default', + 'cubepilot_cubeyellow_default', 'freefly_can-rtk-gps_canbootloader', + 'freefly_can-rtk-gps_default', 'holybro_kakutef7_default', + 'holybro_pix32v5_default', 'modalai_fc-v1_default', 'mro_ctrl-zero-f7_default', + 'mro_ctrl-zero-f7-oem_default', 'mro_x21-777_default', 'radiolink_PIX6_default', + 'sky-drones_smartap-airlink_default', +] + + +class AssignSlotsTest(unittest.TestCase): + def test_one_slot_keeps_order(self): + self.assertEqual(assign_slots(MISC_STM32F7, 1), [MISC_STM32F7]) + + def test_two_slots_split_evenly_in_order(self): + plan = assign_slots(MISC_STM32F7, 2) + self.assertEqual([len(p) for p in plan], [7, 7]) + self.assertEqual(plan[0] + plan[1], MISC_STM32F7) + + def test_uneven_count_differs_by_at_most_one(self): + plan = assign_slots(MISC_STM32F7[:13], 3) + sizes = [len(p) for p in plan] + self.assertEqual(sum(sizes), 13) + self.assertLessEqual(max(sizes) - min(sizes), 1) + + def test_deb_stays_with_default(self): + # make modalai_voxl2_deb builds into build/modalai_voxl2_default + plan = assign_slots(['modalai_voxl2_default', 'modalai_voxl2_deb'], 2) + self.assertEqual(plan, [['modalai_voxl2_default', 'modalai_voxl2_deb']]) + + def test_metadata_stays_with_sitl_default(self): + # the metadata rules all build inside build/px4_sitl_default + base = ['airframe_metadata', 'parameters_metadata', 'extract_events', + 'px4_sitl_allyes', 'px4_sitl_default', 'px4_sitl_sih'] + for slots in (2, 3, 4): + plan = assign_slots(base, slots) + sitl = [p for p in plan if 'px4_sitl_default' in p][0] + self.assertTrue({'airframe_metadata', 'parameters_metadata', + 'extract_events'} <= set(sitl), plan) + + def test_plan_splitting_a_build_dir_is_refused(self): + with self.assertRaises(SystemExit): + check_plan([['modalai_voxl2_default'], ['modalai_voxl2_deb']]) + with self.assertRaises(SystemExit): + check_plan([['px4_sitl_default'], ['parameters_metadata']]) + check_plan(assign_slots(['airframe_metadata', 'px4_sitl_allyes', + 'px4_sitl_default', 'modalai_voxl2_default', + 'modalai_voxl2_deb'], 4)) + + def test_more_slots_than_targets(self): + self.assertEqual(assign_slots(['a', 'b'], 4), [['a'], ['b']]) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/ci/test_generate_board_targets_json.py b/Tools/ci/test_generate_board_targets_json.py new file mode 100644 index 00000000000..cbd934098e3 --- /dev/null +++ b/Tools/ci/test_generate_board_targets_json.py @@ -0,0 +1,97 @@ +"""Check the build_all_targets matrix that generate_board_targets_json.py emits. + +The generator decides what CI builds, and its mistakes are silent: a board +dropped from every group is never built, two groups with the same name +overwrite each other's artifacts and caches, and a group whose cache +namespace has no seeder always starts cold. These tests run the generator +the way the workflow does and check its output against the board tree. +""" + +import json +import subprocess +import sys +import unittest +from collections import Counter +from pathlib import Path + +from build_all_runner import METADATA_TARGETS, build_dir_name + +ROOT = Path(__file__).resolve().parents[2] +GENERATOR = ROOT / 'Tools' / 'ci' / 'generate_board_targets_json.py' +BOARDS = ROOT / 'boards' + +# Workflow expressions read these keys from every build matrix entry. +MATRIX_KEYS = {'container', 'runner', 'group', 'targets', 'slots', + 'cache_prefix', 'cache_size'} + + +def generate(*args): + # the generator resolves boards/ relative to the repo root, as in CI + output = subprocess.run([sys.executable, str(GENERATOR), *args], cwd=ROOT, + check=True, capture_output=True, text=True).stdout + return json.loads(output)['include'] + + +def companion_targets(): + """Targets a parent target builds itself (boards/*/*/companion_targets).""" + companions = set() + for path in BOARDS.glob('*/*/companion_targets'): + companions |= {line.strip() for line in path.read_text().splitlines() + if line.strip() and not line.startswith('#')} + return companions + + +class BuildMatrixTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.flat = [entry['target'] for entry in generate()] + cls.groups = generate('--group') + cls.seeders = generate('--group', '--seeders') + cls.grouped = [t for g in cls.groups for t in g['targets']] + + def test_every_target_built_exactly_once(self): + duplicates = [t for t, n in Counter(self.grouped).items() if n > 1] + self.assertEqual(duplicates, []) + companions = companion_targets() + missing = set(self.flat) - companions - set(self.grouped) + self.assertEqual(missing, set(), 'targets no build job compiles') + # a companion is already built by its parent target's make rule + self.assertEqual(companions & set(self.grouped), set()) + + def test_runner_knows_every_non_board_target(self): + # Targets that are not board configs build inside another target's + # build/ directory; the runner must map them, or two slots write the + # same directory and one overwrites the other. + extra = set(self.grouped) - set(self.flat) + self.assertLessEqual(extra, METADATA_TARGETS) + for target in extra: + self.assertEqual(build_dir_name(target), 'px4_sitl_default') + + def test_group_names_unique(self): + # group names key the uploaded artifact and the ccache namespace + names = Counter(g['group'] for g in self.groups) + self.assertEqual([n for n, count in names.items() if count > 1], []) + + def test_matrix_entries_match_workflow(self): + for g in self.groups: + with self.subTest(group=g.get('group')): + self.assertLessEqual(MATRIX_KEYS, set(g)) + self.assertIsInstance(g['targets'], list) + self.assertTrue(g['targets']) + self.assertEqual(g['len'], len(g['targets'])) + self.assertIsInstance(g['slots'], int) + self.assertGreaterEqual(g['slots'], 1) + + def test_every_cache_namespace_has_a_seeder(self): + # the generator seeds special groups from nothing on purpose; every + # other group must restore from a seeder in its own namespace + seeded = {s['cache_prefix'] for s in self.seeders} + for g in self.groups: + if g['chip_family'] == 'special': + continue + with self.subTest(group=g['group']): + self.assertIn(g['cache_prefix'], seeded) + + +if __name__ == '__main__': + unittest.main()