diff --git a/scripts/cmake_bootstrap.sh b/scripts/cmake_bootstrap.sh index eb9bee1b..855a7900 100755 --- a/scripts/cmake_bootstrap.sh +++ b/scripts/cmake_bootstrap.sh @@ -61,7 +61,11 @@ function generate() { } function build() { - cmake --build build/$1 + # -k 0 keeps Ninja going after a target fails, so one broken target no + # longer decides whether the targets after it exist. ctest reports a + # missing binary as a failing test, which turned a single link error into + # a failure count that varied with build scheduling order. + cmake --build build/$1 -- -k 0 } function build_libs() { @@ -74,7 +78,10 @@ function build_libs() { } function test() { - pushd build/$1 + # Guard the pushd: with the caller capturing this function's status, set -e + # no longer aborts here, so a missing build directory would otherwise let + # ctest run in the source tree and report "no tests" as success. + pushd build/$1 || return 1 [ -z "${CTEST_PARALLEL_LEVEL}" ] && parallel="-j$2" if [ -z "${CTEST_REPEAT_FAIL}" ]; then @@ -198,11 +205,16 @@ if [ "$command" == "build" ]; then echo "" done + # A failing configuration must not stop the ones after it: under set -e + # the loop would abort and leave them unbuilt, which then reads as a wall + # of missing-binary test failures. The status is accumulated and returned. + build_status=0 for item in $builds; do echo "Building $item" - build $item + build $item || build_status=$? echo "" done + [ $build_status -eq 0 ] || exit $build_status elif [ "$command" == "test" ]; then cores=$(nproc) if [ -z "${CTEST_PARALLEL_LEVEL}" ]; diff --git a/test/freertos/cmake/run.sh b/test/freertos/cmake/run.sh index bb1c70f8..896a5c37 100755 --- a/test/freertos/cmake/run.sh +++ b/test/freertos/cmake/run.sh @@ -44,11 +44,18 @@ function generate() { } function build() { - cmake --build build/$1 + # -k 0 keeps Ninja going after a target fails, so one broken target no + # longer decides whether the targets after it exist. ctest reports a + # missing binary as a failing test, which turned a single link error into + # a failure count that varied with build scheduling order. + cmake --build build/$1 -- -k 0 } function test() { - pushd build/$1 + # Guard the pushd: with the caller capturing this function's status, set -e + # no longer aborts here, so a missing build directory would otherwise let + # ctest run in the source tree and report "no tests" as success. + pushd build/$1 || return 1 [ -z "${CTEST_PARALLEL_LEVEL}" ] && parallel="-j$2" if [ -z "${CTEST_REPEAT_FAIL}" ]; then @@ -56,10 +63,17 @@ function test() { else repeat_fail=${CTEST_REPEAT_FAIL} fi - ctest $parallel --timeout 300 -O $1.txt -T test --no-compress-output --test-output-size-passed 4194304 --test-output-size-failed 4194304 --output-on-failure --repeat until-pass:${repeat_fail} --output-junit $1.xml + # ctest's status is captured and returned past the popd and the summary + # steps, which all succeed. Without this the function would return 0 and a + # failing configuration would be reported as a pass. + local status=0 + ctest $parallel --timeout 300 -O $1.txt -T test --no-compress-output --test-output-size-passed 4194304 --test-output-size-failed 4194304 --output-on-failure --repeat until-pass:${repeat_fail} --output-junit $1.xml || status=$? popd - grep -E "^(\s*[0-9]+|Total)" build/$1/$1.txt >build/$1.txt + # Tolerated: this is a summary for humans, and a ctest that died early + # enough to leave no matching line must not mask its own status. + grep -E "^(\s*[0-9]+|Total)" build/$1/$1.txt >build/$1.txt || true sed -i "s/\x1B\[[0-9;]*[JKmsu]//g" build/$1.txt + return $status } cd $(dirname $0) @@ -92,21 +106,30 @@ if [ "$command" == "build" ]; then echo "" done + # A failing configuration must not stop the ones after it: under set -e + # the loop would abort and leave them unbuilt, which then reads as a wall + # of missing-binary test failures. The status is accumulated and returned. + build_status=0 for item in $builds; do echo "Building $item" - build $item + build $item || build_status=$? echo "" done + [ $build_status -eq 0 ] || exit $build_status elif [ "$command" == "test" ]; then cores=$(nproc) if [ -z "${CTEST_PARALLEL_LEVEL}" ]; then parallel_jobs=$(($cores + 2)) fi + # ctest's status is accumulated rather than allowed to abort the loop, so + # a failing configuration does not leave the ones after it untested. + exit_code=0 for item in $builds; do echo "Testing $item" - test $item $parallel_jobs + test $item $parallel_jobs || exit_code=$? done + exit $exit_code else help fi diff --git a/test/posix/cmake/run.sh b/test/posix/cmake/run.sh index aa65f7a7..76536cf7 100755 --- a/test/posix/cmake/run.sh +++ b/test/posix/cmake/run.sh @@ -65,7 +65,11 @@ function generate() { function build() { local arch=$1 local build=$2 - cmake --build "build/${arch}_${build}" + # -k 0 keeps Ninja going after a target fails, so one broken target no + # longer decides whether the targets after it exist. ctest reports a + # missing binary as a failing test, which turned a single link error into + # a failure count that varied with build scheduling order. + cmake --build "build/${arch}_${build}" -- -k 0 } function run_test() { @@ -73,13 +77,20 @@ function run_test() { local build=$2 local parallel_jobs=$3 - pushd "build/${arch}_${build}" + # Guard the pushd: with the caller capturing this function's status, set -e + # no longer aborts here, so a missing build directory would otherwise let + # ctest run in the source tree and report "no tests" as success. + pushd "build/${arch}_${build}" || return 1 [ -z "${CTEST_PARALLEL_LEVEL}" ] && parallel="-j${parallel_jobs}" if [ -z "${CTEST_REPEAT_FAIL}" ]; then repeat_fail=2 else repeat_fail=${CTEST_REPEAT_FAIL} fi + # ctest's status is captured and returned past the popd and the summary + # steps, which all succeed. Without this the function would return 0 and a + # failing configuration would be reported as a pass. + local status=0 ctest $parallel --timeout 1000 \ -O "${arch}_${build}.txt" \ -T test --no-compress-output \ @@ -87,8 +98,9 @@ function run_test() { --test-output-size-failed 4194304 \ --output-on-failure \ --repeat until-pass:${repeat_fail} \ - --output-junit "${arch}_${build}.xml" + --output-junit "${arch}_${build}.xml" || status=$? popd + return $status } # Determine repo root and script directory @@ -133,11 +145,16 @@ if [ "$command" == "build" ]; then echo "" done + # A failing configuration must not stop the ones after it: under set -e + # the loop would abort and leave them unbuilt, which then reads as a wall + # of missing-binary test failures. The status is accumulated and returned. + build_status=0 for item in $builds; do echo "Building ${arch} ${item}" - build "$arch" "$item" + build "$arch" "$item" || build_status=$? echo "" done + [ $build_status -eq 0 ] || exit $build_status elif [ "$command" == "test" ]; then cores=$(nproc) if [ -z "${CTEST_PARALLEL_LEVEL}" ]; then @@ -156,10 +173,16 @@ elif [ "$command" == "test" ]; then done exit $exit_code else + # ctest's status is accumulated rather than allowed to abort the loop. + # set -e would otherwise stop at the first failing configuration and + # leave every configuration after it untested. This mirrors what the + # parallel branch above already does with wait. + exit_code=0 for item in $builds; do echo "Testing ${arch} ${item}" - run_test "$arch" "$item" "$parallel_jobs" + run_test "$arch" "$item" "$parallel_jobs" || exit_code=$? done + exit $exit_code fi else help diff --git a/test/tx/cmake/riscv/run.sh b/test/tx/cmake/riscv/run.sh index 3b0905e7..0e4fb7a1 100755 --- a/test/tx/cmake/riscv/run.sh +++ b/test/tx/cmake/riscv/run.sh @@ -61,7 +61,11 @@ function generate() { function build() { local arch=$1 local build=$2 - cmake --build "build/${arch}_${build}" + # -k 0 keeps Ninja going after a target fails, so one broken target no + # longer decides whether the targets after it exist. ctest reports a + # missing binary as a failing test, which turned a single link error into + # a failure count that varied with build scheduling order. + cmake --build "build/${arch}_${build}" -- -k 0 } function run_test() { @@ -69,13 +73,20 @@ function run_test() { local build=$2 local parallel_jobs=$3 - pushd "build/${arch}_${build}" + # Guard the pushd: with the caller capturing this function's status, set -e + # no longer aborts here, so a missing build directory would otherwise let + # ctest run in the source tree and report "no tests" as success. + pushd "build/${arch}_${build}" || return 1 [ -z "${CTEST_PARALLEL_LEVEL}" ] && parallel="-j${parallel_jobs}" if [ -z "${CTEST_REPEAT_FAIL}" ]; then repeat_fail=2 else repeat_fail=${CTEST_REPEAT_FAIL} fi + # ctest's status is captured and returned past the popd. popd succeeds, so + # without this the function would return 0 and a failing configuration + # would be reported as a pass. + local status=0 ctest $parallel --timeout 1000 \ -O "${arch}_${build}.txt" \ -T test --no-compress-output \ @@ -83,8 +94,9 @@ function run_test() { --test-output-size-failed 4194304 \ --output-on-failure \ --repeat until-pass:${repeat_fail} \ - --output-junit "${arch}_${build}.xml" + --output-junit "${arch}_${build}.xml" || status=$? popd + return $status } # Determine repo root and script directory @@ -129,11 +141,16 @@ if [ "$command" == "build" ]; then echo "" done + # A failing configuration must not stop the ones after it: under set -e + # the loop would abort and leave them unbuilt, which then reads as a wall + # of missing-binary test failures. The status is accumulated and returned. + build_status=0 for item in $builds; do echo "Building ${arch} ${item}" - build "$arch" "$item" + build "$arch" "$item" || build_status=$? echo "" done + [ $build_status -eq 0 ] || exit $build_status elif [ "$command" == "test" ]; then cores=$(nproc) if [ -z "${CTEST_PARALLEL_LEVEL}" ]; then @@ -152,10 +169,16 @@ elif [ "$command" == "test" ]; then done exit $exit_code else + # ctest's status is accumulated rather than allowed to abort the loop. + # set -e would otherwise stop at the first failing configuration and + # leave every configuration after it untested. This mirrors what the + # parallel branch above already does with wait. + exit_code=0 for item in $builds; do echo "Testing ${arch} ${item}" - run_test "$arch" "$item" "$parallel_jobs" + run_test "$arch" "$item" "$parallel_jobs" || exit_code=$? done + exit $exit_code fi else help