diff --git a/.gitignore b/.gitignore index d094ba0fb4..ce2e6b4bf1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ config.h defconfig env/* docs-venv +tests/.ccache/ diff --git a/docs/src/contributing/meta.json b/docs/src/contributing/meta.json index 9b4af2d73d..7bd1673795 100644 --- a/docs/src/contributing/meta.json +++ b/docs/src/contributing/meta.json @@ -5,6 +5,7 @@ "ways_to_contribute", "writing_docs", "pull_requests", + "running_tests", "dco", "coding_style", "assertions", diff --git a/docs/src/contributing/running_tests.mdx b/docs/src/contributing/running_tests.mdx new file mode 100644 index 0000000000..f928694ac4 --- /dev/null +++ b/docs/src/contributing/running_tests.mdx @@ -0,0 +1,85 @@ +--- +title: Running Tests +description: "How to build and run LVGL's unit, build-only and performance tests locally or in Docker before opening a pull request." +--- + +LVGL has an extensive test suite that runs automatically in CI on every push and pull +request. Running it locally before opening a PR catches most problems long before a +reviewer sees them. + +Everything on this page is covered in more detail in +[`tests/README.md`](https://github.com/lvgl/lvgl/blob/master/tests/README.md) — including +how to write new test cases, the layout of the `tests/` folder, and the available custom +asserts. Read that file if you need anything beyond the basics below. + +## Test Types + +- **Unit tests** — functional tests in `tests/src/test_cases/`, with screenshot comparison + against reference images in `tests/ref_imgs/`. +- **Build-only tests** — verify that LVGL still compiles and links with a range of + `lv_conf.h` configurations. +- **Performance tests** — ARM-emulated benchmarks in `tests/src/test_cases_perf/`, run + under QEMU/SO3 so timings are consistent across machines. +- **Emulated benchmarks** — automated `lv_demo_benchmark` runs in the same ARM emulation, + used to catch performance regressions. + +## Running Locally + +Install the prerequisites once: + +```sh +scripts/install-prerequisites.sh +``` + +Then, from the repository root: + +```sh +# Run all executable tests +./tests/main.py test + +# Build all build-only tests +./tests/main.py build + +# Clean, build everything, run the tests and generate a coverage report +./tests/main.py --clean --report build test +``` + +Useful options: + +- `--test-suite ` — run a single test suite instead of all of them. +- `--build-options ` — build or run only one build configuration. +- `--update-image` — re-generate the screenshot reference images. This uses + `scripts/LVGLImage.py`, which needs `pngquant` and `pypng`. Note that different + `pngquant` versions produce different images; CI currently uses pngquant 2.13.1-1. + +Run `./tests/main.py --help` for the full list. + +## Running in Docker + +To test in an environment matching CI exactly: + +```sh +docker build . -f tests/Dockerfile -t lvgl_test_env +docker run --rm -it -v $(pwd):/work lvgl_test_env "./tests/main.py" +``` + +`scripts/run_tests_docker.sh` automates both steps; run it with `--help` for details. + +## Performance Tests and Benchmarks + +These require **Docker** and a **Linux host** (WSL may work but is untested), since they +boot an ARM emulated environment: + +```sh +./tests/perf.py test # performance tests +./tests/benchmark_emu.py run # emulated lv_demo_benchmark +``` + +Both scripts accept `--help`. + + +New test files go into `tests/src/test_cases/` and are named `test_.c`. Start from +`_test_template.c`. See +[`tests/README.md`](https://github.com/lvgl/lvgl/blob/master/tests/README.md#add-new-tests) +for the LVGL-specific asserts such as `TEST_ASSERT_EQUAL_SCREENSHOT()`. + diff --git a/scripts/prerequisites-apt.txt b/scripts/prerequisites-apt.txt index dc3042348f..b14f2777d9 100644 --- a/scripts/prerequisites-apt.txt +++ b/scripts/prerequisites-apt.txt @@ -1,4 +1,5 @@ git +ccache gcc gcc-multilib g++-multilib diff --git a/scripts/run_tests_docker.sh b/scripts/run_tests_docker.sh new file mode 100755 index 0000000000..d09be736f2 --- /dev/null +++ b/scripts/run_tests_docker.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# +# Build the LVGL test Docker image (linux/amd64) and run the test suite +# inside it. Works on non-x86_64 hosts (e.g. Apple Silicon) via QEMU. +# +# Package prerequisite checks from scripts/run_tests.sh are omitted: the +# container is built from tests/Dockerfile which guarantees they are present. +# +# Usage: +# scripts/run_tests_docker.sh [options] [-- ] +# +# Options: +# --32 Build/run as 32-bit (sets NON_AMD64_BUILD=1). +# --64 Build/run as 64-bit (default). +# --both Run both 64-bit and 32-bit, back to back. +# --build-option Pass --build-options= to tests/main.py +# (e.g. OPTIONS_TEST_SYSHEAP, OPTIONS_16BIT, ...). +# When omitted, main.py runs its full matrix. +# --test-suite Pass --test-suite= to tests/main.py +# (ctest --tests-regex filter). +# --actions Space-separated list of actions for main.py +# (default: "build test"). E.g. --actions test. +# --no-clean Do not pass --clean to tests/main.py. +# --report Generate the coverage report (main.py --report) +# and run scripts/check_gcov_coverage.py afterwards. +# --rebuild Force rebuild of the Docker image (--no-cache). +# --shell Drop into an interactive bash shell in the container. +# -- Everything after -- is forwarded verbatim to +# tests/main.py. +# +# Environment: +# LVGL_TEST_IMAGE Override the image name (default: lvgl-tests:local). +# +# Examples: +# # Full 64-bit build+test matrix with coverage +# scripts/run_tests_docker.sh --report +# +# # Single 32-bit test config, only run tests (no build-only matrix) +# scripts/run_tests_docker.sh --32 --build-option OPTIONS_TEST_SYSHEAP \ +# --actions test +# +# # Run only tests whose ctest name matches a regex +# scripts/run_tests_docker.sh --test-suite 'test_obj.*' +# +# # Interactive shell +# scripts/run_tests_docker.sh --shell + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd) + +IMAGE_NAME=${LVGL_TEST_IMAGE:-lvgl-tests:local} +DOCKERFILE="$REPO_ROOT/tests/Dockerfile" +PLATFORM="linux/amd64" + +bits="64" +build_option="" +test_suite="" +actions="build test" +clean="--clean" +report="" +rebuild=false +shell=false +extra_args=() + +usage() { sed -n '2,47p' "$0"; } + +while [ "$#" -gt 0 ]; do + case "$1" in + --32) bits="32" ;; + --64) bits="64" ;; + --both) bits="both" ;; + --build-option) + if [ "$#" -lt 2 ]; then + echo "Error: --build-option requires a value." >&2 + usage >&2 + exit 1 + fi + build_option="$2" + shift + ;; + --test-suite) + if [ "$#" -lt 2 ]; then + echo "Error: --test-suite requires a value." >&2 + usage >&2 + exit 1 + fi + test_suite="$2" + shift + ;; + --actions) + if [ "$#" -lt 2 ]; then + echo "Error: --actions requires a value." >&2 + usage >&2 + exit 1 + fi + actions="$2" + shift + ;; + --no-clean) clean="" ;; + --report) report="--report" ;; + --rebuild) rebuild=true ;; + --shell) shell=true ;; + --) shift; extra_args=("$@"); break ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;; + esac + shift +done + +if ! command -v docker >/dev/null 2>&1; then + echo "Error: docker is not installed or not in PATH." >&2 + exit 1 +fi + +# --- Build image if needed ------------------------------------------------- +build_args=(--platform "$PLATFORM" -f "$DOCKERFILE" -t "$IMAGE_NAME") +if [ "$rebuild" = true ]; then + build_args+=(--no-cache) +fi + +if [ "$rebuild" = true ] || ! docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then + echo "Building Docker image '$IMAGE_NAME' (platform=$PLATFORM)..." + docker build "${build_args[@]}" "$REPO_ROOT" +else + echo "Using existing Docker image '$IMAGE_NAME' (use --rebuild to force rebuild)." +fi + +# --- Docker run ------------------------------------------------------------ +run_args=( + --rm + --platform "$PLATFORM" + --user "$(id -u):$(id -g)" + -e HOME=/tmp + # Persist the ccache across runs. The repo is bind-mounted at /work, so a + # cache dir beneath it survives the --rm container without an extra mount. + -e CCACHE_DIR=/work/tests/.ccache + -v "$REPO_ROOT":/work + -w /work +) + +if [ -t 0 ] && [ -t 1 ]; then + run_args+=(-it) +fi + +if [ "$shell" = true ]; then + exec docker run "${run_args[@]}" "$IMAGE_NAME" bash +fi + +# --- Build the command to run inside the container ------------------------- +# Assemble tests/main.py arguments. +main_args=() +[ -n "$clean" ] && main_args+=("$clean") +[ -n "$report" ] && main_args+=("$report") +[ -n "$build_option" ] && main_args+=("--build-options=$build_option") +[ -n "$test_suite" ] && main_args+=("--test-suite=$test_suite") +# Actions go last, as positional arguments. +# shellcheck disable=SC2206 +actions_arr=($actions) +main_args+=("${actions_arr[@]}") +main_args+=("${extra_args[@]}") + +# Quote arguments safely for the inline bash script. +quote() { printf "%q " "$@"; } +main_args_q=$(quote "${main_args[@]}") + +report_flag=$([ -n "$report" ] && echo 1 || echo 0) + +# Inline script executed in the container. Mirrors relevant setup from +# scripts/run_tests.sh (gcov selection, NON_AMD64_BUILD, report renaming, +# coverage check) but skips the package-presence checks. +INNER=$(cat </dev/null 2>&1; then + export GCOV="gcov-\${gcc_major}" +else + export GCOV="gcov" +fi +echo "Using GCOV=\$GCOV" + +run_one() { + local bits="\$1" + if [ "\$bits" = "32" ]; then + export NON_AMD64_BUILD=1 + echo "=== Running tests/main.py for 32-bit build ===" + else + unset NON_AMD64_BUILD + echo "=== Running tests/main.py for 64-bit build ===" + fi + + ./tests/main.py ${main_args_q} + + if [ "${report_flag}" = "1" ] && [ -d tests/report ]; then + rm -rf "tests/report-\${bits}bit" + mv tests/report "tests/report-\${bits}bit" + echo "Coverage report: tests/report-\${bits}bit/index.html" + fi +} + +BITS="${bits}" +if [ "\$BITS" = "both" ]; then + run_one 32 + run_one 64 +else + run_one "\$BITS" +fi + +if [ "${report_flag}" = "1" ]; then + echo "=== Running scripts/check_gcov_coverage.py ===" + ./scripts/check_gcov_coverage.py +fi +EOS +) + +exec docker run "${run_args[@]}" "$IMAGE_NAME" bash -lc "$INNER" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 71cc29b37a..2ac9208b9b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -519,9 +519,9 @@ if (NOT LIBINPUT_FOUND) endif() find_package(PkgConfig) -pkg_check_modules(xkbcommon pkg_check_modules xkbcommon) +pkg_check_modules(XKB xkbcommon) -if (NOT xkbcommon_FOUND) +if (NOT XKB_FOUND) message("xkbcommon not found, defaulting to 0") add_definitions(-DLV_LIBINPUT_XKB=0) endif() diff --git a/tests/README.md b/tests/README.md index d65f62810f..39b769c87e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -59,6 +59,8 @@ docker run --rm -it -v $(pwd):/work lvgl_test_env "./tests/main.py" This ensures you are testing in a consistent environment with the same dependencies as the CI pipeline. +There is a script which automates these steps: `scripts/run_tests_docker.sh`. It will build a Docker container and run tests in that. Run the script with `--help` for more detail. + ## Running automatically GitHub's CI automatically runs these tests on pushes and pull requests to `master` and `release/v8.*` branches. diff --git a/tests/main.py b/tests/main.py index 06c2d6e32c..33c359b285 100755 --- a/tests/main.py +++ b/tests/main.py @@ -134,8 +134,16 @@ def build_tests(options_name, build_type, clean): created_build_dir = True os.chdir(build_dir) if created_build_dir: - subprocess.check_call(['cmake', '-GNinja', '-DCMAKE_BUILD_TYPE=%s' % build_type, - '-D%s=1' % options_name, '..']) + cmake_args = ['cmake', '-GNinja', '-DCMAKE_BUILD_TYPE=%s' % build_type, + '-D%s=1' % options_name] + # Use ccache as a compiler launcher when available. This dramatically + # speeds up rebuilds across the build matrix and repeated invocations + # (notably under emulation), and is a no-op when ccache is absent. + if shutil.which('ccache'): + cmake_args += ['-DCMAKE_C_COMPILER_LAUNCHER=ccache', + '-DCMAKE_CXX_COMPILER_LAUNCHER=ccache'] + cmake_args.append('..') + subprocess.check_call(cmake_args) subprocess.check_call(['cmake', '--build', build_dir, '--parallel', str(os.cpu_count())])