ci(tests) run tests using ctest (#2503)

Simplifies `tests/main.py` as it no longer needs to invoke
tests individually or keep track of and report their pass/fail
status.

Also enables the ability to run tests in parallel, support timeouts,
and re-run flaky tests.

https://cmake.org/cmake/help/latest/manual/ctest.1.html

tests/CMakeLists.txt

@@ -9,6 +9,8 @@ else()
 cmake_minimum_required(VERSION 3.13)
 project(lvgl_tests LANGUAGES C)
 
+include(CTest)
+
 set(LVGL_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 set(LVGL_TEST_OPTIONS_MINIMAL_MONOCHROME
@@ -289,6 +291,11 @@ foreach( test_case_fname ${TEST_CASE_FILES} )
     target_link_libraries(${test_name} test_common lvgl_examples lvgl png ${TEST_LIBS})
     target_include_directories(${test_name} PUBLIC ${TEST_INCLUDE_DIRS})
     target_compile_options(${test_name} PUBLIC ${COMPILE_OPTIONS})
+
+    add_test(
+        NAME ${test_name} 
+        WORKING_DIRECTORY ${LVGL_TEST_DIR}
+        COMMAND ${test_name})
 endforeach( test_case_fname ${TEST_CASE_FILES} )
 
 endif()

tests/main.py

@@ -7,7 +7,6 @@ import sys
 import os
 
 lvgl_test_dir = os.path.dirname(os.path.realpath(__file__))
-num_test_case_failures = 0
 
 # Key values must match variable names in CMakeLists.txt.
 options = {
@@ -33,9 +32,7 @@ def delete_dir_ignore_missing(dir_path):
 
 
 def generate_test_runners():
-    '''Generate the test runner source code.
+    '''Generate the test runner source code.'''
-
-    Returns a list of test names.'''
     global lvgl_test_dir
     os.chdir(lvgl_test_dir)
     delete_dir_ignore_missing('src/test_runners')
@@ -43,14 +40,11 @@ def generate_test_runners():
 
     # TODO: Intermediate files should be in the build folders, not alongside
     #       the other repo source.
-    test_names = []
     for f in glob.glob("./src/test_cases/test_*.c"):
-        test_names.append(os.path.splitext(os.path.basename(f))[0])
         r = f[:-2] + "_Runner.c"
         r = r.replace("/test_cases/", "/test_runners/")
         subprocess.check_call(['ruby', 'unity/generate_test_runner.rb',
                                f, r, 'config.yml'])
-    return test_names
 
 
 def options_abbrev(options_name):
@@ -60,18 +54,24 @@ def options_abbrev(options_name):
     return options_name[len(prefix):].lower()
 
 
-def get_build_dir(options_name):
+def get_base_buid_dir(options_name):
     '''Given the build options name, return the build directory name.
 
     Does not return the full path to the directory - just the base name.'''
     return 'build_%s' % options_abbrev(options_name)
 
 
+def get_build_dir(options_name):
+    '''Given the build options name, return the build directory name.
+
+    Returns absolute path to the build directory.'''
+    global lvgl_test_dir
+    return os.path.join(lvgl_test_dir, get_base_buid_dir(options_name))
+
+
 def delete_build_dir(options_name):
     '''Recursively delete the build directory for the given options name.'''
-    global lvgl_test_dir
+    delete_dir_ignore_missing(get_build_dir(options_name))
-    build_dir = os.path.join(lvgl_test_dir, get_build_dir(options_name))
-    delete_dir_ignore_missing(build_dir)
 
 
 def build_tests(options_name, build_type):
@@ -100,30 +100,23 @@ def build_tests(options_name, build_type):
     if created_build_dir:
         subprocess.check_call(['cmake', '-DCMAKE_BUILD_TYPE=%s' % build_type,
                                '-D%s=1' % options_name, '..'])
-    subprocess.check_call(['cmake', '--build', os.path.join(lvgl_test_dir, build_dir),
+    subprocess.check_call(['cmake', '--build', build_dir,
                            '--parallel', str(os.cpu_count())])
 
 
-def run_tests(options_name, test_names):
+def run_tests(options_name):
     '''Run the tests for the given options name.'''
-    global num_test_case_failures
-    relative_bd = get_build_dir(options_name)
-    abs_bd = os.path.join(lvgl_test_dir, relative_bd)
-    for test_name in test_names:
 
-        print()
+    print()
-        print()
+    print()
-        label = 'Running: %s' % os.path.join(
+    label = 'Running tests for %s' % options_abbrev(options_name)
-            options_abbrev(options_name), test_name)
+    print('=' * len(label))
-        print('=' * len(label))
+    print(label)
-        print(label)
+    print('=' * len(label), flush=True)
-        print('=' * len(label), flush=True)
 
-        try:
+    os.chdir(get_build_dir(options_name))
-            os.chdir(lvgl_test_dir)
+    subprocess.check_call(
-            subprocess.check_call([os.path.join(abs_bd, test_name)])
+        ['ctest', '--parallel', str(os.cpu_count()), '--output-on-failure'])
-        except subprocess.CalledProcessError as e:
-            num_test_case_failures += 1
 
 
 def generate_code_coverage_report():
@@ -158,7 +151,7 @@ run_test_only = "test" in sys.argv
 generate_gcov_report = "report" in sys.argv
 test_noclean = "noclean" in sys.argv
 
-test_names = generate_test_runners()
+generate_test_runners()
 
 for options_name in options.keys():
     is_test = options_name == run_tests_option_name
@@ -166,15 +159,12 @@ for options_name in options.keys():
     if is_test or not run_test_only:
         build_tests(options_name, build_type)
     if options_name == run_tests_option_name:
-        run_tests(options_name, test_names)
+        try:
+            run_tests(options_name)
+        except subprocess.CalledProcessError as e:
+            sys.exit(e.returncode)
+
 
 if generate_gcov_report:
     generate_code_coverage_report()
 
-print()
-if num_test_case_failures:
-    print('There were %d test case failures.' %
-          num_test_case_failures, file=sys.stderr)
-else:
-    print('All test cases passed.')
-sys.exit(num_test_case_failures)