diff --git a/.github/workflows/mosquitto-make-asan.yml b/.github/workflows/mosquitto-make-asan.yml index b4cc7dc6..1ed6acfa 100644 --- a/.github/workflows/mosquitto-make-asan.yml +++ b/.github/workflows/mosquitto-make-asan.yml @@ -45,4 +45,4 @@ jobs: - name: make test run: | - make WITH_ASAN=yes test + make WITH_ASAN=yes ptest diff --git a/.github/workflows/mosquitto-make.yml b/.github/workflows/mosquitto-make.yml index b16c988f..0cfba5c7 100644 --- a/.github/workflows/mosquitto-make.yml +++ b/.github/workflows/mosquitto-make.yml @@ -44,4 +44,4 @@ jobs: - name: make test run: | - make test + make ptest diff --git a/test/ptest.py b/test/ptest.py index 0781683d..732b2980 100755 --- a/test/ptest.py +++ b/test/ptest.py @@ -4,48 +4,71 @@ import subprocess import time import sys +class PTest(): + def __init__(self, testdef): + self.ports = testdef[0] + self.cmd = str(testdef[1]) + self.attempts = 0 + try: + if isinstance(testdef[2], (list,)): + self.args = [self.cmd] + testdef[2] + else: + self.args = [self.cmd] + [testdef[2]] + except IndexError: + self.args = [self.cmd] + self.start_time = 0 + self.proc = None + self.mosq_port = None + self.runtime = 0 + + def start(self): + self.run_args = self.args.copy() + for p in self.mosq_port: + self.run_args.append(str(p)) + + self.proc = subprocess.Popen(self.run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.start_time = time.time() + + def print_result(self, col): + cmd = " ".join(self.run_args) + print(f"{self.runtime:0.3f}s : \033[{col}m{cmd}\033[0m") + + def next_test(tests, ports): if len(tests) == 0 or len(ports) == 0: return test = tests.pop() - proc_ports = () + test.mosq_port = [] - if len(ports) < test[0]: + if len(ports) < test.ports: tests.insert(0, test) return None else: - if isinstance(test[1], (list,)): - args = test[1] - else: - args = [test[1]] - try: - args.append(test[2]) - except IndexError: - pass - - for i in range(0, test[0]): + for i in range(0, test.ports): proc_port = ports.pop() - proc_ports = proc_ports + (proc_port,) - args.append(str(proc_port)) + test.mosq_port.append(proc_port) - proc = subprocess.Popen(args) - proc.start_time = time.time() - proc.mosq_port = proc_ports - return proc + test.start() + return test -def run_tests(tests, minport=1888, max_running=20): +def run_tests(test_list, minport=1888, max_running=20): ports = list(range(minport, minport+max_running+1)) start_time = time.time() passed = 0 + retried = 0 failed = 0 - failed_tests = [] + tests = [] + for t in test_list: + tests.append(PTest(t)) + failed_tests = [] running_tests = [] - while len(tests) > 0 or len(running_tests) > 0: + retry_tests = [] + while len(tests) > 0 or len(running_tests) > 0 or len(retry_tests) > 0: if len(running_tests) <= max_running: t = next_test(tests, ports) if t is None: @@ -53,32 +76,45 @@ def run_tests(tests, minport=1888, max_running=20): else: running_tests.append(t) + if len(running_tests) == 0 and len(tests) == 0 and len(retry_tests) > 0: + # Only retry tests after everything else has finished to reduce load + tests = retry_tests + retry_tests = [] + for t in running_tests: - t.poll() - if t.returncode is not None: + t.proc.poll() + if t.proc.returncode is not None: + t.runtime = time.time() - t.start_time running_tests.remove(t) - if isinstance(t.mosq_port, tuple): - for portret in t.mosq_port: - ports.append(portret) - else: - ports.append(t.mosq_port) - t.terminate() - t.wait() - runtime = time.time() - t.start_time - #(stdo, stde) = t.communicate() - if t.returncode == 1: - print("%0.3fs : \033[31m%s\033[0m" % (runtime, t.args[0])) + + for portret in t.mosq_port: + ports.append(portret) + t.proc.terminate() + t.proc.wait() + + if t.proc.returncode == 1 and t.attempts < 5: + t.print_result(33) + retried += 1 + t.attempts += 1 + t.proc = None + t.mosq_port = None + retry_tests.append(t) + continue + + if t.proc.returncode == 1: + t.print_result(31) failed = failed + 1 - failed_tests.append(t.args[0]) + failed_tests.append(t.cmd) + print(f"{t.cmd}:") + (stdo, stde) = t.proc.communicate() else: passed = passed + 1 - print("%0.3fs : \033[32m%s\033[0m" % (runtime, t.args[0])) + t.print_result(32) - print("Passed: %d\nFailed: %d\nTotal: %d\nTotal time: %0.2f" % (passed, failed, passed+failed, time.time()-start_time)) + print("Passed: %d\nRetried: %d\nFailed: %d\nTotal: %d\nTotal time: %0.2f" % (passed, retried, failed, passed+failed, time.time()-start_time)) if failed > 0: print("Failing tests:") failed_tests.sort() for f in failed_tests: print(f) sys.exit(1) -