Make ptest retry tests, enable for make workflows

.github/workflows/mosquitto-make-asan.yml

@@ -45,4 +45,4 @@ jobs:
     -
       name: make test
       run: |
-        make WITH_ASAN=yes test
+        make WITH_ASAN=yes ptest

.github/workflows/mosquitto-make.yml

@@ -44,4 +44,4 @@ jobs:
     -
       name: make test
       run: |
-        make test
+        make ptest

test/ptest.py

@@ -4,48 +4,71 @@ import subprocess
 import time
 import sys
 
+class PTest():
+    def __init__(self, testdef):
+        self.ports = testdef[0]
+        self.cmd = str(testdef[1])
+        self.attempts = 0
+        try:
+            if isinstance(testdef[2], (list,)):
+                self.args = [self.cmd] + testdef[2]
+            else:
+                self.args = [self.cmd] + [testdef[2]]
+        except IndexError:
+            self.args = [self.cmd]
+        self.start_time = 0
+        self.proc = None
+        self.mosq_port = None
+        self.runtime = 0
+
+    def start(self):
+        self.run_args = self.args.copy()
+        for p in self.mosq_port:
+            self.run_args.append(str(p))
+
+        self.proc = subprocess.Popen(self.run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.start_time = time.time()
+
+    def print_result(self, col):
+        cmd = " ".join(self.run_args)
+        print(f"{self.runtime:0.3f}s : \033[{col}m{cmd}\033[0m")
+
+
 def next_test(tests, ports):
     if len(tests) == 0 or len(ports) == 0:
         return
 
     test = tests.pop()
-    proc_ports = ()
+    test.mosq_port = []
 
-    if len(ports) < test[0]:
+    if len(ports) < test.ports:
         tests.insert(0, test)
         return None
     else:
-        if isinstance(test[1], (list,)):
-            args = test[1]
-        else:
-            args = [test[1]]
 
-        try:
-            args.append(test[2])
-        except IndexError:
-            pass
-
-        for i in range(0, test[0]):
+        for i in range(0, test.ports):
             proc_port = ports.pop()
-            proc_ports = proc_ports + (proc_port,)
-            args.append(str(proc_port))
+            test.mosq_port.append(proc_port)
 
-        proc = subprocess.Popen(args)
-        proc.start_time = time.time()
-        proc.mosq_port = proc_ports
-        return proc
+        test.start()
+        return test
 
 
-def run_tests(tests, minport=1888, max_running=20):
+def run_tests(test_list, minport=1888, max_running=20):
     ports = list(range(minport, minport+max_running+1))
     start_time = time.time()
     passed = 0
+    retried = 0
     failed = 0
 
-    failed_tests = []
+    tests = []
+    for t in test_list:
+        tests.append(PTest(t))
 
+    failed_tests = []
     running_tests = []
-    while len(tests) > 0 or len(running_tests) > 0:
+    retry_tests = []
+    while len(tests) > 0 or len(running_tests) > 0 or len(retry_tests) > 0:
         if len(running_tests) <= max_running:
             t = next_test(tests, ports)
             if t is None:
@@ -53,32 +76,45 @@ def run_tests(tests, minport=1888, max_running=20):
             else:
                 running_tests.append(t)
 
+        if len(running_tests) == 0 and len(tests) == 0 and len(retry_tests) > 0:
+            # Only retry tests after everything else has finished to reduce load
+            tests = retry_tests
+            retry_tests = []
+
         for t in running_tests:
-            t.poll()
-            if t.returncode is not None:
+            t.proc.poll()
+            if t.proc.returncode is not None:
+                t.runtime = time.time() - t.start_time
                 running_tests.remove(t)
-                if isinstance(t.mosq_port, tuple):
-                    for portret in t.mosq_port:
-                        ports.append(portret)
-                else:
-                    ports.append(t.mosq_port)
-                t.terminate()
-                t.wait()
-                runtime = time.time() - t.start_time
-                #(stdo, stde) = t.communicate()
-                if t.returncode == 1:
-                    print("%0.3fs : \033[31m%s\033[0m" % (runtime, t.args[0]))
+
+                for portret in t.mosq_port:
+                    ports.append(portret)
+                t.proc.terminate()
+                t.proc.wait()
+
+                if t.proc.returncode == 1 and t.attempts < 5:
+                    t.print_result(33)
+                    retried += 1
+                    t.attempts += 1
+                    t.proc = None
+                    t.mosq_port = None
+                    retry_tests.append(t)
+                    continue
+
+                if t.proc.returncode == 1:
+                    t.print_result(31)
                     failed = failed + 1
-                    failed_tests.append(t.args[0])
+                    failed_tests.append(t.cmd)
+                    print(f"{t.cmd}:")
+                    (stdo, stde) = t.proc.communicate()
                 else:
                     passed = passed + 1
-                    print("%0.3fs : \033[32m%s\033[0m" % (runtime, t.args[0]))
+                    t.print_result(32)
 
-    print("Passed: %d\nFailed: %d\nTotal: %d\nTotal time: %0.2f" % (passed, failed, passed+failed, time.time()-start_time))
+    print("Passed: %d\nRetried: %d\nFailed: %d\nTotal: %d\nTotal time: %0.2f" % (passed, retried, failed, passed+failed, time.time()-start_time))
     if failed > 0:
         print("Failing tests:")
         failed_tests.sort()
         for f in failed_tests:
             print(f)
         sys.exit(1)
-