Combine all ptest tests into a single run, for speed.

Old `make ptest` locally: 1m34
New `make ptest` locally: 55s

Doesn't work on CI for some reason, so actually disabled for
`make ptest`. Use ./run_tests.py instead

run_tests.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+import inspect
+import os
+import sys
+
+sys.path.insert(0, "test")
+
+import ptest
+import test.apps.ctrl.test as p_ctrl
+import test.apps.db_dump.test as p_db_dump
+import test.apps.passwd.test as p_passwd
+import test.apps.signal.test as p_signal
+import test.broker.test as p_broker
+import test.client.test as p_client
+import test.lib.test as p_lib
+
+test = ptest.PTest()
+test.add_tests(p_client.tests, "test/client")
+test.add_tests(p_lib.tests, "test/lib")
+test.add_tests(p_ctrl.tests, "test/apps/ctrl")
+test.add_tests(p_db_dump.tests, "test/apps/db_dump")
+test.add_tests(p_passwd.tests, "test/apps/ctrl")
+test.add_tests(p_signal.tests, "test/apps/signal")
+test.add_tests(p_broker.tests, "test/broker")
+
+test.run()

test/apps/ctrl/test.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-import mosq_test_helper
-import pathlib
+import sys
+sys.path.insert(0, "../..")
 import ptest
 
 tests = [
@@ -11,5 +11,6 @@ tests = [
     (2, './ctrl-dynsec.py'),
 ]
 
-test = ptest.PTest()
-test.run_tests(tests)
+if __name__ == "__main__":
+    test = ptest.PTest()
+    test.run_tests(tests)

test/apps/db_dump/test.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env python3
 
-import mosq_test_helper
+import os
 import pathlib
+import sys
+sys.path.insert(0, "../..")
 import ptest
 
 tests = []
 
-for test_file in pathlib.Path('.').glob('db-dump-*.py'):
+for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('db-dump-*.py'):
     tests.append((1, test_file.resolve()))
 
-test = ptest.PTest()
-test.run_tests(tests)
+if __name__ == "__main__":
+    test = ptest.PTest()
+    test.run_tests(tests)

test/apps/passwd/test.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env python3
 
-import mosq_test_helper
+import os
 import pathlib
+import sys
+sys.path.insert(0, "../..")
 import ptest
 
 tests = []
 
-for test_file in pathlib.Path('.').glob('passwd-*.py'):
+for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('passwd-*.py'):
     tests.append((1, test_file.resolve()))
 
-test = ptest.PTest()
-test.run_tests(tests)
+if __name__ == "__main__":
+    test = ptest.PTest()
+    test.run_tests(tests)

test/apps/signal/test.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env python3
 
-import mosq_test_helper
+import os
 import pathlib
+import sys
+sys.path.insert(0, "../..")
 import ptest
 
 tests = []
 
-for test_file in pathlib.Path('.').glob('signal-*.py'):
+for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('signal-*.py'):
     tests.append((1, test_file.resolve()))
 
-test = ptest.PTest()
-test.run_tests(tests)
+if __name__ == "__main__":
+    test = ptest.PTest()
+    test.run_tests(tests)

test/broker/Makefile

@@ -15,7 +15,7 @@ clean :
 test-compile :
 	$(MAKE) -C c
 
-ptest : test-compile msg_sequence_test
+ptest : test-compile
 	./test.py
 
 test : test-compile msg_sequence_test 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 20 21

test/broker/msg_sequence_test.py

@@ -209,7 +209,7 @@ def do_test(hostname, port, protocol):
                     failed_tests.append(this_test)
                     sock = mosq_test.client_connect_only(hostname=hostname, port=port, timeout=2, protocol=protocol)
                     this_test.process_all(sock)
-                    print("\033[32m" + tname + "\033[0m")
+                    #print("\033[32m" + tname + "\033[0m")
                     succeeded += 1
                     sock.close()
                     failed_tests.pop()

test/broker/test.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 
-import mosq_test_helper
+import sys
+sys.path.insert(0, "..")
 import ptest
 
 tests = [
     #(ports required, 'path'),
+    (1, './msg_sequence_test.py'),
     (1, './01-bad-initial-packets.py'),
     (1, './01-connect-575314.py'),
     (1, './01-connect-accept-protocol.py'),
@@ -289,5 +291,6 @@ tests = [
     (1, './21-proxy-v2-ssl-require-tls-success.py'),
 ]
 
-test = ptest.PTest()
-test.run_tests(tests)
+if __name__ == "__main__":
+    test = ptest.PTest()
+    test.run_tests(tests)

test/client/test.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-import mosq_test_helper
+import sys
+sys.path.insert(0, "..")
 import ptest
 
 tests = [
@@ -47,5 +48,6 @@ tests = [
     (2, './04-rr-qos1-ws.py'),
     ]
 
-test = ptest.PTest()
-test.run_tests(tests)
+if __name__ == "__main__":
+    test = ptest.PTest()
+    test.run_tests(tests)

test/lib/Makefile

@@ -10,7 +10,7 @@ all :
 
 check : test
 
-ptest : test-compile msg_sequence_test
+ptest : test-compile
 	./test.py
 
 msg_sequence_test: test-compile-c

test/lib/test.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 
-import mosq_test_helper
+import sys
+sys.path.insert(0, "..")
 import ptest
 
 tests = [
+    (1, './msg_sequence_test.py'),
     (1, './01-con-discon-success-v5.py'),
     (1, './01-con-discon-success.py'),
     (1, './01-con-discon-will-clear.py'),
@@ -63,5 +65,6 @@ tests = [
     ]
 
 
-test = ptest.PTest()
-test.run_tests(tests)
+if __name__ == "__main__":
+    test = ptest.PTest()
+    test.run_tests(tests)

test/ptest.py

@@ -1,20 +1,19 @@
 #!/usr/bin/env python3
 
+from pathlib import Path
 import subprocess
 import time
 import sys
 
 class PTestCase():
-    def __init__(self, testdef):
-        self.ports = testdef[0]
-        self.cmd = str(testdef[1])
+    def __init__(self, path, ports, cmd, args=None):
+        self.path = path
+        self.ports = ports
+        self.cmd = str(cmd)
         self.attempts = 0
-        try:
-            if isinstance(testdef[2], (list,)):
-                self.args = [self.cmd] + testdef[2]
-            else:
-                self.args = [self.cmd] + [testdef[2]]
-        except IndexError:
+        if args is not None:
+            self.args = [self.cmd] + args
+        else:
             self.args = [self.cmd]
         self.start_time = 0
         self.proc = None
@@ -26,12 +25,12 @@ class PTestCase():
         for p in self.mosq_port:
             self.run_args.append(str(p))
 
-        self.proc = subprocess.Popen(self.run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.proc = subprocess.Popen(self.run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.path)
         self.start_time = time.time()
 
     def print_result(self, col):
         cmd = " ".join(self.run_args)
-        print(f"{self.runtime:0.3f}s : \033[{col}m{cmd}\033[0m")
+        print(f"{self.runtime:0.3f}s : \033[{col}m{self.path}/{cmd}\033[0m")
 
 
 class PTest():
@@ -40,18 +39,28 @@ class PTest():
         self.max_running = 20
         self.tests = []
 
-    def next_test(self, tests, ports):
-        if len(tests) == 0 or len(ports) == 0:
+    def add_tests(self, test_list, path=".", label=""):
+        for testdef in test_list:
+            try:
+                if isinstance(testdef[2], (list,)):
+                    args = testdef[2]
+                else:
+                    args = [testdef[2]]
+            except IndexError:
+                args = None
+            self.tests.append(PTestCase(path, testdef[0], testdef[1], args))
+
+    def _next_test(self, ports):
+        if len(self.tests) == 0 or len(ports) == 0:
             return
 
-        test = tests.pop()
+        test = self.tests.pop()
         test.mosq_port = []
 
         if len(ports) < test.ports:
-            tests.insert(0, test)
+            self.tests.insert(0, test)
             return None
         else:
-
             for i in range(0, test.ports):
                 proc_port = ports.pop()
                 test.mosq_port.append(proc_port)
@@ -59,32 +68,31 @@ class PTest():
             test.start()
             return test
 
-
     def run_tests(self, test_list):
+        self.add_tests(test_list)
+        self.run()
+
+    def run(self):
         ports = list(range(self.minport, self.minport+self.max_running+1))
         start_time = time.time()
         passed = 0
         retried = 0
         failed = 0
 
-        tests = []
-        for t in test_list:
-            tests.append(PTestCase(t))
-
         failed_tests = []
         running_tests = []
         retry_tests = []
-        while len(tests) > 0 or len(running_tests) > 0 or len(retry_tests) > 0:
+        while len(self.tests) > 0 or len(running_tests) > 0 or len(retry_tests) > 0:
             if len(running_tests) <= self.max_running:
-                t = self.next_test(tests, ports)
+                t = self._next_test(ports)
                 if t is None:
                     time.sleep(0.1)
                 else:
                     running_tests.append(t)
 
-            if len(running_tests) == 0 and len(tests) == 0 and len(retry_tests) > 0:
+            if len(running_tests) == 0 and len(self.tests) == 0 and len(retry_tests) > 0:
                 # Only retry tests after everything else has finished to reduce load
-                tests = retry_tests
+                self.tests = retry_tests
                 retry_tests = []
 
             for t in running_tests: