From adea690b9db2e4610b247f80f7c5ae02743b54de Mon Sep 17 00:00:00 2001 From: "Roger A. Light" Date: Thu, 21 Nov 2024 12:52:58 +0000 Subject: [PATCH] Combine all ptest tests into a single run, for speed. Old `make ptest` locally: 1m34 New `make ptest` locally: 55s Doesn't work on CI for some reason, so actually disabled for `make ptest`. Use ./run_tests.py instead --- run_tests.py | 27 +++++++++++++++ test/__init__.py | 0 test/apps/ctrl/test.py | 9 ++--- test/apps/db_dump/test.py | 11 +++--- test/apps/passwd/test.py | 11 +++--- test/apps/signal/test.py | 11 +++--- test/broker/Makefile | 2 +- test/broker/msg_sequence_test.py | 2 +- test/broker/test.py | 9 +++-- test/client/test.py | 8 +++-- test/lib/Makefile | 2 +- test/lib/test.py | 9 +++-- test/ptest.py | 58 ++++++++++++++++++-------------- 13 files changed, 106 insertions(+), 53 deletions(-) create mode 100755 run_tests.py create mode 100644 test/__init__.py diff --git a/run_tests.py b/run_tests.py new file mode 100755 index 000000000..58116481b --- /dev/null +++ b/run_tests.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +import inspect +import os +import sys + +sys.path.insert(0, "test") + +import ptest +import test.apps.ctrl.test as p_ctrl +import test.apps.db_dump.test as p_db_dump +import test.apps.passwd.test as p_passwd +import test.apps.signal.test as p_signal +import test.broker.test as p_broker +import test.client.test as p_client +import test.lib.test as p_lib + +test = ptest.PTest() +test.add_tests(p_client.tests, "test/client") +test.add_tests(p_lib.tests, "test/lib") +test.add_tests(p_ctrl.tests, "test/apps/ctrl") +test.add_tests(p_db_dump.tests, "test/apps/db_dump") +test.add_tests(p_passwd.tests, "test/apps/ctrl") +test.add_tests(p_signal.tests, "test/apps/signal") +test.add_tests(p_broker.tests, "test/broker") + +test.run() diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/apps/ctrl/test.py b/test/apps/ctrl/test.py index 6afe970cc..463ddf08f 100755 --- a/test/apps/ctrl/test.py +++ b/test/apps/ctrl/test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -import mosq_test_helper -import pathlib +import sys +sys.path.insert(0, "../..") import ptest tests = [ @@ -11,5 +11,6 @@ tests = [ (2, './ctrl-dynsec.py'), ] -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/apps/db_dump/test.py b/test/apps/db_dump/test.py index e19669393..0c2ec032f 100755 --- a/test/apps/db_dump/test.py +++ b/test/apps/db_dump/test.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import mosq_test_helper +import os import pathlib +import sys +sys.path.insert(0, "../..") import ptest tests = [] -for test_file in pathlib.Path('.').glob('db-dump-*.py'): +for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('db-dump-*.py'): tests.append((1, test_file.resolve())) -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/apps/passwd/test.py b/test/apps/passwd/test.py index 680a8b09f..a51e19fb3 100755 --- a/test/apps/passwd/test.py +++ b/test/apps/passwd/test.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import mosq_test_helper +import os import pathlib +import sys +sys.path.insert(0, "../..") import ptest tests = [] -for test_file in pathlib.Path('.').glob('passwd-*.py'): +for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('passwd-*.py'): tests.append((1, test_file.resolve())) -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/apps/signal/test.py b/test/apps/signal/test.py index 2ee50bdd8..bd5381560 100755 --- a/test/apps/signal/test.py +++ b/test/apps/signal/test.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import mosq_test_helper +import os import pathlib +import sys +sys.path.insert(0, "../..") import ptest tests = [] -for test_file in pathlib.Path('.').glob('signal-*.py'): +for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('signal-*.py'): tests.append((1, test_file.resolve())) -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/broker/Makefile b/test/broker/Makefile index 3aeca36ce..f93ee2c85 100644 --- a/test/broker/Makefile +++ b/test/broker/Makefile @@ -15,7 +15,7 @@ clean : test-compile : $(MAKE) -C c -ptest : test-compile msg_sequence_test +ptest : test-compile ./test.py test : test-compile msg_sequence_test 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 20 21 diff --git a/test/broker/msg_sequence_test.py b/test/broker/msg_sequence_test.py index 696bc9199..87f224aec 100755 --- a/test/broker/msg_sequence_test.py +++ b/test/broker/msg_sequence_test.py @@ -209,7 +209,7 @@ def do_test(hostname, port, protocol): failed_tests.append(this_test) sock = mosq_test.client_connect_only(hostname=hostname, port=port, timeout=2, protocol=protocol) this_test.process_all(sock) - print("\033[32m" + tname + "\033[0m") + #print("\033[32m" + tname + "\033[0m") succeeded += 1 sock.close() failed_tests.pop() diff --git a/test/broker/test.py b/test/broker/test.py index 58592c782..cc5bd406c 100755 --- a/test/broker/test.py +++ b/test/broker/test.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 -import mosq_test_helper +import sys +sys.path.insert(0, "..") import ptest tests = [ #(ports required, 'path'), + (1, './msg_sequence_test.py'), (1, './01-bad-initial-packets.py'), (1, './01-connect-575314.py'), (1, './01-connect-accept-protocol.py'), @@ -289,5 +291,6 @@ tests = [ (1, './21-proxy-v2-ssl-require-tls-success.py'), ] -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/client/test.py b/test/client/test.py index 0adb02f12..5c3bd83ca 100755 --- a/test/client/test.py +++ b/test/client/test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -import mosq_test_helper +import sys +sys.path.insert(0, "..") import ptest tests = [ @@ -47,5 +48,6 @@ tests = [ (2, './04-rr-qos1-ws.py'), ] -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/lib/Makefile b/test/lib/Makefile index 9016d2a28..ab3a6b7c9 100644 --- a/test/lib/Makefile +++ b/test/lib/Makefile @@ -10,7 +10,7 @@ all : check : test -ptest : test-compile msg_sequence_test +ptest : test-compile ./test.py msg_sequence_test: test-compile-c diff --git a/test/lib/test.py b/test/lib/test.py index eca05067d..d3ab6c1c2 100755 --- a/test/lib/test.py +++ b/test/lib/test.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -import mosq_test_helper +import sys +sys.path.insert(0, "..") import ptest tests = [ + (1, './msg_sequence_test.py'), (1, './01-con-discon-success-v5.py'), (1, './01-con-discon-success.py'), (1, './01-con-discon-will-clear.py'), @@ -63,5 +65,6 @@ tests = [ ] -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/ptest.py b/test/ptest.py index 616d4710f..00d90e686 100755 --- a/test/ptest.py +++ b/test/ptest.py @@ -1,20 +1,19 @@ #!/usr/bin/env python3 +from pathlib import Path import subprocess import time import sys class PTestCase(): - def __init__(self, testdef): - self.ports = testdef[0] - self.cmd = str(testdef[1]) + def __init__(self, path, ports, cmd, args=None): + self.path = path + self.ports = ports + self.cmd = str(cmd) self.attempts = 0 - try: - if isinstance(testdef[2], (list,)): - self.args = [self.cmd] + testdef[2] - else: - self.args = [self.cmd] + [testdef[2]] - except IndexError: + if args is not None: + self.args = [self.cmd] + args + else: self.args = [self.cmd] self.start_time = 0 self.proc = None @@ -26,12 +25,12 @@ class PTestCase(): for p in self.mosq_port: self.run_args.append(str(p)) - self.proc = subprocess.Popen(self.run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.proc = subprocess.Popen(self.run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.path) self.start_time = time.time() def print_result(self, col): cmd = " ".join(self.run_args) - print(f"{self.runtime:0.3f}s : \033[{col}m{cmd}\033[0m") + print(f"{self.runtime:0.3f}s : \033[{col}m{self.path}/{cmd}\033[0m") class PTest(): @@ -40,18 +39,28 @@ class PTest(): self.max_running = 20 self.tests = [] - def next_test(self, tests, ports): - if len(tests) == 0 or len(ports) == 0: + def add_tests(self, test_list, path=".", label=""): + for testdef in test_list: + try: + if isinstance(testdef[2], (list,)): + args = testdef[2] + else: + args = [testdef[2]] + except IndexError: + args = None + self.tests.append(PTestCase(path, testdef[0], testdef[1], args)) + + def _next_test(self, ports): + if len(self.tests) == 0 or len(ports) == 0: return - test = tests.pop() + test = self.tests.pop() test.mosq_port = [] if len(ports) < test.ports: - tests.insert(0, test) + self.tests.insert(0, test) return None else: - for i in range(0, test.ports): proc_port = ports.pop() test.mosq_port.append(proc_port) @@ -59,32 +68,31 @@ class PTest(): test.start() return test - def run_tests(self, test_list): + self.add_tests(test_list) + self.run() + + def run(self): ports = list(range(self.minport, self.minport+self.max_running+1)) start_time = time.time() passed = 0 retried = 0 failed = 0 - tests = [] - for t in test_list: - tests.append(PTestCase(t)) - failed_tests = [] running_tests = [] retry_tests = [] - while len(tests) > 0 or len(running_tests) > 0 or len(retry_tests) > 0: + while len(self.tests) > 0 or len(running_tests) > 0 or len(retry_tests) > 0: if len(running_tests) <= self.max_running: - t = self.next_test(tests, ports) + t = self._next_test(ports) if t is None: time.sleep(0.1) else: running_tests.append(t) - if len(running_tests) == 0 and len(tests) == 0 and len(retry_tests) > 0: + if len(running_tests) == 0 and len(self.tests) == 0 and len(retry_tests) > 0: # Only retry tests after everything else has finished to reduce load - tests = retry_tests + self.tests = retry_tests retry_tests = [] for t in running_tests: