From 19ce8004aacb158ce58db76ed32ab8748f9d3a59 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 7 Nov 2024 09:40:27 +0000 Subject: [PATCH 1/4] Check if test is in test pack --- .../python/src/test_orc/test_orchestrator.py | 18 ++++++++++++++++-- framework/python/src/test_orc/test_pack.py | 14 +++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 4085e91ad..84f6e9d23 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -128,6 +128,10 @@ def run_test_modules(self): # Duplicate test obj so we don't alter the source test_copy = copy.deepcopy(test) + # Do not add test if it is not enabled + if not self._is_test_enabled(test_copy.name, device): + continue + # Set result to Not Started test_copy.result = TestResult.NOT_STARTED @@ -146,7 +150,7 @@ def run_test_modules(self): # Increment number of tests that will be run self.get_session().add_total_tests(len(module.tests)) - # Store enabled test modules in the TestsOrchectrator object + # Store enabled test modules in the TestOrchectrator object self._test_modules_running = test_modules self._current_module = 0 @@ -399,6 +403,7 @@ def _is_module_enabled(self, module, device): # Enable module as fallback enabled = True + if device.test_modules is not None: test_modules = device.test_modules if module.name in test_modules: @@ -410,6 +415,13 @@ def _is_module_enabled(self, module, device): return enabled + def _is_test_enabled(self, test, device): + + test_pack_name = device.test_pack + test_pack = self.get_test_pack(test_pack_name) + + return test_pack.get_test(test) is not None + def _run_test_module(self, module): """Start the test container and extract the results.""" @@ -441,7 +453,9 @@ def _run_test_module(self, module): if hasattr(test_copy, "recommendations"): test_copy.recommendations = None - self.get_session().add_test_result(test_copy) + # Only add/update the test if it is enabled + if self._is_test_enabled(test_copy.name, device): + self.get_session().add_test_result(test_copy) # Start the test module module.start(device) diff --git a/framework/python/src/test_orc/test_pack.py b/framework/python/src/test_orc/test_pack.py index b1e3d5d3c..b743cdb58 100644 --- a/framework/python/src/test_orc/test_pack.py +++ b/framework/python/src/test_orc/test_pack.py @@ -27,12 +27,20 @@ class TestPack: # pylint: disable=too-few-public-methods,too-many-instance-attr tests: List[dict] = field(default_factory=lambda: []) language: Dict = field(default_factory=lambda: defaultdict(dict)) - def get_required_result(self, test_name: str) -> str: + def get_test(self, test_name: str) -> str: + """Get details of a test from the test pack""" for test in self.tests: if "name" in test and test["name"].lower() == test_name.lower(): - if "required_result" in test: - return test["required_result"] + return test + + def get_required_result(self, test_name: str) -> str: + """Fetch the required result of the test""" + + test = self.get_test(test_name) + + if test is not None and "required_result" in test: + return test["required_result"] return "Informational" From 1180245636bc8ae5e2f097666dac3c82bcae973a Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 13 Nov 2024 10:21:17 +0000 Subject: [PATCH 2/4] First commit --- .../src/core/docker/test_docker_module.py | 9 ++- .../python/src/test_orc/test_orchestrator.py | 12 +++- framework/python/src/test_orc/test_pack.py | 8 +++ modules/test/base/python/src/test_module.py | 64 ++++++++++++++----- 4 files changed, 74 insertions(+), 19 deletions(-) diff --git a/framework/python/src/core/docker/test_docker_module.py b/framework/python/src/core/docker/test_docker_module.py index 4bbf72594..4904d9536 100644 --- a/framework/python/src/core/docker/test_docker_module.py +++ b/framework/python/src/core/docker/test_docker_module.py @@ -27,11 +27,13 @@ class TestModule(Module): """Represents a test module.""" - def __init__(self, module_config_file, session, extra_hosts): + def __init__(self, module_config_file, test_orc, session, extra_hosts): super().__init__(module_config_file=module_config_file, session=session, extra_hosts=extra_hosts) + self._test_orc = test_orc + # Set IP Index for all test modules self.ip_index = 9 @@ -93,12 +95,17 @@ def _setup_runtime(self, device): util.run_command(f'chown -R {host_user} {self.device_monitor_capture}') def get_environment(self, device): + + # Obtain the test pack + test_pack = self._test_orc.get_test_pack(device.test_pack) + environment = { 'TZ': self.get_session().get_timezone(), 'HOST_USER': self.get_session().get_host_user(), 'DEVICE_MAC': device.mac_addr, 'IPV4_ADDR': device.ip_addr, 'DEVICE_TEST_MODULES': json.dumps(device.test_modules), + 'DEVICE_TEST_PACK': json.dumps(test_pack.to_dict()), 'IPV4_SUBNET': self.get_session().get_ipv4_subnet(), 'IPV6_SUBNET': self.get_session().get_ipv6_subnet(), 'DEV_IFACE': self.get_session().get_device_interface(), diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 84f6e9d23..d133fefd9 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -120,6 +120,8 @@ def run_test_modules(self): if not self._is_module_enabled(module, device): continue + num_tests = 0 + # Add module to list of modules to run test_modules.append(module) @@ -147,8 +149,11 @@ def run_test_modules(self): # Add test result to the session self.get_session().add_test_result(test_copy) + # Increment number of tests being run by this module + num_tests += 1 + # Increment number of tests that will be run - self.get_session().add_total_tests(len(module.tests)) + self.get_session().add_total_tests(num_tests) # Store enabled test modules in the TestOrchectrator object self._test_modules_running = test_modules @@ -651,7 +656,10 @@ def _load_test_module(self, module_dir): module_conf_file = os.path.join(self._root_path, modules_dir, module_dir, MODULE_CONFIG) - module = TestModule(module_conf_file, self.get_session(), extra_hosts) + module = TestModule(module_conf_file, + self, + self.get_session(), + extra_hosts) if module.depends_on is not None: self._load_test_module(module.depends_on) self._test_modules.append(module) diff --git a/framework/python/src/test_orc/test_pack.py b/framework/python/src/test_orc/test_pack.py index b743cdb58..a2e7c5f97 100644 --- a/framework/python/src/test_orc/test_pack.py +++ b/framework/python/src/test_orc/test_pack.py @@ -48,3 +48,11 @@ def get_message(self, name: str) -> str: if name in self.language: return self.language[name] return "Message not found" + + def to_dict(self): + return { + "name": self.name, + "description": self.description, + "tests": self.tests, + "language": self.language + } diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index 6b0170ba4..9e2461718 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -41,12 +41,15 @@ def __init__(self, self._ipv4_subnet = os.environ.get('IPV4_SUBNET', '') self._ipv6_subnet = os.environ.get('IPV6_SUBNET', '') self._dev_iface_mac = os.environ.get('DEV_IFACE_MAC', '') + self._device_test_pack = json.loads(os.environ.get('DEVICE_TEST_PACK', '')) self._add_logger(log_name=log_name) self._config = self._read_config( conf_file=conf_file if conf_file is not None else CONF_FILE) self._device_ipv4_addr = None self._device_ipv6_addr = None + print(self._device_test_pack) + def _add_logger(self, log_name): global LOGGER LOGGER = logger.get_logger(name=log_name) @@ -63,22 +66,51 @@ def _get_tests(self): def _get_device_tests(self, device_test_module): module_tests = self._config['config']['tests'] - if device_test_module is None: - return module_tests - elif not device_test_module['enabled']: - return [] - else: - for test in module_tests: - # Resolve device specific configurations for the test if it exists - # and update module test config with device config options - if 'tests' in device_test_module: - if test['name'] in device_test_module['tests']: - dev_test_config = device_test_module['tests'][test['name']] - if 'enabled' in dev_test_config: - test['enabled'] = dev_test_config['enabled'] - if 'config' in test and 'config' in dev_test_config: - test['config'].update(dev_test_config['config']) - return module_tests + tests_to_run = module_tests + + # If no device specific tests have been provided, add all + if device_test_module is not None: + # Do not run any tests if module is disabled for this device + if not device_test_module['enabled']: + print("Test module is not enabled for the device") + return [] + + # Check if all tests are in the test pack and enabled for the device + for test in tests_to_run: + + print("Test to run: " + test['name']) + + # Resolve device specific configurations for the test if it exists + # and update module test config with device config options + if 'tests' in device_test_module: + + if test['name'] in device_test_module['tests']: + dev_test_config = device_test_module['tests'][test['name']] + + # Check if the test is enabled in the device config + if 'enabled' in dev_test_config: + test['enabled'] = dev_test_config['enabled'] + + # Copy over any device specific test configuration + if 'config' in test and 'config' in dev_test_config: + test['config'].update(dev_test_config['config']) + + # Search for the module test in the test pack + found = False + for test_pack_test in self._device_test_pack['tests']: + print("Checking test name " + test_pack_test['name']) + if test_pack_test['name'] == test['name']: + # Test is in the test pack + print("Found the test in this module") + found = True + + if not found: + tests_to_run.remove(test) + + print("Tests to run:") + print(tests_to_run) + + return tests_to_run def _get_device_test_module(self): if 'DEVICE_TEST_MODULES' in os.environ: From 52eeae79b9230e2c3fea3f6599566266ea0c95c8 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 13 Nov 2024 16:01:16 +0000 Subject: [PATCH 3/4] Fix disabling tests --- modules/test/base/python/src/test_module.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index 9e2461718..21de78143 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -48,8 +48,6 @@ def __init__(self, self._device_ipv4_addr = None self._device_ipv6_addr = None - print(self._device_test_pack) - def _add_logger(self, log_name): global LOGGER LOGGER = logger.get_logger(name=log_name) @@ -72,14 +70,14 @@ def _get_device_tests(self, device_test_module): if device_test_module is not None: # Do not run any tests if module is disabled for this device if not device_test_module['enabled']: - print("Test module is not enabled for the device") return [] + # Tests that will be removed because they are not in the test pack + remove_tests = [] + # Check if all tests are in the test pack and enabled for the device for test in tests_to_run: - print("Test to run: " + test['name']) - # Resolve device specific configurations for the test if it exists # and update module test config with device config options if 'tests' in device_test_module: @@ -98,17 +96,14 @@ def _get_device_tests(self, device_test_module): # Search for the module test in the test pack found = False for test_pack_test in self._device_test_pack['tests']: - print("Checking test name " + test_pack_test['name']) if test_pack_test['name'] == test['name']: # Test is in the test pack - print("Found the test in this module") found = True if not found: - tests_to_run.remove(test) - - print("Tests to run:") - print(tests_to_run) + remove_tests.append(test) + for test in remove_tests: + tests_to_run.remove(test) return tests_to_run From 6b9d0f55e6447616af14b9ac5413e50d96270f62 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 13 Nov 2024 16:36:40 -0700 Subject: [PATCH 4/4] Pass test pack with env var for unit tests --- testing/unit/run_test_module.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/testing/unit/run_test_module.sh b/testing/unit/run_test_module.sh index 37516ee72..8e31e6860 100644 --- a/testing/unit/run_test_module.sh +++ b/testing/unit/run_test_module.sh @@ -15,6 +15,10 @@ # limitations under the License. # Must be run from the root directory of Testrun + +# Read the JSON file into a variable +DEVICE_TEST_PACK=$(