diff --git a/framework/python/src/core/docker/test_docker_module.py b/framework/python/src/core/docker/test_docker_module.py index 13acd4857..3198ef1ba 100644 --- a/framework/python/src/core/docker/test_docker_module.py +++ b/framework/python/src/core/docker/test_docker_module.py @@ -27,11 +27,13 @@ class TestModule(Module): """Represents a test module.""" - def __init__(self, module_config_file, session, extra_hosts): + def __init__(self, module_config_file, test_orc, session, extra_hosts): super().__init__(module_config_file=module_config_file, session=session, extra_hosts=extra_hosts) + self._test_orc = test_orc + # Set IP Index for all test modules self.ip_index = 9 @@ -91,12 +93,17 @@ def _setup_runtime(self, device): util.run_command(f'chown -R {host_user} {self.device_monitor_capture}') def get_environment(self, device): + + # Obtain the test pack + test_pack = self._test_orc.get_test_pack(device.test_pack) + environment = { 'TZ': self.get_session().get_timezone(), 'HOST_USER': self.get_session().get_host_user(), 'DEVICE_MAC': device.mac_addr, 'IPV4_ADDR': device.ip_addr, 'DEVICE_TEST_MODULES': json.dumps(device.test_modules), + 'DEVICE_TEST_PACK': json.dumps(test_pack.to_dict()), 'IPV4_SUBNET': self.get_session().get_ipv4_subnet(), 'IPV6_SUBNET': self.get_session().get_ipv6_subnet(), 'DEV_IFACE': self.get_session().get_device_interface(), diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 4085e91ad..d133fefd9 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -120,6 +120,8 @@ def run_test_modules(self): if not self._is_module_enabled(module, device): continue + num_tests = 0 + # Add module to list of modules to run test_modules.append(module) @@ -128,6 +130,10 @@ def run_test_modules(self): # Duplicate test obj so we don't alter the source test_copy = copy.deepcopy(test) + # Do not add test if it is not enabled + if not self._is_test_enabled(test_copy.name, device): + continue + # Set result to Not Started test_copy.result = TestResult.NOT_STARTED @@ -143,10 +149,13 @@ def run_test_modules(self): # Add test result to the session self.get_session().add_test_result(test_copy) + # Increment number of tests being run by this module + num_tests += 1 + # Increment number of tests that will be run - self.get_session().add_total_tests(len(module.tests)) + self.get_session().add_total_tests(num_tests) - # Store enabled test modules in the TestsOrchectrator object + # Store enabled test modules in the TestOrchectrator object self._test_modules_running = test_modules self._current_module = 0 @@ -399,6 +408,7 @@ def _is_module_enabled(self, module, device): # Enable module as fallback enabled = True + if device.test_modules is not None: test_modules = device.test_modules if module.name in test_modules: @@ -410,6 +420,13 @@ def _is_module_enabled(self, module, device): return enabled + def _is_test_enabled(self, test, device): + + test_pack_name = device.test_pack + test_pack = self.get_test_pack(test_pack_name) + + return test_pack.get_test(test) is not None + def _run_test_module(self, module): """Start the test container and extract the results.""" @@ -441,7 +458,9 @@ def _run_test_module(self, module): if hasattr(test_copy, "recommendations"): test_copy.recommendations = None - self.get_session().add_test_result(test_copy) + # Only add/update the test if it is enabled + if self._is_test_enabled(test_copy.name, device): + self.get_session().add_test_result(test_copy) # Start the test module module.start(device) @@ -637,7 +656,10 @@ def _load_test_module(self, module_dir): module_conf_file = os.path.join(self._root_path, modules_dir, module_dir, MODULE_CONFIG) - module = TestModule(module_conf_file, self.get_session(), extra_hosts) + module = TestModule(module_conf_file, + self, + self.get_session(), + extra_hosts) if module.depends_on is not None: self._load_test_module(module.depends_on) self._test_modules.append(module) diff --git a/framework/python/src/test_orc/test_pack.py b/framework/python/src/test_orc/test_pack.py index b1e3d5d3c..a2e7c5f97 100644 --- a/framework/python/src/test_orc/test_pack.py +++ b/framework/python/src/test_orc/test_pack.py @@ -27,12 +27,20 @@ class TestPack: # pylint: disable=too-few-public-methods,too-many-instance-attr tests: List[dict] = field(default_factory=lambda: []) language: Dict = field(default_factory=lambda: defaultdict(dict)) - def get_required_result(self, test_name: str) -> str: + def get_test(self, test_name: str) -> str: + """Get details of a test from the test pack""" for test in self.tests: if "name" in test and test["name"].lower() == test_name.lower(): - if "required_result" in test: - return test["required_result"] + return test + + def get_required_result(self, test_name: str) -> str: + """Fetch the required result of the test""" + + test = self.get_test(test_name) + + if test is not None and "required_result" in test: + return test["required_result"] return "Informational" @@ -40,3 +48,11 @@ def get_message(self, name: str) -> str: if name in self.language: return self.language[name] return "Message not found" + + def to_dict(self): + return { + "name": self.name, + "description": self.description, + "tests": self.tests, + "language": self.language + } diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index 6b0170ba4..21de78143 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -41,6 +41,7 @@ def __init__(self, self._ipv4_subnet = os.environ.get('IPV4_SUBNET', '') self._ipv6_subnet = os.environ.get('IPV6_SUBNET', '') self._dev_iface_mac = os.environ.get('DEV_IFACE_MAC', '') + self._device_test_pack = json.loads(os.environ.get('DEVICE_TEST_PACK', '')) self._add_logger(log_name=log_name) self._config = self._read_config( conf_file=conf_file if conf_file is not None else CONF_FILE) @@ -63,22 +64,48 @@ def _get_tests(self): def _get_device_tests(self, device_test_module): module_tests = self._config['config']['tests'] - if device_test_module is None: - return module_tests - elif not device_test_module['enabled']: - return [] - else: - for test in module_tests: - # Resolve device specific configurations for the test if it exists - # and update module test config with device config options - if 'tests' in device_test_module: - if test['name'] in device_test_module['tests']: - dev_test_config = device_test_module['tests'][test['name']] - if 'enabled' in dev_test_config: - test['enabled'] = dev_test_config['enabled'] - if 'config' in test and 'config' in dev_test_config: - test['config'].update(dev_test_config['config']) - return module_tests + tests_to_run = module_tests + + # If no device specific tests have been provided, add all + if device_test_module is not None: + # Do not run any tests if module is disabled for this device + if not device_test_module['enabled']: + return [] + + # Tests that will be removed because they are not in the test pack + remove_tests = [] + + # Check if all tests are in the test pack and enabled for the device + for test in tests_to_run: + + # Resolve device specific configurations for the test if it exists + # and update module test config with device config options + if 'tests' in device_test_module: + + if test['name'] in device_test_module['tests']: + dev_test_config = device_test_module['tests'][test['name']] + + # Check if the test is enabled in the device config + if 'enabled' in dev_test_config: + test['enabled'] = dev_test_config['enabled'] + + # Copy over any device specific test configuration + if 'config' in test and 'config' in dev_test_config: + test['config'].update(dev_test_config['config']) + + # Search for the module test in the test pack + found = False + for test_pack_test in self._device_test_pack['tests']: + if test_pack_test['name'] == test['name']: + # Test is in the test pack + found = True + + if not found: + remove_tests.append(test) + for test in remove_tests: + tests_to_run.remove(test) + + return tests_to_run def _get_device_test_module(self): if 'DEVICE_TEST_MODULES' in os.environ: diff --git a/testing/unit/run_test_module.sh b/testing/unit/run_test_module.sh index 37516ee72..8e31e6860 100644 --- a/testing/unit/run_test_module.sh +++ b/testing/unit/run_test_module.sh @@ -15,6 +15,10 @@ # limitations under the License. # Must be run from the root directory of Testrun + +# Read the JSON file into a variable +DEVICE_TEST_PACK=$(