Only run tests if they are included in the device's test pack

framework/python/src/core/docker/test_docker_module.py

@@ -27,11 +27,13 @@
 class TestModule(Module):
   """Represents a test module."""
 
-  def __init__(self, module_config_file, session, extra_hosts):
+  def __init__(self, module_config_file, test_orc, session, extra_hosts):
     super().__init__(module_config_file=module_config_file,
                      session=session,
                      extra_hosts=extra_hosts)
 
+    self._test_orc = test_orc
+
     # Set IP Index for all test modules
     self.ip_index = 9
 
@@ -91,12 +93,17 @@ def _setup_runtime(self, device):
     util.run_command(f'chown -R {host_user} {self.device_monitor_capture}')
 
   def get_environment(self, device):
+
+    # Obtain the test pack
+    test_pack = self._test_orc.get_test_pack(device.test_pack)
+
     environment = {
         'TZ': self.get_session().get_timezone(),
         'HOST_USER': self.get_session().get_host_user(),
         'DEVICE_MAC': device.mac_addr,
         'IPV4_ADDR': device.ip_addr,
         'DEVICE_TEST_MODULES': json.dumps(device.test_modules),
+        'DEVICE_TEST_PACK': json.dumps(test_pack.to_dict()),
         'IPV4_SUBNET': self.get_session().get_ipv4_subnet(),
         'IPV6_SUBNET': self.get_session().get_ipv6_subnet(),
         'DEV_IFACE': self.get_session().get_device_interface(),

framework/python/src/test_orc/test_orchestrator.py

@@ -120,6 +120,8 @@ def run_test_modules(self):
       if not self._is_module_enabled(module, device):
         continue
 
+      num_tests = 0
+
       # Add module to list of modules to run
       test_modules.append(module)
 
@@ -128,6 +130,10 @@ def run_test_modules(self):
         # Duplicate test obj so we don't alter the source
         test_copy = copy.deepcopy(test)
 
+        # Do not add test if it is not enabled
+        if not self._is_test_enabled(test_copy.name, device):
+          continue
+
         # Set result to Not Started
         test_copy.result = TestResult.NOT_STARTED
 
@@ -143,10 +149,13 @@ def run_test_modules(self):
         # Add test result to the session
         self.get_session().add_test_result(test_copy)
 
+        # Increment number of tests being run by this module
+        num_tests += 1
+
       # Increment number of tests that will be run
-      self.get_session().add_total_tests(len(module.tests))
+      self.get_session().add_total_tests(num_tests)
 
-    # Store enabled test modules in the TestsOrchectrator object
+    # Store enabled test modules in the TestOrchectrator object
     self._test_modules_running = test_modules
     self._current_module = 0
 
@@ -399,6 +408,7 @@ def _is_module_enabled(self, module, device):
 
     # Enable module as fallback
     enabled = True
+
     if device.test_modules is not None:
       test_modules = device.test_modules
       if module.name in test_modules:
@@ -410,6 +420,13 @@ def _is_module_enabled(self, module, device):
 
     return enabled
 
+  def _is_test_enabled(self, test, device):
+
+    test_pack_name = device.test_pack
+    test_pack = self.get_test_pack(test_pack_name)
+
+    return test_pack.get_test(test) is not None
+
   def _run_test_module(self, module):
     """Start the test container and extract the results."""
 
@@ -441,7 +458,9 @@ def _run_test_module(self, module):
       if hasattr(test_copy, "recommendations"):
         test_copy.recommendations = None
 
-      self.get_session().add_test_result(test_copy)
+      # Only add/update the test if it is enabled
+      if self._is_test_enabled(test_copy.name, device):
+        self.get_session().add_test_result(test_copy)
 
     # Start the test module
     module.start(device)
@@ -637,7 +656,10 @@ def _load_test_module(self, module_dir):
       module_conf_file = os.path.join(self._root_path, modules_dir, module_dir,
                                       MODULE_CONFIG)
 
-      module = TestModule(module_conf_file, self.get_session(), extra_hosts)
+      module = TestModule(module_conf_file,
+                          self,
+                          self.get_session(),
+                          extra_hosts)
       if module.depends_on is not None:
         self._load_test_module(module.depends_on)
       self._test_modules.append(module)

framework/python/src/test_orc/test_pack.py

@@ -27,16 +27,32 @@ class TestPack:  # pylint: disable=too-few-public-methods,too-many-instance-attr
   tests: List[dict] = field(default_factory=lambda: [])
   language: Dict = field(default_factory=lambda: defaultdict(dict))
 
-  def get_required_result(self, test_name: str) -> str:
+  def get_test(self, test_name: str) -> str:
+    """Get details of a test from the test pack"""
 
     for test in self.tests:
       if "name" in test and test["name"].lower() == test_name.lower():
-        if "required_result" in test:
-          return test["required_result"]
+        return test
+
+  def get_required_result(self, test_name: str) -> str:
+    """Fetch the required result of the test"""
+
+    test = self.get_test(test_name)
+
+    if test is not None and "required_result" in test:
+      return test["required_result"]
 
     return "Informational"
 
   def get_message(self, name: str) -> str:
     if name in self.language:
       return self.language[name]
     return "Message not found"
+
+  def to_dict(self):
+    return {
+      "name": self.name,
+      "description": self.description,
+      "tests": self.tests,
+      "language": self.language
+    }

modules/test/base/python/src/test_module.py

@@ -41,6 +41,7 @@ def __init__(self,
     self._ipv4_subnet = os.environ.get('IPV4_SUBNET', '')
     self._ipv6_subnet = os.environ.get('IPV6_SUBNET', '')
     self._dev_iface_mac = os.environ.get('DEV_IFACE_MAC', '')
+    self._device_test_pack = json.loads(os.environ.get('DEVICE_TEST_PACK', ''))
     self._add_logger(log_name=log_name)
     self._config = self._read_config(
         conf_file=conf_file if conf_file is not None else CONF_FILE)
@@ -63,22 +64,48 @@ def _get_tests(self):
 
   def _get_device_tests(self, device_test_module):
     module_tests = self._config['config']['tests']
-    if device_test_module is None:
-      return module_tests
-    elif not device_test_module['enabled']:
-      return []
-    else:
-      for test in module_tests:
-        # Resolve device specific configurations for the test if it exists
-        # and update module test config with device config options
-        if 'tests' in device_test_module:
-          if test['name'] in device_test_module['tests']:
-            dev_test_config = device_test_module['tests'][test['name']]
-            if 'enabled' in dev_test_config:
-              test['enabled'] = dev_test_config['enabled']
-            if 'config' in test and 'config' in dev_test_config:
-              test['config'].update(dev_test_config['config'])
-      return module_tests
+    tests_to_run = module_tests
+
+    # If no device specific tests have been provided, add all
+    if device_test_module is not None:
+      # Do not run any tests if module is disabled for this device
+      if not device_test_module['enabled']:
+        return []
+
+    # Tests that will be removed because they are not in the test pack
+    remove_tests = []
+
+    # Check if all tests are in the test pack and enabled for the device
+    for test in tests_to_run:
+
+      # Resolve device specific configurations for the test if it exists
+      # and update module test config with device config options
+      if 'tests' in device_test_module:
+
+        if test['name'] in device_test_module['tests']:
+          dev_test_config = device_test_module['tests'][test['name']]
+
+          # Check if the test is enabled in the device config
+          if 'enabled' in dev_test_config:
+            test['enabled'] = dev_test_config['enabled']
+
+          # Copy over any device specific test configuration
+          if 'config' in test and 'config' in dev_test_config:
+            test['config'].update(dev_test_config['config'])
+
+      # Search for the module test in the test pack
+      found = False
+      for test_pack_test in self._device_test_pack['tests']:
+        if test_pack_test['name'] == test['name']:
+          # Test is in the test pack
+          found = True
+
+      if not found:
+        remove_tests.append(test)
+    for test in remove_tests:
+      tests_to_run.remove(test)
+
+    return tests_to_run
 
   def _get_device_test_module(self):
     if 'DEVICE_TEST_MODULES' in os.environ:

testing/unit/run_test_module.sh

@@ -15,6 +15,10 @@
 # limitations under the License.
 
 # Must be run from the root directory of Testrun
+
+# Read the JSON file into a variable
+DEVICE_TEST_PACK=$(<resources/test_packs/qualification.json)
+
 # Function to run tests inside Docker
 run_test() {
   local MODULE_NAME=$1
@@ -29,26 +33,24 @@ run_test() {
   local UNIT_TEST_DIR_DST="/testing/unit/$MODULE_NAME"
   local UNIT_TEST_FILE_DST="/testrun/python/src/module_test.py"
 
-  # Build the docker run command
-  local DOCKER_CMD="sudo docker run --rm --name ${MODULE_NAME}-unit-test"
-
-  # Add volume mount for the main test file
-  DOCKER_CMD="$DOCKER_CMD -v $UNIT_TEST_FILE_SRC:$UNIT_TEST_FILE_DST"
+  # Build the docker run command using an array
+  DOCKER_CMD=(
+    sudo docker run --rm --name "${MODULE_NAME}-unit-test"
+    -e "DEVICE_TEST_PACK=$DEVICE_TEST_PACK"
+    -v "$UNIT_TEST_FILE_SRC:$UNIT_TEST_FILE_DST"
+  )
 
   # Add volume mounts for additional directories if provided
   for DIR in "${DIRS[@]}"; do
-    DOCKER_CMD="$DOCKER_CMD -v $UNIT_TEST_DIR_SRC/$DIR:$UNIT_TEST_DIR_DST/$DIR"
+    DOCKER_CMD+=("-v" "$UNIT_TEST_DIR_SRC/$DIR:$UNIT_TEST_DIR_DST/$DIR")
   done
 
   # Add the container image and entry point
-  DOCKER_CMD="$DOCKER_CMD testrun/${MODULE_NAME}-test $UNIT_TEST_FILE_DST"
-
-  # Temporarily disable 'set -e' to capture exit code
-  set +e
+  DOCKER_CMD+=("testrun/${MODULE_NAME}-test" "$UNIT_TEST_FILE_DST")
 
-  # Execute the Docker command
+  # Run the Docker command
   echo "Running test for ${MODULE_NAME}..."
-  eval $DOCKER_CMD
+  "${DOCKER_CMD[@]}"
 
   # Capture the exit code
   local exit_code=$?
@@ -77,4 +79,4 @@ else
 fi
 
 # Exit with the captured exit code
-exit $exit_code
+exit $exit_code