Merge release v2.2.2 into main

docs/additional_config.md

@@ -78,7 +78,7 @@ set the log_level property for:
     ```
     "test_modules":{
         "connection":{
-          "log_level": "DEGUG"
+          "log_level": "DEBUG"
         }
       }
       ```

framework/python/src/api/api.py

@@ -599,17 +599,19 @@ async def save_device(self, request: Request, response: Response):
       if device is None:
 
         # Create new device
-        device = Device()
-        device.mac_addr = device_json.get(DEVICE_MAC_ADDR_KEY).lower()
-        device.manufacturer = device_json.get(DEVICE_MANUFACTURER_KEY)
-        device.model = device_json.get(DEVICE_MODEL_KEY)
-        device.test_pack = device_json.get(DEVICE_TEST_PACK_KEY)
-        device.type = device_json.get(DEVICE_TYPE_KEY)
-        device.technology = device_json.get(DEVICE_TECH_KEY)
-        device.additional_info = device_json.get(DEVICE_ADDITIONAL_INFO_KEY)
-
-        device.device_folder = device.manufacturer + " " + device.model
-        device.test_modules = device_json.get(DEVICE_TEST_MODULES_KEY)
+        device_manufacturer = device_json.get(DEVICE_MANUFACTURER_KEY)
+        device_model = device_json.get(DEVICE_MODEL_KEY)
+        additional_info=device_json.get(DEVICE_ADDITIONAL_INFO_KEY)
+        device = Device(mac_addr=device_json.get(DEVICE_MAC_ADDR_KEY).lower(),
+                        manufacturer=device_manufacturer,
+                        model=device_model,
+                        test_pack=device_json.get(DEVICE_TEST_PACK_KEY),
+                        type=device_json.get(DEVICE_TYPE_KEY),
+                        technology=device_json.get(DEVICE_TECH_KEY),
+                        additional_info=additional_info,
+                        device_folder=f"{device_manufacturer} {device_model}",
+                        test_modules=device_json.get(DEVICE_TEST_MODULES_KEY)
+                        )
 
         self._testrun.create_device(device)
         response.status_code = status.HTTP_201_CREATED
@@ -1103,13 +1105,13 @@ async def upload_cert(self, file: UploadFile, response: Response):
         response.status_code = status.HTTP_400_BAD_REQUEST
         return self._generate_msg(
           False,
-          "Failed to upload certificate. Is it in the correct format?")
+          "Failed to upload certificate. The file is corrupted.")
 
     # Return error if something went wrong
     if cert_obj is None:
       response.status_code = 500
       return self._generate_msg(
-          False, "Failed to upload certificate. Is it in the correct format?")
+          False, "Failed to upload certificate. An error occurred.")
 
     response.status_code = status.HTTP_201_CREATED
 

framework/python/src/common/device.py

@@ -38,6 +38,11 @@ class Device():
   device_folder: str = None
   reports: List[TestReport] = field(default_factory=list)
   max_device_reports: int = None
+  created_at: datetime = field(default_factory=datetime.now)
+  modified_at: datetime = field(default_factory=datetime.now)
+
+  # Store the original values to detect changes
+  _initial_values: dict = field(init=False, repr=False, default_factory=dict)
 
   def add_report(self, report):
     self.reports.append(report)
@@ -66,6 +71,8 @@ def to_dict(self):
     device_json['technology'] = self.technology
     device_json['test_pack'] = self.test_pack
     device_json['additional_info'] = self.additional_info
+    device_json['created_at'] = self.created_at.isoformat()
+    device_json['modified_at'] = self.modified_at.isoformat()
 
     if self.firmware is not None:
       device_json['firmware'] = self.firmware
@@ -85,4 +92,20 @@ def to_config_json(self):
     device_json['test_pack'] = self.test_pack
     device_json['test_modules'] = self.test_modules
     device_json['additional_info'] = self.additional_info
+    device_json['created_at'] = self.created_at.isoformat()
+    device_json['modified_at'] = self.modified_at.isoformat()
+
     return device_json
+
+  def __post_init__(self):
+    # Store initial values after creation
+    for f in self.__dataclass_fields__:
+      if f not in ['created_at', 'modified_at', '_initial_values']:
+        self._initial_values[f] = getattr(self, f)
+
+  def __setattr__(self, name: str, value: any) -> None:
+    if (name not in ['created_at', 'modified_at', '_initial_values'] and
+      hasattr(self, name) and getattr(self, name) != value):
+      # Update the last_updated timestamp
+      super().__setattr__('modified_at', datetime.now())
+    super().__setattr__(name, value)

framework/python/src/common/risk_profile.py

@@ -95,8 +95,7 @@ def __init__(self, profile_json=None, profile_format=None):
   # but still validate the profile
   def load(self, profile_json, profile_format):
     self.name = profile_json['name']
-    self.created = datetime.strptime(
-      profile_json['created'], '%Y-%m-%d')
+    self.created = datetime.fromisoformat(profile_json['created'])
     self.version = profile_json['version']
     self.questions = profile_json['questions']
     self.status = None
@@ -329,7 +328,7 @@ def to_json(self, pretty=False):
     json_dict = {
         'name': self.name,
         'version': self.version,
-        'created': self.created.strftime('%Y-%m-%d'),
+        'created': self.created.isoformat(),
         'status': self.status,
         'risk': self.risk,
         'questions': self.questions

framework/python/src/common/testreport.py

@@ -25,6 +25,7 @@
 from jinja2 import Environment, FileSystemLoader, BaseLoader
 from collections import OrderedDict
 from bs4 import BeautifulSoup
+import math
 
 
 DATE_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
@@ -35,6 +36,8 @@
 TEMPLATES_FOLDER = 'report_templates'
 TEST_REPORT_TEMPLATE = 'report_template.html'
 ICON = 'icon.png'
+RESULTS_SPACE_FIRST_PAGE = 440
+RESULTS_SPACE = 800
 
 
 LOGGER = logger.get_logger('REPORT')
@@ -133,12 +136,21 @@ def to_json(self):
 
     test_results = []
     for test in self._results:
+      details = test.details
+      if isinstance(details, str):
+        details = ' '.join(list(filter(lambda s: s != '', details.split('\n'))))
+      if isinstance(details, list):
+        details = [str(d) for d in details]
+        details = ' '.join(details)
+      else:
+        details = str(details)
       test_dict = {
         'name': test.name,
         'description': test.description,
         'expected_behavior': test.expected_behavior,
         'required_result': test.required_result,
-        'result': test.result
+        'result': test.result,
+        'details': details
       }
 
       if test.recommendations is not None and len(test.recommendations) > 0:
@@ -217,6 +229,8 @@ def from_json(self, json_file):
       if 'optional_recommendations' in test_result:
         test_case.optional_recommendations = test_result[
           'optional_recommendations']
+      if 'details' in test_result:
+        test_case.details = test_result['details']
 
       self.add_test(test_case)
 
@@ -284,9 +298,15 @@ def to_html(self):
 
     module_reports = self._module_reports
     env_module = Environment(loader=BaseLoader())
-    pages_num = self._pages_num(json_data)
+    manufacturer_length = len(json_data['device']['manufacturer'])
+    device_name_length = len(json_data['device']['model'])
+    title_length = manufacturer_length + device_name_length + 1
+    results = json_data['tests']['results']
+    results_pages = self._generate_result_pages(title_length, results)
+
     module_templates = [
         env_module.from_string(s).render(
+          title = 'Testrun report',
           name=current_test_pack.name,
           device=json_data['device'],
           logo=logo,
@@ -302,19 +322,31 @@ def to_html(self):
                            json_data=json_data,
                            device=json_data['device'],
                            modules=self._device_modules(json_data['device']),
+                           results_pages = results_pages,
                            test_status=json_data['status'],
                            duration=duration,
                            successful_tests=successful_tests,
                            total_tests=self._total_tests,
                            test_results=json_data['tests']['results'],
                            steps_to_resolve=steps_to_resolve_,
                            module_reports=module_reports,
-                           pages_num=pages_num,
-                           tests_first_page=TESTS_FIRST_PAGE,
                            tests_per_page=TESTS_PER_PAGE,
                            module_templates=module_templates
                            ))
 
+  def _calculate_space_first_page(self, title_length):
+    # Calculation of test results lines at first page
+    # Average chars per line is 25
+    estimated_lines = title_length // 25
+    if title_length % 25 > 0:
+      estimated_lines += 1
+    if estimated_lines > 1:
+      # Line height is 60 px
+      title_px = (estimated_lines - 1) * 60
+      return RESULTS_SPACE_FIRST_PAGE - title_px
+    else:
+      return RESULTS_SPACE_FIRST_PAGE
+
   def _add_page_counter(self, html):
     # Add page nums and total page
     soup = BeautifulSoup(html, features='html5lib')
@@ -324,25 +356,43 @@ def _add_page_counter(self, html):
       div.string = f'Page {index+1}/{total_pages}'
     return str(soup)
 
-  def _pages_num(self, json_data):
-
-    # Calculate pages
-    test_count = len(json_data['tests']['results'])
-
-    # Multiple pages required
-    if test_count > TESTS_FIRST_PAGE:
-      # First page
-      pages = 1
-
-      # Remaining testsgenerate
-      test_count -= TESTS_FIRST_PAGE
-      pages += (int)(test_count / TESTS_PER_PAGE)
-      pages = pages + 1 if test_count % TESTS_PER_PAGE > 0 else pages
-
-    # 1 page required
-    else:
-      pages = 1
-
+  def _calc_details_height(self, text):
+    # Calculate a details line height
+    lines = math.ceil(len(text) / 45)
+    return (lines * 14) + 12
+
+  def _calc_text_line_height(self, text):
+    # Calculate result lines count
+    lines = math.ceil(len(text) / 52)
+    return 40 + 15 * (lines -1)
+
+  def _gen_result_page(self, results, space):
+    # Build results page content
+    page = []
+    page_space = 0
+    while results:
+      result = results[0]
+      line_height = self._calc_text_line_height(result['description'])
+      if line_height > 40:
+        result['height'] = line_height
+      if result['details']:
+        line_height += self._calc_details_height(result['details'])
+      page_space += line_height
+      if page_space > space:
+        break
+      else:
+        page.append(results.pop(0))
+    return page
+
+  def _generate_result_pages(self, title_length,  results):
+    # Generating pages with tests results
+    pages = []
+    pages.append(
+      self._gen_result_page(results,
+                            self._calculate_space_first_page(title_length))
+                )
+    while results:
+      pages.append(self._gen_result_page(results, RESULTS_SPACE))
     return pages
 
   def _device_modules(self, device):

framework/python/src/core/session.py

@@ -453,6 +453,14 @@ def add_test_result(self, result):
         if len(result.description) != 0:
           test_result.description = result.description
 
+        # Add details to test result
+        details = result.details
+        if isinstance(details, str):
+          details = list(filter(lambda s: s!='', details.split('\n')))
+        if isinstance(details, list):
+          details = ' '.join(details)
+        test_result.details = details
+
         # Add recommendations if provided
         if result.recommendations is not None:
           test_result.recommendations = result.recommendations

framework/python/src/test_orc/test_case.py

@@ -28,12 +28,14 @@ class TestCase:  # pylint: disable=too-few-public-methods,too-many-instance-attr
   result: str = TestResult.NON_COMPLIANT
   recommendations: list = field(default_factory=lambda: [])
   optional_recommendations: list = field(default_factory=lambda: [])
+  details: str = ""
 
   def to_dict(self):
 
     test_dict = {
       "name": self.name,
       "description": self.description,
+      "details": self.details,
       "expected_behavior": self.expected_behavior,
       "required_result": self.required_result,
       "result": self.result
@@ -47,3 +49,6 @@ def to_dict(self):
       test_dict["optional_recommendations"] = self.optional_recommendations
 
     return test_dict
+
+  def __post_init__(self):
+    self.details = self.details.replace("\n", "", 1)

framework/python/src/test_orc/test_orchestrator.py

@@ -611,14 +611,18 @@ def _run_test_module(self, module):
           # Convert dict from json into TestCase object
           test_case = TestCase(name=test_result["name"],
                                result=test_result["result"],
-                               description=test_result["description"])
+                               description=test_result["description"]
+                               )
 
           # Add steps to resolve if test is non-compliant
           if (test_case.result == TestResult.NON_COMPLIANT
               and "recommendations" in test_result):
             test_case.recommendations = test_result["recommendations"]
           else:
             test_case.recommendations = []
+          # Add details to the test case if presented
+          if "details" in test_result:
+            test_case.details = test_result["details"]
 
           self.get_session().add_test_result(test_case)
 

framework/requirements.txt

@@ -1,5 +1,5 @@
 # Requirements for the core module
-requests==2.32.3
+requests==2.32.5
 
 # Requirements for the net_orc module
 docker==7.1.0

make/DEBIAN/control

@@ -1,7 +1,7 @@
 Package: Testrun
-Version: 2.2.1
+Version: 2.2.2
 Architecture: amd64
-Maintainer: Google <boddey@google.com>
+Maintainer: Google <ssm-orcas@google.com>
 Homepage: https://github.com/google/testrun
 Bugs: https://github.com/google/testrun/issues
 Description: Automatically verify IoT device network behavior

modules/network/base/python/requirements.txt

@@ -1,10 +1,10 @@
 # Dependencies to user defined packages
 # Package dependencies should always be defined before the user defined
 # packages to prevent auto-upgrades of stable dependencies
-protobuf==5.28.3
+protobuf==6.32.1
 
 # User defined packages
-grpcio==1.67.1
-grpcio-tools==1.67.1
+grpcio==1.75.1
+grpcio-tools==1.75.1
 netifaces==0.11.0
 

modules/network/radius/python/requirements.txt

@@ -6,6 +6,6 @@ greenlet==3.0.3
 six==1.16.0
 
 # User defined packages
-eventlet==0.36.1
+eventlet==0.40.3
 pbr==6.1.0
 transitions==0.9.2

modules/test/base/python/requirements.txt

@@ -1,11 +1,11 @@
 # Dependencies to user defined packages
 # Package dependencies should always be defined before the user defined
 # packages to prevent auto-upgrades of stable dependencies
-protobuf==5.28.0
+protobuf==6.32.1
 
 # User defined packages
-grpcio==1.67.1
-grpcio-tools==1.67.1
+grpcio==1.75.1
+grpcio-tools==1.75.1
 netifaces==0.11.0
 
 # Requirements for reports generation