diff --git a/framework/python/src/common/statuses.py b/framework/python/src/common/statuses.py index e4bb629ac..568e1416d 100644 --- a/framework/python/src/common/statuses.py +++ b/framework/python/src/common/statuses.py @@ -15,6 +15,7 @@ class TestrunStatus: + """Statuses for overall testing""" IDLE = "Idle" WAITING_FOR_DEVICE = "Waiting for Device" MONITORING = "Monitoring" @@ -27,6 +28,7 @@ class TestrunStatus: class TestResult: + """Statuses for test results""" IN_PROGRESS = "In Progress" COMPLIANT = "Compliant" NON_COMPLIANT = "Non-Compliant" diff --git a/framework/python/src/common/testreport.py b/framework/python/src/common/testreport.py index befd463b7..d572fdb61 100644 --- a/framework/python/src/common/testreport.py +++ b/framework/python/src/common/testreport.py @@ -248,12 +248,13 @@ def to_html(self): module_reports = self._get_module_pages() pages_num = self._pages_num(json_data) total_pages = pages_num + len(module_reports) + 1 - if len(steps_to_resolve) > 0: - total_pages += 1 - if (len(optional_steps_to_resolve) > 0 + is_pilot = json_data['device']['test_pack'] == 'Pilot Assessment' + if is_pilot and (len(optional_steps_to_resolve) > 0 and json_data['device']['test_pack'] == 'Pilot Assessment' ): total_pages += len(optional_steps_to_resolve) + elif len(steps_to_resolve) > 0: + total_pages += len(steps_to_resolve) return template.render(styles=styles, logo=logo, @@ -275,6 +276,7 @@ def to_html(self): total_pages=total_pages, tests_first_page=TESTS_FIRST_PAGE, tests_per_page=TESTS_PER_PAGE, + is_pilot = is_pilot, ) def _pages_num(self, json_data): @@ -324,7 +326,8 @@ def _get_steps_to_resolve(self, json_data): if 'recommendations' in test: tests_with_recommendations.append(test) - return tests_with_recommendations + return self._split_steps_to_resolve_to_pages( + tests_with_recommendations, 4, 4) def _get_optional_steps_to_resolve(self, json_data): tests_with_recommendations = [] @@ -334,20 +337,21 @@ def _get_optional_steps_to_resolve(self, json_data): if 'optional_recommendations' in test: tests_with_recommendations.append(test) - return self._split_steps_to_resolve_to_pages(tests_with_recommendations) + return self._split_steps_to_resolve_to_pages( + tests_with_recommendations, 3, 4) - def _split_steps_to_resolve_to_pages(self, steps): + def _split_steps_to_resolve_to_pages(self, steps, start_page=4, page=4): # Split steps to resolve to pages. - # First page 3 steps, 4 steps on other pages. - if len(steps) < 3: + # First steps, steps on other pages. + if len(steps) < start_page: return [steps] splitted = [steps[:3]] - index = 3 + index = start_page while index < len(steps): - splitted.append(steps[index:index + 4]) - index += 4 + splitted.append(steps[index:index + page]) + index += page return splitted diff --git a/resources/report/test_report_styles.css b/resources/report/test_report_styles.css index b1ed9d33c..f0f8d45e1 100644 --- a/resources/report/test_report_styles.css +++ b/resources/report/test_report_styles.css @@ -24,6 +24,13 @@ unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; } + @font-face { + font-family: 'Material Symbols Outlined'; + font-style: normal; + font-weight: 400; + src: url(https://fonts.gstatic.com/icon/font?kit=kJF1BvYX7BgnkSrUwT8OhrdQw4oELdPIeeII9v6oDMzByHX9rA6RzaxHMPdY43zj-jCxv3fzvRNU22ZXGJpEpjC_1v-p_4MrImHCIJIZrDCvHOejHdIa31RJq7xr9O779sC3DdMx&skey=b8dc2088854b122f&v=v222) format('woff2'); + } + /* Define some common body formatting*/ body { font-family: 'Google Sans', sans-serif; @@ -415,6 +422,10 @@ border-top: 0; } + .result-line-result-non-compliant-required { + background: #FCE8E6; + } + .result-list-header-label { position: absolute; font-size: 12px; @@ -467,7 +478,7 @@ margin-top: 8px; padding: 4px 4px 7px 5px; border-radius: 2px; - left: 6.85in; + left: 5.4in; } .result-test-result-compliant { @@ -480,6 +491,71 @@ color: #393939; } + .material-symbols-outlined { + font-family: 'Material Symbols Outlined'; + font-weight: normal; + font-style: normal; + line-height: 1; + letter-spacing: normal; + text-transform: none; + white-space: nowrap; + word-wrap: normal; + direction: ltr; + text-align: center; + font-size: 12px; + display: inline-flex; + width: 14px; + height: 14px; + line-height: 14px; + flex-direction: column; + justify-content: center; + align-items: center; + flex-shrink: 0; + border-radius: 100%; + -webkit-font-feature-settings: 'liga'; + -webkit-font-smoothing: antialiased; + vertical-align: bottom; + } + + .result-test-required-result { + display: flex; + align-items: center; + overflow: visible; + } + + .result-test-required-result-text { + line-height: 14px; + flex: 0 0 120px; + padding-left: 8px; + } + + .result-test-required-result-informational{ + color: #0D652D; + } + + .result-test-required-result-informational .material-symbols-outlined { + background: #0D652D; + color: #ffffff; + } + + .result-test-required-result-required { + color: #000000; + } + + .result-test-required-result-required .material-symbols-outlined { + background: #000000; + color: #ffffff; + } + + .result-test-required-result-required-if-applicable { + color: #174EA6; + } + + .result-test-required-result-required-if-applicable .material-symbols-outlined { + background: #174EA6; + color: #ffffff; + } + /* CSS for the footer */ .footer { position: absolute; @@ -639,4 +715,4 @@ width: 8.5in; height: 11in; } - } \ No newline at end of file + } diff --git a/resources/report/test_report_template.html b/resources/report/test_report_template.html index 4f7c6c264..19765f894 100644 --- a/resources/report/test_report_template.html +++ b/resources/report/test_report_template.html @@ -11,6 +11,7 @@ {% set page_index = namespace(value=0) %} + {% set step_index = namespace(value=0) %} {% set opt_step_index = namespace(value=0) %} {# Test Results #} {% for page in range(pages_num) %} @@ -56,7 +57,7 @@

Device Configuration

{% endif %}
Test Status
Complete
- {% if json_data['device']['test_pack'] == 'Pilot Assessment' %} + {% if is_pilot %}
Pilot Recommendation
{% else %}
Test Result
@@ -84,27 +85,54 @@

Device Configuration

Results List ({{ successful_tests }}/{{ total_tests }})

Name
-
Description
-
Result
+
Description
+
Result
+
Required result
{% for i in range(results_from, results_to) %} + {% if test_results[i]['result'] == 'Non-Compliant' and test_results[i]['required_result'] == "Required" %} +
+ {% else %}
+ {% endif %}
{{ test_results[i]['name'] }}
-
{{ test_results[i]['description'] }}
+
{{ test_results[i]['description'] }}
+
+ result-test-result-non-compliant"> {% elif test_results[i]['result'] == 'Compliant' %} -
+ result-test-result-compliant"> {% elif test_results[i]['result'] == 'Error' %} -
+ result-test-result-error"> {% elif test_results[i]['result'] == 'Feature Not Detected' %} -
+ result-test-result-feature-not-detected"> {% elif test_results[i]['result'] == 'Informational' %} -
+ result-test-result-informational"> {% else %} -
+ result-test-result-skipped"> {% endif %} {{ test_results[i]['result'] }}
+ {# Required resul badges #} + {% if test_results[i]['required_result'] == "Required" %} +
+ asterisk + {{ test_results[i]['required_result'] }} +
+ {% elif test_results[i]['required_result'] == "Required if Applicable" %} +
+ asterisk + {{ test_results[i]['required_result'] }} +
+ {% elif test_results[i]['required_result'] == "Informational" %} +
+ info_i + {{ test_results[i]['required_result'] }} +
+ {% else %} +
+ {{ test_results[i]['required_result'] }} +
+ {% endif %}
{% endfor %}
@@ -116,25 +144,29 @@

Results List ({{ successful_tests }}/{{ tot
{% endfor %} {# Steps to resolve Device qualification #} - {% if steps_to_resolve|length > 0 and json_data['device']['test_pack'] == 'Device Qualification' %} - {% set page_index.value = page_index.value+1 %} + {% if steps_to_resolve|length > 0 and not is_pilot %} + {% for step in steps_to_resolve%} + {% set page_index.value = page_index.value + 1 %}
{{ header_macros.header(False, "Testrun report", json_data, device, logo, icon_qualification, icon_pilot)}} -

Non-compliant tests and suggested steps to resolve

- {% for step in steps_to_resolve %} + {% if loop.first %} +

Non-compliant tests and suggested steps to resolve

+ {% endif %} + {% for line in step %} + {% set step_index.value = step_index.value + 1 %}
- {{ loop.index }}. + {{ step_index.value }}.
- Name
{{ step['name'] }} + Name
{{ line['name'] }}
- Description
{{ step["description"] }} + Description
{{ line["description"] }}
Steps to resolve - {% for recommedtation in step['recommendations'] %} + {% for recommedtation in line['recommendations'] %}
{{ loop.index }}. {{ recommedtation }} {% endfor %}
@@ -145,11 +177,12 @@

Non-compliant tests and suggested steps to resolve

Page {{ page_index.value }}/{{ total_pages }}
+ {% endfor %} {% endif %} {# Pilot steps to resolve#} - {% if json_data['device']['test_pack'] == 'Pilot Assessment' and optional_steps_to_resolve|length > 0 %} - {% for step in optional_steps_to_resolve%} - {% set page_index.value = page_index.value + 1 %} + {% if is_pilot and optional_steps_to_resolve|length > 0 %} + {% for step in optional_steps_to_resolve%} + {% set page_index.value = page_index.value + 1 %}
{{ header_macros.header(False, "Testrun report", json_data, device, logo, icon_qualification, icon_pilot)}} {% if loop.first %} diff --git a/testing/api/reports/report.json b/testing/api/reports/report.json index bd697654d..df7ac0d50 100644 --- a/testing/api/reports/report.json +++ b/testing/api/reports/report.json @@ -1,16 +1,16 @@ { "testrun": { - "version": "1.3.1" + "version": "2.1" }, "mac_addr": null, "device": { "mac_addr": "00:1e:42:35:73:c4", "manufacturer": "Teltonika", "model": "TRB140", - "firmware": "1.2.3", + "firmware": "1", "test_modules": { "protocol": { - "enabled": true + "enabled": false }, "services": { "enabled": false @@ -22,38 +22,51 @@ "enabled": true }, "ntp": { - "enabled": true + "enabled": false }, "dns": { - "enabled": true + "enabled": false } - } - }, - "status": "Non-Compliant", - "started": "2024-08-05 13:37:53", - "finished": "2024-08-05 13:39:35", - "tests": { - "total": 12, - "results": [ + }, + "test_pack": "Device Qualification", + "device_profile": [ { - "name": "protocol.valid_bacnet", - "description": "BACnet device could not be discovered", - "expected_behavior": "BACnet traffic can be seen on the network and packets are valid and not malformed", - "required_result": "Recommended", - "result": "Feature Not Detected" + "question": "What type of device is this?", + "answer": "Building Automation Gateway" }, { - "name": "protocol.bacnet.version", - "description": "Device did not respond to BACnet discovery", - "expected_behavior": "The BACnet client implements an up to date version of BACnet", - "required_result": "Recommended", - "result": "Feature Not Detected" + "question": "Please select the technology this device falls into", + "answer": "Hardware - Access Control" }, { - "name": "protocol.valid_modbus", - "description": "Device did not respond to Modbus connection", - "expected_behavior": "Any Modbus functionality works as expected and valid Modbus traffic can be observed", - "required_result": "Recommended", + "question": "Does your device process any sensitive information? ", + "answer": "No" + }, + { + "question": "Can all non-essential services be disabled on your device?", + "answer": "Yes" + }, + { + "question": "Is there a second IP port on the device?", + "answer": "Yes" + }, + { + "question": "Can the second IP port on your device be disabled?", + "answer": "No" + } + ] + }, + "status": "Non-Compliant", + "started": "2024-12-10 16:06:42", + "finished": "2024-12-10 16:08:12", + "tests": { + "total": 5, + "results": [ + { + "name": "security.tls.v1_0_client", + "description": "No outbound connections were found", + "expected_behavior": "The packet indicates a TLS connection with at least TLS 1.0 and support", + "required_result": "Informational", "result": "Feature Not Detected" }, { @@ -70,65 +83,31 @@ }, { "name": "security.tls.v1_2_client", - "description": "TLS 1.2 client connections valid", + "description": "An error occured whilst running this test", "expected_behavior": "The packet indicates a TLS connection with at least TLS 1.2 and support for ECDH and ECDSA ciphers", "required_result": "Required if Applicable", - "result": "Compliant" + "result": "Error" }, { "name": "security.tls.v1_3_server", "description": "TLS 1.3 certificate is invalid", "expected_behavior": "TLS 1.3 certificate is issued to the web browser client when accessed", "required_result": "Informational", - "result": "Informational" + "result": "Informational", + "optional_recommendations": [ + "Enable TLS 1.3 support in the web server configuration", + "Disable TLS 1.0 and 1.1", + "Sign the certificate used by the web server" + ] }, { "name": "security.tls.v1_3_client", - "description": "TLS 1.3 client connections valid", + "description": "An error occured whilst running this test", "expected_behavior": "The packet indicates a TLS connection with at least TLS 1.3", "required_result": "Informational", - "result": "Informational" - }, - { - "name": "ntp.network.ntp_support", - "description": "Device sent NTPv3 packets. NTPv3 is not allowed", - "expected_behavior": "The device sends an NTPv4 request to the configured NTP server.", - "required_result": "Required", - "result": "Non-Compliant", - "recommendations": [ - "Set the NTP version to v4 in the NTP client", - "Install an NTP client that supports NTPv4" - ] - }, - { - "name": "ntp.network.ntp_dhcp", - "description": "Device sent NTP request to non-DHCP provided server", - "expected_behavior": "Device can accept NTP server address, provided by the DHCP server (DHCP OFFER PACKET)", - "required_result": "Roadmap", - "result": "Feature Not Detected" - }, - { - "name": "dns.network.hostname_resolution", - "description": "DNS traffic detected from device", - "expected_behavior": "The device sends DNS requests.", - "required_result": "Required", - "result": "Compliant" - }, - { - "name": "dns.network.from_dhcp", - "description": "DNS traffic detected only to DHCP provided server", - "expected_behavior": "The device sends DNS requests to the DNS server provided by the DHCP server", - "required_result": "Informational", - "result": "Informational" - }, - { - "name": "dns.mdns", - "description": "No MDNS traffic detected from the device", - "expected_behavior": "Device may send MDNS requests", - "required_result": "Informational", - "result": "Informational" + "result": "Error" } ] }, - "report": "http://localhost:8000/report/Teltonika TRB140/2024-08-05T13:37:53" + "report": "http://localhost:8000/report/Teltonika TRB140/2024-12-10T16:06:42" } \ No newline at end of file diff --git a/testing/api/reports/report.pdf b/testing/api/reports/report.pdf index 0e449f196..1a3dc27f9 100644 Binary files a/testing/api/reports/report.pdf and b/testing/api/reports/report.pdf differ