Updated tester schema generation to use msgspec data types

.gitignore

@@ -8,6 +8,7 @@ __pycache__
 .eggs
 venv
 venv2
+.venv
 build
 docker-compose.override.yml
 /workspace

Changelog.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented here.
 - Improved robustness of tester installation scripts and Docker configuration (#688)
 - Moved tidyverse installation steps from server Dockerfile into R tester requirements.system (#688)
 - Fixed Haskell tester installation using ghcup to install stack system-wide (#688)
+- Updated tester schema generation to use msgspec datatypes (#689)
 
 ## [v2.9.0]
 - Install stack with GHCup (#626)
@@ -45,13 +46,13 @@ All notable changes to this project will be documented here.
 - Update R tester to allow a renv.lock file (#581)
 - Improve display of Python package installation errors when creating environment (#585)
 - Update "setting up test environment" message with http response of status code 503 (#589)
-- Change rlimit resource settings to apply each worker individually (#587) 
+- Change rlimit resource settings to apply each worker individually (#587)
 - Drop support for Python 3.8 (#590)
 - Use Python 3.13 in development (#590)
 - Update Docker configuration to install dependencies in a separate service (#590)
 - Improve error reporting with handled assertion errors (#591)
 - Add custom pytest markers to Python tester to record MarkUs metadata (#592)
-- Stop the autotester from running tests if there are errors in test settings (#593) 
+- Stop the autotester from running tests if there are errors in test settings (#593)
 - Implement Redis backoff strategy (#594)
 
 ## [v2.6.0]
@@ -125,7 +126,7 @@ All notable changes to this project will be documented here.
 - Add ability to clean up test scripts that haven't been used for X days (#379)
 
 ## [v2.1.2]
-- Support dependencies on specific package versions and non-CRAN sources for R tester (#323) 
+- Support dependencies on specific package versions and non-CRAN sources for R tester (#323)
 
 ## [v2.1.1]
 - Remove the requirement for clients to send unique user name (#318)
@@ -146,7 +147,7 @@ All notable changes to this project will be documented here.
 - Add Jupyter tester (#284)
 
 ## [v1.10.3]
-- Fix bug where zip archive was unpacked two levels deep instead of just one (#271) 
+- Fix bug where zip archive was unpacked two levels deep instead of just one (#271)
 - Pass PGHOST and PGINFO environment variables to tests (#272)
 - Update to new version of markus-api that supports uploading binary files (#273)
 - Fix bug where environment variables were not string types (#274)

client/autotest_client/form_management.py

@@ -135,6 +135,5 @@ def validate_against_schema(test_specs: Dict, schema: Dict, filenames: List[str]
     schema["definitions"]["files_list"]["enum"] = filenames
     # don't validate based on categories
     schema["definitions"]["test_data_categories"].pop("enum")
-    schema["definitions"]["test_data_categories"].pop("enumNames")
     error = _validate_with_defaults(schema, test_specs, best_only=True)
     return str(error) if error else None

server/autotest_server/schema_skeleton.json

@@ -6,8 +6,7 @@
     },
     "test_data_categories": {
       "type": "string",
-      "enum": [],
-      "enumNames": []
+      "enum": []
     },
     "extra_group_data": {},
     "installed_testers": {

server/autotest_server/testers/__init__.py

@@ -1,12 +1,13 @@
+from __future__ import annotations
+
+import importlib
 import os
 
 _TESTERS = ("ai", "custom", "haskell", "java", "jupyter", "py", "pyta", "r", "racket")
 
 
-def install(testers=_TESTERS):
-    import importlib
-
-    settings = {}
+def install(testers: list[str] = _TESTERS) -> tuple[dict, dict]:
+    installed_testers = []
     for tester in testers:
         mod = importlib.import_module(f".{tester}.setup", package="autotest_server.testers")
         try:
@@ -20,5 +21,25 @@ def install(testers=_TESTERS):
                 " and then rerunning this function."
             )
             raise Exception(msg) from e
-        settings[tester] = mod.settings()
-    return settings
+        installed_testers.append(tester)
+    return get_settings(installed_testers)
+
+
+def get_settings(testers: list[str] = _TESTERS) -> tuple[dict, dict]:
+    """Return JSON schemas for the settings for the given testers.
+
+    The return values are:
+    1. A dictionary mapping tester name to JSON schema
+    2. A dictionary of JSON schema definitions used by the tester schemas
+    """
+    schemas = {}
+    definitions = {}
+    for tester in testers:
+        mod = importlib.import_module(f".{tester}.setup", package="autotest_server.testers")
+        tester_schema, tester_definitions = mod.settings()
+        if "title" in tester_schema and f"{tester_schema['title']}TesterSettings" in tester_definitions:
+            tester_definitions.pop(f"{tester_schema['title']}TesterSettings")
+        schemas[tester] = tester_schema
+        definitions.update(tester_definitions)
+
+    return schemas, definitions

server/autotest_server/testers/ai/setup.py

@@ -48,7 +48,7 @@ def create_environment(settings_, env_dir, _default_env_dir):
 def settings():
     with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f:
         settings_ = json.load(f)
-    return settings_
+    return settings_, {}
 
 
 def install():

server/autotest_server/testers/custom/schema.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+from typing import Annotated
+
+from msgspec import Meta
+from markus_autotesting_core.types import AutotestFile, BaseTestData, BaseTesterSettings
+
+
+class CustomTesterSettings(BaseTesterSettings, tag="custom"):
+    """The settings for the custom tester."""
+
+    test_data: Annotated[list[CustomTestData], Meta(title="Test Groups", min_length=1)]
+
+
+class CustomTestData(BaseTestData, kw_only=True):
+    """The `test_data` specification for the custom tester."""
+
+    script_files: Annotated[
+        list[AutotestFile],
+        Meta(title="Test files", min_length=1, extra_json_schema={"uniqueItems": True}),
+    ]
+    """The file(s) that contain the tests to execute."""

server/autotest_server/testers/custom/setup.py

@@ -1,5 +1,7 @@
 import os
-import json
+
+from ..schema import generate_schema
+from .schema import CustomTesterSettings
 
 
 def create_environment(_settings, _env_dir, default_env_dir):
@@ -11,5 +13,4 @@ def install():
 
 
 def settings():
-    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f:
-        return json.load(f)
+    return generate_schema(CustomTesterSettings)

server/autotest_server/testers/haskell/config.py

@@ -0,0 +1,2 @@
+HASKELL_TEST_DEPS = ["tasty-discover", "tasty-quickcheck", "tasty-hunit"]
+STACK_RESOLVER = "lts-21.21"

server/autotest_server/testers/haskell/requirements.system

@@ -2,7 +2,7 @@
 set -euxo pipefail
 
 # Install a system-wide ghc, which can be used as a default version in the Haskell tester.
-# This should be synchronized with the LTS version and dependencies in setup.py
+# This should be synchronized with the LTS version and dependencies in config.py
 if ! dpkg -l ghc cabal-install &> /dev/null; then
   apt-get -y update
   DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install

server/autotest_server/testers/haskell/schema.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+from typing import Annotated
+
+from msgspec import Meta, Struct
+from markus_autotesting_core.types import BaseTestData, BaseTesterSettings
+
+from .config import STACK_RESOLVER
+
+
+class HaskellTesterSettings(BaseTesterSettings):
+    """The settings for the Haskell tester."""
+
+    env_data: Annotated[HaskellEnvData, Meta(title="Haskell environment")]
+    test_data: Annotated[list[HaskellTestData], Meta(title="Test Groups", min_length=1)]
+
+
+class HaskellTestData(BaseTestData, kw_only=True):
+    """The `test_data` specification for the Haskell tester."""
+
+    test_timeout: Annotated[int, Meta(title="Per-test timeout")] = 10
+    test_cases: Annotated[int, Meta(title="Number of test cases (QuickCheck)")] = 100
+
+
+class HaskellEnvData(Struct, kw_only=True):
+    """Settings for the Haskell environment"""
+
+    resolver_version: Annotated[str, Meta(title="Stackage LTS resolver version")] = STACK_RESOLVER

server/autotest_server/testers/haskell/setup.py

@@ -1,9 +1,9 @@
 import os
-import json
 import subprocess
 
-HASKELL_TEST_DEPS = ["tasty-discover", "tasty-quickcheck", "tasty-hunit"]
-STACK_RESOLVER = "lts-21.21"
+from .config import HASKELL_TEST_DEPS, STACK_RESOLVER
+from ..schema import generate_schema
+from .schema import HaskellTesterSettings
 
 
 def create_environment(_settings, _env_dir, default_env_dir):
@@ -51,8 +51,4 @@ def install():
 
 
 def settings():
-    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f:
-        settings_ = json.load(f)
-    resolver_versions = settings_["properties"]["env_data"]["properties"]["resolver_version"]
-    resolver_versions["default"] = STACK_RESOLVER
-    return settings_
+    return generate_schema(HaskellTesterSettings)

server/autotest_server/testers/java/schema.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+from typing import Annotated
+
+from msgspec import Meta
+from markus_autotesting_core.types import BaseTestData, BaseTesterSettings
+
+
+class JavaTesterSettings(BaseTesterSettings):
+    """The settings for the Java tester."""
+
+    test_data: Annotated[list[JavaTestData], Meta(title="Test Groups", min_length=1)]
+
+
+class JavaTestData(BaseTestData, kw_only=True):
+    """The `test_data` specification for the Java tester."""
+
+    classpath: Annotated[str, Meta(title="Java Class Path")] = "."
+    sources_path: Annotated[str, Meta(title="Java Sources (glob)")] = ""