chore: formalize and test conformance test file naming

.github/workflows/conformance_suite_validation.yml

@@ -0,0 +1,27 @@
+name: Conformance Test Suite Validation
+
+on:
+  push:
+    branches: [ mainline ]
+    paths:
+      - conformance-tests/**
+  pull_request:
+    branches: [ mainline ]
+    paths:
+      - conformance-tests/**
+  workflow_dispatch:
+
+jobs:
+  validate-naming:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Setup Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.12'
+
+    - name: Validate conformance test suite
+      working-directory: conformance-tests
+      run: python test_conformance_suite.py

conformance-tests/README.md

@@ -40,24 +40,33 @@ conformance-tests/
         └── jobs/
 ```
 
-### Naming Convention
+### File Naming Rules
 
-Filenames encode the spec section they test:
+Test filenames follow strict conventions that encode metadata used by test runners.
 
 ```
-<spec-section>--<description>[.invalid][.suffix].yaml
+[<doc>-][<section>]--<description>[.invalid][.test].<ext>
 ```
 
-- `<spec-section>` - Reference to the [Template Schema](../wiki/2023-09-Template-Schemas.md) section (e.g., `1.1`, `3.3.2`, `7.3`)
-- `.invalid` - Test should FAIL validation/execution
-- `.test` - Job execution test (in `jobs/` directory)
+| Component | Required | Description |
+|---|---|---|
+| `<doc>` | No | Document prefix, required for extension specs. Omitted for the base [Template Schema](../wiki/2023-09-Template-Schemas.md) (a leading number implies it). |
+| `<section>` | No | Section reference within the document (e.g., `1.1`, `3.4.1.5`). Omitted when the test isn't tied to a specific numbered section. |
+| `--` | Yes | Double-hyphen separator before the description. |
+| `<description>` | Yes | Lowercase kebab-case description of what is being tested. |
+| `.invalid` | No | Present when the test should FAIL validation or execution. |
+| `.test` | No | Present for job execution tests (files in `jobs/` directories). |
+| `<ext>` | Yes | `.yaml` or `.json`. |
+
+When both `.invalid` and `.test` are present, `.invalid` comes first: `1.1--desc.invalid.test.yaml`
+
+Each extension spec should use a short, consistent prefix (e.g., `chunk` for TASK_CHUNKING, `redact` for REDACTED_ENV_VARS). The base [Template Schema](../wiki/2023-09-Template-Schemas.md) needs no prefix — a leading number implies it.
 
 Examples:
-- `1.1--minimal-job-template.yaml` - Section 1.1 (Job Template root)
-- `3.3.2--allof.yaml` - Section 3.3.2 (AttributeRequirement)
-- `5--cancelation-notify-then-terminate.yaml` - Section 5 (Action)
-- `2.1--missing-name.invalid.yaml` - Invalid test for Section 2.1
-- `contiguous-even.test.yaml` - TASK_CHUNKING extension execution test (in `TASK_CHUNKING/jobs/`)
+- `1.1--minimal-job-template.yaml` — base spec section 1.1, valid template
+- `7.3--nested-braces.invalid.test.yaml` — base spec section 7.3, job execution test that should fail
+- `chunk-3.4.1.5--noncontiguous.yaml` — TASK_CHUNKING section 3.4.1.5, valid template
+- `redact--mixed-env-vars.test.yaml` — REDACTED_ENV_VARS, no specific section, job execution test
 
 ### Extension Tests
 

conformance-tests/test_conformance_suite.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Validates that all conformance test filenames follow the naming convention documented in README.md.
+
+The naming scheme is:
+    [<doc>-][<section>]--<description>[.invalid][.test].<ext>
+
+- Base spec tests (under base/): no doc prefix, section required (leading number).
+- Extension tests: doc prefix required (chunk, redact, etc.), section optional.
+- If an extension test is testing a base spec rule, it uses base spec naming (leading number, no doc prefix).
+"""
+
+import re
+import sys
+from pathlib import Path
+
+CONFORMANCE_DIR = Path(__file__).parent
+
+# Map extension directory names to their doc prefix
+EXTENSION_DOC_PREFIX = {
+    "TASK_CHUNKING": "chunk",
+    "REDACTED_ENV_VARS": "redact",
+}
+
+# Base spec pattern: starts with a digit (section number)
+#   <section>--<description>[.invalid][.test].<ext>
+BASE_RE = re.compile(
+    r"^(?P<section>\d+(?:\.\d+)*)--(?P<desc>[a-z0-9]+(?:-[a-z0-9]+)*)(?P<invalid>\.invalid)?(?P<test>\.test)?\.(?P<ext>yaml|json)$"
+)
+
+# Extension pattern: starts with a doc prefix
+#   <doc>[-<section>]--<description>[.invalid][.test].<ext>
+def extension_re(prefix: str) -> re.Pattern:
+    return re.compile(
+        rf"^{re.escape(prefix)}(?:-(?P<section>\d+(?:\.\d+)*))?--(?P<desc>[a-z0-9]+(?:-[a-z0-9]+)*)(?P<invalid>\.invalid)?(?P<test>\.test)?\.(?P<ext>yaml|json)$"
+    )
+
+
+def validate_filename(name: str, component: str, test_type: str) -> list[str]:
+    """Returns a list of error strings (empty if valid)."""
+    errors = []
+
+    if component == "base":
+        m = BASE_RE.match(name)
+        if not m:
+            errors.append(f"does not match base naming pattern: [<section>]--<description>[.invalid][.test].<ext>")
+            return errors
+    else:
+        prefix = EXTENSION_DOC_PREFIX.get(component)
+        if prefix is None:
+            errors.append(f"unknown extension directory '{component}', add it to EXTENSION_DOC_PREFIX")
+            return errors
+        ext_pattern = extension_re(prefix)
+        m = BASE_RE.match(name) or ext_pattern.match(name)
+        if not m:
+            errors.append(
+                f"does not match naming pattern: "
+                f"<section>--<desc>... (base spec rule) or "
+                f"{prefix}[-<section>]--<desc>... (extension)"
+            )
+            return errors
+
+    if test_type == "jobs" and not m.group("test"):
+        errors.append("files in jobs/ must have .test suffix")
+    if test_type in ("job_templates", "env_templates") and m.group("test"):
+        errors.append(f"files in {test_type}/ must not have .test suffix")
+
+    return errors
+
+
+def main() -> int:
+    failures = []
+
+    for spec_dir in sorted(CONFORMANCE_DIR.iterdir()):
+        if not spec_dir.is_dir() or not spec_dir.name[0].isdigit():
+            continue
+        for component_dir in sorted(spec_dir.iterdir()):
+            if not component_dir.is_dir():
+                continue
+            component = component_dir.name
+            for test_type in ("job_templates", "env_templates", "jobs"):
+                type_dir = component_dir / test_type
+                if not type_dir.is_dir():
+                    continue
+                for f in sorted(type_dir.iterdir()):
+                    if not f.is_file():
+                        continue
+                    errs = validate_filename(f.name, component, test_type)
+                    for e in errs:
+                        failures.append(f"{f.relative_to(CONFORMANCE_DIR)}: {e}")
+
+    if failures:
+        print(f"FAIL: {len(failures)} file(s) with invalid names:\n")
+        for msg in failures:
+            print(f"  {msg}")
+        return 1
+
+    print("OK: all conformance test filenames are valid")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())