From 0531b1b633ea7dc5bf0379fb4088ae6a1000ba36 Mon Sep 17 00:00:00 2001 From: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> Date: Wed, 18 Feb 2026 15:39:23 -0800 Subject: [PATCH] chore: formalize and test conformance test file naming Signed-off-by: Stephen Crowe <6042774+crowecawcaw@users.noreply.github.com> --- .../conformance_suite_validation.yml | 27 +++++ ... redact--fixed-length-redaction.test.yaml} | 0 ...ct--future-occurrences-redacted.test.yaml} | 0 ...> redact--initial-line-redacted.test.yaml} | 0 ....yaml => redact--mixed-env-vars.test.yaml} | 0 ...aml => redact--multiple-secrets.test.yaml} | 0 ...ml => redact--secret-not-logged.test.yaml} | 0 ... redact--unset-takes-precedence.test.yaml} | 0 ...ml => redact--without-extension.test.yaml} | 0 ... => 1.1.2--missing-extension.invalid.yaml} | 0 ...1.5--default-task-count-zero.invalid.yaml} | 0 ....5--invalid-range-constraint.invalid.yaml} | 0 ...hunk-3.4.1.5--missing-chunks.invalid.yaml} | 0 ...--missing-default-task-count.invalid.yaml} | 0 ....5--missing-range-constraint.invalid.yaml} | 0 ...4.1.5--multiple-chunk-params.invalid.yaml} | 0 ...1.5--negative-target-runtime.invalid.yaml} | 0 ...yaml => chunk-3.4.1.5--noncontiguous.yaml} | 0 ...aml => chunk-3.4.1.5--target-runtime.yaml} | 0 ...4.3--associative-combination.invalid.yaml} | 0 ... chunk-3.4.1.5--contiguous-even.test.yaml} | 0 ...hunk-3.4.1.5--contiguous-format.test.yaml} | 0 ...hunk-3.4.1.5--contiguous-uneven.test.yaml} | 0 ...hunk-3.4.1.5--larger-than-range.test.yaml} | 0 ...k-3.4.1.5--noncontiguous-format.test.yaml} | 0 ...=> chunk-3.4.1.5--noncontiguous.test.yaml} | 0 ...nk-3.4.1.5--with-target-runtime.test.yaml} | 0 conformance-tests/README.md | 31 ++++-- conformance-tests/test_conformance_suite.py | 102 ++++++++++++++++++ 29 files changed, 149 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/conformance_suite_validation.yml rename conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/{fixed-length-redaction.test.yaml => redact--fixed-length-redaction.test.yaml} (100%) rename conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/{future-occurrences-redacted.test.yaml => redact--future-occurrences-redacted.test.yaml} (100%) rename conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/{initial-line-redacted.test.yaml => redact--initial-line-redacted.test.yaml} (100%) rename conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/{redaction-mixed-env-vars.test.yaml => redact--mixed-env-vars.test.yaml} (100%) rename conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/{redaction-multiple-secrets.test.yaml => redact--multiple-secrets.test.yaml} (100%) rename conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/{redaction-secret-not-logged.test.yaml => redact--secret-not-logged.test.yaml} (100%) rename conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/{unset-takes-precedence.test.yaml => redact--unset-takes-precedence.test.yaml} (100%) rename conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/{redaction-without-extension.test.yaml => redact--without-extension.test.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{missing-extension.invalid.yaml => 1.1.2--missing-extension.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{default-task-count-zero.invalid.yaml => chunk-3.4.1.5--default-task-count-zero.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{invalid-range-constraint.invalid.yaml => chunk-3.4.1.5--invalid-range-constraint.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{missing-chunks.invalid.yaml => chunk-3.4.1.5--missing-chunks.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{missing-default-task-count.invalid.yaml => chunk-3.4.1.5--missing-default-task-count.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{missing-range-constraint.invalid.yaml => chunk-3.4.1.5--missing-range-constraint.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{multiple-chunk-params.invalid.yaml => chunk-3.4.1.5--multiple-chunk-params.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{negative-target-runtime.invalid.yaml => chunk-3.4.1.5--negative-target-runtime.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{noncontiguous.yaml => chunk-3.4.1.5--noncontiguous.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{target-runtime.yaml => chunk-3.4.1.5--target-runtime.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/job_templates/{associative-combination.invalid.yaml => chunk-3.4.3--associative-combination.invalid.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/jobs/{contiguous-even.test.yaml => chunk-3.4.1.5--contiguous-even.test.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/jobs/{contiguous-format.test.yaml => chunk-3.4.1.5--contiguous-format.test.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/jobs/{contiguous-uneven.test.yaml => chunk-3.4.1.5--contiguous-uneven.test.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/jobs/{larger-than-range.test.yaml => chunk-3.4.1.5--larger-than-range.test.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/jobs/{noncontiguous-format.test.yaml => chunk-3.4.1.5--noncontiguous-format.test.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/jobs/{noncontiguous.test.yaml => chunk-3.4.1.5--noncontiguous.test.yaml} (100%) rename conformance-tests/2023-09/TASK_CHUNKING/jobs/{with-target-runtime.test.yaml => chunk-3.4.1.5--with-target-runtime.test.yaml} (100%) create mode 100644 conformance-tests/test_conformance_suite.py diff --git a/.github/workflows/conformance_suite_validation.yml b/.github/workflows/conformance_suite_validation.yml new file mode 100644 index 0000000..c54481a --- /dev/null +++ b/.github/workflows/conformance_suite_validation.yml @@ -0,0 +1,27 @@ +name: Conformance Test Suite Validation + +on: + push: + branches: [ mainline ] + paths: + - conformance-tests/** + pull_request: + branches: [ mainline ] + paths: + - conformance-tests/** + workflow_dispatch: + +jobs: + validate-naming: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Validate conformance test suite + working-directory: conformance-tests + run: python test_conformance_suite.py diff --git a/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/fixed-length-redaction.test.yaml b/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--fixed-length-redaction.test.yaml similarity index 100% rename from conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/fixed-length-redaction.test.yaml rename to conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--fixed-length-redaction.test.yaml diff --git a/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/future-occurrences-redacted.test.yaml b/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--future-occurrences-redacted.test.yaml similarity index 100% rename from conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/future-occurrences-redacted.test.yaml rename to conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--future-occurrences-redacted.test.yaml diff --git a/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/initial-line-redacted.test.yaml b/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--initial-line-redacted.test.yaml similarity index 100% rename from conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/initial-line-redacted.test.yaml rename to conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--initial-line-redacted.test.yaml diff --git a/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redaction-mixed-env-vars.test.yaml b/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--mixed-env-vars.test.yaml similarity index 100% rename from conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redaction-mixed-env-vars.test.yaml rename to conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--mixed-env-vars.test.yaml diff --git a/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redaction-multiple-secrets.test.yaml b/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--multiple-secrets.test.yaml similarity index 100% rename from conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redaction-multiple-secrets.test.yaml rename to conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--multiple-secrets.test.yaml diff --git a/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redaction-secret-not-logged.test.yaml b/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--secret-not-logged.test.yaml similarity index 100% rename from conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redaction-secret-not-logged.test.yaml rename to conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--secret-not-logged.test.yaml diff --git a/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/unset-takes-precedence.test.yaml b/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--unset-takes-precedence.test.yaml similarity index 100% rename from conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/unset-takes-precedence.test.yaml rename to conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--unset-takes-precedence.test.yaml diff --git a/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redaction-without-extension.test.yaml b/conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--without-extension.test.yaml similarity index 100% rename from conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redaction-without-extension.test.yaml rename to conformance-tests/2023-09/REDACTED_ENV_VARS/jobs/redact--without-extension.test.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/missing-extension.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/1.1.2--missing-extension.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/missing-extension.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/1.1.2--missing-extension.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/default-task-count-zero.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--default-task-count-zero.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/default-task-count-zero.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--default-task-count-zero.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/invalid-range-constraint.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--invalid-range-constraint.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/invalid-range-constraint.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--invalid-range-constraint.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/missing-chunks.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--missing-chunks.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/missing-chunks.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--missing-chunks.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/missing-default-task-count.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--missing-default-task-count.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/missing-default-task-count.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--missing-default-task-count.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/missing-range-constraint.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--missing-range-constraint.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/missing-range-constraint.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--missing-range-constraint.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/multiple-chunk-params.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--multiple-chunk-params.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/multiple-chunk-params.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--multiple-chunk-params.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/negative-target-runtime.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--negative-target-runtime.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/negative-target-runtime.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--negative-target-runtime.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/noncontiguous.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--noncontiguous.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/noncontiguous.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--noncontiguous.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/target-runtime.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--target-runtime.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/target-runtime.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.1.5--target-runtime.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/job_templates/associative-combination.invalid.yaml b/conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.3--associative-combination.invalid.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/job_templates/associative-combination.invalid.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/job_templates/chunk-3.4.3--associative-combination.invalid.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/jobs/contiguous-even.test.yaml b/conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--contiguous-even.test.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/jobs/contiguous-even.test.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--contiguous-even.test.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/jobs/contiguous-format.test.yaml b/conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--contiguous-format.test.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/jobs/contiguous-format.test.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--contiguous-format.test.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/jobs/contiguous-uneven.test.yaml b/conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--contiguous-uneven.test.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/jobs/contiguous-uneven.test.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--contiguous-uneven.test.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/jobs/larger-than-range.test.yaml b/conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--larger-than-range.test.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/jobs/larger-than-range.test.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--larger-than-range.test.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/jobs/noncontiguous-format.test.yaml b/conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--noncontiguous-format.test.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/jobs/noncontiguous-format.test.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--noncontiguous-format.test.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/jobs/noncontiguous.test.yaml b/conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--noncontiguous.test.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/jobs/noncontiguous.test.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--noncontiguous.test.yaml diff --git a/conformance-tests/2023-09/TASK_CHUNKING/jobs/with-target-runtime.test.yaml b/conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--with-target-runtime.test.yaml similarity index 100% rename from conformance-tests/2023-09/TASK_CHUNKING/jobs/with-target-runtime.test.yaml rename to conformance-tests/2023-09/TASK_CHUNKING/jobs/chunk-3.4.1.5--with-target-runtime.test.yaml diff --git a/conformance-tests/README.md b/conformance-tests/README.md index b3b070d..6cc8d48 100644 --- a/conformance-tests/README.md +++ b/conformance-tests/README.md @@ -40,24 +40,33 @@ conformance-tests/ └── jobs/ ``` -### Naming Convention +### File Naming Rules -Filenames encode the spec section they test: +Test filenames follow strict conventions that encode metadata used by test runners. ``` ---[.invalid][.suffix].yaml +[-][
]--[.invalid][.test]. ``` -- `` - Reference to the [Template Schema](../wiki/2023-09-Template-Schemas.md) section (e.g., `1.1`, `3.3.2`, `7.3`) -- `.invalid` - Test should FAIL validation/execution -- `.test` - Job execution test (in `jobs/` directory) +| Component | Required | Description | +|---|---|---| +| `` | No | Document prefix, required for extension specs. Omitted for the base [Template Schema](../wiki/2023-09-Template-Schemas.md) (a leading number implies it). | +| `
` | No | Section reference within the document (e.g., `1.1`, `3.4.1.5`). Omitted when the test isn't tied to a specific numbered section. | +| `--` | Yes | Double-hyphen separator before the description. | +| `` | Yes | Lowercase kebab-case description of what is being tested. | +| `.invalid` | No | Present when the test should FAIL validation or execution. | +| `.test` | No | Present for job execution tests (files in `jobs/` directories). | +| `` | Yes | `.yaml` or `.json`. | + +When both `.invalid` and `.test` are present, `.invalid` comes first: `1.1--desc.invalid.test.yaml` + +Each extension spec should use a short, consistent prefix (e.g., `chunk` for TASK_CHUNKING, `redact` for REDACTED_ENV_VARS). The base [Template Schema](../wiki/2023-09-Template-Schemas.md) needs no prefix — a leading number implies it. Examples: -- `1.1--minimal-job-template.yaml` - Section 1.1 (Job Template root) -- `3.3.2--allof.yaml` - Section 3.3.2 (AttributeRequirement) -- `5--cancelation-notify-then-terminate.yaml` - Section 5 (Action) -- `2.1--missing-name.invalid.yaml` - Invalid test for Section 2.1 -- `contiguous-even.test.yaml` - TASK_CHUNKING extension execution test (in `TASK_CHUNKING/jobs/`) +- `1.1--minimal-job-template.yaml` — base spec section 1.1, valid template +- `7.3--nested-braces.invalid.test.yaml` — base spec section 7.3, job execution test that should fail +- `chunk-3.4.1.5--noncontiguous.yaml` — TASK_CHUNKING section 3.4.1.5, valid template +- `redact--mixed-env-vars.test.yaml` — REDACTED_ENV_VARS, no specific section, job execution test ### Extension Tests diff --git a/conformance-tests/test_conformance_suite.py b/conformance-tests/test_conformance_suite.py new file mode 100644 index 0000000..6b7e6ce --- /dev/null +++ b/conformance-tests/test_conformance_suite.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Validates that all conformance test filenames follow the naming convention documented in README.md. + +The naming scheme is: + [-][
]--[.invalid][.test]. + +- Base spec tests (under base/): no doc prefix, section required (leading number). +- Extension tests: doc prefix required (chunk, redact, etc.), section optional. +- If an extension test is testing a base spec rule, it uses base spec naming (leading number, no doc prefix). +""" + +import re +import sys +from pathlib import Path + +CONFORMANCE_DIR = Path(__file__).parent + +# Map extension directory names to their doc prefix +EXTENSION_DOC_PREFIX = { + "TASK_CHUNKING": "chunk", + "REDACTED_ENV_VARS": "redact", +} + +# Base spec pattern: starts with a digit (section number) +#
--[.invalid][.test]. +BASE_RE = re.compile( + r"^(?P
\d+(?:\.\d+)*)--(?P[a-z0-9]+(?:-[a-z0-9]+)*)(?P\.invalid)?(?P\.test)?\.(?Pyaml|json)$" +) + +# Extension pattern: starts with a doc prefix +# [-
]--[.invalid][.test]. +def extension_re(prefix: str) -> re.Pattern: + return re.compile( + rf"^{re.escape(prefix)}(?:-(?P
\d+(?:\.\d+)*))?--(?P[a-z0-9]+(?:-[a-z0-9]+)*)(?P\.invalid)?(?P\.test)?\.(?Pyaml|json)$" + ) + + +def validate_filename(name: str, component: str, test_type: str) -> list[str]: + """Returns a list of error strings (empty if valid).""" + errors = [] + + if component == "base": + m = BASE_RE.match(name) + if not m: + errors.append(f"does not match base naming pattern: [
]--[.invalid][.test].") + return errors + else: + prefix = EXTENSION_DOC_PREFIX.get(component) + if prefix is None: + errors.append(f"unknown extension directory '{component}', add it to EXTENSION_DOC_PREFIX") + return errors + ext_pattern = extension_re(prefix) + m = BASE_RE.match(name) or ext_pattern.match(name) + if not m: + errors.append( + f"does not match naming pattern: " + f"
--... (base spec rule) or " + f"{prefix}[-
]--... (extension)" + ) + return errors + + if test_type == "jobs" and not m.group("test"): + errors.append("files in jobs/ must have .test suffix") + if test_type in ("job_templates", "env_templates") and m.group("test"): + errors.append(f"files in {test_type}/ must not have .test suffix") + + return errors + + +def main() -> int: + failures = [] + + for spec_dir in sorted(CONFORMANCE_DIR.iterdir()): + if not spec_dir.is_dir() or not spec_dir.name[0].isdigit(): + continue + for component_dir in sorted(spec_dir.iterdir()): + if not component_dir.is_dir(): + continue + component = component_dir.name + for test_type in ("job_templates", "env_templates", "jobs"): + type_dir = component_dir / test_type + if not type_dir.is_dir(): + continue + for f in sorted(type_dir.iterdir()): + if not f.is_file(): + continue + errs = validate_filename(f.name, component, test_type) + for e in errs: + failures.append(f"{f.relative_to(CONFORMANCE_DIR)}: {e}") + + if failures: + print(f"FAIL: {len(failures)} file(s) with invalid names:\n") + for msg in failures: + print(f" {msg}") + return 1 + + print("OK: all conformance test filenames are valid") + return 0 + + +if __name__ == "__main__": + sys.exit(main())