From f560d6d0eb93ab472e282ccc72e324629ddbce01 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Tue, 14 Oct 2025 11:48:03 +0100 Subject: [PATCH 1/5] build(pyproject.toml): update tag_format for cz to match current tag --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index df5e416..dd2188b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -673,7 +673,7 @@ redefining-builtins-modules = [ [tool.commitizen] name = "cz_conventional_commits" -tag_format = "$version" +tag_format = "v$major.$minor.$patch" version_scheme = "pep440" version_provider = "poetry" update_changelog_on_bump = true From 3e79fd975b555eb8b533a7e15801daa67f626049 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:04:05 +0100 Subject: [PATCH 2/5] docs: add issue templates --- .github/ISSUE_TEMPLATE/BUG-REPORT.yml | 80 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml | 63 +++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 1 + 3 files changed, 144 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/BUG-REPORT.yml create mode 100644 .github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml new file mode 100644 index 0000000..cc4709c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml @@ -0,0 +1,80 @@ +name: "🐛 Bug Report" +description: Create a new ticket for a bug. +title: "🐛 [BUG] - " +labels: [ + "bug" +] +body: + - type: textarea + id: description + attributes: + label: "Description" + description: Please enter an explicit description of your issue + placeholder: Short and explicit description of your incident... + validations: + required: true + - type: input + id: reprod-url + attributes: + label: "Reproduction URL" + description: Please enter your GitHub URL to provide a reproduction of the issue + placeholder: ex. https://github.com/USERNAME/REPO-NAME + validations: + required: true + - type: textarea + id: reprod + attributes: + label: "Reproduction steps" + description: Please enter an explicit description of your issue + value: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + render: bash + validations: + required: true + - type: textarea + id: screenshot + attributes: + label: "Screenshots" + description: If applicable, add screenshots to help explain your problem. + value: | + ![DESCRIPTION](LINK.png) + render: bash + validations: + required: false + - type: textarea + id: logs + attributes: + label: "Logs" + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: bash + validations: + required: false + - type: dropdown + id: browsers + attributes: + label: "Browsers" + description: What browsers are you seeing the problem on ? + multiple: true + options: + - Firefox + - Chrome + - Safari + - Microsoft Edge + - Opera + validations: + required: false + - type: dropdown + id: os + attributes: + label: "OS" + description: What is the impacted environment ? + multiple: true + options: + - Windows + - Linux + - Mac + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml new file mode 100644 index 0000000..c869cf2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml @@ -0,0 +1,63 @@ +name: "💡 Feature Request" +description: Create a new ticket for a new feature request +title: "💡 [REQUEST] - <title>" +labels: [ + "feature" +] +body: + - type: input + id: start_date + attributes: + label: "Start Date" + description: Start of development + placeholder: "month/day/year" + validations: + required: false + - type: textarea + id: implementation_pr + attributes: + label: "Implementation PR" + description: Pull request used + placeholder: "#Pull Request ID" + validations: + required: false + - type: textarea + id: reference_issues + attributes: + label: "Reference Issues" + description: Common issues + placeholder: "#Issues IDs" + validations: + required: false + - type: textarea + id: summary + attributes: + label: "Summary" + description: Provide a brief explanation of the feature + placeholder: Describe in a few lines your feature request + validations: + required: true + - type: textarea + id: basic_example + attributes: + label: "Basic Example" + description: Indicate here some basic examples of your feature. + placeholder: A few specific words about your feature request. + validations: + required: true + - type: textarea + id: drawbacks + attributes: + label: "Drawbacks" + description: What are the drawbacks/impacts of your feature request ? + placeholder: Identify the drawbacks and impacts while being neutral on your feature request + validations: + required: true + - type: textarea + id: unresolved_question + attributes: + label: "Unresolved questions" + description: What questions still remain unresolved ? + placeholder: Identify any unresolved issues. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..ec4bb38 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false \ No newline at end of file From 649091c44b7eca852b58f72aaabbb4ae4d4dbd7b Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:04:47 +0100 Subject: [PATCH 3/5] docs(README.md): fix and update info in the README.md --- README.md | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 62f3f12..1626107 100644 --- a/README.md +++ b/README.md @@ -2,39 +2,47 @@ The Data Validation Engine (DVE) is a configuration driven data validation library built and utilised by NHS England. -As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is also typically known as a `dischema` file and example files can be accessed [here](./tests/testdata/). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here](./docs/). +As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` file and example files can be accessed [here](./tests/testdata/). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here](./docs/). -Once a dischema file has been defined, you are ready to use the DVE. The DVE is typically orchestrated based on the four key "services". These are... +Once a dischema file has been defined, you are ready to use the DVE. The DVE is typically orchestrated based on four key "services". These are... -| Order | Service | Purpose | -| ----- | ------- | ------- | -| 1. | File Transformation | This service will take ingest submitted files and turn them into stringified parquet files to ensure that a consistent data structure can be passed through the DVE. | -| 2. | Data Contract | This service will validate and cast a stringified parquet submission against a [pyantic model](https://docs.pydantic.dev/latest/). | +| | Service | Purpose | +| -- | ------- | ------- | +| 1. | File Transformation | This service will take submitted files and turn them into stringified parquet file(s) to ensure that a consistent data structure can be passed through the other services. | +| 2. | Data Contract | This service will validate and peform type casting against a stringified parquet file using [pydantic models](https://docs.pydantic.dev/1.10/). | | 3. | Business Rules | The business rules service will perform more complex validations such as comparisons between fields and tables, aggregations, filters etc to generate new entities. | -| 4. | Error Reports | The error reports service will take all the errors raised in previous services and surface them into a readable format for a downstream users/service. Currently, this implemented to be an excel spreadsheet but could be reconfigure to meet other requirements/use cases. | +| 4. | Error Reports | The error reports service will take all the errors raised in previous services and surface them into a readable format for a downstream users/service. Currently, this implemented to be an excel spreadsheet but could be reconfigured to meet other requirements/use cases. | -We have more detailed documentation around these services [here](./docs/). +If you'd like more detailed documentation around these services the please read the extended documentation [here](./docs/). + +The DVE has been designed in a way that's modular and can support users who just want to utilise specific "services" from the DVE (i.e. just the file transformation + data contract). Additionally, the DVE is designed to support different backend implementations. As part of the base installation of DVE, you will find backend support for `Spark` and `DuckDB`. So, if you need a `MySQL` backend implementation, you can implement this yourself. Given our organisations requirements, it will be unlikely that we add anymore specific backend implementations into the base package beyond Spark and DuckDB. So, if you are unable to implement this yourself, I would recommend reading the guidance on [requesting new features and raising bug reports here](#requesting-new-features-and-raising-bug-reports). + +Additionally, if you'd like to contribute a new backend implementation into the base DVE package, then please look at the [Contributing][#Contributing] section. ## Installation and usage -The DVE is a Python package and can be installed using `pip`. As of release (version 1+) only supports Python 3.7, with Spark version 3.2.1 and DuckDB version of 1.1.0. We are currently working on upgrading the DVE to work on Python 3.11+ and this will be made available asap with version 2.0.0 release. +The DVE is a Python package and can be installed using `pip`. As of release v1.0.0 we currently only supports Python 3.7, with Spark version 3.2.1 and DuckDB version of 1.1.0. We are currently working on upgrading the DVE to work on Python 3.11+ and this will be made available asap with version 2.0.0 release. -In addition to a working Python 3.7+ installation you will need OpenJDK 11 installed. +In addition to a working Python 3.7+ installation you will need OpenJDK 11 installed if you're planning to use the Spark backend implementation. Python dependencies are listed in `pyproject.toml`. To install the DVE package you can simply install using a package manager such as [pip](https://pypi.org/project/pip/). ``` -pip install https://github.com/nhsengland/Data-Validation-Engine +pip install git+https://github.com/NHSDigital/data-validation-engine.git@v1.0.0 ``` -Once you have installed the DVE you are ready to use it. For guidance on how to create your dischema json document (configuration), please read the [documentation](/docs/). +Once you have installed the DVE you are ready to use it. For guidance on how to create your dischema json document (configuration), please read the [documentation](./docs/). -The long term aim is to make the DVE available via PyPi and Conda but we are not quite there yet. Once available this documentation will be updated to reflect the new installation options. +Please note - The long term aim is to make the DVE available via PyPi and Conda but we are not quite there yet. Once available this documentation will be updated to contain the new installation options. ## Requesting new features and raising bug reports -If you have spotted a bug with the DVE then please raise an issue [here](https://github.com/nhsengland/Data-Validation-Engine/issues). Same for any feature requests. +**Before creating new issues, please check to see if the same bug/feature has been created already. Where a duplicate is created, the ticket will be closed and referenced to an existing issue.** + +If you have spotted a bug with the DVE then please raise an issue [here](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template". + +If you have feature request then please follow the same process whilst using the "Feature request template". ## Upcoming features Below is a list of features that we would like to implement or have been requested. @@ -45,7 +53,7 @@ Below is a list of features that we would like to implement or have been request | Upgrade to Pydantic 2.0 | Not yet confirmed | No | | Create a more user friendly interface for building and modifying dischema files | Not yet confirmed | No | -Beyond the Python upgrade, we cannot confirm the other features will be made available any time soon. Therefore, if you have the interest and desire to make these features available, then please feel free to read the [contributing section](#contributing) and get involved. +Beyond the Python upgrade, we cannot confirm the other features will be made available anytime soon. Therefore, if you have the interest and desire to make these features available, then please read the [Contributing](#contributing) section and get involved. ## Contributing Please see guidance [here](./CONTRIBUTE.md). From 150f7bbb6bc10c70353bd4ae6b2ad09f46ea76b2 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:07:47 +0100 Subject: [PATCH 4/5] docs(BUG-REPORT.yml): remove browser section from template --- .github/ISSUE_TEMPLATE/BUG-REPORT.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml index cc4709c..0e700cd 100644 --- a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml +++ b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml @@ -52,20 +52,6 @@ body: render: bash validations: required: false - - type: dropdown - id: browsers - attributes: - label: "Browsers" - description: What browsers are you seeing the problem on ? - multiple: true - options: - - Firefox - - Chrome - - Safari - - Microsoft Edge - - Opera - validations: - required: false - type: dropdown id: os attributes: From c755f50fb6b0aff7e196b0e3965d9aa8fc683c29 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:16:45 +0100 Subject: [PATCH 5/5] docs(BUG-REPORT.yml): make reproduction url optional --- .github/ISSUE_TEMPLATE/BUG-REPORT.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml index 0e700cd..c3d04c8 100644 --- a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml +++ b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml @@ -20,7 +20,7 @@ body: description: Please enter your GitHub URL to provide a reproduction of the issue placeholder: ex. https://github.com/USERNAME/REPO-NAME validations: - required: true + required: false - type: textarea id: reprod attributes: