diff --git a/cove/cove_360/templates/cove_360/explore.html b/cove/cove_360/templates/cove_360/explore.html index 1fc40c5..477c30d 100644 --- a/cove/cove_360/templates/cove_360/explore.html +++ b/cove/cove_360/templates/cove_360/explore.html @@ -275,9 +275,9 @@

- {% if validation_errors %} + {% if validation_errors or additional_closed_codelist_values %} - {% trans "This data does not use the 360Giving Data Standard correctly" %} {% blocktrans count n_errors=validation_errors_count %} {{n_errors}} Error.{% plural %}{{n_errors}} Errors {% endblocktrans %} + {% trans "This data does not use the 360Giving Data Standard correctly" %} {% blocktrans count n_errors=validation_and_closed_codelist_errors_count %} {{n_errors}} Error.{% plural %}{{n_errors}} Errors {% endblocktrans %} {% else %} {% trans "This data uses the 360Giving Data Standard correctly" %} @@ -288,7 +288,7 @@

- {% if validation_errors %} + {% if validation_errors or additional_closed_codelist_values %} {% trans "Sorry your data is not yet using the 360Giving Data Standard. We used the " %} {% else %} {% trans "Congratulations! Your data is using the 360Giving Data Standard. We used the " %} @@ -296,7 +296,7 @@

360Giving JSON Package Schema {% trans "to check this." %}

{% trans "This means that the data" %} - {% if validation_errors %} + {% if validation_errors or additional_closed_codelist_values %} {% trans " does not meet " %} {% else %} {% trans " meets " %} @@ -305,7 +305,7 @@

{% blocktrans %}the requirements of the 360Giving Data Standard.{% endblocktrans %} {% blocktrans %} Making sure your data uses the standard correctly is important. Otherwise it cannot be used alongside other valid 360Giving data and cannot be included in 360Giving tools, such as GrantNav and 360Insights.{% endblocktrans %}

- {% if validation_errors %} + {% if validation_errors or additional_closed_codelist_values %}

 {% trans "The following errors are preventing your data from being valid 360Giving data. Please use the feedback below to find and resolve the issues in your file" %}


@@ -345,6 +345,24 @@

Incorrect Formats

{% endfor %} {% endif %} {% endwith %} + {% if additional_closed_codelist_values %} +
+
+

{% trans 'Codelist Errors' %}

+
+
+
+

Some or all of your entries do not use the 360Giving Data Standard codelists correctly.

+

{% blocktrans %}The fields below use closed codelists. When using these fields, you must use one of the pre-defined codelist values. If you use a value that is not on the relevant codelist, your data will not pass structural checks.{% endblocktrans %}

+
+

+ {% with additional_codelist_values=additional_closed_codelist_values %} +
{% include "additional_codelist_values.html" %}
+ {% endwith %} +
+
+ {% endif %} + {% endif %} {% with validation_errors=validation_errors_grouped.other error_prefix='other-' %} {% if validation_errors %}
@@ -363,12 +381,12 @@

Other

{% endfor %} {% endif %} {% endwith %} - {% endif %}
{% endblock validation %} + {% block quality_accuracy %} {% if quality_accuracy_checks_count %}
diff --git a/lib360dataquality/cove/schema.py b/lib360dataquality/cove/schema.py index 7ba2620..330b5d4 100644 --- a/lib360dataquality/cove/schema.py +++ b/lib360dataquality/cove/schema.py @@ -10,3 +10,4 @@ class Schema360(SchemaJsonMixin): pkg_schema_name = config['schema_name'] schema_url = urljoin(schema_host, schema_name) pkg_schema_url = urljoin(schema_host, pkg_schema_name) + codelists = config['codelists_host'] diff --git a/lib360dataquality/cove/settings.py b/lib360dataquality/cove/settings.py index c26eb31..81cc67d 100644 --- a/lib360dataquality/cove/settings.py +++ b/lib360dataquality/cove/settings.py @@ -11,6 +11,7 @@ 'schema_name': '360-giving-package-schema.json', 'schema_item_name': '360-giving-schema.json', 'schema_host': f'https://raw.githubusercontent.com/ThreeSixtyGiving/standard/{os.environ.get("SCHEMA_BRANCH", "master")}/schema/', + 'codelists_host': f'https://raw.githubusercontent.com/ThreeSixtyGiving/standard/{os.environ.get("SCHEMA_BRANCH", "master")}/codelists/', 'schema_version': None, 'schema_version_choices': None, 'root_list_path': 'grants', diff --git a/lib360dataquality/cove/threesixtygiving.py b/lib360dataquality/cove/threesixtygiving.py index b48d6e9..5a39b9f 100644 --- a/lib360dataquality/cove/threesixtygiving.py +++ b/lib360dataquality/cove/threesixtygiving.py @@ -10,7 +10,7 @@ import openpyxl import pytz from dateutil.relativedelta import relativedelta -from libcove.lib.common import common_checks_context, get_orgids_prefixes +from libcove.lib.common import common_checks_context, get_additional_codelist_values, get_orgids_prefixes from rangedict import RangeDict as range_dict try: @@ -350,6 +350,22 @@ def common_checks_360( } ) + additional_codelist_values = get_additional_codelist_values(schema_obj, json_data) + closed_codelist_values = { + key: value for key, value in additional_codelist_values.items() if not value["isopen"] + } + open_codelist_values = {key: value for key, value in additional_codelist_values.items() if value["isopen"]} + + closed_codelist_errors_count = sum(len(value['values']) for value in closed_codelist_values.values()) + + context.update( + { + "additional_closed_codelist_values": closed_codelist_values, + "additional_open_codelist_values": open_codelist_values, + "validation_and_closed_codelist_errors_count": context["validation_errors_count"] + closed_codelist_errors_count + } + ) + return context diff --git a/requirements_cove.in b/requirements_cove.in index e606cf1..ac1f4dc 100644 --- a/requirements_cove.in +++ b/requirements_cove.in @@ -2,7 +2,7 @@ Django>3.2,<3.3 #^^ rq.filter: <1.12 flattentool>=0.17.0 -libcove +-e git+https://github.com/OpenDataServices/lib-cove.git@05ed94580885e7f01c752dc1d02b3b4f8ad1fe55#egg=libcove libcoveweb django-bootstrap3 django-debug-toolbar diff --git a/requirements_cove.txt b/requirements_cove.txt index 4864b93..e9fde65 100644 --- a/requirements_cove.txt +++ b/requirements_cove.txt @@ -6,6 +6,11 @@ # -e file:./ # via -r requirements_cove.in +-e git+https://github.com/OpenDataServices/lib-cove.git@05ed94580885e7f01c752dc1d02b3b4f8ad1fe55#egg=libcove + # via + # -r requirements_cove.in + # lib360dataquality + # libcoveweb asgiref==3.5.2 # via django attrs==22.1.0 @@ -75,11 +80,6 @@ jsonschema==3.2.0 # via # -r requirements_cove.in # libcove -libcove==0.27.0 - # via - # -r requirements_cove.in - # lib360dataquality - # libcoveweb libcoveweb==0.25.0 # via -r requirements_cove.in lxml==4.9.1 diff --git a/requirements_cove_dev.txt b/requirements_cove_dev.txt index e062eea..bb5e82c 100644 --- a/requirements_cove_dev.txt +++ b/requirements_cove_dev.txt @@ -6,6 +6,11 @@ # -e file:./ # via -r requirements_cove.in +-e git+https://github.com/OpenDataServices/lib-cove.git@05ed94580885e7f01c752dc1d02b3b4f8ad1fe55#egg=libcove + # via + # -r requirements_cove.in + # lib360dataquality + # libcoveweb aiosmtpd==1.4.2 # via pytest-localserver alabaster==0.7.12 @@ -144,11 +149,6 @@ jsonschema==3.2.0 # via # -r requirements_cove.in # libcove -libcove==0.27.0 - # via - # -r requirements_cove.in - # lib360dataquality - # libcoveweb libcoveweb==0.25.0 # via -r requirements_cove.in libsass==0.21.0 diff --git a/requirements_cove_dokku.txt b/requirements_cove_dokku.txt index 545e942..98eb677 100644 --- a/requirements_cove_dokku.txt +++ b/requirements_cove_dokku.txt @@ -6,6 +6,11 @@ # -e file:./ # via -r requirements_cove.txt +-e git+https://github.com/OpenDataServices/lib-cove.git@05ed94580885e7f01c752dc1d02b3b4f8ad1fe55#egg=libcove + # via + # -r requirements_cove.txt + # lib360dataquality + # libcoveweb asgiref==3.5.2 # via # -r requirements_cove.txt @@ -100,11 +105,6 @@ jsonschema==3.2.0 # via # -r requirements_cove.txt # libcove -libcove==0.27.0 - # via - # -r requirements_cove.txt - # lib360dataquality - # libcoveweb libcoveweb==0.25.0 # via -r requirements_cove.txt lxml==4.9.1 diff --git a/tools/cove_checks.py b/tools/cove_checks.py index d3738b8..2b33e93 100755 --- a/tools/cove_checks.py +++ b/tools/cove_checks.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +import sys import argparse import json import pprint @@ -68,6 +69,7 @@ def main(): file_type, lib_cove_config, schema.schema_url, + schema.pkg_schema_url ) ) with open(context["converted_path"], "r") as fp_data: @@ -90,7 +92,12 @@ def main(): # We don't actually want to show the json data again del context["json_data"] - pprint.pprint(context) + def serialize_sets(obj): + if isinstance(obj, set): + return list(obj) + return obj + + json.dump(context, sys.stdout, default=serialize_sets) if __name__ == "__main__":