Add JSON coutput to tools/cove_checks.py

cove/cove_360/templates/cove_360/explore.html

@@ -275,9 +275,9 @@ <h3 class="panel-title panel-title-explore" id="conversion-title">
   <div class="panel panel-default">
     <div class="panel-heading" data-toggle="collapse" data-target="#validation-body">
       <h3 class="panel-title panel-title-explore">
-        {% if validation_errors %}
+        {% if validation_errors or additional_closed_codelist_values %}
           <span class="font-tick cross" aria-hidden="true"></span>
-            {% trans "This data does not use the 360Giving Data Standard correctly" %} <small>{% blocktrans count n_errors=validation_errors_count %} {{n_errors}} Error.{% plural %}{{n_errors}} Errors {% endblocktrans %}</small>
+            {% trans "This data does not use the 360Giving Data Standard correctly" %} <small>{% blocktrans count n_errors=validation_and_closed_codelist_errors_count %} {{n_errors}} Error.{% plural %}{{n_errors}} Errors {% endblocktrans %}</small>
         {% else %}
           <span class="font-tick tick" aria-hidden="true"></span>
           {% trans "This data uses the 360Giving Data Standard correctly" %}
@@ -288,15 +288,15 @@ <h3 class="panel-title panel-title-explore">
     <div id="validation-body" class="collapse">
       <div class="panel-body">
         <p class="explanation">
-          {% if validation_errors %}
+          {% if validation_errors or additional_closed_codelist_values %}
             {% trans "Sorry your data is not yet using the 360Giving Data Standard. We used the " %}
           {% else %}
             {% trans "Congratulations! Your data is using the 360Giving Data Standard. We used the " %}
           {% endif %}
           <a href="https://www.threesixtygiving.org/standard/reference/#toc-360giving-json-schemas"> 360Giving JSON Package Schema</a> {% trans "to check this." %}
         </p>
         <div class="explore-help">{% trans "This means that the data" %}
-          {% if validation_errors %}
+          {% if validation_errors or additional_closed_codelist_values %}
             {% trans " does not meet " %}
           {% else %}
             {% trans " meets " %}
@@ -305,7 +305,7 @@ <h3 class="panel-title panel-title-explore">
           {% blocktrans %}the requirements of the <a href="https://standard.threesixtygiving.org/en/latest/technical/reference/">360Giving Data Standard</a>.{% endblocktrans %}
           {% blocktrans %} Making sure your data uses the standard correctly is important. Otherwise it cannot be used alongside other valid 360Giving data and cannot be included in 360Giving tools, such as GrantNav and 360Insights.{% endblocktrans %}
         </diV>
-        {% if validation_errors %}
+        {% if validation_errors or additional_closed_codelist_values %}
           <br>
           <p class="explanation">&nbsp;{% trans "The following <strong>errors</strong> are preventing your data from being valid 360Giving data. Please use the feedback below to find and resolve the issues in your file" %}</p>
           <br>
@@ -345,6 +345,24 @@ <h4>Incorrect Formats</h4>
               {% endfor %}
             {% endif %}
           {% endwith %}
+          {% if additional_closed_codelist_values %}
+              <div class="panel panel-default">
+                <div class="panel-heading">
+                  <h4>{% trans 'Codelist Errors' %}</h4>
+                </div>
+                <div>
+                  <div class="panel-body">
+                    <p>Some or all of your entries do not use the 360Giving Data Standard codelists correctly.</p>
+                    <p>{% blocktrans %}The fields below use closed codelists. When using these fields, you <strong>must</strong> use one of the pre-defined codelist values. If you use a value that is not on the relevant codelist, your data will not pass structural checks.{% endblocktrans %}</p>
+                  </div>
+                  </p>
+                  {% with additional_codelist_values=additional_closed_codelist_values %}
+                  <div>{% include "additional_codelist_values.html" %}</div>
+                  {% endwith %}
+                </div>
+              </div>
+          {% endif %}
+        {% endif %}
           {% with validation_errors=validation_errors_grouped.other error_prefix='other-' %}
             {% if validation_errors %}
               <div class="panel panel-default">
@@ -363,12 +381,12 @@ <h4>Other</h4>
               {% endfor %}
             {% endif %}
           {% endwith %}
-        {% endif %}
       </div>
     </div>
   </div>
   {% endblock validation %}
 
+
   {% block quality_accuracy %}
   {% if quality_accuracy_checks_count %}
   <div class="panel panel-default">

lib360dataquality/cove/schema.py

@@ -10,3 +10,4 @@ class Schema360(SchemaJsonMixin):
     pkg_schema_name = config['schema_name']
     schema_url = urljoin(schema_host, schema_name)
     pkg_schema_url = urljoin(schema_host, pkg_schema_name)
+    codelists = config['codelists_host']

lib360dataquality/cove/settings.py

@@ -11,6 +11,7 @@
     'schema_name': '360-giving-package-schema.json',
     'schema_item_name': '360-giving-schema.json',
     'schema_host': f'https://raw.githubusercontent.com/ThreeSixtyGiving/standard/{os.environ.get("SCHEMA_BRANCH", "master")}/schema/',
+    'codelists_host': f'https://raw.githubusercontent.com/ThreeSixtyGiving/standard/{os.environ.get("SCHEMA_BRANCH", "master")}/codelists/',
     'schema_version': None,
     'schema_version_choices': None,
     'root_list_path': 'grants',

lib360dataquality/cove/threesixtygiving.py

@@ -10,7 +10,7 @@
 import openpyxl
 import pytz
 from dateutil.relativedelta import relativedelta
-from libcove.lib.common import common_checks_context, get_orgids_prefixes
+from libcove.lib.common import common_checks_context, get_additional_codelist_values, get_orgids_prefixes
 from rangedict import RangeDict as range_dict
 
 try:
@@ -350,6 +350,22 @@ def common_checks_360(
             }
         )
 
+    additional_codelist_values = get_additional_codelist_values(schema_obj, json_data)
+    closed_codelist_values = {
+        key: value for key, value in additional_codelist_values.items() if not value["isopen"]
+    }
+    open_codelist_values = {key: value for key, value in additional_codelist_values.items() if value["isopen"]}
+
+    closed_codelist_errors_count = sum(len(value['values']) for value in closed_codelist_values.values())
+
+    context.update(
+        {
+            "additional_closed_codelist_values": closed_codelist_values,
+            "additional_open_codelist_values": open_codelist_values,
+            "validation_and_closed_codelist_errors_count": context["validation_errors_count"] + closed_codelist_errors_count
+        }
+    )
+
     return context
 
 

requirements_cove.in

@@ -2,7 +2,7 @@
 Django>3.2,<3.3
 #^^ rq.filter: <1.12
 flattentool>=0.17.0
-libcove
+-e git+https://github.com/OpenDataServices/lib-cove.git@05ed94580885e7f01c752dc1d02b3b4f8ad1fe55#egg=libcove
 libcoveweb
 django-bootstrap3
 django-debug-toolbar

requirements_cove.txt

@@ -6,6 +6,11 @@
 #
 -e file:./
     # via -r requirements_cove.in
+-e git+https://github.com/OpenDataServices/lib-cove.git@05ed94580885e7f01c752dc1d02b3b4f8ad1fe55#egg=libcove
+    # via
+    #   -r requirements_cove.in
+    #   lib360dataquality
+    #   libcoveweb
 asgiref==3.5.2
     # via django
 attrs==22.1.0
@@ -75,11 +80,6 @@ jsonschema==3.2.0
     # via
     #   -r requirements_cove.in
     #   libcove
-libcove==0.27.0
-    # via
-    #   -r requirements_cove.in
-    #   lib360dataquality
-    #   libcoveweb
 libcoveweb==0.25.0
     # via -r requirements_cove.in
 lxml==4.9.1

requirements_cove_dev.txt

@@ -6,6 +6,11 @@
 #
 -e file:./
     # via -r requirements_cove.in
+-e git+https://github.com/OpenDataServices/lib-cove.git@05ed94580885e7f01c752dc1d02b3b4f8ad1fe55#egg=libcove
+    # via
+    #   -r requirements_cove.in
+    #   lib360dataquality
+    #   libcoveweb
 aiosmtpd==1.4.2
     # via pytest-localserver
 alabaster==0.7.12
@@ -144,11 +149,6 @@ jsonschema==3.2.0
     # via
     #   -r requirements_cove.in
     #   libcove
-libcove==0.27.0
-    # via
-    #   -r requirements_cove.in
-    #   lib360dataquality
-    #   libcoveweb
 libcoveweb==0.25.0
     # via -r requirements_cove.in
 libsass==0.21.0

requirements_cove_dokku.txt

@@ -6,6 +6,11 @@
 #
 -e file:./
     # via -r requirements_cove.txt
+-e git+https://github.com/OpenDataServices/lib-cove.git@05ed94580885e7f01c752dc1d02b3b4f8ad1fe55#egg=libcove
+    # via
+    #   -r requirements_cove.txt
+    #   lib360dataquality
+    #   libcoveweb
 asgiref==3.5.2
     # via
     #   -r requirements_cove.txt
@@ -100,11 +105,6 @@ jsonschema==3.2.0
     # via
     #   -r requirements_cove.txt
     #   libcove
-libcove==0.27.0
-    # via
-    #   -r requirements_cove.txt
-    #   lib360dataquality
-    #   libcoveweb
 libcoveweb==0.25.0
     # via -r requirements_cove.txt
 lxml==4.9.1

tools/cove_checks.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import sys
 import argparse
 import json
 import pprint
@@ -68,6 +69,7 @@ def main():
                 file_type,
                 lib_cove_config,
                 schema.schema_url,
+                schema.pkg_schema_url
             )
         )
         with open(context["converted_path"], "r") as fp_data:
@@ -90,7 +92,12 @@ def main():
     # We don't actually want to show the json data again
     del context["json_data"]
 
-    pprint.pprint(context)
+    def serialize_sets(obj):
+        if isinstance(obj, set):
+            return list(obj)
+        return obj
+
+    json.dump(context, sys.stdout, default=serialize_sets)
 
 
 if __name__ == "__main__":