Implement optional count in Feature Providers

docs/source/configuration.rst

@@ -271,6 +271,7 @@ default.
                 include_extra_query_parameters: false  # include extra query parameters that are not part of the collection properties (default: false)
                 # editable transactions: DO NOT ACTIVATE unless you have setup access control beyond pygeoapi
                 editable: true  # optional: if backend is writable, default is false
+                count: true  # optional: perform a count query for collection queries, default is true
                 # coordinate reference systems (CRS) section is optional
                 # default CRSs are http://www.opengis.net/def/crs/OGC/1.3/CRS84 (coordinates without height)
                 # and http://www.opengis.net/def/crs/OGC/1.3/CRS84h (coordinates with ellipsoidal height)

pygeoapi/provider/csv_.py

@@ -194,8 +194,9 @@ def _load(self, offset=0, limit=10, resulttype='results',
 
                 feature_collection['features'].append(feature)
 
-                feature_collection['numberMatched'] = \
-                    len(feature_collection['features'])
+                if self.count:
+                    feature_collection['numberMatched'] = \
+                        len(feature_collection['features'])
 
         if identifier is not None and not found:
             return None

pygeoapi/provider/esri.py

@@ -156,9 +156,13 @@ def query(self, offset=0, limit=10, resulttype='results',
         fc = {
             'type': 'FeatureCollection',
             'features': [],
-            'numberMatched': self._get_count(params)
         }
 
+        if self.count or resulttype == 'hits':
+            matched = self._get_count(params)
+            LOGGER.debug(f'Found {matched} result(s)')
+            fc['numberMatched'] = matched
+
         if resulttype == 'hits':
             return fc
 
@@ -168,7 +172,7 @@ def query(self, offset=0, limit=10, resulttype='results',
         params['resultOffset'] = offset
         params['resultRecordCount'] = limit
 
-        hits_ = min(limit, fc['numberMatched'])
+        hits_ = min(limit, matched) if self.count else limit
         fc['features'] = self._get_all(params, hits_)
 
         fc['numberReturned'] = len(fc['features'])

pygeoapi/provider/geojson.py

@@ -188,7 +188,8 @@ def query(self, offset=0, limit=10, resulttype='results',
                           properties=properties,
                           select_properties=select_properties)
 
-        data['numberMatched'] = len(data['features'])
+        if self.count or resulttype == 'hits':
+            data['numberMatched'] = len(data['features'])
 
         if resulttype == 'hits':
             data['features'] = []

pygeoapi/provider/mongo.py

@@ -102,7 +102,6 @@ def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1,
         if sortList:
             featurecursor = featurecursor.sort(sortList)
 
-        matchCount = self.featuredb[self.collection].count_documents(filterObj)
         featurecursor.skip(skip)
         featurecursor.limit(maxitems)
         featurelist = list(featurecursor)
@@ -111,7 +110,7 @@ def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1,
             if skip_geometry:
                 item['geometry'] = None
 
-        return featurelist, matchCount
+        return featurelist
 
     @crs_transform
     def query(self, offset=0, limit=10, resulttype='results',
@@ -144,20 +143,28 @@ def query(self, offset=0, limit=10, resulttype='results',
                       ASCENDING if (sort['order'] == '+') else DESCENDING)
                      for sort in sortby]
 
-        featurelist, matchcount = self._get_feature_list(
-            filterobj, sortList=sort_list, skip=offset, maxitems=limit,
-            skip_geometry=skip_geometry)
-
-        if resulttype == 'hits':
-            featurelist = []
-
         feature_collection = {
             'type': 'FeatureCollection',
-            'features': featurelist,
-            'numberMatched': matchcount,
-            'numberReturned': len(featurelist)
+            'features': []
         }
 
+        if self.count or resulttype == 'hits':
+            matched = self.featuredb[self.collection].count_documents(
+                filterobj)
+            LOGGER.debug(f'Found {matched} result(s)')
+            feature_collection['numberMatched'] = matched
+
+        if resulttype == 'hits':
+            return feature_collection
+
+        featurelist = self._get_feature_list(
+            filterobj, sortList=sort_list, skip=offset, maxitems=limit,
+            skip_geometry=skip_geometry
+        )
+
+        feature_collection['features'] = featurelist
+        feature_collection['numberReturned'] = len(featurelist)
+
         return feature_collection
 
     @crs_transform
@@ -168,8 +175,7 @@ def get(self, identifier, **kwargs):
         :param identifier: feature id
         :returns: dict of single GeoJSON feature
         """
-        featurelist, matchcount = self._get_feature_list(
-                                    {'_id': ObjectId(identifier)})
+        featurelist = self._get_feature_list({'_id': ObjectId(identifier)})
         if featurelist:
             return featurelist[0]
         else:

pygeoapi/provider/socrata.py

@@ -123,10 +123,14 @@ def query(self, offset=0, limit=10, resulttype='results',
 
         fc = {
             'type': 'FeatureCollection',
-            'features': [],
-            'numberMatched': self._get_count(params)
+            'features': []
         }
 
+        if self.count or resulttype == 'hits':
+            matched = self._get_count(params)
+            LOGGER.debug(f'Found {matched} result(s)')
+            fc['numberMatched'] = matched
+
         if resulttype == 'hits':
             # Return hits
             LOGGER.debug('Returning hits')

pygeoapi/provider/sql.py

@@ -231,6 +231,7 @@ def query(
 
             crs_transform_out = get_transform_from_spec(crs_transform_spec)
 
+            response['numberReturned'] = 0
             for item in (
                 results.order_by(*order_by_clauses).offset(offset).limit(limit)
             ):

pygeoapi/provider/tinydb_.py

@@ -226,7 +226,10 @@ def query(self, offset=0, limit=10, resulttype='results',
         else:
             results = self.db.all()
 
-        feature_collection['numberMatched'] = len(results)
+        if self.count or resulttype == 'hits':
+            matched = len(results)
+            LOGGER.debug(f'Found {matched} result(s)')
+            feature_collection['numberMatched'] = matched
 
         if resulttype == 'hits':
             return feature_collection

pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml

@@ -540,6 +540,10 @@ properties:
                                           type: boolean
                                           description: whether the resource is editable
                                           default: false
+                                      count:
+                                          type: boolean
+                                          description: whether to perform a count query for collection queries
+                                          default: true
                                       table:
                                           type: string
                                           description: table name for RDBMS-based providers

tests/provider/test_csv__provider.py

@@ -144,6 +144,19 @@ def test_query(config):
     assert len(results['features'][0]['properties']) == 2
 
 
+def test_no_count(config):
+    p = CSVProvider(config)
+    results = p.query()
+    assert results['numberMatched'] == 5
+    assert results['numberReturned'] == 5
+
+    config['count'] = False
+    p = CSVProvider(config)
+    results = p.query()
+    assert 'numberMatched' not in results
+    assert results['numberReturned'] == 5
+
+
 def test_get_invalid_property(config):
     """Testing query for an invalid property name"""
     p = CSVProvider(config)

tests/provider/test_esri_provider.py

@@ -85,6 +85,19 @@ def test_query(config):
     assert results['numberMatched'] == 406
 
 
+def test_no_count(config):
+    p = ESRIServiceProvider(config)
+    results = p.query()
+    assert results['numberMatched'] == 406
+    assert results['numberReturned'] == 10
+
+    config['count'] = False
+    p = ESRIServiceProvider(config)
+    results = p.query()
+    assert 'numberMatched' not in results
+    assert results['numberReturned'] == 10
+
+
 def test_geometry(config):
     p = ESRIServiceProvider(config)
 

tests/provider/test_geojson_provider.py

@@ -117,6 +117,20 @@ def test_get(fixture, config):
     assert 'Dinagat' in results['properties']['name']
 
 
+def test_no_count(fixture, config):
+    p = GeoJSONProvider(config)
+
+    results = p.query()
+    assert results['numberMatched'] == 1
+    assert results['numberReturned'] == 1
+
+    config['count'] = False
+    p = GeoJSONProvider(config)
+    results = p.query()
+    assert 'numberMatched' not in results
+    assert results['numberReturned'] == 1
+
+
 def test_get_not_existing_item_raise_exception(
     fixture, config
 ):

tests/provider/test_mongo_provider.py

@@ -111,6 +111,20 @@ def test_get(config):
     assert 'Reykjavik' in result['properties']['ls_name']
 
 
+def test_no_count(config):
+    p = MongoProvider(config)
+
+    results = p.query()
+    assert results['numberMatched'] == 243
+    assert results['numberReturned'] == 10
+
+    config['count'] = False
+    p = MongoProvider(config)
+    results = p.query()
+    assert 'numberMatched' not in results
+    assert results['numberReturned'] == 10
+
+
 def test_get_not_existing_item_raise_exception(config):
     """Testing query for a not existing object"""
     p = MongoProvider(config)

tests/provider/test_mysql_provider.py

@@ -132,6 +132,20 @@ def test_query_with_paging(config):
     assert feature_collection['numberReturned'] == ALL_ITEMS_IN_DB - 3
 
 
+def test_no_count(config):
+    """Test query with no count"""
+    p = MySQLProvider(config)
+    feature_collection = p.query()
+    assert feature_collection['numberMatched'] == 5
+    assert feature_collection['numberReturned'] == 5
+
+    config['count'] = False
+    p = MySQLProvider(config)
+    feature_collection = p.query()
+    assert 'numberMatched' not in feature_collection
+    assert feature_collection['numberReturned'] == 5
+
+
 def test_query_bbox(config):
     """Test query with a specified bounding box"""
     p = MySQLProvider(config)

tests/provider/test_postgresql_provider.py

@@ -216,6 +216,20 @@ def test_query_with_property_filter(config):
     assert feature_collection['numberReturned'] == 50
 
 
+def test_no_count(config):
+    """Test query with count disabled"""
+    p = PostgreSQLProvider(config)
+    results = p.query()
+    assert results['numberMatched'] == 14776
+    assert results['numberReturned'] == 10
+
+    config['count'] = False
+    p = PostgreSQLProvider(config)
+    results = p.query()
+    assert 'numberMatched' not in results
+    assert results['numberReturned'] == 10
+
+
 def test_query_with_paging(config):
     """Test query valid features with paging"""
     p = PostgreSQLProvider(config)

tests/provider/test_socrata_provider.py

@@ -191,6 +191,20 @@ def test_query(config, mock_socrata):
     assert results['numberMatched'] == 1006
 
 
+def test_no_count(config, mock_socrata):
+    p = SODAServiceProvider(config)
+
+    results = p.query()
+    assert results['numberMatched'] == 1006
+    assert results['numberReturned'] == 10
+
+    config['count'] = False
+    p = SODAServiceProvider(config)
+    results = p.query()
+    assert 'numberMatched' not in results
+    assert results['numberReturned'] == 10
+
+
 def test_geometry(config, mock_socrata):
     p = SODAServiceProvider(config)
 

tests/provider/test_tinydb_provider.py

@@ -194,6 +194,20 @@ def test_get(config):
     assert result['properties']['FLOW'] == 2.059999942779541
 
 
+def test_no_count(config):
+    p = TinyDBProvider(config)
+
+    results = p.query()
+    assert results['numberMatched'] == 50
+    assert results['numberReturned'] == 10
+
+    config['count'] = False
+    p = TinyDBProvider(config)
+    results = p.query()
+    assert 'numberMatched' not in results
+    assert results['numberReturned'] == 10
+
+
 def test_get_not_existing_item_raise_exception(config):
     """Testing query for a not existing object"""
     p = TinyDBProvider(config)