From 95e8854960b27661d6f28252ae982f5e03158af4 Mon Sep 17 00:00:00 2001 From: Benjamin Webb Date: Tue, 20 Jan 2026 19:29:55 -0500 Subject: [PATCH 1/2] Implement optional count in Feature Providers --- docs/source/configuration.rst | 1 + pygeoapi/provider/csv_.py | 5 +-- pygeoapi/provider/esri.py | 8 +++-- pygeoapi/provider/geojson.py | 3 +- pygeoapi/provider/mongo.py | 34 +++++++++++-------- pygeoapi/provider/socrata.py | 8 +++-- pygeoapi/provider/sql.py | 1 + pygeoapi/provider/tinydb_.py | 5 ++- .../schemas/config/pygeoapi-config-0.x.yml | 4 +++ tests/provider/test_csv__provider.py | 13 +++++++ tests/provider/test_esri_provider.py | 13 +++++++ tests/provider/test_geojson_provider.py | 14 ++++++++ tests/provider/test_mongo_provider.py | 14 ++++++++ tests/provider/test_mysql_provider.py | 14 ++++++++ tests/provider/test_postgresql_provider.py | 14 ++++++++ tests/provider/test_socrata_provider.py | 14 ++++++++ tests/provider/test_tinydb_provider.py | 14 ++++++++ tests/pygeoapi-test-config.yml | 1 + 18 files changed, 158 insertions(+), 22 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 8cab94f52..4b67a75fc 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -271,6 +271,7 @@ default. include_extra_query_parameters: false # include extra query parameters that are not part of the collection properties (default: false) # editable transactions: DO NOT ACTIVATE unless you have setup access control beyond pygeoapi editable: true # optional: if backend is writable, default is false + count: true # optional: perform a count query for collection queries, default is true # coordinate reference systems (CRS) section is optional # default CRSs are http://www.opengis.net/def/crs/OGC/1.3/CRS84 (coordinates without height) # and http://www.opengis.net/def/crs/OGC/1.3/CRS84h (coordinates with ellipsoidal height) diff --git a/pygeoapi/provider/csv_.py b/pygeoapi/provider/csv_.py index cffa54734..d66d6fd1b 100644 --- a/pygeoapi/provider/csv_.py +++ b/pygeoapi/provider/csv_.py @@ -194,8 +194,9 @@ def _load(self, offset=0, limit=10, resulttype='results', feature_collection['features'].append(feature) - feature_collection['numberMatched'] = \ - len(feature_collection['features']) + if self.count: + feature_collection['numberMatched'] = \ + len(feature_collection['features']) if identifier is not None and not found: return None diff --git a/pygeoapi/provider/esri.py b/pygeoapi/provider/esri.py index c84b6e1f1..f41171e43 100644 --- a/pygeoapi/provider/esri.py +++ b/pygeoapi/provider/esri.py @@ -156,9 +156,13 @@ def query(self, offset=0, limit=10, resulttype='results', fc = { 'type': 'FeatureCollection', 'features': [], - 'numberMatched': self._get_count(params) } + if self.count or resulttype == 'hits': + matched = self._get_count(params) + LOGGER.debug(f'Found {matched} result(s)') + fc['numberMatched'] = matched + if resulttype == 'hits': return fc @@ -168,7 +172,7 @@ def query(self, offset=0, limit=10, resulttype='results', params['resultOffset'] = offset params['resultRecordCount'] = limit - hits_ = min(limit, fc['numberMatched']) + hits_ = min(limit, matched) if self.count else limit fc['features'] = self._get_all(params, hits_) fc['numberReturned'] = len(fc['features']) diff --git a/pygeoapi/provider/geojson.py b/pygeoapi/provider/geojson.py index 9109c8a15..5062182fe 100644 --- a/pygeoapi/provider/geojson.py +++ b/pygeoapi/provider/geojson.py @@ -188,7 +188,8 @@ def query(self, offset=0, limit=10, resulttype='results', properties=properties, select_properties=select_properties) - data['numberMatched'] = len(data['features']) + if self.count or resulttype == 'hits': + data['numberMatched'] = len(data['features']) if resulttype == 'hits': data['features'] = [] diff --git a/pygeoapi/provider/mongo.py b/pygeoapi/provider/mongo.py index 6c1bab729..049174f3f 100644 --- a/pygeoapi/provider/mongo.py +++ b/pygeoapi/provider/mongo.py @@ -102,7 +102,6 @@ def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1, if sortList: featurecursor = featurecursor.sort(sortList) - matchCount = self.featuredb[self.collection].count_documents(filterObj) featurecursor.skip(skip) featurecursor.limit(maxitems) featurelist = list(featurecursor) @@ -111,7 +110,7 @@ def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1, if skip_geometry: item['geometry'] = None - return featurelist, matchCount + return featurelist @crs_transform def query(self, offset=0, limit=10, resulttype='results', @@ -144,20 +143,28 @@ def query(self, offset=0, limit=10, resulttype='results', ASCENDING if (sort['order'] == '+') else DESCENDING) for sort in sortby] - featurelist, matchcount = self._get_feature_list( - filterobj, sortList=sort_list, skip=offset, maxitems=limit, - skip_geometry=skip_geometry) - - if resulttype == 'hits': - featurelist = [] - feature_collection = { 'type': 'FeatureCollection', - 'features': featurelist, - 'numberMatched': matchcount, - 'numberReturned': len(featurelist) + 'features': [] } + if self.count or resulttype == 'hits': + matched = self.featuredb[self.collection].count_documents( + filterobj) + LOGGER.debug(f'Found {matched} result(s)') + feature_collection['numberMatched'] = matched + + if resulttype == 'hits': + return feature_collection + + featurelist = self._get_feature_list( + filterobj, sortList=sort_list, skip=offset, maxitems=limit, + skip_geometry=skip_geometry + ) + + feature_collection['features'] = featurelist + feature_collection['numberReturned'] = len(featurelist) + return feature_collection @crs_transform @@ -168,8 +175,7 @@ def get(self, identifier, **kwargs): :param identifier: feature id :returns: dict of single GeoJSON feature """ - featurelist, matchcount = self._get_feature_list( - {'_id': ObjectId(identifier)}) + featurelist = self._get_feature_list({'_id': ObjectId(identifier)}) if featurelist: return featurelist[0] else: diff --git a/pygeoapi/provider/socrata.py b/pygeoapi/provider/socrata.py index 4ffdd4f58..c96c207cc 100644 --- a/pygeoapi/provider/socrata.py +++ b/pygeoapi/provider/socrata.py @@ -123,10 +123,14 @@ def query(self, offset=0, limit=10, resulttype='results', fc = { 'type': 'FeatureCollection', - 'features': [], - 'numberMatched': self._get_count(params) + 'features': [] } + if self.count or resulttype == 'hits': + matched = self._get_count(params) + LOGGER.debug(f'Found {matched} result(s)') + fc['numberMatched'] = matched + if resulttype == 'hits': # Return hits LOGGER.debug('Returning hits') diff --git a/pygeoapi/provider/sql.py b/pygeoapi/provider/sql.py index a955f06db..55c3b5506 100644 --- a/pygeoapi/provider/sql.py +++ b/pygeoapi/provider/sql.py @@ -231,6 +231,7 @@ def query( crs_transform_out = get_transform_from_spec(crs_transform_spec) + response['numberReturned'] = 0 for item in ( results.order_by(*order_by_clauses).offset(offset).limit(limit) ): diff --git a/pygeoapi/provider/tinydb_.py b/pygeoapi/provider/tinydb_.py index 5453f70c8..38ebe0298 100644 --- a/pygeoapi/provider/tinydb_.py +++ b/pygeoapi/provider/tinydb_.py @@ -226,7 +226,10 @@ def query(self, offset=0, limit=10, resulttype='results', else: results = self.db.all() - feature_collection['numberMatched'] = len(results) + if self.count or resulttype == 'hits': + matched = len(results) + LOGGER.debug(f'Found {matched} result(s)') + feature_collection['numberMatched'] = matched if resulttype == 'hits': return feature_collection diff --git a/pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml index d772f000b..6ebd1f4fb 100644 --- a/pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml +++ b/pygeoapi/resources/schemas/config/pygeoapi-config-0.x.yml @@ -540,6 +540,10 @@ properties: type: boolean description: whether the resource is editable default: false + count: + type: boolean + description: whether to perform a count query for collection queries + default: true table: type: string description: table name for RDBMS-based providers diff --git a/tests/provider/test_csv__provider.py b/tests/provider/test_csv__provider.py index d4427e85c..481d197e9 100644 --- a/tests/provider/test_csv__provider.py +++ b/tests/provider/test_csv__provider.py @@ -144,6 +144,19 @@ def test_query(config): assert len(results['features'][0]['properties']) == 2 +def test_no_count(config): + p = CSVProvider(config) + results = p.query() + assert results['numberMatched'] == 5 + assert results['numberReturned'] == 5 + + config['count'] = False + p = CSVProvider(config) + results = p.query() + assert 'numberMatched' not in results + assert results['numberReturned'] == 5 + + def test_get_invalid_property(config): """Testing query for an invalid property name""" p = CSVProvider(config) diff --git a/tests/provider/test_esri_provider.py b/tests/provider/test_esri_provider.py index 65870de79..1be9bddb4 100644 --- a/tests/provider/test_esri_provider.py +++ b/tests/provider/test_esri_provider.py @@ -85,6 +85,19 @@ def test_query(config): assert results['numberMatched'] == 406 +def test_no_count(config): + p = ESRIServiceProvider(config) + results = p.query() + assert results['numberMatched'] == 406 + assert results['numberReturned'] == 10 + + config['count'] = False + p = ESRIServiceProvider(config) + results = p.query() + assert 'numberMatched' not in results + assert results['numberReturned'] == 10 + + def test_geometry(config): p = ESRIServiceProvider(config) diff --git a/tests/provider/test_geojson_provider.py b/tests/provider/test_geojson_provider.py index 7db4dd0fd..4961e3e5f 100644 --- a/tests/provider/test_geojson_provider.py +++ b/tests/provider/test_geojson_provider.py @@ -117,6 +117,20 @@ def test_get(fixture, config): assert 'Dinagat' in results['properties']['name'] +def test_no_count(fixture, config): + p = GeoJSONProvider(config) + + results = p.query() + assert results['numberMatched'] == 1 + assert results['numberReturned'] == 1 + + config['count'] = False + p = GeoJSONProvider(config) + results = p.query() + assert 'numberMatched' not in results + assert results['numberReturned'] == 1 + + def test_get_not_existing_item_raise_exception( fixture, config ): diff --git a/tests/provider/test_mongo_provider.py b/tests/provider/test_mongo_provider.py index 279f672d3..b4a7eeda0 100644 --- a/tests/provider/test_mongo_provider.py +++ b/tests/provider/test_mongo_provider.py @@ -111,6 +111,20 @@ def test_get(config): assert 'Reykjavik' in result['properties']['ls_name'] +def test_no_count(config): + p = MongoProvider(config) + + results = p.query() + assert results['numberMatched'] == 243 + assert results['numberReturned'] == 10 + + config['count'] = False + p = MongoProvider(config) + results = p.query() + assert 'numberMatched' not in results + assert results['numberReturned'] == 10 + + def test_get_not_existing_item_raise_exception(config): """Testing query for a not existing object""" p = MongoProvider(config) diff --git a/tests/provider/test_mysql_provider.py b/tests/provider/test_mysql_provider.py index 0f470d750..3aa7ed7d2 100644 --- a/tests/provider/test_mysql_provider.py +++ b/tests/provider/test_mysql_provider.py @@ -132,6 +132,20 @@ def test_query_with_paging(config): assert feature_collection['numberReturned'] == ALL_ITEMS_IN_DB - 3 +def test_no_count(config): + """Test query with no count""" + p = MySQLProvider(config) + feature_collection = p.query() + assert feature_collection['numberMatched'] == 5 + assert feature_collection['numberReturned'] == 5 + + config['count'] = False + p = MySQLProvider(config) + feature_collection = p.query() + assert 'numberMatched' not in feature_collection + assert feature_collection['numberReturned'] == 5 + + def test_query_bbox(config): """Test query with a specified bounding box""" p = MySQLProvider(config) diff --git a/tests/provider/test_postgresql_provider.py b/tests/provider/test_postgresql_provider.py index c27660caf..d52a846f7 100644 --- a/tests/provider/test_postgresql_provider.py +++ b/tests/provider/test_postgresql_provider.py @@ -216,6 +216,20 @@ def test_query_with_property_filter(config): assert feature_collection['numberReturned'] == 50 +def test_no_count(config): + """Test query with count disabled""" + p = PostgreSQLProvider(config) + results = p.query() + assert results['numberMatched'] == 14776 + assert results['numberReturned'] == 10 + + config['count'] = False + p = PostgreSQLProvider(config) + results = p.query() + assert 'numberMatched' not in results + assert results['numberReturned'] == 10 + + def test_query_with_paging(config): """Test query valid features with paging""" p = PostgreSQLProvider(config) diff --git a/tests/provider/test_socrata_provider.py b/tests/provider/test_socrata_provider.py index 51667ef4b..bfe8b9979 100644 --- a/tests/provider/test_socrata_provider.py +++ b/tests/provider/test_socrata_provider.py @@ -191,6 +191,20 @@ def test_query(config, mock_socrata): assert results['numberMatched'] == 1006 +def test_no_count(config, mock_socrata): + p = SODAServiceProvider(config) + + results = p.query() + assert results['numberMatched'] == 1006 + assert results['numberReturned'] == 10 + + config['count'] = False + p = SODAServiceProvider(config) + results = p.query() + assert 'numberMatched' not in results + assert results['numberReturned'] == 10 + + def test_geometry(config, mock_socrata): p = SODAServiceProvider(config) diff --git a/tests/provider/test_tinydb_provider.py b/tests/provider/test_tinydb_provider.py index 19d8d8f4d..109dbf4ab 100644 --- a/tests/provider/test_tinydb_provider.py +++ b/tests/provider/test_tinydb_provider.py @@ -194,6 +194,20 @@ def test_get(config): assert result['properties']['FLOW'] == 2.059999942779541 +def test_no_count(config): + p = TinyDBProvider(config) + + results = p.query() + assert results['numberMatched'] == 50 + assert results['numberReturned'] == 10 + + config['count'] = False + p = TinyDBProvider(config) + results = p.query() + assert 'numberMatched' not in results + assert results['numberReturned'] == 10 + + def test_get_not_existing_item_raise_exception(config): """Testing query for a not existing object""" p = TinyDBProvider(config) diff --git a/tests/pygeoapi-test-config.yml b/tests/pygeoapi-test-config.yml index 13dc63aa9..8dc3c1ec1 100644 --- a/tests/pygeoapi-test-config.yml +++ b/tests/pygeoapi-test-config.yml @@ -242,6 +242,7 @@ resources: - type: feature name: GeoJSON data: tests/data/ne_110m_lakes.geojson + count: false id_field: id crs: - http://www.opengis.net/def/crs/OGC/1.3/CRS84 From 9ecaa5d37cc5abdad6a763524e38ba5dc89efb9d Mon Sep 17 00:00:00 2001 From: Benjamin Webb Date: Tue, 20 Jan 2026 20:28:47 -0500 Subject: [PATCH 2/2] Revert collection entry --- tests/pygeoapi-test-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pygeoapi-test-config.yml b/tests/pygeoapi-test-config.yml index 8dc3c1ec1..13dc63aa9 100644 --- a/tests/pygeoapi-test-config.yml +++ b/tests/pygeoapi-test-config.yml @@ -242,7 +242,6 @@ resources: - type: feature name: GeoJSON data: tests/data/ne_110m_lakes.geojson - count: false id_field: id crs: - http://www.opengis.net/def/crs/OGC/1.3/CRS84