From cf8b8bde791e92145a0ce77d9d1c7d00ce3834a0 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 31 Jan 2026 15:23:54 -0500 Subject: [PATCH 1/4] move provider util functions to pygeoapi.provider --- pygeoapi/api/__init__.py | 5 +- pygeoapi/api/coverages.py | 5 +- pygeoapi/api/environmental_data_retrieval.py | 8 +-- pygeoapi/api/itemtypes.py | 7 +-- pygeoapi/api/maps.py | 6 +- pygeoapi/api/stac.py | 7 +-- pygeoapi/api/tiles.py | 6 +- pygeoapi/provider/__init__.py | 63 +++++++++++++++++++- pygeoapi/util.py | 58 +----------------- 9 files changed, 81 insertions(+), 84 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index ffc0a8810..c4bf94fd4 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -61,13 +61,14 @@ from pygeoapi.log import setup_logger from pygeoapi.plugin import load_plugin from pygeoapi.process.manager.base import get_manager +from pygeoapi.provider import ( + filter_providers_by_type, get_provider_by_type, get_provider_default) from pygeoapi.provider.base import ( ProviderConnectionError, ProviderGenericError, ProviderTypeError) from pygeoapi.util import ( TEMPLATESDIR, UrlPrefetcher, dategetter, - filter_dict_by_key_value, filter_providers_by_type, get_api_rules, - get_base_url, get_provider_by_type, get_provider_default, get_typed_value, + filter_dict_by_key_value, get_api_rules, get_base_url, get_typed_value, render_j2_template, to_json, get_choice_from_headers, get_from_headers, get_dataset_formatters ) diff --git a/pygeoapi/api/coverages.py b/pygeoapi/api/coverages.py index b44a15fce..327744a01 100644 --- a/pygeoapi/api/coverages.py +++ b/pygeoapi/api/coverages.py @@ -46,9 +46,8 @@ from pygeoapi.openapi import get_oas_30_parameters from pygeoapi.plugin import load_plugin from pygeoapi.provider.base import ProviderGenericError, ProviderTypeError -from pygeoapi.util import ( - filter_dict_by_key_value, get_provider_by_type, to_json -) +from pygeoapi.provider import get_provider_by_type +from pygeoapi.util import filter_dict_by_key_value, to_json from . import ( APIRequest, API, F_JSON, SYSTEM_LOCALE, validate_bbox, validate_datetime, diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index 853559156..7e1ef1f51 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -53,12 +53,12 @@ from pygeoapi.crs import (create_crs_transform_spec, set_content_crs_header) from pygeoapi.openapi import get_oas_30_parameters from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider import filter_providers_by_type, get_provider_by_type from pygeoapi.provider.base import ( ProviderGenericError, ProviderItemNotFoundError) -from pygeoapi.util import ( - filter_providers_by_type, get_dataset_formatters, get_provider_by_type, - get_typed_value, render_j2_template, to_json, filter_dict_by_key_value -) +from pygeoapi.util import (get_dataset_formatters, get_typed_value, + render_j2_template, to_json, + filter_dict_by_key_value) from . import (APIRequest, API, F_COVERAGEJSON, F_HTML, F_JSON, F_JSONLD, validate_datetime, validate_bbox) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 19306f57e..2aa85a973 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -58,14 +58,13 @@ from pygeoapi.linked_data import geojson2jsonld from pygeoapi.openapi import get_oas_30_parameters from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider import filter_providers_by_type, get_provider_by_type from pygeoapi.provider.base import ( ProviderGenericError, ProviderItemNotFoundError, ProviderTypeError, SchemaType) -from pygeoapi.util import (filter_providers_by_type, to_json, - filter_dict_by_key_value, str2bool, - get_provider_by_type, render_j2_template, - get_dataset_formatters) +from pygeoapi.util import (to_json, filter_dict_by_key_value, str2bool, + render_j2_template, get_dataset_formatters) from . import ( APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py index 888073938..bcdda4b7a 100644 --- a/pygeoapi/api/maps.py +++ b/pygeoapi/api/maps.py @@ -46,13 +46,11 @@ from pygeoapi.crs import transform_bbox from pygeoapi.openapi import get_oas_30_parameters from pygeoapi.plugin import load_plugin +from pygeoapi.provider import filter_providers_by_type, get_provider_by_type from pygeoapi.provider.base import ( ProviderGenericError, ProviderInvalidDataError ) -from pygeoapi.util import ( - get_provider_by_type, to_json, filter_providers_by_type, - filter_dict_by_key_value -) +from pygeoapi.util import to_json, filter_dict_by_key_value from . import ( APIRequest, API, F_JSON, FORMAT_TYPES, validate_datetime, validate_subset diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py index 37dfbedc9..a9227da26 100644 --- a/pygeoapi/api/stac.py +++ b/pygeoapi/api/stac.py @@ -51,13 +51,12 @@ from pygeoapi.api import itemtypes as itemtypes_api from pygeoapi.plugin import load_plugin +from pygeoapi.provider import get_provider_by_type from pygeoapi.provider.base import ( ProviderConnectionError, ProviderNotFoundError, ProviderTypeError ) -from pygeoapi.util import ( - filter_dict_by_key_value, get_current_datetime, get_provider_by_type, - render_j2_template, to_json -) +from pygeoapi.util import (filter_dict_by_key_value, get_current_datetime, + render_j2_template, to_json) from . import APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index 2e31a6faf..fb8a39dbb 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -46,15 +46,13 @@ from pygeoapi.plugin import load_plugin from pygeoapi.models.provider.base import (TilesMetadataFormat, TileMatrixSetEnum) +from pygeoapi.provider import get_provider_by_type, filter_providers_by_type from pygeoapi.provider.base import ( ProviderGenericError, ProviderTypeError ) from pygeoapi.provider.tile import ProviderTileNotFoundError -from pygeoapi.util import ( - get_provider_by_type, to_json, filter_dict_by_key_value, - filter_providers_by_type, render_j2_template -) +from pygeoapi.util import to_json, filter_dict_by_key_value, render_j2_template from . import ( APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML, SYSTEM_LOCALE, F_JSONLD diff --git a/pygeoapi/provider/__init__.py b/pygeoapi/provider/__init__.py index 7595c47ca..1ebd319cf 100644 --- a/pygeoapi/provider/__init__.py +++ b/pygeoapi/provider/__init__.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2019 Tom Kralidis +# Copyright (c) 2026 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -27,4 +27,63 @@ # # ================================================================= -"""Provider module containing the plugins wrapping data sources""" +import logging + +from pygeoapi.provider.base import ProviderTypeError + +LOGGER = logging.getLogger(__name__) + + +def filter_providers_by_type(providers: list, type: str) -> dict: + """ + helper function to filter a list of providers by type + + :param providers: ``list`` + :param type: str + + :returns: filtered ``dict`` provider + """ + + providers_ = {provider['type']: provider for provider in providers} + return providers_.get(type) + + +def get_provider_by_type(providers: list, provider_type: str) -> dict: + """ + helper function to load a provider by a provider type + + :param providers: ``list`` of providers + :param provider_type: type of provider (e.g. feature) + + :returns: provider based on type + """ + + LOGGER.debug(f'Searching for provider type {provider_type}') + try: + p = (next(d for i, d in enumerate(providers) + if d['type'] == provider_type)) + except (RuntimeError, StopIteration): + raise ProviderTypeError('Invalid provider type requested') + + return p + + +def get_provider_default(providers: list) -> dict: + """ + helper function to get a resource's default provider + + :param providers: ``list`` of providers + + :returns: filtered ``dict`` + """ + + try: + default = (next(d for i, d in enumerate(providers) if 'default' in d + and d['default'])) + LOGGER.debug('found default provider type') + except StopIteration: + LOGGER.debug('no default provider type. Returning first provider') + default = providers[0] + + LOGGER.debug(f"Default provider: {default['type']}") + return default diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 44406e5b5..c91a4c3e0 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -65,8 +65,7 @@ from pygeoapi import l10n from pygeoapi.models import config as config_models from pygeoapi.plugin import load_plugin, PLUGINS -from pygeoapi.provider.base import ProviderTypeError - +from pygeoapi.provider import get_provider_default LOGGER = logging.getLogger(__name__) @@ -527,61 +526,6 @@ def filter_dict_by_key_value(dict_: dict, key: str, value: str) -> dict: return {k: v for (k, v) in dict_.items() if v[key] == value} -def filter_providers_by_type(providers: list, type: str) -> dict: - """ - helper function to filter a list of providers by type - - :param providers: ``list`` - :param type: str - - :returns: filtered ``dict`` provider - """ - - providers_ = {provider['type']: provider for provider in providers} - return providers_.get(type) - - -def get_provider_by_type(providers: list, provider_type: str) -> dict: - """ - helper function to load a provider by a provider type - - :param providers: ``list`` of providers - :param provider_type: type of provider (e.g. feature) - - :returns: provider based on type - """ - - LOGGER.debug(f'Searching for provider type {provider_type}') - try: - p = (next(d for i, d in enumerate(providers) - if d['type'] == provider_type)) - except (RuntimeError, StopIteration): - raise ProviderTypeError('Invalid provider type requested') - - return p - - -def get_provider_default(providers: list) -> dict: - """ - helper function to get a resource's default provider - - :param providers: ``list`` of providers - - :returns: filtered ``dict`` - """ - - try: - default = (next(d for i, d in enumerate(providers) if 'default' in d - and d['default'])) - LOGGER.debug('found default provider type') - except StopIteration: - LOGGER.debug('no default provider type. Returning first provider') - default = providers[0] - - LOGGER.debug(f"Default provider: {default['type']}") - return default - - class ProcessExecutionMode(Enum): sync_execute = 'sync-execute' async_execute = 'async-execute' From 90e7d456b7d390e5bd386c1d0fc65b6b11346105 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 31 Jan 2026 15:33:27 -0500 Subject: [PATCH 2/4] fix util tests --- tests/other/test_util.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/other/test_util.py b/tests/other/test_util.py index 6d3d70aa6..2876844c2 100644 --- a/tests/other/test_util.py +++ b/tests/other/test_util.py @@ -38,6 +38,7 @@ from pygeoapi import util from pygeoapi.api import __version__ +from pygeoapi.provider import get_provider_by_type, get_provider_default from pygeoapi.provider.base import ProviderTypeError from ..util import get_test_file_path @@ -181,7 +182,7 @@ def test_filter_dict_by_key_value(config): def test_get_provider_by_type(config): - p = util.get_provider_by_type(config['resources']['obs']['providers'], + p = get_provider_by_type(config['resources']['obs']['providers'], 'feature') assert isinstance(p, dict) @@ -189,17 +190,17 @@ def test_get_provider_by_type(config): assert p['name'] == 'CSV' with pytest.raises(ProviderTypeError): - p = util.get_provider_by_type(config['resources']['obs']['providers'], + p = get_provider_by_type(config['resources']['obs']['providers'], 'something-else') def test_get_provider_default(config): - pd = util.get_provider_default(config['resources']['obs']['providers']) + pd = get_provider_default(config['resources']['obs']['providers']) assert pd['type'] == 'feature' assert pd['name'] == 'CSV' - pd = util.get_provider_default(config['resources']['obs']['providers']) + pd = get_provider_default(config['resources']['obs']['providers']) def test_read_data(): From 99059c72da9ebf1ed4187e126180321418a73d95 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 31 Jan 2026 15:34:58 -0500 Subject: [PATCH 3/4] fix util tests --- tests/other/test_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/other/test_util.py b/tests/other/test_util.py index 2876844c2..4e54840c6 100644 --- a/tests/other/test_util.py +++ b/tests/other/test_util.py @@ -183,7 +183,7 @@ def test_filter_dict_by_key_value(config): def test_get_provider_by_type(config): p = get_provider_by_type(config['resources']['obs']['providers'], - 'feature') + 'feature') assert isinstance(p, dict) assert p['type'] == 'feature' @@ -191,7 +191,7 @@ def test_get_provider_by_type(config): with pytest.raises(ProviderTypeError): p = get_provider_by_type(config['resources']['obs']['providers'], - 'something-else') + 'something-else') def test_get_provider_default(config): From ef67e0bbbac02adf78e265293e9d62bd3b520969 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 31 Jan 2026 15:43:37 -0500 Subject: [PATCH 4/4] test str2bool from pygeoapi.provider.sql --- pygeoapi/provider/sql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pygeoapi/provider/sql.py b/pygeoapi/provider/sql.py index cba5abaea..631d41c99 100644 --- a/pygeoapi/provider/sql.py +++ b/pygeoapi/provider/sql.py @@ -91,6 +91,7 @@ ProviderQueryError, ProviderItemNotFoundError ) +from pygeoapi.util import str2bool LOGGER = logging.getLogger(__name__) @@ -127,6 +128,7 @@ def __init__( self.id_field = provider_def['id_field'] self.geom = provider_def.get('geom_field', 'geom') self.driver_name = driver_name + self.count = str2bool(provider_def.get('count', True)) LOGGER.debug(f'Name: {self.name}') LOGGER.debug(f'Table: {self.table}')