From 8c317d2deacd3c6f8cdcbd9883339e27deef7570 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Fri, 5 Dec 2025 13:28:21 +0100 Subject: [PATCH 1/3] MAINT: Removed dependency on direct mongo plugin --- dserver_dependency_graph_plugin/__init__.py | 5 +- dserver_dependency_graph_plugin/graph.py | 13 +-- dserver_dependency_graph_plugin/utils.py | 117 ++++++++++++++++++++ dserver_dependency_graph_plugin/version.py | 34 ++++++ pyproject.toml | 2 +- 5 files changed, 161 insertions(+), 10 deletions(-) create mode 100644 dserver_dependency_graph_plugin/utils.py create mode 100644 dserver_dependency_graph_plugin/version.py diff --git a/dserver_dependency_graph_plugin/__init__.py b/dserver_dependency_graph_plugin/__init__.py index 3f9c937..c3d09af 100644 --- a/dserver_dependency_graph_plugin/__init__.py +++ b/dserver_dependency_graph_plugin/__init__.py @@ -33,7 +33,7 @@ from dservercore import AuthenticationError, ExtensionABC from dservercore.sql_models import DatasetSchema from dservercore.utils import _preprocess_privileges -from dserver_direct_mongo_plugin.utils import _dict_to_mongo_query +from .utils import _dict_to_mongo_query from .schemas import DependencyKeysSchema @@ -246,7 +246,8 @@ def dependency_graph_by_user_and_uuid(username, uuid, dependency_keys=Config.DEP mongo_aggregation = query_dependency_graph(pre_query=pre_query, post_query=post_query, dependency_keys=dependency_keys, - mongo_dependency_view=dependency_view) + mongo_dependency_view=dependency_view, + mongo_collection=current_app.config['MONGO_COLLECTION']) logger.debug("Constructed mongo aggregation: {}".format(mongo_aggregation)) cx = DependencyGraphExtension.db[current_app.config['MONGO_COLLECTION']].aggregate(mongo_aggregation) diff --git a/dserver_dependency_graph_plugin/graph.py b/dserver_dependency_graph_plugin/graph.py index 6b3aa6e..68eefd5 100644 --- a/dserver_dependency_graph_plugin/graph.py +++ b/dserver_dependency_graph_plugin/graph.py @@ -1,7 +1,6 @@ """Aggregation pipelines for graph operations.""" -from dserver_dependency_graph_plugin.config import Config as dependency_graph_plugin_config -from dserver_direct_mongo_plugin.config import Config as direct_mongo_plugin_config +from .config import Config # a regular expression to filter valid v4 UUIDs UUID_v4_REGEX = '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[4][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}' @@ -9,7 +8,7 @@ # most of those 'functions' are pretty static and just wrapped in function # definitions for convenience. -def unwind_dependencies(dependency_keys=dependency_graph_plugin_config.DEPENDENCY_KEYS): +def unwind_dependencies(dependency_keys=Config.DEPENDENCY_KEYS): """Create parallel aggregation pipelines for unwinding all configured dependency keys.""" parallel_aggregations = [] @@ -41,7 +40,7 @@ def unwind_dependencies(dependency_keys=dependency_graph_plugin_config.DEPENDENC return parallel_aggregations -def merge_dependencies(dependency_keys=dependency_graph_plugin_config.DEPENDENCY_KEYS): +def merge_dependencies(dependency_keys=Config.DEPENDENCY_KEYS): """Aggregate (directed) dependency graph edges. All configured dependency keys are merged in a key-agnostic 'dependencies' @@ -117,7 +116,7 @@ def group_inverse_dependencies(): return aggregation -def build_undirected_adjecency_lists(dependency_keys=dependency_graph_plugin_config.DEPENDENCY_KEYS): +def build_undirected_adjecency_lists(dependency_keys=Config.DEPENDENCY_KEYS): """Aggregate undirected adjacency lists.""" aggregation = [ *merge_dependencies(dependency_keys), @@ -200,8 +199,8 @@ def build_undirected_adjecency_lists(dependency_keys=dependency_graph_plugin_con # behavior would be to yield all redundant dataset entries for a uuid. def query_dependency_graph(mongo_dependency_view, pre_query, post_query=None, - dependency_keys=dependency_graph_plugin_config.DEPENDENCY_KEYS, - mongo_collection=direct_mongo_plugin_config.MONGO_COLLECTION): + dependency_keys=Config.DEPENDENCY_KEYS, + mongo_collection=None): """Aggregation pipeline for querying dependency view on datasets collection. :param pre_query: selects all documents for whicht to query the dependency graph. diff --git a/dserver_dependency_graph_plugin/utils.py b/dserver_dependency_graph_plugin/utils.py new file mode 100644 index 0000000..4fd2de6 --- /dev/null +++ b/dserver_dependency_graph_plugin/utils.py @@ -0,0 +1,117 @@ +"""Utility functions for MongoDB query construction. + +These functions were originally part of dserver-direct-mongo-plugin but are +copied here to decouple the dependency-graph-plugin from that package. +""" + +import logging + +logger = logging.getLogger(__name__) + + +VALID_MONGO_QUERY_KEYS = ( + "free_text", + "creator_usernames", + "base_uris", + "uuids", + "tags", +) + +MONGO_QUERY_LIST_KEYS = ( + "creator_usernames", + "base_uris", + "uuids", + "tags", +) + + +def _dict_to_mongo(query_dict): + """Convert a query dictionary to a MongoDB query. + + :param query_dict: Dictionary with query parameters + :returns: MongoDB query dictionary + """ + def _sanitise(query_dict): + for key in list(query_dict.keys()): + if key not in VALID_MONGO_QUERY_KEYS: + del query_dict[key] + for lk in MONGO_QUERY_LIST_KEYS: + if lk in query_dict: + if len(query_dict[lk]) == 0: + del query_dict[lk] + + def _deal_with_possible_or_statment(a_list, key): + if len(a_list) == 1: + return {key: a_list[0]} + else: + return {"$or": [{key: v} for v in a_list]} + + def _deal_with_possible_and_statement(a_list, key): + if len(a_list) == 1: + return {key: a_list[0]} + else: + return {key: {"$all": a_list}} + + _sanitise(query_dict) + + sub_queries = [] + if "free_text" in query_dict: + sub_queries.append({"$text": {"$search": query_dict["free_text"]}}) + if "creator_usernames" in query_dict: + sub_queries.append( + _deal_with_possible_or_statment( + query_dict["creator_usernames"], "creator_username" + ) + ) + if "base_uris" in query_dict: + sub_queries.append( + _deal_with_possible_or_statment(query_dict["base_uris"], "base_uri") + ) + if "uuids" in query_dict: + sub_queries.append(_deal_with_possible_or_statment(query_dict["uuids"], "uuid")) + if "tags" in query_dict: + sub_queries.append( + _deal_with_possible_and_statement(query_dict["tags"], "tags") + ) + + if len(sub_queries) == 0: + return {} + elif len(sub_queries) == 1: + return sub_queries[0] + else: + return {"$and": [q for q in sub_queries]} + + +def _dict_to_mongo_query(query_dict): + """Construct mongo query, allowing embedding of a raw mongo query. + + Converts a query dictionary to a MongoDB query format. If the query_dict + contains a 'query' key with a dict value, that raw MongoDB query is + merged with the constructed query. + + :param query_dict: Dictionary with query parameters. May contain: + - free_text: Text search string + - creator_usernames: List of creator usernames + - base_uris: List of base URIs + - uuids: List of UUIDs + - tags: List of tags + - query: Raw MongoDB query dict (optional) + :returns: MongoDB query dictionary + """ + if "query" in query_dict and isinstance(query_dict["query"], dict): + raw_mongo = query_dict["query"] + del query_dict["query"] + else: + raw_mongo = {} + + mongo_query = _dict_to_mongo(query_dict) + + if len(raw_mongo) > 0 and len(mongo_query) == 0: + mongo_query = raw_mongo + elif len(raw_mongo) > 0 and len(mongo_query) == 1 and "$and" in mongo_query: + mongo_query["$and"].append(raw_mongo) + elif len(raw_mongo) > 0: + mongo_query = {"$and": [mongo_query, raw_mongo]} + + logger.debug("Constructed mongo query: {}".format(mongo_query)) + return mongo_query diff --git a/dserver_dependency_graph_plugin/version.py b/dserver_dependency_graph_plugin/version.py new file mode 100644 index 0000000..88daef1 --- /dev/null +++ b/dserver_dependency_graph_plugin/version.py @@ -0,0 +1,34 @@ +# file generated by setuptools-scm +# don't change, don't track in version control + +__all__ = [ + "__version__", + "__version_tuple__", + "version", + "version_tuple", + "__commit_id__", + "commit_id", +] + +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple + from typing import Union + + VERSION_TUPLE = Tuple[Union[int, str], ...] + COMMIT_ID = Union[str, None] +else: + VERSION_TUPLE = object + COMMIT_ID = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE +commit_id: COMMIT_ID +__commit_id__: COMMIT_ID + +__version__ = version = '0.4.3.dev1' +__version_tuple__ = version_tuple = (0, 4, 3, 'dev1') + +__commit_id__ = commit_id = 'gcde515931' diff --git a/pyproject.toml b/pyproject.toml index d168a75..569df7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dynamic = ["version"] dependencies = [ "dtoolcore>=3.18.0", "dservercore>=0.20.0", - "dserver-direct-mongo-plugin", + "pymongo", ] [project.optional-dependencies] From 652501f643e73bf5a1f87788d4c4b707dde34ce7 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 7 Dec 2025 23:25:45 +0100 Subject: [PATCH 2/3] BUILD: Switched build system to flit --- dserver_dependency_graph_plugin/version.py | 6 ++--- pyproject.toml | 29 +++++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/dserver_dependency_graph_plugin/version.py b/dserver_dependency_graph_plugin/version.py index 88daef1..6825d3b 100644 --- a/dserver_dependency_graph_plugin/version.py +++ b/dserver_dependency_graph_plugin/version.py @@ -28,7 +28,7 @@ commit_id: COMMIT_ID __commit_id__: COMMIT_ID -__version__ = version = '0.4.3.dev1' -__version_tuple__ = version_tuple = (0, 4, 3, 'dev1') +__version__ = version = '0.4.3.dev2' +__version_tuple__ = version_tuple = (0, 4, 3, 'dev2') -__commit_id__ = commit_id = 'gcde515931' +__commit_id__ = commit_id = 'g8c317d2de' diff --git a/pyproject.toml b/pyproject.toml index 569df7a..27b687f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,21 +1,22 @@ [build-system] -requires = ["setuptools>=42", "setuptools_scm[toml]>=6.3"] -build-backend = "setuptools.build_meta" +requires = ["flit_scm"] +build-backend = "flit_scm:buildapi" [project] name = "dserver-dependency-graph-plugin" description = "dserver plugin for receiving s3 notifications on updated objects." readme = "README.rst" -license = {file = "LICENSE"} +license = {text = "MIT"} authors = [ {name = "Johannes L. Hörmann", email = "johannes.laurin@gmail.com"}, ] dynamic = ["version"] +requires-python = ">=3.8" dependencies = [ - "dtoolcore>=3.18.0", - "dservercore>=0.20.0", - "pymongo", - ] + "dtoolcore>=3.18.0", + "dservercore>=0.20.0", + "pymongo", +] [project.optional-dependencies] test = [ @@ -31,13 +32,17 @@ Documentation = "https://github.com/livMatS/dserver-dependency-graph-plugin/blob Repository = "https://github.com/livMatS/dserver-dependency-graph-plugin" Changelog = "https://github.com/livMatS/dserver-dependency-graph-plugin/blob/main/CHANGELOG.rst" +[project.entry-points."dservercore.extension"] +DependencyGraphExtension = "dserver_dependency_graph_plugin:DependencyGraphExtension" + +[tool.flit.module] +name = "dserver_dependency_graph_plugin" + [tool.setuptools_scm] version_scheme = "guess-next-dev" local_scheme = "no-local-version" write_to = "dserver_dependency_graph_plugin/version.py" -[tool.setuptools] -packages = ["dserver_dependency_graph_plugin"] - -[project.entry-points."dservercore.extension"] -"DependencyGraphExtension" = "dserver_dependency_graph_plugin:DependencyGraphExtension" +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--cov=dserver_dependency_graph_plugin --cov-report=term-missing" From 4d201d6b5d89c25cc41994c5470c2033d77aa73d Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Mon, 8 Dec 2025 16:16:41 +0100 Subject: [PATCH 3/3] MAINT: Mongo configuration --- dserver_dependency_graph_plugin/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dserver_dependency_graph_plugin/config.py b/dserver_dependency_graph_plugin/config.py index d50ca9c..98b9c12 100644 --- a/dserver_dependency_graph_plugin/config.py +++ b/dserver_dependency_graph_plugin/config.py @@ -8,6 +8,11 @@ class Config(object): + # MongoDB connection settings + # These are required for the dependency graph plugin to connect to MongoDB + MONGO_URI = os.environ.get("MONGO_URI") + MONGO_DB = os.environ.get("MONGO_DB") + MONGO_COLLECTION = os.environ.get("MONGO_COLLECTION") # If enabled, the underlying database will offer dependency graph views on # the server's default collection. Those views offer on-the-fly-generated # collections of undirected per-dataset adjacency lists in order to