From 03a747772b50e4f54d4e67f660ab80450e71a3a3 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Wed, 7 Jan 2026 23:58:39 +0000 Subject: [PATCH 1/5] Implement middleware to make url_for available from a context variable. This adds: * a middleware function that pushes the `url_for` method of the `Request` object to a context variable. * a `pydantic`-compatible class that calls `url_for` to generate URLs at serialisation time. * a test harness that allows this to be used outside of an HTTP request handler. The middleware is tested on its own and in the context of a FastAPI app, but isolated from the rest of LabThings. --- src/labthings_fastapi/exceptions.py | 11 ++ src/labthings_fastapi/middleware/__init__.py | 1 + src/labthings_fastapi/middleware/url_for.py | 164 +++++++++++++++++++ src/labthings_fastapi/testing.py | 10 ++ tests/test_middleware_url_for.py | 105 ++++++++++++ 5 files changed, 291 insertions(+) create mode 100644 src/labthings_fastapi/middleware/__init__.py create mode 100644 src/labthings_fastapi/middleware/url_for.py create mode 100644 tests/test_middleware_url_for.py diff --git a/src/labthings_fastapi/exceptions.py b/src/labthings_fastapi/exceptions.py index 42a17094..0cbcf419 100644 --- a/src/labthings_fastapi/exceptions.py +++ b/src/labthings_fastapi/exceptions.py @@ -147,6 +147,17 @@ class NoBlobManagerError(RuntimeError): """ +class NoUrlForContextError(RuntimeError): + """Raised if URLFor is serialised without a url_for context variable being set. + + This usually indicates that URLFor is being serialised somewhere other than in + an HTTP response, + for example in test code or in a background task. In these cases, you should + set up the url_for context variable manually, for example using the + `.testing.use_dummy_url_for` context manager. + """ + + class UnsupportedConstraintError(ValueError): """A constraint argument is not supported. diff --git a/src/labthings_fastapi/middleware/__init__.py b/src/labthings_fastapi/middleware/__init__.py new file mode 100644 index 00000000..160876de --- /dev/null +++ b/src/labthings_fastapi/middleware/__init__.py @@ -0,0 +1 @@ +"""Middleware for use with LabThings.""" diff --git a/src/labthings_fastapi/middleware/url_for.py b/src/labthings_fastapi/middleware/url_for.py new file mode 100644 index 00000000..511b22ed --- /dev/null +++ b/src/labthings_fastapi/middleware/url_for.py @@ -0,0 +1,164 @@ +r"""Middleware to make url_for available as a context variable. + +There are several places in LabThings where we need to be able to include URLs +to other endpoints in the LabThings server, most notably in the output of +Actions. For example, if an Action outputs a `.Blob`\ , the URL to download +that `.Blob` would need to be generated. + +Actions are particularly complicated, as they are often invoked by one HTTP +request, and polled by subsequent requests. In order to ensure that the URL +we generate is consistent with the URL being requested, we should always use +the ``url_for`` method from the HTTP request we are responding to. This means +it is, in general, not a great idea to generate URLs within an Action and hold +on to them as strings. While it will work most of the time, it would be better +to store the endpoint name, and only convert it to a URL when the action's +output is serialised by FastAPI. + +This module includes a `.ContextVar` for the ``url_for`` function, and provides +a middleware function that sets the context variable for every request, and a +custom type that works with `pydantic` to convert endpoint names to URLs at +serialisation time. +""" + +from collections.abc import Awaitable, Callable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any, Self +from fastapi import Request, Response +from pydantic import GetCoreSchemaHandler +from pydantic.networks import AnyUrl +from pydantic_core import core_schema +from starlette.datastructures import URL + +from labthings_fastapi.exceptions import NoUrlForContextError + +url_for_ctx: ContextVar[Callable[..., URL]] = ContextVar("url_for_ctx") +"""Context variable storing the url_for function for the current request.""" + + +@contextmanager +def set_url_for_context( + url_for_function: Callable[..., URL], +) -> Iterator[None]: + """Set the url_for context variable for the duration of the context. + + :param url_for_function: The url_for function to set in the context variable. + """ + token = url_for_ctx.set(url_for_function) + try: + yield + finally: + url_for_ctx.reset(token) + + +def dummy_url_for(endpoint: str, **params: Any) -> URL: + r"""Generate a fake URL as a placeholder for a real ``url_for`` function. + + This is intended for use in test code. + + :param endpoint: The name of the endpoint. + :param \**params: The path parameters. + :return: A fake URL. + """ + param_str = "&".join(f"{k}={v}" for k, v in params.items()) + return URL(f"urlfor://{endpoint}/?{param_str}") + + +def url_for(endpoint_name: str, **params: Any) -> URL: + r"""Get a URL for the given endpoint name and path parameters. + + This function uses the ``url_for`` function stored in a context variable + to convert endpoint names and parameters to URLs. It is intended to have + the same signature as `fastapi.Request.url_for`\ . + + :param endpoint_name: The name of the endpoint to generate a URL for. + :param \**params: The path parameters to use in the URL. + :return: The generated URL. + :raises NoUrlForContextError: if there is no url_for function in the context. + """ + try: + url_for_func = url_for_ctx.get() + except LookupError as err: + raise NoUrlForContextError("No url_for context available.") from err + return url_for_func(endpoint_name, **params) + + +async def url_for_middleware( + request: Request, call_next: Callable[[Request], Awaitable[Response]] +) -> Response: + """Middleware to set the url_for context variable for each request. + + This middleware retrieves the ``url_for`` function from the incoming + request, and sets it in the context variable for the duration of the + request. + + :param request: The incoming FastAPI request. + :param call_next: The next middleware or endpoint handler to call. + :return: The response from the next handler. + """ + token = url_for_ctx.set(request.url_for) + try: + response = await call_next(request) + finally: + url_for_ctx.reset(token) + return response + + +class URLFor: + """A pydantic-compatible type that converts endpoint names to URLs.""" + + def __init__(self, endpoint_name: str, **params: Any) -> None: + r"""Create a URLFor instance. + + :param endpoint_name: The name of the endpoint to generate a URL for. + :param \**params: The path parameters to use in the URL. + """ + self.endpoint_name = endpoint_name + self.params = params + + def __str__(self) -> str: + """Convert the URLFor instance to a URL string. + + :return: The generated URL as a string. + """ + url = url_for(self.endpoint_name, **self.params) + return str(url) + + @classmethod + def __get_pydantic_core_schema__( + cls, source: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + """Get the pydantic core schema for the URLFor type. + + This magic method allows `pydantic` to serialise URLFor + instances, and generate a JSONSchema for them. Currently, + URLFor instances may not be validated from strings, and + attempting to do so will raise an error. + + The "core schema" we generate describes the field as a + string, and serialises it by calling ``str(obj)`` which in + turn calls our ``__str__`` method to generate the URL. + + :param source: The source type being converted. + :param handler: The pydantic core schema handler. + :return: The pydantic core schema for the URLFor type. + """ + return core_schema.no_info_wrap_validator_function( + cls._validate, + AnyUrl.__get_pydantic_core_schema__(AnyUrl, handler), + serialization=core_schema.to_string_ser_schema(when_used="always"), + ) + + @classmethod + def _validate(cls, value: Any, handler: Callable[[Any], Self]) -> Self: + """Validate and convert a value to a URLFor instance. + + :param value: The value to validate. + :param handler: The handler to convert the value if needed. + :return: The validated URLFor instance. + :raises TypeError: if the value is not a URLFor instance. + """ + if isinstance(value, cls): + return value + else: + raise TypeError("URLFor instances may not be created from strings.") diff --git a/src/labthings_fastapi/testing.py b/src/labthings_fastapi/testing.py index 57f99bf3..02f51ca4 100644 --- a/src/labthings_fastapi/testing.py +++ b/src/labthings_fastapi/testing.py @@ -1,7 +1,9 @@ """Test harnesses to help with writitng tests for things..""" from __future__ import annotations +from collections.abc import Iterator from concurrent.futures import Future +from contextlib import contextmanager from typing import ( TYPE_CHECKING, Any, @@ -18,6 +20,7 @@ from .utilities import class_attributes from .thing_slots import ThingSlot from .thing_server_interface import ThingServerInterface +from .middleware.url_for import set_url_for_context, dummy_url_for if TYPE_CHECKING: from .thing import Thing @@ -217,3 +220,10 @@ def _mock_slots(thing: Thing) -> None: for _attr_name, attr in class_attributes(thing): if isinstance(attr, ThingSlot): attr.connect(thing, mocks, ...) + + +@contextmanager +def use_dummy_url_for() -> Iterator[None]: + """Use the dummy URL for function in the context variable.""" + with set_url_for_context(dummy_url_for): + yield diff --git a/tests/test_middleware_url_for.py b/tests/test_middleware_url_for.py new file mode 100644 index 00000000..a2c01e74 --- /dev/null +++ b/tests/test_middleware_url_for.py @@ -0,0 +1,105 @@ +"""Test the URLFor class and associated supporting code.""" + +import pytest +from pydantic import BaseModel +from pydantic_core import PydanticSerializationError +from fastapi import FastAPI +from starlette.testclient import TestClient + +from labthings_fastapi.middleware import url_for +from labthings_fastapi.middleware.url_for import URLFor, url_for_middleware +from labthings_fastapi.testing import use_dummy_url_for +from labthings_fastapi.exceptions import NoUrlForContextError + + +class ModelWithURL(BaseModel): + """A model containing a URLFor field.""" + + url: URLFor + + +def test_url_for(): + """Check that the `url_for` function uses the context var as expected.""" + with pytest.raises(NoUrlForContextError): + url_for.url_for("my_endpoint", id=123) + with use_dummy_url_for(): + assert url_for.url_for("my_endpoint", id=123) == "urlfor://my_endpoint/?id=123" + + +def test_string_conversion(mocker): + """Test that URLFor can be converted to a string.""" + url_for_spy = mocker.spy(url_for, "url_for") + u = URLFor("my_endpoint", id=123) + with pytest.raises(NoUrlForContextError): + _ = str(u) + with use_dummy_url_for(): + assert str(u) == "urlfor://my_endpoint/?id=123" + assert url_for_spy.call_count == 2 + + +def test_serialisation(mocker): + """Test that URLFor is serialised by calling str() on it.""" + u = URLFor("my_endpoint", id=123) + m = ModelWithURL(url=u) + + # Check that serialisation fails without a url_for context + # and that it tries to call `url_for` + with pytest.raises(NoUrlForContextError) as excinfo: + _ = m.model_dump() + assert "url_for" in [frame.name for frame in excinfo.traceback] + with pytest.raises(PydanticSerializationError, match="NoUrlForContextError"): + _ = m.model_dump_json() + with use_dummy_url_for(): + assert m.model_dump()["url"] == "urlfor://my_endpoint/?id=123" + + +def test_validation(): + """Test that URLFor validation works as expected.""" + # URLFor is a custom type, so the initialiser works normally + u = URLFor("my_endpoint", id=123) + + # Initialising with an instance should leave it unchanged + m = ModelWithURL(url=u) + assert m.url is u + + # Trying to initialise with anything else should raise an error + with pytest.raises(TypeError): + _ = ModelWithURL(url="https://example.com") + with pytest.raises(TypeError): + _ = ModelWithURL(url="endpoint_name") + with pytest.raises(TypeError): + _ = ModelWithURL(url=None) + + +def test_middleware(): + """Check the middleware function works as expected.""" + app = FastAPI() + app.middleware("http")(url_for_middleware) + + class Model(BaseModel): + url: str + + @app.get("/test-endpoint/{item_id}/", name="test-endpoint") + async def test_endpoint(item_id: int) -> URLFor: + return URLFor("test-endpoint", item_id=item_id) + + @app.get("/sync-endpoint/{item_id}/", name="sync-endpoint") + async def sync_endpoint(item_id: int) -> URLFor: + return URLFor("sync-endpoint", item_id=item_id) + + @app.get("/model-endpoint/{item_id}/", name="model-endpoint") + async def model_endpoint(item_id: int) -> URLFor: + return URLFor("model-endpoint", item_id=item_id) + + with TestClient(app) as client: + response = client.get("/test-endpoint/42/") + assert response.status_code == 200 + assert response.json() == "http://testserver/test-endpoint/42/" + + response = client.get("/sync-endpoint/42/") + assert response.status_code == 200 + assert response.json() == "http://testserver/sync-endpoint/42/" + + response = client.get("/model-endpoint/42/") + assert response.status_code == 200 + assert response.json() == "http://testserver/model-endpoint/42/" From e9009101dbc330c99013e8b3a4c3c22e5da117f0 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Thu, 8 Jan 2026 12:41:25 +0000 Subject: [PATCH 2/5] Ignore a codespell false positive --- src/labthings_fastapi/middleware/url_for.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/labthings_fastapi/middleware/url_for.py b/src/labthings_fastapi/middleware/url_for.py index 511b22ed..e22d5750 100644 --- a/src/labthings_fastapi/middleware/url_for.py +++ b/src/labthings_fastapi/middleware/url_for.py @@ -146,7 +146,9 @@ def __get_pydantic_core_schema__( return core_schema.no_info_wrap_validator_function( cls._validate, AnyUrl.__get_pydantic_core_schema__(AnyUrl, handler), - serialization=core_schema.to_string_ser_schema(when_used="always"), + serialization=core_schema.to_string_ser_schema( # codespell:ignore ser + when_used="always" + ), ) @classmethod From d96b204114e99b2747866792aab92feff3d28f58 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Thu, 8 Jan 2026 12:54:12 +0000 Subject: [PATCH 3/5] Python 3.10 compatible Self import --- src/labthings_fastapi/middleware/url_for.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/labthings_fastapi/middleware/url_for.py b/src/labthings_fastapi/middleware/url_for.py index e22d5750..e77bc7bd 100644 --- a/src/labthings_fastapi/middleware/url_for.py +++ b/src/labthings_fastapi/middleware/url_for.py @@ -23,7 +23,8 @@ from collections.abc import Awaitable, Callable, Iterator from contextlib import contextmanager from contextvars import ContextVar -from typing import Any, Self +from typing import Any +from typing_extensions import Self from fastapi import Request, Response from pydantic import GetCoreSchemaHandler from pydantic.networks import AnyUrl From 16278f1735529505d9c083657eaf2131ee7de139 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 12 Jan 2026 14:16:24 +0000 Subject: [PATCH 4/5] Better testing of middleware Verify that both `URLFor` serialisation and `url_for` are OK in request handlers, but the latter fails in a thread. This should help make it clear where you can (and can't) use the two ways of getting URLs. --- tests/test_middleware_url_for.py | 72 +++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/tests/test_middleware_url_for.py b/tests/test_middleware_url_for.py index a2c01e74..e8f0dc88 100644 --- a/tests/test_middleware_url_for.py +++ b/tests/test_middleware_url_for.py @@ -1,5 +1,6 @@ """Test the URLFor class and associated supporting code.""" +import threading import pytest from pydantic import BaseModel from pydantic_core import PydanticSerializationError @@ -77,29 +78,80 @@ def test_middleware(): app.middleware("http")(url_for_middleware) class Model(BaseModel): - url: str + url: URLFor @app.get("/test-endpoint/{item_id}/", name="test-endpoint") async def test_endpoint(item_id: int) -> URLFor: + """An async endpoint that returns a URLFor instance.""" return URLFor("test-endpoint", item_id=item_id) - @app.get("/sync-endpoint/{item_id}/", name="sync-endpoint") - async def sync_endpoint(item_id: int) -> URLFor: - return URLFor("sync-endpoint", item_id=item_id) + @app.get("/sync-endpoint/{item_id}/") + def sync_endpoint(item_id: int) -> URLFor: + """A sync endpoint that returns a URLFor instance.""" + return URLFor("test-endpoint", item_id=item_id) + + @app.get("/model-endpoint/{item_id}/") + async def model_endpoint(item_id: int) -> Model: + """An async endpoint that returns a model containing a URLFor.""" + return Model(url=URLFor("test-endpoint", item_id=item_id)) + + @app.get("/direct-async-endpoint/{item_id}/") + async def direct_async_endpoint(item_id: int) -> str: + """An async endpoint that calls `url_for` directly.""" + return str(url_for.url_for("test-endpoint", item_id=item_id)) + + @app.get("/direct_sync-endpoint/{item_id}/") + def direct_sync_endpoint(item_id: int) -> str: + """A sync endpoint that calls `url_for` directly.""" + return str(url_for.url_for("test-endpoint", item_id=item_id)) + + def assert_url_for_fails(item_id: int): + with pytest.raises(NoUrlForContextError): + _ = url_for.url_for("test-endpoint", item_id=item_id) + + def append_from_thread(item_id: int, output: list) -> None: + output.append(URLFor("test-endpoint", item_id=item_id)) + + @app.get("/assert_fails_in_thread/{item_id}/") + async def assert_fails_in_thread(item_id: int) -> bool: + t = threading.Thread(target=assert_url_for_fails, args=(item_id,)) + t.start() + t.join() + return True + + @app.get("/return_from_thread/{item_id}/") + async def return_from_thread(item_id: int) -> URLFor: + output = [] + append_from_thread(item_id, output) + return output[0] - @app.get("/model-endpoint/{item_id}/", name="model-endpoint") - async def model_endpoint(item_id: int) -> URLFor: - return URLFor("model-endpoint", item_id=item_id) + URL = "http://testserver/test-endpoint/42/" with TestClient(app) as client: response = client.get("/test-endpoint/42/") assert response.status_code == 200 - assert response.json() == "http://testserver/test-endpoint/42/" + assert response.json() == URL response = client.get("/sync-endpoint/42/") assert response.status_code == 200 - assert response.json() == "http://testserver/sync-endpoint/42/" + assert response.json() == URL response = client.get("/model-endpoint/42/") assert response.status_code == 200 - assert response.json() == "http://testserver/model-endpoint/42/" + assert response.json() == {"url": URL} + + response = client.get("/direct-async-endpoint/42/") + assert response.status_code == 200 + assert response.json() == URL + + response = client.get("/direct_sync-endpoint/42/") + assert response.status_code == 200 + assert response.json() == URL + + response = client.get("/assert_fails_in_thread/42/") + assert response.status_code == 200 + assert response.json() is True + + response = client.get("/return_from_thread/42/") + assert response.status_code == 200 + assert response.json() == URL From 3ec2b8b12b51ef532a5df7d604ed1b3e43425dba Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 12 Jan 2026 14:16:50 +0000 Subject: [PATCH 5/5] Improve docstrings Make it clearer why URLFor exists and when it's appropriate to call `url_for`. --- src/labthings_fastapi/middleware/url_for.py | 42 ++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/labthings_fastapi/middleware/url_for.py b/src/labthings_fastapi/middleware/url_for.py index e77bc7bd..09ab2c77 100644 --- a/src/labthings_fastapi/middleware/url_for.py +++ b/src/labthings_fastapi/middleware/url_for.py @@ -1,5 +1,17 @@ r"""Middleware to make url_for available as a context variable. +This module is intended mostly for internal use within LabThings. The short +summary is that, if you need to refer to other endpoints in the LabThings +server, you should not return hard-coded URLs, but instead use a `URLFor` +object. This will be converted to a URL when it's serialised by FastAPI, using +the correct ``url_for`` function for the current request. + +Under the hood, this module defines a `url_for` function that performs the +conversion. This function may only be run in certain places in the code, as +it relies on a context variable. As a rule of thumb, it's OK to call +`url_for` from a serializer of a `pydantic` model, but you should not call +it from within an Action or Property. + There are several places in LabThings where we need to be able to include URLs to other endpoints in the LabThings server, most notably in the output of Actions. For example, if an Action outputs a `.Blob`\ , the URL to download @@ -72,6 +84,16 @@ def url_for(endpoint_name: str, **params: Any) -> URL: to convert endpoint names and parameters to URLs. It is intended to have the same signature as `fastapi.Request.url_for`\ . + This function will raise a `NoUrlForContextError` if there is no + ``url_for`` function in the context variable. This will be the case if + the function is called outside of a request handler. As a rule, this + function should not be called from within Actions or Properties. + + `URLFor` is provided as a safe way to return URLs: it ensures that the + URL is only generated at serialisation time, when there is a valid + ``url_for`` function in the context. This also means the URL is always + correct for the request being handled. + :param endpoint_name: The name of the endpoint to generate a URL for. :param \**params: The path parameters to use in the URL. :return: The generated URL. @@ -106,7 +128,25 @@ async def url_for_middleware( class URLFor: - """A pydantic-compatible type that converts endpoint names to URLs.""" + """A pydantic-compatible type that converts endpoint names to URLs. + + This class is intended to be used as a field type in `pydantic` models + or as a return type from actions or properties. It does not convert + endpoint names to URLs immediately, but instead stores the endpoint name + and parameters, and only generates the URL when it is serialised by + FastAPI. + + It is safe to *create* a `URLFor` instance anywhere, but converting it + to a string (i.e. generating the URL) requires a valid `url_for` function + and should generally be left for FastAPI. + + Fields or return values annotated as `.URLFor` will only accept a `.URLFor` + instance, but will be serialised to JSON as a string, and will show up in + the JSONSchema as a string. + + Validating a string, i.e. converting a string to a `.URLFor` instance, is + not supported, and will raise a `TypeError`. + """ def __init__(self, endpoint_name: str, **params: Any) -> None: r"""Create a URLFor instance.