From ea6884940beb7e0e3ac06ad51fc699260626d9b2 Mon Sep 17 00:00:00 2001 From: Kazuya Takei Date: Mon, 24 Nov 2025 23:09:51 +0900 Subject: [PATCH 1/4] ci: Reproduce failure in parallel build Refs: #12 --- tests/roots/test-parallel/conf.py | 10 ++++++++++ tests/roots/test-parallel/index.rst | 8 ++++++++ tests/roots/test-parallel/sub1.rst | 5 +++++ tests/roots/test-parallel/sub2.rst | 7 +++++++ tests/roots/test-parallel/sub3.rst | 3 +++ tests/roots/test-parallel/sub4.rst | 3 +++ tests/roots/test-parallel/sub5.rst | 3 +++ tests/test_adapters/test_sphinx.py | 6 ++++++ 8 files changed, 45 insertions(+) create mode 100644 tests/roots/test-parallel/conf.py create mode 100644 tests/roots/test-parallel/index.rst create mode 100644 tests/roots/test-parallel/sub1.rst create mode 100644 tests/roots/test-parallel/sub2.rst create mode 100644 tests/roots/test-parallel/sub3.rst create mode 100644 tests/roots/test-parallel/sub4.rst create mode 100644 tests/roots/test-parallel/sub5.rst diff --git a/tests/roots/test-parallel/conf.py b/tests/roots/test-parallel/conf.py new file mode 100644 index 0000000..fcc6a9a --- /dev/null +++ b/tests/roots/test-parallel/conf.py @@ -0,0 +1,10 @@ +"""Configuration is cases for default behavior.""" + +extensions = [ + "oembedpy.adapters.sphinx", +] + +# To skip toctree +rst_prolog = """ +:orphan: +""" diff --git a/tests/roots/test-parallel/index.rst b/tests/roots/test-parallel/index.rst new file mode 100644 index 0000000..5aafd10 --- /dev/null +++ b/tests/roots/test-parallel/index.rst @@ -0,0 +1,8 @@ +============= +Test document +============= + +.. toctree:: + + sub1 + sub2 diff --git a/tests/roots/test-parallel/sub1.rst b/tests/roots/test-parallel/sub1.rst new file mode 100644 index 0000000..dcc6c16 --- /dev/null +++ b/tests/roots/test-parallel/sub1.rst @@ -0,0 +1,5 @@ +============== +Sub document 1 +============== + +.. oembed:: https://mastodon.cloud/@attakei/109368512525772407 diff --git a/tests/roots/test-parallel/sub2.rst b/tests/roots/test-parallel/sub2.rst new file mode 100644 index 0000000..306e481 --- /dev/null +++ b/tests/roots/test-parallel/sub2.rst @@ -0,0 +1,7 @@ +============== +Sub document 2 +============== + +.. oembed:: https://www.youtube.com/watch?v=Oyh8nuaLASA + :maxwidth: 1200 + :maxheight: 1200 diff --git a/tests/roots/test-parallel/sub3.rst b/tests/roots/test-parallel/sub3.rst new file mode 100644 index 0000000..7c3679c --- /dev/null +++ b/tests/roots/test-parallel/sub3.rst @@ -0,0 +1,3 @@ +============== +Sub document 3 +============== diff --git a/tests/roots/test-parallel/sub4.rst b/tests/roots/test-parallel/sub4.rst new file mode 100644 index 0000000..5ca896c --- /dev/null +++ b/tests/roots/test-parallel/sub4.rst @@ -0,0 +1,3 @@ +============== +Sub document 4 +============== diff --git a/tests/roots/test-parallel/sub5.rst b/tests/roots/test-parallel/sub5.rst new file mode 100644 index 0000000..d8b835d --- /dev/null +++ b/tests/roots/test-parallel/sub5.rst @@ -0,0 +1,3 @@ +============== +Sub document 5 +============== diff --git a/tests/test_adapters/test_sphinx.py b/tests/test_adapters/test_sphinx.py index 71be6f9..d841f4e 100644 --- a/tests/test_adapters/test_sphinx.py +++ b/tests/test_adapters/test_sphinx.py @@ -55,3 +55,9 @@ def test_caches(app: SphinxTestApp): # noqa def test_use_caches(app: SphinxTestApp): # noqa app.build() assert len(app.env.get_domain("oembedpy").caches) == 1 # type: ignore[attr-defined] + + +@pytest.mark.webtest +@pytest.mark.sphinx("html", testroot="parallel", parallel=2) +def test_build_parallel(app: SphinxTestApp, status): # noqa + app.build() From 3320b4bec1425d5378f4a73d9735bd4c33426195 Mon Sep 17 00:00:00 2001 From: Kazuya Takei Date: Sun, 21 Dec 2025 18:03:15 +0900 Subject: [PATCH 2/4] feat: Support merge caches from multiple environments --- oembedpy/adapters/sphinx.py | 12 +++++++- tests/test_adapters/test_sphinx.py | 49 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/oembedpy/adapters/sphinx.py b/oembedpy/adapters/sphinx.py index 9fe8538..e4d315b 100644 --- a/oembedpy/adapters/sphinx.py +++ b/oembedpy/adapters/sphinx.py @@ -92,6 +92,16 @@ def resolve_any_xref( # because this does not have roles and directives that are refered by outside. return [] + def merge_domaindata(self, docnames: list[str], otherdata) -> None: + other_caches = otherdata.get("caches", {}) + for cache_key, content in other_caches.items(): + if cache_key not in self.caches: + self.caches[cache_key] = content + continue + exist = self.caches[cache_key] + if content._expired > exist._expired: + self.caches[cache_key] = content + class oembed(nodes.General, nodes.Element): # noqa: D101,E501 pass @@ -138,6 +148,6 @@ def setup(app: Sphinx): # noqa: D103 app.add_domain(OembedDomain) return { "version": __version__, - "parallel_read_safe": False, + "parallel_read_safe": True, "parallel_write_safe": True, } diff --git a/tests/test_adapters/test_sphinx.py b/tests/test_adapters/test_sphinx.py index d841f4e..4461e96 100644 --- a/tests/test_adapters/test_sphinx.py +++ b/tests/test_adapters/test_sphinx.py @@ -4,8 +4,12 @@ import pytest from bs4 import BeautifulSoup +from sphinx.environment import BuildEnvironment from sphinx.testing.util import SphinxTestApp +from oembedpy.adapters import sphinx as T +from oembedpy.types import Link + @pytest.fixture(scope="module") def rootdir(): @@ -61,3 +65,48 @@ def test_use_caches(app: SphinxTestApp): # noqa @pytest.mark.sphinx("html", testroot="parallel", parallel=2) def test_build_parallel(app: SphinxTestApp, status): # noqa app.build() + + +class TestFor_OembedDomain__merge_domaindata: + CACHE_KEY = ("http://example.com", 1, 1) + + @pytest.mark.sphinx("html", testroot="default") + def test_difference_items(self, app: SphinxTestApp): + domain1 = T.OembedDomain(app.env) + domain1.caches[self.CACHE_KEY] = Link(type="link", version="1.0", _extra={}) + domain2 = T.OembedDomain(BuildEnvironment(app)) + domain2.caches[("http://example.com", 1, 2)] = Link( + type="link", version="1.0", _extra={} + ) + domain1.merge_domaindata([], domain2.data) + assert len(domain1.caches) == 2 + + @pytest.mark.sphinx("html", testroot="default") + def test_keep_main_domain(self, app: SphinxTestApp): + domain1 = T.OembedDomain(app.env) + domain1.caches[self.CACHE_KEY] = Link( + type="link", version="1.0", title="Hello", _extra={} + ) + domain2 = T.OembedDomain(BuildEnvironment(app)) + domain2.caches[self.CACHE_KEY] = Link( + type="link", version="1.0", title="World", _extra={} + ) + print(domain1.caches) + domain1.merge_domaindata([], domain2.data) + assert len(domain1.caches) == 1 + print(domain1.caches) + assert domain1.caches[self.CACHE_KEY].title == "Hello" + + @pytest.mark.sphinx("html", testroot="default") + def test_keep_overrides(self, app: SphinxTestApp): + domain1 = T.OembedDomain(app.env) + link1 = Link(type="link", version="1.0", title="Hello", _extra={}) + link1._expired = 3600 + domain1.caches[self.CACHE_KEY] = link1 + domain2 = T.OembedDomain(BuildEnvironment(app)) + link2 = Link(type="link", version="1.0", title="World", _extra={}) + link2._expired = 3601 + domain2.caches[self.CACHE_KEY] = link2 + domain1.merge_domaindata([], domain2.data) + assert len(domain1.caches) == 1 + assert domain1.caches[("http://example.com", 1, 1)].title == "World" From 9aa6d8471f305b17191f54f9f8d4465a4d148ff7 Mon Sep 17 00:00:00 2001 From: Kazuya Takei Date: Sun, 21 Dec 2025 18:14:29 +0900 Subject: [PATCH 3/4] ci: Update for indepndent each tests --- tests/test_adapters/test_sphinx.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_adapters/test_sphinx.py b/tests/test_adapters/test_sphinx.py index 4461e96..fb9ede8 100644 --- a/tests/test_adapters/test_sphinx.py +++ b/tests/test_adapters/test_sphinx.py @@ -72,7 +72,7 @@ class TestFor_OembedDomain__merge_domaindata: @pytest.mark.sphinx("html", testroot="default") def test_difference_items(self, app: SphinxTestApp): - domain1 = T.OembedDomain(app.env) + domain1 = T.OembedDomain(BuildEnvironment(app)) domain1.caches[self.CACHE_KEY] = Link(type="link", version="1.0", _extra={}) domain2 = T.OembedDomain(BuildEnvironment(app)) domain2.caches[("http://example.com", 1, 2)] = Link( @@ -83,7 +83,7 @@ def test_difference_items(self, app: SphinxTestApp): @pytest.mark.sphinx("html", testroot="default") def test_keep_main_domain(self, app: SphinxTestApp): - domain1 = T.OembedDomain(app.env) + domain1 = T.OembedDomain(BuildEnvironment(app)) domain1.caches[self.CACHE_KEY] = Link( type="link", version="1.0", title="Hello", _extra={} ) @@ -99,7 +99,7 @@ def test_keep_main_domain(self, app: SphinxTestApp): @pytest.mark.sphinx("html", testroot="default") def test_keep_overrides(self, app: SphinxTestApp): - domain1 = T.OembedDomain(app.env) + domain1 = T.OembedDomain(BuildEnvironment(app)) link1 = Link(type="link", version="1.0", title="Hello", _extra={}) link1._expired = 3600 domain1.caches[self.CACHE_KEY] = link1 From d2c972c17dbe871a0a49938fa977888928c729e3 Mon Sep 17 00:00:00 2001 From: Kazuya Takei Date: Sun, 21 Dec 2025 18:25:44 +0900 Subject: [PATCH 4/4] refactor: Add docstring --- oembedpy/adapters/sphinx.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/oembedpy/adapters/sphinx.py b/oembedpy/adapters/sphinx.py index e4d315b..60fcd6e 100644 --- a/oembedpy/adapters/sphinx.py +++ b/oembedpy/adapters/sphinx.py @@ -93,6 +93,12 @@ def resolve_any_xref( return [] def merge_domaindata(self, docnames: list[str], otherdata) -> None: + """Merge domain data from parallel builds. + + This method merges cached oEmbed content from parallel build environments. + The merge strategy prioritizes entries with longer expiration times to ensure + the most up-to-date and longest-lived cached content is retained. + """ other_caches = otherdata.get("caches", {}) for cache_key, content in other_caches.items(): if cache_key not in self.caches: