From 398329f6ec6237cb0f29149863485cccec65258a Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Wed, 15 Oct 2025 05:49:07 -0700 Subject: [PATCH 1/4] add xfail test for issue 11632 (#11651) --- tests/test_cookie_helpers.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_cookie_helpers.py b/tests/test_cookie_helpers.py index 0ec393e2b79..575bbe54d01 100644 --- a/tests/test_cookie_helpers.py +++ b/tests/test_cookie_helpers.py @@ -1137,6 +1137,33 @@ def test_parse_cookie_header_empty() -> None: assert parse_cookie_header(" ") == [] +@pytest.mark.xfail(reason="https://github.com/aio-libs/aiohttp/issues/11632") +def test_parse_cookie_gstate_header() -> None: + header = ( + "_ga=ga; " + "ajs_anonymous_id=0anonymous; " + "analytics_session_id=session; " + "cookies-analytics=true; " + "cookies-functional=true; " + "cookies-marketing=true; " + "cookies-preferences=true; " + 'g_state={"i_l":0,"i_ll":12345,"i_b":"blah"}; ' + "analytics_session_id.last_access=1760128947692; " + "landingPageURLRaw=landingPageURLRaw; " + "landingPageURL=landingPageURL; " + "referrerPageURLRaw=; " + "referrerPageURL=; " + "formURLRaw=formURLRaw; " + "formURL=formURL; " + "fbnAuthExpressCheckout=fbnAuthExpressCheckout; " + "is_express_checkout=1; " + ) + + result = parse_cookie_header(header) + assert result[7][0] == "g_state" + assert result[8][0] == "analytics_session_id.last_access" + + def test_parse_cookie_header_quoted_values() -> None: """Test parse_cookie_header handles quoted values correctly.""" header = 'name="quoted value"; session="with;semicolon"; data="with\\"escaped\\""' From 302243e37274bd9e5a32e405ca11b1b0306f75b1 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 15 Oct 2025 14:05:22 +0100 Subject: [PATCH 2/4] Remove cherry-picker from requirements (#11654) --- requirements/constraints.txt | 14 -------------- requirements/dev.in | 1 - requirements/dev.txt | 14 -------------- 3 files changed, 29 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index f4a41abc189..90d9d1ae4ca 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -47,11 +47,8 @@ cfgv==3.4.0 # via pre-commit charset-normalizer==3.4.4 # via requests -cherry-picker==2.6.0 - # via -r requirements/dev.in click==8.3.0 # via - # cherry-picker # pip-tools # slotscheck # towncrier @@ -86,8 +83,6 @@ frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal -gidgethub==5.4.0 - # via cherry-picker gunicorn==23.0.0 # via -r requirements/base.in identify==2.6.15 @@ -174,7 +169,6 @@ pygments==2.19.2 # sphinx pyjwt==2.8.0 # via - # gidgethub # pyjwt pyproject-hooks==1.2.0 # via @@ -210,7 +204,6 @@ pyyaml==6.0.3 # via pre-commit requests==2.32.5 # via - # cherry-picker # sphinx # sphinxcontrib-spelling rich==14.2.0 @@ -244,14 +237,9 @@ sphinxcontrib-spelling==8.0.1 ; platform_system != "Windows" # via -r requirements/doc-spelling.in sphinxcontrib-towncrier==0.5.0a0 # via -r requirements/doc.in -stamina==25.1.0 - # via cherry-picker -tenacity==9.1.2 - # via stamina tomli==2.3.0 # via # build - # cherry-picker # coverage # mypy # pip-tools @@ -281,8 +269,6 @@ typing-extensions==4.15.0 # virtualenv typing-inspection==0.4.2 # via pydantic -uritemplate==4.2.0 - # via gidgethub urllib3==2.5.0 # via requests uvloop==0.21.0 ; platform_system != "Windows" diff --git a/requirements/dev.in b/requirements/dev.in index 780eae45f5e..75185757c40 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,5 +2,4 @@ -r test.in -r doc.in -cherry_picker pip-tools diff --git a/requirements/dev.txt b/requirements/dev.txt index 1754f9472ca..1f022fa22fa 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -47,11 +47,8 @@ cfgv==3.4.0 # via pre-commit charset-normalizer==3.4.4 # via requests -cherry-picker==2.6.0 - # via -r requirements/dev.in click==8.3.0 # via - # cherry-picker # pip-tools # slotscheck # towncrier @@ -84,8 +81,6 @@ frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal -gidgethub==5.4.0 - # via cherry-picker gunicorn==23.0.0 # via -r requirements/base.in identify==2.6.15 @@ -169,7 +164,6 @@ pygments==2.19.2 # sphinx pyjwt==2.8.0 # via - # gidgethub # pyjwt pyproject-hooks==1.2.0 # via @@ -205,7 +199,6 @@ pyyaml==6.0.3 # via pre-commit requests==2.32.5 # via - # cherry-picker # sphinx rich==14.2.0 # via pytest-codspeed @@ -235,14 +228,9 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx sphinxcontrib-towncrier==0.5.0a0 # via -r requirements/doc.in -stamina==25.1.0 - # via cherry-picker -tenacity==9.1.2 - # via stamina tomli==2.3.0 # via # build - # cherry-picker # coverage # mypy # pip-tools @@ -272,8 +260,6 @@ typing-extensions==4.15.0 # virtualenv typing-inspection==0.4.2 # via pydantic -uritemplate==4.2.0 - # via gidgethub urllib3==2.5.0 # via requests uvloop==0.21.0 ; platform_system != "Windows" and implementation_name == "cpython" From cde03b98c647272cf12f6e36cce35b5dc252171d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 03:08:04 -1000 Subject: [PATCH 3/4] Fix blocking I/O to load netrc when creating requests (#11634) --- CHANGES/11634.bugfix.rst | 1 + aiohttp/client.py | 29 ++++++++ aiohttp/client_reqrep.py | 6 -- tests/conftest.py | 32 +++++++-- tests/test_client_functional.py | 122 ++++++++++++++++++++++++-------- tests/test_client_request.py | 22 +----- tests/test_client_session.py | 77 ++++++++++++++++++++ 7 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 CHANGES/11634.bugfix.rst diff --git a/CHANGES/11634.bugfix.rst b/CHANGES/11634.bugfix.rst new file mode 100644 index 00000000000..649577c50b9 --- /dev/null +++ b/CHANGES/11634.bugfix.rst @@ -0,0 +1 @@ +Fixed blocking I/O in the event loop when using netrc authentication by moving netrc file lookup to an executor -- by :user:`bdraco`. diff --git a/aiohttp/client.py b/aiohttp/client.py index a7da3ff0c57..7a4ad715362 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -86,8 +86,10 @@ EMPTY_BODY_METHODS, BasicAuth, TimeoutHandle, + basicauth_from_netrc, frozen_dataclass_decorator, get_env_proxy_for_url, + netrc_from_env, sentinel, strip_auth_from_url, ) @@ -586,6 +588,20 @@ async def _request( ) ): auth = self._default_auth + + # Try netrc if auth is still None and trust_env is enabled. + # Only check if NETRC environment variable is set to avoid + # creating an expensive executor job unnecessarily. + if ( + auth is None + and self._trust_env + and url.host is not None + and os.environ.get("NETRC") + ): + auth = await self._loop.run_in_executor( + None, self._get_netrc_auth, url.host + ) + # It would be confusing if we support explicit # Authorization header with auth argument if auth is not None and hdrs.AUTHORIZATION in headers: @@ -1131,6 +1147,19 @@ def _prepare_headers(self, headers: LooseHeaders | None) -> "CIMultiDict[str]": added_names.add(key) return result + def _get_netrc_auth(self, host: str) -> BasicAuth | None: + """ + Get auth from netrc for the given host. + + This method is designed to be called in an executor to avoid + blocking I/O in the event loop. + """ + netrc_obj = netrc_from_env() + try: + return basicauth_from_netrc(netrc_obj, host) + except LookupError: + return None + if sys.version_info >= (3, 11) and TYPE_CHECKING: def get( diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 880e1085bab..050d3a259e1 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -40,10 +40,8 @@ BasicAuth, HeadersMixin, TimerNoop, - basicauth_from_netrc, frozen_dataclass_decorator, is_expected_content_type, - netrc_from_env, parse_mimetype, reify, sentinel, @@ -1068,10 +1066,6 @@ def update_auth(self, auth: BasicAuth | None, trust_env: bool = False) -> None: """Set basic auth.""" if auth is None: auth = self.auth - if auth is None and trust_env and self.url.host is not None: - netrc_obj = netrc_from_env() - with contextlib.suppress(LookupError): - auth = basicauth_from_netrc(netrc_obj, self.url.host) if auth is None: return diff --git a/tests/conftest.py b/tests/conftest.py index 5e872dec5c7..6833d2c1653 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -71,10 +71,6 @@ def blockbuster(request: pytest.FixtureRequest) -> Iterator[None]: with blockbuster_ctx( "aiohttp", excluded_modules=["aiohttp.pytest_plugin", "aiohttp.test_utils"] ) as bb: - # TODO: Fix blocking call in ClientRequest's constructor. - # https://github.com/aio-libs/aiohttp/issues/10435 - for func in ["io.TextIOWrapper.read", "os.stat"]: - bb.functions[func].can_block_in("aiohttp/client_reqrep.py", "update_auth") for func in [ "os.getcwd", "os.readlink", @@ -292,6 +288,34 @@ def netrc_contents( return netrc_file_path +@pytest.fixture +def netrc_default_contents(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """Create a temporary netrc file with default test credentials and set NETRC env var.""" + netrc_file = tmp_path / ".netrc" + netrc_file.write_text("default login netrc_user password netrc_pass\n") + + monkeypatch.setenv("NETRC", str(netrc_file)) + + return netrc_file + + +@pytest.fixture +def no_netrc(monkeypatch: pytest.MonkeyPatch) -> None: + """Ensure NETRC environment variable is not set.""" + monkeypatch.delenv("NETRC", raising=False) + + +@pytest.fixture +def netrc_other_host(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """Create a temporary netrc file with credentials for a different host and set NETRC env var.""" + netrc_file = tmp_path / ".netrc" + netrc_file.write_text("machine other.example.com login user password pass\n") + + monkeypatch.setenv("NETRC", str(netrc_file)) + + return netrc_file + + @pytest.fixture def start_connection() -> Iterator[mock.Mock]: with mock.patch( diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 5006a745346..731878d7c1b 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -70,6 +70,23 @@ def fname(here: pathlib.Path) -> pathlib.Path: return here / "conftest.py" +@pytest.fixture +def headers_echo_client( + aiohttp_client: AiohttpClient, +) -> Callable[..., Awaitable[TestClient[web.Request, web.Application]]]: + """Create a client with an app that echoes request headers as JSON.""" + + async def factory(**kwargs: Any) -> TestClient[web.Request, web.Application]: + async def handler(request: web.Request) -> web.Response: + return web.json_response({"headers": dict(request.headers)}) + + app = web.Application() + app.router.add_get("/", handler) + return await aiohttp_client(app, **kwargs) + + return factory + + async def test_keepalive_two_requests_success(aiohttp_client: AiohttpClient) -> None: async def handler(request: web.Request) -> web.Response: body = await request.read() @@ -3702,14 +3719,12 @@ async def handler(request: web.Request) -> web.Response: assert resp.status == 200 -async def test_session_auth(aiohttp_client: AiohttpClient) -> None: - async def handler(request: web.Request) -> web.Response: - return web.json_response({"headers": dict(request.headers)}) - - app = web.Application() - app.router.add_get("/", handler) - - client = await aiohttp_client(app, auth=aiohttp.BasicAuth("login", "pass")) +async def test_session_auth( + headers_echo_client: Callable[ + ..., Awaitable[TestClient[web.Request, web.Application]] + ], +) -> None: + client = await headers_echo_client(auth=aiohttp.BasicAuth("login", "pass")) async with client.get("/") as r: assert r.status == 200 @@ -3717,14 +3732,12 @@ async def handler(request: web.Request) -> web.Response: assert content["headers"]["Authorization"] == "Basic bG9naW46cGFzcw==" -async def test_session_auth_override(aiohttp_client: AiohttpClient) -> None: - async def handler(request: web.Request) -> web.Response: - return web.json_response({"headers": dict(request.headers)}) - - app = web.Application() - app.router.add_get("/", handler) - - client = await aiohttp_client(app, auth=aiohttp.BasicAuth("login", "pass")) +async def test_session_auth_override( + headers_echo_client: Callable[ + ..., Awaitable[TestClient[web.Request, web.Application]] + ], +) -> None: + client = await headers_echo_client(auth=aiohttp.BasicAuth("login", "pass")) async with client.get("/", auth=aiohttp.BasicAuth("other_login", "pass")) as r: assert r.status == 200 @@ -3746,30 +3759,77 @@ async def handler(request: web.Request) -> NoReturn: await client.get("/", headers=headers) -async def test_session_headers(aiohttp_client: AiohttpClient) -> None: - async def handler(request: web.Request) -> web.Response: - return web.json_response({"headers": dict(request.headers)}) - - app = web.Application() - app.router.add_get("/", handler) +@pytest.mark.usefixtures("netrc_default_contents") +async def test_netrc_auth_from_env( # type: ignore[misc] + headers_echo_client: Callable[ + ..., Awaitable[TestClient[web.Request, web.Application]] + ], +) -> None: + """Test that netrc authentication works when NETRC env var is set and trust_env=True.""" + client = await headers_echo_client(trust_env=True) + async with client.get("/") as r: + assert r.status == 200 + content = await r.json() + # Base64 encoded "netrc_user:netrc_pass" is "bmV0cmNfdXNlcjpuZXRyY19wYXNz" + assert content["headers"]["Authorization"] == "Basic bmV0cmNfdXNlcjpuZXRyY19wYXNz" - client = await aiohttp_client(app, headers={"X-Real-IP": "192.168.0.1"}) +@pytest.mark.usefixtures("no_netrc") +async def test_netrc_auth_skipped_without_env_var( # type: ignore[misc] + headers_echo_client: Callable[ + ..., Awaitable[TestClient[web.Request, web.Application]] + ], +) -> None: + """Test that netrc authentication is skipped when NETRC env var is not set.""" + client = await headers_echo_client(trust_env=True) async with client.get("/") as r: assert r.status == 200 content = await r.json() - assert content["headers"]["X-Real-IP"] == "192.168.0.1" + # No Authorization header should be present + assert "Authorization" not in content["headers"] -async def test_session_headers_merge(aiohttp_client: AiohttpClient) -> None: - async def handler(request: web.Request) -> web.Response: - return web.json_response({"headers": dict(request.headers)}) +@pytest.mark.usefixtures("netrc_default_contents") +async def test_netrc_auth_overridden_by_explicit_auth( # type: ignore[misc] + headers_echo_client: Callable[ + ..., Awaitable[TestClient[web.Request, web.Application]] + ], +) -> None: + """Test that explicit auth parameter overrides netrc authentication.""" + client = await headers_echo_client(trust_env=True) + # Make request with explicit auth (should override netrc) + async with client.get( + "/", auth=aiohttp.BasicAuth("explicit_user", "explicit_pass") + ) as r: + assert r.status == 200 + content = await r.json() + # Base64 encoded "explicit_user:explicit_pass" is "ZXhwbGljaXRfdXNlcjpleHBsaWNpdF9wYXNz" + assert ( + content["headers"]["Authorization"] + == "Basic ZXhwbGljaXRfdXNlcjpleHBsaWNpdF9wYXNz" + ) - app = web.Application() - app.router.add_get("/", handler) - client = await aiohttp_client( - app, headers=[("X-Real-IP", "192.168.0.1"), ("X-Sent-By", "requests")] +async def test_session_headers( + headers_echo_client: Callable[ + ..., Awaitable[TestClient[web.Request, web.Application]] + ], +) -> None: + client = await headers_echo_client(headers={"X-Real-IP": "192.168.0.1"}) + + async with client.get("/") as r: + assert r.status == 200 + content = await r.json() + assert content["headers"]["X-Real-IP"] == "192.168.0.1" + + +async def test_session_headers_merge( + headers_echo_client: Callable[ + ..., Awaitable[TestClient[web.Request, web.Application]] + ], +) -> None: + client = await headers_echo_client( + headers=[("X-Real-IP", "192.168.0.1"), ("X-Sent-By", "requests")] ) async with client.get("/", headers={"X-Sent-By": "aiohttp"}) as r: diff --git a/tests/test_client_request.py b/tests/test_client_request.py index ef444f1008f..e05b3198a79 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -14,7 +14,7 @@ from yarl import URL import aiohttp -from aiohttp import BaseConnector, hdrs, helpers, payload +from aiohttp import BaseConnector, hdrs, payload from aiohttp.abc import AbstractStreamWriter from aiohttp.base_protocol import BaseProtocol from aiohttp.client_exceptions import ClientConnectionError @@ -1574,26 +1574,6 @@ def test_gen_default_accept_encoding( assert _gen_default_accept_encoding() == expected -@pytest.mark.parametrize( - ("netrc_contents", "expected_auth"), - [ - ( - "machine example.com login username password pass\n", - helpers.BasicAuth("username", "pass"), - ) - ], - indirect=("netrc_contents",), -) -@pytest.mark.usefixtures("netrc_contents") -def test_basicauth_from_netrc_present( # type: ignore[misc] - make_request: _RequestMaker, - expected_auth: helpers.BasicAuth, -) -> None: - """Test appropriate Authorization header is sent when netrc is not empty.""" - req = make_request("get", "http://example.com", trust_env=True) - assert req.headers[hdrs.AUTHORIZATION] == expected_auth.encode() - - @pytest.mark.parametrize( "netrc_contents", ("machine example.com login username password pass\n",), diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 11a815a325e..84a417f9219 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -26,6 +26,7 @@ from aiohttp.cookiejar import CookieJar from aiohttp.http import RawResponseMessage from aiohttp.pytest_plugin import AiohttpClient, AiohttpServer +from aiohttp.test_utils import TestServer from aiohttp.tracing import Trace @@ -89,6 +90,21 @@ def params() -> _Params: ) +@pytest.fixture +async def auth_server(aiohttp_server: AiohttpServer) -> TestServer: + """Create a server with an auth handler that returns auth header or 'no_auth'.""" + + async def handler(request: web.Request) -> web.Response: + auth_header = request.headers.get(hdrs.AUTHORIZATION) + if auth_header: + return web.Response(text=f"auth:{auth_header}") + return web.Response(text="no_auth") + + app = web.Application() + app.router.add_get("/", handler) + return await aiohttp_server(app) + + async def test_close_coro( create_session: Callable[..., Awaitable[ClientSession]], ) -> None: @@ -1326,3 +1342,64 @@ async def test_properties( value = uuid4() setattr(session, inner_name, value) assert value == getattr(session, outer_name) + + +@pytest.mark.usefixtures("netrc_default_contents") +async def test_netrc_auth_with_trust_env(auth_server: TestServer) -> None: + """Test that netrc authentication works with ClientSession when NETRC env var is set.""" + async with ( + ClientSession(trust_env=True) as session, + session.get(auth_server.make_url("/")) as resp, + ): + text = await resp.text() + # Base64 encoded "netrc_user:netrc_pass" is "bmV0cmNfdXNlcjpuZXRyY19wYXNz" + assert text == "auth:Basic bmV0cmNfdXNlcjpuZXRyY19wYXNz" + + +@pytest.mark.usefixtures("netrc_default_contents") +async def test_netrc_auth_skipped_without_trust_env(auth_server: TestServer) -> None: + """Test that netrc authentication is skipped when trust_env=False.""" + async with ( + ClientSession(trust_env=False) as session, + session.get(auth_server.make_url("/")) as resp, + ): + text = await resp.text() + assert text == "no_auth" + + +@pytest.mark.usefixtures("no_netrc") +async def test_netrc_auth_skipped_without_netrc_env(auth_server: TestServer) -> None: + """Test that netrc authentication is skipped when NETRC env var is not set.""" + async with ( + ClientSession(trust_env=True) as session, + session.get(auth_server.make_url("/")) as resp, + ): + text = await resp.text() + assert text == "no_auth" + + +@pytest.mark.usefixtures("netrc_default_contents") +async def test_netrc_auth_overridden_by_explicit_auth(auth_server: TestServer) -> None: + """Test that explicit auth parameter overrides netrc authentication.""" + async with ( + ClientSession(trust_env=True) as session, + session.get( + auth_server.make_url("/"), + auth=aiohttp.BasicAuth("explicit_user", "explicit_pass"), + ) as resp, + ): + text = await resp.text() + # Base64 encoded "explicit_user:explicit_pass" is "ZXhwbGljaXRfdXNlcjpleHBsaWNpdF9wYXNz" + assert text == "auth:Basic ZXhwbGljaXRfdXNlcjpleHBsaWNpdF9wYXNz" + + +@pytest.mark.usefixtures("netrc_other_host") +async def test_netrc_auth_host_not_in_netrc(auth_server: TestServer) -> None: + """Test that netrc lookup returns None when host is not in netrc file.""" + async with ( + ClientSession(trust_env=True) as session, + session.get(auth_server.make_url("/")) as resp, + ): + text = await resp.text() + # Should not have auth since the host is not in netrc + assert text == "no_auth" From 149a8105e7de73888df11ab1689315f9482aaeae Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 15 Oct 2025 15:30:06 +0100 Subject: [PATCH 4/4] Fix type annotation (#11661) --- aiohttp/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 4aded6cd7e0..7368cf1b170 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -364,7 +364,7 @@ def __init__(self) -> None: # https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5 self.set_default_type("application/octet-stream") - def get_content_type(self) -> Any: + def get_content_type(self) -> str: """Re-implementation from Message Returns application/octet-stream in place of plain/text when