[pull] master from aio-libs:master

.mypy.ini

@@ -2,7 +2,7 @@
 files = aiohttp, examples, tests
 check_untyped_defs = True
 follow_imports_for_stubs = True
-#disallow_any_decorated = True
+disallow_any_decorated = True
 disallow_any_generics = True
 disallow_any_unimported = True
 disallow_incomplete_defs = True

CHANGES/10433.feature.rst

@@ -0,0 +1 @@
+Detect blocking calls in coroutines using BlockBuster -- by :user:`cbornet`.

CHANGES/10464.bugfix.rst

@@ -0,0 +1 @@
+Changed connection creation to explicitly close sockets if an exception is raised in the event loop's ``create_connection`` method -- by :user:`top-oai`.

CONTRIBUTORS.txt

@@ -43,6 +43,7 @@ Andrej Antonov
 Andrew Leech
 Andrew Lytvyn
 Andrew Svetlov
+Andrew Top
 Andrew Zhou
 Andrii Soldatenko
 Anes Abismail
@@ -82,6 +83,7 @@ Chris AtLee
 Chris Laws
 Chris Moore
 Chris Shucksmith
+Christophe Bornet
 Christopher Schmitt
 Claudiu Popa
 Colin Dunklau

aiohttp/abc.py

@@ -81,7 +81,7 @@ def http_exception(self) -> Optional[HTTPException]:
         """HTTPException instance raised on router's resolving, or None"""
 
     @abstractmethod  # pragma: no branch
-    def get_info(self) -> Dict[str, Any]:
+    def get_info(self) -> Dict[str, Any]:  # type: ignore[misc]
         """Return a dict with additional info useful for introspection"""
 
     @property  # pragma: no branch
@@ -120,7 +120,7 @@ def request(self) -> Request:
         return self._request
 
     @abstractmethod
-    def __await__(self) -> Generator[Any, None, StreamResponse]:
+    def __await__(self) -> Generator[None, None, StreamResponse]:
         """Execute the view handler."""
 
 

aiohttp/client.py

@@ -607,11 +607,14 @@ async def _request(
                         if req_cookies:
                             all_cookies.load(req_cookies)
 
+                    proxy_: Optional[URL] = None
                     if proxy is not None:
-                        proxy = URL(proxy)
+                        proxy_ = URL(proxy)
                     elif self._trust_env:
                         with suppress(LookupError):
-                            proxy, proxy_auth = get_env_proxy_for_url(url)
+                            proxy_, proxy_auth = await asyncio.to_thread(
+                                get_env_proxy_for_url, url
+                            )
 
                     req = self._request_class(
                         method,
@@ -628,7 +631,7 @@ async def _request(
                         expect100=expect100,
                         loop=self._loop,
                         response_class=self._response_class,
-                        proxy=proxy,
+                        proxy=proxy_,
                         proxy_auth=proxy_auth,
                         timer=timer,
                         session=self,
@@ -1244,7 +1247,7 @@ def skip_auto_headers(self) -> FrozenSet[istr]:
         return self._skip_auto_headers
 
     @property
-    def auth(self) -> Optional[BasicAuth]:
+    def auth(self) -> Optional[BasicAuth]:  # type: ignore[misc]
         """An object that represents HTTP Basic Authorization"""
         return self._default_auth
 
@@ -1281,7 +1284,7 @@ def trust_env(self) -> bool:
         return self._trust_env
 
     @property
-    def trace_configs(self) -> List[TraceConfig[Any]]:
+    def trace_configs(self) -> List[TraceConfig[Any]]:  # type: ignore[misc]
         """A list of TraceConfig instances used for client tracing"""
         return self._trace_configs
 

aiohttp/client_reqrep.py

@@ -322,7 +322,7 @@ def ssl(self) -> Union["SSLContext", bool, Fingerprint]:
         return self._ssl
 
     @property
-    def connection_key(self) -> ConnectionKey:
+    def connection_key(self) -> ConnectionKey:  # type: ignore[misc]
         if proxy_headers := self.proxy_headers:
             h: Optional[int] = hash(tuple(proxy_headers.items()))
         else:

aiohttp/connector.py

@@ -1101,6 +1101,7 @@ async def _wrap_create_connection(
         client_error: Type[Exception] = ClientConnectorError,
         **kwargs: Any,
     ) -> Tuple[asyncio.Transport, ResponseHandler]:
+        sock: Union[socket.socket, None] = None
         try:
             async with ceil_timeout(
                 timeout.sock_connect, ceil_threshold=timeout.ceil_threshold
@@ -1112,7 +1113,11 @@ async def _wrap_create_connection(
                     interleave=self._interleave,
                     loop=self._loop,
                 )
-                return await self._loop.create_connection(*args, **kwargs, sock=sock)
+                connection = await self._loop.create_connection(
+                    *args, **kwargs, sock=sock
+                )
+                sock = None
+                return connection
         except cert_errors as exc:
             raise ClientConnectorCertificateError(req.connection_key, exc) from exc
         except ssl_errors as exc:
@@ -1121,6 +1126,12 @@ async def _wrap_create_connection(
             if exc.errno is None and isinstance(exc, asyncio.TimeoutError):
                 raise
             raise client_error(req.connection_key, exc) from exc
+        finally:
+            if sock is not None:
+                # Will be hit if an exception is thrown before the event loop takes the socket.
+                # In that case, proactively close the socket to guard against event loop leaks.
+                # For example, see https://github.com/MagicStack/uvloop/issues/653.
+                sock.close()
 
     def _warn_about_tls_in_tls(
         self,

aiohttp/helpers.py

@@ -145,7 +145,7 @@ def __new__(
         return super().__new__(cls, login, password, encoding)
 
     @classmethod
-    def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth":
+    def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth":  # type: ignore[misc]
         """Create a BasicAuth object from an Authorization HTTP header."""
         try:
             auth_type, encoded_credentials = auth_header.split(" ", 1)
@@ -174,7 +174,7 @@ def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth":
         return cls(username, password, encoding=encoding)
 
     @classmethod
-    def from_url(cls, url: URL, *, encoding: str = "latin1") -> Optional["BasicAuth"]:
+    def from_url(cls, url: URL, *, encoding: str = "latin1") -> Optional["BasicAuth"]:  # type: ignore[misc]
         """Create BasicAuth from url."""
         if not isinstance(url, URL):
             raise TypeError("url should be yarl.URL instance")
@@ -245,7 +245,7 @@ def netrc_from_env() -> Optional[netrc.netrc]:
 
 
 @frozen_dataclass_decorator
-class ProxyInfo:
+class ProxyInfo:  # type: ignore[misc]
     proxy: URL
     proxy_auth: Optional[BasicAuth]
 
@@ -870,7 +870,7 @@ def __init_subclass__(cls) -> None:
     def __getitem__(self, key: AppKey[_T]) -> _T: ...
 
     @overload
-    def __getitem__(self, key: str) -> Any: ...
+    def __getitem__(self, key: str) -> Any: ...  # type: ignore[misc]
 
     def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any:
         for mapping in self._maps:
@@ -887,7 +887,7 @@ def get(self, key: AppKey[_T], default: _S) -> Union[_T, _S]: ...
     def get(self, key: AppKey[_T], default: None = ...) -> Optional[_T]: ...
 
     @overload
-    def get(self, key: str, default: Any = ...) -> Any: ...
+    def get(self, key: str, default: Any = ...) -> Any: ...  # type: ignore[misc]
 
     def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any:
         try:

aiohttp/pytest_plugin.py

@@ -40,16 +40,17 @@
 
 
 class AiohttpClient(Protocol):
+    # TODO(PY311): Use Unpack to specify ClientSession kwargs.
     @overload
-    async def __call__(
+    async def __call__(  # type: ignore[misc]
         self,
         __param: Application,
         *,
         server_kwargs: Optional[Dict[str, Any]] = None,
         **kwargs: Any,
     ) -> TestClient[Request, Application]: ...
     @overload
-    async def __call__(
+    async def __call__(  # type: ignore[misc]
         self,
         __param: BaseTestServer[_Request],
         *,
@@ -153,19 +154,19 @@ def finalizer():  # type: ignore[no-untyped-def]
 
 
 @pytest.fixture
-def fast(request):  # type: ignore[no-untyped-def]
+def fast(request: pytest.FixtureRequest) -> bool:
     """--fast config option"""
-    return request.config.getoption("--aiohttp-fast")
+    return request.config.getoption("--aiohttp-fast")  # type: ignore[no-any-return]
 
 
 @pytest.fixture
-def loop_debug(request):  # type: ignore[no-untyped-def]
+def loop_debug(request: pytest.FixtureRequest) -> bool:
     """--enable-loop-debug config option"""
-    return request.config.getoption("--aiohttp-enable-loop-debug")
+    return request.config.getoption("--aiohttp-enable-loop-debug")  # type: ignore[no-any-return]
 
 
 @contextlib.contextmanager
-def _runtime_warning_context():  # type: ignore[no-untyped-def]
+def _runtime_warning_context() -> Iterator[None]:
     """Context manager which checks for RuntimeWarnings.
 
     This exists specifically to
@@ -195,7 +196,9 @@ def _runtime_warning_context():  # type: ignore[no-untyped-def]
 
 
 @contextlib.contextmanager
-def _passthrough_loop_context(loop, fast=False):  # type: ignore[no-untyped-def]
+def _passthrough_loop_context(
+    loop: Optional[asyncio.AbstractEventLoop], fast: bool = False
+) -> Iterator[asyncio.AbstractEventLoop]:
     """Passthrough loop context.
 
     Sets up and tears down a loop unless one is passed in via the loop
@@ -268,7 +271,11 @@ def pytest_generate_tests(metafunc):  # type: ignore[no-untyped-def]
 
 
 @pytest.fixture
-def loop(loop_factory, fast, loop_debug):  # type: ignore[no-untyped-def]
+def loop(
+    loop_factory: Callable[[], asyncio.AbstractEventLoopPolicy],
+    fast: bool,
+    loop_debug: bool,
+) -> Iterator[asyncio.AbstractEventLoop]:
     """Return an instance of the event loop."""
     policy = loop_factory()
     asyncio.set_event_loop_policy(policy)
@@ -280,7 +287,7 @@ def loop(loop_factory, fast, loop_debug):  # type: ignore[no-untyped-def]
 
 
 @pytest.fixture
-def proactor_loop():  # type: ignore[no-untyped-def]
+def proactor_loop() -> Iterator[asyncio.AbstractEventLoop]:
     policy = asyncio.WindowsProactorEventLoopPolicy()  # type: ignore[attr-defined]
     asyncio.set_event_loop_policy(policy)
 
@@ -353,7 +360,7 @@ async def finalize() -> None:
 
 
 @pytest.fixture
-def aiohttp_client_cls() -> Type[TestClient[Any, Any]]:
+def aiohttp_client_cls() -> Type[TestClient[Any, Any]]:  # type: ignore[misc]
     """
     Client class to use in ``aiohttp_client`` factory.
 
@@ -380,7 +387,7 @@ def test_login(aiohttp_client):
 
 
 @pytest.fixture
-def aiohttp_client(
+def aiohttp_client(  # type: ignore[misc]
     loop: asyncio.AbstractEventLoop, aiohttp_client_cls: Type[TestClient[Any, Any]]
 ) -> Iterator[AiohttpClient]:
     """Factory to create a TestClient instance.
@@ -392,14 +399,14 @@ def aiohttp_client(
     clients = []
 
     @overload
-    async def go(
+    async def go(  # type: ignore[misc]
         __param: Application,
         *,
         server_kwargs: Optional[Dict[str, Any]] = None,
         **kwargs: Any,
     ) -> TestClient[Request, Application]: ...
     @overload
-    async def go(
+    async def go(  # type: ignore[misc]
         __param: BaseTestServer[_Request],
         *,
         server_kwargs: Optional[Dict[str, Any]] = None,
@@ -411,6 +418,7 @@ async def go(
         server_kwargs: Optional[Dict[str, Any]] = None,
         **kwargs: Any,
     ) -> TestClient[Any, Any]:
+        # TODO(PY311): Use Unpack to specify ClientSession kwargs and server_kwargs.
         if isinstance(__param, Application):
             server_kwargs = server_kwargs or {}
             server = TestServer(__param, **server_kwargs)

aiohttp/test_utils.py

@@ -149,16 +149,17 @@ async def start_server(self, **kwargs: Any) -> None:
         await site.start()
         server = site._server
         assert server is not None
-        sockets = server.sockets  # type: ignore[attr-defined]
+        sockets = server.sockets
         assert sockets is not None
         self.port = sockets[0].getsockname()[1]
         if not self.scheme:
             self.scheme = "https" if self._ssl else "http"
         self._root = URL(f"{self.scheme}://{absolute_host}:{self.port}")
 
     @abstractmethod
-    async def _make_runner(self, **kwargs: Any) -> BaseRunner[_Request]:
+    async def _make_runner(self, **kwargs: Any) -> BaseRunner[_Request]:  # type: ignore[misc]
         """Return a new runner for the server."""
+        # TODO(PY311): Use Unpack to specify Server kwargs.
 
     def make_url(self, path: StrOrURL) -> URL:
         assert self._root is not None
@@ -232,6 +233,7 @@ def __init__(
         super().__init__(scheme=scheme, host=host, port=port, **kwargs)
 
     async def _make_runner(self, **kwargs: Any) -> AppRunner:
+        # TODO(PY311): Use Unpack to specify Server kwargs.
         return AppRunner(self.app, **kwargs)
 
 
@@ -249,6 +251,7 @@ def __init__(
         super().__init__(scheme=scheme, host=host, port=port, **kwargs)
 
     async def _make_runner(self, **kwargs: Any) -> ServerRunner:
+        # TODO(PY311): Use Unpack to specify Server kwargs.
         srv = Server(self._handler, **kwargs)
         return ServerRunner(srv, **kwargs)
 
@@ -264,15 +267,15 @@ class TestClient(Generic[_Request, _ApplicationNone]):
     __test__ = False
 
     @overload
-    def __init__(
+    def __init__(  # type: ignore[misc]
         self: "TestClient[Request, Application]",
         server: TestServer,
         *,
         cookie_jar: Optional[AbstractCookieJar] = None,
         **kwargs: Any,
     ) -> None: ...
     @overload
-    def __init__(
+    def __init__(  # type: ignore[misc]
         self: "TestClient[_Request, None]",
         server: BaseTestServer[_Request],
         *,
@@ -286,6 +289,7 @@ def __init__(  # type: ignore[misc]
         cookie_jar: Optional[AbstractCookieJar] = None,
         **kwargs: Any,
     ) -> None:
+        # TODO(PY311): Use Unpack to specify ClientSession kwargs.
         if not isinstance(server, BaseTestServer):
             raise TypeError(
                 "server must be TestServer instance, found type: %r" % type(server)

aiohttp/web_app.py

@@ -164,7 +164,7 @@ def __eq__(self, other: object) -> bool:
     def __getitem__(self, key: AppKey[_T]) -> _T: ...
 
     @overload
-    def __getitem__(self, key: str) -> Any: ...
+    def __getitem__(self, key: str) -> Any: ...  # type: ignore[misc]
 
     def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any:
         return self._state[key]
@@ -179,7 +179,7 @@ def _check_frozen(self) -> None:
     def __setitem__(self, key: AppKey[_T], value: _T) -> None: ...
 
     @overload
-    def __setitem__(self, key: str, value: Any) -> None: ...
+    def __setitem__(self, key: str, value: Any) -> None: ...  # type: ignore[misc]
 
     def __setitem__(self, key: Union[str, AppKey[_T]], value: Any) -> None:
         self._check_frozen()
@@ -213,7 +213,7 @@ def get(self, key: AppKey[_T], default: None = ...) -> Optional[_T]: ...
     def get(self, key: AppKey[_T], default: _U) -> Union[_T, _U]: ...
 
     @overload
-    def get(self, key: str, default: Any = ...) -> Any: ...
+    def get(self, key: str, default: Any = ...) -> Any: ...  # type: ignore[misc]
 
     def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any:
         return self._state.get(key, default)

aiohttp/web_request.py

@@ -563,7 +563,7 @@ def cookies(self) -> Mapping[str, str]:
         return MappingProxyType({key: val.value for key, val in parsed.items()})
 
     @reify
-    def http_range(self) -> slice:
+    def http_range(self) -> "slice[int, int, int]":
         """The content of Range HTTP header.
 
         Return a slice instance.