[pull] master from aio-libs:master

CHANGES/11483.feature.rst

@@ -0,0 +1,2 @@
+Added ``StreamReader.total_raw_bytes`` to check the number of bytes downloaded
+-- by :user:`robpats`.

CONTRIBUTORS.txt

@@ -287,6 +287,7 @@ Pahaz Blinov
 Panagiotis Kolokotronis
 Pankaj Pandey
 Parag Jain
+Patrick Lee
 Pau Freixes
 Paul Colomiets
 Paul J. Dorn

README.rst

@@ -17,10 +17,6 @@ Async http client/server framework
    :target: https://codecov.io/gh/aio-libs/aiohttp
    :alt: codecov.io status for master branch
 
-.. image:: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json
-   :target: https://codspeed.io/aio-libs/aiohttp
-   :alt: Codspeed.io status for aiohttp
-
 .. image:: https://badge.fury.io/py/aiohttp.svg
    :target: https://pypi.org/project/aiohttp
    :alt: Latest PyPI package version
@@ -33,13 +29,9 @@ Async http client/server framework
    :target: https://docs.aiohttp.org/
    :alt: Latest Read The Docs
 
-.. image:: https://img.shields.io/matrix/aio-libs:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat
-   :target: https://matrix.to/#/%23aio-libs:matrix.org
-   :alt: Matrix Room — #aio-libs:matrix.org
-
-.. image:: https://img.shields.io/matrix/aio-libs-space:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs-space%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat
-   :target: https://matrix.to/#/%23aio-libs-space:matrix.org
-   :alt: Matrix Space — #aio-libs-space:matrix.org
+.. image:: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json
+   :target: https://codspeed.io/aio-libs/aiohttp
+   :alt: Codspeed.io status for aiohttp
 
 
 Key Features
@@ -202,3 +194,17 @@ Benchmarks
 If you are interested in efficiency, the AsyncIO community maintains a
 list of benchmarks on the official wiki:
 https://github.com/python/asyncio/wiki/Benchmarks
+
+--------
+
+.. image:: https://img.shields.io/matrix/aio-libs:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat
+   :target: https://matrix.to/#/%23aio-libs:matrix.org
+   :alt: Matrix Room — #aio-libs:matrix.org
+
+.. image:: https://img.shields.io/matrix/aio-libs-space:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs-space%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat
+   :target: https://matrix.to/#/%23aio-libs-space:matrix.org
+   :alt: Matrix Space — #aio-libs-space:matrix.org
+
+.. image:: https://insights.linuxfoundation.org/api/badge/health-score?project=aiohttp
+   :target: https://insights.linuxfoundation.org/project/aiohttp
+   :alt: LFX Health Score

aiohttp/http_parser.py

@@ -936,6 +936,7 @@ class DeflateBuffer:
     def __init__(self, out: StreamReader, encoding: Optional[str]) -> None:
         self.out = out
         self.size = 0
+        out.total_compressed_bytes = self.size
         self.encoding = encoding
         self._started_decoding = False
 
@@ -969,6 +970,7 @@ def feed_data(self, chunk: bytes) -> None:
             return
 
         self.size += len(chunk)
+        self.out.total_compressed_bytes = self.size
 
         # RFC1950
         # bits 0..3 = CM = 0b1000 = 8 = "deflate"

aiohttp/streams.py

@@ -132,6 +132,7 @@ class StreamReader(AsyncStreamReaderMixin):
         "_eof_callbacks",
         "_eof_counter",
         "total_bytes",
+        "total_compressed_bytes",
     )
 
     def __init__(
@@ -159,6 +160,7 @@ def __init__(
         self._eof_callbacks: List[Callable[[], None]] = []
         self._eof_counter = 0
         self.total_bytes = 0
+        self.total_compressed_bytes: Optional[int] = None
 
     def __repr__(self) -> str:
         info = [self.__class__.__name__]
@@ -250,6 +252,12 @@ async def wait_eof(self) -> None:
         finally:
             self._eof_waiter = None
 
+    @property
+    def total_raw_bytes(self) -> int:
+        if self.total_compressed_bytes is None:
+            return self.total_bytes
+        return self.total_compressed_bytes
+
     def unread_data(self, data: bytes) -> None:
         """rollback reading some data from stream, inserting it to buffer head."""
         warnings.warn(

docs/streams.rst

@@ -20,8 +20,8 @@ Streaming API
    :attr:`aiohttp.ClientResponse.content` properties for accessing raw
    BODY data.
 
-Reading Methods
----------------
+Reading Attributes and Methods
+------------------------------
 
 .. method:: StreamReader.read(n=-1)
       :async:
@@ -109,6 +109,13 @@ Reading Methods
                                to the end of a HTTP chunk.
 
 
+.. attribute:: StreamReader.total_raw_bytes
+
+   The number of bytes of raw data downloaded (before decompression).
+
+   Readonly :class:`int` property.
+
+
 Asynchronous Iteration Support
 ------------------------------
 

requirements/base-ft.txt

@@ -16,7 +16,7 @@ brotli==1.1.0 ; platform_python_implementation == "CPython"
     # via -r requirements/runtime-deps.in
 cffi==2.0.0
     # via pycares
-frozenlist==1.7.0
+frozenlist==1.8.0
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -30,7 +30,7 @@ multidict==6.6.4
     #   yarl
 packaging==25.0
     # via gunicorn
-propcache==0.3.2
+propcache==0.4.0
     # via
     #   -r requirements/runtime-deps.in
     #   yarl

requirements/base.txt

@@ -16,7 +16,7 @@ brotli==1.1.0 ; platform_python_implementation == "CPython"
     # via -r requirements/runtime-deps.in
 cffi==2.0.0
     # via pycares
-frozenlist==1.7.0
+frozenlist==1.8.0
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -30,7 +30,7 @@ multidict==6.6.4
     #   yarl
 packaging==25.0
     # via gunicorn
-propcache==0.3.2
+propcache==0.4.0
     # via
     #   -r requirements/runtime-deps.in
     #   yarl

requirements/constraints.txt

@@ -78,7 +78,7 @@ freezegun==1.5.5
     # via
     #   -r requirements/lint.in
     #   -r requirements/test-common.in
-frozenlist==1.7.0
+frozenlist==1.8.0
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -145,7 +145,7 @@ pluggy==1.6.0
     #   pytest-cov
 pre-commit==4.3.0
     # via -r requirements/lint.in
-propcache==0.3.2
+propcache==0.4.0
     # via
     #   -r requirements/runtime-deps.in
     #   yarl

requirements/dev.txt

@@ -76,7 +76,7 @@ freezegun==1.5.5
     # via
     #   -r requirements/lint.in
     #   -r requirements/test-common.in
-frozenlist==1.7.0
+frozenlist==1.8.0
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -142,7 +142,7 @@ pluggy==1.6.0
     #   pytest-cov
 pre-commit==4.3.0
     # via -r requirements/lint.in
-propcache==0.3.2
+propcache==0.4.0
     # via
     #   -r requirements/runtime-deps.in
     #   yarl

requirements/runtime-deps.txt

@@ -16,7 +16,7 @@ brotli==1.1.0 ; platform_python_implementation == "CPython"
     # via -r requirements/runtime-deps.in
 cffi==2.0.0
     # via pycares
-frozenlist==1.7.0
+frozenlist==1.8.0
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -26,7 +26,7 @@ multidict==6.6.4
     # via
     #   -r requirements/runtime-deps.in
     #   yarl
-propcache==0.3.2
+propcache==0.4.0
     # via
     #   -r requirements/runtime-deps.in
     #   yarl

requirements/test-ft.txt

@@ -39,7 +39,7 @@ forbiddenfruit==0.1.4
     # via blockbuster
 freezegun==1.5.5
     # via -r requirements/test-common.in
-frozenlist==1.7.0
+frozenlist==1.8.0
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -77,7 +77,7 @@ pluggy==1.6.0
     # via
     #   pytest
     #   pytest-cov
-propcache==0.3.2
+propcache==0.4.0
     # via
     #   -r requirements/runtime-deps.in
     #   yarl

requirements/test.txt

@@ -39,7 +39,7 @@ forbiddenfruit==0.1.4
     # via blockbuster
 freezegun==1.5.5
     # via -r requirements/test-common.in
-frozenlist==1.7.0
+frozenlist==1.8.0
     # via
     #   -r requirements/runtime-deps.in
     #   aiosignal
@@ -77,7 +77,7 @@ pluggy==1.6.0
     # via
     #   pytest
     #   pytest-cov
-propcache==0.3.2
+propcache==0.4.0
     # via
     #   -r requirements/runtime-deps.in
     #   yarl

tests/test_client_functional.py

@@ -5586,3 +5586,46 @@ async def handler(request: web.Request) -> web.Response:
 
     finally:
         await asyncio.to_thread(f.close)
+
+
+async def test_stream_reader_total_raw_bytes(aiohttp_client: AiohttpClient) -> None:
+    """Test whether StreamReader.total_raw_bytes returns the number of bytes downloaded"""
+    source_data = b"@dKal^pH>1h|YW1:c2J$" * 4096
+
+    async def handler(request: web.Request) -> web.Response:
+        response = web.Response(body=source_data)
+        response.enable_compression()
+        return response
+
+    app = web.Application()
+    app.router.add_get("/", handler)
+
+    client = await aiohttp_client(app)
+
+    # Check for decompressed data
+    async with client.get(
+        "/", headers={"Accept-Encoding": "gzip"}, auto_decompress=True
+    ) as resp:
+        assert resp.headers["Content-Encoding"] == "gzip"
+        assert int(resp.headers["Content-Length"]) < len(source_data)
+        data = await resp.content.read()
+        assert len(data) == len(source_data)
+        assert resp.content.total_raw_bytes == int(resp.headers["Content-Length"])
+
+    # Check for compressed data
+    async with client.get(
+        "/", headers={"Accept-Encoding": "gzip"}, auto_decompress=False
+    ) as resp:
+        assert resp.headers["Content-Encoding"] == "gzip"
+        data = await resp.content.read()
+        assert resp.content.total_raw_bytes == len(data)
+        assert resp.content.total_raw_bytes == int(resp.headers["Content-Length"])
+
+    # Check for non-compressed data
+    async with client.get(
+        "/", headers={"Accept-Encoding": "identity"}, auto_decompress=True
+    ) as resp:
+        assert "Content-Encoding" not in resp.headers
+        data = await resp.content.read()
+        assert resp.content.total_raw_bytes == len(data)
+        assert resp.content.total_raw_bytes == int(resp.headers["Content-Length"])