diff --git a/CHANGES/11483.feature.rst b/CHANGES/11483.feature.rst new file mode 100644 index 00000000000..a8ef8b62c44 --- /dev/null +++ b/CHANGES/11483.feature.rst @@ -0,0 +1,2 @@ +Added ``StreamReader.total_raw_bytes`` to check the number of bytes downloaded +-- by :user:`robpats`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 033af03c21a..f123d1543fe 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -287,6 +287,7 @@ Pahaz Blinov Panagiotis Kolokotronis Pankaj Pandey Parag Jain +Patrick Lee Pau Freixes Paul Colomiets Paul J. Dorn diff --git a/README.rst b/README.rst index 28ee627f26c..ab9b444601d 100644 --- a/README.rst +++ b/README.rst @@ -17,10 +17,6 @@ Async http client/server framework :target: https://codecov.io/gh/aio-libs/aiohttp :alt: codecov.io status for master branch -.. image:: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json - :target: https://codspeed.io/aio-libs/aiohttp - :alt: Codspeed.io status for aiohttp - .. image:: https://badge.fury.io/py/aiohttp.svg :target: https://pypi.org/project/aiohttp :alt: Latest PyPI package version @@ -33,13 +29,9 @@ Async http client/server framework :target: https://docs.aiohttp.org/ :alt: Latest Read The Docs -.. image:: https://img.shields.io/matrix/aio-libs:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat - :target: https://matrix.to/#/%23aio-libs:matrix.org - :alt: Matrix Room — #aio-libs:matrix.org - -.. image:: https://img.shields.io/matrix/aio-libs-space:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs-space%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat - :target: https://matrix.to/#/%23aio-libs-space:matrix.org - :alt: Matrix Space — #aio-libs-space:matrix.org +.. image:: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json + :target: https://codspeed.io/aio-libs/aiohttp + :alt: Codspeed.io status for aiohttp Key Features @@ -202,3 +194,17 @@ Benchmarks If you are interested in efficiency, the AsyncIO community maintains a list of benchmarks on the official wiki: https://github.com/python/asyncio/wiki/Benchmarks + +-------- + +.. image:: https://img.shields.io/matrix/aio-libs:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs:matrix.org + :alt: Matrix Room — #aio-libs:matrix.org + +.. image:: https://img.shields.io/matrix/aio-libs-space:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs-space%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs-space:matrix.org + :alt: Matrix Space — #aio-libs-space:matrix.org + +.. image:: https://insights.linuxfoundation.org/api/badge/health-score?project=aiohttp + :target: https://insights.linuxfoundation.org/project/aiohttp + :alt: LFX Health Score diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 84b59afc486..e50fc5fdcc1 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -936,6 +936,7 @@ class DeflateBuffer: def __init__(self, out: StreamReader, encoding: Optional[str]) -> None: self.out = out self.size = 0 + out.total_compressed_bytes = self.size self.encoding = encoding self._started_decoding = False @@ -969,6 +970,7 @@ def feed_data(self, chunk: bytes) -> None: return self.size += len(chunk) + self.out.total_compressed_bytes = self.size # RFC1950 # bits 0..3 = CM = 0b1000 = 8 = "deflate" diff --git a/aiohttp/streams.py b/aiohttp/streams.py index db22f162396..1b675a1b73d 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -132,6 +132,7 @@ class StreamReader(AsyncStreamReaderMixin): "_eof_callbacks", "_eof_counter", "total_bytes", + "total_compressed_bytes", ) def __init__( @@ -159,6 +160,7 @@ def __init__( self._eof_callbacks: List[Callable[[], None]] = [] self._eof_counter = 0 self.total_bytes = 0 + self.total_compressed_bytes: Optional[int] = None def __repr__(self) -> str: info = [self.__class__.__name__] @@ -250,6 +252,12 @@ async def wait_eof(self) -> None: finally: self._eof_waiter = None + @property + def total_raw_bytes(self) -> int: + if self.total_compressed_bytes is None: + return self.total_bytes + return self.total_compressed_bytes + def unread_data(self, data: bytes) -> None: """rollback reading some data from stream, inserting it to buffer head.""" warnings.warn( diff --git a/docs/streams.rst b/docs/streams.rst index 6b65b59475b..8cb573d8edf 100644 --- a/docs/streams.rst +++ b/docs/streams.rst @@ -20,8 +20,8 @@ Streaming API :attr:`aiohttp.ClientResponse.content` properties for accessing raw BODY data. -Reading Methods ---------------- +Reading Attributes and Methods +------------------------------ .. method:: StreamReader.read(n=-1) :async: @@ -109,6 +109,13 @@ Reading Methods to the end of a HTTP chunk. +.. attribute:: StreamReader.total_raw_bytes + + The number of bytes of raw data downloaded (before decompression). + + Readonly :class:`int` property. + + Asynchronous Iteration Support ------------------------------ diff --git a/requirements/base-ft.txt b/requirements/base-ft.txt index 5c1e43c5bda..a900f979cfa 100644 --- a/requirements/base-ft.txt +++ b/requirements/base-ft.txt @@ -16,7 +16,7 @@ brotli==1.1.0 ; platform_python_implementation == "CPython" # via -r requirements/runtime-deps.in cffi==2.0.0 # via pycares -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal @@ -30,7 +30,7 @@ multidict==6.6.4 # yarl packaging==25.0 # via gunicorn -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/runtime-deps.in # yarl diff --git a/requirements/base.txt b/requirements/base.txt index ec4c8e4e6f8..3b4cdd1f37f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -16,7 +16,7 @@ brotli==1.1.0 ; platform_python_implementation == "CPython" # via -r requirements/runtime-deps.in cffi==2.0.0 # via pycares -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal @@ -30,7 +30,7 @@ multidict==6.6.4 # yarl packaging==25.0 # via gunicorn -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/runtime-deps.in # yarl diff --git a/requirements/constraints.txt b/requirements/constraints.txt index f067c0e30ab..f76f14c96fd 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -78,7 +78,7 @@ freezegun==1.5.5 # via # -r requirements/lint.in # -r requirements/test-common.in -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal @@ -145,7 +145,7 @@ pluggy==1.6.0 # pytest-cov pre-commit==4.3.0 # via -r requirements/lint.in -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/runtime-deps.in # yarl diff --git a/requirements/dev.txt b/requirements/dev.txt index 4afc488c8f8..8147207c1d2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -76,7 +76,7 @@ freezegun==1.5.5 # via # -r requirements/lint.in # -r requirements/test-common.in -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal @@ -142,7 +142,7 @@ pluggy==1.6.0 # pytest-cov pre-commit==4.3.0 # via -r requirements/lint.in -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/runtime-deps.in # yarl diff --git a/requirements/runtime-deps.txt b/requirements/runtime-deps.txt index de0e7181da5..0f641deba57 100644 --- a/requirements/runtime-deps.txt +++ b/requirements/runtime-deps.txt @@ -16,7 +16,7 @@ brotli==1.1.0 ; platform_python_implementation == "CPython" # via -r requirements/runtime-deps.in cffi==2.0.0 # via pycares -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal @@ -26,7 +26,7 @@ multidict==6.6.4 # via # -r requirements/runtime-deps.in # yarl -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/runtime-deps.in # yarl diff --git a/requirements/test-ft.txt b/requirements/test-ft.txt index 2eda8248d9a..842052f2cf2 100644 --- a/requirements/test-ft.txt +++ b/requirements/test-ft.txt @@ -39,7 +39,7 @@ forbiddenfruit==0.1.4 # via blockbuster freezegun==1.5.5 # via -r requirements/test-common.in -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal @@ -77,7 +77,7 @@ pluggy==1.6.0 # via # pytest # pytest-cov -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/runtime-deps.in # yarl diff --git a/requirements/test.txt b/requirements/test.txt index 8f938b90f1c..66ce3327cdf 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -39,7 +39,7 @@ forbiddenfruit==0.1.4 # via blockbuster freezegun==1.5.5 # via -r requirements/test-common.in -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/runtime-deps.in # aiosignal @@ -77,7 +77,7 @@ pluggy==1.6.0 # via # pytest # pytest-cov -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/runtime-deps.in # yarl diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index e671aef180a..3433226db49 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -5586,3 +5586,46 @@ async def handler(request: web.Request) -> web.Response: finally: await asyncio.to_thread(f.close) + + +async def test_stream_reader_total_raw_bytes(aiohttp_client: AiohttpClient) -> None: + """Test whether StreamReader.total_raw_bytes returns the number of bytes downloaded""" + source_data = b"@dKal^pH>1h|YW1:c2J$" * 4096 + + async def handler(request: web.Request) -> web.Response: + response = web.Response(body=source_data) + response.enable_compression() + return response + + app = web.Application() + app.router.add_get("/", handler) + + client = await aiohttp_client(app) + + # Check for decompressed data + async with client.get( + "/", headers={"Accept-Encoding": "gzip"}, auto_decompress=True + ) as resp: + assert resp.headers["Content-Encoding"] == "gzip" + assert int(resp.headers["Content-Length"]) < len(source_data) + data = await resp.content.read() + assert len(data) == len(source_data) + assert resp.content.total_raw_bytes == int(resp.headers["Content-Length"]) + + # Check for compressed data + async with client.get( + "/", headers={"Accept-Encoding": "gzip"}, auto_decompress=False + ) as resp: + assert resp.headers["Content-Encoding"] == "gzip" + data = await resp.content.read() + assert resp.content.total_raw_bytes == len(data) + assert resp.content.total_raw_bytes == int(resp.headers["Content-Length"]) + + # Check for non-compressed data + async with client.get( + "/", headers={"Accept-Encoding": "identity"}, auto_decompress=True + ) as resp: + assert "Content-Encoding" not in resp.headers + data = await resp.content.read() + assert resp.content.total_raw_bytes == len(data) + assert resp.content.total_raw_bytes == int(resp.headers["Content-Length"])