diff --git a/CHANGES/11623.bugfix b/CHANGES/11623.bugfix new file mode 100644 index 00000000000..447dd56388c --- /dev/null +++ b/CHANGES/11623.bugfix @@ -0,0 +1 @@ +Switched to `backports.zstd` for Python <3.14 and fixed zstd decompression for chunked zstd streams -- by :user:`ZhaoMJ`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index b64d99fb632..84692200b6e 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -270,6 +270,7 @@ Mikhail Burshteyn Mikhail Kashkin Mikhail Lukyanchenko Mikhail Nacharov +Mingjie Zhao Misha Behersky Mitchell Ferree Morgan Delahaye-Prat diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index fcaee591fe5..c6c6f0d71fc 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -22,18 +22,14 @@ HAS_BROTLI = False try: - from compression.zstd import ( # type: ignore[import-not-found] # noqa: I900 - ZstdDecompressor, - ) + if sys.version_info >= (3, 14): + from compression.zstd import ZstdDecompressor # noqa: I900 + else: # TODO(PY314): Remove mentions of backports.zstd across codebase + from backports.zstd import ZstdDecompressor HAS_ZSTD = True except ImportError: - try: - from zstandard import ZstdDecompressor - - HAS_ZSTD = True - except ImportError: - HAS_ZSTD = False + HAS_ZSTD = False MAX_SYNC_CHUNK_SIZE = 1024 @@ -298,12 +294,12 @@ def __init__(self) -> None: if not HAS_ZSTD: raise RuntimeError( "The zstd decompression is not available. " - "Please install `zstandard` module" + "Please install `backports.zstd` module" ) self._obj = ZstdDecompressor() def decompress_sync(self, data: bytes) -> bytes: - return self._obj.decompress(data) # type: ignore[no-any-return] + return self._obj.decompress(data) def flush(self) -> bytes: return b"" diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index ce143819141..9e4d5a9ce0c 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -938,7 +938,7 @@ def __init__(self, out: StreamReader, encoding: str | None) -> None: if not HAS_ZSTD: raise ContentEncodingError( "Can not decode content-encoding: zstandard (zstd). " - "Please install `zstandard`" + "Please install `backports.zstd`" ) self.decompressor = ZSTDDecompressor() else: diff --git a/docs/client_quickstart.rst b/docs/client_quickstart.rst index 48f123b94bd..fc6d3875ea1 100644 --- a/docs/client_quickstart.rst +++ b/docs/client_quickstart.rst @@ -191,7 +191,7 @@ just install `Brotli `_ or `brotlicffi `_. You can enable ``zstd`` transfer-encodings support, -install `zstandard `_. +install `backports.zstd `_. If you are using Python >= 3.14, no dependency should be required. JSON Request diff --git a/requirements/base-ft.txt b/requirements/base-ft.txt index 5615e306c84..a851bf38281 100644 --- a/requirements/base-ft.txt +++ b/requirements/base-ft.txt @@ -12,6 +12,8 @@ aiosignal==1.4.0 # via -r requirements/runtime-deps.in async-timeout==5.0.1 ; python_version < "3.11" # via -r requirements/runtime-deps.in +backports-zstd==0.5.0 ; platform_python_implementation == "CPython" and python_version < "3.14" + # via -r requirements/runtime-deps.in brotli==1.1.0 ; platform_python_implementation == "CPython" # via -r requirements/runtime-deps.in cffi==2.0.0 @@ -30,7 +32,7 @@ multidict==6.7.0 # yarl packaging==25.0 # via gunicorn -propcache==0.4.0 +propcache==0.4.1 # via # -r requirements/runtime-deps.in # yarl @@ -42,7 +44,5 @@ typing-extensions==4.15.0 # via # aiosignal # multidict -yarl==1.21.0 - # via -r requirements/runtime-deps.in -zstandard==0.25.0 ; platform_python_implementation == "CPython" and python_version < "3.14" +yarl==1.22.0 # via -r requirements/runtime-deps.in diff --git a/requirements/base.txt b/requirements/base.txt index f3e3fe2649b..7e9da23c158 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -12,6 +12,8 @@ aiosignal==1.4.0 # via -r requirements/runtime-deps.in async-timeout==5.0.1 ; python_version < "3.11" # via -r requirements/runtime-deps.in +backports-zstd==0.5.0 ; platform_python_implementation == "CPython" and python_version < "3.14" + # via -r requirements/runtime-deps.in brotli==1.1.0 ; platform_python_implementation == "CPython" # via -r requirements/runtime-deps.in cffi==2.0.0 @@ -30,7 +32,7 @@ multidict==6.7.0 # yarl packaging==25.0 # via gunicorn -propcache==0.4.0 +propcache==0.4.1 # via # -r requirements/runtime-deps.in # yarl @@ -44,7 +46,5 @@ typing-extensions==4.15.0 # multidict uvloop==0.21.0 ; platform_system != "Windows" and implementation_name == "cpython" # via -r requirements/base.in -yarl==1.21.0 - # via -r requirements/runtime-deps.in -zstandard==0.25.0 ; platform_python_implementation == "CPython" and python_version < "3.14" +yarl==1.22.0 # via -r requirements/runtime-deps.in diff --git a/requirements/constraints.txt b/requirements/constraints.txt index fd73a37c401..50e5906ad55 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -24,6 +24,10 @@ async-timeout==5.0.1 ; python_version < "3.11" # valkey babel==2.17.0 # via sphinx +backports-zstd==0.5.0 ; implementation_name == "cpython" + # via + # -r requirements/lint.in + # -r requirements/runtime-deps.in blockbuster==1.5.25 # via # -r requirements/lint.in @@ -145,7 +149,7 @@ pluggy==1.6.0 # pytest-cov pre-commit==4.3.0 # via -r requirements/lint.in -propcache==0.4.0 +propcache==0.4.1 # via # -r requirements/runtime-deps.in # yarl @@ -293,16 +297,12 @@ wait-for-it==2.3.0 # via -r requirements/test-common.in wheel==0.46.0 # via pip-tools -yarl==1.21.0 +yarl==1.22.0 # via -r requirements/runtime-deps.in zlib-ng==1.0.0 # via # -r requirements/lint.in # -r requirements/test-common.in -zstandard==0.25.0 ; implementation_name == "cpython" - # via - # -r requirements/lint.in - # -r requirements/runtime-deps.in # The following packages are considered to be unsafe in a requirements file: pip==25.2 diff --git a/requirements/dev.txt b/requirements/dev.txt index f0ac2f57a03..73a8fb911d2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -24,6 +24,10 @@ async-timeout==5.0.1 ; python_version < "3.11" # valkey babel==2.17.0 # via sphinx +backports-zstd==0.5.0 ; platform_python_implementation == "CPython" and python_version < "3.14" + # via + # -r requirements/lint.in + # -r requirements/runtime-deps.in blockbuster==1.5.25 # via # -r requirements/lint.in @@ -142,7 +146,7 @@ pluggy==1.6.0 # pytest-cov pre-commit==4.3.0 # via -r requirements/lint.in -propcache==0.4.0 +propcache==0.4.1 # via # -r requirements/runtime-deps.in # yarl @@ -284,16 +288,12 @@ wait-for-it==2.3.0 # via -r requirements/test-common.in wheel==0.46.0 # via pip-tools -yarl==1.21.0 +yarl==1.22.0 # via -r requirements/runtime-deps.in zlib-ng==1.0.0 # via # -r requirements/lint.in # -r requirements/test-common.in -zstandard==0.25.0 ; platform_python_implementation == "CPython" and python_version < "3.14" - # via - # -r requirements/lint.in - # -r requirements/runtime-deps.in # The following packages are considered to be unsafe in a requirements file: pip==25.2 diff --git a/requirements/lint.in b/requirements/lint.in index e36b5f28cd6..c83d1dcec88 100644 --- a/requirements/lint.in +++ b/requirements/lint.in @@ -1,4 +1,5 @@ aiodns +backports.zstd; implementation_name == "cpython" blockbuster freezegun isal @@ -14,4 +15,3 @@ trustme uvloop; platform_system != "Windows" valkey zlib_ng -zstandard; implementation_name == "cpython" diff --git a/requirements/lint.txt b/requirements/lint.txt index 087a2f63e90..dbe3dcf4013 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -123,5 +123,5 @@ virtualenv==20.34.0 # via pre-commit zlib-ng==1.0.0 # via -r requirements/lint.in -zstandard==0.25.0 ; implementation_name == "cpython" +backports.zstd==0.5.0 ; implementation_name == "cpython" # via -r requirements/lint.in diff --git a/requirements/runtime-deps.in b/requirements/runtime-deps.in index f849b448cf6..f6c6f2e8caa 100644 --- a/requirements/runtime-deps.in +++ b/requirements/runtime-deps.in @@ -4,10 +4,10 @@ aiodns >= 3.3.0 aiohappyeyeballs >= 2.5.0 aiosignal >= 1.4.0 async-timeout >= 4.0, < 6.0 ; python_version < "3.11" +backports.zstd; platform_python_implementation == 'CPython' and python_version < "3.14" Brotli; platform_python_implementation == 'CPython' brotlicffi; platform_python_implementation != 'CPython' frozenlist >= 1.1.1 multidict >=4.5, < 7.0 propcache >= 0.2.0 yarl >= 1.17.0, < 2.0 -zstandard; platform_python_implementation == 'CPython' and python_version < "3.14" diff --git a/requirements/runtime-deps.txt b/requirements/runtime-deps.txt index 9af01567e76..c97bd0cd4bb 100644 --- a/requirements/runtime-deps.txt +++ b/requirements/runtime-deps.txt @@ -12,6 +12,8 @@ aiosignal==1.4.0 # via -r requirements/runtime-deps.in async-timeout==5.0.1 ; python_version < "3.11" # via -r requirements/runtime-deps.in +backports-zstd==0.5.0 ; platform_python_implementation == "CPython" and python_version < "3.14" + # via -r requirements/runtime-deps.in brotli==1.1.0 ; platform_python_implementation == "CPython" # via -r requirements/runtime-deps.in cffi==2.0.0 @@ -26,7 +28,7 @@ multidict==6.7.0 # via # -r requirements/runtime-deps.in # yarl -propcache==0.4.0 +propcache==0.4.1 # via # -r requirements/runtime-deps.in # yarl @@ -38,7 +40,5 @@ typing-extensions==4.15.0 # via # aiosignal # multidict -yarl==1.21.0 - # via -r requirements/runtime-deps.in -zstandard==0.25.0 ; platform_python_implementation == "CPython" and python_version < "3.14" +yarl==1.22.0 # via -r requirements/runtime-deps.in diff --git a/requirements/test-ft.txt b/requirements/test-ft.txt index 83762508513..f1491d4ed03 100644 --- a/requirements/test-ft.txt +++ b/requirements/test-ft.txt @@ -14,6 +14,8 @@ annotated-types==0.7.0 # via pydantic async-timeout==5.0.1 ; python_version < "3.11" # via -r requirements/runtime-deps.in +backports-zstd==0.5.0 ; platform_python_implementation == "CPython" and python_version < "3.14" + # via -r requirements/runtime-deps.in blockbuster==1.5.25 # via -r requirements/test-common.in brotli==1.1.0 ; platform_python_implementation == "CPython" @@ -77,7 +79,7 @@ pluggy==1.6.0 # via # pytest # pytest-cov -propcache==0.4.0 +propcache==0.4.1 # via # -r requirements/runtime-deps.in # yarl @@ -142,9 +144,7 @@ typing-inspection==0.4.2 # via pydantic wait-for-it==2.3.0 # via -r requirements/test-common.in -yarl==1.21.0 +yarl==1.22.0 # via -r requirements/runtime-deps.in zlib-ng==1.0.0 # via -r requirements/test-common.in -zstandard==0.25.0 ; platform_python_implementation == "CPython" and python_version < "3.14" - # via -r requirements/runtime-deps.in diff --git a/requirements/test.txt b/requirements/test.txt index 101f2cb9018..7a149924182 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,6 +14,8 @@ annotated-types==0.7.0 # via pydantic async-timeout==5.0.1 ; python_version < "3.11" # via -r requirements/runtime-deps.in +backports-zstd==0.5.0 ; platform_python_implementation == "CPython" and python_version < "3.14" + # via -r requirements/runtime-deps.in blockbuster==1.5.25 # via -r requirements/test-common.in brotli==1.1.0 ; platform_python_implementation == "CPython" @@ -77,7 +79,7 @@ pluggy==1.6.0 # via # pytest # pytest-cov -propcache==0.4.0 +propcache==0.4.1 # via # -r requirements/runtime-deps.in # yarl @@ -144,9 +146,7 @@ uvloop==0.21.0 ; platform_system != "Windows" and implementation_name == "cpytho # via -r requirements/base.in wait-for-it==2.3.0 # via -r requirements/test-common.in -yarl==1.21.0 +yarl==1.22.0 # via -r requirements/runtime-deps.in zlib-ng==1.0.0 # via -r requirements/test-common.in -zstandard==0.25.0 ; platform_python_implementation == "CPython" and python_version < "3.14" - # via -r requirements/runtime-deps.in diff --git a/setup.cfg b/setup.cfg index 7fd0f0c59c6..cd12bc3013c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,7 +69,7 @@ speedups = aiodns >= 3.3.0 Brotli; platform_python_implementation == 'CPython' brotlicffi; platform_python_implementation != 'CPython' - zstandard; platform_python_implementation == 'CPython' and python_version < "3.14" + backports.zstd; platform_python_implementation == 'CPython' and python_version < "3.14" [options.packages.find] exclude = diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 4aa965e0ae4..3daff5cd206 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -37,13 +37,13 @@ except ImportError: brotli = None -if sys.version_info >= (3, 14): - import compression.zstd as zstandard # noqa: I900 -else: - try: - import zstandard - except ImportError: - zstandard = None # type: ignore[assignment] +try: + if sys.version_info >= (3, 14): + import compression.zstd as zstandard # noqa: I900 + else: + import backports.zstd as zstandard +except ImportError: + zstandard = None # type: ignore[assignment] REQUEST_PARSERS = [HttpRequestParserPy] RESPONSE_PARSERS = [HttpResponseParserPy]