From 95b28c71fce258f3383ab6ef7f9b6a2a0a65ce46 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 11:10:08 -0600 Subject: [PATCH 1/9] Restore total_bytes to EmptyStreamReader (#10387) --- CHANGES/10387.bugfix.rst | 1 + aiohttp/streams.py | 1 + tests/test_streams.py | 1 + 3 files changed, 3 insertions(+) create mode 100644 CHANGES/10387.bugfix.rst diff --git a/CHANGES/10387.bugfix.rst b/CHANGES/10387.bugfix.rst new file mode 100644 index 00000000000..ad1ead9e363 --- /dev/null +++ b/CHANGES/10387.bugfix.rst @@ -0,0 +1 @@ +Restored the missing ``total_bytes`` attribute to ``EmptyStreamReader`` -- by :user:`bdraco`. diff --git a/aiohttp/streams.py b/aiohttp/streams.py index ca7a420c6d5..db22f162396 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -540,6 +540,7 @@ class EmptyStreamReader(StreamReader): # lgtm [py/missing-call-to-init] def __init__(self) -> None: self._read_eof_chunk = False + self.total_bytes = 0 def __repr__(self) -> str: return "<%s>" % self.__class__.__name__ diff --git a/tests/test_streams.py b/tests/test_streams.py index fe591ea3c64..4305f892eea 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -1119,6 +1119,7 @@ async def test_empty_stream_reader() -> None: with pytest.raises(asyncio.IncompleteReadError): await s.readexactly(10) assert s.read_nowait() == b"" + assert s.total_bytes == 0 async def test_empty_stream_reader_iter_chunks() -> None: From 25c7f2382be9b5e03b8d22671f36a2f6fb07221a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 11:18:30 -0600 Subject: [PATCH 2/9] Restore zero copy writes on Python 3.12.9+/3.13.2+ (#10137) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- .github/workflows/ci-cd.yml | 4 +- CHANGES/10137.misc.rst | 3 + aiohttp/http_writer.py | 17 +++++- tests/test_http_writer.py | 109 +++++++++++++++++++++++++++++++++++- 4 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 CHANGES/10137.misc.rst diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 89a5bf7df02..5f46c0270af 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -266,11 +266,11 @@ jobs: uses: actions/checkout@v4 with: submodules: true - - name: Setup Python 3.13 + - name: Setup Python 3.13.2 id: python-install uses: actions/setup-python@v5 with: - python-version: 3.13 + python-version: 3.13.2 cache: pip cache-dependency-path: requirements/*.txt - name: Update pip, wheel, setuptools, build, twine diff --git a/CHANGES/10137.misc.rst b/CHANGES/10137.misc.rst new file mode 100644 index 00000000000..43b19c33f32 --- /dev/null +++ b/CHANGES/10137.misc.rst @@ -0,0 +1,3 @@ +Restored support for zero copy writes when using Python 3.12 versions 3.12.9 and later or Python 3.13.2+ -- by :user:`bdraco`. + +Zero copy writes were previously disabled due to :cve:`2024-12254` which is resolved in these Python versions. diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 28b14f7a791..e031a97708d 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -1,6 +1,7 @@ """Http related parsers and protocol.""" import asyncio +import sys import zlib from typing import ( # noqa Any, @@ -24,6 +25,17 @@ __all__ = ("StreamWriter", "HttpVersion", "HttpVersion10", "HttpVersion11") +MIN_PAYLOAD_FOR_WRITELINES = 2048 +IS_PY313_BEFORE_313_2 = (3, 13, 0) <= sys.version_info < (3, 13, 2) +IS_PY_BEFORE_312_9 = sys.version_info < (3, 12, 9) +SKIP_WRITELINES = IS_PY313_BEFORE_313_2 or IS_PY_BEFORE_312_9 +# writelines is not safe for use +# on Python 3.12+ until 3.12.9 +# on Python 3.13+ until 3.13.2 +# and on older versions it not any faster than write +# CVE-2024-12254: https://github.com/python/cpython/pull/127656 + + class HttpVersion(NamedTuple): major: int minor: int @@ -90,7 +102,10 @@ def _writelines(self, chunks: Iterable[bytes]) -> None: transport = self._protocol.transport if transport is None or transport.is_closing(): raise ClientConnectionResetError("Cannot write to closing transport") - transport.write(b"".join(chunks)) + if SKIP_WRITELINES or size < MIN_PAYLOAD_FOR_WRITELINES: + transport.write(b"".join(chunks)) + else: + transport.writelines(chunks) async def write( self, diff --git a/tests/test_http_writer.py b/tests/test_http_writer.py index 65afc05ae10..95dabeab377 100644 --- a/tests/test_http_writer.py +++ b/tests/test_http_writer.py @@ -2,7 +2,7 @@ import array import asyncio import zlib -from typing import Any, Iterable +from typing import Any, Generator, Iterable from unittest import mock import pytest @@ -13,6 +13,18 @@ from aiohttp.test_utils import make_mocked_coro +@pytest.fixture +def enable_writelines() -> Generator[None, None, None]: + with mock.patch("aiohttp.http_writer.SKIP_WRITELINES", False): + yield + + +@pytest.fixture +def force_writelines_small_payloads() -> Generator[None, None, None]: + with mock.patch("aiohttp.http_writer.MIN_PAYLOAD_FOR_WRITELINES", 1): + yield + + @pytest.fixture def buf() -> bytearray: return bytearray() @@ -136,6 +148,33 @@ async def test_write_large_payload_deflate_compression_data_in_eof( assert zlib.decompress(content) == (b"data" * 4096) + payload +@pytest.mark.usefixtures("enable_writelines") +async def test_write_large_payload_deflate_compression_data_in_eof_writelines( + protocol: BaseProtocol, + transport: asyncio.Transport, + loop: asyncio.AbstractEventLoop, +) -> None: + msg = http.StreamWriter(protocol, loop) + msg.enable_compression("deflate") + + await msg.write(b"data" * 4096) + assert transport.write.called # type: ignore[attr-defined] + chunks = [c[1][0] for c in list(transport.write.mock_calls)] # type: ignore[attr-defined] + transport.write.reset_mock() # type: ignore[attr-defined] + assert not transport.writelines.called # type: ignore[attr-defined] + + # This payload compresses to 20447 bytes + payload = b"".join( + [bytes((*range(0, i), *range(i, 0, -1))) for i in range(255) for _ in range(64)] + ) + await msg.write_eof(payload) + assert not transport.write.called # type: ignore[attr-defined] + assert transport.writelines.called # type: ignore[attr-defined] + chunks.extend(transport.writelines.mock_calls[0][1][0]) # type: ignore[attr-defined] + content = b"".join(chunks) + assert zlib.decompress(content) == (b"data" * 4096) + payload + + async def test_write_payload_chunked_filter( protocol: BaseProtocol, transport: asyncio.Transport, @@ -207,6 +246,26 @@ async def test_write_payload_deflate_compression_chunked( assert content == expected +@pytest.mark.usefixtures("enable_writelines") +@pytest.mark.usefixtures("force_writelines_small_payloads") +async def test_write_payload_deflate_compression_chunked_writelines( + protocol: BaseProtocol, + transport: asyncio.Transport, + loop: asyncio.AbstractEventLoop, +) -> None: + expected = b"2\r\nx\x9c\r\na\r\nKI,I\x04\x00\x04\x00\x01\x9b\r\n0\r\n\r\n" + msg = http.StreamWriter(protocol, loop) + msg.enable_compression("deflate") + msg.enable_chunking() + await msg.write(b"data") + await msg.write_eof() + + chunks = [b"".join(c[1][0]) for c in list(transport.writelines.mock_calls)] # type: ignore[attr-defined] + assert all(chunks) + content = b"".join(chunks) + assert content == expected + + async def test_write_payload_deflate_and_chunked( buf: bytearray, protocol: BaseProtocol, @@ -243,6 +302,26 @@ async def test_write_payload_deflate_compression_chunked_data_in_eof( assert content == expected +@pytest.mark.usefixtures("enable_writelines") +@pytest.mark.usefixtures("force_writelines_small_payloads") +async def test_write_payload_deflate_compression_chunked_data_in_eof_writelines( + protocol: BaseProtocol, + transport: asyncio.Transport, + loop: asyncio.AbstractEventLoop, +) -> None: + expected = b"2\r\nx\x9c\r\nd\r\nKI,IL\xcdK\x01\x00\x0b@\x02\xd2\r\n0\r\n\r\n" + msg = http.StreamWriter(protocol, loop) + msg.enable_compression("deflate") + msg.enable_chunking() + await msg.write(b"data") + await msg.write_eof(b"end") + + chunks = [b"".join(c[1][0]) for c in list(transport.writelines.mock_calls)] # type: ignore[attr-defined] + assert all(chunks) + content = b"".join(chunks) + assert content == expected + + async def test_write_large_payload_deflate_compression_chunked_data_in_eof( protocol: BaseProtocol, transport: asyncio.Transport, @@ -269,6 +348,34 @@ async def test_write_large_payload_deflate_compression_chunked_data_in_eof( assert zlib.decompress(content) == (b"data" * 4096) + payload +@pytest.mark.usefixtures("enable_writelines") +@pytest.mark.usefixtures("force_writelines_small_payloads") +async def test_write_large_payload_deflate_compression_chunked_data_in_eof_writelines( + protocol: BaseProtocol, + transport: asyncio.Transport, + loop: asyncio.AbstractEventLoop, +) -> None: + msg = http.StreamWriter(protocol, loop) + msg.enable_compression("deflate") + msg.enable_chunking() + + await msg.write(b"data" * 4096) + # This payload compresses to 1111 bytes + payload = b"".join([bytes((*range(0, i), *range(i, 0, -1))) for i in range(255)]) + await msg.write_eof(payload) + assert not transport.write.called # type: ignore[attr-defined] + + chunks = [] + for write_lines_call in transport.writelines.mock_calls: # type: ignore[attr-defined] + chunked_payload = list(write_lines_call[1][0])[1:] + chunked_payload.pop() + chunks.extend(chunked_payload) + + assert all(chunks) + content = b"".join(chunks) + assert zlib.decompress(content) == (b"data" * 4096) + payload + + async def test_write_payload_deflate_compression_chunked_connection_lost( protocol: BaseProtocol, transport: asyncio.Transport, From 9057364b78259fdd77bb10480813b00adc49ee43 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 13:08:27 -0600 Subject: [PATCH 3/9] Revert "Start building riscv64 platform wheels in CI/CD" (#10393) --- .github/workflows/ci-cd.yml | 2 -- CHANGES/10330.packaging.rst | 1 - 2 files changed, 3 deletions(-) delete mode 100644 CHANGES/10330.packaging.rst diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 5f46c0270af..aae1fc5f5cd 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -377,8 +377,6 @@ jobs: qemu: aarch64 - os: ubuntu qemu: ppc64le - - os: ubuntu - qemu: riscv64 - os: ubuntu qemu: s390x steps: diff --git a/CHANGES/10330.packaging.rst b/CHANGES/10330.packaging.rst deleted file mode 100644 index c159cf3a57d..00000000000 --- a/CHANGES/10330.packaging.rst +++ /dev/null @@ -1 +0,0 @@ -Started publishing ``riscv64`` wheels -- by :user:`eshattow`. From 481a8374a6fa7b75269f6e523348b2ce7cb498b6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 13:33:35 -0600 Subject: [PATCH 4/9] Switch to native arm runners for wheel builds (#10396) --- .github/workflows/ci-cd.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index aae1fc5f5cd..dcb70e66869 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -369,16 +369,27 @@ jobs: needs: pre-deploy strategy: matrix: - os: [ubuntu, windows, macos] + os: [ubuntu, windows, macos, "ubuntu-24.04-arm"] qemu: [''] + musl: [""] include: - # Split ubuntu job for the sake of speed-up + # Split ubuntu/musl jobs for the sake of speed-up - os: ubuntu - qemu: aarch64 + qemu: ppc64le + musl: "" + - os: ubuntu + qemu: s390x + musl: "" - os: ubuntu qemu: ppc64le + musl: musllinux - os: ubuntu qemu: s390x + musl: musllinux + - os: ubuntu + musl: musllinux + - os: ubuntu-24.04-arm + musl: musllinux steps: - name: Checkout uses: actions/checkout@v4 @@ -420,12 +431,13 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: + CIBW_SKIP: ${{ matrix.musl == 'musllinux' && '*manylinux*' || '*musllinux*' }} CIBW_ARCHS_MACOS: x86_64 arm64 universal2 - name: Upload wheels uses: actions/upload-artifact@v4 with: name: >- - dist-${{ matrix.os }}-${{ + dist-${{ matrix.os }}-${{ matrix.musl }}-${{ matrix.qemu && matrix.qemu || 'native' From 9b33be33d169f19842ae0a0f537163625fe3af77 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 14:11:42 -0600 Subject: [PATCH 5/9] Add workaround for segfaults during wheel builds (#10400) --- .github/workflows/ci-cd.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index dcb70e66869..34599b30a5b 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -364,7 +364,7 @@ jobs: permissions: contents: read # to fetch code (actions/checkout) - name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }} + name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }} ${{ matrix.musl }} runs-on: ${{ matrix.os }}-latest needs: pre-deploy strategy: @@ -400,6 +400,10 @@ jobs: uses: docker/setup-qemu-action@v3 with: platforms: all + # This should be temporary + # xref https://github.com/docker/setup-qemu-action/issues/188 + # xref https://github.com/tonistiigi/binfmt/issues/215 + image: tonistiigi/binfmt:qemu-v8.1.5 id: qemu - name: Prepare emulation run: | From 908145c97546afb717807d3bcad1f63110fdaa4a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 14:57:44 -0600 Subject: [PATCH 6/9] Disable wheel builds on PyPy (#10403) --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 34599b30a5b..de7d6534ab0 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -435,7 +435,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: - CIBW_SKIP: ${{ matrix.musl == 'musllinux' && '*manylinux*' || '*musllinux*' }} + CIBW_SKIP: pp* ${{ matrix.musl == 'musllinux' && '*manylinux*' || '*musllinux*' }} CIBW_ARCHS_MACOS: x86_64 arm64 universal2 - name: Upload wheels uses: actions/upload-artifact@v4 From 6ee81df6dd2f50e3df8dc2aafcc4cab035c71b9a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 15:04:29 -0600 Subject: [PATCH 7/9] Start build wheels on armv7l musllinux (#10404) --- .github/workflows/ci-cd.yml | 5 ++++- CHANGES/10404.packaging.rst | 1 + docs/spelling_wordlist.txt | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 CHANGES/10404.packaging.rst diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index de7d6534ab0..4326bf051a9 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -381,7 +381,10 @@ jobs: qemu: s390x musl: "" - os: ubuntu - qemu: ppc64le + qemu: armv7l + musl: musllinux + - os: ubuntu + qemu: s390x musl: musllinux - os: ubuntu qemu: s390x diff --git a/CHANGES/10404.packaging.rst b/CHANGES/10404.packaging.rst new file mode 100644 index 00000000000..e27ca91989f --- /dev/null +++ b/CHANGES/10404.packaging.rst @@ -0,0 +1 @@ +Started building armv7l musllinux wheels -- by :user:`bdraco`. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 121d49aab20..5eabd185d05 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -14,6 +14,7 @@ app’s apps arg args +armv Arsenic async asyncio @@ -201,6 +202,7 @@ multidicts Multidicts multipart Multipart +musllinux mypy Nagle Nagle’s From 1009c066dc88de1360bfb1d011cd0cc9addeb534 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 15:40:11 -0600 Subject: [PATCH 8/9] Fix runs-on for wheel builds for native arm (#10410) --- .github/workflows/ci-cd.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 4326bf051a9..cf804f0afde 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -365,31 +365,31 @@ jobs: contents: read # to fetch code (actions/checkout) name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }} ${{ matrix.musl }} - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: pre-deploy strategy: matrix: - os: [ubuntu, windows, macos, "ubuntu-24.04-arm"] + os: ["ubuntu-latest", "windows-latest", "macos-latest", "ubuntu-24.04-arm"] qemu: [''] musl: [""] include: # Split ubuntu/musl jobs for the sake of speed-up - - os: ubuntu + - os: ubuntu-latest qemu: ppc64le musl: "" - - os: ubuntu + - os: ubuntu-latest qemu: s390x musl: "" - - os: ubuntu + - os: ubuntu-latest qemu: armv7l musl: musllinux - - os: ubuntu + - os: ubuntu-latest qemu: s390x musl: musllinux - - os: ubuntu + - os: ubuntu-latest qemu: s390x musl: musllinux - - os: ubuntu + - os: ubuntu-latest musl: musllinux - os: ubuntu-24.04-arm musl: musllinux From f6dae31db7538882be97e6d56f79c527636f62f3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 5 Feb 2025 16:17:17 -0600 Subject: [PATCH 9/9] Fix missing ppc64le musllinux wheels (#10413) --- .github/workflows/ci-cd.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index cf804f0afde..01b27abefc2 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -377,17 +377,17 @@ jobs: - os: ubuntu-latest qemu: ppc64le musl: "" + - os: ubuntu-latest + qemu: ppc64le + musl: musllinux - os: ubuntu-latest qemu: s390x musl: "" - - os: ubuntu-latest - qemu: armv7l - musl: musllinux - os: ubuntu-latest qemu: s390x musl: musllinux - os: ubuntu-latest - qemu: s390x + qemu: armv7l musl: musllinux - os: ubuntu-latest musl: musllinux