From 45bc2d7972e0c66e7aa026158b703a6da853dbc7 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 12 Nov 2025 15:40:24 +0100 Subject: [PATCH] ref(consumer): Make autocommit the default It has shown to not make things actively worse, and in a later PR we will be able to simplify the codebase a lot once we actually get rid of it (see https://github.com/getsentry/arroyo/pull/506) --- arroyo/backends/kafka/configuration.py | 2 +- docs/source/what_for.rst | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arroyo/backends/kafka/configuration.py b/arroyo/backends/kafka/configuration.py index 3b4eb089..1abc5eac 100644 --- a/arroyo/backends/kafka/configuration.py +++ b/arroyo/backends/kafka/configuration.py @@ -237,7 +237,7 @@ def build_kafka_consumer_configuration( bootstrap_servers: Optional[Sequence[str]] = None, override_params: Optional[Mapping[str, Any]] = None, strict_offset_reset: Optional[bool] = None, - enable_auto_commit: bool = False, + enable_auto_commit: bool = True, retry_handle_destroyed: bool = False, ) -> KafkaBrokerConfig: diff --git a/docs/source/what_for.rst b/docs/source/what_for.rst index 7e2d04fd..c7d721f6 100644 --- a/docs/source/what_for.rst +++ b/docs/source/what_for.rst @@ -79,11 +79,15 @@ reached its destination in the following way: # add this value to the config: "enable.auto.commit": "false" + "enable.auto.offset.store": "false" # ------- message = consumer.poll(timeout=0) send_to_destination(process_message(message)) consumer.commit(message.offset()) +Note: Arroyo still uses rdkafka's autocommit, but only to commit stored +offsets. Offsets are still stored explicitly rather than on poll. + High Throughput ---------------