From 69403afa876521d1eb15156b9d8dae215b62773a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 12 Jul 2020 22:39:39 -0600 Subject: [PATCH 1/8] Speed up appservice transactions when sending them This should be done at an earlier level such as where the transactions are calculated, however that doesn't solve the immediate backlog problem on t2bot.io Instead, this alters the HTTP layer to short-circuit the transaction away from the appservice where possible to avoid delaying it. --- synapse/appservice/api.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 0963fb3bb4fb..cc54115e683c 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import time import urllib.parse from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Tuple @@ -308,6 +309,10 @@ async def push_bulk( "left": list(device_list_summary.left), } + if len(serialized_events) == 0 and len(ephemeral) == 0: + logger.info("Returning early on transaction: no events to send") + return True + try: await self.put_json( uri=uri, @@ -348,7 +353,19 @@ async def push_bulk( def _serialize( self, service: "ApplicationService", events: Iterable[EventBase] ) -> List[JsonDict]: + new_events = [] time_now = self.clock.time_msec() + + for event in events: + if int(round(time.time() * 1000)) - event.origin_server_ts > (15 * 60 * 1000): + logger.warning("Dropping event (due to age) %s" % event.event_id) + continue + if service.id != "github" and service.is_interested_in_user(event.sender) and event.sender.endswith(":t2bot.io"): + logger.warning("Dropping event (due to echo) %s" % event.event_id) + continue + logger.info("Allowing @ fallback: %s" % event.event_id) + new_events.append(event) + return [ serialize_event( e, @@ -367,5 +384,5 @@ def _serialize( ), ), ) - for e in events + for e in new_events ] From 6d0269457b07fb2667776913f196f06b5711b2ff Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 8 Dec 2020 19:43:41 -0700 Subject: [PATCH 2/8] Raise concurrency limit for rooms during /send --- synapse/federation/federation_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 907940e19eb0..075c9dd85148 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -85,7 +85,7 @@ # when processing incoming transactions, we try to handle multiple rooms in # parallel, up to this limit. -TRANSACTION_CONCURRENCY_LIMIT = 10 +TRANSACTION_CONCURRENCY_LIMIT = 50 # T2B: Raise from 10 logger = logging.getLogger(__name__) From 2437e3a384888b37b481d0896179301db9bc4f9a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 7 Jun 2021 15:10:44 -0600 Subject: [PATCH 3/8] Adjust timeouts --- synapse/federation/federation_client.py | 2 +- synapse/federation/transport/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 4dca711cd28d..9ee21de8437e 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -355,7 +355,7 @@ async def get_pdu( destinations: Iterable[str], event_id: str, room_version: RoomVersion, - timeout: Optional[int] = None, + timeout: Optional[int] = 15000, ) -> Optional[EventBase]: """Requests the PDU with given origin and ID from the remote home servers. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 32074b8ca690..4893f5ccf7d3 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -105,7 +105,7 @@ async def get_room_state( ) async def get_event( - self, destination: str, event_id: str, timeout: Optional[int] = None + self, destination: str, event_id: str, timeout: Optional[int] = 15000 ) -> JsonDict: """Requests the pdu with give id and origin from the given server. From d01183dc4b6076ce7865ab684023d2fd70f3eb73 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Aug 2021 10:59:53 -0600 Subject: [PATCH 4/8] Selectively handle EDUs --- .../federation/transport/server/federation.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index 6bb4659c4c9e..6caea0ee6cfe 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -100,14 +100,32 @@ async def on_PUT( logger.debug("Decoded %s: %s", transaction_id, str(transaction_data)) + edus_before_filter = len(transaction_data.get("edus", [])) + + filtered_edus = [] + for edu in transaction_data.get("edus", []): + edu_type = edu.get('edu_type', 'io.t2bot.ignored') + if edu_type == 'io.t2bot.ignored': + continue + if edu_type == 'm.presence': + continue + if edu_type == 'm.receipt': + continue + if edu_type == 'm.typing': + continue + filtered_edus.append(edu) + logger.info( - "Received txn %s from %s. (PDUs: %d, EDUs: %d)", + "Received txn %s from %s. (PDUs: %d, Accepted EDUs: %d, Ignored EDUs: %d)", transaction_id, origin, len(transaction_data.get("pdus", [])), - len(transaction_data.get("edus", [])), + len(filtered_edus), + edus_before_filter - len(filtered_edus), ) + transaction_data["edus"] = filtered_edus + if issue_8631_logger.isEnabledFor(logging.DEBUG): DEVICE_UPDATE_EDUS = [ EduTypes.DEVICE_LIST_UPDATE, From c0cb7af4fa0b7ffef76014153e9fc4a24bf194ea Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Sep 2021 10:48:13 -0600 Subject: [PATCH 5/8] Disable push processing --- synapse/handlers/federation_event.py | 2 +- synapse/handlers/message.py | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 778d8869b3c7..6315cb06f2d1 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -2101,7 +2101,7 @@ async def _run_push_actions_and_persist_event( # persist_events_and_notify directly.) assert not event.internal_metadata.outlier - if not backfilled and not context.rejected: + if False and not backfilled and not context.rejected: min_depth = await self._store.get_min_depth(event.room_id) if min_depth is None or min_depth > event.depth: # XXX richvdh 2021/10/07: I don't really understand what this diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 00e7645ba5cc..61cba88fb93c 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1425,17 +1425,18 @@ async def _persist_events( a room that has been un-partial stated. """ - for event, context in events_and_context: - # Skip push notification actions for historical messages - # because we don't want to notify people about old history back in time. - # The historical messages also do not have the proper `context.current_state_ids` - # and `state_groups` because they have `prev_events` that aren't persisted yet - # (historical messages persisted in reverse-chronological order). - if not event.internal_metadata.is_historical(): - with opentracing.start_active_span("calculate_push_actions"): - await self._bulk_push_rule_evaluator.action_for_event_by_user( - event, context - ) + # T2B: Disable push processing. + #for event, context in events_and_context: + # # Skip push notification actions for historical messages + # # because we don't want to notify people about old history back in time. + # # The historical messages also do not have the proper `context.current_state_ids` + # # and `state_groups` because they have `prev_events` that aren't persisted yet + # # (historical messages persisted in reverse-chronological order). + # if not event.internal_metadata.is_historical(): + # with opentracing.start_active_span("calculate_push_actions"): + # await self._bulk_push_rule_evaluator.action_for_event_by_user( + # event, context + # ) try: # If we're a worker we need to hit out to the master. From 5fb59658ca580a1cf99dd719f8a255ccd3d5e085 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 10 Mar 2022 21:08:53 -0700 Subject: [PATCH 6/8] Don't send presence to matrix.org --- synapse/federation/sender/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index a6cb3ba58f0b..6f91a785418f 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -695,6 +695,11 @@ def send_presence_to_destinations( for destination in destinations: if destination == self.server_name: continue + + # T2B: Skip sending presence to servers we know don't support it + if destination == "matrix.org": + continue + if not self._federation_shard_config.should_handle( self._instance_name, destination ): From b244c2b006fa2fc2c508dc2640f8caa7bad6f544 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 19 May 2022 22:38:22 -0600 Subject: [PATCH 7/8] Hardcode the user directory to be disabled --- synapse/handlers/user_directory.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 3610b6bf785e..bdda5e2f0f6e 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -70,7 +70,8 @@ def __init__(self, hs: "HomeServer"): # Guard to ensure we only process deltas one at a time self._is_processing = False - if self.update_user_directory: + # T2B: Disable user directory + if self.update_user_directory and False: self.notifier.add_replication_callback(self.notify_new_event) # We kick this off so that we don't have to wait for a change before @@ -109,6 +110,11 @@ async def search_users( def notify_new_event(self) -> None: """Called when there may be more deltas to process""" + + # T2B: Disable user directory + if True: + return + if not self.update_user_directory: return @@ -133,6 +139,10 @@ async def handle_local_profile_change( # FIXME(#3714): We should probably do this in the same worker as all # the other changes. + # T2B: Disable user directory + if True: + return + if await self.store.should_include_local_user_in_dir(user_id): await self.store.update_profile_in_user_dir( user_id, profile.display_name, profile.avatar_url @@ -142,6 +152,11 @@ async def handle_local_user_deactivated(self, user_id: str) -> None: """Called when a user ID is deactivated""" # FIXME(#3714): We should probably do this in the same worker as all # the other changes. + + # T2B: Disable user directory + if True: + return + await self.store.remove_from_user_dir(user_id) async def _unsafe_process(self) -> None: From f646cebbf1fe70b2c644d8f63f5534a542d9409f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 21:35:05 +0000 Subject: [PATCH 8/8] Bump types-bleach from 4.1.4 to 5.0.3 Bumps [types-bleach](https://github.com/python/typeshed) from 4.1.4 to 5.0.3. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-bleach dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- poetry.lock | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/poetry.lock b/poetry.lock index 63ef8573a075..7ab6d245fcb6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1367,7 +1367,7 @@ python-versions = ">=3.6" [[package]] name = "types-bleach" -version = "4.1.4" +version = "5.0.3" description = "Typing stubs for bleach" category = "dev" optional = false @@ -1993,10 +1993,7 @@ keyring = [ {file = "keyring-23.5.0.tar.gz", hash = "sha256:9012508e141a80bd1c0b6778d5c610dd9f8c464d75ac6774248500503f972fb9"}, ] ldap3 = [ - {file = "ldap3-2.9.1-py2.6.egg", hash = "sha256:5ab7febc00689181375de40c396dcad4f2659cd260fc5e94c508b6d77c17e9d5"}, - {file = "ldap3-2.9.1-py2.7.egg", hash = "sha256:2bc966556fc4d4fa9f445a1c31dc484ee81d44a51ab0e2d0fd05b62cac75daa6"}, {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, - {file = "ldap3-2.9.1-py3.9.egg", hash = "sha256:5630d1383e09ba94839e253e013f1aa1a2cf7a547628ba1265cb7b9a844b5687"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, ] lxml = [ @@ -2287,34 +2284,12 @@ psycopg2cffi-compat = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] pyasn1 = [ - {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, - {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"}, - {file = "pyasn1-0.4.8-py2.6.egg", hash = "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00"}, - {file = "pyasn1-0.4.8-py2.7.egg", hash = "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8"}, {file = "pyasn1-0.4.8-py2.py3-none-any.whl", hash = "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d"}, - {file = "pyasn1-0.4.8-py3.1.egg", hash = "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86"}, - {file = "pyasn1-0.4.8-py3.2.egg", hash = "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7"}, - {file = "pyasn1-0.4.8-py3.3.egg", hash = "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576"}, - {file = "pyasn1-0.4.8-py3.4.egg", hash = "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12"}, - {file = "pyasn1-0.4.8-py3.5.egg", hash = "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2"}, - {file = "pyasn1-0.4.8-py3.6.egg", hash = "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359"}, - {file = "pyasn1-0.4.8-py3.7.egg", hash = "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776"}, {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"}, ] pyasn1-modules = [ {file = "pyasn1-modules-0.2.8.tar.gz", hash = "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e"}, - {file = "pyasn1_modules-0.2.8-py2.4.egg", hash = "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199"}, - {file = "pyasn1_modules-0.2.8-py2.5.egg", hash = "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405"}, - {file = "pyasn1_modules-0.2.8-py2.6.egg", hash = "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb"}, - {file = "pyasn1_modules-0.2.8-py2.7.egg", hash = "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8"}, {file = "pyasn1_modules-0.2.8-py2.py3-none-any.whl", hash = "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74"}, - {file = "pyasn1_modules-0.2.8-py3.1.egg", hash = "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d"}, - {file = "pyasn1_modules-0.2.8-py3.2.egg", hash = "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45"}, - {file = "pyasn1_modules-0.2.8-py3.3.egg", hash = "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4"}, - {file = "pyasn1_modules-0.2.8-py3.4.egg", hash = "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811"}, - {file = "pyasn1_modules-0.2.8-py3.5.egg", hash = "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed"}, - {file = "pyasn1_modules-0.2.8-py3.6.egg", hash = "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0"}, - {file = "pyasn1_modules-0.2.8-py3.7.egg", hash = "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd"}, ] pycodestyle = [ {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, @@ -2452,6 +2427,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -2720,8 +2702,8 @@ typed-ast = [ {file = "typed_ast-1.5.2.tar.gz", hash = "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27"}, ] types-bleach = [ - {file = "types-bleach-4.1.4.tar.gz", hash = "sha256:2d30c2c4fb6854088ac636471352c9a51bf6c089289800d2a8060820a01cd43a"}, - {file = "types_bleach-4.1.4-py3-none-any.whl", hash = "sha256:edffe173ed6d7b6f3543036a96204a9319c3bf6c3645917b14274e43f000cc9b"}, + {file = "types-bleach-5.0.3.tar.gz", hash = "sha256:f7b3df8278efe176d9670d0f063a66c866c77577f71f54b9c7a320e31b1a7bbd"}, + {file = "types_bleach-5.0.3-py3-none-any.whl", hash = "sha256:5931525d03571f36b2bb40210c34b662c4d26c8fd6f2b1e1e83fe4d2d2fd63c7"}, ] types-commonmark = [ {file = "types-commonmark-0.9.2.tar.gz", hash = "sha256:b894b67750c52fd5abc9a40a9ceb9da4652a391d75c1b480bba9cef90f19fc86"},