From 3c65c07472ec4225e4440c340daa85e335b3bd3d Mon Sep 17 00:00:00 2001 From: Andi-Bogdan Postelnicu Date: Wed, 25 Jan 2023 18:09:02 +0200 Subject: [PATCH 1/5] Use a more generic way to retry http/s failures for different implementations. Fixes #95 --- libmozevent/phabricator.py | 9 +++++---- libmozevent/utils.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/libmozevent/phabricator.py b/libmozevent/phabricator.py index a011e4b..df1f4e4 100644 --- a/libmozevent/phabricator.py +++ b/libmozevent/phabricator.py @@ -5,6 +5,7 @@ import structlog from libmozdata.phabricator import PhabricatorAPI +from libmozevent.utils import get_session logger = structlog.get_logger(__name__) @@ -21,10 +22,10 @@ class PhabricatorBuild(object): """ def __init__(self, request): - self.diff_id = int(request.rel_url.query.get("diff", 0)) - self.repo_phid = request.rel_url.query.get("repo") - self.revision_id = int(request.rel_url.query.get("revision", 0)) - self.target_phid = request.rel_url.query.get("target") + self.diff_id = int(get_session("phabricator", request).rel_url.query.get("diff", 0)) + self.repo_phid = get_session("phabricator", request).rel_url.query.get("repo") + self.revision_id = int(get_session("phabricator", request).rel_url.query.get("revision", 0)) + self.target_phid = get_session("phabricator", request).rel_url.query.get("target") self.state = PhabricatorBuildState.Queued # Incremented on an unexpected failure during build's push to try self.retries = 0 diff --git a/libmozevent/utils.py b/libmozevent/utils.py index c8c6d2b..769f713 100644 --- a/libmozevent/utils.py +++ b/libmozevent/utils.py @@ -5,6 +5,7 @@ import asyncio import contextvars import fcntl +from functools import lru_cache import os import signal import time @@ -26,6 +27,24 @@ log = structlog.get_logger(__name__) +@lru_cache(maxsize=None) +def get_session(name: str, requests): + session = requests.Session() + + retry = Retry(total=9, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]) + + # Default HTTPAdapter uses 10 connections. Mount custom adapter to increase + # that limit. Connections are established as needed, so using a large value + # should not negatively impact performance. + http_adapter = requests.adapters.HTTPAdapter( + pool_connections=50, pool_maxsize=50, max_retries=retry + ) + session.mount("https://", http_adapter) + session.mount("http://", http_adapter) + + return session + + def run_tasks(awaitables: Iterable, bus_to_restore=None): """ Helper to run tasks concurrently. From 92712cee930195729914a3cef4d0e3ef436975ab Mon Sep 17 00:00:00 2001 From: Andi-Bogdan Postelnicu Date: Wed, 25 Jan 2023 18:13:31 +0200 Subject: [PATCH 2/5] Fix litning issues. --- libmozevent/phabricator.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libmozevent/phabricator.py b/libmozevent/phabricator.py index df1f4e4..f718307 100644 --- a/libmozevent/phabricator.py +++ b/libmozevent/phabricator.py @@ -22,10 +22,16 @@ class PhabricatorBuild(object): """ def __init__(self, request): - self.diff_id = int(get_session("phabricator", request).rel_url.query.get("diff", 0)) + self.diff_id = int( + get_session("phabricator", request).rel_url.query.get("diff", 0) + ) self.repo_phid = get_session("phabricator", request).rel_url.query.get("repo") - self.revision_id = int(get_session("phabricator", request).rel_url.query.get("revision", 0)) - self.target_phid = get_session("phabricator", request).rel_url.query.get("target") + self.revision_id = int( + get_session("phabricator", request).rel_url.query.get("revision", 0) + ) + self.target_phid = get_session("phabricator", request).rel_url.query.get( + "target" + ) self.state = PhabricatorBuildState.Queued # Incremented on an unexpected failure during build's push to try self.retries = 0 From 8291a13ae97a33181cdd06d153b80e7b24cc6cc6 Mon Sep 17 00:00:00 2001 From: Andi-Bogdan Postelnicu Date: Wed, 25 Jan 2023 18:19:27 +0200 Subject: [PATCH 3/5] Fix litning issues. --- libmozevent/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmozevent/utils.py b/libmozevent/utils.py index 769f713..11355ae 100644 --- a/libmozevent/utils.py +++ b/libmozevent/utils.py @@ -5,10 +5,10 @@ import asyncio import contextvars import fcntl -from functools import lru_cache import os import signal import time +from functools import lru_cache from typing import Iterable import hglib From 5e32faf858c000e464a3de4b9183e3f4a6a1084e Mon Sep 17 00:00:00 2001 From: Andi-Bogdan Postelnicu Date: Wed, 25 Jan 2023 18:23:37 +0200 Subject: [PATCH 4/5] Fix litning issues. --- libmozevent/phabricator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libmozevent/phabricator.py b/libmozevent/phabricator.py index f718307..daf2b2c 100644 --- a/libmozevent/phabricator.py +++ b/libmozevent/phabricator.py @@ -5,6 +5,7 @@ import structlog from libmozdata.phabricator import PhabricatorAPI + from libmozevent.utils import get_session logger = structlog.get_logger(__name__) From a81efd3c9f4f0518d402eed14f6d83b2c073a7cc Mon Sep 17 00:00:00 2001 From: Andi-Bogdan Postelnicu Date: Wed, 25 Jan 2023 18:28:27 +0200 Subject: [PATCH 5/5] Fix litning issues. --- libmozevent/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libmozevent/utils.py b/libmozevent/utils.py index 11355ae..8e2bbaa 100644 --- a/libmozevent/utils.py +++ b/libmozevent/utils.py @@ -23,6 +23,7 @@ from redis.exceptions import ResponseError from redis.exceptions import TimeoutError from redis.exceptions import WatchError +from requests.packages.urllib3.util.retry import Retry log = structlog.get_logger(__name__)