From 5fd4abb6db3b46b7a9eb31258f151fa9a9411202 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 15 Jan 2015 16:09:23 -0600 Subject: [PATCH] Close streamed requests explicitly If we don't explicitly close a response after streaming its download, then we can run into HTTPConnectionPool full warnings. It also will hurt performance if we have to continuously create new sockets for new responses. Calling close will return the connection to the pool so it can be reused. Note this is only necessary when streaming a response. If we don't stream it, then requests will return the connection to the pool for us. Change-Id: I803bd4dd0e769c233501d5e5ff07a19705fbe233 Closes-bug: 1341777 (cherry picked from commit f2107512ee4be7bfbc1aaefdef12e1cba1147777) --- glanceclient/common/http.py | 13 ++++++++++++- tests/utils.py | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index 0909e9598..518dd1195 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -234,7 +234,7 @@ def chunk_body(body): if content_type == 'application/octet-stream': # Do not read all response in memory when # downloading an image. - body_iter = resp.iter_content(chunk_size=CHUNKSIZE) + body_iter = _close_after_stream(resp, CHUNKSIZE) self.log_http_response(resp) else: content = resp.content @@ -269,3 +269,14 @@ def patch(self, url, **kwargs): def delete(self, url, **kwargs): return self._request('DELETE', url, **kwargs) + + +def _close_after_stream(response, chunk_size): + """Iterate over the content and ensure the response is closed after.""" + # Yield each chunk in the response body + for chunk in response.iter_content(chunk_size=chunk_size): + yield chunk + # Once we're done streaming the body, ensure everything is closed. + # This will return the connection to the HTTPConnectionPool in urllib3 + # and ideally reduce the number of HTTPConnectionPool full warnings. + response.close() diff --git a/tests/utils.py b/tests/utils.py index b2849e5f9..3b4b51a0f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -126,6 +126,9 @@ def ok(self): def read(self, amt): return self.body.read(amt) + def close(self): + pass + @property def content(self): if hasattr(self.body, "read"):