From bbcc374cb4b56a71aa6433ae3297bd5f26087e7d Mon Sep 17 00:00:00 2001 From: John Ford Date: Mon, 15 Feb 2016 12:23:48 +0100 Subject: [PATCH 1/2] Use sessions for requests --- taskcluster/client.py | 4 ++++ taskcluster/utils.py | 9 +++++---- test/test_client.py | 6 ++++++ test/test_utils.py | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/taskcluster/client.py b/taskcluster/client.py index 0959f26..c3d1e3e 100644 --- a/taskcluster/client.py +++ b/taskcluster/client.py @@ -68,6 +68,10 @@ def __init__(self, options=None): if 'credentials' in o: log.debug('credentials key scrubbed from logging output') log.debug(dict((k, v) for k, v in o.items() if k != 'credentials')) + self.session = requests.Session() + + def resetSession(self): + self.session = requests.Session() def makeHawkExt(self): """ Make an 'ext' for Hawk authentication """ diff --git a/taskcluster/utils.py b/taskcluster/utils.py index 415a8f4..6b3a1b5 100644 --- a/taskcluster/utils.py +++ b/taskcluster/utils.py @@ -138,7 +138,7 @@ def scope_match(assumed_scopes, required_scope_sets): return False # none of the required_scope_sets were satisfied -def makeHttpRequest(method, url, payload, headers, retries=MAX_RETRIES): +def makeHttpRequest(method, url, payload, headers, retries=MAX_RETRIES, session=None): """ Make an HTTP request and retry it until success, return request """ retry = -1 response = None @@ -156,7 +156,7 @@ def makeHttpRequest(method, url, payload, headers, retries=MAX_RETRIES): log.debug('Making attempt %d', retry) try: - response = makeSingleHttpRequest(method, url, payload, headers) + response = makeSingleHttpRequest(method, url, payload, headers, session) except requests.exceptions.RequestException as rerr: if retry < retries: log.warn('Retrying because of: %s' % rerr) @@ -180,12 +180,13 @@ def makeHttpRequest(method, url, payload, headers, retries=MAX_RETRIES): assert False, "Error from last retry should have been raised!" -def makeSingleHttpRequest(method, url, payload, headers): +def makeSingleHttpRequest(method, url, payload, headers, session=None): method = method.upper() log.debug('Making a %s request to %s', method, url) log.debug('HTTP Headers: %s' % str(headers)) log.debug('HTTP Payload: %s (limit 100 char)' % str(payload)[:100]) - response = requests.request(method.upper(), url, data=payload, headers=headers) + obj = session if session else requests + response = obj.request(method.upper(), url, data=payload, headers=headers) log.debug('Received HTTP Status: %s' % response.status_code) log.debug('Received HTTP Headers: %s' % str(response.headers)) log.debug('Received HTTP Payload: %s (limit 1024 char)' % str(response.text)[:1024]) diff --git a/test/test_client.py b/test/test_client.py index 7a7af01..b18fa73 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -628,6 +628,12 @@ def test_insert_to_index(self): } result = self.i.insertTask('testing', payload) self.assertEqual(payload['expires'], result['expires']) + payload['rank'] += 1 + result = self.i.insertTask('testing', payload) + self.assertEqual(payload['expires'], result['expires']) + payload['rank'] += 1 + result = self.i.insertTask('testing', payload) + self.assertEqual(payload['expires'], result['expires']) def test_listworkertypes_signed_url(self): surl = self.a.buildSignedUrl('listWorkerTypes') diff --git a/test/test_utils.py b/test/test_utils.py index a9ae364..f39ea03 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -167,7 +167,7 @@ class TestPutfile(base.TCTest): def test_success_put_file(self): with mock.patch.object(subject, 'makeSingleHttpRequest') as p: subject.putFile('setup.py', 'http://www.example.com', 'text/plain') - p.assert_called_once_with('put', 'http://www.example.com', mock.ANY, mock.ANY) + p.assert_called_once_with('put', 'http://www.example.com', mock.ANY, mock.ANY, mock.ANY) class TestStableSlugIdClosure(TestCase): From f537e432c1161ec39871b0eeb150e56fbb8ad4e0 Mon Sep 17 00:00:00 2001 From: John Ford Date: Wed, 17 Feb 2016 15:55:40 +0100 Subject: [PATCH 2/2] allow passing session into client --- taskcluster/client.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/taskcluster/client.py b/taskcluster/client.py index c3d1e3e..c8f2656 100644 --- a/taskcluster/client.py +++ b/taskcluster/client.py @@ -41,6 +41,13 @@ } +def createSession(*args, **kwargs): + """ Create a new requests session. This passes through all positional and + keyword arguments to the requests.Session() constructor + """ + return requests.Session(*args, **kwargs) + + class BaseClient(object): """ Base Class for API Client Classes. Each individual Client class needs to set up its own methods for REST endpoints and Topic Exchange @@ -48,7 +55,7 @@ class BaseClient(object): help with this. """ - def __init__(self, options=None): + def __init__(self, options=None, session=None): o = copy.deepcopy(self.classOptions) o.update(_defaultConfig) if options: @@ -68,10 +75,11 @@ def __init__(self, options=None): if 'credentials' in o: log.debug('credentials key scrubbed from logging output') log.debug(dict((k, v) for k, v in o.items() if k != 'credentials')) - self.session = requests.Session() - def resetSession(self): - self.session = requests.Session() + if session: + self.session = options.session + else: + self.session = createSession() def makeHawkExt(self): """ Make an 'ext' for Hawk authentication """