From 9a65826ee6f8dcec3594e32b3fd17f6f98344c7b Mon Sep 17 00:00:00 2001 From: smaeda-ks Date: Wed, 17 Jul 2019 17:51:39 +0900 Subject: [PATCH 1/6] Remove x-cost-rate-limit-* reference --- twitter_ads/http.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/twitter_ads/http.py b/twitter_ads/http.py index b1b7a08..b337731 100644 --- a/twitter_ads/http.py +++ b/twitter_ads/http.py @@ -13,7 +13,6 @@ else: import http.client as httplib -import dateutil.parser from datetime import datetime from requests_oauthlib import OAuth1Session from twitter_ads.utils import get_version @@ -143,10 +142,6 @@ def __init__(self, code, headers, **kwargs): self._rate_limit = int(headers['x-rate-limit-limit']) self._rate_limit_remaining = int(headers['x-rate-limit-remaining']) self._rate_limit_reset = datetime.fromtimestamp(int(headers['x-rate-limit-reset'])) - elif 'x-cost-rate-limit-reset' in headers: - self._rate_limit = int(headers['x-cost-rate-limit-limit']) - self._rate_limit_remaining = int(headers['x-cost-rate-limit-remaining']) - self._rate_limit_reset = dateutil.parser.parse(headers['x-cost-rate-limit-reset'].first) else: self._rate_limit = None self._rate_limit_remaining = None From 21a2963e909eec251141858c068f0574b792f025 Mon Sep 17 00:00:00 2001 From: smaeda-ks Date: Wed, 17 Jul 2019 18:01:37 +0900 Subject: [PATCH 2/6] Update inaccurate comment --- twitter_ads/http.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/twitter_ads/http.py b/twitter_ads/http.py index b337731..33fbcb0 100644 --- a/twitter_ads/http.py +++ b/twitter_ads/http.py @@ -123,12 +123,9 @@ def __init__(self, code, headers, **kwargs): self._raw_body = kwargs.get('raw_body', None) if headers.get('content-type') == 'application/gzip': - # hack because Twitter TON API doesn't return headers as it should - # and instead returns a gzipp'd file rather than a gzipp encoded response - # Content-Encoding: gzip - # Content-Type: application/json - # instead it returns: - # Content-Type: application/gzip + # Async analytics data arrives as a gzipped file so decompress it on-the-fly. + # Note: might need to consider using zlib.decompressobj() instead + # in case data streams gets large enough (data size doesn't fit into memory at once) raw_response_body = zlib.decompress(self._raw_body, 16 + zlib.MAX_WBITS).decode('utf-8') else: raw_response_body = self._raw_body From 9c5d8e7977b72479a7b099bd9e8f589411624d7c Mon Sep 17 00:00:00 2001 From: smaeda-ks Date: Thu, 18 Jul 2019 16:50:44 +0900 Subject: [PATCH 3/6] Expose rate limit headers into instance variables --- twitter_ads/cursor.py | 16 ++++++++++++++++ twitter_ads/http.py | 22 ---------------------- twitter_ads/resource.py | 19 ++++++++++++++++++- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/twitter_ads/cursor.py b/twitter_ads/cursor.py index d254e63..3768457 100644 --- a/twitter_ads/cursor.py +++ b/twitter_ads/cursor.py @@ -93,6 +93,22 @@ def __from_response(self, response): if 'total_count' in response.body: self._total_count = int(response.body['total_count']) + if 'x-rate-limit-reset' in response.headers: + setattr(self, 'rate_limit', + response.headers['x-rate-limit-limit']) + setattr(self, 'rate_limit_remaining', + response.headers['x-rate-limit-remaining']) + setattr(self, 'rate_limit_reset', + response.headers['x-rate-limit-reset']) + + if 'x-account-rate-limit-reset' in response.headers: + setattr(self, 'account_rate_limit', + response.headers['x-account-rate-limit-limit']) + setattr(self, 'account_rate_limit_remaining', + response.headers['x-account-rate-limit-remaining']) + setattr(self, 'account_rate_limit_reset', + response.headers['x-account-rate-limit-reset']) + for item in response.body['data']: if 'from_response' in dir(self._klass): init_with = self._options.get('init_with', None) diff --git a/twitter_ads/http.py b/twitter_ads/http.py index 33fbcb0..23d6db9 100644 --- a/twitter_ads/http.py +++ b/twitter_ads/http.py @@ -13,7 +13,6 @@ else: import http.client as httplib -from datetime import datetime from requests_oauthlib import OAuth1Session from twitter_ads.utils import get_version from twitter_ads.error import Error @@ -135,15 +134,6 @@ def __init__(self, code, headers, **kwargs): except ValueError: self._body = raw_response_body - if 'x-rate-limit-reset' in headers: - self._rate_limit = int(headers['x-rate-limit-limit']) - self._rate_limit_remaining = int(headers['x-rate-limit-remaining']) - self._rate_limit_reset = datetime.fromtimestamp(int(headers['x-rate-limit-reset'])) - else: - self._rate_limit = None - self._rate_limit_remaining = None - self._rate_limit_reset = None - @property def code(self): return self._code @@ -160,18 +150,6 @@ def body(self): def raw_body(self): return self._raw_body - @property - def rate_limit(self): - return self._rate_limit - - @property - def rate_limit_remaining(self): - return self._rate_limit_remaining - - @property - def rate_limit_reset(self): - return self._rate_limit_reset - @property def error(self): return True if (self._code >= 400 and self._code <= 599) else False diff --git a/twitter_ads/resource.py b/twitter_ads/resource.py index c5f2492..83d4977 100644 --- a/twitter_ads/resource.py +++ b/twitter_ads/resource.py @@ -42,12 +42,29 @@ def __init__(self, account): def account(self): return self._account - def from_response(self, response): + def from_response(self, response, headers=None): """ Populates a given objects attributes from a parsed JSON API response. This helper handles all necessary type coercions as it assigns attribute values. """ + if headers is not None: + if 'x-rate-limit-reset' in headers: + setattr(self, 'rate_limit', + headers['x-rate-limit-limit']) + setattr(self, 'rate_limit_remaining', + headers['x-rate-limit-remaining']) + setattr(self, 'rate_limit_reset', + headers['x-rate-limit-reset']) + + if 'x-account-rate-limit-reset' in headers: + setattr(self, 'account_rate_limit', + headers['x-account-rate-limit-limit']) + setattr(self, 'account_rate_limit_remaining', + headers['x-account-rate-limit-remaining']) + setattr(self, 'account_rate_limit_reset', + headers['x-account-rate-limit-reset']) + for name in self.PROPERTIES: attr = '_{0}'.format(name) transform = self.PROPERTIES[name].get('transform', None) From f0a10bed1d21399a9b4704145eca1928081e95de Mon Sep 17 00:00:00 2001 From: smaeda-ks Date: Thu, 18 Jul 2019 20:18:46 +0900 Subject: [PATCH 4/6] Add test - rate limit access --- tests/test_rate_limit.py | 91 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tests/test_rate_limit.py diff --git a/tests/test_rate_limit.py b/tests/test_rate_limit.py new file mode 100644 index 0000000..ba8d5d9 --- /dev/null +++ b/tests/test_rate_limit.py @@ -0,0 +1,91 @@ +import responses +import unittest + +from tests.support import with_resource, with_fixture, characters + +from twitter_ads.account import Account +from twitter_ads.campaign import Campaign +from twitter_ads.client import Client +from twitter_ads.cursor import Cursor +from twitter_ads.http import Request +from twitter_ads.resource import Resource +from twitter_ads import API_VERSION + + +@responses.activate +def test_rate_limit_cursor_class_access(): + responses.add(responses.GET, + with_resource('/' + API_VERSION + '/accounts/2iqph'), + body=with_fixture('accounts_load'), + content_type='application/json') + + responses.add(responses.GET, + with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'), + body=with_fixture('campaigns_all'), + content_type='application/json', + headers={ + 'x-account-rate-limit-limit': '10000', + 'x-account-rate-limit-remaining': '9999', + 'x-account-rate-limit-reset': '1546300800' + }) + + client = Client( + characters(40), + characters(40), + characters(40), + characters(40) + ) + + account = Account.load(client, '2iqph') + + cursor = Campaign.all(account) + assert cursor is not None + assert isinstance(cursor, Cursor) + assert cursor.account_rate_limit == '10000' + assert cursor.account_rate_limit_remaining == '9999' + assert cursor.account_rate_limit_reset == '1546300800' + + +@responses.activate +def test_rate_limit_resource_class_access(): + responses.add(responses.GET, + with_resource('/' + API_VERSION + '/accounts/2iqph'), + body=with_fixture('accounts_load'), + content_type='application/json') + + responses.add(responses.GET, + with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns/2wap7'), + body=with_fixture('campaigns_load'), + content_type='application/json', + headers={ + 'x-account-rate-limit-limit': '10000', + 'x-account-rate-limit-remaining': '9999', + 'x-account-rate-limit-reset': '1546300800' + }) + + client = Client( + characters(40), + characters(40), + characters(40), + characters(40) + ) + + account = Account.load(client, '2iqph') + campaign = Campaign.load(account, '2wap7') + + resource = '/' + API_VERSION + '/accounts/2iqph/campaigns/2wap7' + params = {} + + response = Request(client, 'get', resource, params=params).perform() + # from_response() is a staticmethod, so passing campaign instance as dummy. + # We can later change this test case to not call this manually + # once we changed existing classes to pass the header argument. + data = campaign.from_response(response.body['data'], response.headers) + + assert data is not None + assert isinstance(data, Resource) + assert data.id == '2wap7' + assert data.entity_status == 'ACTIVE' + assert data.account_rate_limit == '10000' + assert data.account_rate_limit_remaining == '9999' + assert data.account_rate_limit_reset == '1546300800' From 94854a51b7accfae8921723c0b151a6907d53ce2 Mon Sep 17 00:00:00 2001 From: smaeda-ks Date: Sun, 21 Jul 2019 02:36:53 +0900 Subject: [PATCH 5/6] code refactoring remove duplicate code --- twitter_ads/cursor.py | 19 ++++--------------- twitter_ads/resource.py | 19 ++++--------------- twitter_ads/utils.py | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/twitter_ads/cursor.py b/twitter_ads/cursor.py index 3768457..543ab1d 100644 --- a/twitter_ads/cursor.py +++ b/twitter_ads/cursor.py @@ -4,6 +4,7 @@ # from twitter_ads import * from twitter_ads.http import Request +from twitter_ads.utils import extract_rate_limit class Cursor(object): @@ -93,21 +94,9 @@ def __from_response(self, response): if 'total_count' in response.body: self._total_count = int(response.body['total_count']) - if 'x-rate-limit-reset' in response.headers: - setattr(self, 'rate_limit', - response.headers['x-rate-limit-limit']) - setattr(self, 'rate_limit_remaining', - response.headers['x-rate-limit-remaining']) - setattr(self, 'rate_limit_reset', - response.headers['x-rate-limit-reset']) - - if 'x-account-rate-limit-reset' in response.headers: - setattr(self, 'account_rate_limit', - response.headers['x-account-rate-limit-limit']) - setattr(self, 'account_rate_limit_remaining', - response.headers['x-account-rate-limit-remaining']) - setattr(self, 'account_rate_limit_reset', - response.headers['x-account-rate-limit-reset']) + limits = extract_rate_limit(response.headers) + for k in limits: + setattr(self, k, limits[k]) for item in response.body['data']: if 'from_response' in dir(self._klass): diff --git a/twitter_ads/resource.py b/twitter_ads/resource.py index 83d4977..cc32704 100644 --- a/twitter_ads/resource.py +++ b/twitter_ads/resource.py @@ -15,6 +15,7 @@ from twitter_ads.http import Request from twitter_ads.cursor import Cursor from twitter_ads import API_VERSION +from twitter_ads.utils import extract_rate_limit def resource_property(klass, name, **kwargs): @@ -49,21 +50,9 @@ def from_response(self, response, headers=None): attribute values. """ if headers is not None: - if 'x-rate-limit-reset' in headers: - setattr(self, 'rate_limit', - headers['x-rate-limit-limit']) - setattr(self, 'rate_limit_remaining', - headers['x-rate-limit-remaining']) - setattr(self, 'rate_limit_reset', - headers['x-rate-limit-reset']) - - if 'x-account-rate-limit-reset' in headers: - setattr(self, 'account_rate_limit', - headers['x-account-rate-limit-limit']) - setattr(self, 'account_rate_limit_remaining', - headers['x-account-rate-limit-remaining']) - setattr(self, 'account_rate_limit_reset', - headers['x-account-rate-limit-reset']) + limits = extract_rate_limit(headers) + for k in limits: + setattr(self, k, limits[k]) for name in self.PROPERTIES: attr = '_{0}'.format(name) diff --git a/twitter_ads/utils.py b/twitter_ads/utils.py index 4b42ccc..5b329e0 100644 --- a/twitter_ads/utils.py +++ b/twitter_ads/utils.py @@ -65,3 +65,18 @@ def validate_whole_hours(time): # Times must be expressed in whole hours if time.minute > 0 or time.second > 0: raise ValueError("'start_time' and 'end_time' must be expressed in whole hours.") + + +def extract_rate_limit(headers): + limits = {} + if 'x-rate-limit-reset' in headers: + limits['rate_limit'] = headers['x-rate-limit-limit'] + limits['rate_limit_remaining'] = headers['x-rate-limit-remaining'] + limits['rate_limit_reset'] = headers['x-rate-limit-reset'] + + if 'x-account-rate-limit-reset' in headers: + limits['account_rate_limit'] = headers['x-account-rate-limit-limit'] + limits['account_rate_limit_remaining'] = headers['x-account-rate-limit-remaining'] + limits['account_rate_limit_reset'] = headers['x-account-rate-limit-reset'] + + return limits From cab4f6e0ea47864b6aa571f0fd790bc772e650b1 Mon Sep 17 00:00:00 2001 From: smaeda-ks Date: Fri, 2 Aug 2019 12:49:04 +0900 Subject: [PATCH 6/6] generalize extract_rate_limit() func --- tests/test_rate_limit.py | 2 ++ twitter_ads/cursor.py | 4 ++-- twitter_ads/resource.py | 4 ++-- twitter_ads/utils.py | 25 ++++++++++++------------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/test_rate_limit.py b/tests/test_rate_limit.py index ba8d5d9..6a992d9 100644 --- a/tests/test_rate_limit.py +++ b/tests/test_rate_limit.py @@ -41,6 +41,7 @@ def test_rate_limit_cursor_class_access(): cursor = Campaign.all(account) assert cursor is not None assert isinstance(cursor, Cursor) + assert cursor.rate_limit is None assert cursor.account_rate_limit == '10000' assert cursor.account_rate_limit_remaining == '9999' assert cursor.account_rate_limit_reset == '1546300800' @@ -86,6 +87,7 @@ def test_rate_limit_resource_class_access(): assert isinstance(data, Resource) assert data.id == '2wap7' assert data.entity_status == 'ACTIVE' + assert data.rate_limit is None assert data.account_rate_limit == '10000' assert data.account_rate_limit_remaining == '9999' assert data.account_rate_limit_reset == '1546300800' diff --git a/twitter_ads/cursor.py b/twitter_ads/cursor.py index 543ab1d..31be004 100644 --- a/twitter_ads/cursor.py +++ b/twitter_ads/cursor.py @@ -4,7 +4,7 @@ # from twitter_ads import * from twitter_ads.http import Request -from twitter_ads.utils import extract_rate_limit +from twitter_ads.utils import extract_response_headers class Cursor(object): @@ -94,7 +94,7 @@ def __from_response(self, response): if 'total_count' in response.body: self._total_count = int(response.body['total_count']) - limits = extract_rate_limit(response.headers) + limits = extract_response_headers(response.headers) for k in limits: setattr(self, k, limits[k]) diff --git a/twitter_ads/resource.py b/twitter_ads/resource.py index c89b957..41e11fe 100644 --- a/twitter_ads/resource.py +++ b/twitter_ads/resource.py @@ -15,7 +15,7 @@ from twitter_ads.http import Request from twitter_ads.cursor import Cursor from twitter_ads import API_VERSION -from twitter_ads.utils import extract_rate_limit +from twitter_ads.utils import extract_response_headers def resource_property(klass, name, **kwargs): @@ -50,7 +50,7 @@ def from_response(self, response, headers=None): attribute values. """ if headers is not None: - limits = extract_rate_limit(headers) + limits = extract_response_headers(headers) for k in limits: setattr(self, k, limits[k]) diff --git a/twitter_ads/utils.py b/twitter_ads/utils.py index 5b329e0..3bdb2db 100644 --- a/twitter_ads/utils.py +++ b/twitter_ads/utils.py @@ -67,16 +67,15 @@ def validate_whole_hours(time): raise ValueError("'start_time' and 'end_time' must be expressed in whole hours.") -def extract_rate_limit(headers): - limits = {} - if 'x-rate-limit-reset' in headers: - limits['rate_limit'] = headers['x-rate-limit-limit'] - limits['rate_limit_remaining'] = headers['x-rate-limit-remaining'] - limits['rate_limit_reset'] = headers['x-rate-limit-reset'] - - if 'x-account-rate-limit-reset' in headers: - limits['account_rate_limit'] = headers['x-account-rate-limit-limit'] - limits['account_rate_limit_remaining'] = headers['x-account-rate-limit-remaining'] - limits['account_rate_limit_reset'] = headers['x-account-rate-limit-reset'] - - return limits +def extract_response_headers(headers): + values = {} + + values['rate_limit'] = headers.get('x-rate-limit-limit') + values['rate_limit_remaining'] = headers.get('x-rate-limit-remaining') + values['rate_limit_reset'] = headers.get('x-rate-limit-reset') + + values['account_rate_limit'] = headers.get('x-account-rate-limit-limit') + values['account_rate_limit_remaining'] = headers.get('x-account-rate-limit-remaining') + values['account_rate_limit_reset'] = headers.get('x-account-rate-limit-reset') + + return values