Skip to content

handle rate limit and implement retry logic #213

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Aug 20, 2019
Merged

handle rate limit and implement retry logic #213

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6746a8f
handle rate limit and implement retry logic
smaeda-ks Aug 3, 2019
b50c715
Add test for client options
smaeda-ks Aug 4, 2019
812e416
Add more test cases for rate limit & retry options
smaeda-ks Aug 4, 2019
ed17e53
Use monkeypatch to stub sleep
smaeda-ks Aug 8, 2019
b584302
Do not use unnecessary temp variables
smaeda-ks Aug 9, 2019
883f899
Merge branch 'master' into smaeda-ks/develop/handle-rate-limit
smaeda-ks Aug 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions tests/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
# Copyright (C) 2015 Twitter, Inc.

import pytest
import requests_oauthlib
import responses

from tests.support import with_resource, with_fixture, characters
Expand All @@ -11,11 +8,12 @@
from twitter_ads.cursor import Cursor
from twitter_ads import API_VERSION


@responses.activate
def test_accounts_with_no_id():
responses.add(responses.GET, with_resource('/' + API_VERSION + '/accounts'),
body=with_fixture('accounts_all'),
content_type='application/json')
body=with_fixture('accounts_all'),
content_type='application/json')

client = Client(
characters(40),
Expand All @@ -29,11 +27,12 @@ def test_accounts_with_no_id():
assert isinstance(cursor, Cursor)
assert cursor.count == 5


@responses.activate
def test_accounts_with_id():
responses.add(responses.GET, with_resource('/' + API_VERSION + '/accounts/2iqph'),
body=with_fixture('accounts_load'),
content_type='application/json')
body=with_fixture('accounts_load'),
content_type='application/json')

client = Client(
characters(40),
Expand All @@ -46,3 +45,24 @@ def test_accounts_with_id():
assert account is not None
assert isinstance(account, Account)
assert account.id == '2iqph'


@responses.activate
def test_accounts_with_options():

client = Client(
characters(40),
characters(40),
characters(40),
characters(40),
options={
'handle_rate_limit': True,
'retry_max': 1,
'retry_delay': 3000,
'retry_on_status': [404, 500, 503]
}
)

assert client is not None
assert isinstance(client, Client)
assert len(client.options) == 4
246 changes: 246 additions & 0 deletions tests/test_rate_limit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import responses
import unittest
import time

from tests.support import with_resource, with_fixture, characters

Expand All @@ -10,6 +11,251 @@
from twitter_ads.http import Request
from twitter_ads.resource import Resource
from twitter_ads import API_VERSION
from twitter_ads.error import RateLimit


@responses.activate
def test_rate_limit_handle_with_retry_success_1(monkeypatch):
# scenario:
# - 500 (retry) -> 429 (handle rate limit) -> 200 (end)
monkeypatch.setattr(time, 'sleep', lambda s: None)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

monkeypatch to stub time.sleep() in http.py


responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph'),
body=with_fixture('accounts_load'),
content_type='application/json')

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=500,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '0',
'x-account-rate-limit-reset': '1546300800'
})

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=429,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '0',
'x-account-rate-limit-reset': str(int(time.time()) + 5)
})

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=200,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '9999',
'x-account-rate-limit-reset': '1546300800'
})

client = Client(
characters(40),
characters(40),
characters(40),
characters(40),
options={
'handle_rate_limit': True,
'retry_max': 1,
'retry_delay': 3000,
'retry_on_status': [500]
}
)

account = Account.load(client, '2iqph')

cursor = Campaign.all(account)
assert len(responses.calls) == 4
assert cursor is not None
assert isinstance(cursor, Cursor)
assert cursor.rate_limit is None
assert cursor.account_rate_limit == '10000'
assert cursor.account_rate_limit_remaining == '9999'
assert cursor.account_rate_limit_reset == '1546300800'


@responses.activate
def test_rate_limit_handle_with_retry_success_2(monkeypatch):
# scenario:
# - 429 (handle rate limit) -> 500 (retry) -> 200 (end)
monkeypatch.setattr(time, 'sleep', lambda s: None)

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph'),
body=with_fixture('accounts_load'),
content_type='application/json')

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=429,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '0',
'x-account-rate-limit-reset': '1546300800'
})

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=500,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '0',
'x-account-rate-limit-reset': str(int(time.time()) + 5)
})

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=200,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '9999',
'x-account-rate-limit-reset': '1546300800'
})

client = Client(
characters(40),
characters(40),
characters(40),
characters(40),
options={
'handle_rate_limit': True,
'retry_max': 1,
'retry_delay': 3000,
'retry_on_status': [500]
}
)

account = Account.load(client, '2iqph')

cursor = Campaign.all(account)
assert len(responses.calls) == 4
assert cursor is not None
assert isinstance(cursor, Cursor)
assert cursor.rate_limit is None
assert cursor.account_rate_limit == '10000'
assert cursor.account_rate_limit_remaining == '9999'
assert cursor.account_rate_limit_reset == '1546300800'


@responses.activate
def test_rate_limit_handle_success(monkeypatch):
monkeypatch.setattr(time, 'sleep', lambda s: None)

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph'),
body=with_fixture('accounts_load'),
content_type='application/json')

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=429,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '0',
'x-account-rate-limit-reset': str(int(time.time()) + 5)
})

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=200,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '9999',
'x-account-rate-limit-reset': '1546300800'
})

client = Client(
characters(40),
characters(40),
characters(40),
characters(40),
options={
'handle_rate_limit': True
}
)

account = Account.load(client, '2iqph')

cursor = Campaign.all(account)
assert len(responses.calls) == 3
assert cursor is not None
assert isinstance(cursor, Cursor)
assert cursor.rate_limit is None
assert cursor.account_rate_limit == '10000'
assert cursor.account_rate_limit_remaining == '9999'
assert cursor.account_rate_limit_reset == '1546300800'


@responses.activate
def test_rate_limit_handle_error(monkeypatch):
monkeypatch.setattr(time, 'sleep', lambda s: None)

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph'),
body=with_fixture('accounts_load'),
content_type='application/json')

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=429,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '0',
'x-account-rate-limit-reset': str(int(time.time()) + 5)
})

responses.add(responses.GET,
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
status=429,
body=with_fixture('campaigns_all'),
content_type='application/json',
headers={
'x-account-rate-limit-limit': '10000',
'x-account-rate-limit-remaining': '0',
'x-account-rate-limit-reset': '1546300800'
})

client = Client(
characters(40),
characters(40),
characters(40),
characters(40),
options={
'handle_rate_limit': True
}
)

account = Account.load(client, '2iqph')

try:
cursor = Campaign.all(account)
except Exception as e:
error = e
print(error)
assert len(responses.calls) == 3
assert isinstance(error, RateLimit)
assert error.reset_at == '1546300800'


@responses.activate
Expand Down
Loading