Skip to content

Retry fetch languages/translations on 202 #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kbairak merged 1 commit into from
Sep 3, 2020
Merged

Retry fetch languages/translations on 202 #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 47 additions & 10 deletions tests/native/core/test_cds.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ class TestCDSHandler(object):

def _lang_lists_equal(self, list_1, list_2):
try:
sorted_1, sorted_2 = [
sorted(l, key=itemgetter('code')) for l in (list_1, list_2)
]
sorted_1, sorted_2 = [sorted(language, key=itemgetter('code'))
for language in (list_1, list_2)]
pairs = zip(sorted_1, sorted_2)
difference = any(x != y for x, y in pairs)
except Exception:
Expand Down Expand Up @@ -100,8 +99,8 @@ def test_fetch_languages(self, patched_logger):

assert cds_handler.fetch_languages() == []
patched_logger.error.assert_called_with(
'Error retrieving languages from CDS: UnknownError '
'(`400 Client Error: Bad Request for url: https://some.host/languages`)'
'Error retrieving languages from CDS: UnknownError (`400 Client '
'Error: Bad Request for url: https://some.host/languages`)'
)
responses.reset()

Expand All @@ -125,8 +124,8 @@ def test_fetch_languages(self, patched_logger):

assert cds_handler.fetch_languages() == []
patched_logger.error.assert_called_with(
'Error retrieving languages from CDS: UnknownError '
'(`403 Client Error: Forbidden for url: https://some.host/languages`)'
'Error retrieving languages from CDS: UnknownError (`403 Client '
'Error: Forbidden for url: https://some.host/languages`)'
)
responses.reset()

Expand Down Expand Up @@ -333,9 +332,8 @@ def test_fetch_translations_etags_management(self, patched_logger):

responses.add(
responses.GET, cds_host + '/content/en',
json={
# whatever, we don't care about the content of json repsone atm.
},
# whatever, we don't care about the content of json repsone atm.
json={},
status=304
)

Expand Down Expand Up @@ -451,3 +449,42 @@ def test_get_headers(self):
'Accept-Encoding': 'gzip',
'If-None-Match': 'something'
}

@responses.activate
def test_retry_fetch_languages(self):
cds_host = 'https://some.host'
cds_handler = CDSHandler(
['el', 'en'],
'some_token',
host=cds_host,
)
responses.add(responses.GET, cds_host + '/languages', status=202)
responses.add(responses.GET, cds_host + '/languages', status=202)
responses.add(responses.GET, cds_host + '/languages',
json={'data': [{'code': "el"},
{'code': "en"}],
'meta': {'some_key': "some_value"}},
status=200)
languages_response = cds_handler.fetch_languages()
assert self._lang_lists_equal(
languages_response,
[{'code': 'el'}, {'code': 'en'}]
)

@responses.activate
def test_retry_fetch_translations(self):
cds_host = 'https://some.host'
cds_handler = CDSHandler(
['el', 'en'],
'some_token',
host=cds_host,
)
responses.add(responses.GET, cds_host + '/content/el', status=202)
responses.add(responses.GET, cds_host + '/content/el', status=202)
responses.add(responses.GET,
cds_host + '/content/el',
json={'data': {'source': {'string': "translation"}}},
status=200)
translations = cds_handler.fetch_translations('el')
assert (translations ==
{'el': (True, {'source': {'string': "translation"}})})
73 changes: 37 additions & 36 deletions transifex/native/cds.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ def get(self, key):
class CDSHandler(object):
"""Handles communication with the Content Delivery Service."""

def __init__(self, configured_languages, token, secret=None, host=TRANSIFEX_CDS_HOST):
def __init__(self, configured_languages, token, secret=None,
host=TRANSIFEX_CDS_HOST):
"""Constructor.

:param list configured_languages: a list of language codes for the
Expand All @@ -72,10 +73,13 @@ def fetch_languages(self):
languages = []

try:
response = requests.get(
self.host + cds_url,
headers=self._get_headers(),
)
last_response_status = 202
while last_response_status == 202:
response = requests.get(
self.host + cds_url,
headers=self._get_headers(),
)
last_response_status = response.status_code

if not response.ok:
logger.error(
Expand All @@ -89,18 +93,16 @@ def fetch_languages(self):
languages = json_content['data']

except (KeyError, ValueError):
# Compatibility with python2.7 where `JSONDecodeError` doesn't exist
# Compatibility with python2.7 where `JSONDecodeError` doesn't
# exist
logger.error(
'Error retrieving languages from CDS: Malformed response')
except requests.ConnectionError:
logger.error(
'Error retrieving languages from CDS: ConnectionError')
except Exception as e:
logger.error(
'Error retrieving languages from CDS: UnknownError (`{}`)'.format(
str(e)
)
)
logger.error('Error retrieving languages from CDS: UnknownError '
'(`{}`)'.format(str(e)))

return languages

Expand Down Expand Up @@ -128,12 +130,16 @@ def fetch_translations(self, language_code=None):
set(self.configured_language_codes):

try:
response = requests.get(
self.host + cds_url.format(language_code=language_code),
headers=self._get_headers(
etag=self.etags.get(language_code)
last_response_status = 202
while last_response_status == 202:
response = requests.get(
(self.host +
cds_url.format(language_code=language_code)),
headers=self._get_headers(
etag=self.etags.get(language_code)
)
)
)
last_response_status = response.status_code

if not response.ok:
logger.error(
Expand All @@ -155,19 +161,19 @@ def fetch_translations(self, language_code=None):
)

except (KeyError, ValueError):
# Compatibility with python2.7 where `JSONDecodeError` doesn't exist
logger.error(
'Error retrieving translations from CDS: Malformed response') # pragma no cover
# Compatibility with python2.7 where `JSONDecodeError` doesn't
# exist
logger.error('Error retrieving translations from CDS: '
'Malformed response') # pragma no cover
translations[language_code] = (False, {}) # pragma no cover
except requests.ConnectionError:
logger.error(
'Error retrieving translations from CDS: ConnectionError')
translations[language_code] = (False, {})
except Exception as e:
logger.error(
'Error retrieving translations from CDS: UnknownError (`{}`)'.format(
str(e)
)
'Error retrieving translations from CDS: UnknownError '
'(`{}`)'.format(str(e))
) # pragma no cover
translations[language_code] = (False, {})

Expand All @@ -176,19 +182,17 @@ def fetch_translations(self, language_code=None):
def push_source_strings(self, strings, purge=False):
"""Push source strings to CDS.

:param list(SourceString) strings: a list of `SourceString` objects holding
source strings
:param bool purge: True deletes destination source content not included in
pushed content.
False appends the pushed content to destination source
content.
:param list(SourceString) strings: a list of `SourceString` objects
holding source strings
:param bool purge: True deletes destination source content not included
in pushed content. False appends the pushed content to destination
source content.
:return: the HTTP response object
:rtype: requests.Response
"""
if not self.secret:
raise Exception(
'You need to use `TRANSIFEX_SECRET` when pushing source content'
)
raise Exception('You need to use `TRANSIFEX_SECRET` when pushing '
'source content')

cds_url = TRANSIFEX_CDS_URLS['PUSH_SOURCE_STRINGS']

Expand All @@ -208,11 +212,8 @@ def push_source_strings(self, strings, purge=False):
logger.error(
'Error pushing source strings to CDS: ConnectionError')
except Exception as e:
logger.error(
'Error pushing source strings to CDS: UnknownError (`{}`)'.format(
str(e)
)
)
logger.error('Error pushing source strings to CDS: UnknownError '
'(`{}`)'.format(str(e)))

return response

Expand Down