Skip to content

Commit

Permalink
ステータスコードが429のときは、Retry-Afterヘッダ値だけ待つようにする (#431)
Browse files Browse the repository at this point in the history 
* update retry

* format

* version up
  • Loading branch information
@yuji38kwmt
yuji38kwmt committed Mar 18, 2022
1 parent ad6f376 commit 3aa3493
Show file tree
Hide file tree
Showing 4 changed files with 187 additions and 23 deletions.
2 changes: 1 addition & 1 deletion annofabapi/__version__.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.55.0"
__version__ = "0.55.1"
110 changes: 96 additions & 14 deletions annofabapi/api.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import copy
import json
import logging
import time
from functools import wraps
from json import JSONDecodeError
from typing import Any, Dict, Optional, Tuple
Expand All @@ -18,6 +19,9 @@
DEFAULT_ENDPOINT_URL = "https://annofab.com"
"""AnnoFab WebAPIのデフォルトのエンドポイントURL"""

DEFAULT_WAITING_TIME_SECONDS_WITH_429_STATUS_CODE = 300
"""HTTP Status Codeが429のときの、デフォルト（Retry-Afterヘッダがないとき）の待ち時間です。"""


def _raise_for_status(response: requests.Response) -> None:
"""
Expand Down Expand Up @@ -157,13 +161,13 @@ def mask_key(d, key: str):


def _should_retry_with_status(status_code: int) -> bool:
"""HTTP Status Codeからリトライすべきかどうかを返す。"""
if status_code == 429:
return True
elif 500 <= status_code < 600:
"""
HTTP Status Codeからリトライすべきかどうかを返す。
"""
# 注意：429(Too many requests)の場合は、backoffモジュール外でリトライするため、このメソッドでは判定しない
if 500 <= status_code < 600:
return True
else:
return False
return False


def my_backoff(function):
Expand Down Expand Up @@ -363,8 +367,9 @@ def _execute_http_request(
raise_for_status: bool = True,
**kwargs,
) -> requests.Response:
"""Session情報を使って、HTTP Requestを投げる。
引数は ``requests.Session.request`` にそのまま渡す。
"""
Session情報を使って、HTTP Requestを投げます。AnnoFab WebAPIで取得したAWS S3のURLなどに、アクセスすることを想定しています。
引数は ``requests.Session.request`` にそのまま渡します。
Args:
raise_for_status: Trueの場合HTTP Status Codeが4XX,5XXのときはHTTPErrorをスローします
Expand Down Expand Up @@ -398,7 +403,47 @@ def _execute_http_request(
},
},
)
# リトライすべき場合はExceptionを返す

# リクエスト過多の場合、待ってから再度アクセスする
if response.status_code == requests.codes.too_many_requests:
retry_after_value = response.headers.get("Retry-After")
waiting_time_seconds = (
float(retry_after_value)
if retry_after_value is not None
else DEFAULT_WAITING_TIME_SECONDS_WITH_429_STATUS_CODE
)

logger.warning(
"HTTPステータスコードが'%s'なので、%s秒待ってからリトライします。 :: %s",
response.status_code,
waiting_time_seconds,
{
"response": {
"status_code": response.status_code,
"text": response.text,
"headers": {"Retry-After": retry_after_value},
},
"request": {
"http_method": http_method,
"url": url,
"query_params": _create_query_params_for_logger(params) if params is not None else None,
},
},
)

time.sleep(float(waiting_time_seconds))
return self._execute_http_request(
http_method=http_method,
url=url,
params=params,
data=data,
json=json,
headers=headers,
raise_for_status=raise_for_status,
**kwargs,
)

# リトライすべき場合はExceptionをスローする
if raise_for_status or _should_retry_with_status(response.status_code):
_log_error_response(logger, response)
_raise_for_status(response)
Expand All @@ -417,14 +462,14 @@ def _request_wrapper(
raise_for_status: bool = True,
) -> Tuple[Any, requests.Response]:
"""
HTTP Requestを投げて、Responseを返す。
AnnoFab WebAPIにアクセスして、レスポンスの中身とレスポンスを取得します。
Args:
http_method:
url_path:
query_params:
header_params:
request_body:
url_path: AnnoFab WebAPIのパス（例：``/my/account``）
query_params: クエリパラメタ
header_params: リクエストヘッダ
request_body: リクエストボディ
raise_for_status: Trueの場合HTTP Status Codeが4XX,5XXのときはHTTPErrorをスローします。Falseの場合はtuple[None, Response]を返します。
Returns:
Expand All @@ -435,6 +480,8 @@ def _request_wrapper(
HTTPError: 引数 ``raise_for_status`` がTrueで、HTTP status codeが4xxx,5xxのときにスローします。
"""

# TODO 判定条件が不明
if url_path.startswith("/internal/"):
url = f"{self.endpoint_url}/api{url_path}"
else:
Expand Down Expand Up @@ -471,6 +518,41 @@ def _request_wrapper(
request_body=request_body,
raise_for_status=raise_for_status,
)
elif response.status_code == requests.codes.too_many_requests:
retry_after_value = response.headers.get("Retry-After")
waiting_time_seconds = (
float(retry_after_value)
if retry_after_value is not None
else DEFAULT_WAITING_TIME_SECONDS_WITH_429_STATUS_CODE
)

logger.warning(
"HTTPステータスコードが'%s'なので、%s秒待ってからリトライします。 :: %s",
response.status_code,
waiting_time_seconds,
{
"response": {
"status_code": response.status_code,
"text": response.text,
"headers": {"Retry-After": retry_after_value},
},
"request": {
"http_method": http_method.lower(),
"url": url,
"query_params": query_params,
},
},
)

time.sleep(waiting_time_seconds)
return self._request_wrapper(
http_method,
url_path,
query_params=query_params,
header_params=header_params,
request_body=request_body,
raise_for_status=raise_for_status,
)

response.encoding = "utf-8"
content = self._response_to_content(response)
Expand Down
96 changes: 89 additions & 7 deletions annofabapi/api2.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,19 @@
import logging
import time
from typing import Any, Dict, Optional, Tuple

import requests
from requests.cookies import RequestsCookieJar

import annofabapi.utils
from annofabapi.api import AnnofabApi, _log_error_response, _raise_for_status
from annofabapi.api import (
DEFAULT_WAITING_TIME_SECONDS_WITH_429_STATUS_CODE,
AnnofabApi,
_create_request_body_for_logger,
_log_error_response,
_raise_for_status,
_should_retry_with_status,
)
from annofabapi.generated_api2 import AbstractAnnofabApi2

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -39,9 +47,11 @@ def _request_wrapper(
self,
http_method: str,
url_path: str,
*,
query_params: Optional[Dict[str, Any]] = None,
header_params: Optional[Dict[str, Any]] = None,
request_body: Optional[Any] = None,
raise_for_status: bool = True,
) -> Tuple[Any, requests.Response]:
"""
HTTP　Requestを投げて、Responseを返す。
Expand All @@ -51,6 +61,7 @@ def _request_wrapper(
query_params:
header_params:
request_body:
raise_for_status: Trueの場合HTTP Status Codeが4XX,5XXのときはHTTPErrorをスローします。Falseの場合はtuple[None, Response]を返します。
Returns:
Tuple[content, Response]. contentはcontent_typeにより型が変わる。
Expand All @@ -68,26 +79,97 @@ def _request_wrapper(
# Unauthorized Errorならば、ログイン後に再度実行する
if response.status_code == requests.codes.unauthorized:
self.api.login()
return self._request_wrapper(http_method, url_path, query_params, header_params, request_body)
return self._request_wrapper(
http_method,
url_path,
query_params=query_params,
header_params=header_params,
request_body=request_body,
raise_for_status=raise_for_status,
)

else:
kwargs.update({"cookies": self.cookies})

# HTTP Requestを投げる
response = self.api.session.request(method=http_method.lower(), url=url, **kwargs)

logger.debug(
"Sent a request :: %s",
{
"request": {
"http_method": http_method.lower(),
"url": url,
"query_params": query_params,
"header_params": header_params,
"request_body": _create_request_body_for_logger(request_body)
if request_body is not None
else None,
},
"response": {
"status_code": response.status_code,
"content_length": len(response.content),
},
},
)

# CloudFrontから403 Errorが発生したとき
if response.status_code == requests.codes.forbidden and response.headers.get("server") == "CloudFront":

self._get_signed_access_v2(url_path)
return self._request_wrapper(http_method, url_path, query_params, header_params, request_body)

_log_error_response(logger, response)
return self._request_wrapper(
http_method,
url_path,
query_params=query_params,
header_params=header_params,
request_body=request_body,
raise_for_status=raise_for_status,
)

elif response.status_code == requests.codes.too_many_requests:
retry_after_value = response.headers.get("Retry-After")
waiting_time_seconds = (
float(retry_after_value)
if retry_after_value is not None
else DEFAULT_WAITING_TIME_SECONDS_WITH_429_STATUS_CODE
)

logger.warning(
"HTTPステータスコードが'%s'なので、%s秒待ってからリトライします。 :: %s",
response.status_code,
waiting_time_seconds,
{
"response": {
"status_code": response.status_code,
"text": response.text,
"headers": {"Retry-After": retry_after_value},
},
"request": {
"http_method": http_method.lower(),
"url": url,
"query_params": query_params,
},
},
)

time.sleep(waiting_time_seconds)
return self._request_wrapper(
http_method,
url_path,
query_params=query_params,
header_params=header_params,
request_body=request_body,
raise_for_status=raise_for_status,
)

response.encoding = "utf-8"
_raise_for_status(response)

content = self.api._response_to_content(response)

# リトライすべき場合はExceptionを返す
if raise_for_status or _should_retry_with_status(response.status_code):
_log_error_response(logger, response)
_raise_for_status(response)

return content, response

def _get_signed_access_v2(self, url_path: str):
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "annofabapi"
version = "0.55.0"
version = "0.55.1"
description = "Python Clinet Library of AnnoFab WebAPI (https://annofab.com/docs/api/)"
authors = ["yuji38kwmt"]
license = "MIT"
Expand Down

0 comments on commit 3aa3493

Please sign in to comment.