Skip to content

Commit

Permalink
added exponential backoff
Browse files Browse the repository at this point in the history
  • Loading branch information
darynaishchenko committed Apr 21, 2023
1 parent feadb38 commit 6918def
Showing 1 changed file with 10 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#

import time
from abc import ABC
from datetime import timedelta
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional
Expand Down Expand Up @@ -30,6 +29,7 @@ def url_base(self):

# https://help.mixpanel.com/hc/en-us/articles/115004602563-Rate-Limits-for-Export-API-Endpoints#api-export-endpoint-rate-limits
reqs_per_hour_limit: int = 60 # 1 query per minute
retries: int = 0

def __init__(
self,
Expand Down Expand Up @@ -99,18 +99,18 @@ def parse_response(
# parse the whole response
yield from self.process_response(response, stream_state=stream_state, **kwargs)

if self.reqs_per_hour_limit > 0:
# we skip this block, if self.reqs_per_hour_limit = 0,
# in all other cases wait for X seconds to match API limitations
self.logger.info("Sleep for %s seconds to match API limitations", 3600 / self.reqs_per_hour_limit)
time.sleep(3600 / self.reqs_per_hour_limit)

def backoff_time(self, response: requests.Response) -> float:
"""
Some API endpoints do not return "Retry-After" header
some endpoints return a strangely low number
Some API endpoints do not return "Retry-After" header.
https://developer.mixpanel.com/reference/import-events#rate-limits (exponential backoff)
"""
return max(int(response.headers.get("Retry-After", 600)), 60)

retry_after = response.headers.get("Retry-After")
if retry_after:
return float(retry_after)

self.retries += 1
return 2**self.retries * 60

def get_stream_params(self) -> Mapping[str, Any]:
"""
Expand Down

0 comments on commit 6918def

Please sign in to comment.