diff --git a/rdl/data_sources/AWSLambdaDataSource.py b/rdl/data_sources/AWSLambdaDataSource.py index adb6949..b47acea 100644 --- a/rdl/data_sources/AWSLambdaDataSource.py +++ b/rdl/data_sources/AWSLambdaDataSource.py @@ -2,6 +2,7 @@ import pandas import json import boto3 +import time from rdl.data_sources.ChangeTrackingInfo import ChangeTrackingInfo from rdl.data_sources.SourceTableInfo import SourceTableInfo @@ -149,9 +150,14 @@ def __get_data_frame(self, data: [[]], column_names: []): def __invoke_lambda(self, pay_load): max_attempts = Constants.MAX_AWS_LAMBDA_INVOKATION_ATTEMPTS + retry_delay = Constants.AWS_LAMBDA_RETRY_DELAY_SECONDS response_payload = None for current_attempt in list(range(1, max_attempts+1, 1)): + if current_attempt > 1: + self.logger.debug(f"\nDelaying retry for {(current_attempt - 1) ^ retry_delay} seconds") + time.sleep((current_attempt - 1) ^ retry_delay) + self.logger.debug(f"\nRequest being sent to Lambda, attempt {current_attempt} of {max_attempts}:") self.logger.debug(pay_load) diff --git a/rdl/shared/Constants.py b/rdl/shared/Constants.py index a8b1309..5ee1722 100644 --- a/rdl/shared/Constants.py +++ b/rdl/shared/Constants.py @@ -1,6 +1,7 @@ APP_NAME = "Relational Data Loader" DATA_PIPELINE_EXECUTION_SCHEMA_NAME = "rdl" -MAX_AWS_LAMBDA_INVOKATION_ATTEMPTS = 3 +MAX_AWS_LAMBDA_INVOKATION_ATTEMPTS = 4 # 1 + 3 retries +AWS_LAMBDA_RETRY_DELAY_SECONDS = 4 # 4 ^ retry attempt, so retry attempt 3 is delayed 64 seconds class FullRefreshReason: