diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index c305a306fdfef..643567796f5c6 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -1326,3 +1326,90 @@ definitions: $parameters: type: object additionalProperties: true +interpolation: + variables: + - title: config + description: The connector configuration. The config's keys match the keys defined in the spec. + - title: headers + description: The HTTP headers from the last response received from the API. + - title: last_records + description: List of records extracted from the last response received from the API. + - title: record + description: The record being processed. + - title: response + description: The body of the last response received from the API. + - title: stream_interval + description: + - title: stream_partition + description: The current partition being processed. + - title: stream_slice + description: This variable is deprecated. Use stream_interval or stream_partition instead. + - title: stream_state + description: The current state of the stream. + macros: + - title: Now (Local) + description: Returns the current date and time using the local to the machine running the sync. We recommend using now_utc() instead of now_local() because it is more predictable. + arguments: {} + return_type: Datetime + examples: + - "{{ now_local() }}" + - title: Now (UTC) + description: Returns the current date and time in the UTC timezone. + arguments: {} + return_type: Datetime + examples: + - "{{ now_utc() }}" + - "{{ now_utc().strftime('%Y-%m-%d') }}" + - title: Today (UTC) + description: Returns the current date in UTC timezone. The output is a date object. + arguments: {} + return_type: Date + examples: + - "{{ today_utc() }}" + - title: Timestamp + description: Converts a number or a string representing a datetime (formatted as ISO8601) to a timestamp. If the input is a number, it is converted to an int. If no timezone is specified, the string is interpreted as UTC. + arguments: + datetime: A string formatted as ISO8601 or an integer representing a unix timestamp + return_type: int + examples: + - "{{ timestamp(1646006400) }}" + - "{{ timestamp('2022-02-28') }}" + - "{{ timestamp('2022-02-28T00:00:00Z') }}" + - "{{ timestamp('2022-02-28 00:00:00Z') }}" + - "{{ timestamp('2022-02-28T00:00:00:-08:00') }}" + - title: Max + description: Returns the largest object of a iterable, or or two or more arguments. + arguments: + args: iterable or a sequence of two or more arguments + return_type: Any + examples: + - "{{ max(2, 3) }}" + - "{{ max([2, 3]) }}" + - title: Day Delta + description: Returns the datetime of now() + num_days. + arguments: + num_days: The number of days to add to now + format: How to format the output string + return_type: str + examples: + - "{{ day_delta(25) }}" + - "{{ day_delta(25, format='%Y-%m-%d') }}" + - "{{ config['start_time'] + day_delta(2) }}" + - title: Duration + description: Converts an ISO8601 duratioin to datetime.timedelta. + arguments: + duration_string: "A string representing an ISO8601 duration. See https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm for more details." + return_type: datetime.timedelta + examples: + - "{{ duration('P1D') }}" + - "{{ duration('P6DT23H') }}" + - "{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + - title: Format Datetime + description: Converts a datetime or a datetime-string to the specified format. + arguments: + datetime: The datetime object or a string to convert. If datetime is a string, it must be formatted as ISO8601. + format: The datetime format + return_type: str + examples: + - "{{ format_datetime(config['start_time'], '%Y-%m-%d') }}" + - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py index 1cf49a89d50fc..4a576e7950f54 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py @@ -61,7 +61,15 @@ def timestamp(dt: Union[numbers.Number, str]): if isinstance(dt, numbers.Number): return int(dt) else: - return int(parser.parse(dt).replace(tzinfo=datetime.timezone.utc).timestamp()) + return _str_to_datetime(dt).astimezone(datetime.timezone.utc).timestamp() + + +def _str_to_datetime(s: str) -> datetime.datetime: + parsed_date = parser.isoparse(s) + if not parsed_date.tzinfo: + # Assume UTC if the input does not contain a timezone + parsed_date = parsed_date.replace(tzinfo=datetime.timezone.utc) + return parsed_date.astimezone(datetime.timezone.utc) def max(*args): @@ -97,7 +105,7 @@ def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str: return (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)).strftime(format) -def duration(datestring: str): +def duration(datestring: str) -> datetime.timedelta: """ Converts ISO8601 duration to datetime.timedelta @@ -107,7 +115,7 @@ def duration(datestring: str): return parse_duration(datestring) -def format_datetime(dt: Union[str, datetime.datetime], format: str): +def format_datetime(dt: Union[str, datetime.datetime], format: str) -> str: """ Converts datetime to another format @@ -116,7 +124,7 @@ def format_datetime(dt: Union[str, datetime.datetime], format: str): """ if isinstance(dt, datetime.datetime): return dt.strftime(format) - return parser.parse(dt).strftime(format) + return _str_to_datetime(dt).strftime(format) _macros_list = [now_local, now_utc, today_utc, timestamp, max, day_delta, duration, format_datetime] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py index 42050e17a1ed6..91bf89ce17c29 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py @@ -6,6 +6,7 @@ import pytest from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation +from freezegun import freeze_time interpolation = JinjaInterpolation() @@ -56,6 +57,15 @@ def test_positive_day_delta(): assert val > (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=24, hours=23)).strftime("%Y-%m-%dT%H:%M:%S.%f%z") +def test_positive_day_delta_with_format(): + delta_template = "{{ day_delta(25,format='%Y-%m-%d') }}" + interpolation = JinjaInterpolation() + + with freeze_time("2021-01-01 03:04:05"): + val = interpolation.eval(delta_template, {}) + assert val == '2021-01-26' + + def test_negative_day_delta(): delta_template = "{{ day_delta(-25) }}" interpolation = JinjaInterpolation() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py index 77fd06d5dfa28..012eb36f1ca51 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py @@ -28,12 +28,45 @@ def test_macros_export(test_name, fn_name, found_in_macros): assert fn_name not in macros -def test_format_datetime(): +@pytest.mark.parametrize("test_name, input_value, format, expected_output", [ + ("test_datetime_string_to_date", "2022-01-01T01:01:01Z", "%Y-%m-%d","2022-01-01"), + ("test_date_string_to_date", "2022-01-01", "%Y-%m-%d", "2022-01-01"), + ("test_datetime_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"), + ("test_datetime_with_tz_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"), + ("test_datetime_string_to_datetime", "2022-01-01T01:01:01Z", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), + ("test_datetime_string_with_tz_to_datetime", "2022-01-01T01:01:01-0800", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T09:01:01Z"), + ("test_datetime_object_tz_to_date", datetime.datetime(2022,1,1,1,1,1), "%Y-%m-%d", "2022-01-01"), + ("test_datetime_object_tz_to_datetime", datetime.datetime(2022,1,1,1,1,1), "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), +]) +def test_format_datetime(test_name, input_value, format, expected_output): format_datetime = macros["format_datetime"] - assert format_datetime("2022-01-01T01:01:01Z", "%Y-%m-%d") == "2022-01-01" - assert format_datetime(datetime.datetime(2022, 1, 1, 1, 1, 1), "%Y-%m-%d") == "2022-01-01" + assert format_datetime(input_value, format) == expected_output -def test_duration(): - duration = macros["duration"] - assert duration("P1D") == datetime.timedelta(days=1) +@pytest.mark.parametrize( + "test_name, input_value, expected_output", [ + ("test_one_day", "P1D", datetime.timedelta(days=1)), + ("test_6_days_23_hours", "P6DT23H", datetime.timedelta(days=6, hours=23)) + ] +) +def test_duration(test_name, input_value, expected_output): + duration_fn = macros["duration"] + assert duration_fn(input_value) == expected_output + + +@pytest.mark.parametrize( + "test_name, input_value, expected_output",[ + ("test_int_input", 1646006400, 1646006400), + ("test_float_input", 100.0, 100), + ("test_float_input_is_floored", 100.9, 100), + ("test_string_date_iso8601", "2022-02-28", 1646006400), + ("test_string_datetime_midnight_iso8601", "2022-02-28T00:00:00Z", 1646006400), + ("test_string_datetime_midnight_iso8601_with_tz", "2022-02-28T00:00:00-08:00", 1646035200), + ("test_string_datetime_midnight_iso8601_no_t", "2022-02-28 00:00:00Z", 1646006400), + ("test_string_datetime_iso8601", "2022-02-28T10:11:12", 1646043072), + ] +) +def test_timestamp(test_name, input_value, expected_output): + timestamp_function = macros["timestamp"] + actual_output = timestamp_function(input_value) + assert actual_output == expected_output