Fix and document macros and interpolation variables (airbytehq#25305)

* Fix and document macros

* cleanup

* dots

* Add tests and refactor

* Update

* Add an example

* Document variables

* Mention now_local is not recommended

airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1953,3 +1953,90 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+interpolation:
+  variables:
+    - title: config
+      description: The connector configuration. The config's keys match the keys defined in the spec.
+    - title: headers
+      description: The HTTP headers from the last response received from the API.
+    - title: last_records
+      description: List of records extracted from the last response received from the API.
+    - title: record
+      description: The record being processed.
+    - title: response
+      description: The body of the last response received from the API.
+    - title: stream_interval
+      description:
+    - title: stream_partition
+      description: The current partition being processed.
+    - title: stream_slice
+      description: This variable is deprecated. Use stream_interval or stream_partition instead.
+    - title: stream_state
+      description: The current state of the stream.
+  macros:
+    - title: Now (Local)
+      description: Returns the current date and time using the local to the machine running the sync. We recommend using now_utc() instead of now_local() because it is more predictable.
+      arguments: {}
+      return_type: Datetime
+      examples:
+        - "{{ now_local() }}"
+    - title: Now (UTC)
+      description: Returns the current date and time in the UTC timezone.
+      arguments: {}
+      return_type: Datetime
+      examples:
+        - "{{ now_utc() }}"
+        - "{{ now_utc().strftime('%Y-%m-%d') }}"
+    - title: Today (UTC)
+      description: Returns the current date in UTC timezone. The output is a date object.
+      arguments: {}
+      return_type: Date
+      examples:
+        - "{{ today_utc() }}"
+    - title: Timestamp
+      description: Converts a number or a string representing a datetime (formatted as ISO8601) to a timestamp. If the input is a number, it is converted to an int. If no timezone is specified, the string is interpreted as UTC.
+      arguments:
+        datetime: A string formatted as ISO8601 or an integer representing a unix timestamp
+      return_type: int
+      examples:
+        - "{{ timestamp(1646006400) }}"
+        - "{{ timestamp('2022-02-28') }}"
+        - "{{ timestamp('2022-02-28T00:00:00Z') }}"
+        - "{{ timestamp('2022-02-28 00:00:00Z') }}"
+        - "{{ timestamp('2022-02-28T00:00:00:-08:00') }}"
+    - title: Max
+      description: Returns the largest object of a iterable, or or two or more arguments.
+      arguments:
+        args: iterable or a sequence of two or more arguments
+      return_type: Any
+      examples:
+        - "{{ max(2, 3) }}"
+        - "{{ max([2, 3]) }}"
+    - title: Day Delta
+      description: Returns the datetime of now() + num_days.
+      arguments:
+        num_days: The number of days to add to now
+        format: How to format the output string
+      return_type: str
+      examples:
+        - "{{ day_delta(25) }}"
+        - "{{ day_delta(25, format='%Y-%m-%d') }}"
+        - "{{ config['start_time'] + day_delta(2) }}"
+    - title: Duration
+      description: Converts an ISO8601 duratioin to datetime.timedelta.
+      arguments:
+        duration_string: "A string representing an ISO8601 duration. See https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm for more details."
+      return_type: datetime.timedelta
+      examples:
+        - "{{ duration('P1D') }}"
+        - "{{ duration('P6DT23H') }}"
+        - "{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}"
+    - title: Format Datetime
+      description: Converts a datetime or a datetime-string to the specified format.
+      arguments:
+        datetime: The datetime object or a string to convert. If datetime is a string, it must be formatted as ISO8601.
+        format: The datetime format
+      return_type: str
+      examples:
+        - "{{ format_datetime(config['start_time'], '%Y-%m-%d') }}"
+        - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}"

airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -61,7 +61,15 @@ def timestamp(dt: Union[numbers.Number, str]):
     if isinstance(dt, numbers.Number):
         return int(dt)
     else:
-        return int(parser.parse(dt).replace(tzinfo=datetime.timezone.utc).timestamp())
+        return _str_to_datetime(dt).astimezone(datetime.timezone.utc).timestamp()
+
+
+def _str_to_datetime(s: str) -> datetime.datetime:
+    parsed_date = parser.isoparse(s)
+    if not parsed_date.tzinfo:
+        # Assume UTC if the input does not contain a timezone
+        parsed_date = parsed_date.replace(tzinfo=datetime.timezone.utc)
+    return parsed_date.astimezone(datetime.timezone.utc)
 
 
 def max(*args):
@@ -97,7 +105,7 @@ def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
     return (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)).strftime(format)
 
 
-def duration(datestring: str):
+def duration(datestring: str) -> datetime.timedelta:
     """
     Converts ISO8601 duration to datetime.timedelta
 
@@ -107,7 +115,7 @@ def duration(datestring: str):
     return parse_duration(datestring)
 
 
-def format_datetime(dt: Union[str, datetime.datetime], format: str):
+def format_datetime(dt: Union[str, datetime.datetime], format: str) -> str:
     """
     Converts datetime to another format
 
@@ -116,7 +124,7 @@ def format_datetime(dt: Union[str, datetime.datetime], format: str):
     """
     if isinstance(dt, datetime.datetime):
         return dt.strftime(format)
-    return parser.parse(dt).strftime(format)
+    return _str_to_datetime(dt).strftime(format)
 
 
 _macros_list = [now_local, now_utc, today_utc, timestamp, max, day_delta, duration, format_datetime]

airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py

@@ -6,6 +6,7 @@
 
 import pytest
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
+from freezegun import freeze_time
 
 interpolation = JinjaInterpolation()
 
@@ -56,6 +57,15 @@ def test_positive_day_delta():
     assert val > (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=24, hours=23)).strftime("%Y-%m-%dT%H:%M:%S.%f%z")
 
 
+def test_positive_day_delta_with_format():
+    delta_template = "{{ day_delta(25,format='%Y-%m-%d') }}"
+    interpolation = JinjaInterpolation()
+
+    with freeze_time("2021-01-01 03:04:05"):
+        val = interpolation.eval(delta_template, {})
+        assert val == '2021-01-26'
+
+
 def test_negative_day_delta():
     delta_template = "{{ day_delta(-25) }}"
     interpolation = JinjaInterpolation()

airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py

@@ -28,12 +28,45 @@ def test_macros_export(test_name, fn_name, found_in_macros):
         assert fn_name not in macros
 
 
-def test_format_datetime():
+@pytest.mark.parametrize("test_name, input_value, format, expected_output", [
+    ("test_datetime_string_to_date", "2022-01-01T01:01:01Z", "%Y-%m-%d","2022-01-01"),
+    ("test_date_string_to_date", "2022-01-01", "%Y-%m-%d", "2022-01-01"),
+    ("test_datetime_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"),
+    ("test_datetime_with_tz_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"),
+    ("test_datetime_string_to_datetime", "2022-01-01T01:01:01Z", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"),
+    ("test_datetime_string_with_tz_to_datetime", "2022-01-01T01:01:01-0800", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T09:01:01Z"),
+    ("test_datetime_object_tz_to_date", datetime.datetime(2022,1,1,1,1,1), "%Y-%m-%d", "2022-01-01"),
+    ("test_datetime_object_tz_to_datetime", datetime.datetime(2022,1,1,1,1,1), "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"),
+])
+def test_format_datetime(test_name, input_value, format, expected_output):
     format_datetime = macros["format_datetime"]
-    assert format_datetime("2022-01-01T01:01:01Z", "%Y-%m-%d") == "2022-01-01"
-    assert format_datetime(datetime.datetime(2022, 1, 1, 1, 1, 1), "%Y-%m-%d") == "2022-01-01"
+    assert format_datetime(input_value, format) == expected_output
 
 
-def test_duration():
-    duration = macros["duration"]
-    assert duration("P1D") == datetime.timedelta(days=1)
+@pytest.mark.parametrize(
+    "test_name, input_value, expected_output", [
+        ("test_one_day", "P1D", datetime.timedelta(days=1)),
+        ("test_6_days_23_hours", "P6DT23H", datetime.timedelta(days=6, hours=23))
+    ]
+)
+def test_duration(test_name, input_value, expected_output):
+    duration_fn = macros["duration"]
+    assert duration_fn(input_value) == expected_output
+
+
+@pytest.mark.parametrize(
+    "test_name, input_value, expected_output",[
+        ("test_int_input", 1646006400, 1646006400),
+        ("test_float_input", 100.0, 100),
+        ("test_float_input_is_floored", 100.9, 100),
+        ("test_string_date_iso8601", "2022-02-28", 1646006400),
+        ("test_string_datetime_midnight_iso8601", "2022-02-28T00:00:00Z", 1646006400),
+        ("test_string_datetime_midnight_iso8601_with_tz", "2022-02-28T00:00:00-08:00", 1646035200),
+        ("test_string_datetime_midnight_iso8601_no_t", "2022-02-28 00:00:00Z", 1646006400),
+        ("test_string_datetime_iso8601", "2022-02-28T10:11:12", 1646043072),
+    ]
+)
+def test_timestamp(test_name, input_value, expected_output):
+    timestamp_function = macros["timestamp"]
+    actual_output = timestamp_function(input_value)
+    assert actual_output == expected_output