airbyte.constants

Constants shared across the PyAirbyte codebase.

 1# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 2"""Constants shared across the PyAirbyte codebase."""
 3
 4from __future__ import annotations
 5
 6
 7DEBUG_MODE = False  # Set to True to enable additional debug logging.
 8
 9
10AB_EXTRACTED_AT_COLUMN = "_airbyte_extracted_at"
11"""A column that stores the timestamp when the record was extracted."""
12
13AB_META_COLUMN = "_airbyte_meta"
14"""A column that stores metadata about the record."""
15
16AB_RAW_ID_COLUMN = "_airbyte_raw_id"
17"""A column that stores a unique identifier for each row in the source data.
18
19Note: The interpretation of this column is slightly different from in Airbyte Dv2 destinations.
20In Airbyte Dv2 destinations, this column points to a row in a separate 'raw' table. In PyAirbyte,
21this column is simply used as a unique identifier for each record as it is received.
22
23PyAirbyte uses ULIDs for this column, which are identifiers that can be sorted by time
24received. This allows us to determine the debug the order of records as they are received, even if
25the source provides records that are tied or received out of order from the perspective of their
26`emitted_at` (`_airbyte_extracted_at`) timestamps.
27"""
28
29AB_INTERNAL_COLUMNS = {
30    AB_RAW_ID_COLUMN,
31    AB_EXTRACTED_AT_COLUMN,
32    AB_META_COLUMN,
33}
34"""A set of internal columns that are reserved for PyAirbyte's internal use."""
DEBUG_MODE = False
AB_EXTRACTED_AT_COLUMN = '_airbyte_extracted_at'

A column that stores the timestamp when the record was extracted.

AB_META_COLUMN = '_airbyte_meta'

A column that stores metadata about the record.

AB_RAW_ID_COLUMN = '_airbyte_raw_id'

A column that stores a unique identifier for each row in the source data.

Note: The interpretation of this column is slightly different from in Airbyte Dv2 destinations. In Airbyte Dv2 destinations, this column points to a row in a separate 'raw' table. In PyAirbyte, this column is simply used as a unique identifier for each record as it is received.

PyAirbyte uses ULIDs for this column, which are identifiers that can be sorted by time received. This allows us to determine the debug the order of records as they are received, even if the source provides records that are tied or received out of order from the perspective of their emitted_at (_airbyte_extracted_at) timestamps.

AB_INTERNAL_COLUMNS = {'_airbyte_meta', '_airbyte_raw_id', '_airbyte_extracted_at'}

A set of internal columns that are reserved for PyAirbyte's internal use.