diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 6c9fb5b0e5919..e6b9315cd21ce 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -130,6 +130,7 @@ def transform_bigquery(config: Dict[str, Any]): "type": "bigquery", "project": config["project_id"], "dataset": config["dataset_id"], + "priority": config.get("transformation_priority", "interactive"), "threads": 32, "retries": 1, } diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py index 1adc28ba4caef..cfd5c0e88cea7 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py @@ -134,6 +134,7 @@ def test_transform_bigquery(self): "project_id": "my_project_id", "dataset_id": "my_dataset_id", "credentials_json": '{ "type": "service_account-json" }', + "transformation_priority": "interactive", "dataset_location": "EU", } @@ -143,6 +144,7 @@ def test_transform_bigquery(self): "method": "service-account-json", "project": "my_project_id", "dataset": "my_dataset_id", + "priority": "interactive", "keyfile_json": {"type": "service_account-json"}, "location": "EU", "retries": 1, @@ -164,6 +166,7 @@ def test_transform_bigquery_no_credentials(self): "method": "oauth", "project": "my_project_id", "dataset": "my_dataset_id", + "priority": "interactive", "retries": 1, "threads": 32, } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json index 4dc5364f36e36..4f98ba0f6619e 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json @@ -73,6 +73,16 @@ "title": "Credentials JSON", "airbyte_secret": true }, + "transformation_priority": { + "type": "string", + "description": "When running custom transformations or Basic normalization, running queries on interactive mode can hit BQ limits, choosing batch will solve those limitss.", + "title": "Transformation Query Run Type", + "default": "interactive", + "enum": [ + "interactive", + "batch" + ] + }, "loading_method": { "type": "object", "title": "Loading Method",