🎉 Add configurable dbt parameter to destination-bigquery (#7118)

* Add configurable dbt parameter to destination-bigquery * Update airbyte-integrations/connectors/destination-bigquery/Dockerfile
airbytehq · Oct 19, 2021 · abf0159 · abf0159
1 parent b18acd6
commit abf0159
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 0 deletions.
diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py
@@ -130,6 +130,7 @@ def transform_bigquery(config: Dict[str, Any]):
             "type": "bigquery",
             "project": config["project_id"],
             "dataset": config["dataset_id"],
+            "priority": config.get("transformation_priority", "interactive"),
             "threads": 32,
             "retries": 1,
         }

diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py
@@ -134,6 +134,7 @@ def test_transform_bigquery(self):
             "project_id": "my_project_id",
             "dataset_id": "my_dataset_id",
             "credentials_json": '{ "type": "service_account-json" }',
+            "transformation_priority": "interactive",
             "dataset_location": "EU",
         }
 
@@ -143,6 +144,7 @@ def test_transform_bigquery(self):
             "method": "service-account-json",
             "project": "my_project_id",
             "dataset": "my_dataset_id",
+            "priority": "interactive",
             "keyfile_json": {"type": "service_account-json"},
             "location": "EU",
             "retries": 1,
@@ -164,6 +166,7 @@ def test_transform_bigquery_no_credentials(self):
             "method": "oauth",
             "project": "my_project_id",
             "dataset": "my_dataset_id",
+            "priority": "interactive",
             "retries": 1,
             "threads": 32,
         }

diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json
@@ -73,6 +73,16 @@
         "title": "Credentials JSON",
         "airbyte_secret": true
       },
+      "transformation_priority": {
+        "type": "string",
+        "description": "When running custom transformations or Basic normalization, running queries on interactive mode can hit BQ limits, choosing batch will solve those limitss.",
+        "title": "Transformation Query Run Type",
+        "default": "interactive",
+        "enum": [
+          "interactive",
+          "batch"
+        ]
+      },
       "loading_method": {
         "type": "object",
         "title": "Loading Method",