Enable full SAT for the BigQuery source

airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml

@@ -5,3 +5,19 @@ tests:
   spec:
     - spec_path: "src/test-integration/resources/expected_spec.json"
       config_path: "src/test-integration/resources/dummy_config.json"
+  connection:
+    - config_path: "secrets/sat-config.json"
+      status: "succeed"
+  discovery:
+    - config_path: "secrets/sat-config.json"
+  basic_read:
+    - config_path: "secrets/sat-config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      expect_records:
+        path: "integration_tests/expected_records.json"
+  full_refresh:
+    - config_path: "secrets/sat-config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+  incremental:
+    - config_path: "secrets/sat-config.json"
+      configured_catalog_path: "integration_tests/configured_catalog_inc.json"

airbyte-integrations/connectors/source-bigquery/integration_tests/basic_dataset.sql

@@ -0,0 +1,33 @@
+CREATE TABLE
+    `sat_test_dataset.sat_basic_dataset`(id NUMERIC(29),
+                                         test_column_1 int64,
+                                         test_column_10 decimal(29,
+                                                9),
+                                         test_column_11 bigdecimal(76,
+                                             38),
+                                         test_column_12 float64,
+                                         test_column_13 bool,
+                                         test_column_15 date,
+                                         test_column_16 datetime,
+                                         test_column_17 timestamp,
+                                         test_column_19 string,
+                                         test_column_2 int,
+                                         test_column_21 time,
+                                         test_column_3 smallint,
+                                         test_column_4 integer,
+                                         test_column_5 bigint,
+                                         test_column_6 tinyint,
+                                         test_column_7 byteint,
+                                         test_column_8 numeric(29,
+                                                9),
+                                         test_column_9 bignumeric(76,
+                                             38) );
+
+
+INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (1, -128, -128, -128, -128, true, date('2021-10-20'), datetime('2021-10-20 11:22:33'), timestamp('2021-10-20 11:22:33'), 'qwe', -128, TIME(15, 30, 00), -128, -128, -128, -128, -128, -128, -128);
+INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (2, 127, 127, 127, 127, false, date('9999-12-31'), datetime('9999-12-31 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 127, 127);
+INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (3, 9223372036854775807, 999999999999999999, 999999999999999999, 0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 999999999999999999, 999999999999999999);
+INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (4, -9223372036854775808, -999999999999999999, -999999999999999999, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -999999999999999999, -999999999999999999);
+INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (5, -9223372036854775808, 0.123456789, 0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, 0.123456789, 0.123456789);
+INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (6, -9223372036854775808, -0.123456789, -0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -0.123456789, -0.123456789);
+INSERT INTO sat_test_dataset.sat_basic_dataset VALUES (7, -9223372036854775808, -0.123456789, -0.123456789, -0.123456789, false, date('0001-01-01'), datetime('0001-01-01 11:22:33'), timestamp('2021-10-20 11:22:33'), 'йцу', 127, TIME(15, 30, 00), 127, 127, 127, 127, 127, -0.123456789, -0.123456789);

airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog.json

@@ -0,0 +1,19 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "sat_basic_dataset",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": null,
+        "default_cursor_field": null,
+        "source_defined_primary_key": [["id"]],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    }
+  ]
+}

airbyte-integrations/connectors/source-bigquery/integration_tests/configured_catalog_inc.json

@@ -0,0 +1,16 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "sat_basic_dataset",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["id"],
+      "destination_sync_mode": "append"
+    }
+  ]
+}

airbyte-integrations/connectors/source-bigquery/integration_tests/expected_records.json

@@ -0,0 +1,7 @@
+{"stream": "sat_basic_dataset", "data": {"id": 1, "test_column_1": -128, "test_column_10": -128, "test_column_11": -128, "test_column_12": -128.0, "test_column_13": true, "test_column_15": "2021-10-20T00:00:00Z", "test_column_16": "2021-10-20T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "qwe", "test_column_2": -128, "test_column_21": "15:30:00", "test_column_3": -128, "test_column_4": -128, "test_column_5": -128, "test_column_6": -128, "test_column_7": -128, "test_column_8": -128, "test_column_9": -128 }, "emitted_at": 1669643054045 }
+{"stream": "sat_basic_dataset", "data": {"id": 2, "test_column_1": 127, "test_column_10": 127, "test_column_11": 127, "test_column_12": 127.0, "test_column_13": false, "test_column_15": "9999-12-31T00:00:00Z", "test_column_16": "9999-12-31T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 127, "test_column_9": 127 }, "emitted_at": 1669643054045 }
+{"stream": "sat_basic_dataset", "data": {"id": 3, "test_column_1": 9223372036854775807, "test_column_10": 999999999999999999, "test_column_11": 999999999999999999, "test_column_12": 0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 999999999999999999, "test_column_9": 999999999999999999 }, "emitted_at": 1669643054045 }
+{"stream": "sat_basic_dataset", "data": {"id": 7, "test_column_1": -9223372036854775808, "test_column_10": -0.123456789, "test_column_11": -0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -0.123456789, "test_column_9": -0.123456789 }, "emitted_at": 1669643054045 }
+{"stream": "sat_basic_dataset", "data": {"id": 4, "test_column_1": -9223372036854775808, "test_column_10": -999999999999999999, "test_column_11": -999999999999999999, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -999999999999999999, "test_column_9": -999999999999999999 }, "emitted_at": 1669643054045 }
+{"stream": "sat_basic_dataset", "data": {"id": 5, "test_column_1": -9223372036854775808, "test_column_10": 0.123456789, "test_column_11": 0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": 0.123456789, "test_column_9": 0.123456789 }, "emitted_at": 1669643054045 }
+{"stream": "sat_basic_dataset", "data": {"id": 6, "test_column_1": -9223372036854775808, "test_column_10": -0.123456789, "test_column_11": -0.123456789, "test_column_12": -0.123456789, "test_column_13": false, "test_column_15": "0001-01-01T00:00:00Z", "test_column_16": "0001-01-01T11:22:33Z", "test_column_17": "2021-10-20T11:22:33Z", "test_column_19": "\u0439\u0446\u0443", "test_column_2": 127, "test_column_21": "15:30:00", "test_column_3": 127, "test_column_4": 127, "test_column_5": 127, "test_column_6": 127, "test_column_7": 127, "test_column_8": -0.123456789, "test_column_9": -0.123456789 }, "emitted_at": 1669643054045 }

airbyte-integrations/connectors/source-bigquery/integration_tests/full_dataset.sql

@@ -0,0 +1,47 @@
+CREATE TABLE
+    sat_test_dataset.sat_full_dataset(id NUMERIC(29),
+                                      test_column_1 int64,
+                                      test_column_10 decimal(29,
+                                             9),
+                                      test_column_11 bigdecimal(76,
+                                          38),
+                                      test_column_12 float64,
+                                      test_column_13 bool,
+                                      test_column_14 bytes,
+                                      test_column_15 date,
+                                      test_column_16 datetime,
+                                      test_column_17 timestamp,
+                                      test_column_18 geography,
+                                      test_column_19 string,
+                                      test_column_2 int,
+                                      test_column_20 STRUCT<course STRING,
+                                      id INT64>,
+                                      test_column_21 time,
+                                      test_column_22 ARRAY<String>,
+                                      test_column_23 STRUCT<frst String,
+                                      sec int64,
+                                      obbj STRUCT<id_col int64,
+                                      mega_obbj STRUCT<last_col time>>>,
+                                      test_column_24 ARRAY<STRUCT<fff String,
+                                      ggg int64>>,
+                                      test_column_25 ARRAY<STRUCT<fff String,
+                                      ggg ARRAY<STRUCT<ooo String,
+                                      kkk int64>>>>,
+                                      test_column_26 INTERVAL,
+                                      test_column_3 smallint,
+                                      test_column_4 integer,
+                                      test_column_5 bigint,
+                                      test_column_6 tinyint,
+                                      test_column_7 byteint,
+                                      test_column_8 numeric(29,
+                                             9),
+                                      test_column_9 bignumeric(76,
+                                          38) );
+
+INSERT INTO sat_test_dataset.sat_full_dataset VALUES (1, null, null, null, null, true, FROM_BASE64("test"), date('2021-10-20'), datetime('2021-10-20 11:22:33'), timestamp('2021-10-20 11:22:33'), ST_GEOGFROMTEXT('POINT(1 2)'), 'qwe', null, STRUCT("B.A",12), TIME(15, 30, 00), ['a', 'b'], STRUCT('s' as frst, 1 as sec, STRUCT(555 as id_col, STRUCT(TIME(15, 30, 00) as time) as mega_obbj) as obbj), [STRUCT('qqq' as fff, 1 as ggg), STRUCT('kkk' as fff, 2 as ggg)], [STRUCT('qqq' as fff, [STRUCT('fff' as ooo, 1 as kkk), STRUCT('hhh' as ooo, 2 as kkk)] as ggg)], MAKE_INTERVAL(2021, 10, 10, 10, 10, 10), null, null, null, null, null, null, null);
+INSERT INTO sat_test_dataset.sat_full_dataset VALUES (2, -128, -128, -128, -128, false, null, date('9999-12-31'), datetime('9999-12-31 11:22:33'), null, null, 'йцу', -128, null, null, null, null, null, null, null, -128, -128, -128, -128, -128, -128, -128);
+INSERT INTO sat_test_dataset.sat_full_dataset VALUES (3, 127, 127, 127, 127, null, null, date('0001-01-01'), datetime('0001-01-01 11:22:33'), null, null, null, 127, null, null, null, null, null, null, null, 127, 127, 127, 127, 127, 127, 127);
+INSERT INTO sat_test_dataset.sat_full_dataset VALUES (4, 9223372036854775807, 999999999999999999, 999999999999999999, 0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 999999999999999999, 999999999999999999);
+INSERT INTO sat_test_dataset.sat_full_dataset VALUES (5, -9223372036854775808, -999999999999999999, -999999999999999999, -0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, -999999999999999999, -999999999999999999);
+INSERT INTO sat_test_dataset.sat_full_dataset VALUES (6, null, 0.123456789, 0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0.123456789, 0.123456789);
+INSERT INTO sat_test_dataset.sat_full_dataset VALUES (7, null, -0.123456789, -0.123456789, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, -0.123456789, -0.123456789);