Add method field on spec.json connectors (snowflake and postgres) (#3960

) * add prop for oneOf snowflake * add method field to pg and update config * correct pg tests * change version in docker file
airbytehq · Jun 8, 2021 · 05b8a2c · 05b8a2c
1 parent b4dd211
commit 05b8a2c
Show file tree

Hide file tree

Showing 9 changed files with 62 additions and 21 deletions.
diff --git a/...esources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/...esources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json
@@ -2,6 +2,6 @@
   "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba",
   "name": "Snowflake",
   "dockerRepository": "airbyte/destination-snowflake",
-  "dockerImageTag": "0.3.7",
+  "dockerImageTag": "0.3.8",
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake"
 }
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750",
   "name": "Postgres",
   "dockerRepository": "airbyte/source-postgres",
-  "dockerImageTag": "0.3.2",
+  "dockerImageTag": "0.3.3",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-postgres",
   "icon": "postgresql.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -22,7 +22,7 @@
 - destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
   name: Snowflake
   dockerRepository: airbyte/destination-snowflake
-  dockerImageTag: 0.3.7
+  dockerImageTag: 0.3.8
   documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
 - destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
   name: S3

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -41,7 +41,7 @@
 - sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   name: Postgres
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 0.3.2
+  dockerImageTag: 0.3.3
   documentationUrl: https://hub.docker.com/r/airbyte/source-postgres
   icon: postgresql.svg
 - sourceDefinitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01

diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile
@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.7
+LABEL io.airbyte.version=0.3.8
 LABEL io.airbyte.name=airbyte/destination-snowflake
diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json
@@ -79,25 +79,38 @@
             "title": "Standard Inserts",
             "additionalProperties": false,
             "description": "Uses <pre>INSERT</pre> statements to send batches of records to Snowflake. Easiest (no setup) but not recommended for large production workloads due to slow speed.",
-            "required": [],
-            "properties": {}
+            "required": ["method"],
+            "properties": {
+              "method": {
+                "type": "string",
+                "enum": ["Standard"],
+                "default": "Standard"
+              }
+            }
           },
           {
             "title": "AWS S3 Staging",
             "additionalProperties": false,
             "description": "Writes large batches of records to a file, uploads the file to S3, then uses <pre>COPY INTO table</pre> to upload the file. Recommended for large production workloads for better speed and scalability.",
             "required": [
+              "method",
               "s3_bucket_name",
               "access_key_id",
               "secret_access_key"
             ],
             "properties": {
+              "method": {
+                "type": "string",
+                "enum": ["S3 Staging"],
+                "default": "S3 Staging",
+                "order": 0
+              },
               "s3_bucket_name": {
                 "title": "S3 Bucket Name",
                 "type": "string",
                 "description": "The name of the staging S3 bucket. Airbyte will write files to this bucket and read them via <pre>COPY</pre> statements on Snowflake.",
                 "examples": ["airbyte.staging"],
-                "order": 0
+                "order": 1
               },
               "s3_bucket_region": {
                 "title": "S3 Bucket Region",
@@ -121,6 +134,7 @@
                   "ca-central-1",
                   "cn-north-1",
                   "cn-northwest-1",
+                  "eu-central-1",
                   "eu-west-1",
                   "eu-west-2",
                   "eu-west-3",
@@ -129,43 +143,54 @@
                   "sa-east-1",
                   "me-south-1"
                 ],
-                "order": 1
+                "order": 2
               },
               "access_key_id": {
                 "type": "string",
                 "description": "The Access Key Id granting allow one to access the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket.",
                 "title": "S3 Key Id",
                 "airbyte_secret": true,
-                "order": 2
+                "order": 3
               },
               "secret_access_key": {
                 "type": "string",
                 "description": "The corresponding secret to the above access key id.",
                 "title": "S3 Access Key",
                 "airbyte_secret": true,
-                "order": 3
+                "order": 4
               }
             }
           },
           {
             "title": "GCS Staging",
             "additionalProperties": false,
             "description": "Writes large batches of records to a file, uploads the file to GCS, then uses <pre>COPY INTO table</pre> to upload the file. Recommended for large production workloads for better speed and scalability.",
-            "required": ["project_id", "bucket_name", "credentials_json"],
+            "required": [
+              "method",
+              "project_id",
+              "bucket_name",
+              "credentials_json"
+            ],
             "properties": {
+              "method": {
+                "type": "string",
+                "enum": ["GCS Staging"],
+                "default": "GCS Staging",
+                "order": 0
+              },
               "project_id": {
                 "title": "GCP Project ID",
                 "type": "string",
                 "description": "The name of the GCP project ID for your credentials.",
                 "examples": ["my-project"],
-                "order": 0
+                "order": 1
               },
               "bucket_name": {
                 "title": "GCS Bucket Name",
                 "type": "string",
                 "description": "The name of the staging GCS bucket. Airbyte will write files to this bucket and read them via <pre>COPY</pre> statements on Snowflake.",
                 "examples": ["airbyte-staging"],
-                "order": 1
+                "order": 2
               },
               "credentials_json": {
                 "title": "Google Application Credentials",

diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile
@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.2
+LABEL io.airbyte.version=0.3.3
 LABEL io.airbyte.name=airbyte/source-postgres
diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json
@@ -59,22 +59,37 @@
             "title": "Standard",
             "additionalProperties": false,
             "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.",
-            "required": [],
-            "properties": {}
+            "required": ["method"],
+            "properties": {
+              "method": {
+                "type": "string",
+                "enum": ["Standard"],
+                "default": "Standard",
+                "order": 0
+              }
+            }
           },
           {
             "title": "Logical Replication (CDC)",
             "additionalProperties": false,
             "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the <a href=\"https://docs.airbyte.io/integrations/sources/postgres\">Postgres Source</a> docs for more information.",
-            "required": ["replication_slot", "publication"],
+            "required": ["method", "replication_slot", "publication"],
             "properties": {
+              "method": {
+                "type": "string",
+                "enum": ["CDC"],
+                "default": "CDC",
+                "order": 0
+              },
               "replication_slot": {
                 "type": "string",
-                "description": "A pgoutput logical replication slot."
+                "description": "A pgoutput logical replication slot.",
+                "order": 1
               },
               "publication": {
                 "type": "string",
-                "description": "A Postgres publication used for consuming changes."
+                "description": "A Postgres publication used for consuming changes.",
+                "order": 2
               }
             }
           }

diff --git a/...urce-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java b/...urce-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java
@@ -53,7 +53,7 @@ public class PostgresSpecTest {
       + "\"port\" : 5432,  "
       + "\"host\" : \"localhost\",  "
       + "\"ssl\" : true, "
-      + "\"replication_method\" : {    \"replication_slot\" : \"ab_slot\", \"publication\" : \"ab_publication\"  }"
+      + "\"replication_method\" : {    \"method\" : \"CDC\", \"replication_slot\" : \"ab_slot\", \"publication\" : \"ab_publication\"  }"
       + "}";
   private static JsonNode schema;
   private static JsonSchemaValidator validator;
@@ -100,6 +100,7 @@ void testWithReplicationMethodStandard() {
     final JsonNode config = Jsons.deserialize(CONFIGURATION);
     ((ObjectNode) config.get("replication_method")).remove("replication_slot");
     ((ObjectNode) config.get("replication_method")).remove("publication");
+    ((ObjectNode) config.get("replication_method")).put("method", "Standard");
     assertTrue(validator.test(schema, config));
 
     final JsonNode configReplicationMethodNotSet = Jsons.deserialize(CONFIGURATION);