🎉 Source and Destination Snowflake: Add jdbc_url_params support for o…

…ptional JDBC parameters (#9623)
airbytehq · Jan 28, 2022 · 5391880 · 5391880
1 parent d70417a
commit 5391880
Show file tree

Hide file tree

Showing 18 changed files with 86 additions and 35 deletions.
diff --git a/...esources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/...esources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json
@@ -2,7 +2,7 @@
   "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba",
   "name": "Snowflake",
   "dockerRepository": "airbyte/destination-snowflake",
-  "dockerImageTag": "0.4.5",
+  "dockerImageTag": "0.4.6",
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake",
   "icon": "snowflake.svg"
 }
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2",
   "name": "Snowflake",
   "dockerRepository": "airbyte/source-snowflake",
-  "dockerImageTag": "0.1.5",
+  "dockerImageTag": "0.1.6",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake",
   "icon": "snowflake.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -179,7 +179,7 @@
 - name: Snowflake
   destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
   dockerRepository: airbyte/destination-snowflake
-  dockerImageTag: 0.4.5
+  dockerImageTag: 0.4.6
   documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
   icon: snowflake.svg
 - name: MariaDB ColumnStore

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -3787,7 +3787,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-snowflake:0.4.5"
+- dockerImage: "airbyte/destination-snowflake:0.4.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake"
     connectionSpecification:
@@ -3856,11 +3856,18 @@
           airbyte_secret: true
           title: "Password"
           order: 6
+        jdbc_url_params:
+          description: "Additional properties to pass to the JDBC URL string when\
+            \ connecting to the database formatted as 'key=value' pairs separated\
+            \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+          title: "JDBC URL Params"
+          type: "string"
+          order: 7
         loading_method:
           type: "object"
           title: "Loading Method"
           description: "The loading method used to send data to Snowflake."
-          order: 7
+          order: 8
           oneOf:
           - title: "[Recommended] Internal Staging"
             additionalProperties: false

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -690,7 +690,7 @@
 - name: Snowflake
   sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2
   dockerRepository: airbyte/source-snowflake
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake
   icon: snowflake.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4978,7 +4978,7 @@
           airbyte_secret: true
           order: 4
         jdbc_url_params:
-          description: "Additional properties to pass to the jdbc url string when\
+          description: "Additional properties to pass to the JDBC URL string when\
             \ connecting to the database formatted as 'key=value' pairs separated\
             \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
           title: "JDBC URL Params"
@@ -7339,7 +7339,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-snowflake:0.1.5"
+- dockerImage: "airbyte/source-snowflake:0.1.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/snowflake"
     connectionSpecification:
@@ -7405,6 +7405,13 @@
           airbyte_secret: true
           title: "Password"
           order: 6
+        jdbc_url_params:
+          description: "Additional properties to pass to the JDBC URL string when\
+            \ connecting to the database formatted as 'key=value' pairs separated\
+            \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+          title: "JDBC URL Params"
+          type: "string"
+          order: 7
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs
@@ -38,7 +38,7 @@
         "order": 4
       },
       "jdbc_url_params": {
-        "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)",
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)",
         "type": "string",
         "order": 5
       },

diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile
@@ -18,6 +18,6 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.4.5
+LABEL io.airbyte.version=0.4.6
 
 LABEL io.airbyte.name=airbyte/destination-snowflake
diff --git a/...wflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/...wflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java
@@ -13,6 +13,8 @@
 import java.sql.SQLException;
 import java.time.Duration;
 import java.util.Properties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * SnowflakeDatabase contains helpers to create connections to and run queries on Snowflake.
@@ -22,9 +24,12 @@ public class SnowflakeDatabase {
   private static final Duration NETWORK_TIMEOUT = Duration.ofMinutes(1);
   private static final Duration QUERY_TIMEOUT = Duration.ofHours(3);
   private static final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer();
+  private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeDatabase.class);
 
   public static Connection getConnection(final JsonNode config) throws SQLException {
-    final String connectUrl = String.format("jdbc:snowflake://%s", config.get("host").asText());
+
+    final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:snowflake://%s/?",
+        config.get("host").asText()));
 
     final Properties properties = new Properties();
 
@@ -47,7 +52,14 @@ public static Connection getConnection(final JsonNode config) throws SQLExceptio
     // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow
     properties.put("JDBC_QUERY_RESULT_FORMAT", "JSON");
 
-    return DriverManager.getConnection(connectUrl, properties);
+    // https://docs.snowflake.com/en/user-guide/jdbc-configure.html#jdbc-driver-connection-string
+    if (config.has("jdbc_url_params")) {
+      jdbcUrl.append(config.get("jdbc_url_params").asText());
+    }
+
+    LOGGER.info(jdbcUrl.toString());
+
+    return DriverManager.getConnection(jdbcUrl.toString(), properties);
   }
 
   public static JdbcDatabase getDatabase(final JsonNode config) {

diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json
@@ -68,11 +68,17 @@
         "title": "Password",
         "order": 6
       },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "title": "JDBC URL Params",
+        "type": "string",
+        "order": 7
+      },
       "loading_method": {
         "type": "object",
         "title": "Loading Method",
         "description": "The loading method used to send data to Snowflake.",
-        "order": 7,
+        "order": 8,
         "oneOf": [
           {
             "title": "[Recommended] Internal Staging",

diff --git a/...integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/...integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json
@@ -43,7 +43,7 @@
         "order": 4
       },
       "jdbc_url_params": {
-        "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
         "title": "JDBC URL Params",
         "type": "string",
         "order": 5

diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json
@@ -43,7 +43,7 @@
         "order": 4
       },
       "jdbc_url_params": {
-        "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
         "title": "JDBC URL Params",
         "type": "string",
         "order": 5

diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json
@@ -38,7 +38,7 @@
         "order": 4
       },
       "jdbc_url_params": {
-        "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)",
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)",
         "type": "string",
         "order": 5
       },

diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-snowflake
diff --git a/...rce-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/...rce-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java
@@ -33,23 +33,34 @@ public static void main(final String[] args) throws Exception {
 
   @Override
   public JsonNode toDatabaseConfig(final JsonNode config) {
-    return Jsons.jsonNode(ImmutableMap.builder()
-        .put("jdbc_url", String.format("jdbc:snowflake://%s/",
-            config.get("host").asText()))
-        .put("host", config.get("host").asText())
+
+    final StringBuilder jdbcUrl = new StringBuilder(String.format("jdbc:snowflake://%s/?",
+        config.get("host").asText()));
+
+    // Add required properties
+    jdbcUrl.append(String.format("role=%s&warehouse=%s&database=%s&schema=%s&JDBC_QUERY_RESULT_FORMAT=%s&CLIENT_SESSION_KEEP_ALIVE=%s",
+        config.get("role").asText(),
+        config.get("warehouse").asText(),
+        config.get("database").asText(),
+        config.get("schema").asText(),
+        // Needed for JDK17 - see
+        // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow
+        "JSON",
+        true));
+
+    // https://docs.snowflake.com/en/user-guide/jdbc-configure.html#jdbc-driver-connection-string
+    if (config.has("jdbc_url_params")) {
+      jdbcUrl.append("&").append(config.get("jdbc_url_params").asText());
+    }
+
+    LOGGER.info(jdbcUrl.toString());
+
+    final ImmutableMap.Builder<Object, Object> configBuilder = ImmutableMap.builder()
         .put("username", config.get("username").asText())
         .put("password", config.get("password").asText())
-        .put("connection_properties",
-            String.format("role=%s;warehouse=%s;database=%s;schema=%s;JDBC_QUERY_RESULT_FORMAT=%s;CLIENT_SESSION_KEEP_ALIVE=%s;",
-                config.get("role").asText(),
-                config.get("warehouse").asText(),
-                config.get("database").asText(),
-                config.get("schema").asText(),
-                // Needed for JDK17 - see
-                // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow
-                "JSON",
-                true))
-        .build());
+        .put("jdbc_url", jdbcUrl.toString());
+
+    return Jsons.jsonNode(configBuilder.build());
   }
 
   @Override

diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/source-snowflake/src/main/resources/spec.json
@@ -63,6 +63,12 @@
         "airbyte_secret": true,
         "title": "Password",
         "order": 6
+      },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "title": "JDBC URL Params",
+        "type": "string",
+        "order": 7
       }
     }
   }

diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md
@@ -113,6 +113,7 @@ You should now have all the requirements needed to configure Snowflake as a dest
 * **Schema**
 * **Username**
 * **Password**
+* **JDBC URL Params** (Optional)
 
 ## Notes about Snowflake Naming Conventions
 
@@ -216,6 +217,7 @@ Finally, you need to add read/write permissions to your bucket with that email.
 
 | Version | Date       | Pull Request | Subject |
 |:--------|:-----------| :-----       | :------ |
+| 0.4.6   | 2022-01-28 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters |
 | 0.4.5   | 2021-12-29 | [#9184](https://github.com/airbytehq/airbyte/pull/9184) | Update connector fields title/description |
 | 0.4.4   | 2022-01-24 | [#9743](https://github.com/airbytehq/airbyte/pull/9743) | Fixed bug with dashes in schema name |
 | 0.4.3   | 2022-01-20 | [#9531](https://github.com/airbytehq/airbyte/pull/9531) | Start using new S3StreamCopier and expose the purgeStagingData option |
@@ -236,4 +238,3 @@ Finally, you need to add read/write permissions to your bucket with that email.
 | 0.3.12  | 2021-07-30 | [#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json                                                                        |
 | 0.3.11  | 2021-07-21 | [#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer                                                                           |
 | 0.3.10  | 2021-07-12 | [#4713](https://github.com/airbytehq/airbyte/pull/4713)| Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake                                |
-
diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md
@@ -30,7 +30,8 @@ The Snowflake source does not alter the schema present in your warehouse. Depend
 6. **Schema**
 7. **Username**
 8. **Password**
-9. Create a dedicated read-only Airbyte user and role with access to all schemas needed for replication.
+9. **JDBC URL Params** (Optional)
+10. Create a dedicated read-only Airbyte user and role with access to all schemas needed for replication.
 
 ### Setup guide
 
@@ -75,9 +76,9 @@ Your database user should now be ready for use with Airbyte.
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.6 | 2022-01-25 | [9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters |
 | 0.1.5 | 2022-01-19 | [9567](https://github.com/airbytehq/airbyte/pull/9567) | Added parameter for keeping JDBC session alive |
 | 0.1.4 | 2021-12-30 | [9203](https://github.com/airbytehq/airbyte/pull/9203) | Update connector fields title/description |
 | 0.1.3 | 2021-01-11 | [9304](https://github.com/airbytehq/airbyte/pull/9304) | Upgrade version of JDBC driver |
 | 0.1.2 | 2021-10-21 | [7257](https://github.com/airbytehq/airbyte/pull/7257) | Fixed parsing of extreme values for FLOAT and NUMBER data types |
 | 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator |
-