[10719] Destination Oracle: custom JDBC parameters (#13841)

* [10719] Destination Oracle: custom JDBC parameters * [10719] Destination Oracle: custom JDBC parameters fixed tests * [10719] Destination Oracle: custom JDBC parameters fixed tests * [10719] Destination Oracle: custom JDBC parameters fix for SSH oracle tests * [10719] Destination Oracle: custom JDBC parameters fixed test * [10719] Destination Oracle: custom JDBC parameters updated image tag * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jul 27, 2022 · 3751be4 · 3751be4
1 parent ce9b1d3
commit 3751be4
Show file tree

Hide file tree

Showing 12 changed files with 90 additions and 27 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -196,7 +196,7 @@
 - name: Oracle
   destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72
   dockerRepository: airbyte/destination-oracle
-  dockerImageTag: 0.1.18
+  dockerImageTag: 0.1.19
   documentationUrl: https://docs.airbyte.io/integrations/destinations/oracle
   icon: oracle.svg
   releaseStage: alpha

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -3070,7 +3070,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-oracle:0.1.18"
+- dockerImage: "airbyte/destination-oracle:0.1.19"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/oracle"
     connectionSpecification:
@@ -3082,6 +3082,7 @@
       - "port"
       - "username"
       - "sid"
+      additionalProperties: true
       properties:
         host:
           title: "Host"
@@ -3116,6 +3117,13 @@
           type: "string"
           airbyte_secret: true
           order: 4
+        jdbc_url_params:
+          description: "Additional properties to pass to the JDBC URL string when\
+            \ connecting to the database formatted as 'key=value' pairs separated\
+            \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+          title: "JDBC URL Params"
+          type: "string"
+          order: 5
         schema:
           title: "Default Schema"
           description: "The default schema is used as the target schema for all statements\
@@ -3127,13 +3135,13 @@
           examples:
           - "airbyte"
           default: "airbyte"
-          order: 5
+          order: 6
         encryption:
           title: "Encryption"
           type: "object"
           description: "The encryption method which is used when communicating with\
             \ the database."
-          order: 6
+          order: 7
           oneOf:
           - title: "Unencrypted"
             description: "Data transfer will not be encrypted."

diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.18
+LABEL io.airbyte.version=0.1.19
 LABEL io.airbyte.name=airbyte/destination-oracle-strict-encrypt
diff --git a/...ations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json b/...ations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json
@@ -46,13 +46,19 @@
         "airbyte_secret": true,
         "order": 4
       },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "title": "JDBC URL Params",
+        "type": "string",
+        "order": 5
+      },
       "schema": {
         "title": "Default Schema",
         "description": "The default schema is used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. The usual value for this field is \"airbyte\".  In Oracle, schemas and users are the same thing, so the \"user\" parameter is used as the login credentials and this is used for the default Airbyte message schema.",
         "type": "string",
         "examples": ["airbyte"],
         "default": "airbyte",
-        "order": 5
+        "order": 6
       }
     }
   }

diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.18
+LABEL io.airbyte.version=0.1.19
 LABEL io.airbyte.name=airbyte/destination-oracle
diff --git a/...on-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/...on-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java
@@ -40,6 +40,8 @@ public class OracleDestination extends AbstractJdbcDestination implements Destin
   private static final String KEY_STORE_PASS = RandomStringUtils.randomAlphanumeric(8);
   public static final String ENCRYPTION_METHOD_KEY = "encryption_method";
 
+  public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params";
+
   enum Protocol {
     TCP,
     TCPS
@@ -99,6 +101,9 @@ public JsonNode toJdbcConfig(final JsonNode config) {
     if (config.has(JdbcUtils.PASSWORD_KEY)) {
       configBuilder.put(JdbcUtils.PASSWORD_KEY, config.get(JdbcUtils.PASSWORD_KEY).asText());
     }
+    if (config.has(JDBC_URL_PARAMS_KEY)) {
+      configBuilder.put(JDBC_URL_PARAMS_KEY, config.get(JDBC_URL_PARAMS_KEY));
+    }
 
     return Jsons.jsonNode(configBuilder.build());
   }

diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json
@@ -46,19 +46,25 @@
         "airbyte_secret": true,
         "order": 4
       },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "title": "JDBC URL Params",
+        "type": "string",
+        "order": 5
+      },
       "schema": {
         "title": "Default Schema",
         "description": "The default schema is used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. The usual value for this field is \"airbyte\".  In Oracle, schemas and users are the same thing, so the \"user\" parameter is used as the login credentials and this is used for the default Airbyte message schema.",
         "type": "string",
         "examples": ["airbyte"],
         "default": "airbyte",
-        "order": 5
+        "order": 6
       },
       "encryption": {
         "title": "Encryption",
         "type": "object",
         "description": "The encryption method which is used when communicating with the database.",
-        "order": 6,
+        "order": 7,
         "oneOf": [
           {
             "title": "Unencrypted",

diff --git a/...n/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java b/...n/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java
@@ -4,6 +4,8 @@
 
 package io.airbyte.integrations.destination.oracle;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -18,7 +20,7 @@
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Map;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 public class NneOracleDestinationAcceptanceTest extends UnencryptedOracleDestinationAcceptanceTest {
 
@@ -48,7 +50,7 @@ public void testEncryption() throws SQLException {
     final List<JsonNode> collect = database.queryJsons(networkServiceBanner);
 
     assertThat(collect.get(2).get("NETWORK_SERVICE_BANNER").asText(),
-        equals("Oracle Advanced Security: " + algorithm + " encryption"));
+        is(equalTo("AES256 Encryption service adapter for Linux: Version 18.0.0.0.0 - Production")));
   }
 
   private Map<String, String> getAdditionalProperties(final String algorithm) {

diff --git a/...o/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/...o/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java
@@ -4,7 +4,11 @@
 
 package io.airbyte.integrations.destination.oracle;
 
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static io.airbyte.integrations.util.HostPortResolver.resolveHost;
+import static io.airbyte.integrations.util.HostPortResolver.resolvePort;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -22,11 +26,12 @@
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
 import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 import javax.sql.DataSource;
 import org.jooq.DSLContext;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 public class UnencryptedOracleDestinationAcceptanceTest extends DestinationAcceptanceTest {
 
@@ -43,8 +48,8 @@ protected String getImageName() {
   private JsonNode getConfig(final OracleContainer db) {
 
     return Jsons.jsonNode(ImmutableMap.builder()
-        .put(JdbcUtils.HOST_KEY, db.getHost())
-        .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort())
+        .put(JdbcUtils.HOST_KEY, resolveHost(db))
+        .put(JdbcUtils.PORT_KEY, resolvePort(db))
         .put("sid", db.getSid())
         .put(JdbcUtils.USERNAME_KEY, db.getUsername())
         .put(JdbcUtils.PASSWORD_KEY, db.getPassword())
@@ -62,9 +67,9 @@ protected JsonNode getConfig() {
 
   @Override
   protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
-                                           final String streamName,
-                                           final String namespace,
-                                           final JsonNode streamSchema)
+      final String streamName,
+      final String namespace,
+      final JsonNode streamSchema)
       throws Exception {
     return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace)
         .stream()
@@ -105,8 +110,8 @@ protected boolean supportObjectDataTypeTest() {
 
   @Override
   protected List<JsonNode> retrieveNormalizedRecords(final TestDestinationEnv env,
-                                                     final String streamName,
-                                                     final String namespace)
+      final String streamName,
+      final String namespace)
       throws Exception {
     final String tableName = namingResolver.getIdentifier(streamName);
     return retrieveRecordsFromTable(tableName, namespace);
@@ -123,11 +128,9 @@ private List<JsonNode> retrieveRecordsFromTable(final String tableName, final St
       throws SQLException {
     try (final DSLContext dslContext = getDSLContext(config)) {
       final List<org.jooq.Record> result = getDatabase(dslContext)
-          .query(ctx -> ctx.fetch(
+          .query(ctx -> new ArrayList<>(ctx.fetch(
               String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName,
-                  OracleDestination.COLUMN_NAME_EMITTED_AT))
-              .stream()
-              .collect(Collectors.toList()));
+                  OracleDestination.COLUMN_NAME_EMITTED_AT))));
       return result
           .stream()
           .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat()))
@@ -194,8 +197,8 @@ public void testNoneEncryption() throws SQLException {
         "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)";
     final List<JsonNode> collect = database.queryJsons(networkServiceBanner);
 
-    assertTrue(collect.get(1).get("NETWORK_SERVICE_BANNER").asText()
-        .contains("Oracle Advanced Security: encryption"));
+    assertThat(collect.get(1).get("NETWORK_SERVICE_BANNER").asText(),
+        is(equalTo("Encryption service for Linux: Version 18.0.0.0.0 - Production")));
   }
 
 }
diff --git a/...racle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java b/...racle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java
@@ -108,4 +108,35 @@ void testInvalidEncryptionMethod() {
     assertThrows(RuntimeException.class, () -> destination.obtainConnectionProtocol(inputConfig));
   }
 
+  @Test
+  void testEmptyExtraParams() {
+    final String extraParam = "";
+    JsonNode config = buildConfigWithExtraJdbcParameters(extraParam);
+    final JsonNode jdbcConfig = new OracleDestination().toJdbcConfig(config);
+    assertNotNull(jdbcConfig.get(OracleDestination.JDBC_URL_PARAMS_KEY).asText());
+    assertEquals(extraParam, jdbcConfig.get(OracleDestination.JDBC_URL_PARAMS_KEY).asText());
+  }
+
+  @Test
+  void testExtraParams() {
+    final String extraParam = "key1=value1&key2=value2&key3=value3";
+    JsonNode config = buildConfigWithExtraJdbcParameters(extraParam);
+    final JsonNode jdbcConfig = new OracleDestination().toJdbcConfig(config);
+    assertNotNull(jdbcConfig.get(OracleDestination.JDBC_URL_PARAMS_KEY).asText());
+    assertEquals(extraParam, jdbcConfig.get(OracleDestination.JDBC_URL_PARAMS_KEY).asText());
+
+  }
+
+  private JsonNode buildConfigWithExtraJdbcParameters(String extraParam) {
+
+    return Jsons.jsonNode(com.google.common.collect.ImmutableMap.of(
+            "host", "localhost",
+            "port", 1773,
+            "sid", "ORCL",
+            "database", "db",
+            "username", "username",
+            "password", "verysecure",
+            "jdbc_url_params", extraParam));
+  }
+
 }
diff --git a/deps.toml b/deps.toml
@@ -17,7 +17,7 @@ connectors-testcontainers-pulsar = "1.16.2"
 connectors-testcontainers-scylla = "1.16.2"
 connectors-testcontainers-tidb = "1.16.3"
 connectors-destination-testcontainers-clickhouse = "1.17.3"
-connectors-destination-testcontainers-oracle-xe = "1.16.0"
+connectors-destination-testcontainers-oracle-xe = "1.17.3"
 connectors-source-testcontainers-clickhouse = "1.17.3"
 platform-testcontainers = "1.17.1"
 

diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md
@@ -92,6 +92,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv
 
 | Version     | Date | Pull Request                                             | Subject                                                                                             |
 |:------------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
+| 0.1.19      | 2022-07-26 | [\#10719](https://github.com/airbytehq/airbyte/pull/)    | Destination Oracle: added custom JDBC parameters support.                                           |
 | 0.1.18      | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors |
 | unpublished | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance |
 | 0.1.16      | 2022-04-06 | [11514](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0                                                                  |
@@ -113,6 +114,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv
 
 | Version | Date       | Pull Request                                            | Subject                                                                                                                     |
 |:--------|:-----------|:--------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------|
+| 0.1.19   | 2022-07-26 | [\#10719](https://github.com/airbytehq/airbyte/pull/) | Destination Oracle: added custom JDBC parameters support.                                        |
 | 0.1.7   | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors |
 | 0.1.5   | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance |
 | 0.1.4   | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support                                                                    |