🎉 Redshift Source and Destination set SSL as default option (#7234)

* Redshift Source and Restination set SSL as default option * add changelog * remove SSL test| add more documentation * bump new version * bump new version Co-authored-by: vmaltsev <vitalii.maltsev@globallogic.com>
airbytehq · Oct 22, 2021 · 0c93274 · 0c93274
1 parent ad6223d
commit 0c93274
Show file tree

Hide file tree

Showing 14 changed files with 30 additions and 79 deletions.
diff --git a/...esources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json b/...esources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json
@@ -2,7 +2,7 @@
   "destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc",
   "name": "Redshift",
   "dockerRepository": "airbyte/destination-redshift",
-  "dockerImageTag": "0.3.18",
+  "dockerImageTag": "0.3.19",
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift",
   "icon": "redshift.svg"
 }
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6",
   "name": "Redshift",
   "dockerRepository": "airbyte/source-redshift",
-  "dockerImageTag": "0.3.3",
+  "dockerImageTag": "0.3.4",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/redshift",
   "icon": "redshift.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -52,7 +52,7 @@
 - destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc
   name: Redshift
   dockerRepository: airbyte/destination-redshift
-  dockerImageTag: 0.3.18
+  dockerImageTag: 0.3.19
   documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift
   icon: redshift.svg
 - destinationDefinitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -180,7 +180,7 @@
 - sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6
   name: Redshift
   dockerRepository: airbyte/source-redshift
-  dockerImageTag: 0.3.3
+  dockerImageTag: 0.3.4
   documentationUrl: https://docs.airbyte.io/integrations/sources/redshift
   icon: redshift.svg
   sourceType: database

diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile
@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.18
+LABEL io.airbyte.version=0.3.19
 LABEL io.airbyte.name=airbyte/destination-redshift
diff --git a/...src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/...src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java
@@ -37,26 +37,21 @@ public JdbcDatabase getDatabase(final JsonNode config) {
     return getJdbcDatabase(config);
   }
 
-  private static void readSsl(final JsonNode redshiftConfig, final List<String> additionalProperties) {
-    final boolean tls = redshiftConfig.has("tls") && redshiftConfig.get("tls").asBoolean(); // for backward compatibility
-    if (!tls) {
-      additionalProperties.add("ssl=false");
-    } else {
-      additionalProperties.add("ssl=true");
-      additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory");
-    }
+  private static void addSsl(final List<String> additionalProperties) {
+    additionalProperties.add("ssl=true");
+    additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory");
   }
 
   public static JdbcDatabase getJdbcDatabase(final JsonNode config) {
     final List<String> additionalProperties = new ArrayList<>();
     final var jdbcConfig = RedshiftInsertDestination.getJdbcConfig(config);
-    readSsl(config, additionalProperties);
+    addSsl(additionalProperties);
     return Databases.createJdbcDatabase(
         jdbcConfig.get("username").asText(),
         jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null,
         jdbcConfig.get("jdbc_url").asText(),
         RedshiftInsertDestination.DRIVER_CLASS,
-        additionalProperties.isEmpty() ? "" : String.join(";", additionalProperties));
+        String.join(";", additionalProperties));
   }
 
   public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) {

diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json
@@ -105,12 +105,6 @@
         "examples": ["10"],
         "description": "Optional. Increase this if syncing tables larger than 100GB. Only relevant for COPY. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default limit of 100GB tables. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care.",
         "title": "Stream Part Size"
-      },
-      "tls": {
-        "title": "TLS connection",
-        "type": "boolean",
-        "description": "Indicates whether TLS encryption protocol will be used to connect to Redshift. It is recommended to use TLS connection if possible",
-        "default": true
       }
     }
   }

diff --git a/...a/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/...a/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java
@@ -142,7 +142,8 @@ protected Database getDatabase() {
             baseConfig.get("host").asText(),
             baseConfig.get("port").asText(),
             baseConfig.get("database").asText()),
-        "com.amazon.redshift.jdbc.Driver", null);
+        "com.amazon.redshift.jdbc.Driver", null,
+        "ssl=true;sslfactory=com.amazon.redshift.ssl.NonValidatingFactory");
   }
 
   @Override

diff --git a/...va/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTestSSL.java b/...va/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTestSSL.java
diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile
@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.3
+LABEL io.airbyte.version=0.3.4
 LABEL io.airbyte.name=airbyte/source-redshift
diff --git a/...source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java b/...source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java
@@ -37,24 +37,16 @@ public JsonNode toDatabaseConfig(final JsonNode redshiftConfig) {
             redshiftConfig.get("host").asText(),
             redshiftConfig.get("port").asText(),
             redshiftConfig.get("database").asText()));
-    readSsl(redshiftConfig, additionalProperties);
+    addSsl(additionalProperties);
+    builder.put("connection_properties", String.join(";", additionalProperties));
 
-    if (!additionalProperties.isEmpty()) {
-      final String connectionParams = String.join(";", additionalProperties);
-      builder.put("connection_properties", connectionParams);
-    }
     return Jsons.jsonNode(builder
         .build());
   }
 
-  private void readSsl(final JsonNode redshiftConfig, final List<String> additionalProperties) {
-    final boolean tls = redshiftConfig.has("tls") && redshiftConfig.get("tls").asBoolean(); // for backward compatibility
-    if (!tls) {
-      additionalProperties.add("ssl=false");
-    } else {
-      additionalProperties.add("ssl=true");
-      additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory");
-    }
+  private void addSsl(final List<String> additionalProperties) {
+    additionalProperties.add("ssl=true");
+    additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory");
   }
 
   @Override

diff --git a/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json
@@ -32,12 +32,6 @@
         "description": "Password associated with the username.",
         "type": "string",
         "airbyte_secret": true
-      },
-      "tls": {
-        "title": "TLS connection",
-        "type": "boolean",
-        "description": "Indicates whether TLS encryption protocol will be used to connect to Redshift. It is recommended to use TLS connection if possible",
-        "default": true
       }
     }
   }

diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md
@@ -105,13 +105,20 @@ Therefore, Airbyte Redshift destination will create tables and schemas using the
 ## Data Size Limitations
 
 Redshift specifies a maximum limit of 65535 bytes to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift destination fails to load such data and currently ignores that record.
+## Data Size Limitations
 
+Redshift specifies a maximum limit of 65535 bytes to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift destination fails to load such data and currently ignores that record.
 See [docs](https://docs.aws.amazon.com/redshift/latest/dg/r_Character_types.html)
 
+## Encryption
+
+All Redshift connections are encrypted using SSL
+
 ## Changelog
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.3.19  | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only |
 | 0.3.17  | 2021-10-12 | [6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support |
 | 0.3.16  | 2021-10-11 | [6949](https://github.com/airbytehq/airbyte/pull/6949) | Each stream was split into files of 10,000 records each for copying using S3 or GCS  |
 | 0.3.14  | 2021-10-08 | [5924](https://github.com/airbytehq/airbyte/pull/5924) | Fixed AWS S3 Staging COPY is writing records from different table in the same raw table  |

diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md
@@ -45,9 +45,14 @@ This is dependent on your networking setup. The easiest way to verify if Airbyte
 
 Next is to provide the necessary information on how to connect to your cluster such as the `host` whcih is part of the connection string or Endpoint accessible [here](https://docs.aws.amazon.com/redshift/latest/gsg/rs-gsg-connect-to-cluster.html#rs-gsg-how-to-get-connection-string) without the `port` and `database` name \(it typically includes the cluster-id, region and end with `.redshift.amazonaws.com`\).
 
+## Encryption
+
+All Redshift connections are encrypted using SSL
+
 ## Changelog
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.3.4   | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only |
 | 0.3.3   | 2021-10-12 | [6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support |
 | 0.3.2   | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator |