0005076: S3 Dialect Implementation

symmetric-assemble/src/asciidoc/appendix/S3.ad

@@ -0,0 +1,18 @@
+
+=== Amazon S3
+
+ifndef::pro[]
+S3 is only supported in the professional version of SymmetricDS.
+endif::pro[]
+
+ifdef::pro[]
+
+Use `symadmin module install S3` to install driver files, or copy your own files into the `lib` sub-directory.
+
+Send changes from your relational database to S3 as a CSV file.  An S3 node can be setup as a <<Load Only Node>> to receive changes from another node that is capturing changes.
+
+Setup the S3 node by using the <<Add Node>> wizard and selecting Amazon S3 as the type. User and password are not needed (or used).
+
+After hitting next you can setup the bucket in S3 that the files go to as well as the directory path desired within that bucket. This will also be where you specify the user credentials that have access to your Amazon S3 instance.
+
+endif::pro[]

symmetric-assemble/src/asciidoc/appendix/databases.ad

@@ -559,5 +559,6 @@ include::snowflake.ad[]
 include::sqlite.ad[]
 include::sybase-ase.ad[]
 include::sybase-sqlanywhere.ad[]
+include::S3.ad[]
 include::teradata.ad[]
 include::tibero.ad[]

symmetric-client/src/main/resources/symmetric-modules.properties

@@ -42,8 +42,7 @@ nuodb=jdbc.nuodb:nuodb-jdbc:3.3.1
 opensearch=org.opensearch.client:opensearch-rest-client:1.0.0,org.elasticsearch.client:elasticsearch-rest-high-level-client:7.10.2,org.elasticsearch:elasticsearch:7.10.2,org.elasticsearch.client:elasticsearch-rest-client:7.10.2,org.elasticsearch.plugin:mapper-extras-client:7.10.2,org.elasticsearch.plugin:parent-join-client:7.10.2,org.elasticsearch.plugin:aggs-matrix-stats-client:7.10.2,org.elasticsearch.plugin:rank-eval-client:7.10.2,org.elasticsearch.plugin:lang-mustache-client:7.10.2,org.elasticsearch:elasticsearch-core:7.10.2,org.elasticsearch:elasticsearch-secure-sm:7.10.2,org.elasticsearch:elasticsearch-x-content:7.10.2,org.elasticsearch:elasticsearch-geo:7.10.2,org.elasticsearch:elasticsearch-cli:7.10.2,org.elasticsearch:jna:5.5.0,org.apache.lucene:lucene-core:8.7.0, org.apache.httpcomponents:httpcore:4.4.12,org.apache.httpcomponents:httpclient:4.5.10,org.apache.httpcomponents:httpasyncclient:4.1.4,org.apache.httpcomponents:httpcore-nio:4.4.12,joda-time:joda-time:2.10.4
 #oracle=jdbc:ojdbc8:18.3.0.0
 #postgres=org.postgresql:postgresql:42.2.8
-redshift=com.amazon.redshift:redshift-jdbc42-no-awssdk:1.2.36.1060, com.amazonaws:aws-java-sdk-s3:1.11.510, com.amazonaws:aws-java-sdk-kms:1.11.510, software.amazon.ion:ion-java:1.0.2, com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7, com.fasterxml.jackson.core:jackson-annotations:2.10.3, joda-time:joda-time:2.8.1, com.amazonaws:jmespath-java:1.11.510, com.amazonaws:aws-java-sdk-core:1.11.510
-snowflake=net.snowflake:snowflake-jdbc:3.6.27, com.microsoft.azure:azure-storage:8.1.0, com.microsoft.azure:azure-keyvault-core:1.0.0, com.google.guava:guava:28.1-android, com.amazonaws:aws-java-sdk-s3:1.11.510, com.amazonaws:aws-java-sdk-kms:1.11.510, software.amazon.ion:ion-java:1.0.2, com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7, com.fasterxml.jackson.core:jackson-annotations:2.10.3, joda-time:joda-time:2.8.1, com.amazonaws:jmespath-java:1.11.510, com.amazonaws:aws-java-sdk-core:1.11.510
+redshift=com.amazon.redshift:redshift-jdbc42:2.1.0.2, com.amazonaws:aws-java-sdk-s3:1.11.510, com.amazonaws:aws-java-sdk-kms:1.11.510, software.amazon.ion:ion-java:1.0.2, com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7, com.fasterxml.jackson.core:jackson-annotations:2.10.3, joda-time:joda-time:2.8.1, com.amazonaws:jmespath-java:1.11.510, com.amazonaws:aws-java-sdk-core:1.11.510snowflake=net.snowflake:snowflake-jdbc:3.6.27, com.microsoft.azure:azure-storage:8.1.0, com.microsoft.azure:azure-keyvault-core:1.0.0, com.google.guava:guava:28.1-android, com.amazonaws:aws-java-sdk-s3:1.11.510, com.amazonaws:aws-java-sdk-kms:1.11.510, software.amazon.ion:ion-java:1.0.2, com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7, com.fasterxml.jackson.core:jackson-annotations:2.10.3, joda-time:joda-time:2.8.1, com.amazonaws:jmespath-java:1.11.510, com.amazonaws:aws-java-sdk-core:1.11.510
 sqlite=org.xerial:sqlite-jdbc:3.25.2
 #sqlserver=net.sourceforge.jtds:jtds:1.3.1
 swaggerui=org.jumpmind.symmetric:symmetric-swagger-ui:$version, io.springfox:springfox-core:2.9.2, io.springfox:springfox-schema:2.9.2, io.springfox:springfox-spi:2.9.2, io.springfox:springfox-spring-web:2.9.2, io.springfox:springfox-swagger2:2.9.2, io.springfox:springfox-swagger-common:2.9.2, io.springfox:springfox-swagger-ui:2.9.2, io.swagger:swagger-models:1.5.20, net.bytebuddy:byte-buddy:1.8.12, org.mapstruct:mapstruct:1.2.0.Final, org.springframework.plugin:spring-plugin-core:1.2.0.RELEASE, org.springframework.plugin:spring-plugin-metadata:1.2.0.RELEASE, com.fasterxml:classmate:1.4.0, com.fasterxml.jackson.core:jackson-databind:2.10.3, com.fasterxml.jackson.core:jackson-annotations:2.10.3, com.fasterxml.jackson.core:jackson-core:2.10.3, com.google.guava:guava:28.2-android, com.google.guava:failureaccess:1.0.1, com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava, com.google.code.findbugs:jsr305:3.0.2, org.checkerframework:checker-compat-qual:2.5.5, com.google.errorprone:error_prone_annotations:2.3.4, com.google.j2objc:j2objc-annotations:1.3
@@ -53,6 +52,8 @@ tibero=jdbc.tibero:tibero:6
 topspeed=nl.cad:tps-parse:1.0.15-SNAPSHOT, com.beust:jcommander:1.30, joda-time:joda-time:2.8.1
 voltdb=org.voltdb:voltdbclient:8.4.1
 cosmos=com.azure:azure-cosmos:4.19.1
+s3=com.amazonaws:aws-java-sdk-s3:1.12.121
+
 
 driver.bigquery=com.google.cloud.bigquery.BigQuery
 driver.cassandra=com.datastax.driver.core.Cluster
@@ -80,3 +81,5 @@ driver.sybase=com.sybase.jdbc4.jdbc.SybDriver
 driver.teradata=com.teradata.jdbc.TeraDriver
 driver.tibero=com.tmax.tibero.jdbc.TbDriver
 driver.voltdb=org.voltdb.jdbc.Driver
+driver.s3=com.amazonaws.services.s3.AmazonS3
+

symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java

@@ -50,6 +50,8 @@ private ParameterConstants() {
     public final static String DB_AWS_ACTIVE_KEY = "target.db.aws.active.key";
     public final static String DB_AWS_SECRET_KEY = "target.db.aws.secret.key";
     public final static String DB_AWS_REGION = "target.db.aws.region";
+    public final static String DB_AWS_BUCKET = "target.db.aws.bucket";
+    public final static String DB_AWS_FOLDER = "target.db.aws.folder";
     public final static String DB_USER = "db.user";
     public final static String DB_PASSWORD = "db.password";
     public final static String START_PULL_JOB = "start.pull.job";
@@ -456,6 +458,12 @@ private ParameterConstants() {
     public final static String COSMOS_DEFAULT_DATABASE = "cosmosdb.default.database";
     public final static String[] ALL_COSMOS_PARAMS = new String[] { COSMOS_DEFAULT_DATABASE };
 
+    public final static String OPENSEARCH_LOAD_AWS_ACCESS_KEY = "opensearch.load.aws.access.key";
+    public final static String OPENSEARCH_LOAD_AWS_SECRET_KEY = "opensearch.load.aws.secret.key";
+    public final static String S3_LOAD_AWS_ACCESS_KEY = "s3.load.aws.access.key";
+    public final static String S3_LOAD_AWS_SECRET_KEY = "s3.load.aws.secret.key";
+
+
     public static Map<String, ParameterMetaData> getParameterMetaData() {
         return parameterMetaData;
     }

symmetric-db/src/main/java/org/jumpmind/db/platform/DatabaseNamesConstants.java

@@ -68,4 +68,6 @@ private DatabaseNamesConstants() {
     public final static String ELASTIC = "elasticsearch";
     public final static String OPENSEARCH = "opensearch";
     public final static String COSMOS = "cosmos";
+    public final static String S3 = "S3";
+
 }