java CDK: improve blobstore module structure (#35285)

airbytehq · Feb 15, 2024 · 4a2bf02 · 4a2bf02
1 parent 40ca9bd
commit 4a2bf02
Show file tree

Hide file tree

Showing 47 changed files with 46 additions and 33 deletions.
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -166,6 +166,7 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.20.8  | 2024-02-15 | [\#35285](https://github.com/airbytehq/airbyte/pull/35285) | Improve blobstore module structure.                                                                                                                            |
 | 0.20.7  | 2024-02-13 | [\#35236](https://github.com/airbytehq/airbyte/pull/35236) | output logs to files in addition to stdout when running tests                                                                                                  |
 | 0.20.6  | 2024-02-12 | [\#35036](https://github.com/airbytehq/airbyte/pull/35036) | Add trace utility to emit analytics messages.                                                                                                                  |
 | 0.20.5  | 2024-02-13 | [\#34869](https://github.com/airbytehq/airbyte/pull/34869) | Don't emit final state in SourceStateIterator there is an underlying stream failure.                                                                           |

diff --git a/...re-blob-storage-destinations/build.gradle → ...rbyte-cdk/azure-destinations/build.gradle b/...re-blob-storage-destinations/build.gradle → ...rbyte-cdk/azure-destinations/build.gradle
diff --git a/...bc/copy/azure/AzureBlobStorageConfig.java → ...bc/copy/azure/AzureBlobStorageConfig.java b/...bc/copy/azure/AzureBlobStorageConfig.java → ...bc/copy/azure/AzureBlobStorageConfig.java
diff --git a/...y/azure/AzureBlobStorageStreamCopier.java → ...y/azure/AzureBlobStorageStreamCopier.java b/...y/azure/AzureBlobStorageStreamCopier.java → ...y/azure/AzureBlobStorageStreamCopier.java
diff --git a/.../AzureBlobStorageStreamCopierFactory.java → .../AzureBlobStorageStreamCopierFactory.java b/.../AzureBlobStorageStreamCopierFactory.java → .../AzureBlobStorageStreamCopierFactory.java
diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.20.7
+version=0.20.8
diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle
@@ -1,6 +1,20 @@
+java {
+    // TODO: rewrite code to avoid javac wornings in the first place
+    compileJava {
+        options.compilerArgs += "-Xlint:-deprecation"
+    }
+    compileTestFixturesJava {
+        options.compilerArgs += "-Xlint:-deprecation"
+    }
+}
+
 dependencies {
     implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies')
     implementation project(':airbyte-cdk:java:airbyte-cdk:core')
+    api project(':airbyte-cdk:java:airbyte-cdk:s3-destinations')
 
     api 'com.google.cloud:google-cloud-storage:2.32.1'
+
+    testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:s3-destinations')
+    testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:s3-destinations'))
 }
diff --git a/...s/destination/gcs/BaseGcsDestination.java → ...s/destination/gcs/BaseGcsDestination.java b/...s/destination/gcs/BaseGcsDestination.java → ...s/destination/gcs/BaseGcsDestination.java
diff --git a/...destination/gcs/GcsDestinationConfig.java → ...destination/gcs/GcsDestinationConfig.java b/...destination/gcs/GcsDestinationConfig.java → ...destination/gcs/GcsDestinationConfig.java
diff --git a/...s/destination/gcs/GcsNameTransformer.java → ...s/destination/gcs/GcsNameTransformer.java b/...s/destination/gcs/GcsNameTransformer.java → ...s/destination/gcs/GcsNameTransformer.java
diff --git a/...destination/gcs/GcsStorageOperations.java → ...destination/gcs/GcsStorageOperations.java b/...destination/gcs/GcsStorageOperations.java → ...destination/gcs/GcsStorageOperations.java
diff --git a/...s/destination/gcs/avro/GcsAvroWriter.java → ...s/destination/gcs/avro/GcsAvroWriter.java b/...s/destination/gcs/avro/GcsAvroWriter.java → ...s/destination/gcs/avro/GcsAvroWriter.java
diff --git a/...n/gcs/credential/GcsCredentialConfig.java → ...n/gcs/credential/GcsCredentialConfig.java b/...n/gcs/credential/GcsCredentialConfig.java → ...n/gcs/credential/GcsCredentialConfig.java
diff --git a/.../gcs/credential/GcsCredentialConfigs.java → .../gcs/credential/GcsCredentialConfigs.java b/.../gcs/credential/GcsCredentialConfigs.java → .../gcs/credential/GcsCredentialConfigs.java
diff --git a/...ion/gcs/credential/GcsCredentialType.java → ...ion/gcs/credential/GcsCredentialType.java b/...ion/gcs/credential/GcsCredentialType.java → ...ion/gcs/credential/GcsCredentialType.java
diff --git a/...redential/GcsHmacKeyCredentialConfig.java → ...redential/GcsHmacKeyCredentialConfig.java b/...redential/GcsHmacKeyCredentialConfig.java → ...redential/GcsHmacKeyCredentialConfig.java
diff --git a/...ons/destination/gcs/csv/GcsCsvWriter.java → ...ons/destination/gcs/csv/GcsCsvWriter.java b/...ons/destination/gcs/csv/GcsCsvWriter.java → ...ons/destination/gcs/csv/GcsCsvWriter.java
diff --git a/...destination/gcs/jsonl/GcsJsonlWriter.java → ...destination/gcs/jsonl/GcsJsonlWriter.java b/...destination/gcs/jsonl/GcsJsonlWriter.java → ...destination/gcs/jsonl/GcsJsonlWriter.java
diff --git a/...ination/gcs/parquet/GcsParquetWriter.java → ...ination/gcs/parquet/GcsParquetWriter.java b/...ination/gcs/parquet/GcsParquetWriter.java → ...ination/gcs/parquet/GcsParquetWriter.java
diff --git a/...destination/gcs/util/GcsS3FileSystem.java → ...destination/gcs/util/GcsS3FileSystem.java b/...destination/gcs/util/GcsS3FileSystem.java → ...destination/gcs/util/GcsS3FileSystem.java
diff --git a/...ations/destination/gcs/util/GcsUtils.java → ...ations/destination/gcs/util/GcsUtils.java b/...ations/destination/gcs/util/GcsUtils.java → ...ations/destination/gcs/util/GcsUtils.java
diff --git a/...destination/gcs/writer/BaseGcsWriter.java → ...destination/gcs/writer/BaseGcsWriter.java b/...destination/gcs/writer/BaseGcsWriter.java → ...destination/gcs/writer/BaseGcsWriter.java
diff --git a/...ination/gcs/GcsDestinationConfigTest.java → ...ination/gcs/GcsDestinationConfigTest.java b/...ination/gcs/GcsDestinationConfigTest.java → ...ination/gcs/GcsDestinationConfigTest.java
diff --git a/...ion/gcs/avro/GcsAvroFormatConfigTest.java → ...ion/gcs/avro/GcsAvroFormatConfigTest.java b/...ion/gcs/avro/GcsAvroFormatConfigTest.java → ...ion/gcs/avro/GcsAvroFormatConfigTest.java
diff --git a/...stination/gcs/avro/GcsAvroWriterTest.java → ...stination/gcs/avro/GcsAvroWriterTest.java b/...stination/gcs/avro/GcsAvroWriterTest.java → ...stination/gcs/avro/GcsAvroWriterTest.java
diff --git a/...ation/gcs/csv/GcsCsvFormatConfigTest.java → ...ation/gcs/csv/GcsCsvFormatConfigTest.java b/...ation/gcs/csv/GcsCsvFormatConfigTest.java → ...ation/gcs/csv/GcsCsvFormatConfigTest.java
diff --git a/...n/gcs/jsonl/GcsJsonlFormatConfigTest.java → ...n/gcs/jsonl/GcsJsonlFormatConfigTest.java b/...n/gcs/jsonl/GcsJsonlFormatConfigTest.java → ...n/gcs/jsonl/GcsJsonlFormatConfigTest.java
diff --git a/...destination/gcs/util/ConfigTestUtils.java → ...destination/gcs/util/ConfigTestUtils.java b/...destination/gcs/util/ConfigTestUtils.java → ...destination/gcs/util/ConfigTestUtils.java
diff --git a/...tions/src/test/resources/test_config.json → ...tions/src/test/resources/test_config.json b/...tions/src/test/resources/test_config.json → ...tions/src/test/resources/test_config.json
diff --git a/...AvroParquetDestinationAcceptanceTest.java → ...AvroParquetDestinationAcceptanceTest.java b/...AvroParquetDestinationAcceptanceTest.java → ...AvroParquetDestinationAcceptanceTest.java
diff --git a/...nation/gcs/GcsAvroTestDataComparator.java → ...nation/gcs/GcsAvroTestDataComparator.java b/...nation/gcs/GcsAvroTestDataComparator.java → ...nation/gcs/GcsAvroTestDataComparator.java
diff --git a/...GcsBaseAvroDestinationAcceptanceTest.java → ...GcsBaseAvroDestinationAcceptanceTest.java b/...GcsBaseAvroDestinationAcceptanceTest.java → ...GcsBaseAvroDestinationAcceptanceTest.java
diff --git a/.../GcsBaseCsvDestinationAcceptanceTest.java → .../GcsBaseCsvDestinationAcceptanceTest.java b/.../GcsBaseCsvDestinationAcceptanceTest.java → .../GcsBaseCsvDestinationAcceptanceTest.java
diff --git a/...BaseCsvGzipDestinationAcceptanceTest.java → ...BaseCsvGzipDestinationAcceptanceTest.java b/...BaseCsvGzipDestinationAcceptanceTest.java → ...BaseCsvGzipDestinationAcceptanceTest.java
diff --git a/...csBaseJsonlDestinationAcceptanceTest.java → ...csBaseJsonlDestinationAcceptanceTest.java b/...csBaseJsonlDestinationAcceptanceTest.java → ...csBaseJsonlDestinationAcceptanceTest.java
diff --git a/...seJsonlGzipDestinationAcceptanceTest.java → ...seJsonlGzipDestinationAcceptanceTest.java b/...seJsonlGzipDestinationAcceptanceTest.java → ...seJsonlGzipDestinationAcceptanceTest.java
diff --git a/...BaseParquetDestinationAcceptanceTest.java → ...BaseParquetDestinationAcceptanceTest.java b/...BaseParquetDestinationAcceptanceTest.java → ...BaseParquetDestinationAcceptanceTest.java
diff --git a/...ion/gcs/GcsDestinationAcceptanceTest.java → ...ion/gcs/GcsDestinationAcceptanceTest.java b/...ion/gcs/GcsDestinationAcceptanceTest.java → ...ion/gcs/GcsDestinationAcceptanceTest.java
diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle
@@ -12,32 +12,32 @@ java {
 }
 
 dependencies {
-    api 'com.amazonaws:aws-java-sdk-s3:1.12.647'
-
     implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies')
     implementation project(':airbyte-cdk:java:airbyte-cdk:core')
     implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')
     implementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations')
 
-    implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'}
-    implementation 'com.github.alexmojaki:s3-stream-upload:2.2.4'
-    implementation 'org.apache.avro:avro:1.11.3'
-    implementation 'org.apache.commons:commons-csv:1.10.0'
-    implementation 'org.apache.commons:commons-text:1.11.0'
-    implementation ('org.apache.hadoop:hadoop-aws:3.3.6') { exclude group: 'com.amazonaws', module: 'aws-java-sdk-bundle' }
-    implementation 'org.apache.hadoop:hadoop-common:3.3.6'
-    implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6'
-    implementation 'org.apache.parquet:parquet-avro:1.13.1'
+    // Re-export dependencies for gcs-destinations.
+    api 'com.amazonaws:aws-java-sdk-s3:1.12.647'
+    api ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'}
+    api 'com.github.alexmojaki:s3-stream-upload:2.2.4'
+    api 'org.apache.avro:avro:1.11.3'
+    api 'org.apache.commons:commons-csv:1.10.0'
+    api 'org.apache.commons:commons-text:1.11.0'
+    api ('org.apache.hadoop:hadoop-aws:3.3.6') { exclude group: 'com.amazonaws', module: 'aws-java-sdk-bundle' }
+    api 'org.apache.hadoop:hadoop-common:3.3.6'
+    api 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6'
+    api 'org.apache.parquet:parquet-avro:1.13.1'
     runtimeOnly 'com.hadoop.gplcompression:hadoop-lzo:0.4.20'
 
     testImplementation 'org.mockito:mockito-inline:5.2.0'
 
-    testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies')
-    testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies'))
-    testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:core')
-    testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core'))
-    testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')
-    testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping'))
-    testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations')
-    testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-destinations'))
+    testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:dependencies')
+    testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies'))
+    testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:core')
+    testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core'))
+    testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')
+    testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping'))
+    testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:db-destinations')
+    testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-destinations'))
 }
diff --git a/airbyte-cdk/java/airbyte-cdk/settings.gradle b/airbyte-cdk/java/airbyte-cdk/settings.gradle
@@ -6,7 +6,7 @@ include ':airbyte-cdk:java:airbyte-cdk:db-sources'
 include ':airbyte-cdk:java:airbyte-cdk:db-destinations'
 include ':airbyte-cdk:java:airbyte-cdk:s3-destinations'
 include ':airbyte-cdk:java:airbyte-cdk:typing-deduping'
-include ':airbyte-cdk:java:airbyte-cdk:azure-blob-storage-destinations'
+include ':airbyte-cdk:java:airbyte-cdk:azure-destinations'
 include ':airbyte-cdk:java:airbyte-cdk:gcs-destinations'
 include ':airbyte-cdk:java:airbyte-cdk:datastore-bigquery'
 include ':airbyte-cdk:java:airbyte-cdk:datastore-mongo'

diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle
@@ -3,13 +3,12 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.20.2'
+    cdkVersionRequired = '0.20.8'
     features = [
-            'datastore-bigquery',
             'db-destinations',
-            'gcs-destinations',
-            's3-destinations',
+            'datastore-bigquery',
             'typing-deduping',
+            'gcs-destinations',
     ]
     useLocalCdk = false
 }

diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
-  dockerImageTag: 2.4.8
+  dockerImageTag: 2.4.9
   dockerRepository: airbyte/destination-bigquery
   documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery
   githubIssueLabel: destination-bigquery

diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle
@@ -3,8 +3,8 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.19.0'
-    features = ['db-destinations', 's3-destinations', 'gcs-destinations']
+    cdkVersionRequired = '0.20.8'
+    features = ['db-destinations', 'gcs-destinations']
     useLocalCdk = false
 }
 

diff --git a/airbyte-integrations/connectors/destination-gcs/metadata.yaml b/airbyte-integrations/connectors/destination-gcs/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: ca8f6566-e555-4b40-943a-545bf123117a
-  dockerImageTag: 0.4.5
+  dockerImageTag: 0.4.6
   dockerRepository: airbyte/destination-gcs
   githubIssueLabel: destination-gcs
   icon: googlecloudstorage.svg

diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md
@@ -210,6 +210,7 @@ tutorials:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                         |
 |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 2.4.9 | 2024-02-15 | [35285](https://github.com/airbytehq/airbyte/pull/35285) | Adopt CDK 0.20.8 |
 | 2.4.8   | 2024-02-12 | [35144](https://github.com/airbytehq/airbyte/pull/35144) | Adopt CDK 0.20.2                                                                                                                                                |
 | 2.4.7   | 2024-02-12 | [35111](https://github.com/airbytehq/airbyte/pull/35111) | Adopt CDK 0.20.1                                                                                                                                                |
 | 2.4.6   | 2024-02-09 | [34575](https://github.com/airbytehq/airbyte/pull/34575) | Adopt CDK 0.20.0                                                                                                                                                |

diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md
@@ -237,6 +237,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                    |
 | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- |
+| 0.4.6 | 2024-02-15 | [35285](https://github.com/airbytehq/airbyte/pull/35285) | Adopt CDK 0.20.8 |
 | 0.4.5 | 2024-02-08 | [34745](https://github.com/airbytehq/airbyte/pull/34745) | Adopt CDK 0.19.0 |
 | 0.4.4   | 2023-07-14 | [#28345](https://github.com/airbytehq/airbyte/pull/28345)  | Increment patch to trigger a rebuild                                                                                       |
 | 0.4.3   | 2023-07-05 | [#27936](https://github.com/airbytehq/airbyte/pull/27936)  | Internal code update                                                                                                       |

diff --git a/settings.gradle b/settings.gradle
@@ -169,15 +169,12 @@ include ':airbyte-cdk:java:airbyte-cdk:db-destinations'
 include ':airbyte-cdk:java:airbyte-cdk:db-sources'
 include ':airbyte-cdk:java:airbyte-cdk:s3-destinations'
 include ':airbyte-cdk:java:airbyte-cdk:typing-deduping'
-include ':airbyte-cdk:java:airbyte-cdk:azure-blob-storage-destinations'
+include ':airbyte-cdk:java:airbyte-cdk:azure-destinations'
 include ':airbyte-cdk:java:airbyte-cdk:gcs-destinations'
 include ':airbyte-cdk:java:airbyte-cdk:datastore-bigquery'
 include ':airbyte-cdk:java:airbyte-cdk:datastore-mongo'
 include ':airbyte-cdk:java:airbyte-cdk:datastore-postgres'
 
-include ':airbyte-integrations:bases:base'
-include ':airbyte-integrations:bases:base-java'
-include ':airbyte-integrations:bases:base-normalization'
 include ':airbyte-integrations:connector-templates:generator'
 include ':airbyte-integrations:connectors-performance:source-harness'
 include ':airbyte-integrations:connectors-performance:destination-harness'