Fix duplicate object names for concurrent uploads: Always use UUID fo… (

#29640) Co-authored-by: Evan Tahler <evan@airbyte.io>
airbytehq · Aug 19, 2023 · 9aad31c · 9aad31c
1 parent ef0a3ef
commit 9aad31c
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 14 deletions.
diff --git a/...ase-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java b/...ase-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java
@@ -156,7 +156,7 @@ public String uploadRecordsToBucket(final SerializableBuffer recordsData,
   private String loadDataIntoBucket(final String objectPath, final SerializableBuffer recordsData) throws IOException {
     final long partSize = DEFAULT_PART_SIZE;
     final String bucket = s3Config.getBucketName();
-    final String partId = getPartId(objectPath);
+    final String partId = UUID.randomUUID().toString();
     final String fileExtension = getExtension(recordsData.getFilename());
     final String fullObjectKey;
     if (StringUtils.isNotBlank(s3Config.getFileNamePattern())) {
@@ -230,17 +230,6 @@ protected static String getExtension(final String filename) {
     return "." + result;
   }
 
-  private String getPartId(final String objectPath) {
-    final String bucket = s3Config.getBucketName();
-    final ObjectListing objects = s3Client.listObjects(bucket, objectPath);
-    if (objects.isTruncated()) {
-      // bucket contains too many objects, use an uuid instead
-      return UUID.randomUUID().toString();
-    } else {
-      return Integer.toString(objects.getObjectSummaries().size());
-    }
-  }
-
   @Override
   public void dropBucketObject(final String objectPath) {
     cleanUpBucketObject(objectPath, List.of());

diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile
@@ -46,7 +46,7 @@ ENV APPLICATION destination-redshift
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.6.4
+LABEL io.airbyte.version=0.6.5
 LABEL io.airbyte.name=airbyte/destination-redshift
 
 ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"

diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc
-  dockerImageTag: 0.6.4
+  dockerImageTag: 0.6.5
   dockerRepository: airbyte/destination-redshift
   githubIssueLabel: destination-redshift
   icon: redshift.svg

diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md
@@ -156,6 +156,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                                                                          |
 |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.6.5   | 2023-08-18 | [\#28619](https://github.com/airbytehq/airbyte/pull/29640) | Fix duplicate staging object names in concurrent environment (e.g. async)                                                                                                                                        |
 | 0.6.4   | 2023-08-10 | [\#28619](https://github.com/airbytehq/airbyte/pull/28619) | Use async method for staging                                                                                                                                                                                     |
 | 0.6.3   | 2023-08-07 | [\#29188](https://github.com/airbytehq/airbyte/pull/29188) | Internal code refactoring                                                                                                                                                                                        |
 | 0.6.2   | 2023-07-24 | [\#28618](https://github.com/airbytehq/airbyte/pull/28618) | Add hooks in preparation for destinations v2 implementation                                                                                                                                                      |