Fix NullBlobStore multipart upload

Complete multipart upload lists the parts to get their sizes but the NullBlobStore reported 8 bytes for the physical size instead of the logical size embedded with the part. There is no readable space to store the logical size so we create a companion single-part object for every part.
gaul · Feb 7, 2019 · 8223f44 · 8223f44
1 parent c07adfc
commit 8223f44
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 8 deletions.
diff --git a/src/main/java/org/gaul/s3proxy/NullBlobStore.java b/src/main/java/org/gaul/s3proxy/NullBlobStore.java
@@ -93,6 +93,7 @@ public Blob getBlob(String container, String name, GetOptions options) {
         payload.getContentMetadata().setContentLength(length);
         payload.getContentMetadata().setContentMD5((HashCode) null);
         blob.setPayload(payload);
+        blob.getMetadata().setSize(length);
         return blob;
     }
 
@@ -140,17 +141,33 @@ public String completeMultipartUpload(final MultipartUpload mpu,
         long length = 0;
         for (MultipartPart part : parts) {
             length += part.partSize();
+            super.removeBlob(mpu.containerName(), mpu.id() + "-" +
+                    part.partNumber());
         }
 
         byte[] array = Longs.toByteArray(length);
         ByteSourcePayload payload = new ByteSourcePayload(
                 ByteSource.wrap(array));
+        payload.getContentMetadata().setContentLength((long) array.length);
 
         super.abortMultipartUpload(mpu);
 
-        MultipartPart part = delegate().uploadMultipartPart(mpu, 1, payload);
+        MultipartUpload mpu2 = super.initiateMultipartUpload(
+                mpu.containerName(), mpu.blobMetadata(), mpu.putOptions());
+
+        MultipartPart part = super.uploadMultipartPart(mpu2, 1, payload);
+
+        return super.completeMultipartUpload(mpu2, ImmutableList.of(part));
+    }
+
+    @Override
+    public void abortMultipartUpload(MultipartUpload mpu) {
+        for (MultipartPart part : super.listMultipartUpload(mpu)) {
+            super.removeBlob(mpu.containerName(), mpu.id() + "-" +
+                    part.partNumber());
+        }
 
-        return delegate().completeMultipartUpload(mpu, ImmutableList.of(part));
+        super.abortMultipartUpload(mpu);
     }
 
     @Override
@@ -170,18 +187,29 @@ public MultipartPart uploadMultipartPart(MultipartUpload mpu,
         newPayload.getContentMetadata().setContentLength((long) array.length);
         newPayload.getContentMetadata().setContentMD5((HashCode) null);
 
+        // create a single-part object which contains the logical length which
+        // list and complete will read later
+        Blob blob = blobBuilder(mpu.id() + "-" + partNumber)
+                .payload(newPayload)
+                .build();
+        super.putBlob(mpu.containerName(), blob);
+
         MultipartPart part = super.uploadMultipartPart(mpu, partNumber,
                 newPayload);
         return MultipartPart.create(part.partNumber(), length, part.partETag(),
                 part.lastModified());
     }
 
-    // Cannot read parts to get the embedded size so return zero instead.
     @Override
     public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) {
         ImmutableList.Builder<MultipartPart> builder = ImmutableList.builder();
         for (MultipartPart part : super.listMultipartUpload(mpu)) {
-            builder.add(MultipartPart.create(part.partNumber(), 0,
+            // get real blob size from stub blob
+            Blob blob = getBlob(mpu.containerName(),
+                    mpu.id() + "-" + part.partNumber());
+            long length = blob.getPayload().getContentMetadata()
+                    .getContentLength();
+            builder.add(MultipartPart.create(part.partNumber(), length,
                     part.partETag(), part.lastModified()));
         }
         return builder.build();

diff --git a/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java b/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java
@@ -141,14 +141,15 @@ public void testCreateMultipartBlobGetBlob() throws Exception {
 
         List<MultipartPart> parts = nullBlobStore.listMultipartUpload(mpu);
         assertThat(parts.get(0).partNumber()).isEqualTo(1);
-        assertThat(parts.get(0).partSize()).isZero();
+        assertThat(parts.get(0).partSize()).isEqualTo(byteSource1.size());
         assertThat(parts.get(0).partETag()).isEqualTo(part1.partETag());
         assertThat(parts.get(1).partNumber()).isEqualTo(2);
-        assertThat(parts.get(1).partSize()).isZero();
+        assertThat(parts.get(1).partSize()).isEqualTo(byteSource2.size());
         assertThat(parts.get(1).partETag()).isEqualTo(part2.partETag());
 
-        nullBlobStore.completeMultipartUpload(mpu, ImmutableList.of(part1,
-                part2));
+        assertThat(nullBlobStore.listMultipartUpload(mpu)).hasSize(2);
+
+        nullBlobStore.completeMultipartUpload(mpu, parts);
 
         Blob newBlob = nullBlobStore.getBlob(containerName, blobName);
         validateBlobMetadata(newBlob.getMetadata());
@@ -162,6 +163,9 @@ public void testCreateMultipartBlobGetBlob() throws Exception {
                     ByteStreams.nullOutputStream());
             assertThat(actualLength).isEqualTo(expectedLength);
         }
+
+        nullBlobStore.removeBlob(containerName, blobName);
+        assertThat(nullBlobStore.list(containerName)).isEmpty();
     }
 
     private static String createRandomContainerName() {