Move append to Upload feature.

iterate-ch · Apr 29, 2024 · c1e1020 · c1e1020
1 parent eff8910
commit c1e1020
Show file tree

Hide file tree

Showing 84 changed files with 595 additions and 423 deletions.
diff --git a/azure/src/main/java/ch/cyberduck/core/azure/AzureSession.java b/azure/src/main/java/ch/cyberduck/core/azure/AzureSession.java
@@ -44,6 +44,7 @@
 import ch.cyberduck.core.features.Move;
 import ch.cyberduck.core.features.Read;
 import ch.cyberduck.core.features.Touch;
+import ch.cyberduck.core.features.Upload;
 import ch.cyberduck.core.features.Write;
 import ch.cyberduck.core.http.DisabledX509HostnameVerifier;
 import ch.cyberduck.core.proxy.Proxy;
@@ -195,6 +196,9 @@ public <T> T _getFeature(final Class<T> type) {
         if(type == Read.class) {
             return (T) new AzureReadFeature(this, context);
         }
+        if(type == Upload.class) {
+            return (T) new AzureUploadFeature(this, context);
+        }
         if(type == Write.class) {
             return (T) new AzureWriteFeature(this, context);
         }

diff --git a/azure/src/main/java/ch/cyberduck/core/azure/AzureUploadFeature.java b/azure/src/main/java/ch/cyberduck/core/azure/AzureUploadFeature.java
@@ -0,0 +1,50 @@
+package ch.cyberduck.core.azure;
+
+/*
+ * Copyright (c) 2002-2024 iterate GmbH. All rights reserved.
+ * https://cyberduck.io/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+import ch.cyberduck.core.Path;
+import ch.cyberduck.core.PathAttributes;
+import ch.cyberduck.core.exception.BackgroundException;
+import ch.cyberduck.core.features.Write;
+import ch.cyberduck.core.shared.DefaultUploadFeature;
+import ch.cyberduck.core.transfer.TransferStatus;
+
+import com.microsoft.azure.storage.OperationContext;
+import com.microsoft.azure.storage.blob.BlobType;
+
+public class AzureUploadFeature extends DefaultUploadFeature {
+
+    private final AzureSession session;
+    private final OperationContext context;
+
+    public AzureUploadFeature(final AzureSession session, final OperationContext context) {
+        super(new AzureWriteFeature(session, context));
+        this.session = session;
+        this.context = context;
+    }
+
+    @Override
+    public Write.Append append(final Path file, final TransferStatus status) throws BackgroundException {
+        final Write.Append append = new Write.Append(status.isExists()).withStatus(status);
+        if(append.append) {
+            final PathAttributes attr = new AzureAttributesFinderFeature(session, context).find(file);
+            if(BlobType.APPEND_BLOB == BlobType.valueOf(attr.getCustom().get(AzureAttributesFinderFeature.KEY_BLOB_TYPE))) {
+                return append;
+            }
+        }
+        return Write.override;
+    }
+}
diff --git a/azure/src/main/java/ch/cyberduck/core/azure/AzureWriteFeature.java b/azure/src/main/java/ch/cyberduck/core/azure/AzureWriteFeature.java
@@ -33,7 +33,6 @@
 import ch.cyberduck.core.io.StatusOutputStream;
 import ch.cyberduck.core.io.VoidStatusOutputStream;
 import ch.cyberduck.core.preferences.HostPreferences;
-import ch.cyberduck.core.shared.AppendWriteFeature;
 import ch.cyberduck.core.transfer.TransferStatus;
 
 import org.apache.commons.lang3.StringUtils;
@@ -57,13 +56,13 @@
 import com.microsoft.azure.storage.blob.CloudBlockBlob;
 import com.microsoft.azure.storage.core.SR;
 
-public class AzureWriteFeature extends AppendWriteFeature<Void> implements Write<Void> {
+public class AzureWriteFeature implements Write<Void> {
     private static final Logger log = LogManager.getLogger(AzureWriteFeature.class);
 
     private final AzureSession session;
     private final OperationContext context;
     private final PathContainerService containerService
-        = new DirectoryDelimiterPathContainerService();
+            = new DirectoryDelimiterPathContainerService();
     private final BlobType blobType;
 
     public AzureWriteFeature(final AzureSession session, final OperationContext context) {
@@ -81,42 +80,30 @@ public ChecksumCompute checksum(final Path file, final TransferStatus status) {
         return ChecksumComputeFactory.get(HashAlgorithm.md5);
     }
 
-    @Override
-    public Append append(final Path file, final TransferStatus status) throws BackgroundException {
-        final Append append = super.append(file, status);
-        if(append.append) {
-            final PathAttributes attr = new AzureAttributesFinderFeature(session, context).find(file);
-            if(BlobType.APPEND_BLOB == BlobType.valueOf(attr.getCustom().get(AzureAttributesFinderFeature.KEY_BLOB_TYPE))) {
-                return append;
-            }
-        }
-        return Write.override;
-    }
-
     @Override
     public StatusOutputStream<Void> write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
         try {
             final CloudBlob blob;
             if(status.isExists()) {
                 if(new HostPreferences(session.getHost()).getBoolean("azure.upload.snapshot")) {
                     session.getClient().getContainerReference(containerService.getContainer(file).getName())
-                        .getBlobReferenceFromServer(containerService.getKey(file)).createSnapshot();
+                            .getBlobReferenceFromServer(containerService.getKey(file)).createSnapshot();
                 }
                 if(status.isAppend()) {
                     // Existing append blob type
                     blob = session.getClient().getContainerReference(containerService.getContainer(file).getName())
-                        .getAppendBlobReference(containerService.getKey(file));
+                            .getAppendBlobReference(containerService.getKey(file));
                 }
                 else {
                     // Existing block blob type
                     final PathAttributes attr = new AzureAttributesFinderFeature(session, context).find(file);
                     if(BlobType.APPEND_BLOB == BlobType.valueOf(attr.getCustom().get(AzureAttributesFinderFeature.KEY_BLOB_TYPE))) {
                         blob = session.getClient().getContainerReference(containerService.getContainer(file).getName())
-                            .getAppendBlobReference(containerService.getKey(file));
+                                .getAppendBlobReference(containerService.getKey(file));
                     }
                     else {
                         blob = session.getClient().getContainerReference(containerService.getContainer(file).getName())
-                            .getBlockBlobReference(containerService.getKey(file));
+                                .getBlockBlobReference(containerService.getKey(file));
                     }
                 }
             }
@@ -125,11 +112,11 @@ public StatusOutputStream<Void> write(final Path file, final TransferStatus stat
                 switch(blobType) {
                     case APPEND_BLOB:
                         blob = session.getClient().getContainerReference(containerService.getContainer(file).getName())
-                            .getAppendBlobReference(containerService.getKey(file));
+                                .getAppendBlobReference(containerService.getKey(file));
                         break;
                     default:
                         blob = session.getClient().getContainerReference(containerService.getContainer(file).getName())
-                            .getBlockBlobReference(containerService.getKey(file));
+                                .getBlockBlobReference(containerService.getKey(file));
                 }
             }
             if(StringUtils.isNotBlank(status.getMime())) {

diff --git a/azure/src/test/java/ch/cyberduck/core/azure/AzureWriteFeatureTest.java b/azure/src/test/java/ch/cyberduck/core/azure/AzureWriteFeatureTest.java
@@ -6,7 +6,6 @@
 import ch.cyberduck.core.Path;
 import ch.cyberduck.core.PathAttributes;
 import ch.cyberduck.core.features.Delete;
-import ch.cyberduck.core.features.Write;
 import ch.cyberduck.core.io.MD5ChecksumCompute;
 import ch.cyberduck.core.io.StreamCopier;
 import ch.cyberduck.core.transfer.TransferStatus;
@@ -54,7 +53,6 @@ public void testWriteOverrideAppendBlob() throws Exception {
         final Map<String, String> metadata = new AzureMetadataFeature(session, context).getMetadata(test);
         assertEquals("text/plain", metadata.get("Content-Type"));
         assertEquals("public,max-age=86400", metadata.get("Cache-Control"));
-        assertEquals(content.length, new AzureWriteFeature(session, context).append(test, status.exists(true).withRemote(attributes)).size, 0L);
         final byte[] buffer = new byte[content.length];
         final InputStream in = new AzureReadFeature(session, context).read(test, new TransferStatus(), new DisabledConnectionCallback());
         IOUtils.readFully(in, buffer);
@@ -92,9 +90,6 @@ public void testWriteOverrideBlockBlob() throws Exception {
         final Map<String, String> metadata = new AzureMetadataFeature(session, context).getMetadata(test);
         assertEquals("text/plain", metadata.get("Content-Type"));
         assertEquals("public,max-age=86400", metadata.get("Cache-Control"));
-        final Write.Append append = new AzureWriteFeature(session, context).append(test, status.withRemote(attributes));
-        assertFalse(append.append);
-        assertEquals(0L, append.size, 0L);
         final byte[] buffer = new byte[content.length];
         final InputStream in = new AzureReadFeature(session, context).read(test, new TransferStatus(), new DisabledConnectionCallback());
         IOUtils.readFully(in, buffer);

diff --git a/azure/src/test/java/ch/cyberduck/core/cryptomator/AzureWriteFeatureTest.java b/azure/src/test/java/ch/cyberduck/core/cryptomator/AzureWriteFeatureTest.java
@@ -87,7 +87,6 @@ public void testWrite() throws Exception {
         assertTrue(cryptomator.getFeature(session, Find.class, new AzureFindFeature(session, null)).find(test));
         final PathAttributes attributes = new CryptoListService(session, new AzureListService(session, context), cryptomator).list(test.getParent(), new DisabledListProgressListener()).get(test).attributes();
         assertEquals(content.length, attributes.getSize());
-        assertEquals(content.length, new CryptoWriteFeature<>(session, new AzureWriteFeature(session, context), cryptomator).append(test, status.withRemote(attributes)).size, 0L);
         final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
         final InputStream in = new CryptoReadFeature(session, new AzureReadFeature(session, context), cryptomator).read(test, new TransferStatus().withLength(content.length), new DisabledConnectionCallback());
         new StreamCopier(status, status).transfer(in, buffer);

diff --git a/backblaze/src/main/java/ch/cyberduck/core/b2/B2AttributesFinderFeature.java b/backblaze/src/main/java/ch/cyberduck/core/b2/B2AttributesFinderFeature.java
@@ -68,9 +68,9 @@ public PathAttributes find(final Path file, final ListProgressListener listener)
         }
         if(file.getType().contains(Path.Type.upload)) {
             // Pending large file upload
-            final Write.Append append = new B2WriteFeature(session, fileid).append(file, new TransferStatus());
+            final Write.Append append = new B2LargeUploadService(session, fileid, new B2WriteFeature(session, fileid)).append(file, new TransferStatus());
             if(append.append) {
-                return new PathAttributes().withSize(append.size);
+                return new PathAttributes().withSize(append.offset);
             }
             return PathAttributes.EMPTY;
         }

diff --git a/backblaze/src/main/java/ch/cyberduck/core/b2/B2LargeUploadService.java b/backblaze/src/main/java/ch/cyberduck/core/b2/B2LargeUploadService.java
@@ -242,6 +242,20 @@ public B2UploadPartResponse call() throws BackgroundException {
         }, overall, counter));
     }
 
+    @Override
+    public Write.Append append(final Path file, final TransferStatus status) throws BackgroundException {
+        final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid);
+        final List<B2FileInfoResponse> upload = partService.find(file);
+        if(!upload.isEmpty()) {
+            Long size = 0L;
+            for(B2UploadPartResponse completed : partService.list(upload.iterator().next().getFileId())) {
+                size += completed.getContentLength();
+            }
+            return new Write.Append(true).withStatus(status).withOffset(size);
+        }
+        return new Write.Append(false).withStatus(status);
+    }
+
     @Override
     public Upload<BaseB2Response> withWriter(final Write<BaseB2Response> writer) {
         this.writer = writer;

diff --git a/backblaze/src/main/java/ch/cyberduck/core/b2/B2ThresholdCopyFeature.java b/backblaze/src/main/java/ch/cyberduck/core/b2/B2ThresholdCopyFeature.java
@@ -47,7 +47,7 @@ public B2ThresholdCopyFeature(final B2Session session, final B2VersionIdProvider
 
     @Override
     public Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback, final StreamListener listener) throws BackgroundException {
-        if(new B2ThresholdUploadService(session, fileid, threshold).threshold(status.getLength())) {
+        if(new B2ThresholdUploadService(session, fileid, threshold).threshold(status)) {
             return new B2LargeCopyFeature(session, fileid).copy(source, target, status, callback, listener);
         }
         else {

diff --git a/backblaze/src/main/java/ch/cyberduck/core/b2/B2ThresholdUploadService.java b/backblaze/src/main/java/ch/cyberduck/core/b2/B2ThresholdUploadService.java
@@ -52,13 +52,16 @@ public B2ThresholdUploadService(final B2Session session, final B2VersionIdProvid
 
     @Override
     public Write.Append append(final Path file, final TransferStatus status) throws BackgroundException {
-        return writer.append(file, status);
+        if(this.threshold(status)) {
+            return new B2LargeUploadService(session, fileid, writer).append(file, status);
+        }
+        return new Write.Append(false).withStatus(status);
     }
 
     @Override
     public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener,
                                  final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
-        if(this.threshold(status.getLength())) {
+        if(this.threshold(status)) {
             return new B2LargeUploadService(session, fileid, writer).upload(file, local, throttle, listener, status, callback);
         }
         else {
@@ -72,12 +75,12 @@ public Upload<BaseB2Response> withWriter(final Write<BaseB2Response> writer) {
         return this;
     }
 
-    protected boolean threshold(final Long length) {
-        if(length > threshold) {
-            if(length > new HostPreferences(session.getHost()).getLong("b2.upload.largeobject.size")) {
+    protected boolean threshold(final TransferStatus status) {
+        if(status.getLength() > threshold) {
+            if(status.getLength() > new HostPreferences(session.getHost()).getLong("b2.upload.largeobject.size")) {
                 if(!new HostPreferences(session.getHost()).getBoolean("b2.upload.largeobject")) {
                     // Disabled by user
-                    if(length < new HostPreferences(session.getHost()).getLong("b2.upload.largeobject.required.threshold")) {
+                    if(status.getLength() < new HostPreferences(session.getHost()).getLong("b2.upload.largeobject.required.threshold")) {
                         log.warn("Large upload is disabled with property b2.upload.largeobject.required.threshold");
                         return false;
                     }

diff --git a/backblaze/src/main/java/ch/cyberduck/core/b2/B2WriteFeature.java b/backblaze/src/main/java/ch/cyberduck/core/b2/B2WriteFeature.java
@@ -37,15 +37,12 @@
 import java.io.IOException;
 import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import synapticloop.b2.exception.B2ApiException;
-import synapticloop.b2.response.B2FileInfoResponse;
 import synapticloop.b2.response.B2FileResponse;
 import synapticloop.b2.response.B2GetUploadPartUrlResponse;
 import synapticloop.b2.response.B2GetUploadUrlResponse;
-import synapticloop.b2.response.B2UploadPartResponse;
 import synapticloop.b2.response.BaseB2Response;
 
 import static ch.cyberduck.core.b2.B2MetadataFeature.X_BZ_INFO_SRC_CREATION_DATE_MILLIS;
@@ -148,20 +145,6 @@ public ChecksumCompute checksum(final Path file, final TransferStatus status) {
         return ChecksumComputeFactory.get(HashAlgorithm.sha1);
     }
 
-    @Override
-    public Append append(final Path file, final TransferStatus status) throws BackgroundException {
-        final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid);
-        final List<B2FileInfoResponse> upload = partService.find(file);
-        if(!upload.isEmpty()) {
-            Long size = 0L;
-            for(B2UploadPartResponse completed : partService.list(upload.iterator().next().getFileId())) {
-                size += completed.getContentLength();
-            }
-            return new Append(true).withStatus(status).withSize(size);
-        }
-        return new Append(false).withStatus(status);
-    }
-
     @Override
     public EnumSet<Flags> features(final Path file) {
         return EnumSet.of(Flags.timestamp);

diff --git a/backblaze/src/test/java/ch/cyberduck/core/b2/B2LargeUploadServiceTest.java b/backblaze/src/test/java/ch/cyberduck/core/b2/B2LargeUploadServiceTest.java
@@ -139,7 +139,7 @@ public BaseB2Response upload(final Path file, final Local local, final Bandwidth
         assertEquals(TransferStatus.UNKNOWN_LENGTH, status.getResponse().getSize());
         final Write.Append resume = service.append(test, status);
         assertTrue(resume.append);
-        assertEquals(0L, resume.size, 0L);
+        assertEquals(0L, resume.offset, 0L);
         final TransferStatus append = new TransferStatus().append(true).withLength(content.length);
         service.upload(test, local,
                 new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), append,
@@ -196,7 +196,7 @@ public BaseB2Response upload(final Path file, final Local local, final Bandwidth
         assertEquals(TransferStatus.UNKNOWN_LENGTH, status.getResponse().getSize());
         final Write.Append appendStatus = feature.append(test, status);
         assertTrue(appendStatus.append);
-        assertEquals(5 * 1000L * 1000L, appendStatus.size, 0L);
+        assertEquals(5 * 1000L * 1000L, appendStatus.offset, 0L);
         final Path upload = new Path(test).withType(EnumSet.of(Path.Type.file, Path.Type.upload));
         assertTrue(new B2FindFeature(session, fileid).find(upload));
         assertEquals(5 * 1000L * 1000L, new B2AttributesFinderFeature(session, fileid).find(upload).getSize(), 0L);

diff --git a/backblaze/src/test/java/ch/cyberduck/core/b2/B2WriteFeatureTest.java b/backblaze/src/test/java/ch/cyberduck/core/b2/B2WriteFeatureTest.java
@@ -22,7 +22,6 @@
 import ch.cyberduck.core.PathAttributes;
 import ch.cyberduck.core.exception.ChecksumException;
 import ch.cyberduck.core.features.Delete;
-import ch.cyberduck.core.features.Write;
 import ch.cyberduck.core.http.HttpResponseOutputStream;
 import ch.cyberduck.core.io.Checksum;
 import ch.cyberduck.core.io.SHA1ChecksumCompute;
@@ -94,9 +93,6 @@ public void testWrite() throws Exception {
         assertEquals(content.length, attributes.getSize());
         assertEquals(new B2AttributesFinderFeature(session, fileid).toAttributes(response), attributes);
         assertEquals(bucketAttr, new B2AttributesFinderFeature(session, fileid).find(bucket));
-        final Write.Append append = new B2WriteFeature(session, fileid).append(test, status.withRemote(attributes));
-        assertFalse(append.append);
-        assertEquals(content.length, append.size, 0L);
         final byte[] buffer = new byte[content.length];
         final InputStream in = new B2ReadFeature(session, fileid).read(test, new TransferStatus(), new DisabledConnectionCallback());
         IOUtils.readFully(in, buffer);

diff --git a/backblaze/src/test/java/ch/cyberduck/core/cryptomator/B2WriteFeatureTest.java b/backblaze/src/test/java/ch/cyberduck/core/cryptomator/B2WriteFeatureTest.java
@@ -88,7 +88,6 @@ public void testWrite() throws Exception {
         assertTrue(cryptomator.getFeature(session, Find.class, new B2FindFeature(session, fileid)).find(test));
         final PathAttributes attributes = cryptomator.getFeature(session, AttributesFinder.class, new B2AttributesFinderFeature(session, fileid)).find(test);
         assertEquals(content.length, attributes.getSize());
-        assertEquals(content.length, writer.append(test, status.withRemote(attributes)).size, 0L);
         final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
         final InputStream in = new CryptoReadFeature(session, new B2ReadFeature(session, fileid), cryptomator).read(test, new TransferStatus().withLength(content.length), new DisabledConnectionCallback());
         new StreamCopier(status, status).transfer(in, buffer);

diff --git a/box/src/main/java/ch/cyberduck/core/box/BoxThresholdUploadService.java b/box/src/main/java/ch/cyberduck/core/box/BoxThresholdUploadService.java
@@ -56,11 +56,6 @@ public File upload(final Path file, final Local local, final BandwidthThrottle t
         return new BoxSmallUploadService(session, fileid, writer).upload(file, local, throttle, listener, status, callback);
     }
 
-    @Override
-    public Write.Append append(final Path file, final TransferStatus status) throws BackgroundException {
-        return new Write.Append(false).withStatus(status);
-    }
-
     @Override
     public Upload<File> withWriter(final Write<File> writer) {
         this.writer = writer;