JCLOUDS-1368: Correct use of slicing algorithm

apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java

@@ -19,6 +19,7 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.io.BaseEncoding.base16;
+import static java.nio.file.Files.createDirectories;
 import static java.nio.file.Files.getFileAttributeView;
 import static java.nio.file.Files.getPosixFilePermissions;
 import static java.nio.file.Files.probeContentType;
@@ -36,6 +37,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.AccessDeniedException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
@@ -75,6 +77,7 @@
 import org.jclouds.io.ContentMetadata;
 import org.jclouds.io.Payload;
 import org.jclouds.logging.Logger;
+import org.jclouds.rest.AuthorizationException;
 import org.jclouds.rest.annotations.ParamValidators;
 
 import com.google.common.base.Function;
@@ -908,8 +911,15 @@ protected boolean createDirectoryWithResult(String container, String directory)
       }
 
       File directoryToCreate = new File(directoryFullName);
-      boolean result = directoryToCreate.mkdirs();
-      return result;
+      try {
+         createDirectories(directoryToCreate.toPath());
+      } catch (AccessDeniedException ade) {
+         throw new AuthorizationException(ade);
+      } catch (IOException ioe) {
+         logger.debug("Could not create directory: %s", ioe.getMessage());
+         return false;
+      }
+      return true;
    }
 
    /** Read the String representation of filesystem attribute, or return null if not present. */

blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java

@@ -354,14 +354,24 @@ protected String putMultipartBlob(String container, Blob blob, PutOptions overri
       MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides);
       try {
          long contentLength = blob.getMetadata().getContentMetadata().getContentLength();
+         // TODO: inject MultipartUploadSlicingAlgorithm to override default part size
          MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
                getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
          long partSize = algorithm.calculateChunkSize(contentLength);
          int partNumber = 1;
-         for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
+         // TODO: for InputStream payloads, this buffers all parts in-memory!
+         while (partNumber < algorithm.getParts()) {
+            Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), partSize);
             BlobUploader b =
                   new BlobUploader(mpu, partNumber++, payload);
             parts.add(executor.submit(b));
+            algorithm.addCopied(partSize);
+         }
+         if (algorithm.getRemaining() != 0) {
+            Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), algorithm.getRemaining());
+            BlobUploader b =
+                  new BlobUploader(mpu, partNumber, payload);
+            parts.add(executor.submit(b));
          }
          return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
       } catch (RuntimeException re) {

blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java

@@ -103,14 +103,10 @@ public long calculateChunkSize(long length) {
          unitPartSize = maximumPartSize;
          parts = (int)(length / unitPartSize);
       }
-      if (parts > maximumNumberOfParts) { // if splits in too many parts or
-                                         // cannot be split
-         unitPartSize = minimumPartSize; // take the minimum part size
-         parts = (int)(length / unitPartSize);
-      }
-      if (parts > maximumNumberOfParts) { // if still splits in too many parts
-         parts = maximumNumberOfParts - 1; // limit them. do not care about not
-                                          // covering
+      if (parts > maximumNumberOfParts) {
+         partSize = length / maximumNumberOfParts;
+         unitPartSize = partSize;
+         parts = maximumNumberOfParts;
       }
       long remainder = length % unitPartSize;
       if (remainder == 0 && parts > 0) {
@@ -132,16 +128,15 @@ public void setCopied(long copied) {
       this.copied = copied;
    }
 
-   @VisibleForTesting
-   protected int getParts() {
+   public int getParts() {
       return parts;
    }
 
    protected int getNextPart() {
       return ++part;
    }
 
-   protected void addCopied(long copied) {
+   public void addCopied(long copied) {
       this.copied += copied;
    }
 
@@ -156,8 +151,7 @@ protected long getChunkSize() {
       return chunkSize;
    }
 
-   @VisibleForTesting
-   protected long getRemaining() {
+   public long getRemaining() {
       return remaining;
    }