Skip to content
Permalink
Browse files
JCLOUDS-1267: Allow B2 streaming uploads
B2 now supports uploads without calculating the SHA-1 hash.  This
allows uploading without a repeatable payload.
  • Loading branch information
gaul committed Apr 8, 2017
1 parent 95f274b commit f0297bdc20c525aa6f32ccb44f84408e82e98e0b
Show file tree
Hide file tree
Showing 6 changed files with 20 additions and 48 deletions.
@@ -33,10 +33,15 @@ public final class UploadFileBinder implements MapBinder {
public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
UploadUrlResponse uploadUrl = (UploadUrlResponse) postParams.get("uploadUrl");
String fileName = (String) postParams.get("fileName");
String contentSha1 = (String) postParams.get("contentSha1");
if (contentSha1 == null) {
contentSha1 = "do_not_verify";
}
Map<String, String> fileInfo = (Map<String, String>) postParams.get("fileInfo");
HttpRequest.Builder builder = request.toBuilder()
.endpoint(uploadUrl.uploadUrl())
.replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken())
.replaceHeader(B2Headers.CONTENT_SHA1, contentSha1)
.replaceHeader(B2Headers.FILE_NAME, escaper.escape(fileName));
for (Map.Entry<String, String> entry : fileInfo.entrySet()) {
builder.replaceHeader(B2Headers.FILE_INFO_PREFIX + entry.getKey(), escaper.escape(entry.getValue()));
@@ -20,6 +20,7 @@

import org.jclouds.http.HttpRequest;
import org.jclouds.b2.domain.GetUploadPartResponse;
import org.jclouds.b2.reference.B2Headers;
import org.jclouds.rest.MapBinder;

import com.google.common.net.HttpHeaders;
@@ -28,9 +29,14 @@ public final class UploadPartBinder implements MapBinder {
@Override
public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
GetUploadPartResponse uploadUrl = (GetUploadPartResponse) postParams.get("response");
String contentSha1 = (String) postParams.get("contentSha1");
if (contentSha1 == null) {
contentSha1 = "do_not_verify";
}
return (R) request.toBuilder()
.endpoint(uploadUrl.uploadUrl())
.replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken())
.replaceHeader(B2Headers.CONTENT_SHA1, contentSha1)
.build();
}

@@ -16,7 +16,6 @@
*/
package org.jclouds.b2.blobstore;

import java.io.IOException;
import java.net.URI;
import java.util.Date;
import java.util.List;
@@ -67,7 +66,6 @@
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.collect.Memoized;
import org.jclouds.domain.Location;
import org.jclouds.io.ByteStreams2;
import org.jclouds.io.ContentMetadata;
import org.jclouds.io.ContentMetadataBuilder;
import org.jclouds.io.MutableContentMetadata;
@@ -78,14 +76,12 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.google.common.hash.Hashing;
import com.google.common.net.HttpHeaders;
import com.google.common.util.concurrent.UncheckedExecutionException;

@@ -254,30 +250,21 @@ public String putBlob(String container, Blob blob) {

@Override
public String putBlob(String container, Blob blob, PutOptions options) {
Preconditions.checkArgument(blob.getPayload().isRepeatable(), "B2 requires repeatable payload to calculate SHA1");

if (options.getBlobAccess() != BlobAccess.PRIVATE) {
throw new UnsupportedOperationException("B2 only supports private access blobs");
}

if (options.isMultipart()) {
return putMultipartBlob(container, blob, options);
} else {
String contentSha1;
try {
contentSha1 = ByteStreams2.hashAndClose(blob.getPayload().openStream(), Hashing.sha1()).toString();
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}

String name = blob.getMetadata().getName();

// B2 versions all files so we store the original fileId to delete it after the upload succeeds
String oldFileId = getFileId(container, name);

Bucket bucket = getBucket(container);
UploadUrlResponse uploadUrl = api.getObjectApi().getUploadUrl(bucket.bucketId());
UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, contentSha1, blob.getMetadata().getUserMetadata(), blob.getPayload());
UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, null, blob.getMetadata().getUserMetadata(), blob.getPayload());

if (oldFileId != null) {
api.getObjectApi().deleteFileVersion(name, oldFileId);
@@ -397,20 +384,15 @@ public String completeMultipartUpload(MultipartUpload mpu, List<MultipartPart> p

@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
Preconditions.checkArgument(payload.isRepeatable());

String contentSha1;
try {
contentSha1 = ByteStreams2.hashAndClose(payload.openStream(), Hashing.sha1()).toString();
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}

GetUploadPartResponse getUploadPart = api.getMultipartApi().getUploadPartUrl(mpu.id());
UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, contentSha1, payload);
UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, null, payload);

Date lastModified = null; // B2 does not return Last-Modified
return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), uploadPart.contentSha1(), lastModified);
String contentSha1 = uploadPart.contentSha1();
if (contentSha1.startsWith("unverified:")) {
contentSha1 = contentSha1.substring("unverified:".length());
}
return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), contentSha1, lastModified);
}

@Override
@@ -84,7 +84,7 @@ public interface MultipartApi {
@Named("b2_upload_part")
@POST
@MapBinder(UploadPartBinder.class)
UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @HeaderParam("X-Bz-Content-Sha1") String sha1, @PayloadParam("payload") Payload payload);
UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @Nullable @PayloadParam("contentSha1") String sha1, @PayloadParam("payload") Payload payload);

@Named("b2_list_parts")
@POST
@@ -24,7 +24,6 @@
import javax.inject.Named;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -68,7 +67,7 @@ public interface ObjectApi {
@POST
@MapBinder(UploadFileBinder.class)
@Consumes(APPLICATION_JSON)
UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @HeaderParam("X-Bz-Content-Sha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload);
UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @Nullable @PayloadParam("contentSha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload);

@Named("b2_delete_file_version")
@POST
@@ -122,26 +122,6 @@ public void testCopyIfUnmodifiedSince() throws Exception {
}
}

@Override
public void testPutInputStream() throws Exception {
try {
super.testPutInputStream();
failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
} catch (IllegalArgumentException iae) {
throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae);
}
}

@Override
public void testPutMultipartInputStream() throws Exception {
try {
super.testPutMultipartInputStream();
failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
} catch (IllegalArgumentException iae) {
throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae);
}
}

@Override
public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
try {

0 comments on commit f0297bd

Please sign in to comment.