Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migration tool ObjectMetadata transforms #5970

Merged
merged 6 commits into from
Mar 28, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -23,17 +23,53 @@
import software.amazon.awssdk.core.async.AsyncRequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GeneratePresignedUrlRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.UploadRequest;

public class S3Transforms {

void upload(S3TransferManager tm, String bucket, String key) {
void upload_streamWithLiteralLength(S3TransferManager tm, String bucket, String key) {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithInputStream = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
PutObjectRequest requestWithStreamAndLiteralLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(333L)
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithInputStream).requestBody(AsyncRequestBody.fromInputStream(inputStream, -1L, newExecutorServiceVariableToDefine)).build());
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndLiteralLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, 333, newExecutorServiceVariableToDefine)).build());
}

void upload_streamWithAssignedLength(S3TransferManager tm, String bucket, String key) {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
long contentLen = 777;
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithStreamAndAssignedLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(contentLen)
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndAssignedLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, contentLen, newExecutorServiceVariableToDefine)).build());
}

void upload_streamWithoutLength(S3TransferManager tm, String bucket, String key) {
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithStreamAndNoLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndNoLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, -1L, newExecutorServiceVariableToDefine)).build());
}

void objectmetadata_unsupportedSetters(Date dateVal) {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();

/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - expirationTimeRuleId - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.expirationTimeRuleId("expirationTimeRuleId");
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - ongoingRestore - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.ongoingRestore(false);
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - requesterCharged - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.requesterCharged(false);

/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - lastModified - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.lastModified(dateVal);
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - expirationTime - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.expirationTime(dateVal);
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - restoreExpirationTime - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.restoreExpirationTime(dateVal);

/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - header - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.header("key", "val");
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - addUserMetadata - is not supported, please manually migrate the code by setting it on the v2 request/response object.*/metadata.addUserMetadata("a", "b");
}

private void generatePresignedUrl(S3Client s3, String bucket, String key, Date expiration) {
@@ -43,7 +79,6 @@ private void generatePresignedUrl(S3Client s3, String bucket, String key, Date e

URL urlPost = /*AWS SDK for Java v2 migration: S3 generatePresignedUrl() with POST HTTP method is not supported in v2. Only GET, PUT, and DELETE are supported - https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/presigner/S3Presigner.html*/s3.generatePresignedUrl(bucket, key, expiration, HttpMethod.POST);


HttpMethod httpMethod = HttpMethod.PUT;
URL urlWithHttpMethodVariable = /*AWS SDK for Java v2 migration: Transform for S3 generatePresignedUrl() with an assigned variable for HttpMethod is not supported. Please manually migrate your code - https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/presigner/S3Presigner.html*/s3.generatePresignedUrl(bucket, key, expiration, httpMethod);

Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@
import com.amazonaws.HttpMethod;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import java.io.ByteArrayInputStream;
@@ -27,11 +28,45 @@

public class S3Transforms {

void upload(TransferManager tm, String bucket, String key) {
void upload_streamWithLiteralLength(TransferManager tm, String bucket, String key) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(333);
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithInputStream = new PutObjectRequest(bucket, key, "location");
requestWithInputStream.setInputStream(inputStream);
tm.upload(requestWithInputStream);
PutObjectRequest requestWithStreamAndLiteralLength = new PutObjectRequest(bucket, key, "location").withMetadata(metadata);
requestWithStreamAndLiteralLength.setInputStream(inputStream);
tm.upload(requestWithStreamAndLiteralLength);
}

void upload_streamWithAssignedLength(TransferManager tm, String bucket, String key) {
ObjectMetadata metadata = new ObjectMetadata();
long contentLen = 777;
metadata.setContentLength(contentLen);
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithStreamAndAssignedLength = new PutObjectRequest(bucket, key, "location").withMetadata(metadata);
requestWithStreamAndAssignedLength.setInputStream(inputStream);
tm.upload(requestWithStreamAndAssignedLength);
}

void upload_streamWithoutLength(TransferManager tm, String bucket, String key) {
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithStreamAndNoLength = new PutObjectRequest(bucket, key, "location");
requestWithStreamAndNoLength.setInputStream(inputStream);
tm.upload(requestWithStreamAndNoLength);
}

void objectmetadata_unsupportedSetters(Date dateVal) {
ObjectMetadata metadata = new ObjectMetadata();

metadata.setExpirationTimeRuleId("expirationTimeRuleId");
metadata.setOngoingRestore(false);
metadata.setRequesterCharged(false);

metadata.setLastModified(dateVal);
metadata.setExpirationTime(dateVal);
metadata.setRestoreExpirationTime(dateVal);

metadata.setHeader("key", "val");
metadata.addUserMetadata("a", "b");
}

private void generatePresignedUrl(AmazonS3 s3, String bucket, String key, Date expiration) {
@@ -41,7 +76,6 @@ private void generatePresignedUrl(AmazonS3 s3, String bucket, String key, Date e

URL urlPost = s3.generatePresignedUrl(bucket, key, expiration, HttpMethod.POST);


HttpMethod httpMethod = HttpMethod.PUT;
URL urlWithHttpMethodVariable = s3.generatePresignedUrl(bucket, key, expiration, httpMethod);

Original file line number Diff line number Diff line change
@@ -18,11 +18,15 @@
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.RequestPayer;
@@ -47,6 +51,18 @@ void putObject_bucketKeyFile(String bucket, String key, File file) {
.build(), RequestBody.fromFile(file));
}

void putObject_bucketKeyStreamMetadata(String bucket, String key, InputStream stream) {
HeadObjectResponse metadataWithLength = HeadObjectResponse.builder()
.build();
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).contentLength(22L)
.build(), RequestBody.fromInputStream(stream, 22L));


HeadObjectResponse metadataWithoutLength = HeadObjectResponse.builder()
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory. If content length must be unknown, we recommend using the CRT-based S3 client - https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html*/s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).build(), RequestBody.fromContentProvider(() -> stream, "application/octet-stream"));
}

/**
* Mixed ordering to ensure the files are assigned correctly
*/
@@ -76,10 +92,12 @@ void putObject_requestPojoWithInputStream(String bucket, String key) {

PutObjectRequest request1 = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory.*/s3.putObject(request1, RequestBody.fromContentProvider(() -> inputStream1, "binary/octet-stream"));
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory. If content length must be unknown, we recommend using the CRT-based S3 client - https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html*/s3.putObject(request1, RequestBody.fromContentProvider(() -> inputStream1, "application/octet-stream"));

/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory.*/s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
.build(), RequestBody.fromContentProvider(() -> inputStream2, "binary/octet-stream"));
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(11L)
.build(), RequestBody.fromInputStream(inputStream2, 11L));
}

void putObject_requestPojoWithoutPayload(String bucket, String key) {
@@ -104,4 +122,53 @@ void putObjectRequesterPaysSetter() {
PutObjectRequest requestWithFalse =PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
.build();
}

void putObjectRequest_setMetadata() {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();

PutObjectRequest request = PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
.build();
request = request.toBuilder().contentLength(66L)
.contentEncoding("UTF-8")
.contentType("text/plain")
.build();
}

void putObjectRequest_withMetadata() {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
long contentLen = 66;
Date expiry = new Date();

Map<String, String> userMetadata = new HashMap<>();
userMetadata.put("key", "value");

PutObjectRequest request = PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location").contentLength(contentLen)
.contentEncoding("UTF-8")
.contentType("text/plain")
.contentLanguage("en-US")
.cacheControl("must-revalidate")
.contentDisposition("inline")
.contentMD5("md5Val")
.serverSideEncryption("sseEncryptionVal")
.serverSideEncryption("sseAlgorithmVal")
.sseCustomerKeyMD5("sseCustomerKeyMd5Val")
.bucketKeyEnabled(true)
.metadata(userMetadata)
.expires(expiry.toInstant())
.build();
}

void putObjectRequester_emptyMetadata() {
HeadObjectResponse emptyMetadata1 = HeadObjectResponse.builder()
.build();
PutObjectRequest request1 =PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
.build();

HeadObjectResponse emptyMetadata2 = HeadObjectResponse.builder()
.build();
PutObjectRequest request2 = PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
.build();
}
}
Original file line number Diff line number Diff line change
@@ -18,11 +18,15 @@
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

public class S3Streaming {

@@ -41,6 +45,16 @@ void putObject_bucketKeyFile(String bucket, String key, File file) {
s3.putObject(bucket, key, file);
}

void putObject_bucketKeyStreamMetadata(String bucket, String key, InputStream stream) {
ObjectMetadata metadataWithLength = new ObjectMetadata();
metadataWithLength.setContentLength(22);
s3.putObject(bucket, key, stream, metadataWithLength);


ObjectMetadata metadataWithoutLength = new ObjectMetadata();
s3.putObject(bucket, key, stream, metadataWithoutLength);
}

/**
* Mixed ordering to ensure the files are assigned correctly
*/
@@ -69,7 +83,9 @@ void putObject_requestPojoWithInputStream(String bucket, String key) {
request1.setInputStream(inputStream1);
s3.putObject(request1);

s3.putObject(new PutObjectRequest(bucket, key, "location").withInputStream(inputStream2));
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(11);
s3.putObject(new PutObjectRequest(bucket, key, "location").withInputStream(inputStream2).withMetadata(metadata));
}

void putObject_requestPojoWithoutPayload(String bucket, String key) {
@@ -90,4 +106,47 @@ void putObjectRequesterPaysSetter() {

PutObjectRequest requestWithFalse = new PutObjectRequest("bucket", "key", "location").withRequesterPays(false);
}

void putObjectRequest_setMetadata() {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(66);
metadata.setContentType("text/plain");
metadata.setContentEncoding("UTF-8");

PutObjectRequest request = new PutObjectRequest("bucket", "key", "location");
request.setMetadata(metadata);
}

void putObjectRequest_withMetadata() {
ObjectMetadata metadata = new ObjectMetadata();
long contentLen = 66;
metadata.setContentLength(contentLen);
metadata.setContentType("text/plain");
metadata.setContentEncoding("UTF-8");
metadata.setContentLanguage("en-US");
metadata.setCacheControl("must-revalidate");
metadata.setContentDisposition("inline");
metadata.setContentMD5("md5Val");
metadata.setSSEAlgorithm("sseAlgorithmVal");
metadata.setServerSideEncryption("sseEncryptionVal");
metadata.setSSECustomerKeyMd5("sseCustomerKeyMd5Val");
metadata.setBucketKeyEnabled(true);
Date expiry = new Date();
metadata.setHttpExpiresDate(expiry);

Map<String, String> userMetadata = new HashMap<>();
userMetadata.put("key", "value");
metadata.setUserMetadata(userMetadata);

PutObjectRequest request = new PutObjectRequest("bucket", "key", "location").withMetadata(metadata);
}

void putObjectRequester_emptyMetadata() {
ObjectMetadata emptyMetadata1 = new ObjectMetadata();
PutObjectRequest request1 = new PutObjectRequest("bucket", "key", "location").withMetadata(emptyMetadata1);

ObjectMetadata emptyMetadata2 = new ObjectMetadata();
PutObjectRequest request2 = new PutObjectRequest("bucket", "key", "location");
request2.setMetadata(emptyMetadata2);
}
}
Loading
Oops, something went wrong.
Loading
Oops, something went wrong.