Skip to content

Commit 998c43e

Browse files
author
Neil Avery
committed
s3 upload and multipart working
1 parent 407725f commit 998c43e

File tree

10 files changed

+392
-119
lines changed

10 files changed

+392
-119
lines changed

‎pom.xml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,10 @@
7171
<!-- </resource>-->
7272
<!-- </resources>-->
7373
<!-- <testSourceDirectory>test</testSourceDirectory>-->
74-
<!-- <testResources>-->
75-
<!-- <testResource>-->
76-
<!-- <directory>test-data</directory>-->
77-
<!-- </testResource></testResources>-->
74+
<testResources>
75+
<testResource>
76+
<directory>test-data</directory>
77+
</testResource></testResources>
7878

7979
<pluginManagement>
8080
<plugins>

‎uploader/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@
4444
<groupId>io.quarkus</groupId>
4545
<artifactId>quarkus-resteasy</artifactId>
4646
</dependency>
47+
<!-- https://mvnrepository.com/artifact/org.jboss.resteasy/resteasy-multipart-provider -->
48+
<dependency>
49+
<groupId>org.jboss.resteasy</groupId>
50+
<artifactId>resteasy-multipart-provider</artifactId>
51+
<version>4.4.1.Final</version>
52+
</dependency>
4753
<dependency>
4854
<groupId>io.quarkus</groupId>
4955
<artifactId>quarkus-junit5</artifactId>

‎uploader/src/main/java/com/liquidlabs/logscapeng/uploader/AWSS3UploaderService.java‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
package com.liquidlabs.logscapeng.uploader;
2+
3+
import com.amazonaws.AmazonServiceException;
4+
import com.amazonaws.SdkClientException;
5+
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
6+
import com.amazonaws.regions.Regions;
7+
import com.amazonaws.services.s3.AmazonS3;
8+
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
9+
import com.amazonaws.services.s3.model.*;
10+
import org.apache.commons.io.IOUtils;
11+
12+
import javax.enterprise.context.ApplicationScoped;
13+
import java.io.File;
14+
import java.io.FileOutputStream;
15+
import java.io.IOException;
16+
import java.util.ArrayList;
17+
import java.util.List;
18+
19+
@ApplicationScoped
20+
public class AWSS3UploaderService {
21+
22+
23+
24+
public String upload(UploadMeta upload) {
25+
Regions clientRegion = Regions.EU_WEST_2;
26+
String bucketName = upload.tenant;
27+
String keyName = upload.resource + "/" + upload.filename;
28+
String filePath = upload.resource + "/" + upload.filename;
29+
30+
ObjectMetadata objectMetadata = new ObjectMetadata();
31+
objectMetadata.addUserMetadata("tags", upload.tags);
32+
objectMetadata.addUserMetadata("tenant", upload.tenant);
33+
objectMetadata.addUserMetadata("length", ""+upload.filecontent.length);
34+
35+
36+
File file = createTempFile(upload.filecontent);
37+
long contentLength = file.length();
38+
long partSize = 5 * 1024 * 1024; // Set part size to 5 MB.
39+
40+
try {
41+
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
42+
.withRegion(clientRegion)
43+
.withCredentials(new ProfileCredentialsProvider())
44+
.build();
45+
46+
47+
if (!s3Client.doesBucketExistV2(upload.tenant)) {
48+
s3Client.createBucket(upload.tenant);
49+
}
50+
51+
// Create a list of ETag objects. You retrieve ETags for each object part uploaded,
52+
// then, after each individual part has been uploaded, pass the list of ETags to
53+
// the request to complete the upload.
54+
List<PartETag> partETags = new ArrayList<PartETag>();
55+
56+
// Initiate the multipart upload.
57+
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName, objectMetadata);
58+
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
59+
60+
61+
// Upload the file parts.
62+
long filePosition = 0;
63+
for (int i = 1; filePosition < upload.filecontent.length; i++) {
64+
// Because the last part could be less than 5 MB, adjust the part size as needed.
65+
partSize = Math.min(partSize, (contentLength - filePosition));
66+
67+
// Create the request to upload a part.
68+
UploadPartRequest uploadRequest = new UploadPartRequest()
69+
.withBucketName(bucketName)
70+
.withKey(keyName)
71+
.withUploadId(initResponse.getUploadId())
72+
.withPartNumber(i)
73+
.withFileOffset(filePosition)
74+
.withFile(file)
75+
.withPartSize(partSize);
76+
77+
// Upload the part and add the response's ETag to our list.
78+
UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest);
79+
partETags.add(uploadResult.getPartETag());
80+
81+
filePosition += partSize;
82+
}
83+
84+
System.out.println("ETags:" + partETags);
85+
// Complete the multipart upload.
86+
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName,
87+
initResponse.getUploadId(), partETags);
88+
s3Client.completeMultipartUpload(compRequest);
89+
} catch (AmazonServiceException e) {
90+
// The call was transmitted successfully, but Amazon S3 couldn't process
91+
// it, so it returned an error response.
92+
e.printStackTrace();
93+
} catch (SdkClientException e) {
94+
// Amazon S3 couldn't be contacted for a response, or the client
95+
// couldn't parse the response from Amazon S3.
96+
e.printStackTrace();
97+
} finally {
98+
file.delete();
99+
}
100+
101+
return "yay";
102+
103+
}
104+
105+
private File createTempFile(byte[] filecontent) {
106+
try {
107+
File tempFile = File.createTempFile("test", ".tmp");
108+
FileOutputStream fos = new FileOutputStream(tempFile);
109+
fos.write(filecontent);
110+
fos.flush();
111+
fos.close();
112+
return tempFile;
113+
} catch (IOException e) {
114+
e.printStackTrace();
115+
}
116+
throw new RuntimeException("Failed to create temp file");
117+
}
118+
119+
120+
public static void main(String[] args) throws IOException {
121+
Regions clientRegion = Regions.DEFAULT_REGION;
122+
String bucketName = "*** Bucket name ***";
123+
String keyName = "*** Key name ***";
124+
String filePath = "*** Path to file to upload ***";
125+
126+
File file = new File(filePath);
127+
long contentLength = file.length();
128+
long partSize = 5 * 1024 * 1024; // Set part size to 5 MB.
129+
130+
try {
131+
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
132+
.withRegion(clientRegion)
133+
.withCredentials(new ProfileCredentialsProvider())
134+
.build();
135+
136+
// Create a list of ETag objects. You retrieve ETags for each object part uploaded,
137+
// then, after each individual part has been uploaded, pass the list of ETags to
138+
// the request to complete the upload.
139+
List<PartETag> partETags = new ArrayList<PartETag>();
140+
141+
// Initiate the multipart upload.
142+
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName);
143+
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
144+
145+
// Upload the file parts.
146+
long filePosition = 0;
147+
for (int i = 1; filePosition < contentLength; i++) {
148+
// Because the last part could be less than 5 MB, adjust the part size as needed.
149+
partSize = Math.min(partSize, (contentLength - filePosition));
150+
151+
// Create the request to upload a part.
152+
UploadPartRequest uploadRequest = new UploadPartRequest()
153+
.withBucketName(bucketName)
154+
.withKey(keyName)
155+
.withUploadId(initResponse.getUploadId())
156+
.withPartNumber(i)
157+
.withFileOffset(filePosition)
158+
.withFile(file)
159+
.withPartSize(partSize);
160+
161+
// Upload the part and add the response's ETag to our list.
162+
UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest);
163+
partETags.add(uploadResult.getPartETag());
164+
165+
filePosition += partSize;
166+
}
167+
168+
// Complete the multipart upload.
169+
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName,
170+
initResponse.getUploadId(), partETags);
171+
s3Client.completeMultipartUpload(compRequest);
172+
} catch (AmazonServiceException e) {
173+
// The call was transmitted successfully, but Amazon S3 couldn't process
174+
// it, so it returned an error response.
175+
e.printStackTrace();
176+
} catch (SdkClientException e) {
177+
// Amazon S3 couldn't be contacted for a response, or the client
178+
// couldn't parse the response from Amazon S3.
179+
e.printStackTrace();
180+
}
181+
}
182+
183+
184+
185+
}

‎uploader/src/main/java/com/liquidlabs/logscapeng/uploader/SimpleServerSideUploader.java‎

Lines changed: 0 additions & 92 deletions
This file was deleted.

‎uploader/src/main/java/com/liquidlabs/logscapeng/uploader/SimpleServersideUploaderResource.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package com.liquidlabs.logscapeng.uploader;
2+
3+
import java.io.File;
4+
import java.io.FileOutputStream;
5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.util.ArrayList;
8+
import java.util.List;
9+
import java.util.Map;
10+
11+
import com.amazonaws.AmazonServiceException;
12+
import com.amazonaws.SdkClientException;
13+
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
14+
import com.amazonaws.regions.Regions;
15+
import com.amazonaws.services.s3.AmazonS3;
16+
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
17+
import com.amazonaws.services.s3.model.*;
18+
import org.acme.quickstart.GreetingService;
19+
import org.apache.commons.io.IOUtils;
20+
import org.jboss.resteasy.annotations.providers.multipart.MultipartForm;
21+
import org.jboss.resteasy.plugins.providers.multipart.InputPart;
22+
import org.jboss.resteasy.plugins.providers.multipart.MultipartFormDataInput;
23+
import org.jboss.resteasy.annotations.providers.multipart.MultipartForm;
24+
25+
import javax.inject.Inject;
26+
import javax.ws.rs.*;
27+
import javax.ws.rs.core.MediaType;
28+
import javax.ws.rs.core.MultivaluedMap;
29+
import javax.ws.rs.core.Response;
30+
31+
/**
32+
* First (naive) implementation.
33+
* Server side uploader the runs with AWS Client credentials.
34+
* Loads directly to S3 bucket, driven by a REST based client that does a binary post.
35+
*
36+
* A Lambda wont handle the volume of data upload.
37+
*/
38+
@Path("/upload")
39+
public class SimpleServersideUploaderResource {
40+
41+
@Inject
42+
AWSS3UploaderService awss3UploaderService;
43+
44+
45+
@GET
46+
@Produces(MediaType.TEXT_PLAIN)
47+
public String id() {
48+
return SimpleServersideUploaderResource.class.getCanonicalName();
49+
}
50+
51+
@POST
52+
@Path("/file")
53+
@Consumes(MediaType.MULTIPART_FORM_DATA)
54+
@Produces(MediaType.TEXT_PLAIN)
55+
public Response uploadFile(@MultipartForm UploadMeta uploadMeta) throws IOException {
56+
System.out.println("GOT FILe:" + uploadMeta);
57+
String upload = awss3UploaderService.upload(uploadMeta);
58+
return Response.status(200).entity(upload).build();
59+
// return Response.status(200).entity("Uploaded file name : " + uploadMeta.filename).build();
60+
// return null;
61+
}
62+
63+
64+
private String getFileName(MultivaluedMap<String, String> header) {
65+
66+
String[] contentDisposition = header.getFirst("Content-Disposition").split(";");
67+
68+
for (String filename : contentDisposition) {
69+
70+
if ((filename.trim().startsWith("filename"))) {
71+
72+
String[] name = filename.split("=");
73+
74+
return name[1].trim().replaceAll("\"", "");
75+
}
76+
}
77+
return "unknown";
78+
}
79+
}

0 commit comments

Comments
 (0)