From 6c75d40b961a120afd222b7a42daf6184244c4ed Mon Sep 17 00:00:00 2001 From: Bethany Seeger Date: Wed, 10 Apr 2024 17:16:24 -0400 Subject: [PATCH 1/3] Add metadata to Cloudfile and populate the S3 version with tags. --- .../java/formflow/library/file/CloudFile.java | 8 +- .../library/file/S3CloudFileRepository.java | 139 ++++++++++-------- .../controllers/FileControllerTest.java | 23 ++- 3 files changed, 97 insertions(+), 73 deletions(-) diff --git a/src/main/java/formflow/library/file/CloudFile.java b/src/main/java/formflow/library/file/CloudFile.java index 665dbba9a..ec73e0979 100644 --- a/src/main/java/formflow/library/file/CloudFile.java +++ b/src/main/java/formflow/library/file/CloudFile.java @@ -1,13 +1,15 @@ package formflow.library.file; +import java.util.Map; + import lombok.AllArgsConstructor; import lombok.Getter; @AllArgsConstructor @Getter public class CloudFile { - - private Long fileSize; - private byte[] fileBytes; + private Long fileSize; + private byte[] fileBytes; + private Map metadata; } diff --git a/src/main/java/formflow/library/file/S3CloudFileRepository.java b/src/main/java/formflow/library/file/S3CloudFileRepository.java index 3dab42437..d187c700f 100644 --- a/src/main/java/formflow/library/file/S3CloudFileRepository.java +++ b/src/main/java/formflow/library/file/S3CloudFileRepository.java @@ -8,15 +8,23 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.DeleteObjectRequest; +import com.amazonaws.services.s3.model.GetObjectTaggingRequest; +import com.amazonaws.services.s3.model.GetObjectTaggingResult; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.services.s3.model.Tag; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import com.amazonaws.services.s3.transfer.Upload; import com.amazonaws.util.IOUtils; + import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; @@ -31,74 +39,81 @@ @Slf4j public class S3CloudFileRepository implements CloudFileRepository { - private final String bucketName; - private final AmazonS3 s3Client; - private final TransferManager transferManager; + private final String bucketName; + private final AmazonS3 s3Client; + private final TransferManager transferManager; - public S3CloudFileRepository(@Value("${form-flow.aws.access_key}") String accessKey, - @Value("${form-flow.aws.secret_key}") String secretKey, - @Value("${form-flow.aws.s3_bucket_name}") String s3BucketName, - @Value("${form-flow.aws.region}") String region) { - AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); - bucketName = s3BucketName; - s3Client = AmazonS3ClientBuilder - .standard() - .withRegion(region) - .withCredentials(new AWSStaticCredentialsProvider(credentials)) - .build(); - transferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build(); - } + public S3CloudFileRepository(@Value("${form-flow.aws.access_key}") String accessKey, + @Value("${form-flow.aws.secret_key}") String secretKey, + @Value("${form-flow.aws.s3_bucket_name}") String s3BucketName, + @Value("${form-flow.aws.region}") String region) { + AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); + bucketName = s3BucketName; + s3Client = AmazonS3ClientBuilder + .standard() + .withRegion(region) + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .build(); + transferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build(); + } - /** - * Takes a filepath and Multipart file to upload the multipart file to AWS S3 where the filepath acts as the path to the file in - * S3. File paths that include "/" will create a folder structure where each string prior to the "/" will represent a folder. - * - * @param filePath File path representing a folder structure and path to the file in S3. - * @param file The multipart file to be uploaded to S3. - */ - public void upload(String filePath, MultipartFile file) { - try { - log.info("Inside the S3 File Repository Upload Call"); - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentType(file.getContentType()); - objectMetadata.setContentLength(file.getSize()); - log.info("Upload Metadata Set"); - Upload upload = transferManager.upload(bucketName, filePath, file.getInputStream(), objectMetadata); - log.info("Upload Called"); - upload.waitForCompletion(); - log.info("Upload complete"); - } catch (AmazonServiceException e) { - // make some noise, something's wrong with our connection to S3 - System.err.println(e.getErrorMessage()); - log.error("AWS S3 exception occurred: " + e.getErrorMessage()); - throw new RuntimeException(e.getErrorMessage()); - } catch (InterruptedException | IOException e) { - log.error("Exception occurred in S3 code: " + e.getMessage()); - throw new RuntimeException(e.getMessage()); + /** + * Takes a filepath and Multipart file to upload the multipart file to AWS S3 where the filepath acts as the path to the file + * in S3. File paths that include "/" will create a folder structure where each string prior to the "/" will represent a + * folder. + * + * @param filePath File path representing a folder structure and path to the file in S3. + * @param file The multipart file to be uploaded to S3. + */ + public void upload(String filePath, MultipartFile file) { + try { + log.info("Inside the S3 File Repository Upload Call"); + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentType(file.getContentType()); + objectMetadata.setContentLength(file.getSize()); + log.info("Upload Metadata Set"); + Upload upload = transferManager.upload(bucketName, filePath, file.getInputStream(), objectMetadata); + log.info("Upload Called"); + upload.waitForCompletion(); + log.info("Upload complete"); + } catch (AmazonServiceException e) { + // make some noise, something's wrong with our connection to S3 + System.err.println(e.getErrorMessage()); + log.error("AWS S3 exception occurred: " + e.getErrorMessage()); + throw new RuntimeException(e.getErrorMessage()); + } catch (InterruptedException | IOException e) { + log.error("Exception occurred in S3 code: " + e.getMessage()); + throw new RuntimeException(e.getMessage()); + } } - } - public CloudFile get(String filepath) { - try { - log.info("Getting file at filepath {} from S3", filepath); - S3Object s3Object = s3Client.getObject(bucketName, filepath); - S3ObjectInputStream inputStream = s3Object.getObjectContent(); + public CloudFile get(String filepath) { + try { + log.info("Getting file at filepath {} from S3", filepath); + S3Object s3Object = s3Client.getObject(bucketName, filepath); + S3ObjectInputStream inputStream = s3Object.getObjectContent(); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - IOUtils.copy(inputStream, outputStream); - byte[] fileBytes = outputStream.toByteArray(); - long fileSize = fileBytes.length; + GetObjectTaggingRequest getTaggingRequest = new GetObjectTaggingRequest(bucketName, s3Object.getKey()); + GetObjectTaggingResult getTagsResult = s3Client.getObjectTagging(getTaggingRequest); - log.info("File {} successfully downloaded", filepath); - return new CloudFile(fileSize, fileBytes); - } catch (IOException e) { - log.error("Exception occurred while attempting to get the file with path %s: " + e.getMessage(), filepath); - throw new RuntimeException(e.getMessage()); + Map metadata = new HashMap<>(); + metadata.put("tags", getTagsResult.getTagSet()); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + IOUtils.copy(inputStream, outputStream); + byte[] fileBytes = outputStream.toByteArray(); + long fileSize = fileBytes.length; + + log.info("File {} successfully downloaded", filepath); + return new CloudFile(fileSize, fileBytes, metadata); + } catch (IOException e) { + log.error("Exception occurred while attempting to get the file with path %s: " + e.getMessage(), filepath); + throw new RuntimeException(e.getMessage()); + } } - } - public void delete(String filepath) throws SdkClientException { - log.info("Deleting file at filepath {} from S3", filepath); - s3Client.deleteObject(new DeleteObjectRequest(bucketName, filepath)); - } + public void delete(String filepath) throws SdkClientException { + log.info("Deleting file at filepath {} from S3", filepath); + s3Client.deleteObject(new DeleteObjectRequest(bucketName, filepath)); + } } diff --git a/src/test/java/formflow/library/controllers/FileControllerTest.java b/src/test/java/formflow/library/controllers/FileControllerTest.java index fd34ba35f..bbac032a3 100644 --- a/src/test/java/formflow/library/controllers/FileControllerTest.java +++ b/src/test/java/formflow/library/controllers/FileControllerTest.java @@ -23,6 +23,7 @@ import formflow.library.file.CloudFileRepository; import formflow.library.utilities.AbstractMockMvcTest; import formflow.library.utils.UserFileMap; + import java.nio.file.Files; import java.nio.file.Paths; import java.time.OffsetDateTime; @@ -32,6 +33,7 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; + import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -171,7 +173,8 @@ void shouldShowFileContainsVirusErrorIfClammitScanFindsVirus() throws Exception .session(session) .contentType(MediaType.MULTIPART_FORM_DATA_VALUE)) .andExpect(status().is(HttpStatus.UNPROCESSABLE_ENTITY.value())) - .andExpect(content().string("We are unable to process this file because a virus was detected. Please try another file.")); + .andExpect(content().string( + "We are unable to process this file because a virus was detected. Please try another file.")); } @Test @@ -352,7 +355,8 @@ void shouldReturnForbiddenStatusIfSessionIdDoesNotMatchSubmissionIdForSingleFile when(userFileRepositoryService.findById(fileId)).thenReturn(Optional.ofNullable(userFile)); mockMvc.perform( MockMvcRequestBuilders - .get("/file-download/{flow}/{submissionId}/{fileId}", "testFlow", submission.getId().toString(), fileId) + .get("/file-download/{flow}/{submissionId}/{fileId}", "testFlow", submission.getId().toString(), + fileId) .session(session)) .andExpect(status().is(HttpStatus.FORBIDDEN.value())); } @@ -368,7 +372,8 @@ void shouldReturnForbiddenIfAFilesSubmissionIdDoesNotMatchSubmissionIdOnTheUserF when(submissionRepositoryService.findById(submission.getId())).thenReturn(Optional.of(submission)); mockMvc.perform( MockMvcRequestBuilders - .get("/file-download/{flow}/{submissionId}/{fileId}", "testFlow", submission.getId().toString(), fileId) + .get("/file-download/{flow}/{submissionId}/{fileId}", "testFlow", submission.getId().toString(), + fileId) .session(session)) .andExpect(status().is(HttpStatus.FORBIDDEN.value())); } @@ -418,7 +423,8 @@ void singleFileEndpointShouldReturnNotFoundIfNoUserFileIsFoundForAGivenFileId() when(userFileRepositoryService.findById(fileId)).thenReturn(Optional.empty()); mockMvc.perform( MockMvcRequestBuilders - .get("/file-download/{flow}/{submissionId}/{fileId}", "testFlow", submission.getId().toString(), fileId) + .get("/file-download/{flow}/{submissionId}/{fileId}", "testFlow", submission.getId().toString(), + fileId) .session(session)) .andExpect(status().is(404)); } @@ -428,7 +434,7 @@ void singleFileEndpointShouldReturnTheSameFileBytesAsTheCloudFileRepository() th setFlowInfoInSession(session, "testFlow", submission.getId()); byte[] testFileBytes = "foo".getBytes(); long fileSize = testFileBytes.length; - CloudFile testcloudFile = new CloudFile(fileSize, testFileBytes); + CloudFile testcloudFile = new CloudFile(fileSize, testFileBytes, null); UserFile testUserFile = UserFile.builder() .originalName("testFileName") .mimeType("image/jpeg") @@ -441,7 +447,8 @@ void singleFileEndpointShouldReturnTheSameFileBytesAsTheCloudFileRepository() th MvcResult mvcResult = mockMvc.perform( MockMvcRequestBuilders - .get("/file-download/{flow}/{submissionId}/{fileId}", "testFlow", submission.getId().toString(), fileId) + .get("/file-download/{flow}/{submissionId}/{fileId}", "testFlow", submission.getId().toString(), + fileId) .session(session)) .andExpect(MockMvcResultMatchers.request().asyncStarted()) .andReturn(); @@ -461,8 +468,8 @@ void multiFileEndpointShouldReturnZipOfUserFilesReturnedByTheCloudFileRepository byte[] secondTestFileBytes = Files.readAllBytes(Paths.get("src/test/resources/test-platypus.gif")); long firstTestFileSize = firstTestFileBytes.length; long secondTestFileSize = secondTestFileBytes.length; - CloudFile firstTestcloudFile = new CloudFile(firstTestFileSize, firstTestFileBytes); - CloudFile secondTestcloudFile = new CloudFile(secondTestFileSize, secondTestFileBytes); + CloudFile firstTestcloudFile = new CloudFile(firstTestFileSize, firstTestFileBytes, null); + CloudFile secondTestcloudFile = new CloudFile(secondTestFileSize, secondTestFileBytes, null); UserFile firstTestUserFile = UserFile.builder() .originalName("test.png") From b411e913120cb8b6382c03e9d978a5fa54ad636c Mon Sep 17 00:00:00 2001 From: Bethany Seeger Date: Thu, 11 Apr 2024 10:58:01 -0400 Subject: [PATCH 2/3] Update readme with CloudFile and Virus Scanning info. --- README.md | 190 +++++++++++------- .../library/file/S3CloudFileRepository.java | 2 - 2 files changed, 119 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 3094801d4..6e36cfa6e 100644 --- a/README.md +++ b/README.md @@ -191,26 +191,26 @@ example using our [live templates for a form screen](#applying-live-templates-to + th:replace="~{fragments/cardHeader :: cardHeader(header='Tell us about yourself')}"/> - -
- + +
+ - - -
- + -
-
- +
+ + ``` @@ -407,8 +407,8 @@ Thymeleaf templates. Template code can run conditions via this object, like so: ```html
-

Conditionally show this element

+ th:with="showCondition=${conditionManager.runCondition('ConditionName', submission, 'data')}"> +

Conditionally show this element

``` @@ -925,18 +925,18 @@ The template HTML can look like:
- - -
-
-
-

-
-
-
-
- -
+ + +
+
+
+

+
+
+
+
+ +
@@ -979,15 +979,15 @@ Example form fragment: ```html - -
- INPUTS GO HERE -
-
- SUBMIT BUTTON GOES HERE -
- + th:replace="~{fragments/form :: form(action=${formAction}, content=~{::exampleForm}, formId='exampleID')}"> + +
+ INPUTS GO HERE +
+
+ SUBMIT BUTTON GOES HERE +
+ ``` @@ -1236,12 +1236,12 @@ Below are examples of both types of checkboxes: label='This label is actually a legend for the checkbox fieldset', fieldsetHelpText='This help text will appear below the legend', content=~{::vehiclesOwnedContent})}"> - - - - + + + + ``` @@ -1256,7 +1256,7 @@ to `checkboxInSet()`: ```html + th:replace="'fragments/inputs/checkboxInSet' :: checkboxInSet(inputName='vehiclesOwned',value='None of the Above', label='None of the Above', noneOfTheAbove=true)"/> ``` Honeycrisp contains JavaScript logic that deselects the other checkboxes when "None of the Above" is @@ -1307,24 +1307,24 @@ An example of a radio input: label='What\'s your favorite color?', fieldsetHelpText='This help text will appear under the legend', content=~{::favoriteColorContent})}"> - - + - - - + ``` @@ -1353,17 +1353,17 @@ An example select input: ```html - - - - - - + th:replace="~{fragments/inputs/select :: select(inputName='favoriteFruit', label='What\'s your favorite fruit?', helpText='Mine is banana', content=~{::favoriteFruitContent})}"> + + + + + + ``` @@ -1468,9 +1468,9 @@ Here is an example of using the `reveal` fragment: linkLabel=~{::revealLabel2}, content=~{::revealContent2}, forceShowContent='true')}"> - -

- + +

+ ``` @@ -1511,12 +1511,35 @@ The library provides a file upload feature using the client side JavaScript library [Dropzone JS](https://www.dropzone.dev/). File uploads need a configured AWS S3 Bucket to upload to and provide functionality for uploading, retrieving and deleting files. +### Cloud File Repository + +The library provides a method for integrating with cloud file repositories, like S3. +Right now the library has only implemented an integration with S3. + +#### CloudFile + +As part of this cloud file repository integration, we provide a generic `CloudFile` path to +hold the file information coming back from the could file repository. + +The `CloudFile` class has three fields: + +```java + Long fileSize; + byte[]fileBytes; + Map metadata; +``` + +The first two represent the file and file size information. The `metadata` field could +be anything the implementation would like to store in this field. + +For example, the AWS S3 Cloud File Repository will put the S3 file's `tag` information +in this metadata field, under the `tags` key. + ### AWS S3 You will need a registered AWS account to set up an S3 bucket. Once you have registered your AWS account you -can [follow the instructions here to create an S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) -. +can [follow the instructions here to create an S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html). Make sure to note your bucket's name and region as well as your AWS access and secret keys as you will need these for configuring file uploads in the library. The bucket and region are configured in @@ -1533,7 +1556,7 @@ input field the file was uploaded from, and a UUID. We then store it in S3, orga submission it is a part of, like so: ``` - `{{submission_id}}/{{flow_name}}_{{input_name}}_UUID.{jpg, png, docx…} ` + {{submission_id}}/{{flow_name}}_{{input_name}}_UUID.{jpg, png, docx…} ``` The `flow_name` is the flow the user was in when they uploaded the file and the `input_name` is the @@ -1648,6 +1671,8 @@ permits. ### Virus Scanning +#### ClamAV Server + File uploads made through form flow can be scanned for viruses. We provide a way to pass files to a ClamAV server. @@ -1670,6 +1695,29 @@ There is a field `virus_scanned` in the `user_files` table with `Boolean` as its > ⚠️ If virus scanning is enabled and a virus is detected in a file, it is rejected and not saved in > our systems. +#### Cloud Storage Security in AWS + +Some projects have chosen to +use [Cloud Storage Security](https://aws.amazon.com/marketplace/pp/prodview-q7oc4shdnpc4w?ref_=aws-mp-console-subscription-detail) +right in AWS. It will allow a file to be scanned once it has been uploaded to AWS. + +When this option is used, the scanner will add a few tags directly to the files once they are scanned. By default +configuration, if a virus is found the file is moved to a quarantine bucket. Please +read the Cloud Storage Security information for more details about that. + +The tags added to the files are: + +* `scan-result` - this is generally set to `Clean` unless there was a virus +* `date-scanned` - a date and time stamp; for example: `2024-03-27 12:38:35Z` + +If this virus scanning tool is used, these (and any other tags on the file) can be +retrieved via the `CloudFile` object's `metadata` field. The key that this data is stored +under is `tags`. The data type is `List`. + +Because of the asynchronous nature of this method of virus scanning, the `user_files` table +is **not** updated with the virus scanning information and is **not** a reliable source for determining if a file has +been scanned or not. + ## Document Download Form flow library allows users to either: @@ -2160,7 +2208,7 @@ public class ApplicantDateOfBirthPreparer implements SubmissionFieldPreparer { @Override public Map prepareSubmissionFields(Submission submission, - PdfMap pdfMap) { + PdfMap pdfMap) { Map submissionFields = new HashMap<>(); String month = submission.getInputData().get("applicantBirthMonth").toString(); @@ -2211,7 +2259,7 @@ public class DataBaseFieldPreparer implements SubmissionFieldPreparer { @Override public Map prepareSubmissionFields(Submission submission, - PdfMap pdfMap) { + PdfMap pdfMap) { Map submissionFields = new HashMap<>(); ArrayList> houseHoldSubflow = (ArrayList>) submission.getInputData() @@ -2422,7 +2470,7 @@ Below is an example of a sendEmail() call being made by an application using the Please note that pdfs is a list of files to be passed as attachments with the email. ```java -MessageResponse response = mailgunEmailClient.sendEmail( +MessageResponse response=mailgunEmailClient.sendEmail( emailSubject, recipientEmail, emailToCc, @@ -2430,7 +2478,7 @@ MessageResponse response = mailgunEmailClient.sendEmail( emailBody, pdfs, requireTls -); + ); ``` The `sendEmail()` method will send an email and return the `MessageResponse` object it receives from diff --git a/src/main/java/formflow/library/file/S3CloudFileRepository.java b/src/main/java/formflow/library/file/S3CloudFileRepository.java index d187c700f..18741fdc4 100644 --- a/src/main/java/formflow/library/file/S3CloudFileRepository.java +++ b/src/main/java/formflow/library/file/S3CloudFileRepository.java @@ -13,7 +13,6 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.Tag; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import com.amazonaws.services.s3.transfer.Upload; @@ -22,7 +21,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; From 9559b0cb06101135ba1bd2c772bcc4742c7bac10 Mon Sep 17 00:00:00 2001 From: Bethany Seeger Date: Thu, 11 Apr 2024 11:02:16 -0400 Subject: [PATCH 3/3] Add javadocs --- .../formflow/library/file/S3CloudFileRepository.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/formflow/library/file/S3CloudFileRepository.java b/src/main/java/formflow/library/file/S3CloudFileRepository.java index 18741fdc4..24663a354 100644 --- a/src/main/java/formflow/library/file/S3CloudFileRepository.java +++ b/src/main/java/formflow/library/file/S3CloudFileRepository.java @@ -85,6 +85,12 @@ public void upload(String filePath, MultipartFile file) { } } + /** + * Retrieves a file from S3. + * + * @param filepath The path of the file + * @return CloudFile containing file, file size, and metadata about the file. + */ public CloudFile get(String filepath) { try { log.info("Getting file at filepath {} from S3", filepath); @@ -110,6 +116,12 @@ public CloudFile get(String filepath) { } } + /** + * Deletes a file from S3 storage. + * + * @param filepath The path of the file to delete + * @throws SdkClientException + */ public void delete(String filepath) throws SdkClientException { log.info("Deleting file at filepath {} from S3", filepath); s3Client.deleteObject(new DeleteObjectRequest(bucketName, filepath));