Skip to content

Commit

Permalink
Update loggin and tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@sree-cfa
sree-cfa committed Jan 5, 2024
1 parent 3851442 commit fdd90d9
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 60 deletions.
2 changes: 1 addition & 1 deletion crontab
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
# 0 19 * * WED java -Dserver.port=9191 -jar /opt/la-doc-uploader/app.jar transmit

# Run every hour
# @hourly java -Dserver.port=9191 -jar /opt/la-doc-uploader/app.jar transferSubmissions
@hourly java -Dserver.port=9191 -jar /opt/la-doc-uploader/app.jar transferSubmissions

43 changes: 31 additions & 12 deletions src/main/java/org/ladocuploader/app/cli/SubmissionTransfer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
import formflow.library.file.CloudFileRepository;
import formflow.library.pdf.PdfService;
import lombok.extern.slf4j.Slf4j;
import org.ladocuploader.app.data.Transmission;
import org.ladocuploader.app.data.TransmissionRepository;
import org.ladocuploader.app.data.enums.TransmissionStatus;
import org.ladocuploader.app.data.enums.TransmissionType;
import org.ladocuploader.app.submission.StringEncryptor;
import org.springframework.data.domain.Sort;
Expand All @@ -20,10 +22,7 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

Expand Down Expand Up @@ -83,31 +82,46 @@ public void transferSubmissions() {
// Get submissions to transfer
String batchSeq = Long.toString(transmissionRepository.nextValueBatchSequence());
String batchIndex = Strings.padStart(batchSeq, BATCH_INDEX_LEN, '0');
UUID uuid = UUID.randomUUID();
String zipFileName = batchIndex + ".zip";
log.info(String.format("Beginning transfer of batch %s", batchIndex));
log.info(String.format("Beginning transfer of %s: batch %s", uuid, batchIndex));

// Stats on transfers
int successful = 0, failed = 0;

// for each submission, add to zipfile
List<Submission> submissionsBatch = transmissionRepository.submissionsToTransmit(Sort.unsorted(), TransmissionType.SNAP);
int subfolderidx = 1;
try (FileOutputStream baos = new FileOutputStream(zipFileName);
ZipOutputStream zos = new ZipOutputStream(baos)) {
long now = new Date().getTime();
StringBuilder docMeta = new StringBuilder();
int subfolderidx = 1;
for (Submission submission : submissionsBatch) {
if (submission.getSubmittedAt().getTime() + TWO_HOURS > now) {
// Give a 2-hour wait for folks to upload documents
continue;
}

Transmission transmission = transmissionRepository.findBySubmissionAndTransmissionType(submission, TransmissionType.SNAP);

String subfolder = Integer.toString(subfolderidx++);
try {
log.info("Generate applicant summary");
packageSnapApplication(batchIndex, zos, docMeta, submission, subfolder);

log.info("Adding uploaded docs");
packageUploadedDocuments(batchIndex, zos, docMeta, submission, subfolder);

transmission.setStatus(TransmissionStatus.Complete);
updateTransmission(uuid, transmission);
successful++;
} catch (Exception e) {
log.error("Error generating file collection for submission ID {}", submission.getId(), e);

transmission.setDocumentationErrors(Map.of("error", e.getMessage()));
transmission.setStatus(TransmissionStatus.Failed);
updateTransmission(uuid, transmission);
failed++;
}
}

Expand All @@ -132,8 +146,13 @@ public void transferSubmissions() {
}
}

// TODO - include some stats, how many transfered, failed, successful
log.info(String.format("Completed transfer of batch %s", batchIndex));
log.info(String.format("Completed transfer of batch %s, total %s, successful %s, failed %s", batchIndex, subfolderidx - 1, successful, failed));
}

private void updateTransmission(UUID uuid, Transmission transmission) {
transmission.setTimeSent(new Date());
transmission.setRunId(uuid);
transmissionRepository.save(transmission);
}

private void packageUploadedDocuments(String batchIndex, ZipOutputStream zos, StringBuilder docMeta, Submission submission, String subfolder) throws IOException {
Expand All @@ -145,7 +164,7 @@ private void packageUploadedDocuments(String batchIndex, ZipOutputStream zos, St
filenameDuplicates.putIfAbsent(docUploadFilename, 0);
filenameDuplicates.computeIfPresent(docUploadFilename, (s, i) -> i + 1);
Integer filecount = filenameDuplicates.get(docUploadFilename);
if(filecount > 1) {
if (filecount > 1) {
docUploadFilename = "%s_%s".formatted(filecount, docUploadFilename);
}

Expand Down Expand Up @@ -207,9 +226,9 @@ private static String removeFileExtension(String filename) {
}

private static String formatBirthdate(Map<String, Object> inputData) {
String day = (String) inputData.get("birthDay");
String month = (String) inputData.get("birthMonth");
String year = (String) inputData.get("birthYear");
String day = (String) inputData.getOrDefault("birthDay", "");
String month = (String) inputData.getOrDefault("birthMonth", "");
String year = (String) inputData.getOrDefault("birthYear", "");

return "%s/%s/%s".formatted(Strings.padStart(day, 2, '0'), Strings.padStart(month, 2, '0'), year);
}
Expand Down
134 changes: 87 additions & 47 deletions src/test/java/org/ladocuploader/app/cli/SubmissionTransferTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
import java.util.zip.ZipInputStream;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.*;
Expand Down Expand Up @@ -63,11 +63,12 @@ class SubmissionTransferTest {
@MockBean
PGPEncryptor encryptor;

Submission submission;
private Submission submissionWithDocs;
private Submission submissionWithoutDocs;
private Submission invalidSubmission;

@BeforeEach
void setup() throws IOException {
when(pdfService.getFilledOutPDF(any())).thenReturn("some bytes".getBytes());
when(encryptor.signAndEncryptPayload(anyString())).thenReturn(new byte[]{});
doAnswer(args -> {
String zipfilename = (String) args.getArguments()[0];
Expand All @@ -79,23 +80,68 @@ void setup() throws IOException {
return null;
}).when(ftpsClient).uploadFile(anyString(), any());

submissionWithoutDocs = queueSubmissionWithoutDocs();
submissionWithDocs = queueSubmissionWithDocs();
invalidSubmission = queueInvalidSubmission();

when(pdfService.getFilledOutPDF(eq(submissionWithoutDocs))).thenReturn("some bytes".getBytes());
when(pdfService.getFilledOutPDF(eq(submissionWithDocs))).thenReturn("some other bytes".getBytes());
when(pdfService.getFilledOutPDF(eq(invalidSubmission))).thenThrow(new IllegalArgumentException("There was an error generating the PDF"));
}

@Test
public void transmitZipFile() throws IOException {
submissionTransfer.transferSubmissions();

File zipFile = new File("mocktransmit_00050000000.zip");
assertTrue(zipFile.exists());

verify(ftpsClient).uploadFile(any(), any());

Transmission transmittedWithDocs = transmissionRepository.findBySubmissionAndTransmissionType(submissionWithDocs, TransmissionType.SNAP);
assertThat(transmittedWithDocs.getStatus(), equalTo(TransmissionStatus.Complete));
assertNull(transmittedWithDocs.getDocumentationErrors());

Transmission transmittedWithoutDocs = transmissionRepository.findBySubmissionAndTransmissionType(submissionWithoutDocs, TransmissionType.SNAP);
assertThat(transmittedWithoutDocs.getStatus(), equalTo(TransmissionStatus.Complete));
assertNull(transmittedWithoutDocs.getDocumentationErrors());

Transmission invalidTransmission = transmissionRepository.findBySubmissionAndTransmissionType(invalidSubmission, TransmissionType.SNAP);
assertThat(invalidTransmission.getStatus(), equalTo(TransmissionStatus.Failed));
assertThat(invalidTransmission.getDocumentationErrors().get("error"), equalTo("There was an error generating the PDF"));

String destDir = "output";
List<String> fileNames = unzip(zipFile.getPath(), destDir);

assertThat(fileNames, hasItem("output/1/"));
assertThat(fileNames, hasItem("output/1/SNAP_application.pdf"));
assertThat(fileNames, hasItem("output/2/"));
assertThat(fileNames, hasItem("output/2/SNAP_application.pdf"));
assertThat(fileNames, hasItem("output/2/originalFilename.png"));
assertThat(fileNames, hasItem("output/2/2_originalFilename.png"));
assertThat(fileNames, hasItem("output/2/weird/:\\filename.jpg"));
assertThat(fileNames, hasItem("output/00050000000.txt"));
assertEquals(8, fileNames.size());

// cleanup
zipFile.delete();
}

private Submission queueInvalidSubmission() {
Date submittedDate = new Date(new Date().getTime() - (1000 * 60 * 60) * 2);
submission = Submission.builder()
Submission submission = Submission.builder()
.submittedAt(submittedDate)
.flow("laDigitalAssister")
.urlParams(new HashMap<>())
.inputData(Map.of(
"firstName", "Tester",
"lastName", "McTest",
"birthDay", "3",
"birthMonth", "12",
"birthYear", "4567",
"signature", "Tester McTest sig"
)).build();
.inputData(new HashMap<>()).build();
submissionRepository.save(submission);
saveTransmissionRecord(submission);
return submission;
}

var submissionWithDocs = Submission.builder()
private Submission queueSubmissionWithDocs() {
Date submittedDate = new Date(new Date().getTime() - (1000 * 60 * 60) * 2);
var submission = Submission.builder()
.submittedAt(submittedDate)
.flow("laDigitalAssister")
.urlParams(new HashMap<>())
Expand All @@ -107,16 +153,16 @@ void setup() throws IOException {
"birthYear", "1111",
"signature", "Other McOtherson sig"
))).build();
submissionRepository.save(submissionWithDocs);
UUID docId1 = saveUserFile(submissionWithDocs, "applicant1_birth-certificate.jpeg", "originalFilename.png");
UUID docId2 = saveUserFile(submissionWithDocs, "applicant1_license.jpeg", "originalFilename.png");
UUID docId3 = saveUserFile(submissionWithDocs, "applicant1_pay-stub.png", "weird/:\\filename.jpg");
saveTransmissionRecord(submissionWithDocs);
submissionWithDocs.getInputData().putAll(Map.of("uploadDocuments", "[\"%s\",\"%s\",\"%s\"]".formatted(docId1, docId2, docId3),
submissionRepository.save(submission);
UUID docId1 = saveUserFile(submission, "applicant1_birth-certificate.jpeg", "originalFilename.png");
UUID docId2 = saveUserFile(submission, "applicant1_license.jpeg", "originalFilename.png");
UUID docId3 = saveUserFile(submission, "applicant1_pay-stub.png", "weird/:\\filename.jpg");
saveTransmissionRecord(submission);
submission.getInputData().putAll(Map.of("uploadDocuments", "[\"%s\",\"%s\",\"%s\"]".formatted(docId1, docId2, docId3),
"docType_wildcard_" + docId1, "BirthCertificate",
"docType_wildcard_" + docId2, "Other",
"docType_wildcard_" + docId3, "DriversLicense"));
submissionRepository.save(submissionWithDocs);
submissionRepository.save(submission);

Stream.of("applicant1_birth-certificate.jpeg", "applicant1_license.jpeg", "applicant1_pay-stub.png")
.forEach(filename -> {
Expand All @@ -127,6 +173,26 @@ void setup() throws IOException {
throw new IllegalStateException(e);
}
});
return submission;
}

private Submission queueSubmissionWithoutDocs() {
Date submittedDate = new Date(new Date().getTime() - (1000 * 60 * 60) * 2);
Submission submission = Submission.builder()
.submittedAt(submittedDate)
.flow("laDigitalAssister")
.urlParams(new HashMap<>())
.inputData(Map.of(
"firstName", "Tester",
"lastName", "McTest",
"birthDay", "3",
"birthMonth", "12",
"birthYear", "4567",
"signature", "Tester McTest sig"
)).build();
submissionRepository.save(submission);
saveTransmissionRecord(submission);
return submission;
}

private void saveTransmissionRecord(Submission submission) {
Expand All @@ -146,32 +212,6 @@ private UUID saveUserFile(Submission submission, String repoPath, String origina
return docFile.getFileId();
}

@Test
public void transmitZipFile() throws IOException {
submissionTransfer.transferSubmissions();

File zipFile = new File("mocktransmit_00050000000.zip");
assertTrue(zipFile.exists());

verify(ftpsClient).uploadFile(any(), any());

String destDir = "output";
List<String> fileNames = unzip(zipFile.getPath(), destDir);

assertThat(fileNames, hasItem("output/1/"));
assertThat(fileNames, hasItem("output/1/SNAP_application.pdf"));
assertThat(fileNames, hasItem("output/2/"));
assertThat(fileNames, hasItem("output/2/SNAP_application.pdf"));
assertThat(fileNames, hasItem("output/2/originalFilename.png"));
assertThat(fileNames, hasItem("output/2/2_originalFilename.png"));
assertThat(fileNames, hasItem("output/2/weird/:\\filename.jpg"));
assertThat(fileNames, hasItem("output/00050000000.txt"));
assertEquals(8, fileNames.size());

// cleanup
zipFile.delete();
}

private static List<String> unzip(String zipFilePath, String destDir) {
List<String> result = new ArrayList<>();
//buffer for read and write data to file
Expand Down

0 comments on commit fdd90d9

Please sign in to comment.