Skip to content

Commit

Permalink
add wic to transmitter commands and environment variables for deploym…
Browse files Browse the repository at this point in the history 
…ent (#490)
  • Loading branch information
@lkemperman-cfa
lkemperman-cfa committed Jan 3, 2024
1 parent 1fcef29 commit 7aed088
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 55 deletions.
5 changes: 3 additions & 2 deletions .github/workflows/deploy-prod.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@ jobs:
'ENCRYPTION_KEY=${{ secrets.ENCRYPTION_KEY }}' \
'SPRING_DATASOURCE_USERNAME=${{ secrets.SPRING_DATASOURCE_USERNAME }}' \
'SPRING_DATASOURCE_PASSWORD=${{ secrets.SPRING_DATASOURCE_PASSWORD }}' \
'DEFAULT_USER=${{ secrets.DEFAULT_USER }}' \
'DEFAULT_PASSWORD=${{ secrets.DEFAULT_PASSWORD }}' \
'SFTP_USERNAME=${{ secrets.SFTP_USERNAME_PROD }}' \
'SFTP_PASSWORD=${{ secrets.SFTP_PASSWORD_PROD }}' \
'SFTP_UPLOAD_URL=${{ secrets.SFTP_UPLOAD_URL }}' \
'demo=false' \
'RELEASE_HEALTHCHECK_TIMEOUT=900'
deploy_es:
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/deploy-staging.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ jobs:
'ENCRYPTION_KEY=${{ secrets.ENCRYPTION_KEY }}' \
'SPRING_DATASOURCE_USERNAME=${{ secrets.SPRING_DATASOURCE_USERNAME_STAGING }}' \
'SPRING_DATASOURCE_PASSWORD=${{ secrets.SPRING_DATASOURCE_PASSWORD_STAGING }}' \
'SFTP_USERNAME=${{ secrets.SFTP_USERNAME_STAGING }}' \
'SFTP_PASSWORD=${{ secrets.SFTP_PASSWORD_STAGING }}' \
'SFTP_UPLOAD_URL=${{ secrets.SFTP_UPLOAD_URL }}' \
'demo=true' \
'RELEASE_HEALTHCHECK_TIMEOUT=900'
draft-release:
Expand Down
107 changes: 56 additions & 51 deletions src/main/java/org/ladocuploader/app/cli/TransmitterCommands.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ public class TransmitterCommands {

private final String successfulSubmissionKey = "success";

private final List<TransmissionType> transmissionTypes = List.of(TransmissionType.ECE, TransmissionType.WIC);

public TransmitterCommands(TransmissionRepository transmissionRepository,
SftpClient sftpClient, CsvService csvService, CloudFileRepository cloudFileRepository) {
this.transmissionRepository = transmissionRepository;
Expand All @@ -59,22 +61,25 @@ public TransmitterCommands(TransmissionRepository transmissionRepository,
@ShellMethod(key = "transmit")
public void transmit() throws IOException, JSchException, SftpException {
log.info("Finding submissions to transmit...");
for (TransmissionType transmissionType : transmissionTypes) {
var submissions = this.transmissionRepository.submissionsToTransmit(Sort.unsorted(), transmissionType);
log.info("Total submissions to transmit for {} is {}", transmissionType.name(), submissions.size());
if (submissions.size() > 0) {
log.info("Transmitting submissions for {}", transmissionType.name());
transmitBatch(submissions, transmissionType);
} else {
log.info("Skipping transmission for {}", transmissionType.name());
}

var allSubmissions = this.transmissionRepository.submissionsToTransmit(Sort.unsorted(), TransmissionType.ECE);
log.info("Total submissions to transmit for ECE is {}", allSubmissions.size());
if (allSubmissions.size() > 0) {
log.info("Transmitting submissions for ECE");
transmitBatch(allSubmissions, TransmissionType.ECE);
} else {
log.info("Skipping transmission for ECE");
}

}

private void transmitBatch(List<Submission> submissions, TransmissionType transmissionType) throws IOException, JSchException, SftpException{

UUID runId = UUID.randomUUID();
String zipFilename = createZipFilename(transmissionType, runId);
Map<String, Object> zipResults = zipFiles(submissions, zipFilename);
Map<String, Object> zipResults = zipFiles(submissions, zipFilename, transmissionType.getPackageType());

List<UUID> successfullySubmittedIds = (List<UUID>) zipResults.get(successfulSubmissionKey);

Expand All @@ -100,20 +105,20 @@ private void transmitBatch(List<Submission> submissions, TransmissionType transm
log.info("Finished transmission of a batch");

failedSubmissions.forEach((id, errorMessages) -> {
Submission submission = Submission.builder().id(id).build();
Transmission transmission = transmissionRepository.findBySubmissionAndTransmissionType(submission, transmissionType);
transmission.setStatus(TransmissionStatus.Failed);
transmission.setRunId(runId);
transmission.setSubmissionErrors(errorMessages);
transmissionRepository.save(transmission);
Submission submission = Submission.builder().id(id).build();
Transmission transmission = transmissionRepository.findBySubmissionAndTransmissionType(submission, transmissionType);
transmission.setStatus(TransmissionStatus.Failed);
transmission.setRunId(runId);
transmission.setSubmissionErrors(errorMessages);
transmissionRepository.save(transmission);
}
);
failedDocumentation.forEach((id, errorMessages) -> {
Submission submission = Submission.builder().id(id).build();
Transmission transmission = transmissionRepository.findBySubmissionAndTransmissionType(submission, transmissionType);
transmission.setDocumentationErrors(errorMessages);
transmissionRepository.save(transmission);
}
Submission submission = Submission.builder().id(id).build();
Transmission transmission = transmissionRepository.findBySubmissionAndTransmissionType(submission, transmissionType);
transmission.setDocumentationErrors(errorMessages);
transmissionRepository.save(transmission);
}
);


Expand Down Expand Up @@ -149,61 +154,61 @@ private void addZipEntries(CsvPackage csvPackage, ZipOutputStream zipOutput){
);
}

private Map<String, Object> zipFiles(List<Submission> submissions, String zipFileName) throws IOException {
private Map<String, Object> zipFiles(List<Submission> submissions, String zipFileName, CsvPackageType packageType) throws IOException {
Map<String, Object> results = new HashMap<>();
List<UUID> successfullySubmittedIds = new ArrayList<>(submissions.stream()
.map(Submission::getId)
.toList());

try (FileOutputStream baos = new FileOutputStream(zipFileName);
ZipOutputStream zos = new ZipOutputStream(baos)) {
CsvPackage ecePackage = csvService.generateCsvPackage(submissions, CsvPackageType.ECE_PACKAGE);
Map<UUID, Map<CsvType, String>> submissionErrors = ecePackage.getErrorMessages();
CsvPackage csvPackage = csvService.generateCsvPackage(submissions, packageType);
Map<UUID, Map<CsvType, String>> submissionErrors = csvPackage.getErrorMessages();
Map<UUID, Map<UUID, String>> documentationErrors = new HashMap<>();

submissionErrors.forEach((submissionId, submissionErrorMessages) -> {
successfullySubmittedIds.remove(submissionId);
successfullySubmittedIds.remove(submissionId);
}
);

addZipEntries(ecePackage, zos);
addZipEntries(csvPackage, zos);
submissions.forEach(submission -> {
List<UserFile> userFiles = transmissionRepository.userFilesBySubmission(submission);
log.info("Found " + userFiles.size() + " files associated with app.");
if (userFiles.size() == 0){
return;
}
int fileCount = 0;
String subfolder = submission.getId() + "_files/";
String subfolder = submission.getId() + "_files/";
try {
zos.putNextEntry(new ZipEntry(subfolder));
} catch (Exception e){
log.error("Error generating file collection for submission ID {}", submission.getId(), e);
}
Map<UUID, String> submissionDocumentationErrors = new HashMap<>();
for (UserFile userFile: userFiles) {
try {
zos.putNextEntry(new ZipEntry(subfolder));
} catch (Exception e){
log.error("Error generating file collection for submission ID {}", submission.getId(), e);
}
Map<UUID, String> submissionDocumentationErrors = new HashMap<>();
for (UserFile userFile: userFiles) {
try {
fileCount += 1;
String fileName = userFile.getOriginalName();
log.info("filename is: " + fileName);
ZipEntry docEntry = new ZipEntry(subfolder + String.format("%02d", fileCount) + "_" + userFile.getOriginalName().replaceAll("[/:\\\\]", "_"));
docEntry.setSize(userFile.getFilesize().longValue());
zos.putNextEntry(docEntry);
CloudFile docFile = fileRepository.get(userFile.getRepositoryPath());
byte[] bytes = new byte[Math.toIntExact(docFile.getFileSize())];

try (ByteArrayInputStream fis = new ByteArrayInputStream(docFile.getFileBytes())) {
fis.read(bytes);
zos.write(bytes);
}
zos.closeEntry();
File file = new File(userFile.getRepositoryPath());
file.delete(); // delete after download and added to zipfile
} catch (Exception e) {
submissionDocumentationErrors.put(userFile.getFileId(), e.getMessage());
log.error("Error generating file collection for submission ID {}", submission.getId(), e);
fileCount += 1;
String fileName = userFile.getOriginalName();
log.info("filename is: " + fileName);
ZipEntry docEntry = new ZipEntry(subfolder + String.format("%02d", fileCount) + "_" + userFile.getOriginalName().replaceAll("[/:\\\\]", "_"));
docEntry.setSize(userFile.getFilesize().longValue());
zos.putNextEntry(docEntry);
CloudFile docFile = fileRepository.get(userFile.getRepositoryPath());
byte[] bytes = new byte[Math.toIntExact(docFile.getFileSize())];

try (ByteArrayInputStream fis = new ByteArrayInputStream(docFile.getFileBytes())) {
fis.read(bytes);
zos.write(bytes);
}
zos.closeEntry();
File file = new File(userFile.getRepositoryPath());
file.delete(); // delete after download and added to zipfile
} catch (Exception e) {
submissionDocumentationErrors.put(userFile.getFileId(), e.getMessage());
log.error("Error generating file collection for submission ID {}", submission.getId(), e);
}
}
documentationErrors.put(submission.getId(), submissionDocumentationErrors);

});
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/org/ladocuploader/app/csv/enums/CsvPackageType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,12 @@
import static org.ladocuploader.app.csv.enums.CsvType.WIC_APPLICATION;

import java.util.List;
import org.ladocuploader.app.csv.CsvService;

public enum CsvPackageType {

// TODO: abstract environment
ECE_PACKAGE (List.of(PARENT_GUARDIAN, STUDENT, RELATIONSHIP, ECE_APPLICATION), "/la-du-moveit-transfer/nola-ps"),
WIC_PACKAGE (List.of(WIC_APPLICATION), "/la-du-moveit-transfer/wic");
WIC_PACKAGE (List.of(WIC_APPLICATION), "/la-du-moveit-transfer/dcfs");

private final List<CsvType> csvTypeList;

Expand Down

0 comments on commit 7aed088

Please sign in to comment.