Ftps transfers for SNAP application and docs

build.gradle

@@ -50,6 +50,11 @@ dependencies {
     }
 
     implementation 'com.amazonaws:aws-encryption-sdk-java:3.0.0'
+    implementation 'org.bouncycastle:bcpg-jdk15on:1.70'
+    implementation 'org.bouncycastle:bcpkix-jdk15on:1.70'
+    implementation 'org.springframework.shell:spring-shell-starter:3.0.4'
+
+    implementation 'commons-net:commons-net:3.10.0'
     testImplementation 'org.projectlombok:lombok:1.18.30'
 
     implementation 'io.sentry:sentry-spring-boot-starter-jakarta:7.0.0'

src/main/java/org/ladocuploader/app/cli/FtpsClient.java

@@ -0,0 +1,8 @@
+package org.ladocuploader.app.cli;
+
+import java.io.IOException;
+
+public interface FtpsClient {
+
+  void uploadFile(String zipFilename, byte[] data) throws IOException;
+}

src/main/java/org/ladocuploader/app/cli/FtpsClientImpl.java

@@ -0,0 +1,62 @@
+package org.ladocuploader.app.cli;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.net.PrintCommandListener;
+import org.apache.commons.net.ftp.FTPReply;
+import org.apache.commons.net.ftp.FTPSClient;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Component;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+
+@Slf4j
+@Component
+@Profile("production")
+public class FtpsClientImpl implements FtpsClient {
+
+  private final String username;
+  private final String password;
+  private final String uploadUrl;
+  private final String uploadDir;
+
+  public FtpsClientImpl(@Value("${ftps.username:}") String username, @Value("${ftps.password:}") String password, @Value("${ftps.upload-url:}") String uploadUrl, @Value("${ftps.upload-dir:}") String uploadDir) {
+    this.username = username;
+    this.password = password;
+    this.uploadUrl = uploadUrl;
+    this.uploadDir = uploadDir;
+  }
+
+  @Override
+  public void uploadFile(String zipFilename, byte[] data) throws IOException {
+    FTPSClient ftp = new FTPSClient();
+
+    ftp.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(System.out)));
+
+    ftp.connect(uploadUrl);
+    int reply = ftp.getReplyCode();
+    if (!FTPReply.isPositiveCompletion(reply)) {
+      ftp.disconnect();
+      throw new IOException("Exception in connecting to FTP Server");
+    }
+
+    ftp.login(username, password);
+
+    ftp.changeWorkingDirectory(uploadDir);
+
+    ftp.execPBSZ(0);
+    ftp.execPROT("P");
+    ftp.enterLocalPassiveMode();
+    ftp.pasv();
+    InputStream local = new ByteArrayInputStream(data);
+    ftp.storeFile(zipFilename, local);
+    local.close();
+    ftp.completePendingCommand();
+
+    ftp.logout();
+    ftp.disconnect();
+  }
+}

src/main/java/org/ladocuploader/app/cli/MockFtpsClientImpl.java

@@ -0,0 +1,17 @@
+package org.ladocuploader.app.cli;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+@Profile("!production")
+public class MockFtpsClientImpl implements FtpsClient {
+
+  @Override
+  public void uploadFile(String zipFilename, byte[] data) {
+    // Do nothing
+    log.info("Mock uploading file " + zipFilename);
+  }
+}

src/main/java/org/ladocuploader/app/cli/PGPEncryptor.java

@@ -0,0 +1,157 @@
+package org.ladocuploader.app.cli;
+
+import lombok.extern.slf4j.Slf4j;
+import org.bouncycastle.bcpg.CompressionAlgorithmTags;
+import org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags;
+import org.bouncycastle.openpgp.*;
+import org.bouncycastle.openpgp.jcajce.JcaPGPPublicKeyRingCollection;
+import org.bouncycastle.openpgp.operator.KeyFingerPrintCalculator;
+import org.bouncycastle.openpgp.operator.jcajce.*;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.io.*;
+import java.security.SecureRandom;
+import java.util.Date;
+import java.util.Iterator;
+
+@Slf4j
+@Component
+public class PGPEncryptor {
+
+  private final String pubkeyFilename;
+  private final String sigkeyFilename;
+  private final String sigkeyPassword;
+
+  public PGPEncryptor(@Value("${pgp.pubkey_filename:}") String pubkeyFilename, @Value("${pgp.sigkey_filename:}") String sigkeyFilename, @Value("${pgp.sigkey_password:}") String sigkeyPassword) {
+    this.pubkeyFilename = pubkeyFilename;
+    this.sigkeyFilename = sigkeyFilename;
+    this.sigkeyPassword = sigkeyPassword;
+  }
+
+  public byte[] signAndEncryptPayload(String filename) throws IOException {
+    FileInputStream instream = new FileInputStream(filename);
+
+    log.info("Retrieving keys for signing and encryption");
+    PGPSecretKey signingKey = getSecretKey();
+    PGPPublicKey pubKey = getPublicKey();
+
+    ByteArrayOutputStream outstream = new ByteArrayOutputStream();
+    try {
+      log.info("Signing and encrypting payload");
+      return signAndEncryptPayload(instream, signingKey, pubKey, outstream);
+    } catch (PGPException e) {
+      throw new IllegalStateException("There was an issue signing and encrypting the file", e);
+    } finally {
+      instream.close();
+      outstream.close();
+      log.info("Completed signing and encrypting payload");
+    }
+  }
+
+  private PGPPublicKey getPublicKey() throws IOException {
+    PGPPublicKey pubKey = null;
+    InputStream inputStream = new FileInputStream(pubkeyFilename);
+    inputStream = PGPUtil.getDecoderStream(inputStream);
+    try {
+      JcaPGPPublicKeyRingCollection ringCollection = new JcaPGPPublicKeyRingCollection(inputStream);
+      Iterator<PGPPublicKeyRing> keyRingsIterator = ringCollection.getKeyRings();
+      while (keyRingsIterator.hasNext()) {
+        PGPPublicKeyRing pgpPublicKeyRing = keyRingsIterator.next();
+        Iterator<PGPPublicKey> pubKeysIterator = pgpPublicKeyRing.getPublicKeys();
+        while (pubKeysIterator.hasNext()) {
+          pubKey = pubKeysIterator.next();
+        }
+      }
+    } catch (PGPException e) {
+      throw new IllegalArgumentException("Invalid public key");
+    } finally {
+      inputStream.close();
+    }
+    return pubKey;
+  }
+
+  private PGPSecretKey getSecretKey() throws IOException {
+    InputStream inputStream = new FileInputStream(sigkeyFilename);
+    inputStream = PGPUtil.getDecoderStream(inputStream);
+    PGPSecretKeyRingCollection pgpSec;
+
+    KeyFingerPrintCalculator fpCalculator = new JcaKeyFingerprintCalculator();
+    try {
+      pgpSec = new PGPSecretKeyRingCollection(inputStream, fpCalculator);
+    } catch (PGPException e) {
+      throw new IllegalArgumentException("Invalid signing key", e);
+    }
+
+    Iterator<PGPSecretKeyRing> keyRingIter = pgpSec.getKeyRings();
+    while (keyRingIter.hasNext()) {
+      PGPSecretKeyRing keyRing = keyRingIter.next();
+
+      Iterator<PGPSecretKey> keyIter = keyRing.getSecretKeys();
+      while (keyIter.hasNext()) {
+        PGPSecretKey key = keyIter.next();
+
+        if (key.isSigningKey()) {
+          return key;
+        }
+      }
+    }
+
+    throw new IllegalArgumentException("Invalid signing key");
+  }
+
+  private PGPPrivateKey getPrivateKey(PGPSecretKey secretKey) throws PGPException {
+    return secretKey.extractPrivateKey(
+        new JcePBESecretKeyDecryptorBuilder().build(sigkeyPassword.toCharArray()));
+  }
+
+  private byte[] signAndEncryptPayload(InputStream inputStream, PGPSecretKey secKey, PGPPublicKey pubKey,
+                                      ByteArrayOutputStream outputStream) throws PGPException, IOException {
+    int BUFFER_SIZE = 1 << 16;
+
+    // Encryption
+    PGPEncryptedDataGenerator encGen = new PGPEncryptedDataGenerator(
+        new JcePGPDataEncryptorBuilder(SymmetricKeyAlgorithmTags.AES_256)
+            .setWithIntegrityPacket(true)
+            .setSecureRandom(new SecureRandom()));
+    encGen.addMethod(new JcePublicKeyKeyEncryptionMethodGenerator(pubKey));
+    OutputStream encOut = encGen.open(outputStream, new byte[BUFFER_SIZE]);
+
+    // Compression
+    PGPCompressedDataGenerator cGen = new PGPCompressedDataGenerator(CompressionAlgorithmTags.ZIP);
+    OutputStream cOut = cGen.open(encOut);
+
+    // Signing
+    PGPSignatureGenerator sGen = new PGPSignatureGenerator(new JcaPGPContentSignerBuilder(secKey.getPublicKey().getAlgorithm(), PGPUtil.SHA1));
+    sGen.init(PGPSignature.BINARY_DOCUMENT, getPrivateKey(secKey));
+
+    Iterator<String> it = secKey.getPublicKey().getUserIDs();
+    if (it.hasNext()) {
+      PGPSignatureSubpacketGenerator spGen = new PGPSignatureSubpacketGenerator();
+      spGen.addSignerUserID(false, it.next());
+      sGen.setHashedSubpackets(spGen.generate());
+    }
+
+    sGen.generateOnePassVersion(false).encode(cOut);
+
+    // Literal Data generator and output stream
+    byte[] data = inputStream.readAllBytes();
+    PGPLiteralDataGenerator lGen = new PGPLiteralDataGenerator();
+    OutputStream lOut = lGen.open(cOut, PGPLiteralData.BINARY, PGPLiteralData.CONSOLE, data.length, new Date());
+
+    lOut.write(data);
+    sGen.update(data);
+
+    lOut.close();
+    lGen.close();
+
+    sGen.generate().encode(cOut);
+    cOut.close();
+    cGen.close();
+    encGen.close();
+
+    return outputStream.toByteArray();
+  }
+
+}
+

src/main/java/org/ladocuploader/app/cli/SubmissionTransfer.java

@@ -0,0 +1,121 @@
+package org.ladocuploader.app.cli;
+
+import formflow.library.data.Submission;
+import formflow.library.data.SubmissionRepository;
+import formflow.library.data.UserFile;
+import formflow.library.data.UserFileRepositoryService;
+import formflow.library.file.CloudFile;
+import formflow.library.file.CloudFileRepository;
+import formflow.library.pdf.PdfService;
+import lombok.extern.slf4j.Slf4j;
+import org.ladocuploader.app.submission.StringEncryptor;
+import org.springframework.shell.standard.ShellComponent;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+@Slf4j
+@ShellComponent
+public class SubmissionTransfer {
+  private final SubmissionRepository submissionRepository;
+  private final UserFileRepositoryService fileRepositoryService;
+  private final CloudFileRepository fileRepository;
+  private final PdfService pdfService;
+  private final PGPEncryptor pgpEncryptor;
+
+  private final StringEncryptor encryptor;
+  private final FtpsClient ftpsClient;
+
+  public SubmissionTransfer(SubmissionRepository submissionRepository, UserFileRepositoryService fileRepositoryService, CloudFileRepository fileRepository, PdfService pdfService, PGPEncryptor pgpEncryptor, StringEncryptor encryptor, FtpsClient ftpsClient) {
+    this.submissionRepository = submissionRepository;
+    this.fileRepositoryService = fileRepositoryService;
+    this.fileRepository = fileRepository;
+    this.pdfService = pdfService;
+    this.pgpEncryptor = pgpEncryptor;
+    this.encryptor = encryptor;
+    this.ftpsClient = ftpsClient;
+  }
+
+  public void transferSubmissions() {
+    // TODO get submissions to transfer
+
+    String batchIndex = "5000"; // TODO Prob needs a counter
+    String zipFileName = batchIndex + ".zip";
+
+    log.info(String.format("Beginning transfer of batch %s", batchIndex));
+
+    // for each, add to zipfile
+    List<Submission> submissionsBatch = new ArrayList<>(); // TODO Get from transmission table
+    try (FileOutputStream docMeta = new FileOutputStream(batchIndex + ".txt");
+         FileOutputStream baos = new FileOutputStream(zipFileName);
+         ZipOutputStream zos = new ZipOutputStream(baos)) {
+      for (Submission submission : submissionsBatch) {
+        String subfolder = "1"; // TODO Something unique
+        try {
+          if ("laDigitalAssister".equals(submission.getFlow())) {
+            log.info("Generate applicant summary");
+            byte[] file = pdfService.getFilledOutPDF(submission); // TODO Handle generate-pdf crashes
+            String fileName = String.format("%s_application.pdf", subfolder);
+            zos.putNextEntry(new ZipEntry(subfolder));
+            ZipEntry entry = new ZipEntry(subfolder + fileName);
+            entry.setSize(file.length);
+            zos.putNextEntry(entry);
+            zos.write(file);
+            zos.closeEntry();
+            byte[] metaEntry = generateMetaDataEntry(batchIndex, subfolder, fileName, "APP-OFS 4APP", submission);
+            docMeta.write(metaEntry);
+
+            log.info("Adding uploaded docs");
+            List<UserFile> userFiles = fileRepositoryService.findAllBySubmission(submission);
+            for (UserFile userFile : userFiles) {
+              String docUploadFilename = "somethingunique"; // TODO this can be anything
+              ZipEntry docEntry = new ZipEntry(subfolder + docUploadFilename);
+              docEntry.setSize(userFile.getFilesize().longValue());
+              zos.putNextEntry(docEntry);
+
+              CloudFile docFile = fileRepository.get(userFile.getRepositoryPath());
+              zos.write(docFile.getFileBytes());
+              zos.closeEntry();
+
+              // write doc metadata - TODO add docType
+              metaEntry = generateMetaDataEntry(batchIndex, userFile.getOriginalName(), "", "", submission);
+              docMeta.write(metaEntry);
+            }
+          }
+        } catch (Exception e) {
+          log.error("Error generating file collection for submission ID {}", submission.getId(), e);
+        }
+        zos.close();
+        baos.close();
+        docMeta.close();
+
+        // Encrypt and transfer
+        byte[] data = pgpEncryptor.signAndEncryptPayload(zipFileName);
+        ftpsClient.uploadFile(zipFileName, data);
+      }
+    } catch (IOException ex) {
+      throw new IllegalStateException(ex);
+    }
+
+    // TODO - include some stats, how many transfered, failed, successful
+    log.info(String.format("Completed transfer of batch %s", batchIndex));
+  }
+
+  private byte[] generateMetaDataEntry(String bactchIndex, String subfolder, String filename, String documentType, Submission submission) {
+    String metaEntry = String.format("\"%s\",", bactchIndex) +
+        String.format("\"%s\",", filename) + // TODO remove file extension
+        String.format("\"%s\",", documentType) +
+        String.format("\"%s\",", submission.getInputData().getOrDefault("firstName", "")) +
+        String.format("\"%s\",", submission.getInputData().getOrDefault("lastName", "")) +
+        String.format("\"%s\",", submission.getInputData().getOrDefault("ssns", "")) + // TODO decrypt
+        String.format("\"%s\",", submission.getInputData().getOrDefault("birthdate", "")) + // TODO format
+        String.format("\"%s\",", submission.getSubmittedAt()) + // TODO format
+        String.format("\"%s/%s\",", bactchIndex, subfolder, filename);
+    return metaEntry.getBytes();
+  }
+
+}

src/main/resources/application.yaml

@@ -89,6 +89,11 @@ server:
   servlet:
     session:
       persistent: true
+ftps:
+  username: ${FTPS_USERNAME:-""}
+  password: ${FTPS_PASSWORD:-""}
+  upload-url: ${FTPS_UPLOAD_URL:-""}
+  upload-dir: ${FTPS_UPLOAD_DIR:-""}
 sentry:
   dsn: ${SENTRY_DSN}
   traces-sample-rate: 0.6