LNK-1871: Some refactors to the FileSystemSender. Also, updates to an…

api/src/main/java/com/lantanagroup/link/api/controller/ValidationController.java

@@ -49,7 +49,6 @@ public class ValidationController extends BaseController {
   /**
    * Validates a Bundle provided in the request body
    *
-   * @param tenantId The id of the tenant
    * @param severity The minimum severity level to report on
    * @return Returns an OperationOutcome resource that provides details about each of the issues found
    */
@@ -73,7 +72,6 @@ public OperationOutcome validate(@RequestBody Bundle bundle, @RequestParam(defau
   /**
    * Validates a Bundle provided in the request body
    *
-   * @param tenantId The id of the tenant
    * @param severity The minimum severity level to report on
    * @return Returns an OperationOutcome resource that provides details about each of the issues found
    */
@@ -92,7 +90,9 @@ public String validateSummary(@RequestBody Bundle bundle, @RequestParam(defaultV
    * @throws IOException
    */
   @GetMapping("/{tenantId}/{reportId}")
-  public OperationOutcome getValidationIssuesForReport(@PathVariable String tenantId, @PathVariable String reportId, @RequestParam(defaultValue = "INFORMATION") OperationOutcome.IssueSeverity severity, @RequestParam(required = false) String code) {
+  public OperationOutcome getValidationIssuesForReport(@PathVariable String tenantId, @PathVariable String reportId,
+                                                       @RequestParam(defaultValue = "INFORMATION") OperationOutcome.IssueSeverity severity,
+                                                       @RequestParam(required = false) String code) {
     TenantService tenantService = TenantService.create(this.sharedService, tenantId);
 
     if (tenantService == null) {

core/src/main/java/com/lantanagroup/link/ApplyConceptMaps.java

@@ -31,10 +31,6 @@ public ApplyConceptMaps() {
     validationSupport.fetchAllStructureDefinitions();
   }
 
-  private FHIRPathEngine getFhirPathEngine() {
-    HapiWorkerContext workerContext = new HapiWorkerContext(FhirContextProvider.getFhirContext(), validationSupport);
-    return new FHIRPathEngine(workerContext);
-  }
 
   private void translateCoding(ConceptMap map, Coding code) {
     map.getGroup().stream().forEach((ConceptMap.ConceptMapGroupComponent group) -> {
@@ -81,7 +77,7 @@ public List<Base> filterResourcesByPathList(DomainResource resource, List<String
     List<Base> results = new ArrayList<>();
     // logger.debug(String.format("FindCodings for resource %s based on path %s", resource.getResourceType() + "/" + resource.getIdElement().getIdPart(), List.of(pathList)));
     pathList.stream().forEach(path -> {
-      results.addAll(getFhirPathEngine().evaluate(resource, path));
+      results.addAll(FhirHelper.getFhirPathEngine().evaluate(resource, path));
     });
     return results;
   }

core/src/main/java/com/lantanagroup/link/Constants.java

@@ -61,4 +61,9 @@ public class Constants {
   public static final String VALIDATION_ISSUE_TASK = "Validation";
   public static final String VALIDATION_ISSUE_CATEGORY = "Validation Issues";
   public static final String REPORT_GENERATION_TASK = "Report Generation";
+
+  //Submission file names
+  public static final String ORGANIZATION_FILE_NAME = "organization.json";
+  public static final String DEVICE_FILE_NAME = "device.json";
+  public static final String QUERY_PLAN_FILE_NAME = "query-plan.yml";
 }

core/src/main/java/com/lantanagroup/link/FhirBundleProcessor.java

@@ -33,21 +33,35 @@ public class FhirBundleProcessor {
   @Getter
   private final List<Bundle.BundleEntryComponent> otherResources = new ArrayList<>();
 
+  @Getter
+  private final HashMap<Integer, String> bundleEntryIndexToFileMap = new HashMap<>();
+
   public FhirBundleProcessor(Bundle bundle) {
     this.bundle = bundle;
 
     this.linkOrganization = this.bundle.getEntry().stream()
             .filter(e -> {
-              return e.getResource().getResourceType().equals(ResourceType.Organization) &&
-                      e.getResource().getMeta().hasProfile(Constants.SubmittingOrganizationProfile);
+              if(e.getResource().getResourceType().equals(ResourceType.Organization) &&
+                      e.getResource().getMeta().hasProfile(Constants.SubmittingOrganizationProfile)){
+                int index = bundle.getEntry().indexOf(e);
+                bundleEntryIndexToFileMap.put(index, Constants.ORGANIZATION_FILE_NAME);
+                return true;
+              }
+              else return false;
             })
             .findFirst()
             .orElse(null);
 
     this.linkDevice = this.bundle.getEntry().stream()
             .filter(e -> {
-              return e.getResource().getResourceType().equals(ResourceType.Device) &&
-                      e.getResource().getMeta().hasProfile(Constants.SubmittingDeviceProfile);
+
+              if(e.getResource().getResourceType().equals(ResourceType.Device) &&
+                      e.getResource().getMeta().hasProfile(Constants.SubmittingDeviceProfile)){
+                int index = bundle.getEntry().indexOf(e);
+                bundleEntryIndexToFileMap.put(index, Constants.DEVICE_FILE_NAME);
+                return true;
+              }
+              else return false;
             })
             .findFirst()
             .orElse(null);
@@ -56,35 +70,59 @@ public FhirBundleProcessor(Bundle bundle) {
             .filter(e -> e.getResource().getResourceType().equals(ResourceType.Library))
             .filter(e -> {
               Library library = (Library) e.getResource();
-              return library.getType().getCoding().stream()
-                      .anyMatch(c -> c.getCode().equals(Constants.LibraryTypeModelDefinitionCode));
+              if(library.getType().getCoding().stream()
+                      .anyMatch(c -> c.getCode().equals(Constants.LibraryTypeModelDefinitionCode))){
+                int index = bundle.getEntry().indexOf(e);
+                bundleEntryIndexToFileMap.put(index, Constants.QUERY_PLAN_FILE_NAME);
+                return true;
+              }
+              else return false;
             })
             .findFirst()
             .orElse(null);
 
     this.linkCensusLists = this.bundle.getEntry().stream()
-            .filter(e ->
-                    e.getResource().getResourceType().equals(ResourceType.List) &&
-                            e.getResource().getMeta().getProfile().stream()
-                                    .anyMatch(p -> p.getValue().equals(Constants.CensusProfileUrl)))
+            .filter(e -> {
+                      if(e.getResource().getResourceType().equals(ResourceType.List) &&
+                      e.getResource().getMeta().getProfile().stream()
+                              .anyMatch(p -> p.getValue().equals(Constants.CensusProfileUrl))){
+                        int index = bundle.getEntry().indexOf(e);
+                        bundleEntryIndexToFileMap.put(index, String.format("census-%s.json",
+                                e.getResource().getIdElement().getIdPart()));
+                        return true;
+                      }
+                      else return false;
+                    })
             .sorted(new FhirBundlerEntrySorter.ResourceComparator())
             .collect(Collectors.toList());
 
     this.aggregateMeasureReports = this.bundle.getEntry().stream()
-            .filter(e -> e.getResource().getResourceType().equals(ResourceType.MeasureReport) && ((MeasureReport) e.getResource()).getType().equals(MeasureReport.MeasureReportType.SUBJECTLIST))
+            .filter(e -> {
+              if(e.getResource().getResourceType().equals(ResourceType.MeasureReport)
+                      && ((MeasureReport) e.getResource()).getType().equals(MeasureReport.MeasureReportType.SUBJECTLIST)){
+                int index = bundle.getEntry().indexOf(e);
+                bundleEntryIndexToFileMap.put(index, String.format("aggregate-%s.json",
+                        e.getResource().getIdElement().getIdPart()));
+                return true;
+              }
+              else return false;
+            })
             .sorted(new FhirBundlerEntrySorter.ResourceComparator())
             .collect(Collectors.toList());
 
     for (Bundle.BundleEntryComponent e : bundle.getEntry()) {
       String patientReference = FhirHelper.getPatientReference(e.getResource());
+      int index = this.bundle.getEntry().indexOf(e);
       if (patientReference != null) {
         String patientId = patientReference.replace("Patient/", "");
         if (!this.patientResources.containsKey(patientId)) {
           this.patientResources.put(patientId, new ArrayList<>());
         }
         this.patientResources.get(patientId).add(e);
+        bundleEntryIndexToFileMap.put(index, String.format("patient-%s.json", patientId));
       } else if (isOtherResource(e)) {
         this.otherResources.add(e);
+        bundleEntryIndexToFileMap.put(index, "other-resources.json");
       }
     }
   }

core/src/main/java/com/lantanagroup/link/FhirHelper.java

@@ -1,5 +1,6 @@
 package com.lantanagroup.link;
 
+import ca.uhn.fhir.context.support.DefaultProfileValidationSupport;
 import ca.uhn.fhir.parser.IParser;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.common.base.Strings;
@@ -15,7 +16,9 @@
 import jakarta.servlet.http.HttpServletRequest;
 import org.apache.commons.lang3.StringUtils;
 import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.r4.hapi.ctx.HapiWorkerContext;
 import org.hl7.fhir.r4.model.*;
+import org.hl7.fhir.r4.utils.FHIRPathEngine;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,6 +33,9 @@
 public class FhirHelper {
   private static final Logger logger = LoggerFactory.getLogger(FhirHelper.class);
 
+  private static final DefaultProfileValidationSupport validationSupport =
+          new DefaultProfileValidationSupport(FhirContextProvider.getFhirContext());
+
   public static org.hl7.fhir.r4.model.Address getFHIRAddress(Address address) {
     org.hl7.fhir.r4.model.Address ret = new org.hl7.fhir.r4.model.Address();
 
@@ -339,6 +345,11 @@ private static void addEventNotesToDevice(Device device, String eventCategory, L
     }
   }
 
+  public static FHIRPathEngine getFhirPathEngine() {
+    HapiWorkerContext workerContext = new HapiWorkerContext(FhirContextProvider.getFhirContext(), validationSupport);
+    return new FHIRPathEngine(workerContext);
+  }
+
   /**
    * Returns patient's reference of the related resource
    *

core/src/main/java/com/lantanagroup/link/db/TenantService.java

@@ -379,4 +379,8 @@ public Integer countUncategorizedValidationResults(String reportId) {
   public List<ValidationResult> getUncategorizedValidationResults(String reportId) {
     return this.validations.getUncategorized(reportId);
   }
+
+  public String getOrganizationID(){
+    return this.config.getCdcOrgId();
+  }
 }

core/src/main/java/com/lantanagroup/link/sender/FileSystemSender.java

@@ -2,16 +2,15 @@
 
 import ca.uhn.fhir.parser.IParser;
 import com.lantanagroup.link.*;
+import com.lantanagroup.link.Constants;
 import com.lantanagroup.link.auth.LinkCredentials;
 import com.lantanagroup.link.config.sender.FileSystemSenderConfig;
 import com.lantanagroup.link.db.TenantService;
 import com.lantanagroup.link.db.model.Report;
 import lombok.Setter;
 import org.apache.commons.lang3.StringUtils;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
-import org.hl7.fhir.r4.model.Bundle;
-import org.hl7.fhir.r4.model.Library;
-import org.hl7.fhir.r4.model.Resource;
+import org.hl7.fhir.r4.model.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,15 +21,16 @@
 import javax.crypto.NoSuchPaddingException;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.SecretKeySpec;
+
 import jakarta.servlet.http.HttpServletRequest;
+
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.security.*;
 import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.Date;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import static com.google.common.primitives.Bytes.concat;
@@ -52,7 +52,7 @@ private FileSystemSenderConfig.Formats getFormat() {
     return this.config.getFormat();
   }
 
-  public Path getFilePath() {
+  public Path getFilePath(String type) {
     String suffix = "";
     String path;
 
@@ -74,7 +74,7 @@ public Path getFilePath() {
       }
     }
 
-    String fileName = "submission-" + (new SimpleDateFormat("yyyyMMdd'T'HHmmss").format(new Date())) + suffix;
+    String fileName = type + "-" + (new SimpleDateFormat("yyyyMMdd'T'HHmmss").format(new Date())) + suffix;
 
     return Paths.get(path, fileName);
   }
@@ -154,7 +154,7 @@ private void saveToFile(Resource resource, String path) throws Exception {
     logger.info("Saved submission bundle to file system: {}", path);
   }
 
-  private void saveToFolder(Bundle bundle, String path) throws Exception {
+  private void saveToFolder(Bundle bundle, OperationOutcome outcome, String path) throws Exception {
     File folder = new File(path);
 
     if (!folder.exists() && !folder.mkdirs()) {
@@ -167,39 +167,47 @@ private void saveToFolder(Bundle bundle, String path) throws Exception {
     // Save link resources
     logger.debug("Saving link resources");
     if (fhirBundleProcessor.getLinkOrganization() != null) {
-      this.saveToFile(fhirBundleProcessor.getLinkOrganization().getResource(), Paths.get(path, "organization.json").toString());
+      this.saveToFile(fhirBundleProcessor.getLinkOrganization().getResource(),
+              Paths.get(path, Constants.ORGANIZATION_FILE_NAME).toString());
     }
 
     if (fhirBundleProcessor.getLinkDevice() != null) {
-      this.saveToFile(fhirBundleProcessor.getLinkDevice().getResource(), Paths.get(path, "device.json").toString());
+      this.saveToFile(fhirBundleProcessor.getLinkDevice().getResource(),
+              Paths.get(path, Constants.DEVICE_FILE_NAME).toString());
     }
 
     if (fhirBundleProcessor.getLinkQueryPlanLibrary() != null) {
       Library library = (Library) fhirBundleProcessor.getLinkQueryPlanLibrary().getResource();
-      this.saveToFile(library.getContentFirstRep().getData(), Paths.get(path, "query-plan.yml").toString());
+      this.saveToFile(library.getContentFirstRep().getData(),
+              Paths.get(path, Constants.QUERY_PLAN_FILE_NAME).toString());
     }
 
     // Save aggregate measure reports
     logger.debug("Saving aggregate measure reports");
-    if (!fhirBundleProcessor.getAggregateMeasureReports().isEmpty()) {
-      for (int i = 0; i < fhirBundleProcessor.getAggregateMeasureReports().size(); i++) {
-        this.saveToFile(fhirBundleProcessor.getAggregateMeasureReports().get(i).getResource(), Paths.get(path, String.format("aggregate-%d.json", i + 1)).toString());
+    List<Bundle.BundleEntryComponent> aggregates = fhirBundleProcessor.getAggregateMeasureReports();
+    if (aggregates != null && !aggregates.isEmpty()) {
+      for (Bundle.BundleEntryComponent aggregate : aggregates) {
+        Resource aggregateReport = aggregate.getResource();
+        this.saveToFile(aggregateReport, Paths.get(path, String.format("aggregate-%s.json", aggregateReport.getIdElement().getIdPart())).toString());
       }
     }
 
     // Save census lists
     logger.debug("Saving census lists");
-    if (fhirBundleProcessor.getLinkCensusLists() != null && !fhirBundleProcessor.getLinkCensusLists().isEmpty()) {
-      for (int i = 0; i < fhirBundleProcessor.getLinkCensusLists().size(); i++) {
-        this.saveToFile(fhirBundleProcessor.getLinkCensusLists().get(i).getResource(), Paths.get(path, String.format("census-%d.json", i + 1)).toString());
+    List<Bundle.BundleEntryComponent> lists = fhirBundleProcessor.getLinkCensusLists();
+    if (lists != null && !lists.isEmpty()) {
+      for (Bundle.BundleEntryComponent entry : lists) {
+        Resource list = entry.getResource();
+        this.saveToFile(list,
+                Paths.get(path, String.format("census-%s.json", list.getIdElement().getIdPart())).toString());
       }
     }
 
     // Save patient resources
     logger.debug("Saving patient resources as patient bundles");
-
-    if (!fhirBundleProcessor.getPatientResources().isEmpty()) {
-      for (String patientId : fhirBundleProcessor.getPatientResources().keySet()) {
+    Set<String> patientIds = fhirBundleProcessor.getPatientResources().keySet();
+    if (!patientIds.isEmpty()) {
+      for (String patientId : patientIds) {
         Bundle patientBundle = new Bundle();
         patientBundle.setType(Bundle.BundleType.COLLECTION);
         patientBundle.getEntry().addAll(fhirBundleProcessor.getPatientResources().get(patientId));
@@ -217,6 +225,19 @@ private void saveToFolder(Bundle bundle, String path) throws Exception {
     if (otherResourcesBundle.hasEntry()) {
       this.saveToFile(otherResourcesBundle, Paths.get(path, "other-resources.json").toString());
     }
+
+    // Annotating validation results with what file each issue occurs in and saving
+    logger.debug("Annotating and saving validation results");
+    List<OperationOutcome.OperationOutcomeIssueComponent> issues = outcome.getIssue();
+    issues.forEach(i -> {
+      String expression = i.getExpression().get(0).toString();
+      if(expression.contains("Bundle.entry[")){
+        String entryIndex = expression.substring(expression.indexOf("Bundle.entry[") + 13, expression.indexOf("]"));
+        i.setDiagnostics(fhirBundleProcessor.getBundleEntryIndexToFileMap().get(Integer.parseInt(entryIndex)));
+      }
+    });
+    this.saveToFile(outcome, Paths.get(path, "validation-results.json").toString());
+
   }
 
   @SuppressWarnings("unused")
@@ -232,12 +253,18 @@ public void send(TenantService tenantService, Bundle submissionBundle, Report re
             this.config.getFormat(),
             StringUtils.isEmpty(this.config.getEncryptSecret()) ? "without" : "with");
 
-    String path = this.getFilePath().toString();
+
+    OperationOutcome outcome =
+            tenantService.getValidationResultsOperationOutcome(report.getId(), OperationOutcome.IssueSeverity.INFORMATION, null);
 
     if (this.config.getIsBundle()) {
-      this.saveToFile(submissionBundle, path);
+      this.saveToFile(submissionBundle, this.getFilePath("submission").toString());
+      this.saveToFile(outcome, this.getFilePath("validation").toString());
     } else {
-      this.saveToFolder(submissionBundle, path);
+      String orgId = tenantService.getOrganizationID();
+      this.saveToFolder(submissionBundle, outcome,
+              (orgId != null && !orgId.isEmpty() ? this.getFilePath(orgId).toString()
+                      : this.getFilePath("submission").toString()));
     }
   }
 }

core/src/test/java/com/lantanagroup/link/sender/FileSystemSenderTests.java

@@ -13,7 +13,7 @@ public class FileSystemSenderTests {
   @Test
   public void getFilePathTest_NoConfigPath() {
     FileSystemSender sender = new FileSystemSender();
-    Assert.assertThrows(IllegalArgumentException.class, () -> { sender.getFilePath(); });
+    Assert.assertThrows(IllegalArgumentException.class, () -> { sender.getFilePath("submission"); });
   }
 
   @Test
@@ -23,7 +23,7 @@ public void getFilePathTest_ConfigPath() {
     FileSystemSender sender = new FileSystemSender();
     sender.setConfig(config);
 
-    Path path = sender.getFilePath();
+    Path path = sender.getFilePath("submission");
     Assert.assertNotNull(path);
     Assert.assertTrue(path.toString().startsWith(config.getPath()));
   }