Merge pull request #64 from monarch-initiative/batchConversion

batch convertion to HPO

loinc2hpo-core/src/main/java/org/monarchinitiative/loinc2hpo/exception/LoincCodeNotAnnotatedException.java

@@ -0,0 +1,16 @@
+package org.monarchinitiative.loinc2hpo.exception;
+
+import org.monarchinitiative.loinc2hpo.loinc.LOINC2HpoAnnotation;
+
+public class LoincCodeNotAnnotatedException extends Loinc2HpoException{
+
+    public LoincCodeNotAnnotatedException () {
+
+    }
+
+    public LoincCodeNotAnnotatedException (String msg) {
+
+        super(msg);
+
+    }
+}

loinc2hpo-core/src/main/java/org/monarchinitiative/loinc2hpo/fhir/FhirObservationAnalyzer.java

@@ -8,9 +8,11 @@
 import org.monarchinitiative.loinc2hpo.loinc.*;
 import org.monarchinitiative.loinc2hpo.testresult.BasicLabTestOutcome;
 import org.monarchinitiative.loinc2hpo.testresult.LabTestOutcome;
+import org.monarchinitiative.phenol.formats.hpo.HpoTerm;
 
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * This class is responsible for analyzing a FHIR observation
@@ -37,112 +39,62 @@ public static void setObservation(Observation aFhirObservation) {
      * @param loincIds
      * @return
      */
-    public static LabTestOutcome getHPO4ObservationOutcome(HashSet<LoincId> loincIds, Map<LoincId, LOINC2HpoAnnotationImpl> loinc2HPOannotationMap) {
+    public static LabTestOutcome getHPO4ObservationOutcome(Set<LoincId> loincIds, Map<LoincId, LOINC2HpoAnnotationImpl> loinc2HPOannotationMap) throws MalformedLoincCodeException, UnsupportedCodingSystemException, LoincCodeNotFoundException, AmbiguousResultsFoundException, AnnotationNotFoundException, UnrecognizedCodeException, LoincCodeNotAnnotatedException, AmbiguousReferenceException, ReferenceNotFoundException, FHIRException {
 
         //first make sure the observation has a valid loinc code; otherwise, we cannot handle it
         if (!hasValidLoincCode(loincIds)) {
             //TODO: consider handling this as a future project
-            return null;
-        }
-        LoincId loincId = null;
-        try {
-            loincId = getLoincIdOfObservation();
-        } catch (MalformedLoincCodeException e) {
-            logger.error("malformed loinc code; should never happen");
-            return null;
-        } catch (LoincCodeNotFoundException e) {
-            logger.error("No loinc code was found in the observation; should never happen");
-            return null;
-        } catch (UnsupportedCodingSystemException e) {
-            logger.error("coding system not recognized");
-            return null;
+            throw new LoincCodeNotFoundException();
         }
 
+        LoincId loincId = getLoincIdOfObservation();
 
         if (!loinc2HPOannotationMap.containsKey(loincId)) {
-            return null;
+            throw new LoincCodeNotAnnotatedException();
         }
 
-
-
+        HpoTerm4TestOutcome hpoterm = null;
         if (observation.hasInterpretation()) {
             logger.debug("enter analyzer using the interpretation field");
             try {
-                //return getHPOFromInterpretation(observation.getInterpretation(), loinc2HPOannotationMap);
-                HpoTerm4TestOutcome hpoterm = new ObservationAnalysisFromInterpretation(getLoincIdOfObservation(), observation.getInterpretation(), loinc2HPOannotationMap).getHPOforObservation();
+                //hpoterm won't be null
+                hpoterm = new ObservationAnalysisFromInterpretation(getLoincIdOfObservation(), observation.getInterpretation(), loinc2HPOannotationMap).getHPOforObservation();
                 return new BasicLabTestOutcome(hpoterm, null, observation.getSubject(), observation.getIdentifier());
-            } catch (UnrecognizedCodeException e) {
-                //this means the interpretation code is not recognized
-                logger.info("The interpretation codes for this loinc code is not annotated; system will try using raw values");
-            } catch (MalformedLoincCodeException e1) {
-                //not going to happen
-                logger.error("malformed loinc code.");
-                return null;
-            } catch (LoincCodeNotFoundException e2) {
-                //not going to happen
-                logger.error("no loinc code is found in the observation");
-                return null;
-            } catch (UnsupportedCodingSystemException e3) {
-                //not going to happen
-                logger.error("The interpretation coding system cannot be recognized.");
-                return null;
-            } catch (AmbiguousResultsFoundException e) {
-                logger.error("The observation has conflicting interpretation codes.");
-                return null;
             } catch (AnnotationNotFoundException e) {
-                logger.error("There is no annotation for the loinc code used in the observation");
-            }
+                logger.trace("Annotation for the interpretation code is not found; try other methods");
+            } //catch (AmbiguousResultsFoundException e) {  //we should not catch this exception, I think --@azhang
+              //  logger.trace("Interpretation code resulted conflicting hpo interpretation; try other methods");
+            //}
         }
 
         //if we failed to analyze the outcome through the interpretation field, we try to analyze the raw value using
         //the reference range
         //Qn will have a value field
         if (observation.hasValueQuantity()) {
-            try {
-                HpoTerm4TestOutcome hpoterm = new ObservationAnalysisFromQnValue(loincId, observation, loinc2HPOannotationMap).getHPOforObservation();
-                if (hpoterm != null) return new BasicLabTestOutcome(hpoterm, null, observation.getSubject(), observation.getIdentifier());
-            } catch (ReferenceNotFoundException e) {
-                //if there is no reference
-                logger.error("The observation has no reference field.");
-                //TODO: make a list of our own references
-            } catch (AmbiguousReferenceException e) {
-                logger.info("There are two reference ranges or more");
-            } catch (UnrecognizedCodeException e) {
-                logger.error("uncognized coding system");
-            }
-
+            hpoterm = new ObservationAnalysisFromQnValue(loincId, observation, loinc2HPOannotationMap).getHPOforObservation();
+            if (hpoterm != null) return new BasicLabTestOutcome(hpoterm, null, observation.getSubject(), observation.getIdentifier());
         }
 
         //Ord will have a ValueCodeableConcept field
         if (observation.hasValueCodeableConcept()) {
-            try {
-                HpoTerm4TestOutcome hpoterm = null;
-                hpoterm = new ObservationAnalysisFromCodedValues(loincId,
-                        observation.getValueCodeableConcept(), loinc2HPOannotationMap).getHPOforObservation();
-                if (hpoterm != null) return new BasicLabTestOutcome(hpoterm, null, observation.getSubject(), observation.getIdentifier());
-            } catch (AmbiguousResultsFoundException e) {
-                logger.error("multiple results are found");
-            } catch (UnrecognizedCodeException e) {
-                logger.error("unrecognized codes");
-            } catch (FHIRException e) {
-                //not going to happen
-                logger.error("Could not get HPO term from coded value");
-            } catch (AnnotationNotFoundException e) {
-                logger.error("There is no annotation for the loinc code used in the observation");
+            hpoterm = new ObservationAnalysisFromCodedValues(loincId,
+                    observation.getValueCodeableConcept(), loinc2HPOannotationMap).getHPOforObservation();
+            if (hpoterm != null) {
+                return new BasicLabTestOutcome(hpoterm, null, observation.getSubject(), observation.getIdentifier());
             }
         }
 
         //if all the above fails, we cannot do nothing
         logger.error("Could not return HPO for observation: " + observation.getId());
-        return null;
+        return new BasicLabTestOutcome(null, null, observation.getSubject(), observation.getIdentifier());
     }
 
     /**
      * Check whether a FHIR observation has a valid Loinc code
      * @param loincIds: a hashset of all loinc codes (just codes)
      * @return false if the observation does not have one, or in wrong format, or recognized in the hashset
      */
-    private static boolean hasValidLoincCode(HashSet<LoincId> loincIds){
+    private static boolean hasValidLoincCode(Set<LoincId> loincIds){
 
         for (Coding coding : observation.getCode().getCoding()) {
             if (coding.getSystem().equals("http://loinc.org")) {

loinc2hpo-core/src/main/java/org/monarchinitiative/loinc2hpo/fhir/FhirResourceRetriever.java

@@ -8,6 +8,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import org.hl7.fhir.dstu3.model.*;
 import org.hl7.fhir.instance.model.api.IBaseBundle;
+import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.monarchinitiative.loinc2hpo.exception.AmbiguousSubjectException;
 import org.monarchinitiative.loinc2hpo.exception.SubjectNotFoundException;
 import org.apache.logging.log4j.Logger;
@@ -31,30 +32,43 @@ public class FhirResourceRetriever {
     public static final IParser jsonParser = ctx.newJsonParser();
 
 
+
     /**
      * This function parses a json file stored locally to an hapi-fhir Observation object
      * @param filepath
      * @return
      */
-    public static Observation parseJsonFile2Observation(String filepath) {
+    public static Observation parseJsonFile2Observation(String filepath) throws IOException, DataFormatException {
         Observation observation = null;
-        try {
-            File file = new File(filepath);
-            byte[] bytes = new byte[(int)file.length()];
-            FileInputStream fileInputStream = new FileInputStream(file);
-            fileInputStream.read(bytes);
-            //logger.debug(new String(bytes));
-            observation = (Observation) jsonParser.parseResource(new String(bytes));
-            fileInputStream.close();
-
-        } catch (IOException e) {
-            e.printStackTrace();
-        } catch (DataFormatException e) {
-            logger.error("Json file " + filepath + " is not a valid observation");
+
+        File file = new File(filepath);
+        byte[] bytes = new byte[(int)file.length()];
+        FileInputStream fileInputStream = new FileInputStream(file);
+        fileInputStream.read(bytes);
+        //logger.debug(new String(bytes));
+        IBaseResource ibaseResource = jsonParser.parseResource(new String(bytes));
+        if (ibaseResource instanceof Observation) {
+            observation = (Observation) ibaseResource;
+        } else {
+            throw new DataFormatException();
         }
+        fileInputStream.close();
+
         return observation;
     }
 
+    public static Observation parseJsonFile2Observation(File file) throws IOException {
+
+        return parseJsonFile2Observation(file.getAbsolutePath());
+
+    }
+
+    public static String toJsonString(Observation observation) {
+
+        return jsonParser.setPrettyPrint(true).encodeResourceToString(observation);
+
+    }
+
     /**
      * @TODO: implement it
      * retrieve a patient's observations from FHIR server

loinc2hpo-core/src/main/java/org/monarchinitiative/loinc2hpo/fhir/ObservationAnalysisFromQnValue.java

@@ -140,6 +140,7 @@ public HpoTerm4TestOutcome getHPOforObservation() throws ReferenceNotFoundExcept
             //We should handle this case
 
 
+            throw new AmbiguousReferenceException();
 
         } else {
             throw new AmbiguousReferenceException();

loinc2hpo-core/src/main/java/org/monarchinitiative/loinc2hpo/testresult/BasicLabTestOutcome.java

@@ -81,6 +81,7 @@ public String toString() {
         if (hpoId.getId()==null) {
             return "error => hpoId.getId() is null in testResult";
         }
+
         return String.format("BasicLabTestOutcome: %s [%s; %s]", hpoId.getId().getIdWithPrefix(),"?", "?");
     }
 

loinc2hpo-core/src/test/java/org/monarchinitiative/loinc2hpo/fhir/FhirObservationAnalyzerTest.java

@@ -9,16 +9,15 @@
 import org.monarchinitiative.loinc2hpo.codesystems.CodeSystemConvertor;
 import org.monarchinitiative.loinc2hpo.codesystems.Loinc2HPOCodedValue;
 import org.monarchinitiative.loinc2hpo.loinc.*;
+import org.monarchinitiative.loinc2hpo.testresult.LabTestOutcome;
 import org.monarchinitiative.phenol.formats.hpo.HpoOntology;
 import org.monarchinitiative.phenol.formats.hpo.HpoTerm;
 import org.monarchinitiative.phenol.io.obo.hpo.HpoOboParser;
 import org.monarchinitiative.phenol.ontology.data.TermId;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import static org.junit.Assert.*;
@@ -29,7 +28,7 @@ public class FhirObservationAnalyzerTest {
     private static Map<String, HpoTerm> hpoTermMap;
 
     @BeforeClass
-    public static void setup(){
+    public static void setup() throws Exception{
         String path = FhirObservationAnalyzerTest.class.getClassLoader().getResource("json/glucoseHigh.fhir").getPath();
         observation = FhirResourceRetriever.parseJsonFile2Observation(path);
 
@@ -104,10 +103,12 @@ public void testUniversalAnnotation() throws Exception {
                 .build();
 
         testmap.put(loincId, glucoseAnnotation);
-        /**
-        LabTestOutcome result = FhirObservationAnalyzer.getHPOFromInterpretation(FhirObservationAnalyzer.getObservation().getInterpretation(), testmap);
+
+        Set<LoincId> loincIdSet = new HashSet<>();
+        loincIdSet.add(loincId);
+        LabTestOutcome result = FhirObservationAnalyzer.getHPO4ObservationOutcome(loincIdSet, testmap);
         System.out.println(result);
-         **/
+
     }
 
 

loinc2hpo-core/src/test/java/org/monarchinitiative/loinc2hpo/fhir/FhirObservationRetrieverTest.java

@@ -1,25 +1,27 @@
 package org.monarchinitiative.loinc2hpo.fhir;
 
+import ca.uhn.fhir.parser.DataFormatException;
 import org.hl7.fhir.dstu3.model.Observation;
 import org.junit.Test;
 
+
+
 import static org.junit.Assert.*;
 
 public class FhirObservationRetrieverTest {
     @Test
-    public void testParseJsonFile2Observation(){
+    public void testParseJsonFile2Observation() throws Exception{
 
         String path = getClass().getClassLoader().getResource("json/glucoseHigh.fhir").getPath();
         Observation observation = FhirResourceRetriever.parseJsonFile2Observation(path);
         assertNotNull(observation);
         assertEquals("Observation", observation.getResourceType().toString());
     }
 
-    @Test
-    public void testParseJsonFile2ObservationException(){
+    @Test (expected = DataFormatException.class)
+    public void testParseJsonFile2ObservationException() throws Exception{
         String path = getClass().getClassLoader().getResource("json/malformedObservation.fhir").getPath();
         Observation observation = FhirResourceRetriever.parseJsonFile2Observation(path);
-        assertNull(observation);
     }
 
     @Test

loinc2hpo-core/src/test/java/org/monarchinitiative/loinc2hpo/fhir/ObservationAnalysisFromCodedValuesTest.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import java.util.zip.DataFormatException;
 
 import static org.junit.Assert.*;
 
@@ -31,7 +32,7 @@ public class ObservationAnalysisFromCodedValuesTest {
 
 
     @BeforeClass
-    public static void setup() throws MalformedLoincCodeException {
+    public static void setup() throws MalformedLoincCodeException, DataFormatException, IOException {
         String path = FhirObservationAnalyzerTest.class.getClassLoader().getResource("json/staphylococcus.fhir").getPath();
         Observation observation1 = FhirResourceRetriever.parseJsonFile2Observation(path);
         path = FhirObservationAnalyzerTest.class.getClassLoader().getResource("json/staphylococcusNoInterpretation.fhir").getPath();

loinc2hpo-core/src/test/java/org/monarchinitiative/loinc2hpo/fhir/ObservationAnalysisFromInterpretationTest.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import java.util.zip.DataFormatException;
 
 import static org.junit.Assert.*;
 
@@ -31,7 +32,7 @@ public class ObservationAnalysisFromInterpretationTest {
     private static Map<LoincId, LOINC2HpoAnnotationImpl> testmap = new HashMap<>();
 
     @BeforeClass
-    public static void setup() throws MalformedLoincCodeException {
+    public static void setup() throws MalformedLoincCodeException, IOException, DataFormatException {
         String path = FhirObservationAnalyzerTest.class.getClassLoader().getResource("json/glucoseHigh.fhir").getPath();
         Observation observation1 = FhirResourceRetriever.parseJsonFile2Observation(path);
         path = FhirObservationAnalyzerTest.class.getClassLoader().getResource("json/glucoseConflictingInterpretation.fhir").getPath();

loinc2hpo-core/src/test/java/org/monarchinitiative/loinc2hpo/fhir/ObservationAnalysisFromQnValueTest.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import java.util.zip.DataFormatException;
 
 import static org.junit.Assert.*;
 
@@ -31,7 +32,7 @@ public class ObservationAnalysisFromQnValueTest {
 
 
     @BeforeClass
-    public static void setup() throws MalformedLoincCodeException {
+    public static void setup() throws MalformedLoincCodeException, IOException, DataFormatException {
         String path = FhirObservationAnalyzerTest.class.getClassLoader().getResource("json/glucoseHighNoInterpretation.fhir").getPath();
         Observation observation1 = FhirResourceRetriever.parseJsonFile2Observation(path);
         path = FhirObservationAnalyzerTest.class.getClassLoader().getResource("json/glucoseNoInterpretationNoReference.fhir").getPath();