Major revision. Using Berkeley DB to store concept and concept

relationship information. No more sorting of vocab files + dedicated
Athena file reader. Allowing names of source concepts to be used when
searching. Showing number of parents and children per concept. Flexible
divider in main UI.

.classpath

@@ -14,5 +14,6 @@
 	<classpathentry kind="lib" path="lib/poi-ooxml-schemas-3.10.1-20140818.jar"/>
 	<classpathentry kind="lib" path="lib/stax-api-1.0.1.jar"/>
 	<classpathentry kind="lib" path="lib/xmlbeans-2.6.0.jar"/>
+	<classpathentry kind="lib" path="lib/je-7.3.7.jar"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

build.xml

@@ -8,7 +8,7 @@
   <property name="build" location="build"/>
   <property name="dist"  location="dist"/>
 
-  <property name="JDK_VERSION" value="1.7"/>
+  <property name="JDK_VERSION" value="1.8"/>
 
   <path id="classpath">
     <fileset dir="${lib}" includes="**/*.jar"/>

src/org/ohdsi/usagi/BerkeleyDbEngine.java

@@ -0,0 +1,173 @@
+package org.ohdsi.usagi;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.ohdsi.utilities.DirectoryUtilities;
+
+import com.sleepycat.je.DatabaseException;
+import com.sleepycat.je.Environment;
+import com.sleepycat.je.EnvironmentConfig;
+import com.sleepycat.persist.EntityCursor;
+import com.sleepycat.persist.EntityIndex;
+import com.sleepycat.persist.EntityStore;
+import com.sleepycat.persist.PrimaryIndex;
+import com.sleepycat.persist.SecondaryIndex;
+import com.sleepycat.persist.StoreConfig;
+
+public class BerkeleyDbEngine {
+	public static String						DATABASE_FOLDER		= "sleepyCat";
+	private Environment							dbEnvironment;
+	private EntityStore							store;
+	private ConceptDataAccessor					conceptDataAccessor;
+	private MapsToRelationshipDataAccessor		mapsToRelationshipDataAccessor;
+	private SubsumesRelationshipDataAccessor	subsumesRelationshipDataAccessor;
+	private String								databaseFolder;
+	private boolean								isOpenForReading	= false;
+	private boolean								isOpenForWriting	= false;
+
+	public BerkeleyDbEngine(String folder) {
+		this.databaseFolder = folder + "/" + DATABASE_FOLDER;
+	}
+
+	public void createDatabase() {
+		File folder = new File(databaseFolder);
+		if (folder.exists())
+			DirectoryUtilities.deleteDir(folder);
+		new File(databaseFolder).mkdir();
+		open(true);
+		isOpenForWriting = true;
+	}
+
+	public void openForReading() {
+		if (!isOpenForReading) {
+			open(false);
+			isOpenForReading = true;
+		}
+	}
+
+	private void open(boolean create) {
+		try {
+			EnvironmentConfig envConfig = new EnvironmentConfig();
+			envConfig.setAllowCreate(create);
+			envConfig.setReadOnly(!create);
+			dbEnvironment = new Environment(new File(databaseFolder), envConfig);
+
+			StoreConfig storeConfig = new StoreConfig();
+			storeConfig.setAllowCreate(create);
+			storeConfig.setReadOnly(!create);
+			store = new EntityStore(dbEnvironment, "EntityStore", storeConfig);
+			conceptDataAccessor = new ConceptDataAccessor();
+			mapsToRelationshipDataAccessor = new MapsToRelationshipDataAccessor();
+			subsumesRelationshipDataAccessor = new SubsumesRelationshipDataAccessor();
+		} catch (DatabaseException dbe) {
+			throw new RuntimeException(dbe);
+		}
+	}
+
+	public void put(Concept concept) {
+		conceptDataAccessor.primaryIndex.put(concept);
+	}
+
+	public void put(MapsToRelationship mapsToRelationship) {
+		mapsToRelationshipDataAccessor.primaryIndex.put(mapsToRelationship);
+	}
+
+	public void put(SubsumesRelationship subsumesRelationship) {
+		subsumesRelationshipDataAccessor.primaryIndex.put(subsumesRelationship);
+	}
+
+	public EntityCursor<Concept> getConceptCursor() {
+		return conceptDataAccessor.primaryIndex.entities();
+	}
+
+	public MapsToRelationship getMapsToRelationship(int conceptId) {
+		return mapsToRelationshipDataAccessor.primaryIndex.get(conceptId);
+	}
+
+	public List<MapsToRelationship> getMapsToRelationshipsByConceptId2(int conceptId) {
+		EntityIndex<Integer, MapsToRelationship> subIndex = mapsToRelationshipDataAccessor.secondaryIndex.subIndex(conceptId);
+		EntityCursor<MapsToRelationship> cursor = subIndex.entities();
+		List<MapsToRelationship> relationships = new ArrayList<MapsToRelationship>();
+		try {
+			for (MapsToRelationship relationship : cursor)
+				relationships.add(relationship);
+		} finally {
+			cursor.close();
+		}
+		return relationships;
+	}
+
+	public List<SubsumesRelationship> getSubsumesRelationshipsByParentConceptId(int conceptId) {
+		EntityIndex<Integer, SubsumesRelationship> subIndex = subsumesRelationshipDataAccessor.secondaryIndexParent.subIndex(conceptId);
+		EntityCursor<SubsumesRelationship> cursor = subIndex.entities();
+		List<SubsumesRelationship> relationships = new ArrayList<SubsumesRelationship>();
+		try {
+			for (SubsumesRelationship relationship : cursor)
+				relationships.add(relationship);
+		} finally {
+			cursor.close();
+		}
+		return relationships;
+	}
+
+	public List<SubsumesRelationship> getSubsumesRelationshipsByChildConceptId(int conceptId) {
+		EntityIndex<Integer, SubsumesRelationship> subIndex = subsumesRelationshipDataAccessor.secondaryIndexChild.subIndex(conceptId);
+		EntityCursor<SubsumesRelationship> cursor = subIndex.entities();
+		List<SubsumesRelationship> relationships = new ArrayList<SubsumesRelationship>();
+		try {
+			for (SubsumesRelationship relationship : cursor)
+				relationships.add(relationship);
+		} finally {
+			cursor.close();
+		}
+		return relationships;
+	}
+
+	public Concept getConcept(int conceptId) {
+		return conceptDataAccessor.primaryIndex.get(conceptId);
+	}
+
+	public void shutdown() throws DatabaseException {
+		try {
+			if (isOpenForReading || isOpenForWriting) {
+				store.close();
+				dbEnvironment.close();
+			}
+		} catch (DatabaseException dbe) {
+			throw new RuntimeException(dbe);
+		}
+	}
+
+	private class ConceptDataAccessor {
+		public PrimaryIndex<Integer, Concept>	primaryIndex;
+
+		public ConceptDataAccessor() throws DatabaseException {
+			primaryIndex = store.getPrimaryIndex(Integer.class, Concept.class);
+		}
+
+	}
+
+	private class MapsToRelationshipDataAccessor {
+		public PrimaryIndex<Integer, MapsToRelationship>			primaryIndex;
+		public SecondaryIndex<Integer, Integer, MapsToRelationship>	secondaryIndex;
+
+		public MapsToRelationshipDataAccessor() throws DatabaseException {
+			primaryIndex = store.getPrimaryIndex(Integer.class, MapsToRelationship.class);
+			secondaryIndex = store.getSecondaryIndex(primaryIndex, Integer.class, "conceptId2");
+		}
+	}
+
+	private class SubsumesRelationshipDataAccessor {
+		public PrimaryIndex<Integer, SubsumesRelationship>				primaryIndex;
+		public SecondaryIndex<Integer, Integer, SubsumesRelationship>	secondaryIndexParent;
+		public SecondaryIndex<Integer, Integer, SubsumesRelationship>	secondaryIndexChild;
+
+		public SubsumesRelationshipDataAccessor() throws DatabaseException {
+			primaryIndex = store.getPrimaryIndex(Integer.class, SubsumesRelationship.class);
+			secondaryIndexParent = store.getSecondaryIndex(primaryIndex, Integer.class, "parentConceptId");
+			secondaryIndexChild = store.getSecondaryIndex(primaryIndex, Integer.class, "childConceptId");
+		}
+	}
+}

src/org/ohdsi/usagi/CodeMapping.java

@@ -29,10 +29,10 @@ public static enum MappingStatus {
 		APPROVED, UNCHECKED, AUTO_MAPPED, AUTO_MAPPED_TO_1
 	};
 
-	public SourceCode			sourceCode;
-	public double				matchScore;
-	public MappingStatus		mappingStatus;
-	public List<TargetConcept>	targetConcepts	= new ArrayList<TargetConcept>(1);
+	public SourceCode		sourceCode;
+	public double			matchScore;
+	public MappingStatus	mappingStatus;
+	public List<Concept>	targetConcepts	= new ArrayList<Concept>(1);
 
 	public CodeMapping(SourceCode sourceCode) {
 		this.sourceCode = sourceCode;

src/org/ohdsi/usagi/Concept.java

@@ -0,0 +1,79 @@
+/*******************************************************************************
+ * Copyright 2017 Observational Health Data Sciences and Informatics
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.ohdsi.usagi;
+
+import org.ohdsi.utilities.files.Row;
+
+import com.sleepycat.persist.model.Entity;
+import com.sleepycat.persist.model.PrimaryKey;
+
+/**
+ * Class for holding information about a single (target) concept in the Vocabulary
+ */
+@Entity
+public class Concept {
+	public static Concept	EMPTY_CONCEPT	= createEmptyConcept();
+
+	@PrimaryKey
+	public int				conceptId;
+	public String			conceptName;
+	public String			domainId;
+	public String			vocabularyId;
+	public String			conceptClassId;
+	public String			standardConcept;
+	public String			conceptCode;
+	public String			validStartDate;
+	public String			validEndDate;
+	public String			invalidReason;
+	public int				parentCount;
+	public int				childCount;
+
+	public String			additionalInformation;
+
+	public Concept(Row row) {
+		conceptId = row.getInt("concept_id");
+		conceptName = row.get("concept_name");
+		domainId = row.get("domain_id");
+		vocabularyId = row.get("vocabulary_id");
+		conceptClassId = row.get("concept_class_id");
+		standardConcept = row.get("standard_concept");
+		conceptCode = row.get("concept_code");
+		validStartDate = row.get("valid_start_date");
+		validEndDate = row.get("valid_end_date");
+		invalidReason = row.get("invalid_reason");
+		additionalInformation = "";
+	}
+
+	public static Concept createEmptyConcept() {
+		Concept concept = new Concept();
+		concept.conceptId = 0;
+		concept.conceptName = "";
+		concept.conceptClassId = "";
+		concept.vocabularyId = "";
+		concept.conceptCode = "";
+		concept.validStartDate = "";
+		concept.validEndDate = "";
+		concept.invalidReason = "";
+		concept.standardConcept = "";
+		concept.additionalInformation = "";
+		concept.parentCount = 0;
+		concept.childCount = 0;
+		return concept;
+	}
+
+	public Concept() {
+	}
+}

src/org/ohdsi/usagi/MapsToRelationship.java

@@ -0,0 +1,24 @@
+package org.ohdsi.usagi;
+
+import static com.sleepycat.persist.model.Relationship.MANY_TO_ONE;
+
+import org.ohdsi.utilities.files.Row;
+
+import com.sleepycat.persist.model.Entity;
+import com.sleepycat.persist.model.PrimaryKey;
+import com.sleepycat.persist.model.SecondaryKey;
+
+@Entity
+public class MapsToRelationship {
+	@PrimaryKey
+	public int	conceptId1;
+	@SecondaryKey(relate=MANY_TO_ONE)
+	public int	conceptId2;
+
+	public MapsToRelationship() {}
+
+	public MapsToRelationship(Row row) {
+		conceptId1 = row.getInt("concept_id_1");
+		conceptId2 = row.getInt("concept_id_2");
+	}
+}

src/org/ohdsi/usagi/ReadCodeMappingsFromFile.java

@@ -18,6 +18,7 @@
 import java.util.Iterator;
 
 import org.ohdsi.usagi.CodeMapping.MappingStatus;
+import org.ohdsi.usagi.ui.Global;
 import org.ohdsi.utilities.files.ReadCSVFileWithHeader;
 import org.ohdsi.utilities.files.Row;
 
@@ -53,12 +54,12 @@ private void readNext() {
 				buffer = null;
 			else {
 				buffer = new CodeMapping(new SourceCode(row));
-//				if (row.get("matchScore").equals("UNCHECKED"))
-//					System.out.println("asdf");
 				buffer.matchScore = row.getDouble("matchScore");
 				buffer.mappingStatus = MappingStatus.valueOf(row.get("mappingStatus"));
-				while (row != null && new SourceCode(row).sourceCode.equals(buffer.sourceCode.sourceCode) && new SourceCode(row).sourceName.equals(buffer.sourceCode.sourceName)) {
-					buffer.targetConcepts.add(new TargetConcept(row));
+				while (row != null && new SourceCode(row).sourceCode.equals(buffer.sourceCode.sourceCode)
+						&& new SourceCode(row).sourceName.equals(buffer.sourceCode.sourceName)) {
+					Concept concept = Global.dbEngine.getConcept(row.getInt("conceptId"));
+					buffer.targetConcepts.add(concept);
 					if (iterator.hasNext())
 						row = iterator.next();
 					else

src/org/ohdsi/usagi/SubsumesRelationship.java

@@ -0,0 +1,27 @@
+package org.ohdsi.usagi;
+
+import org.ohdsi.utilities.files.Row;
+
+import com.sleepycat.persist.model.Entity;
+import com.sleepycat.persist.model.PrimaryKey;
+import com.sleepycat.persist.model.SecondaryKey;
+
+import static com.sleepycat.persist.model.Relationship.*;
+
+@Entity
+public class SubsumesRelationship {
+	@PrimaryKey(sequence="Sequence_Namespace")
+	public int id;
+
+	@SecondaryKey(relate=MANY_TO_ONE)
+	public int parentConceptId;
+	@SecondaryKey(relate=MANY_TO_ONE)
+	public int childConceptId;
+
+	public SubsumesRelationship() {}
+
+	public SubsumesRelationship(Row row) {
+		parentConceptId = row.getInt("concept_id_1");
+		childConceptId = row.getInt("concept_id_2");
+	}
+}