Skip to content

Commit

Permalink
Added option to map to non-standard (classification) concepts)
Browse files Browse the repository at this point in the history
  • Loading branch information
@schuemie
schuemie committed Feb 13, 2017
1 parent bf18111 commit 477bf4b
Show file tree
Hide file tree
Showing 7 changed files with 48 additions and 20 deletions.
12 changes: 12 additions & 0 deletions src/org/ohdsi/usagi/TargetConcept.java
View file
Expand Up @@ -38,6 +38,7 @@ public class TargetConcept {
public String validEndDate;
public String invalidReason;
public String additionalInformation;
public String standardConcept;
public static TargetConcept EMPTY_CONCEPT = createEmptyConcept();

public TargetConcept(Row row) {
Expand All @@ -52,6 +53,10 @@ public TargetConcept(Row row) {
validStartDate = row.get("VALID_START_DATE");
validEndDate = row.get("VALID_END_DATE");
invalidReason = row.get("INVALID_REASON");
if (row.getFieldNames().contains("STANDARDCONCEPT"))
standardConcept = row.get("STANDARDCONCEPT");
else
standardConcept = "";
for (String domain : row.get("DOMAINS").split("/"))
domains.add(domain);
additionalInformation = row.get("ADDITIONAL_INFORMATION");
Expand All @@ -65,6 +70,10 @@ public TargetConcept(Row row) {
validStartDate = row.get("VALIDSTARTDATE");
validEndDate = row.get("VALIDENDDATE");
invalidReason = row.get("INVALIDREASON");
if (row.getFieldNames().contains("STANDARDCONCEPT"))
standardConcept = row.get("STANDARDCONCEPT");
else
standardConcept = "";
for (String domain : row.get("DOMAINS").split("/"))
domains.add(domain);
additionalInformation = row.get("ADDITIONALINFORMATION");
Expand All @@ -82,6 +91,7 @@ private static TargetConcept createEmptyConcept() {
concept.validStartDate = "";
concept.validEndDate = "";
concept.invalidReason = "";
concept.standardConcept = "";
concept.additionalInformation = "This is an empty concept";
return concept;
}
Expand All @@ -100,6 +110,7 @@ public TargetConcept(TargetConcept concept) {
validEndDate = concept.validEndDate;
invalidReason = concept.invalidReason;
domains.addAll(concept.domains);
standardConcept = concept.standardConcept;
additionalInformation = concept.additionalInformation;
}

Expand All @@ -115,6 +126,7 @@ public Row toRow() {
row.add("validEndDate", validEndDate);
row.add("invalidReason", invalidReason);
row.add("domains", StringUtilities.join(domains, "/"));
row.add("standardConcept", standardConcept);
if (additionalInformation == null)
row.add("additionalInformation", "");
else
Expand Down
16 changes: 10 additions & 6 deletions src/org/ohdsi/usagi/UsagiSearchEngine.java
View file
Expand Up @@ -91,7 +91,8 @@ public class UsagiSearchEngine {
private QueryParser conceptClassQueryParser;
private QueryParser vocabularyQueryParser;
private QueryParser keywordsQueryParser;
private QueryParser invalidQueryParser;
// private QueryParser invalidQueryParser;
private QueryParser standardConceptParser;
private QueryParser domainQueryParser;
private int numDocs;
private FieldType textVectorField = getTextVectorFieldType();
Expand Down Expand Up @@ -150,6 +151,7 @@ public void addConceptToIndex(TargetConcept concept) {
document.add(new StringField("VALID_START_DATE", concept.validStartDate, Store.YES));
document.add(new StringField("VALID_END_DATE", concept.validEndDate, Store.YES));
document.add(new StringField("INVALID_REASON", concept.invalidReason, Store.YES));
document.add(new StringField("STANDARD_CONCEPT", concept.standardConcept, Store.YES));
document.add(new TextField("DOMAINS", StringUtilities.join(concept.domains, "\n"), Store.YES));
document.add(new StringField("ADDITIONAL_INFORMATION", concept.additionalInformation, Store.YES));
writer.addDocument(document);
Expand Down Expand Up @@ -260,7 +262,8 @@ public void openIndexForSearching() {
vocabularyQueryParser = new QueryParser(Version.LUCENE_4_9, "VOCABULARY", new KeywordAnalyzer());
keywordsQueryParser = new QueryParser(Version.LUCENE_4_9, "TERM", analyzer);
domainQueryParser = new QueryParser(Version.LUCENE_4_9, "DOMAINS", analyzer);
invalidQueryParser = new QueryParser(Version.LUCENE_4_9, "INVALID_REASON", new KeywordAnalyzer());
// invalidQueryParser = new QueryParser(Version.LUCENE_4_9, "INVALID_REASON", new KeywordAnalyzer());
standardConceptParser = new QueryParser(Version.LUCENE_4_9, "STANDARD_CONCEPT", new KeywordAnalyzer());
numDocs = reader.numDocs();
} catch (Exception e) {
throw new RuntimeException(e);
Expand Down Expand Up @@ -317,7 +320,7 @@ public void close() {
}

public List<ScoredConcept> search(String searchTerm, boolean useMlt, Collection<Integer> filterConceptIds, String filterDomain, String filterConceptClass,
String filterVocabulary, boolean filterInvalid) {
String filterVocabulary, boolean filterStandard) {
List<ScoredConcept> results = new ArrayList<ScoredConcept>();
try {
Query query;
Expand Down Expand Up @@ -365,9 +368,9 @@ public List<ScoredConcept> search(String searchTerm, boolean useMlt, Collection<
Query vocabularyQuery = vocabularyQueryParser.parse("\"" + filterVocabulary.toString() + "\"");
booleanQuery.add(vocabularyQuery, Occur.MUST);
}
if (filterInvalid) {
Query invalidQuery = invalidQueryParser.parse("\"\"");
booleanQuery.add(invalidQuery, Occur.MUST);
if (filterStandard) {
Query standardQuery = standardConceptParser.parse("S");
booleanQuery.add(standardQuery, Occur.MUST);
}
TopDocs topDocs = searcher.search(booleanQuery, 100);

Expand Down Expand Up @@ -402,6 +405,7 @@ private TargetConcept docIdToTargetConcept(int docId) {
targetConcept.validStartDate = document.get("VALID_START_DATE");
targetConcept.validEndDate = document.get("VALID_END_DATE");
targetConcept.invalidReason = document.get("INVALID_REASON");
targetConcept.standardConcept = document.get("STANDARD_CONCEPT");
for (String domain : document.get("DOMAINS").split("\n"))
targetConcept.domains.add(domain);
targetConcept.additionalInformation = document.get("ADDITIONAL_INFORMATION");
Expand Down
3 changes: 2 additions & 1 deletion src/org/ohdsi/usagi/indexBuilding/BuildIndex.java
View file
Expand Up @@ -151,7 +151,7 @@ public void run() {

Row conceptRow = multiRowSet.get("concept").get(0);
if (conceptRow.getCells().size() > 2) // Extra check to catch badly formatted rows (which are in a vocab we don't care about)
if (conceptRow.get("STANDARD_CONCEPT").equals("S")) {
if (conceptRow.get("STANDARD_CONCEPT").equals("S") || conceptRow.get("STANDARD_CONCEPT").equals("C")) {
vocabularies.add(conceptRow.get("VOCABULARY_ID"));
conceptClassIds.add(conceptRow.get("CONCEPT_CLASS_ID"));
domainIds.add(conceptRow.get("DOMAIN_ID"));
Expand Down Expand Up @@ -179,6 +179,7 @@ public void run() {
concept.invalidReason = conceptRow.get("INVALID_REASON");
concept.validEndDate = conceptRow.get("VALID_END_DATE");
concept.validStartDate = conceptRow.get("VALID_START_DATE");
concept.standardConcept = conceptRow.get("STANDARD_CONCEPT");
concept.vocabulary = conceptRow.get("VOCABULARY_ID");
if (loincToInfo != null && concept.vocabulary.equals("LOINC")) {
String info = loincToInfo.get(concept.conceptCode);
Expand Down
15 changes: 8 additions & 7 deletions src/org/ohdsi/usagi/ui/FilterPanel.java
View file
Expand Up @@ -35,7 +35,7 @@ public class FilterPanel extends JPanel {

private static final long serialVersionUID = 1378433878412231259L;
private JCheckBox filterByAutoCheckBox;
private JCheckBox filterInvalidCheckBox;
private JCheckBox filterStandardCheckBox;
private JCheckBox filterByConceptClassCheckBox;
private JCheckBox filterByVocabularyCheckBox;
private JCheckBox filterByDomainCheckBox;
Expand Down Expand Up @@ -70,16 +70,17 @@ public void actionPerformed(ActionEvent arg0) {
c.gridy = 0;
c.weightx = 0.1;
c.gridwidth = 2;
filterInvalidCheckBox = new JCheckBox("Filter invalid concepts", false);
filterInvalidCheckBox.setToolTipText("Limit the search to only valid concepts");
filterInvalidCheckBox.addActionListener(new ActionListener() {
filterStandardCheckBox = new JCheckBox("Filter standard concepts", false);
filterStandardCheckBox.setToolTipText("Limit the search to only standard concepts");
filterStandardCheckBox.addActionListener(new ActionListener() {

@Override
public void actionPerformed(ActionEvent arg0) {
notifyListeners();
}
});
add(filterInvalidCheckBox, c);
filterStandardCheckBox.setSelected(true);
add(filterStandardCheckBox, c);

c.gridx = 0;
c.gridy = 1;
Expand Down Expand Up @@ -197,8 +198,8 @@ public boolean getFilterByAuto() {
return filterByAutoCheckBox.isSelected();
}

public boolean getFilterInvalid() {
return filterInvalidCheckBox.isSelected();
public boolean getFilterStandard() {
return filterStandardCheckBox.isSelected();
}

public boolean getFilterByConceptClass() {
Expand Down
4 changes: 2 additions & 2 deletions src/org/ohdsi/usagi/ui/ImportDialog.java
View file
Expand Up @@ -375,7 +375,7 @@ public void run() {
try {
Global.usagiSearchEngine.createDerivedIndex(sourceCodes, null);

boolean filterInvalid = filterPanel.getFilterInvalid();
boolean filterStandard = filterPanel.getFilterStandard();
String filterConceptClass = null;
if (filterPanel.getFilterByConceptClass())
filterConceptClass = filterPanel.getConceptClass();
Expand All @@ -394,7 +394,7 @@ public void run() {

CodeMapping codeMapping = new CodeMapping(sourceCode);
List<ScoredConcept> concepts = Global.usagiSearchEngine.search(sourceCode.sourceName, true, filterConceptIds, filterDomain,
filterConceptClass, filterVocabulary, filterInvalid);
filterConceptClass, filterVocabulary, filterStandard);
if (concepts.size() > 0) {
codeMapping.targetConcepts.add(concepts.get(0).concept);
codeMapping.matchScore = concepts.get(0).matchScore;
Expand Down
14 changes: 11 additions & 3 deletions src/org/ohdsi/usagi/ui/MappingDetailPanel.java
View file
Expand Up @@ -206,6 +206,10 @@ public void valueChanged(ListSelectionEvent event) {
}

});
// searchTable.hideColumn("Synonym");
searchTable.hideColumn("Valid start date");
searchTable.hideColumn("Valid end date");
searchTable.hideColumn("Invalid reason");
panel.add(new JScrollPane(searchTable));

JPanel buttonPanel = new JPanel();
Expand Down Expand Up @@ -384,7 +388,7 @@ public void run() {
if (filterPanel.getFilterByAuto())
filterConceptIds = codeMapping.sourceCode.sourceAutoAssignedConceptIds;

boolean filterInvalid = filterPanel.getFilterInvalid();
boolean filterStandard = filterPanel.getFilterStandard();
String filterConceptClass = null;
if (filterPanel.getFilterByConceptClass())
filterConceptClass = filterPanel.getConceptClass();
Expand All @@ -400,7 +404,7 @@ public void run() {

if (Global.usagiSearchEngine.isOpenForSearching()) {
List<ScoredConcept> searchResults = Global.usagiSearchEngine.search(query, true, filterConceptIds, filterDomain, filterConceptClass,
filterVocabulary, filterInvalid);
filterVocabulary, filterStandard);

searchTableModel.setScoredConcepts(searchResults);
searchTable.scrollRectToVisible(new Rectangle(searchTable.getCellRect(0, 0, true)));
Expand Down Expand Up @@ -505,7 +509,7 @@ class TargetConceptTableModel extends AbstractTableModel {

private String scoreColumnName = "Score";
private String[] columnNames = { "Synonym", "Concept ID", "Concept name", "Domain", "Concept class", "Vocabulary", "Concept code",
"Valid start date", "Valid end date", "Invalid reason" };
"Valid start date", "Valid end date", "Invalid reason", "Standard concept" };
private List<TargetConcept> targetConcepts = new ArrayList<TargetConcept>();
private boolean hasScoreColumn;
private Double[] scoreColumn;
Expand Down Expand Up @@ -585,6 +589,8 @@ public Object getValueAt(int row, int col) {
return targetConcept.validEndDate;
case 9:
return targetConcept.invalidReason;
case 10:
return targetConcept.standardConcept;
default:
return "";
}
Expand Down Expand Up @@ -617,6 +623,8 @@ public Class<?> getColumnClass(int col) {
return String.class;
case 9:
return String.class;
case 10:
return String.class;
default:
return String.class;
}
Expand Down
4 changes: 3 additions & 1 deletion src/org/ohdsi/usagi/ui/MappingTablePanel.java
View file
Expand Up @@ -94,7 +94,7 @@ class CodeMapTableModel extends AbstractTableModel {

private String[] defaultColumnNames = { "Status", "Source code", "Source term", "Frequency", "Match score", "Synonym", "Concept ID",
"Concept name", "Domain", "Concept class", "Vocabulary", "Concept code", "Valid start date",
"Valid end date", "Invalid reason" };
"Valid end date", "Invalid reason", "Standard concept" };
private String[] columnNames = defaultColumnNames;
private int addInfoColCount = 0;
private int ADD_INFO_START_COL = 4;
Expand Down Expand Up @@ -180,6 +180,8 @@ public Object getValueAt(int row, int col) {
return targetConcept.validEndDate;
case 14:
return targetConcept.invalidReason;
case 15:
return targetConcept.standardConcept;
default:
return "";
}
Expand Down

0 comments on commit 477bf4b

Please sign in to comment.