Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Always apply vs filters correctly #2195

Merged
merged 6 commits into from Nov 24, 2020
Merged

Always apply vs filters correctly #2195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3c38f85
Always apply valueset filters correctly
jamesagnew Nov 24, 2020
8cef42e
Test fix
jamesagnew Nov 24, 2020
06e0132
Testfix
jamesagnew Nov 24, 2020
dd329a8
Test fix
jamesagnew Nov 24, 2020
9639783
Add changelog
jamesagnew Nov 24, 2020
3b12f4f
Test fix
jamesagnew Nov 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions hapi-fhir-base/src/main/java/ca/uhn/fhir/context/support/ValueSetExpansionOptions.java
View file
Open in desktop
Expand Up @@ -93,4 +93,10 @@ public ValueSetExpansionOptions setFailOnMissingCodeSystem(boolean theFailOnMiss
myFailOnMissingCodeSystem = theFailOnMissingCodeSystem;
return this;
}

public static ValueSetExpansionOptions forOffsetAndCount(int theOffset, int theCount) {
return new ValueSetExpansionOptions()
.setOffset(theOffset)
.setCount(theCount);
}
}
6 changes: 6 additions & 0 deletions ...c/main/resources/ca/uhn/hapi/fhir/changelog/5_3_0/2195-handle-vs-filters-on-large-vs.yaml
View file
Open in desktop
@@ -0,0 +1,6 @@
---
type: fix
issue: 2195
title: "When performing a ValueSet expansion with a filter a large pre-expanded
ValueSet (more than 1000 codes), the filter failed to find concepts appearing
after the first thousand. This has been corrected."
2 changes: 1 addition & 1 deletion ...ir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ITermValueSetConceptViewDao.java
View file
Open in desktop
Expand Up @@ -34,7 +34,7 @@ public interface ITermValueSetConceptViewDao extends JpaRepository<TermValueSetC
@Query("SELECT v FROM TermValueSetConceptView v WHERE v.myConceptValueSetPid = :pid AND v.myConceptOrder >= :from AND v.myConceptOrder < :to ORDER BY v.myConceptOrder")
List<TermValueSetConceptView> findByTermValueSetId(@Param("from") int theFrom, @Param("to") int theTo, @Param("pid") Long theValueSetId);

@Query("SELECT v FROM TermValueSetConceptView v WHERE v.myConceptValueSetPid = :pid AND v.myConceptDisplay LIKE :display ORDER BY v.myConceptOrder")
@Query("SELECT v FROM TermValueSetConceptView v WHERE v.myConceptValueSetPid = :pid AND LOWER(v.myConceptDisplay) LIKE :display ORDER BY v.myConceptOrder")
List<TermValueSetConceptView> findByTermValueSetId(@Param("pid") Long theValueSetId, @Param("display") String theDisplay);

}
199 changes: 21 additions & 178 deletions ...-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java
View file
Open in desktop
Expand Up @@ -22,7 +22,6 @@

import ca.uhn.fhir.context.support.ConceptValidationOptions;
import ca.uhn.fhir.context.support.IValidationSupport;
import ca.uhn.fhir.context.support.ValidationSupportContext;
import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoValueSet;
import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao;
Expand All @@ -31,216 +30,60 @@
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.util.ElementUtil;
import org.hl7.fhir.convertors.conv30_40.ValueSet30_40;
import org.hl7.fhir.dstu3.model.CodeableConcept;
import org.hl7.fhir.dstu3.model.Coding;
import org.hl7.fhir.dstu3.model.IntegerType;
import org.hl7.fhir.dstu3.model.ValueSet;
import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent;
import org.hl7.fhir.dstu3.model.ValueSet.FilterOperator;
import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.instance.model.api.IPrimitiveType;

import java.util.Date;
import java.util.List;

import static ca.uhn.fhir.jpa.dao.FhirResourceDaoValueSetDstu2.toStringOrNull;
import static ca.uhn.fhir.jpa.dao.r4.FhirResourceDaoValueSetR4.validateHaveExpansionOrThrowInternalErrorException;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.hl7.fhir.convertors.conv30_40.ValueSet30_40.convertValueSet;

public class FhirResourceDaoValueSetDstu3 extends BaseHapiFhirResourceDao<ValueSet> implements IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> {

private IValidationSupport myValidationSupport;

@Override
public void start() {
super.start();
myValidationSupport = getApplicationContext().getBean(IValidationSupport.class, "myJpaValidationSupportChain");
}

@Override
public ValueSet expand(IIdType theId, String theFilter, RequestDetails theRequestDetails) {
ValueSet source = read(theId, theRequestDetails);
public org.hl7.fhir.dstu3.model.ValueSet expand(IIdType theId, String theFilter, RequestDetails theRequestDetails) {
org.hl7.fhir.dstu3.model.ValueSet source = read(theId, theRequestDetails);
return expand(source, theFilter);
}

@Override
public ValueSet expand(IIdType theId, String theFilter, int theOffset, int theCount, RequestDetails theRequestDetails) {
ValueSet source = read(theId, theRequestDetails);
public org.hl7.fhir.dstu3.model.ValueSet expand(IIdType theId, String theFilter, int theOffset, int theCount, RequestDetails theRequestDetails) {
org.hl7.fhir.dstu3.model.ValueSet source = read(theId, theRequestDetails);
return expand(source, theFilter, theOffset, theCount);
}

private ValueSet doExpand(ValueSet theSource) {
validateIncludes("include", theSource.getCompose().getInclude());
validateIncludes("exclude", theSource.getCompose().getExclude());

IValidationSupport.ValueSetExpansionOutcome retVal = myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), null, theSource);
validateHaveExpansionOrThrowInternalErrorException(retVal);
return (ValueSet) retVal.getValueSet();

}

private ValueSet doExpand(ValueSet theSource, int theOffset, int theCount) {
validateIncludes("include", theSource.getCompose().getInclude());
validateIncludes("exclude", theSource.getCompose().getExclude());

ValueSetExpansionOptions options = new ValueSetExpansionOptions()
.setOffset(theOffset)
.setCount(theCount);
IValidationSupport.ValueSetExpansionOutcome retVal = myValidationSupport.expandValueSet(new ValidationSupportContext(myValidationSupport), options, theSource);
validateHaveExpansionOrThrowInternalErrorException(retVal);
return (ValueSet) retVal.getValueSet();
}

private void validateIncludes(String name, List<ConceptSetComponent> listToValidate) {
for (ConceptSetComponent nextExclude : listToValidate) {
if (isBlank(nextExclude.getSystem()) && nextExclude.getValueSet().isEmpty() && !ElementUtil.isEmpty(nextExclude.getConcept(), nextExclude.getFilter())) {
throw new InvalidRequestException("ValueSet contains " + name + " criteria with no system defined");
}
}
}

@Override
public ValueSet expandByIdentifier(String theUri, String theFilter) {
if (isBlank(theUri)) {
throw new InvalidRequestException("URI must not be blank or missing");
}

ValueSet source = new ValueSet();
source.setUrl(theUri);

if (isNotBlank(theFilter)) {
ConceptSetFilterComponent filter = source.getCompose().addInclude().addValueSet(theUri).addFilter();
filter.setProperty("display");
filter.setOp(FilterOperator.EQUAL);
filter.setValue(theFilter);
} else {
source.getCompose().addInclude().addValueSet(theUri);
}

return doExpand(source);

// if (defaultValueSet != null) {
// source = getContext().newJsonParser().parseResource(ValueSet.class, getContext().newJsonParser().encodeResourceToString(defaultValueSet));
// } else {
// IBundleProvider ids = search(ValueSet.SP_URL, new UriParam(theUri));
// if (ids.size() == 0) {
// throw new InvalidRequestException("Unknown ValueSet URI: " + theUri);
// }
// source = (ValueSet) ids.getResources(0, 1).get(0);
// }
//
// return expand(defaultValueSet, theFilter);
public org.hl7.fhir.dstu3.model.ValueSet expandByIdentifier(String theUri, String theFilter) {
org.hl7.fhir.r4.model.ValueSet canonicalOutput = myTerminologySvc.expandValueSet(null, theUri, theFilter);
return ValueSet30_40.convertValueSet(canonicalOutput);
}

@Override
public ValueSet expandByIdentifier(String theUri, String theFilter, int theOffset, int theCount) {
if (isBlank(theUri)) {
throw new InvalidRequestException("URI must not be blank or missing");
}

ValueSet source = new ValueSet();
source.setUrl(theUri);

if (isNotBlank(theFilter)) {
ConceptSetFilterComponent filter = source.getCompose().addInclude().addValueSet(theUri).addFilter();
filter.setProperty("display");
filter.setOp(FilterOperator.EQUAL);
filter.setValue(theFilter);
} else {
source.getCompose().addInclude().addValueSet(theUri);
}

return doExpand(source, theOffset, theCount);
public org.hl7.fhir.dstu3.model.ValueSet expandByIdentifier(String theUri, String theFilter, int theOffset, int theCount) {
ValueSetExpansionOptions options = ValueSetExpansionOptions.forOffsetAndCount(theOffset, theCount);
org.hl7.fhir.r4.model.ValueSet canonicalOutput = myTerminologySvc.expandValueSet(options, theUri, theFilter);
return ValueSet30_40.convertValueSet(canonicalOutput);
}

@Override
public ValueSet expand(ValueSet theSource, String theFilter) {
ValueSet toExpand = new ValueSet();

// for (UriType next : theSource.getCompose().getInclude()) {
// ConceptSetComponent include = toExpand.getCompose().addInclude();
// include.setSystem(next.getValue());
// addFilterIfPresent(theFilter, include);
// }

for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
toExpand.getCompose().addInclude(next);
addFilterIfPresent(theFilter, next);
}

if (toExpand.getCompose().isEmpty()) {
throw new InvalidRequestException("ValueSet does not have any compose.include or compose.import values, can not expand");
}

toExpand.getCompose().getExclude().addAll(theSource.getCompose().getExclude());

ValueSet retVal = doExpand(toExpand);

if (isNotBlank(theFilter)) {
applyFilter(retVal.getExpansion().getTotalElement(), retVal.getExpansion().getContains(), theFilter);
}

return retVal;
public org.hl7.fhir.dstu3.model.ValueSet expand(org.hl7.fhir.dstu3.model.ValueSet theSource, String theFilter) {
org.hl7.fhir.r4.model.ValueSet canonicalInput = ValueSet30_40.convertValueSet(theSource);
org.hl7.fhir.r4.model.ValueSet canonicalOutput = myTerminologySvc.expandValueSet(null, canonicalInput, theFilter);
return ValueSet30_40.convertValueSet(canonicalOutput);
}

@Override
public ValueSet expand(ValueSet theSource, String theFilter, int theOffset, int theCount) {
ValueSet toExpand = new ValueSet();
toExpand.setId(theSource.getId());
toExpand.setUrl(theSource.getUrl());
if (theSource.getVersion() != null) {
toExpand.setVersion(theSource.getVersion());
}

for (ConceptSetComponent next : theSource.getCompose().getInclude()) {
toExpand.getCompose().addInclude(next);
addFilterIfPresent(theFilter, next);
}

if (toExpand.getCompose().isEmpty()) {
throw new InvalidRequestException("ValueSet does not have any compose.include or compose.import values, can not expand");
}

toExpand.getCompose().getExclude().addAll(theSource.getCompose().getExclude());

ValueSet retVal = doExpand(toExpand, theOffset, theCount);

if (isNotBlank(theFilter)) {
applyFilter(retVal.getExpansion().getTotalElement(), retVal.getExpansion().getContains(), theFilter);
}

return retVal;
}

private void applyFilter(IntegerType theTotalElement, List<ValueSetExpansionContainsComponent> theContains, String theFilter) {

for (int idx = 0; idx < theContains.size(); idx++) {
ValueSetExpansionContainsComponent next = theContains.get(idx);
if (isBlank(next.getDisplay()) || !org.apache.commons.lang3.StringUtils.containsIgnoreCase(next.getDisplay(), theFilter)) {
theContains.remove(idx);
idx--;
if (theTotalElement.getValue() != null) {
theTotalElement.setValue(theTotalElement.getValue() - 1);
}
}
applyFilter(theTotalElement, next.getContains(), theFilter);
}
}

private void addFilterIfPresent(String theFilter, ConceptSetComponent include) {
if (ElementUtil.isEmpty(include.getConcept())) {
if (isNotBlank(theFilter)) {
include.addFilter().setProperty("display").setOp(FilterOperator.EQUAL).setValue(theFilter);
}
}
public org.hl7.fhir.dstu3.model.ValueSet expand(org.hl7.fhir.dstu3.model.ValueSet theSource, String theFilter, int theOffset, int theCount) {
ValueSetExpansionOptions options = ValueSetExpansionOptions.forOffsetAndCount(theOffset, theCount);
org.hl7.fhir.r4.model.ValueSet canonicalInput = ValueSet30_40.convertValueSet(theSource);
org.hl7.fhir.r4.model.ValueSet canonicalOutput = myTerminologySvc.expandValueSet(options, canonicalInput, theFilter);
return ValueSet30_40.convertValueSet(canonicalOutput);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions ...hir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/predicate/PredicateBuilderToken.java
View file
Open in desktop
Expand Up @@ -205,7 +205,7 @@ private Collection<Predicate> createPredicateToken(Collection<IQueryParameterTyp
*/

if (modifier == TokenParamModifier.IN) {
codes.addAll(myTerminologySvc.expandValueSet(null, code));
codes.addAll(myTerminologySvc.expandValueSetIntoConceptList(null, code));
} else if (modifier == TokenParamModifier.ABOVE) {
system = determineSystemIfMissing(theSearchParam, code, system);
validateHaveSystemAndCodeForToken(paramName, code, system);
Expand Down Expand Up @@ -273,7 +273,7 @@ private String determineSystemIfMissing(RuntimeSearchParam theSearchParam, Strin
String valueSet = valueSetUris.iterator().next();
ValueSetExpansionOptions options = new ValueSetExpansionOptions()
.setFailOnMissingCodeSystem(false);
List<FhirVersionIndependentConcept> candidateCodes = myTerminologySvc.expandValueSet(options, valueSet);
List<FhirVersionIndependentConcept> candidateCodes = myTerminologySvc.expandValueSetIntoConceptList(options, valueSet);
for (FhirVersionIndependentConcept nextCandidate : candidateCodes) {
if (nextCandidate.getCode().equals(code)) {
retVal = nextCandidate.getSystem();
Expand Down