Skip to content

Commit

Permalink
Paging now working
Browse files Browse the repository at this point in the history
  • Loading branch information
@jamesagnew
jamesagnew committed Jun 19, 2017
1 parent 526a1fa commit 571045b
Show file tree
Hide file tree
Showing 3 changed files with 227 additions and 35 deletions.
36 changes: 35 additions & 1 deletion hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
View file
Expand Up @@ -87,15 +87,16 @@ public class DaoConfig {

private boolean myEnforceReferentialIntegrityOnWrite = true;

private int myEverythingIncludesFetchPageSize = 50;
/**
* update setter javadoc if default changes
*/
private long myExpireSearchResultsAfterMillis = DateUtils.MILLIS_PER_HOUR;

/**
* update setter javadoc if default changes
*/
private Integer myFetchSizeDefaultMaximum = null;

private int myHardTagListLimit = 1000;
private int myIncludeLimit = 2000;
/**
Expand Down Expand Up @@ -147,6 +148,22 @@ public int getDeferIndexingForCodesystemsOfSize() {
return myDeferIndexingForCodesystemsOfSize;
}

/**
* Unlike with normal search queries, $everything queries have their _includes loaded by the main search thread and these included results
* are added to the normal search results instead of being added on as extras in a page. This means that they will not appear multiple times
* as the search results are paged over.
* <p>
* In order to recursively load _includes, we process the original results in batches of this size. Adjust with caution, increasing this
* value may improve performance but may also cause memory issues.
* </p>
* <p>
* The default value is 50
* </p>
*/
public int getEverythingIncludesFetchPageSize() {
return myEverythingIncludesFetchPageSize;
}

/**
* Sets the number of milliseconds that search results for a given client search
* should be preserved before being purged from the database.
Expand Down Expand Up @@ -537,6 +554,23 @@ public void setEnforceReferentialIntegrityOnWrite(boolean theEnforceReferentialI
myEnforceReferentialIntegrityOnWrite = theEnforceReferentialIntegrityOnWrite;
}

/**
* Unlike with normal search queries, $everything queries have their _includes loaded by the main search thread and these included results
* are added to the normal search results instead of being added on as extras in a page. This means that they will not appear multiple times
* as the search results are paged over.
* <p>
* In order to recursively load _includes, we process the original results in batches of this size. Adjust with caution, increasing this
* value may improve performance but may also cause memory issues.
* </p>
* <p>
* The default value is 50
* </p>
*/
public void setEverythingIncludesFetchPageSize(int theEverythingIncludesFetchPageSize) {
Validate.inclusiveBetween(1, Integer.MAX_VALUE, theEverythingIncludesFetchPageSize);
myEverythingIncludesFetchPageSize = theEverythingIncludesFetchPageSize;
}

/**
* If this is set to <code>false</code> (default is <code>true</code>) the stale search deletion
* task will be disabled (meaning that search results will be retained in the database indefinitely). USE WITH CAUTION.
Expand Down
151 changes: 125 additions & 26 deletions hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java
View file
Expand Up @@ -27,32 +27,86 @@

import java.math.BigDecimal;
import java.math.MathContext;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.*;
import javax.persistence.criteria.AbstractQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaBuilder.In;

import org.apache.commons.lang3.*;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.From;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.persistence.criteria.Order;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import javax.persistence.criteria.Subquery;

import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.tuple.Pair;
import org.hl7.fhir.instance.model.api.*;

import com.google.common.collect.*;

import ca.uhn.fhir.context.*;
import ca.uhn.fhir.jpa.dao.SearchBuilder.IncludesIterator;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;

import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeDeclaredChildDefinition;
import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.FhirVersionEnum;
import ca.uhn.fhir.context.RuntimeChildChoiceDefinition;
import ca.uhn.fhir.context.RuntimeChildResourceDefinition;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.dao.data.IForcedIdDao;
import ca.uhn.fhir.jpa.dao.data.IResourceIndexedSearchParamUriDao;
import ca.uhn.fhir.jpa.entity.*;
import ca.uhn.fhir.jpa.entity.BaseHasResource;
import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamDate;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamNumber;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamQuantity;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamString;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamUri;
import ca.uhn.fhir.jpa.entity.ResourceLink;
import ca.uhn.fhir.jpa.entity.ResourceTable;
import ca.uhn.fhir.jpa.entity.ResourceTag;
import ca.uhn.fhir.jpa.entity.SearchParam;
import ca.uhn.fhir.jpa.entity.SearchParamPresent;
import ca.uhn.fhir.jpa.entity.TagDefinition;
import ca.uhn.fhir.jpa.entity.TagTypeEnum;
import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
import ca.uhn.fhir.jpa.term.VersionIndependentConcept;
import ca.uhn.fhir.jpa.util.StopWatch;
import ca.uhn.fhir.model.api.*;
import ca.uhn.fhir.model.base.composite.*;
import ca.uhn.fhir.model.api.IPrimitiveDatatype;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.model.api.IResource;
import ca.uhn.fhir.model.api.Include;
import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
import ca.uhn.fhir.model.base.composite.BaseCodingDt;
import ca.uhn.fhir.model.base.composite.BaseIdentifierDt;
import ca.uhn.fhir.model.base.composite.BaseQuantityDt;
import ca.uhn.fhir.model.dstu.resource.BaseResource;
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.model.primitive.InstantDt;
Expand All @@ -61,18 +115,33 @@
import ca.uhn.fhir.rest.api.SortOrderEnum;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.method.RestSearchParameterTypeEnum;
import ca.uhn.fhir.rest.param.*;
import ca.uhn.fhir.rest.param.CompositeParam;
import ca.uhn.fhir.rest.param.DateParam;
import ca.uhn.fhir.rest.param.DateRangeParam;
import ca.uhn.fhir.rest.param.HasParam;
import ca.uhn.fhir.rest.param.NumberParam;
import ca.uhn.fhir.rest.param.ParamPrefixEnum;
import ca.uhn.fhir.rest.param.QuantityParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.param.StringParam;
import ca.uhn.fhir.rest.param.TokenParam;
import ca.uhn.fhir.rest.param.TokenParamModifier;
import ca.uhn.fhir.rest.param.UriParam;
import ca.uhn.fhir.rest.param.UriParamQualifierEnum;
import ca.uhn.fhir.rest.server.Constants;
import ca.uhn.fhir.rest.server.exceptions.*;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.util.UrlUtil;

/**
* The SearchBuilder is responsible for actually forming the SQL query that handles
* searchs for resources
*/
public class SearchBuilder implements ISearchBuilder {
private static Long NO_MORE = Long.valueOf(-1);
private static final List<Long> EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<Long>());

private static Long NO_MORE = Long.valueOf(-1);
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchBuilder.class);
private List<Long> myAlsoIncludePids;
private CriteriaBuilder myBuilder;
Expand All @@ -92,6 +161,7 @@ public class SearchBuilder implements ISearchBuilder {
private ISearchParamRegistry mySearchParamRegistry;
private String mySearchUuid;
private IHapiTerminologySvc myTerminologySvc;

/**
* Constructor
*/
Expand Down Expand Up @@ -1967,10 +2037,47 @@ private static List<Long> filterResourceIdsByLastUpdated(EntityManager theEntity
static Predicate[] toArray(List<Predicate> thePredicates) {
return thePredicates.toArray(new Predicate[thePredicates.size()]);
}

public class IncludesIterator implements Iterator<Long>{

private Iterator<Long> myCurrentIterator;
private int myCurrentOffset;
private ArrayList<Long> myCurrentPids;
private Long myNext;
private int myPageSize = myCallingDao.getConfig().getEverythingIncludesFetchPageSize();

public IncludesIterator(Set<Long> thePidSet) {
myCurrentPids = new ArrayList<Long>(thePidSet);
myCurrentIterator = EMPTY_LONG_LIST.iterator();
myCurrentOffset = 0;
}

private void fetchNext() {
while (myNext == null) {

if (myCurrentIterator.hasNext()) {
myNext = myCurrentIterator.next();
break;
}

if (!myCurrentIterator.hasNext()) {
int start = myCurrentOffset;
int end = myCurrentOffset + myPageSize;
if (end > myCurrentPids.size()) {
end = myCurrentPids.size();
}
if (end - start <= 0) {
myNext = NO_MORE;
break;
}
myCurrentOffset = end;
Collection<Long> pidsToScan = myCurrentPids.subList(start, end);
Set<Include> includes = Collections.singleton(new Include("*", true));
Set<Long> newPids = loadReverseIncludes(myCallingDao, myContext, myEntityManager, pidsToScan, includes, false, myParams.getLastUpdated());
myCurrentIterator = newPids.iterator();
}

}
}

@Override
public boolean hasNext() {
Expand All @@ -1986,14 +2093,6 @@ public Long next() {
return retVal;
}

private void fetchNext() {
if (myNext == null) {



}
}

}

private enum JoinEnum {
Expand Down Expand Up @@ -2093,7 +2192,7 @@ private void fetchNext() {

if (myNext == null) {
if (myStillNeedToFetchIncludes) {
myIncludesIterator = new IncludesIterator();
myIncludesIterator = new IncludesIterator(myPidSet);
myStillNeedToFetchIncludes = false;
}
if (myIncludesIterator != null) {
Expand Down

0 comments on commit 571045b

Please sign in to comment.