Permalink
Browse files

#1049 Suspend size limit in ElasticSearch scroll search

-Re-factored:
Moved search constants up one level in abstraction:
from ElasticSearch to Search interface
removed: ElasticSearch.SIZE_GET_ALL_RESULTS,
         ElasticSearch.SEARCH_FROM_START
added: Search.GET_ALL_RESULTS, Search.SEARCH_FROM_START_INDEX
  • Loading branch information...
MPDLbrede committed Sep 5, 2018
1 parent aee4a3d commit b0fc2834fde6b92e4ef198e079022ef273611546
@@ -21,7 +21,7 @@
import de.mpg.imeji.logic.model.ContentVO;
import de.mpg.imeji.logic.model.Item;
import de.mpg.imeji.logic.model.User;
import de.mpg.imeji.logic.search.elasticsearch.ElasticSearch;
import de.mpg.imeji.logic.search.Search;
import de.mpg.imeji.logic.search.model.SearchResult;
import de.mpg.imeji.logic.util.ObjectHelper;
import de.mpg.imeji.logic.util.ObjectHelper.ObjectType;
@@ -108,7 +108,7 @@ private void addFiles(ZipOutputStream zip, List<String> itemUris, String parentP
* @throws ImejiException
*/
private List<CollectionElement> retrieveObjects(CollectionImeji collection) throws ImejiException {
final SearchResult itemUIDs = facade.search(null, collection, super.user, null, ElasticSearch.SIZE_GET_ALL_RESULTS, ElasticSearch.SEARCH_FROM_START);
final SearchResult itemUIDs = facade.search(null, collection, super.user, null, Search.GET_ALL_RESULTS, Search.SEARCH_FROM_START_INDEX);
return facade.retrieveItemsAndCollections(itemUIDs.getResults(), user);
}
@@ -16,7 +16,21 @@
* @version $Revision$ $LastChangedDate$
*/
public interface Search {
/**
/**
* Use this constant to state that you want to retrieve all existing search results
* of a query without any size limit
*/
public static final int GET_ALL_RESULTS= -1000; //
/**
* Use this constant to state that you want to retrieve data starting at
* result index 0
*/
public static final int SEARCH_FROM_START_INDEX = 0;
/**
* Types of search (What objects types are retuned)
*
* @author saquet (initial creation)
@@ -46,9 +46,6 @@
private static final int SEARCH_INTERVALL_MAX_SIZE = 500;
private static final int SEARCH_SCROLL_INTERVALL = SEARCH_INTERVALL_MAX_SIZE;
private static final int SEARCH_TO_INDEX_LIMIT = 10000;
public static final int SIZE_GET_ALL_RESULTS= -1000; // use this constant to state that you want to retrieve all existing search results
// of a query without a size limit
public static final int SEARCH_FROM_START = 0;
private static final int SCROLL_TIMEOUT_MSEC = 60000;
/**
@@ -111,12 +108,11 @@ public SearchResult searchWithFacets(SearchQuery query, SortCriterion sortCri, U
private SearchResult searchElasticSearch(SearchQuery query, List<SortCriterion> sortCriteria, User user, String folderUri,
int from, int size, boolean addFacets) {
// magic number "-1" for size is spread all over the code
// semantic meaning: I don't know how much results I want, give me all?
if(size < 0) { size = SEARCH_INTERVALL_MAX_SIZE; }
// magic number "-1" for size is spread all over the code:
if(size != GET_ALL_RESULTS && size < 0) { size = SEARCH_INTERVALL_MAX_SIZE; }
from = from < 0 ? 0 : from;
if (size != SIZE_GET_ALL_RESULTS &&
if (size != GET_ALL_RESULTS &&
size < SEARCH_INTERVALL_MAX_SIZE &&
from + size < SEARCH_TO_INDEX_LIMIT) {
return searchSinglePage(query, sortCriteria, user, folderUri, from, size, addFacets);
@@ -224,7 +220,7 @@ private SearchResult searchWithScroll(SearchQuery query, List<SortCriterion> sor
List<String> retrievedIDs = new ArrayList<String>();
// scroll all results starting at a given index
if(size == SIZE_GET_ALL_RESULTS) {
if(size == GET_ALL_RESULTS) {
retrievedIDs = scrollAllResults(resp, from);
}
// scroll to a size limit starting at a given index

0 comments on commit b0fc283

Please sign in to comment.