From 7c5b070215647137dde868fed1a47c03da96ebcf Mon Sep 17 00:00:00 2001 From: GERey Date: Mon, 16 Mar 2015 11:10:49 -0700 Subject: [PATCH 1/5] Created test that makes 10 cats and gets them back. --- .../applications/queries/GeoPagingTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java index f9fe5b3bfd..38ebeafe78 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java @@ -197,6 +197,48 @@ public void groupQueriesWithConsistentResults() throws IOException { } } + /** + * Test that geo-query returns co-located entities in expected order. + */ + @Test // USERGRID-1401 + public void groupQueriesWithDistanceOrderedResults() throws IOException { + + int maxRangeLimit = 10; + Entity[] cats = new Entity[maxRangeLimit]; + + // 1. Create several entities + for (int i = 0; i < maxRangeLimit; i++) { + float latDelta = i; + Entity cat = new Entity(); + cat.put("name", "cat" + i); + cat.put("location", new MapUtils.HashMapBuilder() + .map("latitude", 37.0+(latDelta/10)) + .map("longitude", (-75.0))); + cats[i] = cat; + this.app().collection("cats").post(cat); + } + this.refreshIndex(); + + QueryParameters params = new QueryParameters(); + String query = String.format( + "select * where location within 1000000 of 37, -75"); + params.setQuery(query); + Collection collection = this.app().collection("cats").get(params); + assertNotNull( collection ); + List entities = collection.getResponse().getEntities(); + assertNotNull( entities ); + + for (int consistent = 0; consistent < maxRangeLimit; consistent++) { +//got entities back, just need to page through them and make sure that i got them in location order. + Object entity = entities.get( consistent ); + assertNotNull( entity ); + + + + + } + } + /** * Creates a store right on top of the center store and checks to see if we can find that store, then find both From 65ff858dbf9b2e30a7322ad4612fd5d8a4120914 Mon Sep 17 00:00:00 2001 From: GERey Date: Mon, 16 Mar 2015 13:15:23 -0700 Subject: [PATCH 2/5] Added test that checks entities written to a collection in reverse and proves that locations queried are returned in collection order instead of distance order. --- .../applications/queries/GeoPagingTest.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java index 38ebeafe78..a444fbdb68 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java @@ -203,17 +203,16 @@ public void groupQueriesWithConsistentResults() throws IOException { @Test // USERGRID-1401 public void groupQueriesWithDistanceOrderedResults() throws IOException { - int maxRangeLimit = 10; - Entity[] cats = new Entity[maxRangeLimit]; + int maxRangeLimit = 9; + Entity[] cats = new Entity[maxRangeLimit+1]; // 1. Create several entities - for (int i = 0; i < maxRangeLimit; i++) { - float latDelta = i; + for (int i = maxRangeLimit; i >= 0; i--) { Entity cat = new Entity(); cat.put("name", "cat" + i); - cat.put("location", new MapUtils.HashMapBuilder() - .map("latitude", 37.0+(latDelta/10)) - .map("longitude", (-75.0))); + cat.put( "location", + new MapUtils.HashMapBuilder().map( "latitude", 37.0 + i ) + .map( "longitude", ( -75.0 + i) ) ); cats[i] = cat; this.app().collection("cats").post(cat); } @@ -221,7 +220,7 @@ public void groupQueriesWithDistanceOrderedResults() throws IOException { QueryParameters params = new QueryParameters(); String query = String.format( - "select * where location within 1000000 of 37, -75"); + "select * where location within 1500000 of 37, -75"); params.setQuery(query); Collection collection = this.app().collection("cats").get(params); assertNotNull( collection ); @@ -229,12 +228,12 @@ public void groupQueriesWithDistanceOrderedResults() throws IOException { assertNotNull( entities ); for (int consistent = 0; consistent < maxRangeLimit; consistent++) { -//got entities back, just need to page through them and make sure that i got them in location order. - Object entity = entities.get( consistent ); + //got entities back, just need to page through them and make sure that i got them in location order. + Entity entity = (Entity) entities.get( consistent ); assertNotNull( entity ); - - - + LinkedHashMap location = ( LinkedHashMap ) entity.get( "location" ); + assertEquals( 37.0+ consistent,location.get( "latitude" ) ); + assertEquals( -75.0+ consistent , location.get( "longitude" ) ); } } From 65c5e22d18d7eaba6e41ff7d84eb51365da6930d Mon Sep 17 00:00:00 2001 From: GERey Date: Tue, 17 Mar 2015 16:28:55 -0700 Subject: [PATCH 3/5] [USERGRID-460] Added ability to make geo sorts possible. Not Working. Refacorted the query code for filterBuilders and queryBuilders in the Query class. --- .../cassandra/QueryProcessorImpl.java | 15 +++- .../index/impl/EsEntityIndexImpl.java | 77 ++++++++++++------- .../persistence/index/impl/EsQueryVistor.java | 61 ++++++++------- .../persistence/index/query/Query.java | 36 ++------- .../index/query/tree/QueryVisitor.java | 5 +- 5 files changed, 107 insertions(+), 87 deletions(-) diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java index 874ff884f9..7c4ea371b0 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java @@ -71,6 +71,7 @@ import org.apache.usergrid.persistence.schema.CollectionInfo; import org.elasticsearch.index.query.FilterBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -292,7 +293,7 @@ public Results getResults( SearchVisitor visitor ) throws Exception { } } if (logger.isDebugEnabled()) { - logger.debug("Getting result for query: [{}], returning entityIds size: {}", + logger.debug("Getting result for query: [{}], returning entityIds size: {}", getQuery(), entityIds.size()); } @@ -621,12 +622,18 @@ public int getSliceCount() { @Override public QueryBuilder getQueryBuilder() { - throw new UnsupportedOperationException("Not supported by this vistor implementation."); + throw new UnsupportedOperationException("Not supported by this vistor implementation."); } @Override public FilterBuilder getFilterBuilder() { - throw new UnsupportedOperationException("Not supported by this vistor implementation."); + throw new UnsupportedOperationException("Not supported by this vistor implementation."); + } + + + @Override + public GeoDistanceSortBuilder getGeoDistanceSortBuilder() { + throw new UnsupportedOperationException("Not supported by this vistor implementation."); } } @@ -724,4 +731,4 @@ private void checkIndexed( String propertyName ) throws NoIndexException { public EntityManager getEntityManager() { return em; } -} \ No newline at end of file +} diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index c92b2991f8..ed1f03a781 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.index.query.CandidateResult; import org.apache.usergrid.persistence.index.query.CandidateResults; import org.apache.usergrid.persistence.index.query.Query; +import org.apache.usergrid.persistence.index.query.tree.QueryVisitor; import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.persistence.map.MapManagerFactory; @@ -64,11 +65,14 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.client.AdminClient; +import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.*; +import org.elasticsearch.index.search.geo.GeoDistanceFilter; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.indices.InvalidAliasNameException; @@ -397,6 +401,14 @@ public EntityIndexBatch createBatch() { } + /** + * Needs REfactor to make clearer what queries and how the information gets used so we don't need to retraverse + * the query every time we generate some information about it. + * @param indexScope + * @param searchTypes + * @param query + * @return + */ @Override public CandidateResults search( final IndexScope indexScope, final SearchTypes searchTypes, final Query query ) { @@ -404,8 +416,9 @@ public CandidateResults search( final IndexScope indexScope, final SearchTypes s final String context = IndexingUtils.createContextName(indexScope); final String[] entityTypes = searchTypes.getTypeNames(); - QueryBuilder qb = query.createQueryBuilder(context); + final QueryVisitor queryVisitor = query.getQueryVisitor(); + QueryBuilder qb = query.createQueryBuilder( context ); SearchResponse searchResponse; @@ -415,8 +428,7 @@ public CandidateResults search( final IndexScope indexScope, final SearchTypes s .setScroll(cursorTimeout + "m") .setQuery(qb); - final FilterBuilder fb = query.createFilterBuilder(); - + final FilterBuilder fb = queryVisitor.getFilterBuilder(); //we have post filters, apply them if ( fb != null ) { logger.debug( " Filter: {} ", fb.toString() ); @@ -426,6 +438,7 @@ public CandidateResults search( final IndexScope indexScope, final SearchTypes s srb = srb.setFrom( 0 ).setSize( query.getLimit() ); + for ( Query.SortPredicate sp : query.getSortPredicates() ) { final SortOrder order; @@ -436,34 +449,44 @@ public CandidateResults search( final IndexScope indexScope, final SearchTypes s order = SortOrder.DESC; } - // we do not know the type of the "order by" property and so we do not know what - // type prefix to use. So, here we add an order by clause for every possible type - // that you can order by: string, number and boolean and we ask ElasticSearch - // to ignore any fields that are not present. - - final String stringFieldName = STRING_PREFIX + sp.getPropertyName(); - final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName ) - .order( order ).ignoreUnmapped( true ); - srb.addSort( stringSort ); - - logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() ); - - final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName(); - final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName ) - .order( order ).ignoreUnmapped( true ); - srb.addSort( numberSort ); - logger.debug( " Sort: {} order by {}", numberFieldName, order.toString() ); - - final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName(); - final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName ) - .order( order ).ignoreUnmapped( true ); - srb.addSort( booleanSort ); - logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() ); + + //By default we want to order geo location queries to be returned by the order defined above first. + /** + * I'm not sure there is any way to build the sort without traversing the tree. + */ + if(fb instanceof GeoDistanceFilterBuilder){ + srb.addSort(queryVisitor.getGeoDistanceSortBuilder().order( SortOrder.ASC ).unit( DistanceUnit.KILOMETERS ).geoDistance( + GeoDistance.SLOPPY_ARC )); + logger.info( " Geo Sort: {} order by {}", sp.getPropertyName(), order.toString() ); + + } + else { + // we do not know the type of the "order by" property and so we do not know what + // type prefix to use. So, here we add an order by clause for every possible type + // that you can order by: string, number and boolean and we ask ElasticSearch + // to ignore any fields that are not present. + final String stringFieldName = STRING_PREFIX + sp.getPropertyName(); + final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true ); + srb.addSort( stringSort ); + + logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() ); + + final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName(); + final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName ).order( order ).ignoreUnmapped( true ); + srb.addSort( numberSort ); + logger.debug( " Sort: {} order by {}", numberFieldName, order.toString() ); + + final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName(); + final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName ).order( order ) + .ignoreUnmapped( true ); + srb.addSort( booleanSort ); + logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() ); + } } if ( logger.isDebugEnabled() ) { - logger.debug( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ", + logger.info( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ", this.alias.getReadAlias(), context, entityTypes, srb ); } diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java index f012babb0a..2e7b174fdf 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java @@ -30,6 +30,8 @@ import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; +import org.elasticsearch.search.sort.SortBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +60,7 @@ /** - * Visits tree of parsed Query operands and populates + * Visits tree of parsed Query operands and populates * ElasticSearch QueryBuilder that represents the query. */ public class EsQueryVistor implements QueryVisitor { @@ -66,8 +68,9 @@ public class EsQueryVistor implements QueryVisitor { Stack stack = new Stack(); List filterBuilders = new ArrayList(); + GeoDistanceSortBuilder geoSortBuilder = null; + - @Override public void visit( AndOperand op ) throws IndexException { @@ -157,14 +160,14 @@ public void visit( ContainsOperand op ) throws NoFullTextIndexException { Object value = op.getLiteral().getValue(); BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR - qb.minimumNumberShouldMatch(1); + qb.minimumNumberShouldMatch(1); // field is an entity/array that needs no name prefix qb = qb.should( QueryBuilders.matchQuery( name, value ) ); // OR field is a string and needs the prefix on the name qb = qb.should( QueryBuilders.matchQuery( addPrefix( value.toString(), name, true), value)); - + stack.push( qb ); } @@ -186,7 +189,8 @@ public void visit( WithinOperand op ) { FilterBuilder fb = FilterBuilders.geoDistanceFilter( name ) .lat( lat ).lon( lon ).distance( distance, DistanceUnit.METERS ); filterBuilders.add( fb ); - } + geoSortBuilder = SortBuilders.geoDistanceSort( "location" ).point( lat,lon ); + } @Override @@ -222,19 +226,19 @@ public void visit( Equal op ) throws NoIndexException { qb.minimumNumberShouldMatch(1); // field is an entity/array that does not need a prefix on its name - // TODO is this right now that we've updated our doc structure? + // TODO is this right now that we've updated our doc structure? // Should this be "must" instead of should? qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) ); - + // or field is just a string that does need a prefix if ( svalue.indexOf("*") != -1 ) { qb = qb.should( QueryBuilders.wildcardQuery( addPrefix( value, name ), svalue ) ); } else { qb = qb.should( QueryBuilders.termQuery( addPrefix( value, name ), value )); - } + } stack.push( qb ); return; - } + } // assume all other types need prefix stack.push( QueryBuilders.termQuery( addPrefix( value, name ), value )); @@ -276,7 +280,7 @@ private String addPrefix( Object value, String origname, boolean analyzed ) { if ( parts.length > 1 ) { name = parts[ parts.length - 1 ]; } - + if ( value instanceof String && analyzed ) { name = addAnalyzedStringPrefix( name ); @@ -293,12 +297,12 @@ private String addPrefix( Object value, String origname, boolean analyzed ) { name = addStringPrefix( name ); } - // re-create nested property name + // re-create nested property name if ( parts.length > 1 ) { parts[parts.length - 1] = name; Joiner joiner = Joiner.on(".").skipNulls(); return joiner.join(parts); - } + } return name; } @@ -307,34 +311,34 @@ private String addPrefix( Object value, String origname, boolean analyzed ) { private String addAnalyzedStringPrefix( String name ) { if ( name.startsWith( ANALYZED_STRING_PREFIX ) ) { return name; - } + } return ANALYZED_STRING_PREFIX + name; - } - + } + private String addStringPrefix( String name ) { if ( name.startsWith( STRING_PREFIX ) ) { return name; - } + } return STRING_PREFIX + name; - } - + } + private String addNumberPrefix( String name ) { if ( name.startsWith( NUMBER_PREFIX ) ) { return name; - } + } return NUMBER_PREFIX + name; - } - + } + private String addBooleanPrefix( String name ) { if ( name.startsWith( BOOLEAN_PREFIX ) ) { return name; - } + } return BOOLEAN_PREFIX + name; - } - + } + @Override public QueryBuilder getQueryBuilder() { @@ -354,14 +358,19 @@ public FilterBuilder getFilterBuilder() { for ( FilterBuilder fb : filterBuilders ) { if ( andFilter == null ) { andFilter = FilterBuilders.andFilter( fb ); - } else { + } else { andFilter = FilterBuilders.andFilter( andFilter, fb ); } - } + } } else if ( !filterBuilders.isEmpty() ) { return filterBuilders.get(0); } return null; } + + @Override + public GeoDistanceSortBuilder getGeoDistanceSortBuilder(){ + return geoSortBuilder; + } } diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java index da68772a96..ed6d19eec9 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java @@ -103,6 +103,7 @@ public enum Level { private List counterFilters; private String collection; private String ql; + private QueryVisitor visitor; private static ObjectMapper mapper = new ObjectMapper(); @@ -146,27 +147,7 @@ public Query( Query q ) { public QueryBuilder createQueryBuilder( final String context ) { - QueryBuilder queryBuilder = null; - - - //we have a root operand. Translate our AST into an ES search - if ( getRootOperand() != null ) { - // In the case of geo only queries, this will return null into the query builder. - // Once we start using tiles, we won't need this check any longer, since a geo query - // will return a tile query + post filter - QueryVisitor v = new EsQueryVistor(); - - try { - getRootOperand().visit( v ); - } - catch ( IndexException ex ) { - throw new RuntimeException( "Error building ElasticSearch query", ex ); - } - - - queryBuilder = v.getQueryBuilder(); - } - + QueryBuilder queryBuilder = getQueryVisitor().getQueryBuilder(); // Add our filter for context to our query for fast execution. // Fast because it utilizes bitsets internally. See this post for more detail. @@ -188,23 +169,20 @@ public QueryBuilder createQueryBuilder( final String context ) { } - public FilterBuilder createFilterBuilder() { - FilterBuilder filterBuilder = null; + public QueryVisitor getQueryVisitor() { + + QueryVisitor v = new EsQueryVistor(); if ( getRootOperand() != null ) { - QueryVisitor v = new EsQueryVistor(); try { getRootOperand().visit( v ); } catch ( IndexException ex ) { throw new RuntimeException( "Error building ElasticSearch query", ex ); } - filterBuilder = v.getFilterBuilder(); } - - return filterBuilder; - } - + return v; + } /** * Create a query instance from the QL. If the string is null, return an empty query diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java index d295c9267f..943718c8bc 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java @@ -24,6 +24,7 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException; import org.elasticsearch.index.query.FilterBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; /** @@ -93,10 +94,12 @@ public interface QueryVisitor { */ public void visit( GreaterThanEqual op ) throws NoIndexException; - /** + /** * Returns resulting query builder. */ public QueryBuilder getQueryBuilder(); public FilterBuilder getFilterBuilder(); + + public GeoDistanceSortBuilder getGeoDistanceSortBuilder(); } From 6f3824b8b4211b8170f4b504bb32058ebfbcc0d7 Mon Sep 17 00:00:00 2001 From: GERey Date: Tue, 17 Mar 2015 16:45:53 -0700 Subject: [PATCH 4/5] [USERGRID-460] Fixed GeoPaging by adding correct sorting prefix. --- .../apache/usergrid/persistence/index/impl/EsQueryVistor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java index 2e7b174fdf..6d2a1f470a 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java @@ -189,7 +189,7 @@ public void visit( WithinOperand op ) { FilterBuilder fb = FilterBuilders.geoDistanceFilter( name ) .lat( lat ).lon( lon ).distance( distance, DistanceUnit.METERS ); filterBuilders.add( fb ); - geoSortBuilder = SortBuilders.geoDistanceSort( "location" ).point( lat,lon ); + geoSortBuilder = SortBuilders.geoDistanceSort( GEO_PREFIX+"location" ).point( lat,lon ); } From 4964ab62a3c846721aef657b36002b2a549f1674 Mon Sep 17 00:00:00 2001 From: GERey Date: Wed, 18 Mar 2015 13:20:12 -0700 Subject: [PATCH 5/5] Cleaned some of the GeoPagingTest comments and added a test that ensures consistent order from elastic search. --- .../apache/usergrid/persistence/GeoIT.java | 25 +++++++++++++++++-- .../applications/queries/GeoPagingTest.java | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java index a1ac4ff96a..ad8a0dd579 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java @@ -420,8 +420,13 @@ public void testPointPaging() throws Exception { } + /** + * Verify that elasticsearch does a secondary ordering on paging such that we get consistent results + * back from a cursor despite having a geoquery with all the positions in the same location. + * @throws Exception + */ @Test - public void testSamePointPaging() throws Exception { + public void testSamePointConsistantPaging() throws Exception { EntityManager em = app.getEntityManager(); assertNotNull(em); @@ -445,15 +450,31 @@ public void testSamePointPaging() throws Exception { // just to be save query.addFilter("location within 50000000 of 0, 0"); query.setLimit(100); + List names = new ArrayList(); int count = 0; Results results; + //get arraylist of entities from a search + do { + results = em.searchCollection(em.getApplicationRef(), "stores", query); + for (Entity entity : results.getEntities()) { + names.add( count,entity.getName() ); + count++; + } + + // set for the next "page" + query.setCursor(results.getCursor()); + } + while (results.getCursor() != null); + //verify that entities come out in the same order when doing the same query against the same data. + //aka make sure the elasticsearch does a secondary search. + count = 0; do { results = em.searchCollection(em.getApplicationRef(), "stores", query); for (Entity entity : results.getEntities()) { - assertEquals(String.valueOf(count), entity.getName()); + assertEquals( names.get( count ),entity.getName() ); count++; } diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java index a444fbdb68..553b387a97 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java @@ -200,7 +200,7 @@ public void groupQueriesWithConsistentResults() throws IOException { /** * Test that geo-query returns co-located entities in expected order. */ - @Test // USERGRID-1401 + @Test public void groupQueriesWithDistanceOrderedResults() throws IOException { int maxRangeLimit = 9;