fix for 1105: postgis query too large

There are two strategies in this fix: 1. The method that makes the getFeature requests in the spatial filter now limits the number of features to 5000 and will make multiple requests 2. A performance enhancement. It will limit the number of features in get features to numHits. So if we are loading hits 10-15 at most 15 features will be loaded.
geonetwork · Oct 10, 2012 · 76fcaf6 · 76fcaf6
1 parent 99daf8d
commit 76fcaf6
Show file tree

Hide file tree

Showing 20 changed files with 72 additions and 51 deletions.
diff --git a/docs b/docs
diff --git a/gast b/gast
diff --git a/geoserver b/geoserver
diff --git a/installer b/installer
diff --git a/maven_repo b/maven_repo
diff --git a/release b/release
diff --git a/web/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/CatalogSearcher.java b/web/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/CatalogSearcher.java
@@ -368,13 +368,14 @@ private Pair<Element, List<ResultItem>> performSearch(
 		// --- proper search
 		Log.debug(Geonet.CSW_SEARCH, "Lucene query: " + query.toString());
 
+        int numHits = startPosition + maxRecords;
+
 		// TODO Handle NPE creating spatial filter (due to constraint
 		// language version).
-		Filter spatialfilter = sm.getSpatial().filter(query, filterExpr, filterVersion);
+		Filter spatialfilter = sm.getSpatial().filter(query, numHits, filterExpr, filterVersion);
 		CachingWrapperFilter cFilter = null;
 		if (spatialfilter != null) cFilter = new CachingWrapperFilter(spatialfilter);
 		boolean buildSummary = resultType == ResultType.RESULTS_WITH_SUMMARY;
-		int numHits = startPosition + maxRecords;
 
 			// get as many results as instructed or enough for search summary
 		if (buildSummary) {

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/LuceneSearcher.java b/web/src/main/java/org/fao/geonet/kernel/search/LuceneSearcher.java
@@ -185,8 +185,8 @@ public final FieldSelectorResult accept(String name) {
 	public void search(ServiceContext srvContext, Element request, ServiceConfig config) throws Exception {
 		IndexSearcher searcher = getNewSearcher();
 		try {
-			computeQuery(srvContext, request, config, searcher);
 			initSearchRange(srvContext);
+			computeQuery(srvContext, request, config, searcher);
 			performQuery(getFrom()-1, getTo(), searcher);
 		} finally {
 			releaseSearcher();
@@ -435,7 +435,7 @@ else if (userSession.getProfile().equals(Geonet.Profile.REVIEWER)) {
 
 		Geometry geometry = getGeometry(request);
         if (geometry != null) {
-            _filter = new CachingWrapperFilter(_sm.getSpatial().filter(_query, geometry, request));
+            _filter = new CachingWrapperFilter(_sm.getSpatial().filter(_query, getTo() - 1, geometry, request));
         }
 
         String sortBy = Util.getParam(request, Geonet.SearchResult.SORT_BY,

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/SearchManager.java b/web/src/main/java/org/fao/geonet/kernel/search/SearchManager.java
@@ -1218,15 +1218,13 @@ public void run()
          * @return
          * @throws Exception
          */
-        public Filter filter(org.apache.lucene.search.Query query, Element filterExpr, String filterVersion)
-                throws Exception
-        {
-
+        public Filter filter(org.apache.lucene.search.Query query, int numHits, Element filterExpr, String filterVersion)
+                throws Exception {
             _lock.lock();
             try {
             	Parser filterParser = getFilterParser(filterVersion);
                 Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> accessor = new SpatialIndexAccessor();
-                return OgcGenericFilters.create(query, filterExpr, accessor , filterParser);
+                return OgcGenericFilters.create(query, numHits, filterExpr, accessor , filterParser);
             }
             catch (Exception e) {
             	// TODO Handle NPE creating spatial filter (due to constraint language version).
@@ -1247,15 +1245,14 @@ public Filter filter(org.apache.lucene.search.Query query, Element filterExpr, S
          * @return
          * @throws Exception
          */
-        public SpatialFilter filter(org.apache.lucene.search.Query query,
-                Geometry geom, Element request) throws Exception
-        {
+        public SpatialFilter filter(org.apache.lucene.search.Query query, int numHits,
+                Geometry geom, Element request) throws Exception {
             _lock.lock();
             try {
                 String relation = Util.getParam(request,
                         Geonet.SearchResult.RELATION,
                         Geonet.SearchResult.Relation.INTERSECTION);
-                return _types.get(relation).newInstance(query, geom, new SpatialIndexAccessor());
+                return _types.get(relation).newInstance(query, numHits, geom, new SpatialIndexAccessor());
             }
             finally {
                 _lock.unlock();
@@ -1365,7 +1362,7 @@ public void run()
      */
     private static Constructor<? extends SpatialFilter> constructor(Class<? extends SpatialFilter> clazz)
             throws SecurityException, NoSuchMethodException {
-        return clazz.getConstructor(org.apache.lucene.search.Query.class, Geometry.class, Pair.class);
+        return clazz.getConstructor(org.apache.lucene.search.Query.class, int.class, Geometry.class, Pair.class);
     }
 
 }
diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/ContainsFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/ContainsFilter.java
@@ -48,9 +48,9 @@ public class ContainsFilter extends SpatialFilter
 
     private static final long serialVersionUID = 1114543251684147194L;
 
-    public ContainsFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    public ContainsFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
     protected SpatialOperator createGeomFilter(FilterFactory2 filterFactory,

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/CrossesFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/CrossesFilter.java
@@ -48,9 +48,9 @@ public class CrossesFilter extends SpatialFilter
 
     private static final long serialVersionUID = 1114543251684147194L;
 
-    public CrossesFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    public CrossesFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
     protected SpatialOperator createGeomFilter(FilterFactory2 filterFactory,

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/EqualsFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/EqualsFilter.java
@@ -48,9 +48,9 @@ public class EqualsFilter extends SpatialFilter
 
     private static final long serialVersionUID = 1114543251684147194L;
 
-    public EqualsFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    public EqualsFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
     protected SpatialOperator createGeomFilter(FilterFactory2 filterFactory,

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/FullScanFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/FullScanFilter.java
@@ -63,16 +63,16 @@ public class FullScanFilter extends SpatialFilter
     private static final long serialVersionUID = 1114543251684147194L;
     private Set<String>       _matches;
 
-    public FullScanFilter(Query query, Geometry geom,
+    public FullScanFilter(Query query, int numHits, Geometry geom,
             Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
-    protected FullScanFilter(Query query, Envelope bounds,
+    protected FullScanFilter(Query query, int numHits, Envelope bounds,
             Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, bounds, sourceAccessor);
+        super(query, numHits, bounds, sourceAccessor);
     }
 
     public BitSet bits(final IndexReader reader) throws IOException

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/IntersectionFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/IntersectionFilter.java
@@ -48,9 +48,9 @@ public class IntersectionFilter extends SpatialFilter
 
     private static final long serialVersionUID = 1114543251684147194L;
 
-    public IntersectionFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    public IntersectionFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
     protected SpatialOperator createGeomFilter(FilterFactory2 filterFactory,

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/IsFullyOutsideOfFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/IsFullyOutsideOfFilter.java
@@ -42,9 +42,9 @@ public class IsFullyOutsideOfFilter  extends SpatialFilter
 
     private static final long serialVersionUID = 1114543251684147194L;
 
-    public IsFullyOutsideOfFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    public IsFullyOutsideOfFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
     protected SpatialOperator createGeomFilter(FilterFactory2 filterFactory,

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/OgcGenericFilters.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/OgcGenericFilters.java
@@ -87,7 +87,7 @@ public class OgcGenericFilters
      * @throws Exception
      */
     @SuppressWarnings("serial")
-    public static SpatialFilter create(Query query,
+    public static SpatialFilter create(Query query, int numHits,
             Element filterExpr, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor, Parser parser) throws Exception
     {
 		// -- parse Filter and report any validation issues
@@ -136,15 +136,15 @@ public static SpatialFilter create(Query query,
 
         Boolean disjointFilter = (Boolean) finalFilter.accept(new DisjointDetector(), false);
         if( disjointFilter ){
-            return new FullScanFilter(query, bounds, sourceAccessor){
+            return new FullScanFilter(query, numHits, bounds, sourceAccessor){
                 @Override
                 protected Filter createFilter(FeatureSource source)
                 {
                     return finalFilter;
                 }
             };
         }else{
-            return new SpatialFilter(query, bounds, sourceAccessor){
+            return new SpatialFilter(query, numHits, bounds, sourceAccessor){
                 @Override
                 protected Filter createFilter(FeatureSource source)
                 {

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/OverlapsFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/OverlapsFilter.java
@@ -48,9 +48,9 @@ public class OverlapsFilter extends SpatialFilter
 
     private static final long serialVersionUID = 1114543251684147194L;
 
-    public OverlapsFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    public OverlapsFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
     protected SpatialOperator createGeomFilter(FilterFactory2 filterFactory,

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/SpatialFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/SpatialFilter.java
@@ -61,8 +61,10 @@
 
 import java.io.IOException;
 import java.util.BitSet;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -79,6 +81,7 @@ public abstract class SpatialFilter extends Filter
     }
 
 	private static final Geometry WORLD_BOUNDS;
+	private static final int MAX_FIDS_PER_QUERY = 5000;
 	static {
 		GeometryFactory fac = new GeometryFactory();
 		WORLD_BOUNDS = fac.toGeometry(new Envelope(-180,180,-90,90));
@@ -92,12 +95,14 @@ public abstract class SpatialFilter extends Filter
     protected final FieldSelector _selector;
     private Map<String, FeatureId> _unrefinedMatches;
     private boolean warned = false;
+	private int _numHits;
+	private int _hits = 0;
 
-
-    protected SpatialFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    protected SpatialFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
         _query = query;
         _geom = geom;
+        _numHits = numHits;
         this.sourceAccessor = sourceAccessor;
         _filterFactory = CommonFactoryFinder.getFilterFactory2(GeoTools
                 .getDefaultHints());
@@ -110,9 +115,9 @@ public final FieldSelectorResult accept(String name) {
 				};
     }
 
-    protected SpatialFilter(Query query, Envelope bounds, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    protected SpatialFilter(Query query, int numHits, Envelope bounds, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        this(query,JTS.toGeometry(bounds),sourceAccessor);
+        this(query,numHits,JTS.toGeometry(bounds),sourceAccessor);
     }
 
     public DocIdSet getDocIdSet(final IndexReader reader) throws IOException
@@ -123,8 +128,9 @@ public DocIdSet getDocIdSet(final IndexReader reader) throws IOException
         final Set<FeatureId> matches = new HashSet<FeatureId>();
         final Map<FeatureId,Integer> docIndexLookup = new HashMap<FeatureId,Integer>();
 
-        new IndexSearcher(reader).search(_query, new Collector()
-        {
+        if(unrefinedSpatialMatches.isEmpty() || _hits >= _numHits) return new DocIdBitSet(bits);
+
+        new IndexSearcher(reader).search(_query, new Collector() {
 						private int docBase;
 
 						//ignore scorer
@@ -142,7 +148,8 @@ public final void collect(int doc)
                     Document document = reader.document(doc, _selector);
                     String key = document.get("_id");
                     FeatureId featureId = unrefinedSpatialMatches.get(key); 
-                    if (featureId!=null) {
+                    if (featureId!=null && _hits < _numHits) {
+						_hits ++ ;
                         matches.add(featureId);
                         docIndexLookup.put(featureId, doc);
                     }
@@ -165,7 +172,22 @@ public void setNextReader(IndexReader reader, int docBase) {
 
     private BitSet applySpatialFilter(Set<FeatureId> matches, Map<FeatureId, Integer> docIndexLookup, BitSet bits) throws IOException
     {
-        Id fidFilter = _filterFactory.id(matches);
+
+        while (!matches.isEmpty()) {
+        	Id fidFilter;
+        	if(matches.size() > MAX_FIDS_PER_QUERY) {
+        		Set<FeatureId> subset = new HashSet<FeatureId>((int)(MAX_FIDS_PER_QUERY*1.30));
+        		int i = 0;
+        		Iterator<FeatureId> iter = matches.iterator();
+        		while(iter.hasNext() && i < MAX_FIDS_PER_QUERY) {
+        			subset.add(iter.next());
+        			iter.remove();
+        		}
+        		fidFilter = _filterFactory.id(subset);
+	        } else {
+	        	fidFilter = _filterFactory.id(matches);
+	        	matches = Collections.emptySet();
+	        }
         FeatureSource<SimpleFeatureType, SimpleFeature> _featureSource = sourceAccessor.one();
         String ftn = _featureSource.getSchema().getName().getLocalPart();
         String[] geomAtt = {_featureSource.getSchema().getGeometryDescriptor().getLocalName()};
@@ -185,6 +207,7 @@ private BitSet applySpatialFilter(Set<FeatureId> matches, Map<FeatureId, Integer
         } finally {
             iterator.close();
         }
+		}
         return bits;
     }
 

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/TouchesFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/TouchesFilter.java
@@ -48,9 +48,9 @@ public class TouchesFilter extends SpatialFilter
 
     private static final long serialVersionUID = 1114543251684147194L;
 
-    public TouchesFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    public TouchesFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
     protected SpatialOperator createGeomFilter(FilterFactory2 filterFactory,

diff --git a/web/src/main/java/org/fao/geonet/kernel/search/spatial/WithinFilter.java b/web/src/main/java/org/fao/geonet/kernel/search/spatial/WithinFilter.java
@@ -48,9 +48,9 @@ public class WithinFilter extends SpatialFilter
 
     private static final long serialVersionUID = 1114543251684147194L;
 
-    public WithinFilter(Query query, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
+    public WithinFilter(Query query, int numHits, Geometry geom, Pair<FeatureSource<SimpleFeatureType, SimpleFeature>, SpatialIndex> sourceAccessor) throws IOException
     {
-        super(query, geom, sourceAccessor);
+        super(query, numHits, geom, sourceAccessor);
     }
 
     protected SpatialOperator createGeomFilter(FilterFactory2 filterFactory,