WFS: add WEB.METADATA wfs_features_cache_count and wfs_features_cache…

…_size items wfs_features_cache_count determines how many shapes are stores in resultCacheObj*, so as to be used immediately by GML or OGR output, without querying again the dataset. wfs_features_cache_size is the limit in bytes of the space taken by those shapes. Both items can be set. The most limiting will be honoured. CREDITS: Funded by: Regione Toscana - Settore Sistema Informativo Territoriale ed Ambientale (CIG: 6038019AE5)
MapServer · Dec 1, 2016 · 3c39c91 · 3c39c91
1 parent b8b57c9
commit 3c39c91
Show file tree

Hide file tree

Showing 12 changed files with 842 additions and 27 deletions.
diff --git a/mapfile.c b/mapfile.c
@@ -3953,7 +3953,7 @@ int freeLayer(layerObj *layer)
     freeFeatureList(layer->features);
 
   if(layer->resultcache) {
-    if(layer->resultcache->results) free(layer->resultcache->results);
+    cleanupResultCache(layer->resultcache);
     msFree(layer->resultcache);
   }
 
@@ -7409,6 +7409,28 @@ void initResultCache(resultCacheObj *resultcache)
   }
 }
 
+void cleanupResultCache(resultCacheObj *resultcache)
+{
+  if(resultcache) {
+    if(resultcache->results)
+    {
+        int i;
+        for( i = 0; i < resultcache->numresults; i++ )
+        {
+            if( resultcache->results[i].shape )
+            {
+                msFreeShape( resultcache->results[i].shape );
+                msFree( resultcache->results[i].shape );
+            }
+        }
+        free(resultcache->results);
+    }
+    resultcache->results = NULL;
+    initResultCache(resultcache);
+  }
+}
+
+
 static int resolveSymbolNames(mapObj* map)
 {
   int i, j;

diff --git a/mapgml.c b/mapgml.c
@@ -1745,14 +1745,22 @@ int msGMLWriteWFSQuery(mapObj *map, FILE *stream, const char *default_namespace_
       for(j=0; j<lp->resultcache->numresults; j++) {
         char* pszFID;
 
-        status = msLayerGetShape(lp, &shape, &(lp->resultcache->results[j]));
-        if(status != MS_SUCCESS) {
-          msGMLFreeGroups(groupList);
-          msGMLFreeConstants(constantList);
-          msGMLFreeItems(itemList);
-          msGMLFreeGeometries(geometryList);
-          msFree(layerName);
-          return(status);
+        if( lp->resultcache->results[j].shape )
+        {
+            /* msDebug("Using cached shape %ld\n", lp->resultcache->results[j].shapeindex); */
+            msCopyShape(lp->resultcache->results[j].shape, &shape);
+        }
+        else
+        {
+            status = msLayerGetShape(lp, &shape, &(lp->resultcache->results[j]));
+            if(status != MS_SUCCESS) {
+                msGMLFreeGroups(groupList);
+                msGMLFreeConstants(constantList);
+                msGMLFreeItems(itemList);
+                msGMLFreeGeometries(geometryList);
+                msFree(layerName);
+                return(status);
+            }
         }
 
 #ifdef USE_PROJ

diff --git a/mapogcfiltercommon.c b/mapogcfiltercommon.c
@@ -700,14 +700,23 @@ int FLTApplyFilterToLayerCommonExpressionWithRect(mapObj *map, int iLayerIndex,
   int save_startindex;
   int save_maxfeatures;
   int save_only_cache_result_count;
+  int save_cache_shapes;
+  int save_max_cached_shape_count;
+  int save_max_cached_shape_ram_amount;
 
   save_startindex = map->query.startindex;
   save_maxfeatures = map->query.maxfeatures;
   save_only_cache_result_count = map->query.only_cache_result_count;
+  save_cache_shapes = map->query.cache_shapes;
+  save_max_cached_shape_count = map->query.max_cached_shape_count;
+  save_max_cached_shape_ram_amount = map->query.max_cached_shape_ram_amount;
   msInitQuery(&(map->query));
   map->query.startindex = save_startindex;
   map->query.maxfeatures = save_maxfeatures;
   map->query.only_cache_result_count = save_only_cache_result_count;
+  map->query.cache_shapes = save_cache_shapes;
+  map->query.max_cached_shape_count = save_max_cached_shape_count;
+  map->query.max_cached_shape_ram_amount = save_max_cached_shape_ram_amount;
 
   map->query.mode = MS_QUERY_MULTIPLE;
   map->query.layer = iLayerIndex;

diff --git a/mapogroutput.c b/mapogroutput.c
@@ -1079,13 +1079,21 @@ int msOGRWriteFromQuery( mapObj *map, outputFormatObj *format, int sendheaders )
       /*
       ** Read the shape.
       */
-      status = msLayerGetShape(layer, &resultshape, &(layer->resultcache->results[i]));
-      if(status != MS_SUCCESS) {
-        OGR_DS_Destroy( hDS );
-        msOGRCleanupDS( datasource_name );
-        msGMLFreeItems(item_list);
-        msFreeShape(&resultshape);
-        return status;
+      if( layer->resultcache->results[i].shape )
+      {
+          /* msDebug("Using cached shape %ld\n", layer->resultcache->results[i].shapeindex); */
+          msCopyShape(layer->resultcache->results[i].shape, &resultshape);
+      }
+      else
+      {
+        status = msLayerGetShape(layer, &resultshape, &(layer->resultcache->results[i]));
+        if(status != MS_SUCCESS) {
+            OGR_DS_Destroy( hDS );
+            msOGRCleanupDS( datasource_name );
+            msGMLFreeItems(item_list);
+            msFreeShape(&resultshape);
+            return status;
+        }
       }
 
       /*

diff --git a/mapprimitive.c b/mapprimitive.c
@@ -169,6 +169,27 @@ void msFreeShape(shapeObj *shape)
   msInitShape(shape); /* now reset */
 }
 
+int msGetShapeRAMSize(shapeObj* shape)
+{
+    int i;
+    int size = 0;
+    size += sizeof(shapeObj);
+    size += shape->numlines * sizeof(lineObj);
+    for (i = 0; i < shape->numlines; i++)
+    {
+        size += shape->line[i].numpoints * sizeof(pointObj);
+    }
+    size += shape->numvalues * sizeof(char*);
+    for( i = 0; i < shape->numvalues; i++ )
+    {
+        if( shape->values[i] )
+            size += strlen( shape->values[i] ) + 1;
+    }
+    if( shape->text )
+        size += strlen( shape->text ) + 1;
+    return size;
+}
+
 void msFreeLabelPathObj(labelPathObj *path)
 {
   msFreeShape(&(path->bounds));

diff --git a/mapquery.c b/mapquery.c
@@ -30,6 +30,21 @@
 #include "mapserver.h"
 #include "mapows.h"
 
+/* This object is used by the various mapQueryXXXXX() functions. It stores
+ * the total amount of shapes and their RAM footprint, when they are cached
+ * in the resultCacheObj* of layers. This is the total number accross all queried
+ * layers. However this is isn't persistant accross several calls to
+ * mapQueryXXXXX(), if the resultCacheObj* objects weren't cleaned up. This
+ * is not needed in the context of WFS, for which this is used for now.
+ */
+typedef struct
+{
+    int cachedShapeCountWarningEmitted;
+    int cachedShapeCount;
+    int cachedShapeRAMWarningEmitted;
+    int cachedShapeRAM;
+} queryCacheObj;
+
 int msInitQuery(queryObj *query) {
   if(!query) return MS_FAILURE;
 
@@ -56,6 +71,10 @@ int msInitQuery(queryObj *query) {
 
   query->filteritem = NULL;
   msInitExpression(&query->filter);
+
+  query->cache_shapes = MS_FALSE;
+  query->max_cached_shape_count = 0;
+  query->max_cached_shape_ram_amount = 0;
 
   return MS_SUCCESS;
 }
@@ -133,9 +152,60 @@ int msIsLayerQueryable(layerObj *lp)
   return MS_FALSE;
 }
 
-static int addResult(resultCacheObj *cache, shapeObj *shape)
+static void initQueryCache(queryCacheObj* queryCache)
+{
+    queryCache->cachedShapeCountWarningEmitted = MS_FALSE;
+    queryCache->cachedShapeCount = 0;
+    queryCache->cachedShapeRAMWarningEmitted = MS_FALSE;
+    queryCache->cachedShapeRAM = 0;
+}
+
+/** Check whether we should store the shape in resultCacheObj* given the
+ * limits allowed in map->query.max_cached_shape_count and
+ * map->query.max_cached_shape_ram_amount.
+ */
+static int canCacheShape(mapObj* map, queryCacheObj *queryCache,
+                         shapeObj* shape, int shape_ram_size)
+{
+  if( !map->query.cache_shapes )
+      return MS_FALSE;
+  if( queryCache->cachedShapeCountWarningEmitted ||
+      (map->query.max_cached_shape_count > 0 &&
+      queryCache->cachedShapeCount >= map->query.max_cached_shape_count) )
+  {
+      if( !queryCache->cachedShapeCountWarningEmitted )
+      {
+          queryCache->cachedShapeCountWarningEmitted = MS_TRUE;
+          msDebug("map->query.max_cached_shape_count = %d reached. "
+                  "Next features will not be cached.\n",
+                  map->query.max_cached_shape_count);
+      }
+      return MS_FALSE;
+  }
+  if( queryCache->cachedShapeRAMWarningEmitted ||
+      (map->query.max_cached_shape_ram_amount > 0 &&
+       queryCache->cachedShapeRAM + shape_ram_size > map->query.max_cached_shape_ram_amount) )
+  {
+      if( !queryCache->cachedShapeRAMWarningEmitted )
+      {
+          queryCache->cachedShapeRAMWarningEmitted = MS_TRUE;
+          msDebug("map->query.max_cached_shape_ram_amount = %d reached after %d cached features. "
+                  "Next features will not be cached.\n",
+                  map->query.max_cached_shape_ram_amount,
+                  queryCache->cachedShapeCount);
+      }
+      return MS_FALSE;
+  }
+  return MS_TRUE;
+}
+
+static int addResult(mapObj* map, resultCacheObj *cache,
+                     queryCacheObj* queryCache, shapeObj *shape)
 {
   int i;
+  int shape_ram_size = (map->query.max_cached_shape_ram_amount > 0) ? 
+                                            msGetShapeRAMSize( shape ) : 0;
+  int store_shape = canCacheShape (map, queryCache, shape, shape_ram_size);
 
   if(cache->numresults == cache->cachesize) { /* just add it to the end */
     if(cache->cachesize == 0)
@@ -155,6 +225,16 @@ static int addResult(resultCacheObj *cache, shapeObj *shape)
   cache->results[i].tileindex = shape->tileindex;
   cache->results[i].shapeindex = shape->index;
   cache->results[i].resultindex = shape->resultindex;
+  if( store_shape )
+  {
+      cache->results[i].shape = (shapeObj*)msSmallMalloc(sizeof(shapeObj));
+      msInitShape(cache->results[i].shape);
+      msCopyShape(shape, cache->results[i].shape);
+      queryCache->cachedShapeCount ++;
+      queryCache->cachedShapeRAM += shape_ram_size;
+  }
+  else
+      cache->results[i].shape = NULL;
   cache->numresults++;
 
   cache->previousBounds = cache->bounds;
@@ -471,10 +551,13 @@ int msQueryByIndex(mapObj *map)
   int status;
 
   resultObj record;
+  queryCacheObj queryCache;
 
   shapeObj shape;
   double minfeaturesize = -1;
 
+  initQueryCache(&queryCache);
+
   if(map->query.type != MS_QUERY_BY_INDEX) {
     msSetError(MS_QUERYERR, "The query is not properly defined.", "msQueryByIndex()");
     return(MS_FAILURE);
@@ -494,7 +577,7 @@ int msQueryByIndex(mapObj *map)
 
   if(map->query.clear_resultcache) {
     if(lp->resultcache) {
-      if(lp->resultcache->results) free(lp->resultcache->results);
+      cleanupResultCache(lp->resultcache);
       free(lp->resultcache);
       lp->resultcache = NULL;
     }
@@ -568,7 +651,7 @@ int msQueryByIndex(mapObj *map)
     return(MS_FAILURE);
   }
 
-  addResult(lp->resultcache, &shape);
+  addResult(map, lp->resultcache, &queryCache, &shape);
 
   msFreeShape(&shape);
   /* msLayerClose(lp); */
@@ -659,6 +742,9 @@ int msQueryByFilter(mapObj *map)
   int nclasses = 0;
   int *classgroup = NULL;
   double minfeaturesize = -1;
+  queryCacheObj queryCache;
+
+  initQueryCache(&queryCache);
 
   if(map->query.type != MS_QUERY_BY_FILTER) {
     msSetError(MS_QUERYERR, "The query is not properly defined.", "msQueryByFilter()");
@@ -865,7 +951,7 @@ int msQueryByFilter(mapObj *map)
       if( map->query.only_cache_result_count )
         lp->resultcache->numresults ++;
       else
-        addResult(lp->resultcache, &shape);
+        addResult(map, lp->resultcache, &queryCache, &shape);
       msFreeShape(&shape);
 
       if(map->query.mode == MS_QUERY_SINGLE) { /* no need to look any further */
@@ -925,6 +1011,9 @@ int msQueryByRect(mapObj *map)
   int nclasses = 0;
   int *classgroup = NULL;
   double minfeaturesize = -1;
+  queryCacheObj queryCache;
+
+  initQueryCache(&queryCache);
 
   if(map->query.type != MS_QUERY_BY_RECT) {
     msSetError(MS_QUERYERR, "The query is not properly defined.", "msQueryByRect()");
@@ -1156,7 +1245,7 @@ int msQueryByRect(mapObj *map)
         if( map->query.only_cache_result_count )
             lp->resultcache->numresults ++;
         else
-            addResult(lp->resultcache, &shape);
+            addResult(map, lp->resultcache, &queryCache, &shape);
         --map->query.maxfeatures;
       }
       msFreeShape(&shape);
@@ -1218,6 +1307,10 @@ int msQueryByFeatures(mapObj *map)
   int *classgroup = NULL;
   double minfeaturesize = -1;
 
+  queryCacheObj queryCache;
+
+  initQueryCache(&queryCache);
+
   if(map->debug) msDebug("in msQueryByFeatures()\n");
 
   /* is the selection layer valid and has it been queried */
@@ -1474,7 +1567,7 @@ int msQueryByFeatures(mapObj *map)
             msFreeShape(&shape);
             continue;
           }
-          addResult(lp->resultcache, &shape);
+          addResult(map, lp->resultcache, &queryCache, &shape);
         }
         msFreeShape(&shape);
 
@@ -1538,6 +1631,10 @@ int msQueryByPoint(mapObj *map)
   int *classgroup = NULL;
   double minfeaturesize = -1;
 
+  queryCacheObj queryCache;
+
+  initQueryCache(&queryCache);
+
   if(map->query.type != MS_QUERY_BY_POINT) {
     msSetError(MS_QUERYERR, "The query is not properly defined.", "msQueryByPoint()");
     return(MS_FAILURE);
@@ -1693,11 +1790,12 @@ int msQueryByPoint(mapObj *map)
         }
 
         if(map->query.mode == MS_QUERY_SINGLE) {
-          lp->resultcache->numresults = 0;
-          addResult(lp->resultcache, &shape);
+          cleanupResultCache(lp->resultcache);
+          initQueryCache(&queryCache);
+          addResult(map, lp->resultcache, &queryCache, &shape);
           t = d; /* next one must be closer */
         } else {
-          addResult(lp->resultcache, &shape);
+          addResult(map, lp->resultcache, &queryCache, &shape);
         }
       }
 
@@ -1748,6 +1846,9 @@ int msQueryByShape(mapObj *map)
   int nclasses = 0;
   int *classgroup = NULL;
   double minfeaturesize = -1;
+  queryCacheObj queryCache;
+
+  initQueryCache(&queryCache);
 
   if(map->query.type != MS_QUERY_BY_SHAPE) {
     msSetError(MS_QUERYERR, "The query is not properly defined.", "msQueryByShape()");
@@ -1981,7 +2082,7 @@ int msQueryByShape(mapObj *map)
           msFreeShape(&shape);
           continue;
         }
-        addResult(lp->resultcache, &shape);
+        addResult(map, lp->resultcache, &queryCache, &shape);
       }
       msFreeShape(&shape);