Merge branch 'MapServer:main' into map2img

CMakeLists.txt

@@ -182,6 +182,8 @@ option(BUILD_STATIC "Also build a static version of mapserver" OFF)
 option(LINK_STATIC_LIBMAPSERVER "Link to static version of libmapserver (also for mapscripts)" OFF)
 option(WITH_APACHE_MODULE "include (experimental) support for apache module" OFF)
 option(WITH_GENERIC_NINT "generic rounding" OFF)
+# the following requires a customized SWIG build - see https://github.com/swig/swig/pull/1951
+option(WITH_MAPSCRIPT_PROPERTYANNOTATIONS "Add annotations to Python mapscript class properties" OFF)
 
 #TODO: USE_OGL? , USE_SDE, USE_CLUSTER_EXTERNAL USE_CLUSTER_PLUGIN, USE_MSSQL2008, USE_MSSQL2008_PLUGIN
 # SIGNORE_MISSING_DATA, CGI_CL_DEBUG_ARGS, EXTRA DEBUG FLAGS?,
@@ -791,6 +793,9 @@ endif(WITH_EXEMPI)
 if(WITH_PYTHON)
    add_subdirectory("mapscript/python")
    set(USE_PYTHON_MAPSCRIPT 1)
+   if(WITH_MAPSCRIPT_PROPERTYANNOTATIONS)
+     set(MAPSCRIPT_PROPERTYANNOTATIONS 1)
+   endif(WITH_MAPSCRIPT_PROPERTYANNOTATIONS)
 endif(WITH_PYTHON)
 
 if(WITH_V8)
@@ -955,6 +960,7 @@ status_optional_feature("JAVA" "${USE_JAVA_MAPSCRIPT}")
 status_optional_feature("C#" "${USE_CSHARP_MAPSCRIPT}")
 status_optional_feature("V8 Javascript" "${USE_V8_MAPSCRIPT}")
 status_optional_feature("Apache Module (Experimental)" "${USE_APACHE_MODULE}")
+status_optional_feature("Python MapScript Property Annotations" "${MAPSCRIPT_PROPERTYANNOTATIONS}")
 
 message(STATUS "")
 message(STATUS "PROJECT_BINARY_DIR is set to ${PROJECT_BINARY_DIR}")

appveyor.yml

@@ -7,17 +7,17 @@ image: Visual Studio 2019
 cache:
   - '%LOCALAPPDATA%\pip\Cache'
   - '%APPVEYOR_BUILD_FOLDER%\swigwin-3.0.12.zip'
-  - '%APPVEYOR_BUILD_FOLDER%\swigwin-4.0.1.zip'
+  - '%APPVEYOR_BUILD_FOLDER%\swigwin-4.0.2.zip'
 
 environment:
 
   global:
-    SWIG_VER: swigwin-4.0.1
+    SWIG_VER: swigwin-4.0.2
     TWINE_USERNAME: mapserver
     TWINE_PASSWORD:
       secure: mHoJHeXdXbBNoDf7MA4ZEg==
 
-  # VS 2017
+  # VS 2019
   VS_VERSION: Visual Studio 16 2019
   matrix:
   - platform: x86
@@ -63,13 +63,15 @@ build_script:
   - if not exist %SWIG_VER%.zip appveyor DownloadFile https://sourceforge.net/projects/swig/files/swigwin/%SWIG_VER%/%SWIG_VER%.zip
   - set SDK_ZIP=%SDK%-dev.zip
   - set SDK_URL=http://download.gisinternals.com/sdk/downloads/%SDK_ZIP%
-  - echo "%SDK_ZIP%"
-  - echo "%SDK_URL%"
   - mkdir sdk
   - 7z x %SWIG_VER%.zip -osdk > nul
+  # add custom swig.exe
+  - appveyor DownloadFile http://download.osgeo.org/mapserver/docs/swigwin-4.0.2.mapserver.zip
   - cd sdk
   - appveyor DownloadFile "%SDK_URL%"
   - 7z x "%SDK_ZIP%" > nul
+  - cd %BUILD_FOLDER%/sdk/%SWIG_VER%/
+  - if "%platform%" == "x64" 7z e %BUILD_FOLDER%/swigwin-4.0.2.mapserver.zip -y -r swig.exe > nul
   - set SDK_PREFIX=%BUILD_FOLDER%/sdk/%SDK%
   - set SDK_INC=%BUILD_FOLDER%/sdk/%SDK%/include
   - set SDK_LIB=%BUILD_FOLDER%/sdk/%SDK%/lib
@@ -81,7 +83,7 @@ build_script:
   - cd build
   - set PATH=%BUILD_FOLDER%/build/Release;%SDK_BIN%;%PATH%  
   - set "PROJECT_BINARY_DIR=%BUILD_FOLDER%/build"
-  - cmake -G "%VS_FULL%" -A %VS_ARCH% .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=%SDK_PREFIX% -DPNG_LIBRARY=%SDK_LIB%/libpng16_static.lib -DHARFBUZZ_INCLUDE_DIR=%SDK_INC%/harfbuzz -DICONV_DLL=%SDK_BIN%/iconv.dll -DFRIBIDI_INCLUDE_DIR=%SDK_INC%/fribidi -DMS_EXTERNAL_LIBS=%SDK_LIB%/harfbuzz.lib;%SDK_LIB%/uriparser.lib -DSVG_LIBRARY=%SDK_LIB%/libsvg.lib -DSVGCAIRO_LIBRARY=%SDK_LIB%/libsvg-cairo.lib -DREGEX_DIR=%REGEX_DIR% -DSWIG_EXECUTABLE=%SWIG_EXECUTABLE% -DPROTOBUFC_COMPILER=%SDK_BIN%/protoc.exe -DPROTOBUFC_LIBRARY=%SDK_LIB%/protobuf-c.lib -DPROTOBUFC_INCLUDE_DIR=%SDK_INC%/protobuf-c -DWITH_CURL=1 -DWITH_KML=1 -DWITH_SVGCAIRO=1 -DWITH_THREAD_SAFETY=1 -DWITH_SOS=1 -DWITH_CLIENT_WFS=1 -DWITH_CLIENT_WMS=1-DWITH_CSHARP=1 -DWITH_PROTOBUFC=1 -DWITH_POSTGIS=0 -DWITH_PERL=0 -DWITH_MSSQL2008=1 -DWITH_PYTHON=1 -DWITH_PHPNG=0 -DWITH_HARFBUZZ=1
+  - cmake -G "%VS_FULL%" -A %VS_ARCH% .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=%SDK_PREFIX% -DPNG_LIBRARY=%SDK_LIB%/libpng16_static.lib -DHARFBUZZ_INCLUDE_DIR=%SDK_INC%/harfbuzz -DICONV_DLL=%SDK_BIN%/iconv.dll -DFRIBIDI_INCLUDE_DIR=%SDK_INC%/fribidi -DMS_EXTERNAL_LIBS=%SDK_LIB%/harfbuzz.lib;%SDK_LIB%/uriparser.lib -DSVG_LIBRARY=%SDK_LIB%/libsvg.lib -DSVGCAIRO_LIBRARY=%SDK_LIB%/libsvg-cairo.lib -DREGEX_DIR=%REGEX_DIR% -DSWIG_EXECUTABLE=%SWIG_EXECUTABLE% -DPROTOBUFC_COMPILER=%SDK_BIN%/protoc.exe -DPROTOBUFC_LIBRARY=%SDK_LIB%/protobuf-c.lib -DPROTOBUFC_INCLUDE_DIR=%SDK_INC%/protobuf-c -DWITH_CURL=1 -DWITH_KML=1 -DWITH_SVGCAIRO=1 -DWITH_THREAD_SAFETY=1 -DWITH_SOS=1 -DWITH_CLIENT_WFS=1 -DWITH_CLIENT_WMS=1-DWITH_CSHARP=1 -DWITH_PROTOBUFC=1 -DWITH_POSTGIS=0 -DWITH_PERL=0 -DWITH_MSSQL2008=1 -DWITH_PYTHON=1 -DWITH_PHPNG=0 -DWITH_HARFBUZZ=1 -DMAPSCRIPT_PROPERTYANNOTATIONS=1 -DPROJ_INCLUDE_DIR=%SDK_INC%/proj7
   - cmake --build . --config Release
   - cd %BUILD_FOLDER%/build
   # set the MapScript custom environment variable for py 3.8
@@ -94,7 +96,6 @@ build_script:
 before_test:
   - set PATH=%PATH%;%SDK_BIN%/gdal/apps
   - cd %BUILD_FOLDER%/msautotest
-  #- set PROJ_LIB=%SDK_BIN%/proj7/share
   - "./mssql/create_mssql_db.bat"
   - "%Python_ROOT_DIR%/python -m pip install -U -r requirements.txt"
   - "%Python_ROOT_DIR%/python -m pip install --no-index --find-links=file://%BUILD_FOLDER%/build/mapscript/python/Release/dist mapscript"

ci/travis/before_install.sh

@@ -9,7 +9,7 @@ sudo apt-get remove --purge postgresql* libpq-dev libpq5 || /bin/true
 
 sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
 sudo apt-get update
-sudo apt-get install -y --allow-unauthenticated protobuf-c-compiler libprotobuf-c0-dev bison flex libfribidi-dev cmake librsvg2-dev colordiff libpq-dev libpng-dev libjpeg-dev libgif-dev libgeos-dev libfreetype6-dev libfcgi-dev libcurl4-gnutls-dev libcairo2-dev libgdal-dev libproj-dev libxml2-dev libexempi-dev lcov lftp postgis libharfbuzz-dev gdal-bin ccache curl postgresql-server-dev-10 postgresql-10-postgis-3 postgresql-10-postgis-3-scripts swig g++
+sudo apt-get install -y --allow-unauthenticated protobuf-c-compiler libprotobuf-c0-dev bison flex libfribidi-dev cmake librsvg2-dev colordiff libpq-dev libpng-dev libjpeg-dev libgif-dev libgeos-dev libfreetype6-dev libfcgi-dev libcurl4-gnutls-dev libcairo2-dev libgdal-dev libproj-dev libxml2-dev libexempi-dev lcov lftp postgis libharfbuzz-dev gdal-bin ccache curl postgresql-server-dev-10 postgresql-10-postgis-3 postgresql-10-postgis-3-scripts swig g++ ca-certificates
 # following are already installed on Travis CI
 #sudo apt-get install --allow-unauthenticated php-dev python-dev python3-dev
 sudo apt-get install -y --allow-unauthenticated libmono-system-drawing4.0-cil mono-mcs
@@ -27,6 +27,7 @@ pyenv global $PYTHON_VERSION
 pyenv which pip
 pyenv which python
 
+pip install cryptography==3.4.6 # avoid requiring rust compiler for the cryptography dependency
 pip install cpp-coveralls pyflakes lxml
 pip install -r msautotest/requirements.txt
 

mapogcapi.cpp

@@ -106,6 +106,16 @@ static void outputError(int code, const std::string& description)
   msIO_printf("%s\n", j.dump().c_str());
 }
 
+static int includeLayer(mapObj *map, layerObj *layer) {
+  if(!msOWSRequestIsEnabled(map, layer, "AO", "OGCAPI", MS_FALSE) || 
+     !msWFSIsLayerSupported(layer) || 
+     !msIsLayerQueryable(layer)) {
+    return MS_FALSE;
+  } else {
+    return MS_TRUE;
+  }
+}
+
 /*
 ** Get stuff...
 */
@@ -518,7 +528,7 @@ static json getFeature(layerObj *layer, shapeObj *shape, gmlItemListObj *items,
     try {
       json item = getFeatureItem(&(items->items[i]), shape->values[i]);
       if(!item.is_null()) feature["properties"].insert(item.begin(), item.end());
-    } catch (const std::runtime_error &e) {
+    } catch (const std::runtime_error) {
       throw std::runtime_error("Error fetching item.");
     }
   }
@@ -527,7 +537,7 @@ static json getFeature(layerObj *layer, shapeObj *shape, gmlItemListObj *items,
     try {
       json constant = getFeatureConstant(&(constants->constants[i]));
       if(!constant.is_null()) feature["properties"].insert(constant.begin(), constant.end());
-    } catch (const std::runtime_error &e) {
+    } catch (const std::runtime_error) {
       throw std::runtime_error("Error fetching constant.");  
     }
   }
@@ -536,7 +546,7 @@ static json getFeature(layerObj *layer, shapeObj *shape, gmlItemListObj *items,
   try {
     json geometry = getFeatureGeometry(shape, geometry_precision);
     if(!geometry.is_null()) feature["geometry"] = geometry;
-  } catch (const std::runtime_error &e) {
+  } catch (const std::runtime_error) {
     throw std::runtime_error("Error fetching geometry.");
   }
 
@@ -566,12 +576,15 @@ static const char* getCollectionDescription(layerObj* layer)
 {
   const char *description = msOWSLookupMetadata(&(layer->metadata), "A", "description");
   if(!description) description = msOWSLookupMetadata(&(layer->metadata), "OF", "abstract"); // fallback on abstract
+  if(!description) description = "<!-- Warning: unable to set the collection description. -->"; // finally a warning...
   return description;
 }
 
 static const char* getCollectionTitle(layerObj* layer)
 {
-  return msOWSLookupMetadata(&(layer->metadata), "AOF", "title");
+    const char* title = msOWSLookupMetadata(&(layer->metadata), "AOF", "title");
+    if(!title) title = layer->name; // revert to layer name if no title found
+    return title;
 }
 
 static int getGeometryPrecision(mapObj *map, layerObj *layer)
@@ -592,16 +605,18 @@ static json getCollection(mapObj *map, layerObj *layer, OGCAPIFormat format)
 
   if(!map || !layer) return collection;
 
-  if(!msOWSRequestIsEnabled(map, layer, "AO", "OGCAPI", MS_FALSE) || !msWFSIsLayerSupported(layer)) return collection;
+  if(!includeLayer(map, layer)) return collection;
 
   // initialize some things
   std::string api_root = getApiRootUrl(map);
 
   if(msOWSGetLayerExtent(map, layer, "AOF", &bbox) == MS_SUCCESS) {
     if (layer->projection.numargs > 0)
       msOWSProjectToWGS84(&layer->projection, &bbox);
-    else
+    else if (map->projection.numargs > 0)
       msOWSProjectToWGS84(&map->projection, &bbox);
+    else
+      throw std::runtime_error("Unable to transform bounding box, no projection defined.");
   } else {
     throw std::runtime_error("Unable to get collection bounding box."); // might be too harsh since extent is optional
   }
@@ -617,8 +632,8 @@ static json getCollection(mapObj *map, layerObj *layer, OGCAPIFormat format)
   // build collection object
   collection = {
     { "id", id },
-    { "description", description?description:"" },
-    { "title", title?title:"" },
+    { "description", description },
+    { "title", title },
     { "extent", {
         { "spatial", {
             { "bbox", {{ round_down(bbox.minx, geometry_precision),
@@ -724,6 +739,11 @@ static void outputTemplate(const char *directory, const char *filename, const js
   } catch(const inja::RenderError &e) {
     outputError(OGCAPI_CONFIG_ERROR, "Template rendering error. " + std::string(e.what()) + " (" + std::string(filename) + ").");
     return;
+  }
+    catch (const inja::InjaError& e) {
+        outputError(OGCAPI_CONFIG_ERROR, "InjaError error. " + std::string(e.what()) + " (" + std::string(filename) + ")."
+            + " (" + std::string(directory) + ").");
+        return;
   } catch(...) {
     outputError(OGCAPI_SERVER_ERROR, "General template handling error.");
     return;
@@ -769,8 +789,11 @@ static void outputResponse(mapObj *map, cgiRequestObj *request, OGCAPIFormat for
       j["template"]["path"].push_back(request->api_path[i]);
 
     // parameters (optional)
-    for( int i=0; i<request->NumParams; i++)
-      j["template"]["params"].update({{ request->ParamNames[i], request->ParamValues[i] }});
+    for( int i=0; i<request->NumParams; i++) {
+      if(request->ParamValues[i] && strlen(request->ParamValues[i]) > 0) { // skip empty params
+        j["template"]["params"].update({{ request->ParamNames[i], request->ParamValues[i] }});
+      }
+    }
 
     // add custom tags (optional)
     const char *tags = msOWSLookupMetadata(&(map->web.metadata), "A", "html_tags");
@@ -919,7 +942,7 @@ static int processCollectionItemsRequest(mapObj *map, cgiRequestObj *request, co
   layer = map->layers[i]; // for convenience
   layer->status = MS_ON; // force on (do we need to save and reset?)
 
-  if(!msOWSRequestIsEnabled(map, layer, "AO", "OGCAPI", MS_FALSE) || !msWFSIsLayerSupported(layer)) {
+  if(!includeLayer(map, layer)) {
     outputError(OGCAPI_NOT_FOUND_ERROR, "Invalid collection.");
     return MS_SUCCESS;
   }
@@ -1021,7 +1044,12 @@ static int processCollectionItemsRequest(mapObj *map, cgiRequestObj *request, co
         {
             if( msStringToInt(offsetStr, &offset, 10) != MS_SUCCESS )
             {
-              outputError(OGCAPI_PARAM_ERROR, "Bad value fo offset");
+              outputError(OGCAPI_PARAM_ERROR, "Bad value for offset.");
+              return MS_SUCCESS;
+            }
+
+            if(offset < 0 || offset >= numberMatched) {
+              outputError(OGCAPI_PARAM_ERROR, "Offset out of range.");
               return MS_SUCCESS;
             }
 
@@ -1079,6 +1107,19 @@ static int processCollectionItemsRequest(mapObj *map, cgiRequestObj *request, co
         });
     }
 
+    if( offset > 0 ) 
+    {
+        response["links"].push_back({
+	    { "rel", "prev" },
+            { "type", format==OGCAPIFormat::JSON? OGCAPI_MIMETYPE_GEOJSON : OGCAPI_MIMETYPE_HTML },
+	    { "title", "previous page" },
+	    { "href", api_root + "/collections/" + std::string(id_encoded) +
+	              "/items?f=" + (format==OGCAPIFormat::JSON? "json" : "html") +
+		      "&limit=" + std::to_string(limit) +
+		      "&offset=" + std::to_string(MS_MAX(0, (offset - limit))) }
+	});
+    }
+
     msFree(id_encoded); // done
   }
 
@@ -1100,16 +1141,31 @@ static int processCollectionItemsRequest(mapObj *map, cgiRequestObj *request, co
 
     const int geometry_precision = getGeometryPrecision(map, layer);
 
-    // reprojection (layer projection to EPSG:4326)
+    // reprojection to EPSG:4326 (if necessary)
     reprojectionObj *reprojector = NULL;
-    if(msProjectionsDiffer(&(layer->projection), &(map->latlon))) {
-      reprojector = msProjectCreateReprojector(&(layer->projection), &(map->latlon));
-      if(reprojector == NULL) {
-        msGMLFreeItems(items);
-        msGMLFreeConstants(constants);
-        outputError(OGCAPI_SERVER_ERROR, "Error creating re-projector.");
-        return MS_SUCCESS;
+    if(layer->projection.numargs > 0) { 
+      if(msProjectionsDiffer(&(layer->projection), &(map->latlon))) {
+        reprojector = msProjectCreateReprojector(&(layer->projection), &(map->latlon));
+        if(reprojector == NULL) {
+          msGMLFreeItems(items);
+          msGMLFreeConstants(constants);
+          outputError(OGCAPI_SERVER_ERROR, "Error creating re-projector.");
+          return MS_SUCCESS;
+        }
+      }
+    } else if(map->projection.numargs > 0) {
+      if(msProjectionsDiffer(&(map->projection), &(map->latlon))) {
+        reprojector = msProjectCreateReprojector(&(map->projection), &(map->latlon));
+        if(reprojector == NULL) {
+	  msGMLFreeItems(items);
+          msGMLFreeConstants(constants);
+	  outputError(OGCAPI_SERVER_ERROR, "Error creating re-projector.");
+          return MS_SUCCESS;
+        }
       }
+    } else {
+      outputError(OGCAPI_CONFIG_ERROR, "Unable to transform geometries, no projection defined.");
+      return MS_SUCCESS;
     }
 
     for(i=0; i<layer->resultcache->numresults; i++) {
@@ -1160,7 +1216,7 @@ static int processCollectionItemsRequest(mapObj *map, cgiRequestObj *request, co
 
   // extend the response a bit for templating (HERE)
   if(format == OGCAPIFormat::HTML) {
-    const char *title = msOWSLookupMetadata(&(layer->metadata), "AOF", "title");
+    const char *title = getCollectionTitle(layer);
     const char *id = layer->name;
     response["collection"] = {
       { "id", id },
@@ -1339,7 +1395,7 @@ static int processApiRequest(mapObj *map, cgiRequestObj *request, OGCAPIFormat f
           server["description"] = value;
       }
   }
-  response["servers"] = { server };
+  response["servers"].push_back(server);
 
   const std::string oapif_yaml_url = "http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/ogcapi-features-1.yaml";
 
@@ -1407,9 +1463,7 @@ static int processApiRequest(mapObj *map, cgiRequestObj *request, OGCAPIFormat f
 
   for(int i=0; i<map->numlayers; i++) {
       layerObj* layer = map->layers[i];
-      if(!msOWSRequestIsEnabled(map, layer, "AO", "OGCAPI", MS_FALSE) ||
-         !msWFSIsLayerSupported(layer))
-      {
+      if(!includeLayer(map, layer)) {
           continue;
       }
 

mapogr.cpp

@@ -2383,9 +2383,14 @@ static int msOGRFileWhichShapes(layerObj *layer, rectObj rect, msOGRFileInfo *ps
                 filter = msStringConcatenate(filter, "\"");
                 if( isGPKG )
                     filter = msStringConcatenate(filter, ")");
-                filter = msStringConcatenate(filter, ", BuildMbr(");
                 char *points = (char *)msSmallMalloc(30*2*5);
-                snprintf(points, 30*4, "%lf,%lf,%lf,%lf", rect.minx, rect.miny, rect.maxx, rect.maxy);
+                if( rect.minx == rect.maxx && rect.miny == rect.maxy ) {
+                  filter = msStringConcatenate(filter, ",  ST_GeomFromText(");
+                  snprintf(points, 30*4, "'POINT(%lf %lf)'", rect.minx, rect.miny);
+                } else {
+                  filter = msStringConcatenate(filter, ", BuildMbr(");
+                  snprintf(points, 30*4, "%lf,%lf,%lf,%lf", rect.minx, rect.miny, rect.maxx, rect.maxy);
+                }
                 filter = msStringConcatenate(filter, points);
                 msFree(points);
                 filter = msStringConcatenate(filter, "))");
@@ -3422,12 +3427,13 @@ static int  msOGRExtractTopSpatialFilter( msOGRFileInfo *info,
                                           pSpatialFilterNode);
   }
 
-  if( (expr->m_nToken == MS_TOKEN_COMPARISON_INTERSECTS ||
+  if( (((expr->m_nToken == MS_TOKEN_COMPARISON_INTERSECTS ||
       expr->m_nToken == MS_TOKEN_COMPARISON_OVERLAPS ||
       expr->m_nToken == MS_TOKEN_COMPARISON_CROSSES ||
       expr->m_nToken == MS_TOKEN_COMPARISON_WITHIN ||
       expr->m_nToken == MS_TOKEN_COMPARISON_CONTAINS) &&
-      expr->m_aoChildren.size() == 2 &&
+      expr->m_aoChildren.size() == 2) ||
+      (expr->m_nToken == MS_TOKEN_COMPARISON_DWITHIN && expr->m_aoChildren.size() == 3)) &&
       expr->m_aoChildren[1]->m_nToken == MS_TOKEN_LITERAL_SHAPE )
   {
         if( info->rect_is_defined )
@@ -3442,7 +3448,11 @@ static int  msOGRExtractTopSpatialFilter( msOGRFileInfo *info,
         OGRErr e = OGR_G_CreateFromWkt(&wkt, NULL, &hSpatialFilter);
         if (e == OGRERR_NONE) {
             OGREnvelope env;
-            OGR_G_GetEnvelope(hSpatialFilter, &env);
+            if( expr->m_nToken == MS_TOKEN_COMPARISON_DWITHIN ) {
+                OGR_G_GetEnvelope(OGR_G_Buffer(hSpatialFilter, expr->m_aoChildren[2]->m_dfVal, 30), &env);
+            } else {
+                OGR_G_GetEnvelope(hSpatialFilter, &env);
+            }
             info->rect.minx = env.MinX;
             info->rect.miny = env.MinY;
             info->rect.maxx = env.MaxX;