From 1fe78a9ba838c46dd823abb7d36314e1278484fd Mon Sep 17 00:00:00 2001 From: sethg Date: Fri, 31 Oct 2025 15:18:01 +0100 Subject: [PATCH 1/2] Add map --- workshop/exercises/mapfiles/gdalg.map | 47 ++++++++++++++++++++ workshop/exercises/mapfiles/roads.gdalg.json | 4 ++ 2 files changed, 51 insertions(+) create mode 100644 workshop/exercises/mapfiles/gdalg.map create mode 100644 workshop/exercises/mapfiles/roads.gdalg.json diff --git a/workshop/exercises/mapfiles/gdalg.map b/workshop/exercises/mapfiles/gdalg.map new file mode 100644 index 0000000..7f8b800 --- /dev/null +++ b/workshop/exercises/mapfiles/gdalg.map @@ -0,0 +1,47 @@ +MAP + NAME "Roads" + EXTENT 26.668678 58.339241 26.796582 58.409410 + SIZE 800 600 + # CONFIG MS_ERRORFILE "/tmp/error.log" + CONFIG MS_ERRORFILE "C:/temp/error.log" + CONFIG CPL_DEBUG "ON" + PROJECTION + "init=epsg:4326" + END + DEBUG 5 + WEB + METADATA + "ows_enable_request" "*" + "ows_srs" "EPSG:4326 EPSG:3857" + END + END + + LAYER + NAME "roads" + TYPE POLYGON + STATUS ON + DEBUG 5 + + # PATH=C:\GitHub\MapServer\sdk\release-1930-x64\bin;%PATH% + # $env:Path = "C:\GitHub\MapServer\build\RelWithDebInfo;C:\GitHub\MapServer\sdk\release-1930-x64\bin;C:\GitHub\MapServer\sdk\release-1930-x64\bin\gdal\apps;" + $env:Path + CONNECTIONTYPE OGR + CONNECTION "{\"type\": \"gdal_streamed_alg\",\"command_line\": \"gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.000001\"}" + # DATA "0" + + # CONNECTION "{\"type\": \"gdal_streamed_alg\",\"command_line\":\"gdal vector pipeline read data/osm/roads.fgb\"}" + #CONNECTION "{\"type\":\"gdal_streamed_alg\",\"command_line\":\"gdal vector pipeline read --input data/province.shp\"}" + # DATA province + # TYPE POINT + + #CONNECTION "roads.gdalg.json" + DATA "0" + CLASS + STYLE + COLOR 50 50 50 + END + END + END +END + +# map2img -m gdalg.map -o D:\Temp\mymap.png +# gdal vector info '{\"type\": \"gdal_streamed_alg\",\"command_line\": \"gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.000001\"}'o \ No newline at end of file diff --git a/workshop/exercises/mapfiles/roads.gdalg.json b/workshop/exercises/mapfiles/roads.gdalg.json new file mode 100644 index 0000000..18a09f1 --- /dev/null +++ b/workshop/exercises/mapfiles/roads.gdalg.json @@ -0,0 +1,4 @@ +{ + "type": "gdal_streamed_alg", + "command_line": "gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.000001" +} \ No newline at end of file From a993b7795065e18369fa5a68a2ebc524218e1bf3 Mon Sep 17 00:00:00 2001 From: sethg Date: Fri, 14 Nov 2025 19:21:23 +0100 Subject: [PATCH 2/2] Add vector pipeline workshop and demo --- workshop/content/docs/advanced/arcgis.md | 2 +- workshop/content/docs/advanced/gdalg.md | 138 +++++++++++++++++++ workshop/content/mkdocs.yml | 1 + workshop/exercises/app/gdalg.html | 14 ++ workshop/exercises/app/index.html | 3 +- workshop/exercises/app/js/gdalg.js | 26 ++++ workshop/exercises/mapfiles/gdalg.map | 58 +++++--- workshop/exercises/mapfiles/roads.gdalg.json | 2 +- 8 files changed, 219 insertions(+), 25 deletions(-) create mode 100644 workshop/content/docs/advanced/gdalg.md create mode 100644 workshop/exercises/app/gdalg.html create mode 100644 workshop/exercises/app/js/gdalg.js diff --git a/workshop/content/docs/advanced/arcgis.md b/workshop/content/docs/advanced/arcgis.md index 1486565..ebeae88 100644 --- a/workshop/content/docs/advanced/arcgis.md +++ b/workshop/content/docs/advanced/arcgis.md @@ -183,7 +183,7 @@ labelsCheckbox.addEventListener('change', (event) => { --8<-- "arcgis.js" ``` -??? Mapfile "stac.map" +??? Mapfile "arcgis.map" ``` scala --8<-- "arcgis.map" diff --git a/workshop/content/docs/advanced/gdalg.md b/workshop/content/docs/advanced/gdalg.md new file mode 100644 index 0000000..78b1dca --- /dev/null +++ b/workshop/content/docs/advanced/gdalg.md @@ -0,0 +1,138 @@ +# Working with GDAL Vector Pipelines + +## Overview + +MapServer can dynamically run a [GDAL Vector Pipeline](https://gdal.org/en/latest/programs/gdal_vector_pipeline.html), and render its output - +all through a simple Mapfile. + +In this workshop we'll use the Tartu roads dataset used in the [Line Styling](../mapfile/lines.md) exercise, dynamically +buffer it using GDAL, and display the result in OpenLayers using a MapServer WMS. + +This is a simple example of a pipeline, but additional steps can be chained together to create more complex workflows. +MapServer reads a vector pipeline using the [GDALG: GDAL Streamed Algorithm](https://gdal.org/en/latest/drivers/vector/gdalg.html) driver. + +
+ +
+ +## Checking the Pipelines with GDAL + +Before configuring MapServer, it is often easier to test your pipelines directly with GDAL, to ensure they run correctly. +Run the commands below to connect to the MapServer Docker container and use GDAL to get information about the pipelines. + +```bash +# open a shell inside the MapServer container +docker exec -it mapserver /bin/bash + +# check the dataset used in the pipeline +gdal vector info data/osm/roads.fgb + +# inspect the a pipeline JSON file included in the container +gdal vector info roads.gdalg.json + +# test an inline pipeline using a single-quoted string (recommended) +gdal vector info '{"type": "gdal_streamed_alg","command_line": "gdal vector pipeline ! read /etc/mapserver/data/osm/roads.fgb ! geom buffer --distance=0.0001"}' + +# alternatively escape the double quotes +gdal vector info "{\"type\": \"gdal_streamed_alg\",\"command_line\": \"gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.0001\"}" +``` + +## The Mapfile + +### Embedding Pipelines in a Mapfile + +The pipeline `LAYER` in this example uses `CONNECTIONTYPE OGR` and includes the GDAL pipeline "inline" - meaning the pipeline is defined +directly in the Mapfile. Our example pipeline looks like this: + +```scala +CONNECTION '{"type": "gdal_streamed_alg","command_line": "gdal vector pipeline ! read /etc/mapserver/data/osm/roads.fgb ! geom buffer --distance=0.0001"}' +DATA "0" +``` + +Key points to note: + +- `DATA "0"` tells MapServer to use the first (index 0) layer in the connection. Since the pipeline returns a single dataset, this will correspond + to the buffered roads. +- When using an inline GDAL pipeline, you must provide the absolute path to any datasets used by the pipeline. +- GDAL requires a valid JSON string for the pipeline. All property names and string values must use double quotes. + +In a Mapfile, you have two options for embedding JSON: + +1. Wrap the JSON string in single quotes (as in the example above) - this is simpler and easier to read or copy/paste. +2. Escape the double quotes using `\"` as in the example below: + + ```scala + CONNECTION "{\"type\": \"gdal_streamed_alg\",\"command_line\": \"gdal vector pipeline ! read /etc/mapserver/data/osm/roads.fgb ! geom buffer --distance=0.0001\"}" + ``` + +### Referencing Pipelines in a JSON File + +MapServer can also reference a JSON file containing a pipeline, which makes it easy to test and reuse the pipeline with GDAL. +By convention GDALG files should use the `.gdalg.json` extension. + +```scala +CONNECTION "roads.gdalg.json" +DATA "0" +``` + +The contents of pipeline JSON file `roads.gdalg.json` are shown below. Notice that the dataset path `data/osm/roads.fgb` is relative to the JSON file. + +Using relative paths in a JSON file makes the pipeline more portable, because it can be moved to a different folder or system without changing the dataset paths. + +```json +{ + "type": "gdal_streamed_alg", + "command_line": "gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.0001" +} +``` + +### Hatch Styling + +Finally, we use a [hatch symbol](https://mapserver.org/mapfile/symbol.html#mapfile-symbol-type) to style the buffered roads. +Hatch symbols allow you to add patterned fills, and you can adjust their angle, width, and size in the [STYLE](https://mapserver.org/mapfile/style.html) block. + +```scala +SYMBOL + NAME "hatchsymbol" + TYPE hatch +END +... +LAYER + CLASS + STYLE + SYMBOL "hatchsymbol" + COLOR "#78C8FF" + WIDTH 0.1 + ANGLE 45 + SIZE 8 + END +``` + +## Code + +!!! example + + - Direct MapServer request: + - Local OpenLayers example: + +??? JavaScript "gdalg.js" + + ``` js + --8<-- "gdalg.js" + ``` + +??? Mapfile "gdalg.map" + + ``` scala + --8<-- "gdalg.map" + ``` + +## Exercises + +1. Switch to using the JSON file - replace the inline GDAL pipeline with `roads.gdalg.json` in your Mapfile. + +2. Extend the GDAL pipeline - add another processing step to the pipeline. + Refer to the [GDAL Vector Pipeline documentation](https://gdal.org/en/latest/programs/gdal_vector_pipeline.html) for examples of available operations. + +3. Experiment with hatch styling - adjust properties such as `ANGLE` and `SIZE` in your `STYLE` block + to see how the hatch pattern changes. diff --git a/workshop/content/mkdocs.yml b/workshop/content/mkdocs.yml index 6a4738f..481a599 100644 --- a/workshop/content/mkdocs.yml +++ b/workshop/content/mkdocs.yml @@ -30,6 +30,7 @@ nav: - OGC API - Features: outputs/ogcapi-features.md - Advanced: - ArcGIS Feature Server: advanced/arcgis.md + - GDAL Pipeline: advanced/gdalg.md - Vector Symbols: advanced/symbols.md - Clusters: advanced/clusters.md - SLD: advanced/sld.md diff --git a/workshop/exercises/app/gdalg.html b/workshop/exercises/app/gdalg.html new file mode 100644 index 0000000..b3e18e1 --- /dev/null +++ b/workshop/exercises/app/gdalg.html @@ -0,0 +1,14 @@ + + + + + + + + GDAL Vector Pipeline + + +
+ + + diff --git a/workshop/exercises/app/index.html b/workshop/exercises/app/index.html index e4e0c5f..ac0d767 100644 --- a/workshop/exercises/app/index.html +++ b/workshop/exercises/app/index.html @@ -30,9 +30,10 @@

Outputs

  • Vector Tiles
  • OGC API - Features
    • -

    Advanced

    +

    Advanced

    • ArcGIS Feature Server
      • +
    • GDAL Pipelines
    • Vector Symbols (Railways)
    • Clusters
    • Landuse
      • diff --git a/workshop/exercises/app/js/gdalg.js b/workshop/exercises/app/js/gdalg.js new file mode 100644 index 0000000..0051aea --- /dev/null +++ b/workshop/exercises/app/js/gdalg.js @@ -0,0 +1,26 @@ +import '../css/style.css'; +import ImageWMS from 'ol/source/ImageWMS.js'; +import Map from 'ol/Map.js'; +import View from 'ol/View.js'; +import { Image as ImageLayer } from 'ol/layer.js'; + +const mapserverUrl = import.meta.env.VITE_MAPSERVER_BASE_URL; +const mapfilesPath = import.meta.env.VITE_MAPFILES_PATH; + +const layers = [ + new ImageLayer({ + source: new ImageWMS({ + url: mapserverUrl + mapfilesPath + 'gdalg.map&', + params: { 'LAYERS': 'buffered_roads,roads' }, + ratio: 1 + }), + }), +]; +const map = new Map({ + layers: layers, + target: 'map', + view: new View({ + center: [2975862.75916499, 8046369.8646329], + zoom: 17, + }), +}); diff --git a/workshop/exercises/mapfiles/gdalg.map b/workshop/exercises/mapfiles/gdalg.map index 7f8b800..e07be4c 100644 --- a/workshop/exercises/mapfiles/gdalg.map +++ b/workshop/exercises/mapfiles/gdalg.map @@ -1,14 +1,11 @@ MAP - NAME "Roads" + NAME "GDALG" EXTENT 26.668678 58.339241 26.796582 58.409410 SIZE 800 600 - # CONFIG MS_ERRORFILE "/tmp/error.log" - CONFIG MS_ERRORFILE "C:/temp/error.log" - CONFIG CPL_DEBUG "ON" + IMAGECOLOR "#0A1E50" PROJECTION "init=epsg:4326" END - DEBUG 5 WEB METADATA "ows_enable_request" "*" @@ -16,32 +13,49 @@ MAP END END + SYMBOL + NAME "hatchsymbol" + TYPE hatch + END + LAYER - NAME "roads" + NAME "buffered_roads" TYPE POLYGON - STATUS ON - DEBUG 5 - - # PATH=C:\GitHub\MapServer\sdk\release-1930-x64\bin;%PATH% - # $env:Path = "C:\GitHub\MapServer\build\RelWithDebInfo;C:\GitHub\MapServer\sdk\release-1930-x64\bin;C:\GitHub\MapServer\sdk\release-1930-x64\bin\gdal\apps;" + $env:Path + STATUS OFF CONNECTIONTYPE OGR - CONNECTION "{\"type\": \"gdal_streamed_alg\",\"command_line\": \"gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.000001\"}" + # note we need to use the full path to the dataset using the inline pipeline + CONNECTION '{"type": "gdal_streamed_alg","command_line": "gdal vector pipeline ! read /etc/mapserver/data/osm/roads.fgb ! geom buffer --distance=0.0001"}' + DATA "0" + # CONNECTION "roads.gdalg.json" # DATA "0" + CLASS + STYLE + COLOR 50 100 180 + END + STYLE + SYMBOL "hatchsymbol" + COLOR "#78C8FF" + WIDTH 0.1 + ANGLE 45 + SIZE 8 + END + END + END - # CONNECTION "{\"type\": \"gdal_streamed_alg\",\"command_line\":\"gdal vector pipeline read data/osm/roads.fgb\"}" - #CONNECTION "{\"type\":\"gdal_streamed_alg\",\"command_line\":\"gdal vector pipeline read --input data/province.shp\"}" - # DATA province - # TYPE POINT - - #CONNECTION "roads.gdalg.json" - DATA "0" + # original unbuffered roads dataset + LAYER + NAME "roads" + TYPE LINE + STATUS OFF + CONNECTIONTYPE FLATGEOBUF + DATA "data/osm/roads.fgb" CLASS STYLE - COLOR 50 50 50 + WIDTH 0.8 + COLOR "#DCF0FF" END END END + END -# map2img -m gdalg.map -o D:\Temp\mymap.png -# gdal vector info '{\"type\": \"gdal_streamed_alg\",\"command_line\": \"gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.000001\"}'o \ No newline at end of file diff --git a/workshop/exercises/mapfiles/roads.gdalg.json b/workshop/exercises/mapfiles/roads.gdalg.json index 18a09f1..8884047 100644 --- a/workshop/exercises/mapfiles/roads.gdalg.json +++ b/workshop/exercises/mapfiles/roads.gdalg.json @@ -1,4 +1,4 @@ { "type": "gdal_streamed_alg", - "command_line": "gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.000001" + "command_line": "gdal vector pipeline ! read data/osm/roads.fgb ! geom buffer --distance=0.0001" } \ No newline at end of file