feat(data-pipeline): improve tiles (#339)

* feat(data-pipeline): hide gsutil upload logs

* feat(data-pipeline): use nearest neighbor filtering for gdalwarp

* feat(data-pipeline): use dataset bounds from netcdf

* feat(data-pipeline): add script to optionally mask values

* feat(data-pipeline): add snow layer

ci/cloudbuild-tiles-reproject.yaml

@@ -36,6 +36,7 @@ steps:
     id: 'upload-to-storage'
     args:
       - "-m"
+      - "-q"
       - "cp"
       - "-r"
       - "/data/upload/${_LAYER_ID}/*"

ci/cloudbuild-tiles.yaml

@@ -45,6 +45,7 @@ steps:
     id: 'upload-to-storage'
     args:
       - "-m"
+      - "-q"
       - "cp"
       - "-r"
       - "/data/upload/${_LAYER_ID}/*"

data/downloads/odp-ftp-sea-state.sh

@@ -16,4 +16,5 @@ do
   curl --silent $FTP_URL > $FILENAME
 
   python ./data/drop-unused-vars.py --file $FILENAME --variable swh_mean
+  python ./data/mask-values.py --file $FILENAME --variable swh_mean --max 100
 done

data/gdal-reproject.sh

@@ -28,6 +28,7 @@ for file in $(find $FOLDER -name *.nc -type f | sort -n); do
   gdalwarp \
     -t_srs EPSG:4326 \
     -te $OUT_BOUNDS \
+    -r near \
     NETCDF:\"$file\":$VARIABLE \
     ./tmp.tif
 

data/gdal-tiles.sh

@@ -17,6 +17,7 @@ for file in $(find /data/images/* -name 0.png | sort -n); do
     --no-kml \
     --webviewer=none \
     --resampling near \
+    --tmscompatible \
     --s_srs EPSG:4326 \
     --processes=8 \
      --quiet \

data/layers-config.json

@@ -48,7 +48,7 @@
     }
   },
   "sea_surface_salinity.sss": {
-    "colorMap": "inferno",
+    "colorMap": "custom",
     "timeFormat": {
       "year": "numeric",
       "month": "long",
@@ -80,7 +80,7 @@
     }
   },
   "sea_state.swh_mean": {
-    "colorMap": "inferno",
+    "colorMap": "viridis",
     "timeFormat": {
       "year": "numeric",
       "month": "long",
@@ -118,5 +118,13 @@
       "month": "long",
       "day": "2-digit"
     }
+  },
+  "snow.swe": {
+    "colorMap": "ocean",
+    "timeFormat": {
+      "year": "numeric",
+      "month": "long",
+      "day": "2-digit"
+    }
   }
 }

data/mask-values.py

@@ -0,0 +1,23 @@
+import os
+import xarray as xr
+from argparse import ArgumentParser
+
+parser = ArgumentParser()
+parser.add_argument("-f", "--file", dest="file")
+parser.add_argument("-v", "--variable", dest="variable")
+parser.add_argument("--min", dest="min")
+parser.add_argument("--max", dest="max")
+args = parser.parse_args()
+
+ds = xr.open_dataset(args.file, decode_coords=False, decode_cf=False)
+
+if args.min != None:
+  da = ds[args.variable]
+  ds[args.variable] = da.where(da > float(args.min))
+
+if args.max != None:
+  da = ds[args.variable]
+  ds[args.variable] = da.where(da < float(args.max))
+
+os.remove(args.file)
+ds.to_netcdf(args.file, format='NETCDF4_CLASSIC', mode='w')

data/triggers/snow_swe.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+TIMEOUT=8000
+LAYER_ID="snow.swe"
+VARIABLE_ID="swe"
+VERSION="0.4.1"
+ZOOM_LEVELS="0-3"
+MIN="auto"
+MAX="auto"
+MACHINE_TYPE="N1_HIGHCPU_8"
+
+if [ ! -f ./package.json ]; then
+    echo "You have to be in the root folder of the project to run this script!"
+    exit 1
+fi
+
+# --machine-type=$MACHINE_TYPE \
+gcloud builds submit --config ./ci/cloudbuild-tiles.yaml \
+  --timeout=$TIMEOUT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX \
+  .

data/utility.py

@@ -23,17 +23,30 @@ def write_style_file(layer_id, variable_id, min, max):
 
   print(content)
 
-def write_world_file(shape):
-  lon_res = 360 / shape[2]
-  lat_res = 180 / shape[1]
+def write_world_file(shape, attributes):
+  lat_min = -90
+  lat_max = 90
+  lon_min = -180
+  lon_max = 180
+
+  try:
+    lat_min = attributes['geospatial_lat_min']
+    lat_max = attributes['geospatial_lat_max']
+    lon_min = attributes['geospatial_lon_min']
+    lon_max = attributes['geospatial_lon_max']
+  except KeyError:
+    print("Write Worldfile: Could not read geospatil info using defauls!")
+
+  lon_res = (abs(lon_min) + abs(lon_max)) / shape[2]
+  lat_res = (abs(lat_min) + abs(lat_max)) / shape[1]
 
   content = """{a}\n{b}\n{c}\n{d}\n{e}\n{f}""".format(
-    a = lon_res * 2,
+    a = lon_res,
     b = 0,
     c = 0,
-    d = lat_res * -2,
-    e = -180 + lon_res,
-    f = 270 - lat_res
+    d = lat_res * -1,
+    e = lon_min + (lon_res / 2),
+    f = lat_max - (lat_res / 2)
   )
 
   with open('./worldfile.wld', 'w') as f:

data/write-zarr.py

@@ -61,7 +61,7 @@
   print(f'Written zarr in {time.time() - start_time}s')
 
   print('Writing world file...')
-  utility.write_world_file(shape)
+  utility.write_world_file(shape, ds.attrs)
 
   print('Writing style file...')
   utility.write_style_file(args.layer_id, args.variable_id, min, max)

storage/layers/layers-de.json

@@ -88,5 +88,11 @@
     "name": "Sea Surface Salinity - sss",
     "description": "Das ist der ...",
     "link": "http://..."
+  },
+  {
+    "id": "snow.swe",
+    "name": "Snow - swe",
+    "description": "Das ist der ...",
+    "link": "http://..."
   }
 ]

storage/layers/layers-en.json

@@ -88,5 +88,11 @@
     "name": "Sea Surface Salinity - sss",
     "description": "This is the ...",
     "link": "http://..."
+  },
+  {
+    "id": "snow.swe",
+    "name": "Snow - swe",
+    "description": "This is the ...",
+    "link": "http://..."
   }
 ]