feat(data-pipeline): improve tiles and add greenhouse (#340)

* feat(data-pipeline): hide gsutil upload logs

* feat(data-pipeline): use nearest neighbor filtering for gdalwarp

* feat(data-pipeline): use dataset bounds from netcdf

* feat(data-pipeline): add script to optionally mask values

* feat(data-pipeline): add snow layer

* feat(data-pipeline): add greenhouse

data/add-time-coordinate.py

@@ -14,4 +14,4 @@
 ds = ds.expand_dims(dim={'time': new_time}, axis=0)
 
 os.remove(args.file)
-ds.to_netcdf(args.file, format='NETCDF4_CLASSIC', mode='w')
+ds.to_netcdf(args.file, format='NETCDF4', mode='w')

data/downloads/odp-ftp-biomass.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/biomass/data/agb/maps/2017/v1.0/netcdf/ESACCI-BIOMASS-L4-AGB-MERGED-100m-2017-fv1.0.nc"
+START_DATE=2017-01-01
+OUTPUT_FODLER=./download
+
+FILENAME=$OUTPUT_FODLER/$(date +%Y%m%d -d "$START_DATE + $i month").nc
+echo $FTP_URL
+
+curl --silent $FTP_URL > $FILENAME
+
+python ./data/drop-unused-vars.py --file $FILENAME --variable AOD550_mean
+python ./data/add-time-coordinate.py --file $FILENAME --timestamp $NEXT_DATE

data/downloads/odp-ftp-soil-moisture.sh

@@ -1,9 +1,5 @@
 #!/usr/bin/env bash
 
-
-ftp://anon-ftp.ceda.ac.uk/neodc/esacci/soil_moisture/data/daily_files/COMBINED/v04.5/1988/ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-198802000000-fv04.5.nc
-
-
 BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/soil_moisture/data/daily_files/COMBINED/v04.5"
 START_DATE=1987-11-01
 OUTPUT_FODLER=./download

data/downloads/storage-greenhouse-xch4.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# gsutil -m cp gs://esa-cfs-cate-data/greenhouse.xch4/* ./download
+
+START_DATE=2003-01-01
+OUTPUT_FODLER=./download
+
+for i in {0..191}
+do
+  NEXT_MONTH=$(date +%Y-%m -d "$START_DATE + $i month")
+  NEXT_DATE=$(date +%Y%m%d -d "$START_DATE + $i month")
+  FILENAME=$OUTPUT_FODLER/xch4_$NEXT_MONTH.tif
+  NETCDF=$OUTPUT_FODLER/$NEXT_DATE.nc
+  echo $FILENAME
+
+  python ./data/tif2netcdf.py --file $FILENAME --output $NETCDF --variable xch4
+  python ./data/mask-values.py --file $NETCDF --variable xch4 --min "-100"
+  python ./data/add-time-coordinate.py --file $NETCDF --timestamp $NEXT_DATE
+done

data/downloads/storage-greenhouse-xco2.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+gsutil -m cp gs://esa-cfs-cate-data/greenhouse.xco2/* ./download
+
+START_DATE=2003-01-01
+OUTPUT_FODLER=./download
+
+for i in {0..191}
+do
+  NEXT_MONTH=$(date +%Y-%m -d "$START_DATE + $i month")
+  NEXT_DATE=$(date +%Y%m%d -d "$START_DATE + $i month")
+  FILENAME=$OUTPUT_FODLER/xco2_$NEXT_MONTH.tif
+  NETCDF=$OUTPUT_FODLER/$NEXT_DATE.nc
+  echo $FILENAME
+
+  python ./data/tif2netcdf.py --file $FILENAME --output $NETCDF --variable xco2
+  python ./data/mask-values.py --file $NETCDF --variable xco2 --min "-100"
+  python ./data/add-time-coordinate.py --file $NETCDF --timestamp $NEXT_DATE
+done

data/gdal-colors/colors-agb.txt

@@ -0,0 +1,4 @@
+100 0 255 0
+1 255 255 0
+0 0 0 0 0
+nv 0 255 255 0

data/gdal-colors/colors-ice_conc.txt

@@ -1,3 +1,3 @@
-10000 255 255 0
+10000 0 255 0
 0 255 0 0
 nv 0 255 255 0

data/gdal-colors/colors-xch4.txt

@@ -0,0 +1,4 @@
+0.0000020082 255 255 0
+0.00000175 0 255 0
+0.0000015674 0 255 255
+nv 0 255 255 0

data/gdal-colors/colors-xco2.txt

@@ -0,0 +1,3 @@
+0.000426 255 0 0
+0.000360 0 0 255
+nv 0 255 255 0

data/gdal-reproject.sh

@@ -7,7 +7,7 @@ SOURCE_PROJECTION=$4
 SOURCE_BOUNDS=$5
 TRIMMED_SOURCE_BOUNDS=$(echo $SOURCE_BOUNDS | sed 's/ *$//g')
 FOLDER=/data/netcdfs/
-counter=0
+timestamp_counter=0
 
 for file in $(find $FOLDER -name *.nc -type f | sort -n); do
   echo "--------------"
@@ -29,13 +29,17 @@ for file in $(find $FOLDER -name *.nc -type f | sort -n); do
     -t_srs EPSG:4326 \
     -te $OUT_BOUNDS \
     -r near \
+    --config GDAL_CACHEMAX 90% \
+    -co compress=LZW \
     NETCDF:\"$file\":$VARIABLE \
     ./tmp.tif
 
   gdaldem\
     color-relief \
     ./tmp.tif \
     ./data/gdal-colors/colors-$VARIABLE.txt \
+    --config GDAL_CACHEMAX 90% \
+    -co compress=LZW \
     -alpha ./colored.tif
 
   gdal2tiles.py \
@@ -46,12 +50,12 @@ for file in $(find $FOLDER -name *.nc -type f | sort -n); do
     --webviewer=none \
     --resampling near \
     --s_srs EPSG:4326 \
-    ./colored.tif /data/images/$VARIABLE/$counter
+    ./colored.tif /data/images/$VARIABLE/$timestamp_counter
 
   rm ./tmp.tif
   rm ./colored.tif
 
-  counter=$((counter+1))
+  timestamp_counter=$((timestamp_counter+1))
 done
 
 

data/layers-config.json

@@ -126,5 +126,29 @@
       "month": "long",
       "day": "2-digit"
     }
+  },
+  "biomass.agb": {
+    "colorMap": "custom",
+    "timeFormat": {
+      "year": "numeric",
+      "month": "long",
+      "day": "2-digit"
+    }
+  },
+  "greenhouse.xco2": {
+    "colorMap": "custom",
+    "timeFormat": {
+      "year": "numeric",
+      "month": "long",
+      "day": "2-digit"
+    }
+  },
+  "greenhouse.xch4": {
+    "colorMap": "custom",
+    "timeFormat": {
+      "year": "numeric",
+      "month": "long",
+      "day": "2-digit"
+    }
   }
 }

data/mask-values.py

@@ -20,4 +20,4 @@
   ds[args.variable] = da.where(da < float(args.max))
 
 os.remove(args.file)
-ds.to_netcdf(args.file, format='NETCDF4_CLASSIC', mode='w')
+ds.to_netcdf(args.file, format='NETCDF4', mode='w')

data/prepare-tile-upload.sh

@@ -16,5 +16,5 @@ cp -r /data/images/$variable/* /data/upload/$layer/tiles/
 # copy layer metadata
 cp metadata.json /data/upload/$layer/metadata.json
 
-cd /data/upload && zip -r ./$layer/package.zip ./*
+cd /data/upload && zip -r -q ./$layer/package.zip ./*
 

data/tif2netcdf.py

@@ -0,0 +1,13 @@
+import xarray as xr
+import rasterio
+from argparse import ArgumentParser
+
+parser = ArgumentParser()
+parser.add_argument("-f", "--file", dest="file")
+parser.add_argument("-o", "--output", dest="output")
+parser.add_argument("-v", "--variable", dest="variable")
+args = parser.parse_args()
+
+da = xr.open_rasterio(args.file)
+ds = da.to_dataset(name=args.variable)
+ds.to_netcdf(args.output, format='NETCDF4', mode='w')

data/triggers/biomass_agb.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+TIMEOUT=2000
+LAYER_ID="biomass.agb"
+VARIABLE_ID="agb"
+VERSION="0.4.1"
+ZOOM_LEVELS="0-7"
+MIN_LON="-180"
+MAX_LON="180"
+MIN_LAT="-60"
+MAX_LAT="80"
+MIN="0"
+MAX="100"
+MACHINE_TYPE="N1_HIGHCPU_32"
+
+if [ ! -f ./package.json ]; then
+    echo "You have to be in the root folder of the project to run this script!"
+    exit 1
+fi
+
+gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+  --machine-type=$MACHINE_TYPE \
+  --timeout=$TIMEOUT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  .

data/triggers/greenhouse_xch4.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+TIMEOUT=3000
+LAYER_ID="greenhouse.xch4"
+VARIABLE_ID="xch4"
+VERSION="0.4.1"
+ZOOM_LEVELS="0-2"
+MIN_LON="-180"
+MAX_LON="180"
+MIN_LAT="-90"
+MAX_LAT="90"
+MIN="auto"
+MAX="auto"
+MACHINE_TYPE="N1_HIGHCPU_8"
+
+if [ ! -f ./package.json ]; then
+    echo "You have to be in the root folder of the project to run this script!"
+    exit 1
+fi
+
+# --machine-type=$MACHINE_TYPE \
+gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+  --timeout=$TIMEOUT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  .

data/triggers/greenhouse_xco2.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+TIMEOUT=3000
+LAYER_ID="greenhouse.xco2"
+VARIABLE_ID="xco2"
+VERSION="0.4.1"
+ZOOM_LEVELS="0-3"
+MIN_LON="-180"
+MAX_LON="180"
+MIN_LAT="-90"
+MAX_LAT="90"
+MIN="auto"
+MAX="auto"
+MACHINE_TYPE="N1_HIGHCPU_8"
+
+if [ ! -f ./package.json ]; then
+    echo "You have to be in the root folder of the project to run this script!"
+    exit 1
+fi
+
+# --machine-type=$MACHINE_TYPE \
+gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+  --timeout=$TIMEOUT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  .

storage/layers/layers-de.json

@@ -94,5 +94,23 @@
     "name": "Snow - swe",
     "description": "Das ist der ...",
     "link": "http://..."
+  },
+  {
+    "id": "biomass.agb",
+    "name": "Biomass - agb",
+    "description": "Das ist der ...",
+    "link": "http://..."
+  },
+  {
+    "id": "greenhouse.xco2",
+    "name": "Greenhouse - xco2",
+    "description": "Das ist der ...",
+    "link": "http://..."
+  },
+  {
+    "id": "greenhouse.xch4",
+    "name": "Greenhouse - xch4",
+    "description": "Das ist der ...",
+    "link": "http://..."
   }
 ]

storage/layers/layers-en.json

@@ -94,5 +94,23 @@
     "name": "Snow - swe",
     "description": "This is the ...",
     "link": "http://..."
+  },
+  {
+    "id": "biomass.agb",
+    "name": "Biomass - agb",
+    "description": "This is the ...",
+    "link": "http://..."
+  },
+  {
+    "id": "greenhouse.xco2",
+    "name": "Greenhouse - xco2",
+    "description": "This is the ...",
+    "link": "http://..."
+  },
+  {
+    "id": "greenhouse.xch4",
+    "name": "Greenhouse - xch4",
+    "description": "This is the ...",
+    "link": "http://..."
   }
 ]