feat(data-pipeline): add sss dataset (#338)

ci/cloudbuild-tiles-reproject.yaml

@@ -19,8 +19,10 @@ steps:
     args:
        - "./data/gdal-reproject.sh"
        - "${_VARIABLE_ID}"
-       - "${_MIN_LON} ${_MIN_LAT} ${_MAX_LON} ${_MAX_LAT}"
        - "${_ZOOM_LEVELS}"
+       - "${_MIN_LON} ${_MIN_LAT} ${_MAX_LON} ${_MAX_LAT}"
+       - "${_SOURCE_PROJECTION}"
+       - "${_SOURCE_MIN_X} ${_SOURCE_MAX_Y} ${_SOURCE_MAX_X} ${_SOURCE_MIN_Y}"
 
   - name: gcr.io/esa-climate-from-space/tile-mover
     id: 'prepare-upload'
@@ -40,6 +42,7 @@ steps:
       - "gs://esa-cfs-tiles/${_VERSION}/${_LAYER_ID}/"
 
 options:
+  substitution_option: 'ALLOW_LOOSE'
   diskSizeGb: 50
   volumes:
   - name: 'vol1'

data/downloads/odp-ftp-sss.sh → data/downloads/odp-ftp-sea-surface-salinity.sh

@@ -4,7 +4,7 @@ BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/sea_surface_salinity/data/v01.8
 START_DATE=2010-01-01
 OUTPUT_FODLER=./download
 
-for i in {0..3}
+for i in {0..5}
 do
   NEXT_YEAR=$(date +%Y -d "$START_DATE + $i month")
   NEXT_DATE=$(date +%Y%m%d -d "$START_DATE + $i month")

data/gdal-colors/colors-sss.txt

@@ -1,7 +1,3 @@
-100 0 0 255
-80 0 155 100
-60 100 155 0
-40 255 255 0
-20 155 255 0
-1 0 255 0
+50 0 255 0
+1 255 0 0
 nv 0 255 255 0

data/gdal-reproject.sh

@@ -1,18 +1,33 @@
 #!/usr/bin/env bash
 
 VARIABLE=$1
-BOUNDS=$2
-ZOOM_LEVELS=$3
+ZOOM_LEVELS=$2
+OUT_BOUNDS=$3
+SOURCE_PROJECTION=$4
+SOURCE_BOUNDS=$5
+TRIMMED_SOURCE_BOUNDS=$(echo $SOURCE_BOUNDS | sed 's/ *$//g')
 FOLDER=/data/netcdfs/
 counter=0
 
 for file in $(find $FOLDER -name *.nc -type f | sort -n); do
   echo "--------------"
   echo $file
 
+  if [ ! -z "$SOURCE_PROJECTION" ] && [ ! -z "$TRIMMED_SOURCE_BOUNDS" ]
+  then
+      echo "Updating projection information"
+      # if defined set projection and bounds in netcdf file
+      gdal_translate \
+        -of NETCDF \
+        -a_srs $SOURCE_PROJECTION \
+        -a_ullr $TRIMMED_SOURCE_BOUNDS \
+        NETCDF:\"$file\":$VARIABLE \
+        $file
+  fi
+
   gdalwarp \
     -t_srs EPSG:4326 \
-    -te $BOUNDS \
+    -te $OUT_BOUNDS \
     NETCDF:\"$file\":$VARIABLE \
     ./tmp.tif
 

data/triggers/sea_surface_salinity_sss.sh

@@ -7,6 +7,15 @@ VERSION="test"
 ZOOM_LEVELS="0-3"
 MIN="auto"
 MAX="auto"
+MIN_LON="-180"
+MAX_LON="180"
+MIN_LAT="-90"
+MAX_LAT="90"
+SOURCE_PROJECTION="EPSG:3410"
+SOURCE_MIN_X="-17324563.84"
+SOURCE_MAX_X="17324563.84"
+SOURCE_MIN_Y="-7338939.46"
+SOURCE_MAX_Y="7338939.46"
 MACHINE_TYPE="N1_HIGHCPU_8"
 
 if [ ! -f ./package.json ]; then
@@ -15,7 +24,7 @@ if [ ! -f ./package.json ]; then
 fi
 
 # --machine-type=$MACHINE_TYPE \
-gcloud builds submit --config ./ci/cloudbuild-tiles.yaml \
+gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
-  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT,_SOURCE_PROJECTION=$SOURCE_PROJECTION,_SOURCE_BOUNDS=$SOURCE_BOUNDS,_SOURCE_MIN_X=$SOURCE_MIN_X,_SOURCE_MAX_X=$SOURCE_MAX_X,_SOURCE_MIN_Y=$SOURCE_MIN_Y,_SOURCE_MAX_Y=$SOURCE_MAX_Y \
   .

storage/layers/layers-de.json

@@ -82,5 +82,11 @@
     "name": "Land Cover - lccs_class",
     "description": "Das ist der ...",
     "link": "http://..."
+  },
+  {
+    "id": "sea_surface_salinity.sss",
+    "name": "Sea Surface Salinity - sss",
+    "description": "Das ist der ...",
+    "link": "http://..."
   }
 ]

storage/layers/layers-en.json

@@ -82,5 +82,11 @@
     "name": "Land Cover - lccs_class",
     "description": "This is the ...",
     "link": "http://..."
+  },
+  {
+    "id": "sea_surface_salinity.sss",
+    "name": "Sea Surface Salinity - sss",
+    "description": "This is the ...",
+    "link": "http://..."
   }
 ]