feat(data-layer): use single image for some layers (#496)

ubilabs · Aug 20, 2020 · 7d447a3 · 7d447a3
1 parent 9a282ff
commit 7d447a3
Show file tree

Hide file tree

Showing 33 changed files with 236 additions and 102 deletions.
diff --git a/ci/cloudbuild-tiles-reproject.yaml b/ci/cloudbuild-tiles-reproject.yaml
@@ -11,15 +11,17 @@ steps:
     entrypoint: '/bin/bash'
     args:
       - "-c"
-      - "conda run -n cate-env python data/write-zarr.py --layer ${_LAYER_ID} --variable ${_VARIABLE_ID} --zoom-levels ${_ZOOM_LEVELS} --min ${_MIN} --max ${_MAX}"
+      - "conda run -n cate-env python data/write-zarr.py --layer ${_LAYER_ID} --variable ${_VARIABLE_ID} --type ${_LAYER_TYPE} --zoom-levels ${_ZOOM_LEVELS} --min ${_MIN} --max ${_MAX}"
 
-  - name: geographica/gdal2:2.4.0
+  - name: osgeo/gdal:ubuntu-full-3.1.2
     id: 'gdal-generate-tiles'
     entrypoint: '/bin/bash'
     args:
        - "./data/gdal-reproject.sh"
        - "${_VARIABLE_ID}"
        - "${_ZOOM_LEVELS}"
+       - "${_LAYER_TYPE}"
+       - "${_LON_RES} ${_LAT_RES}"
        - "${_MIN_LON} ${_MIN_LAT} ${_MAX_LON} ${_MAX_LAT}"
        - "${_SOURCE_PROJECTION}"
        - "${_SOURCE_MIN_X} ${_SOURCE_MAX_Y} ${_SOURCE_MAX_X} ${_SOURCE_MIN_Y}"

diff --git a/data/convert-colormap.py b/data/convert-colormap.py
@@ -0,0 +1,21 @@
+from matplotlib import cm
+
+stepsNum = 10
+minValue = 0
+maxValue = 6
+cmap = cm.get_cmap('Blues')
+reverse = True
+
+r = range(0, stepsNum + 1)
+
+diff = (maxValue - minValue) / stepsNum
+steps = [n / stepsNum for n in r]
+values = [round(minValue + diff * n, 1) for n in r]
+
+if reverse:
+  values.reverse()
+
+for s, v in zip(steps, values):
+  r,g,b,a = cmap(s)
+  print(v, int(r * 255), int(g * 255), int(b * 255), int(a * 255))
+
diff --git a/data/downloads/odp-ftp-ozone.py b/data/downloads/odp-ftp-ozone.py
@@ -0,0 +1,29 @@
+import os
+from datetime import datetime
+from ftplib import FTP
+
+ftp = FTP('anon-ftp.ceda.ac.uk')
+ftp.login()
+
+BASE_PATH = 'neodc/esacci/ozone/data/total_columns/l3/merged/v0100'
+OUTPUT_PATH = f"{os.getcwd()}/download/ozone"
+
+years = range(1996, 2011)
+
+for year in years:
+  files = ftp.nlst(f'{BASE_PATH}/{year}/')
+
+  for file in files:
+    file_name = file.split('-')[-2]
+    date = datetime.strptime(file_name, '%Y%m%d').strftime('%Y-%m-%d')
+    out = f"{OUTPUT_PATH}/{file_name}.nc"
+
+    with open(out, 'wb') as fp:
+      ftp.retrbinary(f"RETR {file}", fp.write)
+      print(file_name)
+
+    os.system(f"python ./data/drop-unused-vars.py --file {out} --variable atmosphere_mole_content_of_ozone")
+    os.system(f"python {os.getcwd()}/data/add-time-coordinate.py --file {out} --timestamp {date}")
+    os.system(f"python ./data/wrap-longitude.py --file {out}")
+
+ftp.quit()
diff --git a/data/downloads/odp-ftp-ozone.sh b/data/downloads/odp-ftp-ozone.sh
diff --git a/data/downloads/odp-ftp-snow.py b/data/downloads/odp-ftp-snow.py
@@ -0,0 +1,24 @@
+import os
+from ftplib import FTP
+
+ftp = FTP('anon-ftp.ceda.ac.uk')
+ftp.login()
+
+BASE_PATH = '/neodc/esacci/snow/data/swe/MERGED/v1.0'
+OUTPUT_PATH = f"{os.getcwd()}/download/snow"
+
+years = range(1979, 2018)
+
+for year in years:
+  months = ftp.nlst(f'{BASE_PATH}/{year}/')
+
+  for month in months:
+    days = ftp.nlst(month)
+    first_day = days[0]
+    file_name = first_day.split('/').pop().split('-')[0]
+
+    with open(f"{OUTPUT_PATH}/{file_name}.nc", 'wb') as fp:
+      ftp.retrbinary(f"RETR {first_day}", fp.write)
+      print(file_name)
+
+ftp.quit()
diff --git a/data/gdal-colors/colors-AOD550_mean.txt b/data/gdal-colors/colors-AOD550_mean.txt
@@ -1,5 +1,3 @@
-5 192 75 20 255
-2.5 241 162 33 255
-0.5 250 220 40 44
-0 250 220 40 0
+1 184 101 9 255
+0 184 101 9 0
 nv 0 0 0 0
diff --git a/data/gdal-colors/colors-swe.txt b/data/gdal-colors/colors-swe.txt
@@ -0,0 +1,3 @@
+250 255 255 255 255
+0 9 9 12 255
+nv 0 0 0 0
diff --git a/data/gdal-colors/colors-swh_mean.txt b/data/gdal-colors/colors-swh_mean.txt
@@ -0,0 +1,12 @@
+6.0 247 251 255 255
+5.4 227 238 248 255
+4.8 207 225 242 255
+4.2 182 212 233 255
+3.6 147 196 222 255
+3.0 106 173 213 255
+2.4 74 151 201 255
+1.8 45 125 187 255
+1.2 23 100 171 255
+0.6 8 73 145 255
+0.0 8 48 107 255
+nv 0 0 0 0
diff --git a/data/gdal-reproject.sh b/data/gdal-reproject.sh
@@ -2,9 +2,11 @@
 
 VARIABLE=$1
 ZOOM_LEVELS=$2
-OUT_BOUNDS=$3
-SOURCE_PROJECTION=$4
-SOURCE_BOUNDS=$5
+LAYER_TYPE=$3
+RESOLUTION=$4
+OUT_BOUNDS=$5
+SOURCE_PROJECTION=$6
+SOURCE_BOUNDS=$7
 TRIMMED_SOURCE_BOUNDS=$(echo $SOURCE_BOUNDS | sed 's/ *$//g')
 FOLDER=/data/netcdfs/
 timestamp_counter=0
@@ -28,6 +30,7 @@ for file in $(find $FOLDER -name *.nc -type f | sort -n); do
   gdalwarp \
     -t_srs EPSG:4326 \
     -te $OUT_BOUNDS \
+    -ts $RESOLUTION \
     -r near \
     --config GDAL_CACHEMAX 90% \
     -co compress=LZW \
@@ -42,7 +45,8 @@ for file in $(find $FOLDER -name *.nc -type f | sort -n); do
     -co compress=LZW \
     -alpha ./colored.tif
 
-  gdal2tiles.py \
+  if [ "$LAYER_TYPE" = "tiles" ]; then
+    gdal2tiles.py \
     --profile geodetic \
     --zoom=$ZOOM_LEVELS \
     --tmscompatible \
@@ -51,6 +55,14 @@ for file in $(find $FOLDER -name *.nc -type f | sort -n); do
     --resampling near \
     --s_srs EPSG:4326 \
     ./colored.tif /data/images/$VARIABLE/$timestamp_counter
+  else
+    mkdir -p /data/images/$VARIABLE/$timestamp_counter
+
+    gdal_translate \
+      -of PNG \
+      ./colored.tif \
+      /data/images/$VARIABLE/$timestamp_counter/full.png
+  fi
 
   rm ./tmp.tif
   rm ./colored.tif

diff --git a/data/triggers/aerosol_aod550_mean.sh b/data/triggers/aerosol_aod550_mean.sh
@@ -1,16 +1,19 @@
 #!/usr/bin/env bash
 
-TIMEOUT=8000
+TIMEOUT=4000
 LAYER_ID="aerosol.AOD550_mean"
 VARIABLE_ID="AOD550_mean"
-VERSION="0.6.1"
+LAYER_TYPE="image"
+VERSION="0.7.1"
+LON_RES="360"
+LAT_RES="180"
 ZOOM_LEVELS="0-3"
 MIN_LON="-180"
 MAX_LON="180"
 MIN_LAT="-90"
 MAX_LAT="90"
 MIN="0"
-MAX="5"
+MAX="1"
 MACHINE_TYPE="N1_HIGHCPU_8"
 
 if [ ! -f ./package.json ]; then
@@ -21,5 +24,5 @@ fi
 # --machine-type=$MACHINE_TYPE \
 gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
-  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/biomass_agb.sh b/data/triggers/biomass_agb.sh
@@ -3,7 +3,10 @@
 TIMEOUT=8000
 LAYER_ID="biomass.agb"
 VARIABLE_ID="agb"
-VERSION="0.6.1"
+LAYER_TYPE="tiles"
+VERSION="0.7.1"
+LON_RES="81000"
+LAT_RES="31500"
 ZOOM_LEVELS="0-5"
 MIN_LON="-180"
 MAX_LON="180"
@@ -18,11 +21,11 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-# Note! If process takes too long use "-ts 81000 31500 \" in gdalwarp command to
+# Note! Using resolution of "-ts 81000 31500 \" in gdalwarp command to
 # output only a 1/5th-resolution image
 
 gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
   --machine-type=$MACHINE_TYPE \
   --timeout=$TIMEOUT \
-  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/cloud_cfc.sh b/data/triggers/cloud_cfc.sh
@@ -3,7 +3,10 @@
 TIMEOUT=8000
 LAYER_ID="cloud.cfc"
 VARIABLE_ID="cfc"
-VERSION="0.6.1"
+LAYER_TYPE="image"
+VERSION="0.7.1"
+LON_RES="720"
+LAT_RES="360"
 ZOOM_LEVELS="0-3"
 MIN_LON="-180"
 MAX_LON="180"
@@ -21,5 +24,5 @@ fi
 # --machine-type=$MACHINE_TYPE \
 gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
-  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/fire_burned_area.sh b/data/triggers/fire_burned_area.sh
@@ -3,7 +3,10 @@
 TIMEOUT=8000
 LAYER_ID="fire.burned_area"
 VARIABLE_ID="burned_area"
-VERSION="0.6.1"
+LAYER_TYPE="image"
+VERSION="0.7.1"
+LON_RES="1440"
+LAT_RES="720"
 ZOOM_LEVELS="0-3"
 MIN_LON="-180"
 MAX_LON="180"
@@ -21,5 +24,5 @@ fi
 # --machine-type=$MACHINE_TYPE \
 gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
-  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/greenhouse_xch4.sh b/data/triggers/greenhouse_xch4.sh
@@ -3,23 +3,24 @@
 TIMEOUT=3000
 LAYER_ID="greenhouse.xch4"
 VARIABLE_ID="xch4"
-VERSION="0.6.1"
+LAYER_TYPE="image"
+VERSION="0.7.1"
+LON_RES="180"
+LAT_RES="90"
 ZOOM_LEVELS="0-2"
 MIN_LON="-180"
 MAX_LON="180"
 MIN_LAT="-90"
 MAX_LAT="90"
 MIN="auto"
 MAX="auto"
-MACHINE_TYPE="N1_HIGHCPU_8"
 
 if [ ! -f ./package.json ]; then
     echo "You have to be in the root folder of the project to run this script!"
     exit 1
 fi
 
-# --machine-type=$MACHINE_TYPE \
 gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
-  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/greenhouse_xco2.sh b/data/triggers/greenhouse_xco2.sh
@@ -3,23 +3,24 @@
 TIMEOUT=3000
 LAYER_ID="greenhouse.xco2"
 VARIABLE_ID="xco2"
-VERSION="0.6.1"
-ZOOM_LEVELS="0-3"
+LAYER_TYPE="image"
+VERSION="0.7.1"
+LON_RES="180"
+LAT_RES="90"
+ZOOM_LEVELS="0-2"
 MIN_LON="-180"
 MAX_LON="180"
 MIN_LAT="-90"
 MAX_LAT="90"
 MIN="auto"
 MAX="auto"
-MACHINE_TYPE="N1_HIGHCPU_8"
 
 if [ ! -f ./package.json ]; then
     echo "You have to be in the root folder of the project to run this script!"
     exit 1
 fi
 
-# --machine-type=$MACHINE_TYPE \
 gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
-  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/greenland_ice_sec.sh b/data/triggers/greenland_ice_sec.sh
@@ -3,23 +3,22 @@
 TIMEOUT=2000
 LAYER_ID="greenland_ice.sec"
 VARIABLE_ID="SEC"
-VERSION="0.6.1"
+LAYER_TYPE="tiles"
+VERSION="0.7.1"
 ZOOM_LEVELS="0-3"
 MIN_LON="-90"
 MAX_LON="7.594643368591434"
 MIN_LAT="58.854580820213855"
 MAX_LAT="84.00492144822202"
 MIN="-5"
 MAX="5"
-MACHINE_TYPE="N1_HIGHCPU_8"
 
 if [ ! -f ./package.json ]; then
     echo "You have to be in the root folder of the project to run this script!"
     exit 1
 fi
 
-# --machine-type=$MACHINE_TYPE \
 gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
-  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
+  --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .