feat(dataset): update aerosol to v4.3 (#800)

ubilabs · Dec 14, 2020 · 961e0aa · 961e0aa
1 parent 0fc2281
commit 961e0aa
Show file tree

Hide file tree

Showing 20 changed files with 42 additions and 22 deletions.
diff --git a/data/downloads/odp-ftp-aerosol.sh b/data/downloads/odp-ftp-aerosol.sh
@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 
-BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/aerosol/data/AATSR_SU/L3/v4.21/MONTHLY/"
-START_DATE=2002-07-01
+VERSION="4.3"
+BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/aerosol/data/AATSR_SU/L3/v$VERSION/MONTHLY/"
+START_DATE=2012-01-01
 OUTPUT_FODLER=./download/aerosol
 
 mkdir -p $OUTPUT_FODLER
@@ -12,7 +13,7 @@ do
   NEXT_MONTH=$(date +%Y%m -d "$START_DATE + $i month")
   NEXT_DATE=$(date +%Y-%m-%d -d "$START_DATE + $i month")
   FILENAME=$OUTPUT_FODLER/$(date +%Y%m%d -d "$START_DATE + $i month").nc
-  FTP_URL=$BASE_URL$NEXT_YEAR/$NEXT_MONTH-ESACCI-L3C_AEROSOL-AER_PRODUCTS-AATSR_ENVISAT-SU_MONTHLY-v4.21.nc
+  FTP_URL=$BASE_URL$NEXT_YEAR/$NEXT_MONTH-ESACCI-L3C_AEROSOL-AER_PRODUCTS-AATSR_ENVISAT-SU_MONTHLY-v$VERSION.nc
   echo $FTP_URL
 
   curl --silent $FTP_URL > $FILENAME

diff --git a/data/triggers/aerosol_aod550_mean.sh b/data/triggers/aerosol_aod550_mean.sh
@@ -22,7 +22,8 @@ if [ ! -f ./package.json ]; then
 fi
 
 # --machine-type=$MACHINE_TYPE \
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/biomass_agb.sh b/data/triggers/biomass_agb.sh
@@ -21,7 +21,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --machine-type=$MACHINE_TYPE \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \

diff --git a/data/triggers/cloud_cfc.sh b/data/triggers/cloud_cfc.sh
@@ -22,7 +22,8 @@ if [ ! -f ./package.json ]; then
 fi
 
 # --machine-type=$MACHINE_TYPE \
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/fire_burned_area.sh b/data/triggers/fire_burned_area.sh
@@ -22,7 +22,8 @@ if [ ! -f ./package.json ]; then
 fi
 
 # --machine-type=$MACHINE_TYPE \
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/greenland_ice_sec.sh b/data/triggers/greenland_ice_sec.sh
@@ -18,7 +18,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/lakes_chla_mean.sh b/data/triggers/lakes_chla_mean.sh
@@ -21,7 +21,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --machine-type=$MACHINE_TYPE \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \

diff --git a/data/triggers/lakes_lake_surface_water_temperature.sh b/data/triggers/lakes_lake_surface_water_temperature.sh
@@ -21,7 +21,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --machine-type=$MACHINE_TYPE \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \

diff --git a/data/triggers/land_cover_lccs_class.sh b/data/triggers/land_cover_lccs_class.sh
@@ -21,7 +21,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --machine-type=$MACHINE_TYPE \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \

diff --git a/data/triggers/oc_chlor_a.sh b/data/triggers/oc_chlor_a.sh
@@ -21,7 +21,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --machine-type=$MACHINE_TYPE \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \

diff --git a/data/triggers/ozone_total_ozone_column.sh b/data/triggers/ozone_total_ozone_column.sh
@@ -20,7 +20,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud --project esa-climate-from-space builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_LAYER_TYPE=$LAYER_TYPE,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/sea_ice_nh_ice_conc.sh b/data/triggers/sea_ice_nh_ice_conc.sh
@@ -22,7 +22,8 @@ if [ ! -f ./package.json ]; then
 fi
 
 # --machine-type=$MACHINE_TYPE \
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/sea_ice_sh_ice_conc.sh b/data/triggers/sea_ice_sh_ice_conc.sh
@@ -22,7 +22,8 @@ if [ ! -f ./package.json ]; then
 fi
 
 # --machine-type=$MACHINE_TYPE \
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/sea_level_sla.sh b/data/triggers/sea_level_sla.sh
@@ -20,7 +20,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/sea_state_swh_mean.sh b/data/triggers/sea_state_swh_mean.sh
@@ -22,7 +22,8 @@ if [ ! -f ./package.json ]; then
 fi
 
 # --machine-type=$MACHINE_TYPE \
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/sea_surface_salinity_sss.sh b/data/triggers/sea_surface_salinity_sss.sh
@@ -25,7 +25,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT,_SOURCE_PROJECTION=$SOURCE_PROJECTION,_SOURCE_BOUNDS=$SOURCE_BOUNDS,_SOURCE_MIN_X=$SOURCE_MIN_X,_SOURCE_MAX_X=$SOURCE_MAX_X,_SOURCE_MIN_Y=$SOURCE_MIN_Y,_SOURCE_MAX_Y=$SOURCE_MAX_Y \
   .
diff --git a/data/triggers/snow_swe.sh b/data/triggers/snow_swe.sh
@@ -20,7 +20,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/soil_moisture_anomaly.sh b/data/triggers/soil_moisture_anomaly.sh
@@ -20,7 +20,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/soil_moisture_sm_mean.sh b/data/triggers/soil_moisture_sm_mean.sh
@@ -20,7 +20,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \
   .
diff --git a/data/triggers/sst_analysed_sst.sh b/data/triggers/sst_analysed_sst.sh
@@ -21,7 +21,8 @@ if [ ! -f ./package.json ]; then
     exit 1
 fi
 
-gcloud builds submit --config ./ci/cloudbuild-tiles-reproject.yaml \
+gcloud --project esa-climate-from-space builds submit \
+  --config ./ci/cloudbuild-tiles-reproject.yaml \
   --machine-type=$MACHINE_TYPE \
   --timeout=$TIMEOUT \
   --substitutions _LAYER_ID=$LAYER_ID,_VARIABLE_ID=$VARIABLE_ID,_ZOOM_LEVELS=$ZOOM_LEVELS,_LAYER_TYPE=$LAYER_TYPE,_LON_RES=$LON_RES,_LAT_RES=$LAT_RES,_VERSION=$VERSION,_MIN=$MIN,_MAX=$MAX,_MIN_LON=$MIN_LON,_MAX_LON=$MAX_LON,_MIN_LAT=$MIN_LAT,_MAX_LAT=$MAX_LAT \