feat(data): update (#945)

* feast(ocean colour): update download script

* feat(ocean colour): update data set

* feat(soil moisture): update data

* feat(biomass): adjust download

* feat(soil moisture): remove timestamping

* feat(permafrost): adjust download script

* fix(remove files): remove unneeded files

data/downloads/odp-ftp-biomass.sh

@@ -1,13 +1,20 @@
 #!/usr/bin/env bash
 
-URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/biomass/data/agb/maps/2017/v1.0/netcdf/ESACCI-BIOMASS-L4-AGB-MERGED-100m-2017-fv1.0.nc"
-START_DATE=2017-01-01
+DATA_VERSION="v2.0"
+YEARS="2010 2017 2018"
 OUTPUT_FODLER=./download
 
-FILENAME=$OUTPUT_FODLER/$(date +%Y%m%d -d "$START_DATE + $i month").nc
-echo $FTP_URL
 
-curl --silent $FTP_URL > $FILENAME
+for YEAR in $YEARS
+do
+    FTP_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/biomass/data/agb/maps/$DATA_VERSION/netcdf/$YEAR/ESACCI-BIOMASS-L4-AGB-MERGED-100m-$YEAR-f$DATA_VERSION.nc"
+    FILENAME="$OUTPUT_FODLER/$YEAR0101".nc
 
-python ./data/drop-unused-vars.py --file $FILENAME --variable AOD550_mean
-python ./data/add-time-coordinate.py --file $FILENAME --timestamp $NEXT_DATE
+    echo $FTP_URL
+
+    curl --silent $FTP_URL > $FILENAME
+
+    # TODO
+    # python ./data/drop-unused-vars.py --file $FILENAME --variable AOD550_mean
+    # python ./data/add-time-coordinate.py --file $FILENAME --timestamp $(date -d "$YEAR-01-01")
+done

data/downloads/odp-ftp-oc.sh

@@ -1,21 +1,29 @@
 #!/usr/bin/env bash
 
-BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/ocean_colour/data/v4.2-release/geographic/netcdf/chlor_a/monthly/v4.2/"
+VERSION=5.0
 START_DATE=1997-09-01
-OUTPUT_FODLER=./download/oc
+END_DATE=2020-12-01
 
-mkdir -p $OUTPUT_FODLER
+BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/ocean_colour/data/v$VERSION-release/geographic/netcdf/chlor_a/monthly/v$VERSION/"
+OUTPUT_FOLDER=./download/oc
 
-for i in {0..255}
-do
-  NEXT_YEAR=$(date +%Y -d "$START_DATE + $i month")
-  NEXT_MONTH=$(date +%Y%m -d "$START_DATE + $i month")
-  FILENAME=$OUTPUT_FODLER/$(date +%Y%m%d -d "$START_DATE + $i month").nc
-  FTP_URL=$BASE_URL$NEXT_YEAR/ESACCI-OC-L3S-CHLOR_A-MERGED-1M_MONTHLY_4km_GEO_PML_OCx-$NEXT_MONTH-fv4.2.nc
-  echo $FTP_URL
+# timestamped download folder to not overwrite prvious runs
+OUTPUT_FOLDER=$OUTPUT_FOLDER-$(env TZ=Europe/Berlin date +%Y%m%d%H%M)
+mkdir -p $OUTPUT_FOLDER
+
+current_date=$START_DATE
+while [[ "$current_date" < $(date -I -d "$END_DATE + 1 month") ]]; do 
+  echo $current_date
+  NEXT_YEAR=$(date +%Y -d "$current_date")
+  NEXT_MONTH=$(date +%Y%m -d "$current_date")
+  FILENAME=$OUTPUT_FOLDER/$(date +%Y%m%d -d "$current_date").nc
+  FTP_URL=$BASE_URL$NEXT_YEAR/ESACCI-OC-L3S-CHLOR_A-MERGED-1M_MONTHLY_4km_GEO_PML_OCx-$NEXT_MONTH-fv$VERSION.nc
 
+  echo $FTP_URL
   curl --silent $FTP_URL > $FILENAME
 
   python ./data/log-values.py --file $FILENAME --variable chlor_a
   python ./data/drop-unused-vars.py --file $FILENAME --variable chlor_a
+
+  current_date=$(date -I -d "$current_date + 1 month")
 done

data/downloads/odp-ftp-permafrost.sh

@@ -1,20 +1,43 @@
 #!/usr/bin/env bash
 
-BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/permafrost/data/permafrost_extent/L4/area4/pp/v01.0/ESACCI-PERMAFROST-L4-MODIS-PFR-AREA4_PP"
-START_DATE=2003-01-01
+#             ftp://anon-ftp.ceda.ac.uk/neodc/esacci/permafrost/data/permafrost_extent/L4/area4/pp/v03.0/ESACCI-PERMAFROST-L4-PFR-ERA5_MODISLST_BIASCORRECTED-AREA4_PP-1997-fv03.0.nc
+BASE_URL_OLD="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/permafrost/data/permafrost_extent/L4/area4/pp/v03.0/ESACCI-PERMAFROST-L4-PFR-ERA5_MODISLST_BIASCORRECTED-AREA4_PP"
+#             ftp://anon-ftp.ceda.ac.uk/neodc/esacci/permafrost/data/permafrost_extent/L4/area4/pp/v03.0/ESACCI-PERMAFROST-L4-PFR-MODISLST_CRYOGRID-AREA4_PP-2003-fv03.0.nc
+BASE_URL_NEW="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/permafrost/data/permafrost_extent/L4/area4/pp/v03.0/ESACCI-PERMAFROST-L4-PFR-MODISLST_CRYOGRID-AREA4_PP"
+START_DATE=1997-01-01
+END_DATE=2019-12-01
 OUTPUT_FODLER=./download/permafrost
 
 mkdir -p $OUTPUT_FODLER
 
-for i in {0..14}
-do
-  NEXT_YEAR=$(date +%Y -d "$START_DATE + $i year")
-  NEXT_DATE=$(date +%Y-%m-%d -d "$START_DATE + $i year")
-  FILENAME=$OUTPUT_FODLER/$(date +%Y%m%d -d "$START_DATE + $i year").nc
-  FTP_URL=$BASE_URL"-"$NEXT_YEAR"-fv01.0.nc"
+current_date=$START_DATE
+while [[ "$current_date" < $(date -I -d "$END_DATE + 1 month") ]]; do 
+  echo "$current_date"
+  NEXT_YEAR=$(date +%Y -d "$current_date")
+
+  FILENAME=$OUTPUT_FODLER/$NEXT_YEAR.nc
+
+  if [[ $current_date -lt "2003-01-01" ]]
+  then
+    FTP_URL=$BASE_URL_OLD"-"$NEXT_YEAR"-fv03.0.nc"
+  else
+    FTP_URL=$BASE_URL_NEW"-"$NEXT_YEAR"-fv03.0.nc"
+  fi
+
   echo $FTP_URL
+  # echo $FILENAME
+  # curl --silent $FTP_URL > $FILENAME
+
+
+
+  # NEXT_YEAR=$(date +%Y -d "$START_DATE + $i year")
+  # NEXT_DATE=$(date +%Y-%m-%d -d "$START_DATE + $i year")
+
+  # FTP_URL=$BASE_URL"-"$NEXT_YEAR"-fv01.0.nc"
+  # echo $FTP_URL
 
-  curl --silent $FTP_URL > $FILENAME
+  # curl --silent $FTP_URL > $FILENAME
 
-  python ./data/add-time-coordinate.py --file $FILENAME --timestamp $NEXT_DATE
+  python ./data/add-time-coordinate.py --file $FILENAME --timestamp $current_date
+  current_date=$(date -I -d "$current_date + 1 year")
 done

data/downloads/owncloud-soil-moisture-anomaly.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+BASE_URL="https://owncloud.tuwien.ac.at/index.php/s/bY8j2kBgZlkqFYC/download?path=%2F&files=ESACCI-SOILMOISTURE-L3S-SSMV-MONTHLY_MEAN-COMBINED-19781101-20191231-fv05.2.nc"
+OUTPUT_FOLDER=./download/soil_moisture_anomaly
+
+mkdir -p $OUTPUT_FOLDER
+
+FILENAME=$OUTPUT_FOLDER/sma.nc
+curl --silent $BASE_URL > $FILENAME
+
+python ./data/split-time-dim.py --file $FILENAME --folder $OUTPUT_FOLDER --variable Anomaly
+

data/downloads/owncloud-soil-moisture.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+BASE_URL="https://owncloud.tuwien.ac.at/index.php/s/bY8j2kBgZlkqFYC/download?path=%2F&files=ESACCI-SOILMOISTURE-L3S-SSMV-MONTHLY_MEAN-COMBINED-19781101-20191231-fv05.2.nc"
+OUTPUT_FOLDER=./download/soil_moisture
+
+mkdir -p $OUTPUT_FOLDER
+
+FILENAME=$OUTPUT_FOLDER/sm.nc
+curl --silent $BASE_URL > $FILENAME
+
+python ./data/split-time-dim.py --file $FILENAME --folder $OUTPUT_FOLDER --variable sm_mean
+

data/split-time-dim.py

@@ -6,6 +6,7 @@
 
 parser = ArgumentParser()
 parser.add_argument("-f", "--file", dest="file")
+parser.add_argument("-o", "--folder", dest="folder")
 parser.add_argument("-v", "--variable", dest="variable")
 args = parser.parse_args()
 
@@ -15,6 +16,6 @@
 for n in range(len(da.time)):
   da_slice = da.isel(time=n)
   date = pd.to_datetime(str(da_slice.time.values))
-  filename = date.strftime('./download/soil_moisture/%Y%m%d.nc')
+  filename = date.strftime('%s/%%Y%%m%%d.nc' % (args.folder))
   print(filename)
   da_slice.to_dataset().to_netcdf(filename, format='NETCDF4', mode='w')

data/triggers/oc_chlor_a.sh

@@ -4,7 +4,7 @@ TIMEOUT=8000
 LAYER_ID="oc.chlor_a"
 VARIABLE_ID="chlor_a"
 LAYER_TYPE="image"
-VERSION="1.1.1"
+VERSION="1.2.1"
 LON_RES="2048"
 LAT_RES="1024"
 ZOOM_LEVELS="0-3"

data/triggers/soil_moisture_anomaly.sh

@@ -4,7 +4,7 @@ TIMEOUT=4000
 LAYER_ID="soil_moisture.Anomaly"
 VARIABLE_ID="Anomaly"
 LAYER_TYPE="image"
-VERSION="1.1.1"
+VERSION="1.2.1"
 LON_RES="1440"
 LAT_RES="720"
 ZOOM_LEVELS="0-3"

data/triggers/soil_moisture_sm_mean.sh

@@ -4,7 +4,7 @@ TIMEOUT=8000
 LAYER_ID="soil_moisture.sm_mean"
 VARIABLE_ID="sm_mean"
 LAYER_TYPE="image"
-VERSION="1.1.1"
+VERSION="1.2.1"
 LON_RES="1440"
 LAT_RES="720"
 ZOOM_LEVELS="0-3"

storage/layers/layers-de.json

@@ -11,14 +11,14 @@
     "type": "Hydrosphere",
     "name": "Soil Moisture",
     "shortName": "Soil Moisture",
-    "description": "Soil moisture (SM) describes the volumetric water content stored in the first few centimetres of soil. SM takes a key role in the water, energy and carbon cycle due to its feedback on vegetation growth, evapotranspiration and precipitation. It favours the occurrence of floods and droughts and is therefore essential for monitoring and predicting their impact on the environment.\n\nIn CCI a long-term (~40 year long) record is created by merging multiple active (radar) and passive (radiometer) sensor based satellite SM data sets into three harmonised, daily products. The current version contains quality checked data from 1978 to 2019 from 11 different sensors. Unreliable observations - as e.g. under frozen soil conditions - are masked out in the data. The animation in the App shows monthly aggregates of the COMBINED product of ESA CCI SM v04.7.\n\n**Variable Shown:** Volumetric Soil Moisture in m3/m3  \n**Time Span:** November 1978 – December 2019  \n**Temporal resolution:** monthly  \n**Geographic Extent:** global  \n**Spatial Resolution:**  0.25 degrees  \n**Version:** 4.7  \n\n[ESA CCI Soil Moisture ECV Project website](https://climate.esa.int/projects/soil-moisture/)  \n[Data in the Open Data Portal](https://catalogue.ceda.ac.uk/uuid/2d4a50f390064820a9dcc2fcf7ac4b18)"
+    "description": "Soil moisture (SM) describes the volumetric water content stored in the first few centimetres of soil. SM takes a key role in the water, energy and carbon cycle due to its feedback on vegetation growth, evapotranspiration and precipitation. It favours the occurrence of floods and droughts and is therefore essential for monitoring and predicting their impact on the environment.\n\nIn CCI a long-term (~40 year long) record is created by merging multiple active (radar) and passive (radiometer) sensor based satellite SM data sets into three harmonised, daily products. The current version contains quality checked data from 1978 to 2019 from 11 different sensors. Unreliable observations - as e.g. under frozen soil conditions - are masked out in the data. The animation in the App shows monthly aggregates of the COMBINED product of ESA CCI SM v05.2.\n\n**Variable Shown:** Volumetric Soil Moisture in m3/m3  \n**Time Span:** November 1978 – December 2019  \n**Temporal resolution:** monthly  \n**Geographic Extent:** global  \n**Spatial Resolution:**  0.25 degrees  \n**Version:** 5.2  \n\n[ESA CCI Soil Moisture ECV Project website](https://climate.esa.int/projects/soil-moisture/)  \n[Data in the Open Data Portal](https://catalogue.ceda.ac.uk/uuid/2d4a50f390064820a9dcc2fcf7ac4b18)"
   },
   {
     "id": "soil_moisture.Anomaly",
     "type": "Hydrosphere",
     "name": "Soil Moisture – Anomalies",
     "shortName": "Soil Moisture – Anomalies",
-    "description": "Soil Moisture (SM) states are affected by long-term climatological patterns as well as by short-term, local events. SM anomalies separate recurring seasonal/climatological variations from varying regional conditions and represent these deviations from the average conditions at each point in time. Droughts are therefore represented as negative anomalies while positive anomalies indicate “wetter than normal” conditions.\n\nIn CCI SM anomalies are calculated as differences between absolute SM observations and the SM climatology from a 20 year baseline period (1991-2010). This period is selected with regard to the available data density and quality, which can affect the derived deviations from the average conditions. The animation in the App shows monthly SM anomalies from 1991-2019 in the COMBINED product of ESA CCI SM v04.7.\n\n**Variable Shown:** Anomaly  \n**Time Span:** November 1978 – December 2019  \n**Temporal resolution:** monthly  \n**Geographic Extent:** global  \n**Spatial Resolution:**  0.25 degrees  \n**Version:** 4.7   \n\n[ESA CCI Soil Moisture ECV Project website](https://climate.esa.int/projects/soil-moisture/)  \n[Soil Moisture Data in the Open Data Portal](https://catalogue.ceda.ac.uk/uuid/c256fcfeef24460ca6eb14bf0fe09572)"
+    "description": "Soil Moisture (SM) states are affected by long-term climatological patterns as well as by short-term, local events. SM anomalies separate recurring seasonal/climatological variations from varying regional conditions and represent these deviations from the average conditions at each point in time. Droughts are therefore represented as negative anomalies while positive anomalies indicate “wetter than normal” conditions.\n\nIn CCI SM anomalies are calculated as differences between absolute SM observations and the SM climatology from a 20 year baseline period (1991-2010). This period is selected with regard to the available data density and quality, which can affect the derived deviations from the average conditions. The animation in the App shows monthly SM anomalies from 1991-2019 in the COMBINED product of ESA CCI SM v05.2.\n\n**Variable Shown:** Anomaly  \n**Time Span:** November 1978 – December 2019  \n**Temporal resolution:** monthly  \n**Geographic Extent:** global  \n**Spatial Resolution:**  0.25 degrees  \n**Version:** 5.2   \n\n[ESA CCI Soil Moisture ECV Project website](https://climate.esa.int/projects/soil-moisture/)  \n[Soil Moisture Data in the Open Data Portal](https://catalogue.ceda.ac.uk/uuid/c256fcfeef24460ca6eb14bf0fe09572)"
   },
   {
     "id": "sst.analysed_sst",
@@ -32,7 +32,7 @@
     "type": "Ocean Biogeochemistry",
     "name": "Ocean Colour – Chlorophyll-a Concentration",
     "shortName": "Ocean Colour",
-    "description": "Chlorophyll-a is the primary pigment in many plants, including the phytoplankton in the ocean.  It absorbs light, allowing plants to photosynthesise and so generate energy to grow. Ocean colour remote sensing uses this light absorption to quantify the amount of Chlorophyll-a in the surface ocean, the depth to which the light penetrates, and so quantify the amount of phytoplankton. Phytoplankton are essential because they are the foundation of the marine food chain, playing a role in carbon fixation that potentially reduces human-induced carbon dioxide in the atmosphere, but can also increase ocean acidification.\n\nThe Ocean Colour CCI project has created a consistent time-series by merging data from multiple ocean colour satellites, including ESA’s MERIS dataset and NASA’s SeaWiFS, MODIS-Aqua and VIIRS datasets. The project is also in the process of adding the Copernicus/ESA OLCI dataset. Further details can be found online at [climate.esa.int/projects/ocean-colour](https://climate.esa.int/en/projects/ocean-colour/)\n\n**Variable Shown:** Chlorophyll-a concentration in seawater in Milligram/m3  \n**Time Span:** September 1997 – December 2019  \n**Temporal Resolution:** monthly  \n**Geographic Extent:** global  \n**Spatial Resolution:** 4 km  \n**Version:** 4.2  \n**DOI:** \n\n[ESA CCI Ocean Color ECV Project website](https://climate.esa.int/projects/ocean-colour/)  \n[Data in the Open Data Portal](https://catalogue.ceda.ac.uk/uuid/5400de38636d43de9808bfc0b500e863)"
+    "description": "Chlorophyll-a is the primary pigment in many plants, including the phytoplankton in the ocean.  It absorbs light, allowing plants to photosynthesise and so generate energy to grow. Ocean colour remote sensing uses this light absorption to quantify the amount of Chlorophyll-a in the surface ocean, the depth to which the light penetrates, and so quantify the amount of phytoplankton. Phytoplankton are essential because they are the foundation of the marine food chain, playing a role in carbon fixation that potentially reduces human-induced carbon dioxide in the atmosphere, but can also increase ocean acidification.\n\nThe Ocean Colour CCI project has created a consistent time-series by merging data from multiple ocean colour satellites, including ESA’s MERIS dataset and NASA’s SeaWiFS, MODIS-Aqua and VIIRS datasets. The project is also in the process of adding the Copernicus/ESA OLCI dataset. Further details can be found online at [climate.esa.int/projects/ocean-colour](https://climate.esa.int/en/projects/ocean-colour/)\n\n**Variable Shown:** Chlorophyll-a concentration in seawater in Milligram/m3  \n**Time Span:** September 1997 – December 2020  \n**Temporal Resolution:** monthly  \n**Geographic Extent:** global  \n**Spatial Resolution:** 4 km  \n**Version:** 5.0  \n**DOI:** [10.5285/1dbe7a109c0244aaad713e078fd3059a](https://dx.doi.org/10.5285/1dbe7a109c0244aaad713e078fd3059a) \n\n[ESA CCI Ocean Color ECV Project website](https://climate.esa.int/projects/ocean-colour/)  \n[Data in the Open Data Portal](https://catalogue.ceda.ac.uk/uuid/e9f82908fd9c48138b31e5cfaa6d692b)"
   },
   {
     "id": "aerosol.AOD550_mean",