feat(pipeline): add land surface temperature

data/downloads/odp-ftp-land_surface_temperature.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/land_surface_temperature/data/SSMI_SSMIS/L3C/v2.23//monthly/"
+BASE_URL="ftp://anon-ftp.ceda.ac.uk/neodc/esacci/land_surface_temperature/data/SSMI_SSMIS/L3C/v2.33//monthly/"
 START_DATE=1996-01-01
 OUTPUT_FOLDER=./download/lst
 
@@ -11,7 +11,7 @@ do
   NEXT_DATE=$(date +%Y%m -d "$START_DATE + $i month")
   NEXT_YEAR=$(date +%Y -d "$START_DATE + $i month")
   FILENAME=$OUTPUT_FOLDER/$(date +%Y%m%d -d "$START_DATE + $i month").nc
-  FTP_URL=$BASE_URL$NEXT_YEAR/"ESACCI-LST-L3C-LST-SSMI13-0.25deg_1MONTHLY_ASC-"$NEXT_DATE"000000-fv2.23.nc"
+  FTP_URL=$BASE_URL$NEXT_YEAR/"ESACCI-LST-L3C-LST-SSMI13-0.25deg_1MONTHLY_ASC-"$NEXT_DATE"000000-fv2.33.nc"
   echo $FTP_URL
 
   curl --silent $FTP_URL > $FILENAME
@@ -23,7 +23,7 @@ do
   NEXT_DATE=$(date +%Y%m -d "$START_DATE + $i month")
   NEXT_YEAR=$(date +%Y -d "$START_DATE + $i month")
   FILENAME=$OUTPUT_FOLDER/$(date +%Y%m%d -d "$START_DATE + $i month").nc
-  FTP_URL=$BASE_URL$NEXT_YEAR/"ESACCI-LST-L3C-LST-SSMI17-0.25deg_1MONTHLY_ASC-"$NEXT_DATE"000000-fv2.23.nc"
+  FTP_URL=$BASE_URL$NEXT_YEAR/"ESACCI-LST-L3C-LST-SSMI17-0.25deg_1MONTHLY_ASC-"$NEXT_DATE"000000-fv2.33.nc"
   echo $FTP_URL
 
   curl --silent $FTP_URL > $FILENAME

pipeline/dags/land_surface_temperature.lst.py

@@ -0,0 +1,66 @@
+from datetime import datetime
+import task_factories
+from airflow import DAG
+from airflow.models.param import Param
+
+# layer
+LAYER_ID = 'land_surface_temperature'
+LAYER_VARIABLE = 'lst'
+LAYER_VERSION = '1.14.1'
+RESOLUTION = '1440 720'
+METADATA = {
+    "id": f'{LAYER_ID}.{LAYER_VARIABLE}',
+    "version": LAYER_VERSION,
+    "timestamps": [],  # will be injected
+    "min_value": 243.0,
+    "max_value": 323.0,
+    "type": "image",  # 'tiles' or 'image'
+    "zoom_levels": '0-3',
+    "units": 'kelvin',
+    "basemap": None,
+    "legend_values": [ "50 °", "-30" ],
+    "time_format": {
+        "year": "numeric",
+        "month": "long"
+    }
+}
+
+# dev
+BUCKET_ORIGIN = 'esa-cfs-cate-data'
+BUCKET_TMP = 'esa-cfs-pipeline-tmp'
+WORKDIR = '/workdir/files'
+COLOR_FILE = f'/opt/airflow/plugins/colors/{LAYER_ID}.{LAYER_VARIABLE}.txt'
+DEBUG = False
+
+dag_params = {
+    "max_files": Param(2, type=["null", "integer"], minimum=0,),
+    "output_bucket": Param("esa-cfs-pipeline-output", type=["string"])
+}
+
+with DAG(dag_id=METADATA["id"], start_date=datetime(2022, 1, 1), schedule=None, catchup=False, params=dag_params) as dag:
+
+    # create tasks
+    clean_workdir = task_factories.clean_dir(
+        task_id='clean_workdir', dir=WORKDIR)
+    list_files = task_factories.gcs_list_files(
+        bucket_name=BUCKET_ORIGIN, layer_id=LAYER_ID, layer_variable=LAYER_VARIABLE)
+    download = task_factories.gcs_download_file(
+        bucket_name=BUCKET_ORIGIN, dir=WORKDIR, appendix='_downloaded', dry_run=False)
+    legend_image = task_factories.legend_image(
+        workdir=WORKDIR, color_file=COLOR_FILE)
+    metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
+    gdal_transforms = task_factories.gdal_transforms(
+        layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
+    upload = task_factories.upload(
+        WORKDIR, LAYER_ID, LAYER_VARIABLE, LAYER_VERSION, METADATA['type'])
+
+    # connect tasks
+    files = list_files()
+    clean_workdir >> files
+    downloads = download.expand(filename=files)
+    gdal_transforms(downloads) >> upload()
+    clean_workdir >> legend_image
+    metadata(files)
+
+    if DEBUG:
+        downloads >> task_factories.gdal_info()

pipeline/plugins/colors/land_surface_temperature.lst.txt

@@ -0,0 +1,10 @@
+5000 120 10 10
+4000 240 40 40
+2000 240 120 30
+1000 250 210 35
+0 148 200 148
+-1000 100 200 200
+-2000 65 168 240
+-4000 40 40 140
+-5000 10 10 60
+nv 0 0 0 0