Skip to content

Commit

Permalink
feat(pipeline): remove default value for extend
Browse files Browse the repository at this point in the history
  • Loading branch information
Andreas Helms authored and Andreas Helms committed Apr 4, 2024
1 parent 29fa434 commit 221099f
Show file tree
Hide file tree
Showing 12 changed files with 26 additions and 13 deletions.
3 changes: 2 additions & 1 deletion pipeline/dags/biomass.agb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'biomass'
LAYER_VARIABLE = 'agb'
RESOLUTION = '81000 31500'
EXTEND = '-180 -60 180 80'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -52,7 +53,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te='-180 -60 180 80', max_tis_warp=1, max_tis_dem=1, max_tis_translate=1)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND, max_tis_warp=1, max_tis_dem=1, max_tis_translate=1)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/cloud.cfc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'cloud'
LAYER_VARIABLE = 'cfc'
RESOLUTION = '1024 512'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -47,7 +48,7 @@
download = task_factories.gcs_download_file(bucket_name=BUCKET_ORIGIN, dir=WORKDIR, appendix='_downloaded')
legend_image = task_factories.legend_image(workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
gdal_transforms = task_factories.gdal_transforms(layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

# connect tasks
Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/fire.burned_area.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'fire'
LAYER_VARIABLE = 'burned_area'
RESOLUTION = '1440 720'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -57,7 +58,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/greenhouse_xch4.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'greenhouse'
LAYER_VARIABLE = 'xch4'
RESOLUTION = '180 90'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -61,7 +62,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/greenhouse_xco2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'greenhouse'
LAYER_VARIABLE = 'xco2'
RESOLUTION = '180 90'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -61,7 +62,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/land_cover.lccs_class.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'land_cover'
LAYER_VARIABLE = 'lccs_class'
RESOLUTION = '64800 32400'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -49,7 +50,7 @@
download = task_factories.gcs_download_file(bucket_name=BUCKET_ORIGIN, dir=WORKDIR, appendix='_downloaded')
legend_image = task_factories.legend_image(workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, max_tis_warp=1, max_tis_dem=1, max_tis_translate=1)
gdal_transforms = task_factories.gdal_transforms(layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND, max_tis_warp=1, max_tis_dem=1, max_tis_translate=1)
upload = task_factories.upload(WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

# connect tasks
Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/land_surface_temperature.lst.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'land_surface_temperature'
LAYER_VARIABLE = 'lst'
RESOLUTION = '1440 720'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -52,7 +53,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/oc.chlor_a.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'oc'
LAYER_VARIABLE = 'chlor_a'
RESOLUTION = '2048 1024'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -52,7 +53,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/sea_state_swh_mean.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'sea_state'
LAYER_VARIABLE = 'swh_mean'
RESOLUTION = '360 180'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -52,7 +53,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/soil_moisture.Anomaly.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
LAYER_VARIABLE = 'Anomaly'
LAYER_VERSION = '1.14.1'
RESOLUTION = '1440 720'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"version": LAYER_VERSION,
Expand Down Expand Up @@ -54,7 +55,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
3 changes: 2 additions & 1 deletion pipeline/dags/soil_moisture.sm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LAYER_ID = 'soil_moisture'
LAYER_VARIABLE = 'sm'
RESOLUTION = '1440 720'
EXTEND = '-180 -90 180 90'
METADATA = {
"id": f'{LAYER_ID}.{LAYER_VARIABLE}',
"timestamps": [], # will be injected
Expand Down Expand Up @@ -52,7 +53,7 @@
workdir=WORKDIR, color_file=COLOR_FILE)
metadata = task_factories.metadata(workdir=WORKDIR, metadata=METADATA)
gdal_transforms = task_factories.gdal_transforms(
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION)
layer_variable=LAYER_VARIABLE, color_file=COLOR_FILE, layer_type=METADATA['type'], zoom_levels=METADATA['zoom_levels'], gdal_ts=RESOLUTION, gdal_te=EXTEND)
upload = task_factories.upload(
WORKDIR, LAYER_ID, LAYER_VARIABLE, METADATA['type'])

Expand Down
6 changes: 4 additions & 2 deletions pipeline/dags/task_factories.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ def fn(files, **context):
return fn


def gdal_transforms(color_file: str, layer_type: str, zoom_levels: str, gdal_te: str = '-180 -90 180 90', gdal_ts: str = '1024 512', layer_variable: str = '', warp_cmd: str = None, max_tis_warp: int = 4, max_tis_dem: int = 4, max_tis_translate: int = 4):
def gdal_transforms(color_file: str, layer_type: str, zoom_levels: str, gdal_te: str = None, gdal_ts: str = None, layer_variable: str = '', warp_cmd: str = None, max_tis_warp: int = 4, max_tis_dem: int = 4, max_tis_translate: int = 4):
def get_transform_task():
if layer_type == 'image':
return BashOperator.partial(
Expand All @@ -180,7 +180,9 @@ def get_transform_outpath(filename):
@task_group(group_id='gdal_transforms_group')
def fn(downloads):
file_path_in = 'NETCDF:"$FILEPATH_IN":$DATA_VARIABLE' if layer_variable else '$FILEPATH_IN'
warp_command = f'gdalwarp -t_srs EPSG:4326 -te {gdal_te} -ts {gdal_ts} -r near --config GDAL_CACHEMAX 90% -co compress=LZW {file_path_in} $FILEPATH_OUT' if not warp_cmd else warp_cmd
gdal_te_flag = '-te ' + gdal_te if gdal_te else ''
gdal_ts_flag = '-ts ' + gdal_ts if gdal_ts else ''
warp_command = f'gdalwarp -t_srs EPSG:4326 {gdal_te_flag} {gdal_ts_flag} -r near --config GDAL_CACHEMAX 90% -co compress=LZW {file_path_in} $FILEPATH_OUT' if not warp_cmd else warp_cmd
gdal_warp = BashOperator.partial(
task_id='reproject_and_to_tiff',
bash_command=f'rm -f $FILEPATH_OUT && {warp_command} && echo $FILEPATH_OUT',
Expand Down

0 comments on commit 221099f

Please sign in to comment.