Skip to content

Commit

Permalink
Merge 213bbfe into 02f9480
Browse files Browse the repository at this point in the history
  • Loading branch information
@emielverstegen
emielverstegen committed May 27, 2019
2 parents 02f9480 + 213bbfe commit 5b67a89
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 27 deletions.
4 changes: 3 additions & 1 deletion CHANGES.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ Changelog of threedi-scenario-downloader
0.10 (unreleased)
-----------------

- Nothing changed yet.
- Increased download chunk size

- Added bounds_srs as optional argument to define the spatial reference system the bounds are supplied in


0.9 (2019-05-22)
Expand Down
114 changes: 90 additions & 24 deletions threedi_scenario_downloader/downloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,19 +137,20 @@ def get_raster(scenario_uuid, raster_code):
return result["raster"]


def create_raster_task(raster, target_srs, resolution, bounds=None, time=None):
def create_raster_task(
raster, target_srs, resolution, bounds=None, bounds_srs=None, time=None
):
"""create Lizard raster task"""

if bounds == None:
bounds = raster["spatial_bounds"]
bounds_srs = "EPSG:4326"

e = bounds["east"]
w = bounds["west"]
n = bounds["north"]
s = bounds["south"]

source_srs = "EPSG:4326"

bbox = "POLYGON(({} {},{} {},{} {},{} {},{} {}))".format(
w, n, e, n, e, s, w, s, w, n
)
Expand All @@ -160,7 +161,7 @@ def create_raster_task(raster, target_srs, resolution, bounds=None, time=None):
payload = {
"cellsize": resolution,
"geom": bbox,
"srs": source_srs,
"srs": bounds_srs,
"target_srs": target_srs,
"format": "geotiff",
"async": "true",
Expand All @@ -170,7 +171,7 @@ def create_raster_task(raster, target_srs, resolution, bounds=None, time=None):
payload = {
"cellsize": resolution,
"geom": bbox,
"srs": source_srs,
"srs": bounds_srs,
"target_srs": target_srs,
"time": time,
"format": "geotiff",
Expand Down Expand Up @@ -208,7 +209,7 @@ def download_file(url, path):
)
r.raise_for_status()
with open(path, "wb") as file:
for chunk in r.iter_content(100000):
for chunk in r.iter_content(1024 * 1024 * 10):
file.write(chunk)


Expand All @@ -226,7 +227,14 @@ def download_task(task_uuid, pathname=None):


def download_raster(
scenario, raster_code, target_srs, resolution, bounds=None, time=None, pathname=None
scenario,
raster_code,
target_srs,
resolution,
bounds=None,
bounds_srs=None,
time=None,
pathname=None,
):
"""
download raster
Expand All @@ -240,7 +248,14 @@ def download_raster(
else:
log.debug("Invalid scenario: supply a json object or uuid string")

task = create_raster_task(raster, target_srs, resolution, bounds, time)
task = create_raster_task(
raster,
target_srs,
resolution=resolution,
bounds=bounds,
bounds_srs=bounds_srs,
time=time,
)
task_uuid = task["task_id"]

log.debug("Start waiting for task {} to finish".format(task_uuid))
Expand All @@ -266,34 +281,58 @@ def download_raster(


def download_maximum_waterdepth_raster(
scenario_uuid, target_srs, resolution, bounds=None, pathname=None
scenario_uuid, target_srs, resolution, bounds=None, bounds_srs=None, pathname=None
):
"""download Maximum waterdepth raster"""
download_raster(
scenario_uuid, "depth-max-dtri", target_srs, resolution, bounds, None, pathname
scenario_uuid,
"depth-max-dtri",
target_srs,
resolution,
bounds=bounds,
bounds_srs=bounds_srs,
pathname=pathname,
)


def download_maximum_waterlevel_raster(
scenario_uuid, target_srs, resolution, bounds=None, pathname=None
scenario_uuid, target_srs, resolution, bounds=None, bounds_srs=None, pathname=None
):
"""download Maximum waterdepth raster"""
download_raster(
scenario_uuid, "s1-max-dtri", target_srs, resolution, bounds, None, pathname
scenario_uuid,
"s1-max-dtri",
target_srs,
resolution,
bounds=bounds,
bounds_srs=bounds_srs,
pathname=pathname,
)


def download_total_damage_raster(
scenario_uuid, target_srs, resolution, bounds=None, pathname=None
scenario_uuid, target_srs, resolution, bounds=None, bounds_srs=None, pathname=None
):
"""download Total Damage raster"""
download_raster(
scenario_uuid, "total-damage", target_srs, resolution, bounds, None, pathname
scenario_uuid,
"total-damage",
target_srs,
resolution,
bounds=bounds,
bounds_srs=bounds_srs,
pathname=pathname,
)


def download_waterdepth_raster(
scenario_uuid, target_srs, resolution, time, bounds=None, pathname=None
scenario_uuid,
target_srs,
resolution,
time,
bounds=None,
bounds_srs=None,
pathname=None,
):
"""download snapshot of Waterdepth raster"""
download_raster(
Expand All @@ -302,13 +341,20 @@ def download_waterdepth_raster(
target_srs,
resolution,
bounds=bounds,
bounds_srs=bounds_srs,
time=time,
pathname=pathname,
)


def download_waterlevel_raster(
scenario_uuid, target_srs, resolution, time, bounds=None, pathname=None
scenario_uuid,
target_srs,
resolution,
time,
bounds=None,
bounds_srs=None,
pathname=None,
):
"""download snapshot of Waterdepth raster"""
download_raster(
Expand All @@ -317,13 +363,20 @@ def download_waterlevel_raster(
target_srs,
resolution,
bounds=bounds,
bounds_srs=bounds_srs,
time=time,
pathname=pathname,
)


def download_precipitation_raster(
scenario_uuid, target_srs, resolution, time, bounds=None, pathname=None
scenario_uuid,
target_srs,
resolution,
time,
bounds=None,
bounds_srs=None,
pathname=None,
):
"""download snapshot of Waterdepth raster"""
download_raster(
Expand All @@ -332,6 +385,7 @@ def download_precipitation_raster(
target_srs,
resolution,
bounds=bounds,
bounds_srs=bounds_srs,
time=time,
pathname=pathname,
)
Expand Down Expand Up @@ -396,9 +450,11 @@ def rasters_in_scenario(scenario_json):
return static_rasters, temporal_rasters


def get_raster_link(raster, target_srs, resolution, bounds=None, time=None):
def get_raster_link(
raster, target_srs, resolution, bounds=None, bounds_srs=None, time=None
):
"""get url to download raster"""
task = create_raster_task(raster, target_srs, resolution, bounds, time)
task = create_raster_task(raster, target_srs, resolution, bounds, bounds_srs, time)
task_uuid = task["task_id"]

log.debug("Start waiting for task {} to finish".format(task_uuid))
Expand All @@ -418,22 +474,27 @@ def get_raster_link(raster, target_srs, resolution, bounds=None, time=None):


def get_static_rasters_links(
static_rasters, target_srs, resolution, bounds=None, time=None
static_rasters, target_srs, resolution, bounds=None, bounds_srs=None, time=None
):
"""return a dict of urls to geotiff files of static rasters in scenario
the dict items are formatted as result_name: link.tif"""
static_raster_urls = {}
for static_raster in static_rasters:
name = static_raster["name_3di"]
static_raster_url = get_raster_link(
static_raster, target_srs, resolution, bounds, time
static_raster, target_srs, resolution, bounds, bounds_srs, time
)
static_raster_urls[name] = static_raster_url
return static_raster_urls


def get_temporal_raster_links(
temporal_raster, target_srs, resolution, bounds=None, interval_hours=None
temporal_raster,
target_srs,
resolution,
bounds=None,
bounds_srs=None,
interval_hours=None,
):
"""return a dict of urls to geotiff files of a temporal raster
the dict items are formatted as name_3di_datetime: link.tif"""
Expand All @@ -460,13 +521,18 @@ def get_temporal_raster_links(


def get_temporal_rasters_links(
temporal_rasters, target_srs, resolution, bounds=None, interval_hours=None
temporal_rasters,
target_srs,
resolution,
bounds=None,
bounds_srs=None,
interval_hours=None,
):
"""get links to all temporal rasters"""
temporal_rasters_urls = {}
for temporal_raster in temporal_rasters:
temporal_raster_urls = get_temporal_raster_links(
temporal_raster, target_srs, resolution, bounds, interval_hours
temporal_raster, target_srs, resolution, bounds, bounds_srs, interval_hours
)
for name_timestep, download_url in temporal_raster_urls.items():
temporal_rasters_urls.setdefault(name_timestep, download_url)
Expand Down
21 changes: 19 additions & 2 deletions threedi_scenario_downloader/tests/test_downloader_manual.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,29 @@ def test_download_waterdepth_raster():
"EPSG:28992",
1000,
"2018-06-02T06:00:00Z",
None,
"threedi_scenario_downloader/tests/testdata/waterdepth.tif",
bounds=None,
bounds_srs=None,
pathname="threedi_scenario_downloader/tests/testdata/waterdepth.tif",
)
assert os.path.isfile("threedi_scenario_downloader/tests/testdata/waterdepth.tif")


def test_download_waterdepth_raster_reprojected_bounds():
bounds = {"east": 115000, "west": 114000, "north": 561000, "south": 560000}
downloader.download_waterdepth_raster(
"06c38953-31ec-4f6d-ae1f-ccdf31a348ae",
"EPSG:28992",
1000,
"2018-06-02T06:00:00Z",
bounds=bounds,
bounds_srs="EPSG:28992",
pathname="threedi_scenario_downloader/tests/testdata/waterdepth_reprojected.tif",
)
assert os.path.isfile(
"threedi_scenario_downloader/tests/testdata/waterdepth_reprojected.tif"
)


def test_get_netcdf_link():
url = downloader.get_netcdf_link("06c38953-31ec-4f6d-ae1f-ccdf31a348ae")
assert url == "https://demo.lizard.net/api/v3/scenario-results/52331/results_3di.nc"
Expand Down

0 comments on commit 5b67a89

Please sign in to comment.