-
Notifications
You must be signed in to change notification settings - Fork 165
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Linear blending #229
Linear blending #229
Changes from 4 commits
80c4d4b
f01f4f2
7d6008b
d6eb63d
086c41d
d9d0777
6d4a375
758f88b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,162 @@ | ||
# -*- coding: utf-8 -*- | ||
|
||
""" | ||
Plot linear blending | ||
==================== | ||
""" | ||
|
||
from matplotlib import cm, pyplot as plt | ||
import numpy as np | ||
from pprint import pprint | ||
from pysteps.motion.lucaskanade import dense_lucaskanade | ||
from pysteps import io, rcparams, nowcasts | ||
from pysteps.utils import conversion, transformation | ||
from pysteps.visualization import plot_precip_field | ||
from datetime import datetime | ||
from pysteps.utils import dimension | ||
from pysteps.nowcasts.linear_blending import forecast | ||
|
||
|
||
def gaussian(x, max, mean, sigma): | ||
return max * np.exp(-(x - mean) * (x - mean) / sigma / sigma / 2) | ||
|
||
|
||
def dummy_nwp(R, n_leadtimes, max=20, mean=0, sigma=0.25, speed=100): | ||
"""Generates dummy NWP data with the same dimension as the input | ||
precipitation field R. The NWP data is a vertical line with a Gaussian | ||
profile moving to the left""" | ||
|
||
# R is original radar image | ||
rows = R.shape[0] | ||
cols = R.shape[1] | ||
|
||
# Initialise the dummy NWP data | ||
R_nwp = np.zeros((n_leadtimes, rows, cols)) | ||
x = np.linspace(-5, 5, cols) | ||
|
||
for n in range(n_leadtimes): | ||
for i in range(rows): | ||
R_nwp[n, i, :] = gaussian(x, max, mean, sigma) | ||
mean -= speed / rows | ||
|
||
return R_nwp | ||
|
||
|
||
############################################################################### | ||
# Set nowcast parameters | ||
n_ens_members = 10 | ||
n_leadtimes = 18 | ||
start_blending = 30 # in minutes | ||
end_blending = 60 # in minutes | ||
seed = 24 | ||
|
||
# Read precipitation field | ||
# ------------------------ | ||
# | ||
# First thing, the sequence of Swiss radar composites is imported, converted and | ||
# transformed into units of dBR. | ||
|
||
date = datetime.strptime("201701311200", "%Y%m%d%H%M") | ||
data_source = "mch" | ||
|
||
# Load data source config | ||
root_path = rcparams.data_sources[data_source]["root_path"] | ||
path_fmt = rcparams.data_sources[data_source]["path_fmt"] | ||
fn_pattern = rcparams.data_sources[data_source]["fn_pattern"] | ||
fn_ext = rcparams.data_sources[data_source]["fn_ext"] | ||
importer_name = rcparams.data_sources[data_source]["importer"] | ||
importer_kwargs = rcparams.data_sources[data_source]["importer_kwargs"] | ||
timestep = rcparams.data_sources[data_source]["timestep"] | ||
|
||
# Find the radar files in the archive | ||
fns = io.find_by_date( | ||
date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=2 | ||
) | ||
|
||
# Read the data from the archive | ||
importer = io.get_method(importer_name, "importer") | ||
R, _, metadata = io.read_timeseries(fns, importer, legacy=True, **importer_kwargs) | ||
|
||
# Convert to rain rate | ||
R, metadata = conversion.to_rainrate(R, metadata) | ||
|
||
# Upscale data to 2 km to limit memory usage | ||
R, metadata = dimension.aggregate_fields_space(R, metadata, 2000) | ||
|
||
# Import the dummy NWP data (vertical line moving to the left) | ||
R_nwp = dummy_nwp(R[-1, :, :], n_leadtimes + 1, max=7, mean=4, speed=0.2 * 350) | ||
|
||
# Log-transform the data to unit of dBR, set the threshold to 0.1 mm/h, | ||
# set the fill value to -15 dBR | ||
R, metadata = transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0) | ||
|
||
# Set missing values with the fill value | ||
# R[~np.isfinite(R)] = -15.0 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If not used, we could remove these lines. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. True, in the plot_steps_nowcast.py example, this is how the missing data was handled, but here we use the (dummy) NWP data |
||
|
||
# Nicely print the metadata | ||
pprint(metadata) | ||
|
||
############################################################################### | ||
# Linear blending of nowcast and NWP data | ||
|
||
# Estimate the motion field | ||
V = dense_lucaskanade(R) | ||
|
||
# Define nowcast keyword arguments | ||
nowcast_kwargs = { | ||
"n_ens_members": n_ens_members, | ||
"n_cascade_levels": 6, | ||
"R_thr": -10.0, | ||
"kmperpixel": 2.0, | ||
"timestep": timestep, | ||
"noise_method": "nonparametric", | ||
"vel_pert_method": "bps", | ||
"mask_method": "incremental", | ||
} | ||
|
||
# Calculate the blended precipitation field | ||
R_blended = forecast( | ||
R[-3:, :, :], | ||
V, | ||
n_leadtimes, | ||
timestep, | ||
"steps", | ||
R_nwp=R_nwp[1:, :, :], | ||
start_blending=start_blending, | ||
end_blending=end_blending, | ||
nowcast_kwargs=nowcast_kwargs, | ||
) | ||
|
||
""" | ||
extrap_kwargs = {"allow_nonfinite_values": True} | ||
nowcast_kwargs = {"extrap_kwargs": extrap_kwargs} | ||
|
||
# Calculate the blended precipitation field | ||
R_blended = forecast( | ||
R[-1, :, :], | ||
V, | ||
n_leadtimes, | ||
timestep, | ||
"extrapolation", | ||
R_nwp=R_nwp[1: :, :], | ||
start_blending=start_blending, | ||
end_blending=end_blending, | ||
nowcast_kwargs=nowcast_kwargs, | ||
) | ||
""" | ||
|
||
# Calculate the ensemble average | ||
if len(R_blended.shape) == 4: | ||
R_blended_mean = np.mean(R_blended[:, :, :, :], axis=0) | ||
else: | ||
R_blended_mean = np.copy(R_blended) | ||
|
||
# Plot the blended field | ||
for i in range(n_leadtimes): | ||
plot_precip_field( | ||
R_blended_mean[i, :, :], | ||
geodata=metadata, | ||
title="Blended field (+ %i min)" % ((i + 1) * timestep), | ||
) | ||
plt.show() | ||
plt.close() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,172 @@ | ||
# -*- coding: utf-8 -*- | ||
""" | ||
pysteps.nowcasts.linear_blending | ||
================================ | ||
|
||
Implementation of the linear blending between nowcast and NWP data. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @RubenImhoff @wdewettin thanks for implementing this module. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Great job @wdewettin and @RubenImhoff |
||
|
||
.. autosummary:: | ||
:toctree: ../generated/ | ||
|
||
forecast | ||
""" | ||
|
||
import numpy as np | ||
from pysteps import nowcasts | ||
from pysteps.utils import transformation | ||
|
||
|
||
def forecast( | ||
precip, | ||
velocity, | ||
timesteps, | ||
timestep, | ||
nowcast_method, | ||
R_nwp=None, | ||
start_blending=120, | ||
end_blending=240, | ||
nowcast_kwargs=dict(), | ||
): | ||
|
||
"""Generate a forecast by linearly blending nowcasts with NWP data | ||
|
||
Parameters | ||
---------- | ||
precip: array_like | ||
Array containing the input precipitation field(s) ordered by timestamp | ||
from oldest to newest. The time steps between the inputs are assumed | ||
to be regular. | ||
velocity; array_like | ||
Array of shape (2, m, n) containing the x- and y-components of the advection | ||
field. The velocities are assumed to represent one time step between the | ||
inputs. All values are required to be finite. | ||
timesteps: int | ||
Number of time steps to forecast. | ||
timestep: int or float | ||
The time difference (in minutes) between consecutive forecast fields. | ||
nowcast_method: {'anvil', 'eulerian', 'lagrangian', 'extrapolation', 'lagrangian_probability', 'sprog', 'steps', 'sseps'} | ||
Name of the nowcast method used to forecast the precipitation. | ||
R_nwp: array_like or NoneType, optional | ||
Array of shape (timesteps, m, n) in the case of no ensemble or | ||
of shape (n_ens_members, timesteps, m, n) in the case of an ensemble | ||
containing the NWP precipitation fields ordered by timestamp from oldest | ||
to newest. The time steps between the inputs are assumed to be regular | ||
(and identical to the time step between the nowcasts). If no NWP | ||
data is given the value of R_nwp is None and no blending will be performed. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In case of R_nwp = None, the script is similar to steps.py, right? Do we want to use the script as a postprocessing tool, i.e. the user makes a nowcast with steps.py and passes the output on to linear_blending.py, or do we want to keep it this way and have some duplicate code with steps.py? As steps.py may also change to allow for other blending methods, I wonder what everyone thinks about the best way forward. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Here the object returned should be equal to the output of the nowcasting method used if R_nwp=None. |
||
start_blending: int, optional | ||
Time stamp (in minutes) after which the blending should start. Before this | ||
only the nowcast data is used. | ||
end_blending: int, optional | ||
Time stamp (in minutes) after which the blending should end. Between | ||
start_blending and end_blending the nowcasts and NWP data are linearly | ||
merged with each other. After end_blending only the NWP data is used. | ||
nowcast_kwargs: dict, optional | ||
Dictionary containing keyword arguments for the nowcast method. | ||
|
||
Returns | ||
------- | ||
R_blended: ndarray | ||
Array of shape (timesteps, m, n) in the case of no ensemble or | ||
of shape (n_ens_members, timesteps, m, n) in the case of an ensemble | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. With the way it is implemented at this moment: n_ens_members equals the maximum number of ensemble members in the input, so either the number of the nowcast or that of the NWP. Perhaps good to add that here. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. (B.t.w., I can add that too, but I'll wait for the responses - and thus possible changes - of the others) |
||
containing the precipation forecast generated by linearly blending | ||
the nowcasts and the NWP data. | ||
""" | ||
|
||
# Calculate the nowcasts | ||
nowcast_method_func = nowcasts.get_method(nowcast_method) | ||
R_nowcast = nowcast_method_func( | ||
precip, | ||
velocity, | ||
timesteps, | ||
**nowcast_kwargs, | ||
) | ||
|
||
# Transform the precipitation back to mm/h | ||
R_nowcast = transformation.dB_transform(R_nowcast, threshold=-10.0, inverse=True)[0] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I believe the threshold is present in the metadata (the transformer at least returns the metadata). We could add this to the function, then we don't have to hard code it here. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I understand why this is done here, but this is assuming that There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I understand why this is done here, but this is assuming that Said that the returned output |
||
|
||
# Check if NWP data is given as input | ||
if R_nwp is not None: | ||
|
||
if len(R_nowcast.shape) == 4: | ||
n_ens_members_nowcast = R_nowcast.shape[0] | ||
if n_ens_members_nowcast == 1: | ||
R_nowcast = np.squeeze(R_nowcast) | ||
else: | ||
n_ens_members_nowcast = 1 | ||
|
||
if len(R_nwp.shape) == 4: | ||
n_ens_members_nwp = R_nwp.shape[0] | ||
if n_ens_members_nwp == 1: | ||
R_nwp = np.squeeze(R_nwp) | ||
else: | ||
n_ens_members_nwp = 1 | ||
|
||
n_ens_members_max = max(n_ens_members_nowcast, n_ens_members_nwp) | ||
n_ens_members_min = min(n_ens_members_nowcast, n_ens_members_nwp) | ||
|
||
if n_ens_members_min != n_ens_members_max: | ||
if n_ens_members_nwp == 1: | ||
R_nwp = np.repeat(R_nwp[np.newaxis, :, :], n_ens_members_max, axis=0) | ||
elif n_ens_members_nowcast == 1: | ||
R_nowcast = np.repeat( | ||
R_nowcast[np.newaxis, :, :], n_ens_members_max, axis=0 | ||
) | ||
else: | ||
repeats = [ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. After consulting @wdewettin and @ladc, I figured out what this piece of code does. So if you e.g. have 10 ensemble nowcasts members and 3 NWP members, the output will be an ensemble of 10 members. Hence, the three NWP members are blended with the first three members of the nowcast (member one with member one, two with two, etc.), subsequently, the same NWP members are blended with the next three members (NWP member one with member 4, NWP member 2 with member 5, etc.), until 10 is reached. Is this the way to go (works for me) or do we want an output of 30 (10 x 3 members here) in this case? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Oh and @wdewettin, let's add some comments to the code here so that this procedure is clear. :) (B.t.w., I can add that too, but I'll wait for the responses - and thus possible changes - of the others) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes! I will await the responses from the others, and then add some comments There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I agree. It wuld be nice to add additional comments in this part of the script as @RubenImhoff suggested. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I agree. It would be nice to add additional comments in this part of the script as @RubenImhoff suggested. |
||
(n_ens_members_max + i) // n_ens_members_min | ||
for i in range(n_ens_members_min) | ||
] | ||
|
||
if n_ens_members_nwp == n_ens_members_min: | ||
R_nwp = np.repeat(R_nwp, repeats, axis=0) | ||
elif n_ens_members_nowcast == n_ens_members_min: | ||
R_nowcast = np.repeat(R_nowcast, repeats, axis=0) | ||
|
||
# Check if dimensions are correct | ||
assert ( | ||
R_nwp.shape == R_nowcast.shape | ||
), "The dimensions of R_nowcast and R_nwp need to be identical: dimension of R_nwp = {} and dimension of R_nowcast = {}".format( | ||
R_nwp.shape, R_nowcast.shape | ||
) | ||
|
||
# Initialise output | ||
R_blended = np.zeros_like(R_nowcast) | ||
|
||
# Calculate the weights | ||
for i in range(timesteps): | ||
# Calculate what time we are at | ||
t = (i + 1) * timestep | ||
|
||
# Calculate the weight with a linear relation (weight_nwp at start_blending = 0.0) | ||
# and (weight_nwp at end_blending = 1.0) | ||
weight_nwp = (t - start_blending) / (end_blending - start_blending) | ||
|
||
# Set weights at times before start_blending and after end_blending | ||
if weight_nwp < 0.0: | ||
weight_nwp = 0.0 | ||
elif weight_nwp > 1.0: | ||
weight_nwp = 1.0 | ||
|
||
# Calculate weight_nowcast | ||
weight_nowcast = 1.0 - weight_nwp | ||
|
||
# Calculate output by combining R_nwp and R_nowcast, | ||
# while distinguishing between ensemble and non-ensemble methods | ||
if n_ens_members_max == 1: | ||
R_blended[i, :, :] = ( | ||
weight_nwp * R_nwp[i, :, :] + weight_nowcast * R_nowcast[i, :, :] | ||
) | ||
else: | ||
R_blended[:, i, :, :] = ( | ||
weight_nwp * R_nwp[:, i, :, :] | ||
+ weight_nowcast * R_nowcast[:, i, :, :] | ||
) | ||
|
||
# Find where the NaN values are and replace them with NWP data | ||
nan_indices = np.isnan(R_blended) | ||
R_blended[nan_indices] = R_nwp[nan_indices] | ||
ladc marked this conversation as resolved.
Show resolved
Hide resolved
|
||
else: | ||
# If no NWP data is given, the blended field is simply equal to the nowcast field | ||
R_blended = R_nowcast | ||
|
||
return R_blended |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Perhaps plot the radar rainfall and NWP rainfall forecast field somewhere, to give the reader (as this example may end up in the pysteps gallery) and idea of what the example fields look like.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do you mean plotting them for every timestep? Or only the first one(s)?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
First ones should be fine. :)