From e4d61c74044060ff101a581d2dd9d4345cb1a672 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 1 Nov 2021 14:04:13 +0000 Subject: [PATCH 001/197] Start on Optical Flow data source --- .../data_sources/optical_flow/__init__.py | 1 + .../optical_flow/optical_flow_data_source.py | 288 ++++++++++++++++++ .../optical_flow/optical_flow_model.py | 21 ++ .../satellite/satellite_data_source.py | 1 + 4 files changed, 311 insertions(+) create mode 100644 nowcasting_dataset/data_sources/optical_flow/__init__.py create mode 100644 nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py create mode 100644 nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py diff --git a/nowcasting_dataset/data_sources/optical_flow/__init__.py b/nowcasting_dataset/data_sources/optical_flow/__init__.py new file mode 100644 index 00000000..9a3ee67d --- /dev/null +++ b/nowcasting_dataset/data_sources/optical_flow/__init__.py @@ -0,0 +1 @@ +""" Optical Flow data sources and functions """ diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py new file mode 100644 index 00000000..c2e8a159 --- /dev/null +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -0,0 +1,288 @@ +""" Optical Flow Data Source """ +import logging +from concurrent import futures +from dataclasses import InitVar, dataclass +from numbers import Number +from typing import Iterable, Optional + +import cv2 +import numpy as np +import pandas as pd +import xarray as xr + +import nowcasting_dataset.time as nd_time +from nowcasting_dataset.data_sources.data_source import ZarrDataSource +from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput +from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow +from nowcasting_dataset.dataset.xr_utils import join_list_data_array_to_batch_dataset + +_LOG = logging.getLogger("nowcasting_dataset") + + +@dataclass +class OpticalFlowDataSource(ZarrDataSource): + """ + Optical Flow Data Source, computing flow between Satellite data + + zarr_path: Must start with 'gs://' if on GCP. + """ + + zarr_path: str = None + image_size_pixels: InitVar[int] = 128 + meters_per_pixel: InitVar[int] = 2_000 + + def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): + """ Post Init """ + super().__post_init__(image_size_pixels, meters_per_pixel) + self._cache = {} + self._shape_of_example = ( + self._total_seq_length, + image_size_pixels, + image_size_pixels, + 2, + ) + + def open(self) -> None: + """ + Open Satellite data + + We don't want to open_sat_data in __init__. + If we did that, then we couldn't copy SatelliteDataSource + instances into separate processes. Instead, + call open() _after_ creating separate processes. + """ + self._data = self._open_data() + self._data = self._data.sel(variable=list(self.channels)) + + def _open_data(self) -> xr.DataArray: + return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) + + def get_batch( + self, + t0_datetimes: pd.DatetimeIndex, + x_locations: Iterable[Number], + y_locations: Iterable[Number], + ) -> OpticalFlow: + """ + Get batch data + + Load the first _n_timesteps_per_batch concurrently. This + loads the timesteps from disk concurrently, and fills the + cache. If we try loading all examples + concurrently, then SatelliteDataSource will try reading from + empty caches, and things are much slower! + + Args: + t0_datetimes: list of timestamps for the datetime of the batches. The batch will also + include data for historic and future depending on `history_minutes` and + `future_minutes`. + x_locations: x center batch locations + y_locations: y center batch locations + + Returns: Batch data + + """ + # Load the first _n_timesteps_per_batch concurrently. This + # loads the timesteps from disk concurrently, and fills the + # cache. If we try loading all examples + # concurrently, then SatelliteDataSource will try reading from + # empty caches, and things are much slower! + zipped = list(zip(t0_datetimes, x_locations, y_locations)) + batch_size = len(t0_datetimes) + + with futures.ThreadPoolExecutor(max_workers=batch_size) as executor: + future_examples = [] + for coords in zipped[: self.n_timesteps_per_batch]: + t0_datetime, x_location, y_location = coords + future_example = executor.submit( + self.get_example, t0_datetime, x_location, y_location + ) + future_examples.append(future_example) + examples = [future_example.result() for future_example in future_examples] + + # Load the remaining examples. This should hit the DataSource caches. + for coords in zipped[self.n_timesteps_per_batch :]: + t0_datetime, x_location, y_location = coords + example = self.get_example(t0_datetime, x_location, y_location) + examples.append(example) + + output = join_list_data_array_to_batch_dataset(examples) + + self._cache = {} + + return OpticalFlow(output) + + def get_example( + self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number + ) -> DataSourceOutput: + """ + Get Optical Flow Example data + + Args: + t0_dt: list of timestamps for the datetime of the batches. The batch will also include + data for historic and future depending on `history_minutes` and `future_minutes`. + x_meters_center: x center batch locations + y_meters_center: y center batch locations + + Returns: Example Data + + """ + selected_data = self._get_time_slice(t0_dt) + bounding_box = self._square.bounding_box_centered_on( + x_meters_center=x_meters_center, y_meters_center=y_meters_center + ) + selected_data = selected_data.sel( + x=slice(bounding_box.left, bounding_box.right), + y=slice(bounding_box.top, bounding_box.bottom), + ) + + # selected_sat_data is likely to have 1 too many pixels in x and y + # because sel(x=slice(a, b)) is [a, b], not [a, b). So trim: + selected_data = selected_data.isel( + x=slice(0, self._square.size_pixels), y=slice(0, self._square.size_pixels) + ) + + selected_data = self._post_process_example(selected_data, t0_dt) + + if selected_data.shape != self._shape_of_example: + raise RuntimeError( + "Example is wrong shape! " + f"x_meters_center={x_meters_center}\n" + f"y_meters_center={y_meters_center}\n" + f"t0_dt={t0_dt}\n" + f"times are {selected_data.time}\n" + f"expected shape={self._shape_of_example}\n" + f"actual shape {selected_data.shape}" + ) + + # rename 'variable' to 'channels' + selected_data = selected_data.rename({"variable": "channels"}) + + # Compute optical flow for the timesteps + # Get Optical Flow for the pre-t0 time, and applying the t0-1 to t0 optical flow for + # forecast steps in the future + + return selected_data + + def _compute_optical_flow(self, sat_data: np.ndarray, timestep: int) -> np.ndarray: + """ + Args: + sat_data: uint8 numpy array of shape (num_timesteps, height, width) + timestep: The timestep to process. + + Returns: + optical flow field + """ + prev_img = sat_data[timestep] + next_img = sat_data[timestep + 1] + return cv2.calcOpticalFlowFarneback( + prev=prev_img, + next=next_img, + flow=None, + pyr_scale=0.5, + levels=2, + winsize=40, + iterations=3, + poly_n=5, + poly_sigma=0.7, + flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, + ) + + def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: + """Takes an image and warps it forwards in time according to the flow field. + + Args: + image: The grayscale image to warp. + flow: A 3D array. The first two dimensions must be the same size as the first two + dimensions of the image. The third dimension represented the x and y displacement. + + Returns: Warped image. The border has values np.NaN. + """ + # Adapted from https://github.com/opencv/opencv/issues/11068 + height, width = flow.shape[:2] + remap = -flow.copy() + remap[..., 0] += np.arange(width) # map_x + remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y + # cv.remap docs: https://docs.opencv.org/4.5.0/da/d54/group__imgproc__transform.html#gab75ef31ce5cdfb5c44b6da5f3b908ea4 + return cv2.remap( + src=image, + map1=remap, + map2=None, + interpolation=cv2.INTER_LINEAR, + # See BorderTypes: https://docs.opencv.org/4.5.0/d2/de8/group__core__array.html#ga209f2f4869e304c82d07739337eae7c5 + borderMode=cv2.BORDER_CONSTANT, + borderValue=np.NaN, + ) + + def _get_time_slice(self, t0_dt: pd.Timestamp) -> xr.DataArray: + try: + return self._cache[t0_dt] + except KeyError: + start_dt = self._get_start_dt(t0_dt) + end_dt = self._get_end_dt(t0_dt) + data = self.data.sel(time=slice(start_dt, end_dt)) + data = data.load() + self._cache[t0_dt] = data + return data + + def _post_process_example( + self, selected_data: xr.DataArray, t0_dt: pd.Timestamp + ) -> xr.DataArray: + + selected_data.data = selected_data.data.astype(np.float32) + + return selected_data + + def datetime_index(self, remove_night: bool = True) -> pd.DatetimeIndex: + """Returns a complete list of all available datetimes + + Args: + remove_night: If True then remove datetimes at night. + """ + if self._data is None: + sat_data = self._open_data() + else: + sat_data = self._data + + datetime_index = pd.DatetimeIndex(sat_data.time.values) + + if remove_night: + border_locations = self.geospatial_border() + datetime_index = nd_time.select_daylight_datetimes( + datetimes=datetime_index, locations=border_locations + ) + + return datetime_index + + +def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: + """Lazily opens the Zarr store. + + Adds 1 minute to the 'time' coordinates, so the timestamps + are at 00, 05, ..., 55 past the hour. + + Args: + zarr_path: Cloud URL or local path. If GCP URL, must start with 'gs://' + consolidated: Whether or not the Zarr metadata is consolidated. + """ + _LOG.debug("Opening satellite data: %s", zarr_path) + + # We load using chunks=None so xarray *doesn't* use Dask to + # load the Zarr chunks from disk. Using Dask to load the data + # seems to slow things down a lot if the Zarr store has more than + # about a million chunks. + # See https://github.com/openclimatefix/nowcasting_dataset/issues/23 + dataset = xr.open_dataset( + zarr_path, engine="zarr", consolidated=consolidated, mode="r", chunks=None + ) + + data_array = dataset["stacked_eumetsat_data"] + del dataset + + # The 'time' dimension is at 04, 09, ..., 59 minutes past the hour. + # To make it easier to align the satellite data with other data sources + # (which are at 00, 05, ..., 55 minutes past the hour) we add 1 minute to + # the time dimension. + # TODO Remove this as new Zarr already has the time fixed + data_array["time"] = data_array.time + pd.Timedelta("1 minute") + return data_array diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py new file mode 100644 index 00000000..58e504f4 --- /dev/null +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py @@ -0,0 +1,21 @@ +""" Model for output of Optical Flow data """ +from __future__ import annotations + +from xarray.ufuncs import isinf, isnan + +from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput + + +class OpticalFlow(DataSourceOutput): + """ Class to store optical flow data as a xr.Dataset with some validation """ + + __slots__ = () + _expected_dimensions = ("time", "x", "y", "channels") + + @classmethod + def model_validation(cls, v): + """ Check that all values are not NaN, Infinite, or -1.""" + assert (~isnan(v.data)).all(), "Some optical flow data values are NaNs" + assert (~isinf(v.data)).all(), "Some optical flow data values are Infinite" + assert (v.data != -1).all(), "Some optical flow data values are -1's" + return v diff --git a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py index 4dfe64b6..f8254f16 100644 --- a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py +++ b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py @@ -145,5 +145,6 @@ def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: # To make it easier to align the satellite data with other data sources # (which are at 00, 05, ..., 55 minutes past the hour) we add 1 minute to # the time dimension. + # TODO Remove this as new Zarr already has the time fixed data_array["time"] = data_array.time + pd.Timedelta("1 minute") return data_array From a294eb97483500e5391a64eaa02201a969c93dd1 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 1 Nov 2021 15:27:25 +0000 Subject: [PATCH 002/197] Add more to optical flow --- .../optical_flow/optical_flow_data_source.py | 52 +++++++++++++++---- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index c2e8a159..0bf136e8 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -30,6 +30,7 @@ class OpticalFlowDataSource(ZarrDataSource): zarr_path: str = None image_size_pixels: InitVar[int] = 128 meters_per_pixel: InitVar[int] = 2_000 + previous_timestep_for_flow: InitVar[int] = 1 def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): """ Post Init """ @@ -159,25 +160,58 @@ def get_example( selected_data = selected_data.rename({"variable": "channels"}) # Compute optical flow for the timesteps - # Get Optical Flow for the pre-t0 time, and applying the t0-1 to t0 optical flow for - # forecast steps in the future + # Get Optical Flow for the pre-t0 time, and applying the t0-previous_timesteps_per_flow to + # t0 optical flow for forecast steps in the future + # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions + # for all future timesteps for each of them + # Compute optical flow per channel, as it might be different + return selected_data - def _compute_optical_flow(self, sat_data: np.ndarray, timestep: int) -> np.ndarray: + def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp, previous_timestamp: pd.Timestamp): + """ + Compute and return optical flow predictions for the example + + Args: + satellite_data: Satellite DataArray + t0_dt: t0 timestamp + + Returns: + The xr.DataArray with the optical flow predictions for t0 to forecast horizon + """ + + prediction_dictionary = {} + + for channel in satellite_data.coords["channels"]: + channel_images = satellite_data.sel(channel=channel) + t0_image = channel_images.sel(time=t0_dt).values + previous_image = channel_images.sel(time=previous_timestamp).values + optical_flow = self._compute_optical_flow(t0_image, previous_image) + # Do predictions now + predictions = [] + # Number of timesteps before t0 + # TODO Fix this, number of future steps + for prediction_timestep in range(9): + flow = optical_flow * prediction_timestep + warped_image = self._remap_image(t0_image, flow) + predictions.append(warped_image) + prediction_dictionary[channel] = predictions + # TODO Convert to xr.DataArray + return prediction_dictionary + + + def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: """ Args: - sat_data: uint8 numpy array of shape (num_timesteps, height, width) - timestep: The timestep to process. + satellite_data: uint8 numpy array of shape (num_timesteps, height, width) Returns: optical flow field """ - prev_img = sat_data[timestep] - next_img = sat_data[timestep + 1] return cv2.calcOpticalFlowFarneback( - prev=prev_img, - next=next_img, + prev=previous_image, + next=t0_image, flow=None, pyr_scale=0.5, levels=2, From 572e24726664296a4b9a79cd27529aa63b5b485b Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 1 Nov 2021 15:50:05 +0000 Subject: [PATCH 003/197] Get previous timestep flow --- .../optical_flow/optical_flow_data_source.py | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 0bf136e8..4eaa6056 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -30,7 +30,7 @@ class OpticalFlowDataSource(ZarrDataSource): zarr_path: str = None image_size_pixels: InitVar[int] = 128 meters_per_pixel: InitVar[int] = 2_000 - previous_timestep_for_flow: InitVar[int] = 1 + previous_timestep_for_flow: int = 1 def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): """ Post Init """ @@ -165,11 +165,26 @@ def get_example( # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions # for all future timesteps for each of them # Compute optical flow per channel, as it might be different - + selected_data = self._compute_and_return_optical_flow(selected_data, t0_dt = t0_dt) return selected_data - def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp, previous_timestamp: pd.Timestamp): + def _compute_previous_timestep(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp) -> pd.Timestamp: + """ + Get timestamp of previous + + Args: + satellite_data: + t0_dt: + + Returns: + + """ + satellite_data = satellite_data.where(satellite_data.time <= t0_dt, drop = True) + return satellite_data.isel(time=-self.previous_timestep_for_flow).values + + + def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp): """ Compute and return optical flow predictions for the example @@ -182,7 +197,8 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: """ prediction_dictionary = {} - + # Get the previous timestamp + previous_timestamp = self._compute_previous_timestep(satellite_data, t0_dt = t0_dt) for channel in satellite_data.coords["channels"]: channel_images = satellite_data.sel(channel=channel) t0_image = channel_images.sel(time=t0_dt).values From b88ac47af7121d39505277df47d6e7194f22a537 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 1 Nov 2021 16:00:02 +0000 Subject: [PATCH 004/197] Reorder inputs --- .../data_sources/optical_flow/optical_flow_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 4eaa6056..41f7e905 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -28,9 +28,9 @@ class OpticalFlowDataSource(ZarrDataSource): """ zarr_path: str = None + previous_timestep_for_flow: int = 1 image_size_pixels: InitVar[int] = 128 meters_per_pixel: InitVar[int] = 2_000 - previous_timestep_for_flow: int = 1 def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): """ Post Init """ From dc6765e98487bc53f0d95f1acd769fdfd270a0b7 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 1 Nov 2021 16:15:49 +0000 Subject: [PATCH 005/197] Add to fake Batch --- nowcasting_dataset/data_sources/fake.py | 24 ++++++++++++++++++++++++ nowcasting_dataset/dataset/batch.py | 12 ++++++++++++ 2 files changed, 36 insertions(+) diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index 309ea1bf..9695e7dd 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -14,6 +14,7 @@ from nowcasting_dataset.data_sources.satellite.satellite_model import Satellite from nowcasting_dataset.data_sources.sun.sun_model import Sun from nowcasting_dataset.data_sources.topographic.topographic_model import Topographic +from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow from nowcasting_dataset.dataset.xr_utils import ( convert_data_array_to_dataset, join_list_data_array_to_batch_dataset, @@ -119,6 +120,29 @@ def satellite_fake( return Satellite(xr_dataset) +def optical_flow_fake( + batch_size=32, + seq_length_5=19, + satellite_image_size_pixels=64, + number_satellite_channels=7, + ) -> OpticalFlow: + """ Create fake data """ + # make batch of arrays + xr_arrays = [ + create_image_array( + seq_length_5=seq_length_5, + image_size_pixels=satellite_image_size_pixels, + number_channels=number_satellite_channels, + ) + for _ in range(batch_size) + ] + + # make dataset + xr_dataset = join_list_data_array_to_batch_dataset(xr_arrays) + + return OpticalFlow(xr_dataset) + + def sun_fake(batch_size, seq_length_5): """Create fake data""" # create dataset with both azimuth and elevation, index with time diff --git a/nowcasting_dataset/dataset/batch.py b/nowcasting_dataset/dataset/batch.py index 49fc4c11..37622b3a 100644 --- a/nowcasting_dataset/dataset/batch.py +++ b/nowcasting_dataset/dataset/batch.py @@ -20,6 +20,7 @@ satellite_fake, sun_fake, topographic_fake, + optical_flow_fake, ) from nowcasting_dataset.data_sources.gsp.gsp_model import GSP from nowcasting_dataset.data_sources.metadata.metadata_model import Metadata @@ -29,6 +30,7 @@ from nowcasting_dataset.data_sources.sun.sun_model import Sun from nowcasting_dataset.data_sources.topographic.topographic_model import Topographic from nowcasting_dataset.utils import get_netcdf_filename +from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow _LOG = logging.getLogger(__name__) @@ -57,6 +59,7 @@ class Batch(BaseModel): metadata: Optional[Metadata] satellite: Optional[Satellite] topographic: Optional[Topographic] + optical_flow: Optional[OpticalFlow] pv: Optional[PV] sun: Optional[Sun] gsp: Optional[GSP] @@ -68,6 +71,7 @@ def data_sources(self): return [ self.satellite, self.topographic, + self.optical_flow, self.pv, self.sun, self.gsp, @@ -92,6 +96,14 @@ def fake(configuration: Configuration): configuration.input_data.satellite.satellite_channels ), ), + optical_flow=optical_flow_fake( + batch_size=batch_size, + seq_length_5=configuration.input_data.satellite.seq_length_5_minutes, + satellite_image_size_pixels=satellite_image_size_pixels, + number_satellite_channels=len( + configuration.input_data.satellite.satellite_channels + ), + ), nwp=nwp_fake( batch_size=batch_size, seq_length_5=configuration.input_data.nwp.seq_length_5_minutes, From 0d47d0ac32d4e1ceda51cb321c8a91b1988933be Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 1 Nov 2021 16:17:20 +0000 Subject: [PATCH 006/197] Add to init --- nowcasting_dataset/data_sources/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py index 034d9c46..1eb6267f 100644 --- a/nowcasting_dataset/data_sources/__init__.py +++ b/nowcasting_dataset/data_sources/__init__.py @@ -1,6 +1,7 @@ """ Various DataSources """ from nowcasting_dataset.data_sources.data_source import DataSource # noqa: F401 from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource +from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import OpticalFlowDataSource from nowcasting_dataset.data_sources.nwp.nwp_data_source import NWPDataSource from nowcasting_dataset.data_sources.pv.pv_data_source import PVDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource From def12f556f7f9078fa6c7f0d93eb8a2328fd5f85 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 1 Nov 2021 16:29:54 +0000 Subject: [PATCH 007/197] Add to configuration --- nowcasting_dataset/config/model.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 136ca152..3b90d048 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -114,6 +114,21 @@ class Satellite(DataSourceMixin): satellite_meters_per_pixel: int = METERS_PER_PIXEL_FIELD +class OpticalFlow(DataSourceMixin): + """Satellite configuration model""" + + satellite_zarr_path: str = Field( + "gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr", + description="The path which holds the satellite zarr.", + ) + satellite_channels: tuple = Field( + SAT_VARIABLE_NAMES, description="the satellite channels that are used" + ) + satellite_image_size_pixels: int = IMAGE_SIZE_PIXELS_FIELD + satellite_meters_per_pixel: int = METERS_PER_PIXEL_FIELD + previous_timestep_to_use: int = 1 + + class NWP(DataSourceMixin): """NWP configuration model""" @@ -178,6 +193,7 @@ class InputData(BaseModel): pv: Optional[PV] = None satellite: Optional[Satellite] = None + optical_flow: Optional[OpticalFlow] = None nwp: Optional[NWP] = None gsp: Optional[GSP] = None topographic: Optional[Topographic] = None @@ -217,7 +233,7 @@ def set_forecast_and_history_minutes(cls, values): """ # It would be much better to use nowcasting_dataset.data_sources.ALL_DATA_SOURCE_NAMES, # but that causes a circular import. - ALL_DATA_SOURCE_NAMES = ("pv", "satellite", "nwp", "gsp", "topographic", "sun") + ALL_DATA_SOURCE_NAMES = ("pv", "satellite", "nwp", "gsp", "topographic", "sun", "optical_flow") enabled_data_sources = [ data_source_name for data_source_name in ALL_DATA_SOURCE_NAMES @@ -246,6 +262,7 @@ def set_all_to_defaults(cls): gsp=GSP(), topographic=Topographic(), sun=Sun(), + optical_flow=OpticalFlow(), ) From 443124bb87fe4e1eb13678a182170565dc44e988 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 1 Nov 2021 16:33:42 +0000 Subject: [PATCH 008/197] Add padding Optical Flow --- .../optical_flow/optical_flow_data_source.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 41f7e905..63cc3ba2 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -25,6 +25,8 @@ class OpticalFlowDataSource(ZarrDataSource): Optical Flow Data Source, computing flow between Satellite data zarr_path: Must start with 'gs://' if on GCP. + + Pads image size to allow for cropping out NaN values """ zarr_path: str = None @@ -33,13 +35,13 @@ class OpticalFlowDataSource(ZarrDataSource): meters_per_pixel: InitVar[int] = 2_000 def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): - """ Post Init """ - super().__post_init__(image_size_pixels, meters_per_pixel) + """ Post Init Add 16 pixels to each side of the image""" + super().__post_init__(image_size_pixels+32, meters_per_pixel) self._cache = {} self._shape_of_example = ( self._total_seq_length, - image_size_pixels, - image_size_pixels, + image_size_pixels+32, + image_size_pixels+32, 2, ) @@ -211,6 +213,7 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: for prediction_timestep in range(9): flow = optical_flow * prediction_timestep warped_image = self._remap_image(t0_image, flow) + # TODO Crop out center of the flow to match the desired shape predictions.append(warped_image) prediction_dictionary[channel] = predictions # TODO Convert to xr.DataArray From 2a4d9e1b02d5ff7aae43ff53bf57901a3bb0e7d5 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Tue, 2 Nov 2021 11:04:03 +0000 Subject: [PATCH 009/197] Add crop center --- .../optical_flow/optical_flow_data_source.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 63cc3ba2..0148ef18 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -214,6 +214,8 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: flow = optical_flow * prediction_timestep warped_image = self._remap_image(t0_image, flow) # TODO Crop out center of the flow to match the desired shape + warped_image = crop_center(warped_image, self._square.size_pixels, + self._square.size_pixels) predictions.append(warped_image) prediction_dictionary[channel] = predictions # TODO Convert to xr.DataArray @@ -339,3 +341,21 @@ def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: # TODO Remove this as new Zarr already has the time fixed data_array["time"] = data_array.time + pd.Timedelta("1 minute") return data_array + + +def crop_center(img,cropx,cropy): + """ + Crop center of numpy image + + Args: + img: + cropx: + cropy: + + Returns: + + """ + y,x = img.shape + startx = x//2-(cropx//2) + starty = y//2-(cropy//2) + return img[starty:starty+cropy,startx:startx+cropx] \ No newline at end of file From ab30f365bef8226e0f7e3d807606c2f05dd57540 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Tue, 2 Nov 2021 11:08:58 +0000 Subject: [PATCH 010/197] Add getting number of future timesteps --- .../optical_flow/optical_flow_data_source.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 0148ef18..cb0c30f0 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -182,9 +182,23 @@ def _compute_previous_timestep(self, satellite_data: xr.DataArray, t0_dt: pd.Tim Returns: """ - satellite_data = satellite_data.where(satellite_data.time <= t0_dt, drop = True) + satellite_data = satellite_data.where(satellite_data.time < t0_dt, drop = True) return satellite_data.isel(time=-self.previous_timestep_for_flow).values + def _get_number_future_timesteps(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp) -> \ + int: + """ + Get number of future timestamps + + Args: + satellite_data: + t0_dt: + + Returns: + + """ + satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop = True) + return len(satellite_data.coords['time']) def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp): """ @@ -209,11 +223,9 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: # Do predictions now predictions = [] # Number of timesteps before t0 - # TODO Fix this, number of future steps - for prediction_timestep in range(9): + for prediction_timestep in range(self._get_number_future_timesteps(satellite_data, t0_dt)): flow = optical_flow * prediction_timestep warped_image = self._remap_image(t0_image, flow) - # TODO Crop out center of the flow to match the desired shape warped_image = crop_center(warped_image, self._square.size_pixels, self._square.size_pixels) predictions.append(warped_image) From c1a5f977578f43aa15d2b68594b39652926d6ce4 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 10:40:24 +0000 Subject: [PATCH 011/197] Update to newer format --- .../optical_flow/optical_flow_data_source.py | 75 +++++++++++++------ 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index cb0c30f0..c3bf5a02 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -11,6 +11,7 @@ import xarray as xr import nowcasting_dataset.time as nd_time +from nowcasting_dataset.consts import SAT_VARIABLE_NAMES from nowcasting_dataset.data_sources.data_source import ZarrDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow @@ -24,12 +25,9 @@ class OpticalFlowDataSource(ZarrDataSource): """ Optical Flow Data Source, computing flow between Satellite data - zarr_path: Must start with 'gs://' if on GCP. - Pads image size to allow for cropping out NaN values """ - - zarr_path: str = None + channels: Optional[Iterable[str]] = SAT_VARIABLE_NAMES previous_timestep_for_flow: int = 1 image_size_pixels: InitVar[int] = 128 meters_per_pixel: InitVar[int] = 2_000 @@ -37,12 +35,13 @@ class OpticalFlowDataSource(ZarrDataSource): def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): """ Post Init Add 16 pixels to each side of the image""" super().__post_init__(image_size_pixels+32, meters_per_pixel) + n_channels = len(self.channels) self._cache = {} self._shape_of_example = ( self._total_seq_length, image_size_pixels+32, image_size_pixels+32, - 2, + n_channels, ) def open(self) -> None: @@ -281,30 +280,60 @@ def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: borderValue=np.NaN, ) - def _get_time_slice(self, t0_dt: pd.Timestamp) -> xr.DataArray: - try: - return self._cache[t0_dt] - except KeyError: - start_dt = self._get_start_dt(t0_dt) - end_dt = self._get_end_dt(t0_dt) - data = self.data.sel(time=slice(start_dt, end_dt)) - data = data.load() - self._cache[t0_dt] = data - return data - - def _post_process_example( - self, selected_data: xr.DataArray, t0_dt: pd.Timestamp - ) -> xr.DataArray: - - selected_data.data = selected_data.data.astype(np.float32) + def _open_data(self) -> xr.DataArray: + return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) - return selected_data + def _dataset_to_data_source_output(output: xr.Dataset) -> OpticalFlow: + return OpticalFlow(output) + + def _get_time_slice(self, t0_dt: pd.Timestamp) -> xr.DataArray: + start_dt = self._get_start_dt(t0_dt) + end_dt = self._get_end_dt(t0_dt) + data = self.data.sel(time=slice(start_dt, end_dt)) + return data def datetime_index(self, remove_night: bool = True) -> pd.DatetimeIndex: """Returns a complete list of all available datetimes Args: remove_night: If True then remove datetimes at night. + We're interested in forecasting solar power generation, so we + don't care about nighttime data :) + + In the UK in summer, the sun rises first in the north east, and + sets last in the north west [1]. In summer, the north gets more + hours of sunshine per day. + + In the UK in winter, the sun rises first in the south east, and + sets last in the south west [2]. In winter, the south gets more + hours of sunshine per day. + + | | Summer | Winter | + | ---: | :---: | :---: | + | Sun rises first in | N.E. | S.E. | + | Sun sets last in | N.W. | S.W. | + | Most hours of sunlight | North | South | + + Before training, we select timesteps which have at least some + sunlight. We do this by computing the clearsky global horizontal + irradiance (GHI) for the four corners of the satellite imagery, + and for all the timesteps in the dataset. We only use timesteps + where the maximum global horizontal irradiance across all four + corners is above some threshold. + + The 'clearsky solar irradiance' is the amount of sunlight we'd + expect on a clear day at a specific time and location. The SI unit + of irradiance is watt per square meter. The 'global horizontal + irradiance' (GHI) is the total sunlight that would hit a + horizontal surface on the surface of the Earth. The GHI is the + sum of the direct irradiance (sunlight which takes a direct path + from the Sun to the Earth's surface) and the diffuse horizontal + irradiance (the sunlight scattered from the atmosphere). For more + info, see: https://en.wikipedia.org/wiki/Solar_irradiance + + References: + 1. [Video of June 2019](https://www.youtube.com/watch?v=IOp-tj-IJpk) + 2. [Video of Jan 2019](https://www.youtube.com/watch?v=CJ4prUVa2nQ) """ if self._data is None: sat_data = self._open_data() @@ -317,7 +346,7 @@ def datetime_index(self, remove_night: bool = True) -> pd.DatetimeIndex: border_locations = self.geospatial_border() datetime_index = nd_time.select_daylight_datetimes( datetimes=datetime_index, locations=border_locations - ) + ) return datetime_index From c76f8694d810ad9db1f3412995e9cef7b3e4e744 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 10:52:17 +0000 Subject: [PATCH 012/197] Remove get_batch --- .../optical_flow/optical_flow_data_source.py | 80 +++---------------- 1 file changed, 13 insertions(+), 67 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index c3bf5a02..89fa59e2 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -39,8 +39,8 @@ def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): self._cache = {} self._shape_of_example = ( self._total_seq_length, - image_size_pixels+32, - image_size_pixels+32, + image_size_pixels, + image_size_pixels, n_channels, ) @@ -59,60 +59,6 @@ def open(self) -> None: def _open_data(self) -> xr.DataArray: return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) - def get_batch( - self, - t0_datetimes: pd.DatetimeIndex, - x_locations: Iterable[Number], - y_locations: Iterable[Number], - ) -> OpticalFlow: - """ - Get batch data - - Load the first _n_timesteps_per_batch concurrently. This - loads the timesteps from disk concurrently, and fills the - cache. If we try loading all examples - concurrently, then SatelliteDataSource will try reading from - empty caches, and things are much slower! - - Args: - t0_datetimes: list of timestamps for the datetime of the batches. The batch will also - include data for historic and future depending on `history_minutes` and - `future_minutes`. - x_locations: x center batch locations - y_locations: y center batch locations - - Returns: Batch data - - """ - # Load the first _n_timesteps_per_batch concurrently. This - # loads the timesteps from disk concurrently, and fills the - # cache. If we try loading all examples - # concurrently, then SatelliteDataSource will try reading from - # empty caches, and things are much slower! - zipped = list(zip(t0_datetimes, x_locations, y_locations)) - batch_size = len(t0_datetimes) - - with futures.ThreadPoolExecutor(max_workers=batch_size) as executor: - future_examples = [] - for coords in zipped[: self.n_timesteps_per_batch]: - t0_datetime, x_location, y_location = coords - future_example = executor.submit( - self.get_example, t0_datetime, x_location, y_location - ) - future_examples.append(future_example) - examples = [future_example.result() for future_example in future_examples] - - # Load the remaining examples. This should hit the DataSource caches. - for coords in zipped[self.n_timesteps_per_batch :]: - t0_datetime, x_location, y_location = coords - example = self.get_example(t0_datetime, x_location, y_location) - examples.append(example) - - output = join_list_data_array_to_batch_dataset(examples) - - self._cache = {} - - return OpticalFlow(output) def get_example( self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number @@ -132,20 +78,28 @@ def get_example( selected_data = self._get_time_slice(t0_dt) bounding_box = self._square.bounding_box_centered_on( x_meters_center=x_meters_center, y_meters_center=y_meters_center - ) + ) selected_data = selected_data.sel( x=slice(bounding_box.left, bounding_box.right), y=slice(bounding_box.top, bounding_box.bottom), - ) + ) # selected_sat_data is likely to have 1 too many pixels in x and y # because sel(x=slice(a, b)) is [a, b], not [a, b). So trim: selected_data = selected_data.isel( x=slice(0, self._square.size_pixels), y=slice(0, self._square.size_pixels) - ) + ) selected_data = self._post_process_example(selected_data, t0_dt) + # Compute optical flow for the timesteps + # Get Optical Flow for the pre-t0 time, and applying the t0-previous_timesteps_per_flow to + # t0 optical flow for forecast steps in the future + # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions + # for all future timesteps for each of them + # Compute optical flow per channel, as it might be different + selected_data = self._compute_and_return_optical_flow(selected_data, t0_dt = t0_dt) + if selected_data.shape != self._shape_of_example: raise RuntimeError( "Example is wrong shape! " @@ -160,14 +114,6 @@ def get_example( # rename 'variable' to 'channels' selected_data = selected_data.rename({"variable": "channels"}) - # Compute optical flow for the timesteps - # Get Optical Flow for the pre-t0 time, and applying the t0-previous_timesteps_per_flow to - # t0 optical flow for forecast steps in the future - # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions - # for all future timesteps for each of them - # Compute optical flow per channel, as it might be different - selected_data = self._compute_and_return_optical_flow(selected_data, t0_dt = t0_dt) - return selected_data def _compute_previous_timestep(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp) -> pd.Timestamp: From 3bd99a15c813813b37ab4301513d70b77a5919db Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 11:15:41 +0000 Subject: [PATCH 013/197] Misc update --- .../data_sources/optical_flow/optical_flow_data_source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 89fa59e2..0fa48b63 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -176,6 +176,7 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: predictions.append(warped_image) prediction_dictionary[channel] = predictions # TODO Convert to xr.DataArray + # Swap out data for the future part of the dataarray return prediction_dictionary From ee604db72430cb1167b886e00a952eee21fbdbce Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 13:40:01 +0000 Subject: [PATCH 014/197] Add opencv --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8fd4e4aa..46e2b562 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,3 +29,4 @@ s3fs fsspec pathy satip>=2.0.2 +opencv From 1df138ce848d73e3b1bb55f5a71aeab449447c83 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 13:43:01 +0000 Subject: [PATCH 015/197] Fix requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 46e2b562..cdc9ec59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,4 +29,4 @@ s3fs fsspec pathy satip>=2.0.2 -opencv +opencv-python From 6097286d5252783908c16d3a91d4cc5a2af9953c Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 13:44:52 +0000 Subject: [PATCH 016/197] Change to headless OpenCV --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cdc9ec59..0a0f2eef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,4 +29,4 @@ s3fs fsspec pathy satip>=2.0.2 -opencv-python +opencv-contrib-python-headless From 7c438b5d7ed9a6385a1615eeb864d01764682dfb Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 13:56:50 +0000 Subject: [PATCH 017/197] Fix linter errors --- nowcasting_dataset/config/model.py | 16 +++- nowcasting_dataset/data_sources/__init__.py | 4 +- nowcasting_dataset/data_sources/fake.py | 16 ++-- .../optical_flow/optical_flow_data_source.py | 86 ++++++++++--------- nowcasting_dataset/dataset/batch.py | 6 +- 5 files changed, 70 insertions(+), 58 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 3b90d048..016f44d1 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -118,12 +118,12 @@ class OpticalFlow(DataSourceMixin): """Satellite configuration model""" satellite_zarr_path: str = Field( - "gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr", + "gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr", # noqa: E501 description="The path which holds the satellite zarr.", - ) + ) satellite_channels: tuple = Field( SAT_VARIABLE_NAMES, description="the satellite channels that are used" - ) + ) satellite_image_size_pixels: int = IMAGE_SIZE_PIXELS_FIELD satellite_meters_per_pixel: int = METERS_PER_PIXEL_FIELD previous_timestep_to_use: int = 1 @@ -233,7 +233,15 @@ def set_forecast_and_history_minutes(cls, values): """ # It would be much better to use nowcasting_dataset.data_sources.ALL_DATA_SOURCE_NAMES, # but that causes a circular import. - ALL_DATA_SOURCE_NAMES = ("pv", "satellite", "nwp", "gsp", "topographic", "sun", "optical_flow") + ALL_DATA_SOURCE_NAMES = ( + "pv", + "satellite", + "nwp", + "gsp", + "topographic", + "sun", + "optical_flow", + ) enabled_data_sources = [ data_source_name for data_source_name in ALL_DATA_SOURCE_NAMES diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py index 1eb6267f..6fc93310 100644 --- a/nowcasting_dataset/data_sources/__init__.py +++ b/nowcasting_dataset/data_sources/__init__.py @@ -1,8 +1,10 @@ """ Various DataSources """ from nowcasting_dataset.data_sources.data_source import DataSource # noqa: F401 from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource -from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import OpticalFlowDataSource from nowcasting_dataset.data_sources.nwp.nwp_data_source import NWPDataSource +from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( + OpticalFlowDataSource, +) from nowcasting_dataset.data_sources.pv.pv_data_source import PVDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource from nowcasting_dataset.data_sources.sun.sun_data_source import SunDataSource diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index 9695e7dd..87722a36 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -10,11 +10,11 @@ from nowcasting_dataset.data_sources.gsp.gsp_model import GSP from nowcasting_dataset.data_sources.metadata.metadata_model import Metadata from nowcasting_dataset.data_sources.nwp.nwp_model import NWP +from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow from nowcasting_dataset.data_sources.pv.pv_model import PV from nowcasting_dataset.data_sources.satellite.satellite_model import Satellite from nowcasting_dataset.data_sources.sun.sun_model import Sun from nowcasting_dataset.data_sources.topographic.topographic_model import Topographic -from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow from nowcasting_dataset.dataset.xr_utils import ( convert_data_array_to_dataset, join_list_data_array_to_batch_dataset, @@ -121,11 +121,11 @@ def satellite_fake( def optical_flow_fake( - batch_size=32, - seq_length_5=19, - satellite_image_size_pixels=64, - number_satellite_channels=7, - ) -> OpticalFlow: + batch_size=32, + seq_length_5=19, + satellite_image_size_pixels=64, + number_satellite_channels=7, +) -> OpticalFlow: """ Create fake data """ # make batch of arrays xr_arrays = [ @@ -133,9 +133,9 @@ def optical_flow_fake( seq_length_5=seq_length_5, image_size_pixels=satellite_image_size_pixels, number_channels=number_satellite_channels, - ) + ) for _ in range(batch_size) - ] + ] # make dataset xr_dataset = join_list_data_array_to_batch_dataset(xr_arrays) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 0fa48b63..04723ae3 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -1,6 +1,5 @@ """ Optical Flow Data Source """ import logging -from concurrent import futures from dataclasses import InitVar, dataclass from numbers import Number from typing import Iterable, Optional @@ -15,7 +14,6 @@ from nowcasting_dataset.data_sources.data_source import ZarrDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow -from nowcasting_dataset.dataset.xr_utils import join_list_data_array_to_batch_dataset _LOG = logging.getLogger("nowcasting_dataset") @@ -27,6 +25,7 @@ class OpticalFlowDataSource(ZarrDataSource): Pads image size to allow for cropping out NaN values """ + channels: Optional[Iterable[str]] = SAT_VARIABLE_NAMES previous_timestep_for_flow: int = 1 image_size_pixels: InitVar[int] = 128 @@ -34,7 +33,7 @@ class OpticalFlowDataSource(ZarrDataSource): def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): """ Post Init Add 16 pixels to each side of the image""" - super().__post_init__(image_size_pixels+32, meters_per_pixel) + super().__post_init__(image_size_pixels + 32, meters_per_pixel) n_channels = len(self.channels) self._cache = {} self._shape_of_example = ( @@ -59,7 +58,6 @@ def open(self) -> None: def _open_data(self) -> xr.DataArray: return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) - def get_example( self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number ) -> DataSourceOutput: @@ -78,17 +76,17 @@ def get_example( selected_data = self._get_time_slice(t0_dt) bounding_box = self._square.bounding_box_centered_on( x_meters_center=x_meters_center, y_meters_center=y_meters_center - ) + ) selected_data = selected_data.sel( x=slice(bounding_box.left, bounding_box.right), y=slice(bounding_box.top, bounding_box.bottom), - ) + ) # selected_sat_data is likely to have 1 too many pixels in x and y # because sel(x=slice(a, b)) is [a, b], not [a, b). So trim: selected_data = selected_data.isel( x=slice(0, self._square.size_pixels), y=slice(0, self._square.size_pixels) - ) + ) selected_data = self._post_process_example(selected_data, t0_dt) @@ -98,7 +96,7 @@ def get_example( # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions # for all future timesteps for each of them # Compute optical flow per channel, as it might be different - selected_data = self._compute_and_return_optical_flow(selected_data, t0_dt = t0_dt) + selected_data = self._compute_and_return_optical_flow(selected_data, t0_dt=t0_dt) if selected_data.shape != self._shape_of_example: raise RuntimeError( @@ -116,34 +114,37 @@ def get_example( return selected_data - def _compute_previous_timestep(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp) -> pd.Timestamp: + def _compute_previous_timestep( + self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp + ) -> pd.Timestamp: """ Get timestamp of previous Args: - satellite_data: - t0_dt: + satellite_data: Satellite data to use + t0_dt: Timestamp Returns: - + The previous timesteps """ - satellite_data = satellite_data.where(satellite_data.time < t0_dt, drop = True) + satellite_data = satellite_data.where(satellite_data.time < t0_dt, drop=True) return satellite_data.isel(time=-self.previous_timestep_for_flow).values - def _get_number_future_timesteps(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp) -> \ - int: + def _get_number_future_timesteps( + self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp + ) -> int: """ Get number of future timestamps Args: - satellite_data: - t0_dt: + satellite_data: Satellite data to use + t0_dt: The timestamp of the t0 image Returns: - + The number of future timesteps """ - satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop = True) - return len(satellite_data.coords['time']) + satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) + return len(satellite_data.coords["time"]) def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp): """ @@ -159,7 +160,7 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: prediction_dictionary = {} # Get the previous timestamp - previous_timestamp = self._compute_previous_timestep(satellite_data, t0_dt = t0_dt) + previous_timestamp = self._compute_previous_timestep(satellite_data, t0_dt=t0_dt) for channel in satellite_data.coords["channels"]: channel_images = satellite_data.sel(channel=channel) t0_image = channel_images.sel(time=t0_dt).values @@ -168,22 +169,27 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: # Do predictions now predictions = [] # Number of timesteps before t0 - for prediction_timestep in range(self._get_number_future_timesteps(satellite_data, t0_dt)): + for prediction_timestep in range( + self._get_number_future_timesteps(satellite_data, t0_dt) + ): flow = optical_flow * prediction_timestep warped_image = self._remap_image(t0_image, flow) - warped_image = crop_center(warped_image, self._square.size_pixels, - self._square.size_pixels) + warped_image = crop_center( + warped_image, self._square.size_pixels, self._square.size_pixels + ) predictions.append(warped_image) prediction_dictionary[channel] = predictions # TODO Convert to xr.DataArray # Swap out data for the future part of the dataarray return prediction_dictionary - def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: """ + Compute the optical flow for a set of images + Args: - satellite_data: uint8 numpy array of shape (num_timesteps, height, width) + t0_image: t0 image + previous_image: previous image to compute optical flow with Returns: optical flow field @@ -202,7 +208,8 @@ def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray ) def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: - """Takes an image and warps it forwards in time according to the flow field. + """ + Takes an image and warps it forwards in time according to the flow field. Args: image: The grayscale image to warp. @@ -216,20 +223,15 @@ def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: remap = -flow.copy() remap[..., 0] += np.arange(width) # map_x remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y - # cv.remap docs: https://docs.opencv.org/4.5.0/da/d54/group__imgproc__transform.html#gab75ef31ce5cdfb5c44b6da5f3b908ea4 return cv2.remap( src=image, map1=remap, map2=None, interpolation=cv2.INTER_LINEAR, - # See BorderTypes: https://docs.opencv.org/4.5.0/d2/de8/group__core__array.html#ga209f2f4869e304c82d07739337eae7c5 borderMode=cv2.BORDER_CONSTANT, borderValue=np.NaN, ) - def _open_data(self) -> xr.DataArray: - return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) - def _dataset_to_data_source_output(output: xr.Dataset) -> OpticalFlow: return OpticalFlow(output) @@ -293,7 +295,7 @@ def datetime_index(self, remove_night: bool = True) -> pd.DatetimeIndex: border_locations = self.geospatial_border() datetime_index = nd_time.select_daylight_datetimes( datetimes=datetime_index, locations=border_locations - ) + ) return datetime_index @@ -331,19 +333,19 @@ def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: return data_array -def crop_center(img,cropx,cropy): +def crop_center(img, cropx, cropy): """ Crop center of numpy image Args: - img: - cropx: - cropy: + img: Image to crop + cropx: Size in x direction + cropy: Size in y direction Returns: - + The cropped image """ - y,x = img.shape - startx = x//2-(cropx//2) - starty = y//2-(cropy//2) - return img[starty:starty+cropy,startx:startx+cropx] \ No newline at end of file + y, x = img.shape + startx = x // 2 - (cropx // 2) + starty = y // 2 - (cropy // 2) + return img[starty : starty + cropy, startx : startx + cropx] diff --git a/nowcasting_dataset/dataset/batch.py b/nowcasting_dataset/dataset/batch.py index 37622b3a..4cb71e30 100644 --- a/nowcasting_dataset/dataset/batch.py +++ b/nowcasting_dataset/dataset/batch.py @@ -16,21 +16,21 @@ gsp_fake, metadata_fake, nwp_fake, + optical_flow_fake, pv_fake, satellite_fake, sun_fake, topographic_fake, - optical_flow_fake, ) from nowcasting_dataset.data_sources.gsp.gsp_model import GSP from nowcasting_dataset.data_sources.metadata.metadata_model import Metadata from nowcasting_dataset.data_sources.nwp.nwp_model import NWP +from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow from nowcasting_dataset.data_sources.pv.pv_model import PV from nowcasting_dataset.data_sources.satellite.satellite_model import Satellite from nowcasting_dataset.data_sources.sun.sun_model import Sun from nowcasting_dataset.data_sources.topographic.topographic_model import Topographic from nowcasting_dataset.utils import get_netcdf_filename -from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow _LOG = logging.getLogger(__name__) @@ -102,8 +102,8 @@ def fake(configuration: Configuration): satellite_image_size_pixels=satellite_image_size_pixels, number_satellite_channels=len( configuration.input_data.satellite.satellite_channels - ), ), + ), nwp=nwp_fake( batch_size=batch_size, seq_length_5=configuration.input_data.nwp.seq_length_5_minutes, From add230a29d808997c7d3a9488f0fd1ce790a201b Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 14:18:18 +0000 Subject: [PATCH 018/197] Add unit tests --- .../test_optical_flow_data_source.py | 55 +++++++++++++++++++ .../optical_flow/test_optical_flow_model.py | 31 +++++++++++ 2 files changed, 86 insertions(+) create mode 100644 tests/data_sources/optical_flow/test_optical_flow_data_source.py create mode 100644 tests/data_sources/optical_flow/test_optical_flow_model.py diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py new file mode 100644 index 00000000..5d13c488 --- /dev/null +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -0,0 +1,55 @@ +"""Test OpticalFlowDataSource.""" +import numpy as np +import pandas as pd +import pytest + + +def test_satellite_data_source_init(sat_data_source): # noqa: D103 + pass + + +def test_open(sat_data_source): # noqa: D103 + sat_data_source.open() + assert sat_data_source.data is not None + + +def test_datetime_index(sat_data_source): # noqa: D103 + datetimes = sat_data_source.datetime_index() + assert isinstance(datetimes, pd.DatetimeIndex) + assert len(datetimes) > 0 + assert len(np.unique(datetimes)) == len(datetimes) + assert np.all(np.diff(datetimes.view(int)) > 0) + + +@pytest.mark.parametrize( + "x, y, left, right, top, bottom", + [ + (0, 0, -128_000, 126_000, 128_000, -126_000), + (10, 0, -126_000, 128_000, 128_000, -126_000), + (30, 0, -126_000, 128_000, 128_000, -126_000), + (1000, 0, -126_000, 128_000, 128_000, -126_000), + (0, 1000, -128_000, 126_000, 128_000, -126_000), + (1000, 1000, -126_000, 128_000, 128_000, -126_000), + (2000, 2000, -126_000, 128_000, 130_000, -124_000), + (2000, 1000, -126_000, 128_000, 128_000, -126_000), + (2001, 2001, -124_000, 130_000, 130_000, -124_000), + ], +) +def test_get_example(sat_data_source, x, y, left, right, top, bottom): # noqa: D103 + sat_data_source.open() + t0_dt = pd.Timestamp("2019-01-01T13:00") + sat_data = sat_data_source.get_example(t0_dt=t0_dt, x_meters_center=x, y_meters_center=y) + + assert left == sat_data.x.values[0] + assert right == sat_data.x.values[-1] + # sat_data.y is top-to-bottom. + assert top == sat_data.y.values[0] + assert bottom == sat_data.y.values[-1] + assert len(sat_data.x) == pytest.IMAGE_SIZE_PIXELS + assert len(sat_data.y) == pytest.IMAGE_SIZE_PIXELS + + +def test_geospatial_border(sat_data_source): # noqa: D103 + border = sat_data_source.geospatial_border() + correct_border = [(-110000, 1094000), (-110000, -58000), (730000, 1094000), (730000, -58000)] + np.testing.assert_array_equal(border, correct_border) diff --git a/tests/data_sources/optical_flow/test_optical_flow_model.py b/tests/data_sources/optical_flow/test_optical_flow_model.py new file mode 100644 index 00000000..348d9832 --- /dev/null +++ b/tests/data_sources/optical_flow/test_optical_flow_model.py @@ -0,0 +1,31 @@ +"""Test Optical Flow model.""" +import os +import tempfile + +import numpy as np +import pytest + +from nowcasting_dataset.data_sources.fake import optical_flow_fake +from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow + + +def test_optical_flow_init(): # noqa: D103 + _ = optical_flow_fake() + + +def test_optical_flow_validation(): # noqa: D103 + sat = optical_flow_fake() + + OpticalFlow.model_validation(sat) + + sat.data[0, 0] = np.nan + with pytest.raises(Exception): + optical_flow_fake.model_validation(sat) + + +def test_optical_flow_save(): # noqa: D103 + + with tempfile.TemporaryDirectory() as dirpath: + optical_flow_fake().save_netcdf(path=dirpath, batch_i=0) + + assert os.path.exists(f"{dirpath}/satellite/000000.nc") From 9fd22014a09bc8bcbcef1cb1cb409e3f3c9752cc Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 14:21:04 +0000 Subject: [PATCH 019/197] Update for OpticalFlowDataSource --- conftest.py | 13 +++++++++- .../test_optical_flow_data_source.py | 24 ++++++++++--------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/conftest.py b/conftest.py index a4ebb9d3..a95ee4c0 100644 --- a/conftest.py +++ b/conftest.py @@ -7,7 +7,7 @@ import nowcasting_dataset from nowcasting_dataset import consts from nowcasting_dataset.config.load import load_yaml_configuration -from nowcasting_dataset.data_sources import SatelliteDataSource +from nowcasting_dataset.data_sources import OpticalFlowDataSource, SatelliteDataSource from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource @@ -49,6 +49,17 @@ def sat_data_source(sat_filename: Path): # noqa: D103 ) +@pytest.fixture +def optical_flow_data_source(sat_filename: Path): # noqa: D103 + return OpticalFlowDataSource( + image_size_pixels=pytest.IMAGE_SIZE_PIXELS, + zarr_path=sat_filename, + history_minutes=0, + forecast_minutes=5, + channels=("HRV",), + ) + + @pytest.fixture def general_data_source(): # noqa: D103 return MetadataDataSource(history_minutes=0, forecast_minutes=5, object_at_center="GSP") diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 5d13c488..5d9a7f4d 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -4,17 +4,17 @@ import pytest -def test_satellite_data_source_init(sat_data_source): # noqa: D103 +def test_satellite_data_source_init(optical_flow_data_source): # noqa: D103 pass -def test_open(sat_data_source): # noqa: D103 - sat_data_source.open() - assert sat_data_source.data is not None +def test_open(optical_flow_data_source): # noqa: D103 + optical_flow_data_source.open() + assert optical_flow_data_source.data is not None -def test_datetime_index(sat_data_source): # noqa: D103 - datetimes = sat_data_source.datetime_index() +def test_datetime_index(optical_flow_data_source): # noqa: D103 + datetimes = optical_flow_data_source.datetime_index() assert isinstance(datetimes, pd.DatetimeIndex) assert len(datetimes) > 0 assert len(np.unique(datetimes)) == len(datetimes) @@ -35,10 +35,12 @@ def test_datetime_index(sat_data_source): # noqa: D103 (2001, 2001, -124_000, 130_000, 130_000, -124_000), ], ) -def test_get_example(sat_data_source, x, y, left, right, top, bottom): # noqa: D103 - sat_data_source.open() +def test_get_example(optical_flow_data_source, x, y, left, right, top, bottom): # noqa: D103 + optical_flow_data_source.open() t0_dt = pd.Timestamp("2019-01-01T13:00") - sat_data = sat_data_source.get_example(t0_dt=t0_dt, x_meters_center=x, y_meters_center=y) + sat_data = optical_flow_data_source.get_example( + t0_dt=t0_dt, x_meters_center=x, y_meters_center=y + ) assert left == sat_data.x.values[0] assert right == sat_data.x.values[-1] @@ -49,7 +51,7 @@ def test_get_example(sat_data_source, x, y, left, right, top, bottom): # noqa: assert len(sat_data.y) == pytest.IMAGE_SIZE_PIXELS -def test_geospatial_border(sat_data_source): # noqa: D103 - border = sat_data_source.geospatial_border() +def test_geospatial_border(optical_flow_data_source): # noqa: D103 + border = optical_flow_data_source.geospatial_border() correct_border = [(-110000, 1094000), (-110000, -58000), (730000, 1094000), (730000, -58000)] np.testing.assert_array_equal(border, correct_border) From 22939058b8c610a70369d1c9c654c6c79a67a0ac Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 14:49:05 +0000 Subject: [PATCH 020/197] Make new dataarray with the predictions --- .../optical_flow/optical_flow_data_source.py | 62 ++++++++++++++----- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 04723ae3..c5c3236d 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -17,6 +17,8 @@ _LOG = logging.getLogger("nowcasting_dataset") +IMAGE_BUFFER_SIZE = 16 + @dataclass class OpticalFlowDataSource(ZarrDataSource): @@ -33,7 +35,7 @@ class OpticalFlowDataSource(ZarrDataSource): def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): """ Post Init Add 16 pixels to each side of the image""" - super().__post_init__(image_size_pixels + 32, meters_per_pixel) + super().__post_init__(image_size_pixels + 2 * IMAGE_BUFFER_SIZE, meters_per_pixel) n_channels = len(self.channels) self._cache = {} self._shape_of_example = ( @@ -161,27 +163,59 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: prediction_dictionary = {} # Get the previous timestamp previous_timestamp = self._compute_previous_timestep(satellite_data, t0_dt=t0_dt) - for channel in satellite_data.coords["channels"]: - channel_images = satellite_data.sel(channel=channel) - t0_image = channel_images.sel(time=t0_dt).values - previous_image = channel_images.sel(time=previous_timestamp).values - optical_flow = self._compute_optical_flow(t0_image, previous_image) - # Do predictions now + for prediction_timestep in range(self._get_number_future_timesteps(satellite_data, t0_dt)): predictions = [] - # Number of timesteps before t0 - for prediction_timestep in range( - self._get_number_future_timesteps(satellite_data, t0_dt) - ): + for channel in satellite_data.coords["channels"]: + channel_images = satellite_data.sel(channel=channel) + t0_image = channel_images.sel(time=t0_dt).values + previous_image = channel_images.sel(time=previous_timestamp).values + optical_flow = self._compute_optical_flow(t0_image, previous_image) + # Do predictions now flow = optical_flow * prediction_timestep warped_image = self._remap_image(t0_image, flow) warped_image = crop_center( warped_image, self._square.size_pixels, self._square.size_pixels ) predictions.append(warped_image) - prediction_dictionary[channel] = predictions - # TODO Convert to xr.DataArray + # Add the block of predictions for all channels + prediction_dictionary[prediction_timestep] = np.concatenate(predictions, axis=-1) + # Make a block of T, H, W, C ordering + prediction = np.stack( + [prediction_dictionary[k] for k in prediction_dictionary.keys()], axis=0 + ) # Swap out data for the future part of the dataarray - return prediction_dictionary + return prediction + + def _update_dataarray_with_predictions( + self, satellite_data: xr.DataArray, predictions: np.ndarray, t0_dt: pd.Timestamp + ) -> xr.DataArray: + """ + Updates the dataarray with predictions + + Additionally, changes the temporal size to t0+1 to forecast horizon + + Args: + satellite_data: Satellite data + predictions: Predictions from the optical flow + + Returns: + The Xarray dataArray with the optical flow predictions + """ + + # Combine all channels for a single timestep + satellite_data = satellite_data.where(satellite_data.time > t0_dt) + # Make sure its the correct size + satellite_data = satellite_data.isel( + x=slice(0, self._square.size_pixels - IMAGE_BUFFER_SIZE), + y=slice(0, self._square.size_pixels - IMAGE_BUFFER_SIZE), + ) + dataarray = xr.DataArray( + data=predictions, + dims=satellite_data.dims, + coords=satellite_data.coords, + ) + + return dataarray def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: """ From c05b8f38380789b8b4f28386a0d98e31b523bd99 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 14:53:26 +0000 Subject: [PATCH 021/197] Return the correct DataArray --- .../optical_flow/optical_flow_data_source.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index c5c3236d..0bf8f147 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -98,7 +98,9 @@ def get_example( # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions # for all future timesteps for each of them # Compute optical flow per channel, as it might be different - selected_data = self._compute_and_return_optical_flow(selected_data, t0_dt=t0_dt) + selected_data: xr.DataArray = self._compute_and_return_optical_flow( + selected_data, t0_dt=t0_dt + ) if selected_data.shape != self._shape_of_example: raise RuntimeError( @@ -148,7 +150,9 @@ def _get_number_future_timesteps( satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) return len(satellite_data.coords["time"]) - def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp): + def _compute_and_return_optical_flow( + self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp + ) -> xr.DataArray: """ Compute and return optical flow predictions for the example @@ -184,7 +188,10 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray, t0_dt: [prediction_dictionary[k] for k in prediction_dictionary.keys()], axis=0 ) # Swap out data for the future part of the dataarray - return prediction + dataarray = self._update_dataarray_with_predictions( + satellite_data, predictions=prediction, t0_dt=t0_dt + ) + return dataarray def _update_dataarray_with_predictions( self, satellite_data: xr.DataArray, predictions: np.ndarray, t0_dt: pd.Timestamp From ce45ced1082d122fb41d4cd5c1a081203b005229 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 15:02:09 +0000 Subject: [PATCH 022/197] Update test --- tests/data_sources/optical_flow/test_optical_flow_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_model.py b/tests/data_sources/optical_flow/test_optical_flow_model.py index 348d9832..1a61a757 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_model.py +++ b/tests/data_sources/optical_flow/test_optical_flow_model.py @@ -28,4 +28,4 @@ def test_optical_flow_save(): # noqa: D103 with tempfile.TemporaryDirectory() as dirpath: optical_flow_fake().save_netcdf(path=dirpath, batch_i=0) - assert os.path.exists(f"{dirpath}/satellite/000000.nc") + assert os.path.exists(f"{dirpath}/optical_flow/000000.nc") From a3742febbca7b1e82787807b7bf1eba788c8a1ed Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 15:36:09 +0000 Subject: [PATCH 023/197] Fix tests --- .../optical_flow/optical_flow_data_source.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 0bf8f147..3484bddc 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -35,11 +35,11 @@ class OpticalFlowDataSource(ZarrDataSource): def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): """ Post Init Add 16 pixels to each side of the image""" - super().__post_init__(image_size_pixels + 2 * IMAGE_BUFFER_SIZE, meters_per_pixel) + super().__post_init__(image_size_pixels + (2 * IMAGE_BUFFER_SIZE), meters_per_pixel) n_channels = len(self.channels) self._cache = {} self._shape_of_example = ( - self._total_seq_length, + self.forecast_length, image_size_pixels, image_size_pixels, n_channels, @@ -92,6 +92,9 @@ def get_example( selected_data = self._post_process_example(selected_data, t0_dt) + # rename 'variable' to 'channels' + selected_data = selected_data.rename({"variable": "channels"}) + # Compute optical flow for the timesteps # Get Optical Flow for the pre-t0 time, and applying the t0-previous_timesteps_per_flow to # t0 optical flow for forecast steps in the future @@ -113,9 +116,6 @@ def get_example( f"actual shape {selected_data.shape}" ) - # rename 'variable' to 'channels' - selected_data = selected_data.rename({"variable": "channels"}) - return selected_data def _compute_previous_timestep( @@ -131,8 +131,10 @@ def _compute_previous_timestep( Returns: The previous timesteps """ - satellite_data = satellite_data.where(satellite_data.time < t0_dt, drop=True) - return satellite_data.isel(time=-self.previous_timestep_for_flow).values + satellite_data = satellite_data.where(satellite_data.time <= t0_dt, drop=True) + return satellite_data.isel( + time=len(satellite_data.time) - self.previous_timestep_for_flow + ).time.values def _get_number_future_timesteps( self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp @@ -170,7 +172,7 @@ def _compute_and_return_optical_flow( for prediction_timestep in range(self._get_number_future_timesteps(satellite_data, t0_dt)): predictions = [] for channel in satellite_data.coords["channels"]: - channel_images = satellite_data.sel(channel=channel) + channel_images = satellite_data.sel(channels=channel) t0_image = channel_images.sel(time=t0_dt).values previous_image = channel_images.sel(time=previous_timestamp).values optical_flow = self._compute_optical_flow(t0_image, previous_image) @@ -178,7 +180,9 @@ def _compute_and_return_optical_flow( flow = optical_flow * prediction_timestep warped_image = self._remap_image(t0_image, flow) warped_image = crop_center( - warped_image, self._square.size_pixels, self._square.size_pixels + warped_image, + self._square.size_pixels - (2 * IMAGE_BUFFER_SIZE), + self._square.size_pixels - (2 * IMAGE_BUFFER_SIZE), ) predictions.append(warped_image) # Add the block of predictions for all channels @@ -187,6 +191,8 @@ def _compute_and_return_optical_flow( prediction = np.stack( [prediction_dictionary[k] for k in prediction_dictionary.keys()], axis=0 ) + if len(self.channels) == 1: # Only case where another channel needs to be added + prediction = np.expand_dims(prediction, axis=-1) # Swap out data for the future part of the dataarray dataarray = self._update_dataarray_with_predictions( satellite_data, predictions=prediction, t0_dt=t0_dt @@ -210,11 +216,11 @@ def _update_dataarray_with_predictions( """ # Combine all channels for a single timestep - satellite_data = satellite_data.where(satellite_data.time > t0_dt) + satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) # Make sure its the correct size satellite_data = satellite_data.isel( - x=slice(0, self._square.size_pixels - IMAGE_BUFFER_SIZE), - y=slice(0, self._square.size_pixels - IMAGE_BUFFER_SIZE), + x=slice(IMAGE_BUFFER_SIZE, self._square.size_pixels - IMAGE_BUFFER_SIZE), + y=slice(IMAGE_BUFFER_SIZE, self._square.size_pixels - IMAGE_BUFFER_SIZE), ) dataarray = xr.DataArray( data=predictions, @@ -358,9 +364,7 @@ def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: # seems to slow things down a lot if the Zarr store has more than # about a million chunks. # See https://github.com/openclimatefix/nowcasting_dataset/issues/23 - dataset = xr.open_dataset( - zarr_path, engine="zarr", consolidated=consolidated, mode="r", chunks=None - ) + dataset = xr.open_dataset(zarr_path, engine="zarr", mode="r", chunks=None) data_array = dataset["stacked_eumetsat_data"] del dataset From 6340093432931d680960c2995c3ee400dd23ccb2 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 15:52:26 +0000 Subject: [PATCH 024/197] Fix test path --- tests/data_sources/optical_flow/test_optical_flow_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_model.py b/tests/data_sources/optical_flow/test_optical_flow_model.py index 1a61a757..5eb405a9 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_model.py +++ b/tests/data_sources/optical_flow/test_optical_flow_model.py @@ -20,7 +20,7 @@ def test_optical_flow_validation(): # noqa: D103 sat.data[0, 0] = np.nan with pytest.raises(Exception): - optical_flow_fake.model_validation(sat) + OpticalFlow.model_validation(sat) def test_optical_flow_save(): # noqa: D103 @@ -28,4 +28,4 @@ def test_optical_flow_save(): # noqa: D103 with tempfile.TemporaryDirectory() as dirpath: optical_flow_fake().save_netcdf(path=dirpath, batch_i=0) - assert os.path.exists(f"{dirpath}/optical_flow/000000.nc") + assert os.path.exists(f"{dirpath}/opticalflow/000000.nc") From 919d4d2916a733ef6e714a46b9bc09119a29bdad Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 15:56:05 +0000 Subject: [PATCH 025/197] Minor docstring fixes --- .../data_sources/optical_flow/optical_flow_data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 3484bddc..ea4e7cb3 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -212,7 +212,7 @@ def _update_dataarray_with_predictions( predictions: Predictions from the optical flow Returns: - The Xarray dataArray with the optical flow predictions + The Xarray DataArray with the optical flow predictions """ # Combine all channels for a single timestep @@ -239,7 +239,7 @@ def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray previous_image: previous image to compute optical flow with Returns: - optical flow field + Optical Flow field """ return cv2.calcOpticalFlowFarneback( prev=previous_image, From ed42e8ab456cc0561fd5a2959d7cb34f27485322 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 16:04:35 +0000 Subject: [PATCH 026/197] Address PR comments --- .../optical_flow/optical_flow_data_source.py | 115 +----------------- .../satellite/satellite_data_source.py | 1 + 2 files changed, 3 insertions(+), 113 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index ea4e7cb3..31f5b338 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -9,11 +9,10 @@ import pandas as pd import xarray as xr -import nowcasting_dataset.time as nd_time from nowcasting_dataset.consts import SAT_VARIABLE_NAMES -from nowcasting_dataset.data_sources.data_source import ZarrDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow +from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource _LOG = logging.getLogger("nowcasting_dataset") @@ -21,7 +20,7 @@ @dataclass -class OpticalFlowDataSource(ZarrDataSource): +class OpticalFlowDataSource(SatelliteDataSource): """ Optical Flow Data Source, computing flow between Satellite data @@ -45,21 +44,6 @@ def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): n_channels, ) - def open(self) -> None: - """ - Open Satellite data - - We don't want to open_sat_data in __init__. - If we did that, then we couldn't copy SatelliteDataSource - instances into separate processes. Instead, - call open() _after_ creating separate processes. - """ - self._data = self._open_data() - self._data = self._data.sel(variable=list(self.channels)) - - def _open_data(self) -> xr.DataArray: - return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) - def get_example( self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number ) -> DataSourceOutput: @@ -282,101 +266,6 @@ def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: def _dataset_to_data_source_output(output: xr.Dataset) -> OpticalFlow: return OpticalFlow(output) - def _get_time_slice(self, t0_dt: pd.Timestamp) -> xr.DataArray: - start_dt = self._get_start_dt(t0_dt) - end_dt = self._get_end_dt(t0_dt) - data = self.data.sel(time=slice(start_dt, end_dt)) - return data - - def datetime_index(self, remove_night: bool = True) -> pd.DatetimeIndex: - """Returns a complete list of all available datetimes - - Args: - remove_night: If True then remove datetimes at night. - We're interested in forecasting solar power generation, so we - don't care about nighttime data :) - - In the UK in summer, the sun rises first in the north east, and - sets last in the north west [1]. In summer, the north gets more - hours of sunshine per day. - - In the UK in winter, the sun rises first in the south east, and - sets last in the south west [2]. In winter, the south gets more - hours of sunshine per day. - - | | Summer | Winter | - | ---: | :---: | :---: | - | Sun rises first in | N.E. | S.E. | - | Sun sets last in | N.W. | S.W. | - | Most hours of sunlight | North | South | - - Before training, we select timesteps which have at least some - sunlight. We do this by computing the clearsky global horizontal - irradiance (GHI) for the four corners of the satellite imagery, - and for all the timesteps in the dataset. We only use timesteps - where the maximum global horizontal irradiance across all four - corners is above some threshold. - - The 'clearsky solar irradiance' is the amount of sunlight we'd - expect on a clear day at a specific time and location. The SI unit - of irradiance is watt per square meter. The 'global horizontal - irradiance' (GHI) is the total sunlight that would hit a - horizontal surface on the surface of the Earth. The GHI is the - sum of the direct irradiance (sunlight which takes a direct path - from the Sun to the Earth's surface) and the diffuse horizontal - irradiance (the sunlight scattered from the atmosphere). For more - info, see: https://en.wikipedia.org/wiki/Solar_irradiance - - References: - 1. [Video of June 2019](https://www.youtube.com/watch?v=IOp-tj-IJpk) - 2. [Video of Jan 2019](https://www.youtube.com/watch?v=CJ4prUVa2nQ) - """ - if self._data is None: - sat_data = self._open_data() - else: - sat_data = self._data - - datetime_index = pd.DatetimeIndex(sat_data.time.values) - - if remove_night: - border_locations = self.geospatial_border() - datetime_index = nd_time.select_daylight_datetimes( - datetimes=datetime_index, locations=border_locations - ) - - return datetime_index - - -def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: - """Lazily opens the Zarr store. - - Adds 1 minute to the 'time' coordinates, so the timestamps - are at 00, 05, ..., 55 past the hour. - - Args: - zarr_path: Cloud URL or local path. If GCP URL, must start with 'gs://' - consolidated: Whether or not the Zarr metadata is consolidated. - """ - _LOG.debug("Opening satellite data: %s", zarr_path) - - # We load using chunks=None so xarray *doesn't* use Dask to - # load the Zarr chunks from disk. Using Dask to load the data - # seems to slow things down a lot if the Zarr store has more than - # about a million chunks. - # See https://github.com/openclimatefix/nowcasting_dataset/issues/23 - dataset = xr.open_dataset(zarr_path, engine="zarr", mode="r", chunks=None) - - data_array = dataset["stacked_eumetsat_data"] - del dataset - - # The 'time' dimension is at 04, 09, ..., 59 minutes past the hour. - # To make it easier to align the satellite data with other data sources - # (which are at 00, 05, ..., 55 minutes past the hour) we add 1 minute to - # the time dimension. - # TODO Remove this as new Zarr already has the time fixed - data_array["time"] = data_array.time + pd.Timedelta("1 minute") - return data_array - def crop_center(img, cropx, cropy): """ diff --git a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py index f8254f16..afdc4260 100644 --- a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py +++ b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py @@ -146,5 +146,6 @@ def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: # (which are at 00, 05, ..., 55 minutes past the hour) we add 1 minute to # the time dimension. # TODO Remove this as new Zarr already has the time fixed + # See https://github.com/openclimatefix/nowcasting_dataset/issues/313 data_array["time"] = data_array.time + pd.Timedelta("1 minute") return data_array From fa2330c79c86ef189dc29cab6a2f3f85aa3bb0e1 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 3 Nov 2021 16:06:12 +0000 Subject: [PATCH 027/197] Fix from rebase --- nowcasting_dataset/data_sources/fake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index 87722a36..de41e69a 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -132,7 +132,7 @@ def optical_flow_fake( create_image_array( seq_length_5=seq_length_5, image_size_pixels=satellite_image_size_pixels, - number_channels=number_satellite_channels, + channels=SAT_VARIABLE_NAMES[0:number_satellite_channels], ) for _ in range(batch_size) ] From c3d11199a6c7f80e22537555f0a5968f0274a696 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Mon, 8 Nov 2021 13:44:36 +0000 Subject: [PATCH 028/197] Add docstring --- .../optical_flow/optical_flow_data_source.py | 285 ------------------ .../data_sources/transforms/__init__.py | 1 + .../data_sources/transforms/base.py | 30 ++ .../data_sources/transforms/optical_flow.py | 279 +++++++++++++++++ 4 files changed, 310 insertions(+), 285 deletions(-) delete mode 100644 nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py create mode 100644 nowcasting_dataset/data_sources/transforms/__init__.py create mode 100644 nowcasting_dataset/data_sources/transforms/base.py create mode 100644 nowcasting_dataset/data_sources/transforms/optical_flow.py diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py deleted file mode 100644 index 31f5b338..00000000 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ /dev/null @@ -1,285 +0,0 @@ -""" Optical Flow Data Source """ -import logging -from dataclasses import InitVar, dataclass -from numbers import Number -from typing import Iterable, Optional - -import cv2 -import numpy as np -import pandas as pd -import xarray as xr - -from nowcasting_dataset.consts import SAT_VARIABLE_NAMES -from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput -from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow -from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource - -_LOG = logging.getLogger("nowcasting_dataset") - -IMAGE_BUFFER_SIZE = 16 - - -@dataclass -class OpticalFlowDataSource(SatelliteDataSource): - """ - Optical Flow Data Source, computing flow between Satellite data - - Pads image size to allow for cropping out NaN values - """ - - channels: Optional[Iterable[str]] = SAT_VARIABLE_NAMES - previous_timestep_for_flow: int = 1 - image_size_pixels: InitVar[int] = 128 - meters_per_pixel: InitVar[int] = 2_000 - - def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): - """ Post Init Add 16 pixels to each side of the image""" - super().__post_init__(image_size_pixels + (2 * IMAGE_BUFFER_SIZE), meters_per_pixel) - n_channels = len(self.channels) - self._cache = {} - self._shape_of_example = ( - self.forecast_length, - image_size_pixels, - image_size_pixels, - n_channels, - ) - - def get_example( - self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number - ) -> DataSourceOutput: - """ - Get Optical Flow Example data - - Args: - t0_dt: list of timestamps for the datetime of the batches. The batch will also include - data for historic and future depending on `history_minutes` and `future_minutes`. - x_meters_center: x center batch locations - y_meters_center: y center batch locations - - Returns: Example Data - - """ - selected_data = self._get_time_slice(t0_dt) - bounding_box = self._square.bounding_box_centered_on( - x_meters_center=x_meters_center, y_meters_center=y_meters_center - ) - selected_data = selected_data.sel( - x=slice(bounding_box.left, bounding_box.right), - y=slice(bounding_box.top, bounding_box.bottom), - ) - - # selected_sat_data is likely to have 1 too many pixels in x and y - # because sel(x=slice(a, b)) is [a, b], not [a, b). So trim: - selected_data = selected_data.isel( - x=slice(0, self._square.size_pixels), y=slice(0, self._square.size_pixels) - ) - - selected_data = self._post_process_example(selected_data, t0_dt) - - # rename 'variable' to 'channels' - selected_data = selected_data.rename({"variable": "channels"}) - - # Compute optical flow for the timesteps - # Get Optical Flow for the pre-t0 time, and applying the t0-previous_timesteps_per_flow to - # t0 optical flow for forecast steps in the future - # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions - # for all future timesteps for each of them - # Compute optical flow per channel, as it might be different - selected_data: xr.DataArray = self._compute_and_return_optical_flow( - selected_data, t0_dt=t0_dt - ) - - if selected_data.shape != self._shape_of_example: - raise RuntimeError( - "Example is wrong shape! " - f"x_meters_center={x_meters_center}\n" - f"y_meters_center={y_meters_center}\n" - f"t0_dt={t0_dt}\n" - f"times are {selected_data.time}\n" - f"expected shape={self._shape_of_example}\n" - f"actual shape {selected_data.shape}" - ) - - return selected_data - - def _compute_previous_timestep( - self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp - ) -> pd.Timestamp: - """ - Get timestamp of previous - - Args: - satellite_data: Satellite data to use - t0_dt: Timestamp - - Returns: - The previous timesteps - """ - satellite_data = satellite_data.where(satellite_data.time <= t0_dt, drop=True) - return satellite_data.isel( - time=len(satellite_data.time) - self.previous_timestep_for_flow - ).time.values - - def _get_number_future_timesteps( - self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp - ) -> int: - """ - Get number of future timestamps - - Args: - satellite_data: Satellite data to use - t0_dt: The timestamp of the t0 image - - Returns: - The number of future timesteps - """ - satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) - return len(satellite_data.coords["time"]) - - def _compute_and_return_optical_flow( - self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp - ) -> xr.DataArray: - """ - Compute and return optical flow predictions for the example - - Args: - satellite_data: Satellite DataArray - t0_dt: t0 timestamp - - Returns: - The xr.DataArray with the optical flow predictions for t0 to forecast horizon - """ - - prediction_dictionary = {} - # Get the previous timestamp - previous_timestamp = self._compute_previous_timestep(satellite_data, t0_dt=t0_dt) - for prediction_timestep in range(self._get_number_future_timesteps(satellite_data, t0_dt)): - predictions = [] - for channel in satellite_data.coords["channels"]: - channel_images = satellite_data.sel(channels=channel) - t0_image = channel_images.sel(time=t0_dt).values - previous_image = channel_images.sel(time=previous_timestamp).values - optical_flow = self._compute_optical_flow(t0_image, previous_image) - # Do predictions now - flow = optical_flow * prediction_timestep - warped_image = self._remap_image(t0_image, flow) - warped_image = crop_center( - warped_image, - self._square.size_pixels - (2 * IMAGE_BUFFER_SIZE), - self._square.size_pixels - (2 * IMAGE_BUFFER_SIZE), - ) - predictions.append(warped_image) - # Add the block of predictions for all channels - prediction_dictionary[prediction_timestep] = np.concatenate(predictions, axis=-1) - # Make a block of T, H, W, C ordering - prediction = np.stack( - [prediction_dictionary[k] for k in prediction_dictionary.keys()], axis=0 - ) - if len(self.channels) == 1: # Only case where another channel needs to be added - prediction = np.expand_dims(prediction, axis=-1) - # Swap out data for the future part of the dataarray - dataarray = self._update_dataarray_with_predictions( - satellite_data, predictions=prediction, t0_dt=t0_dt - ) - return dataarray - - def _update_dataarray_with_predictions( - self, satellite_data: xr.DataArray, predictions: np.ndarray, t0_dt: pd.Timestamp - ) -> xr.DataArray: - """ - Updates the dataarray with predictions - - Additionally, changes the temporal size to t0+1 to forecast horizon - - Args: - satellite_data: Satellite data - predictions: Predictions from the optical flow - - Returns: - The Xarray DataArray with the optical flow predictions - """ - - # Combine all channels for a single timestep - satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) - # Make sure its the correct size - satellite_data = satellite_data.isel( - x=slice(IMAGE_BUFFER_SIZE, self._square.size_pixels - IMAGE_BUFFER_SIZE), - y=slice(IMAGE_BUFFER_SIZE, self._square.size_pixels - IMAGE_BUFFER_SIZE), - ) - dataarray = xr.DataArray( - data=predictions, - dims=satellite_data.dims, - coords=satellite_data.coords, - ) - - return dataarray - - def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: - """ - Compute the optical flow for a set of images - - Args: - t0_image: t0 image - previous_image: previous image to compute optical flow with - - Returns: - Optical Flow field - """ - return cv2.calcOpticalFlowFarneback( - prev=previous_image, - next=t0_image, - flow=None, - pyr_scale=0.5, - levels=2, - winsize=40, - iterations=3, - poly_n=5, - poly_sigma=0.7, - flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, - ) - - def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: - """ - Takes an image and warps it forwards in time according to the flow field. - - Args: - image: The grayscale image to warp. - flow: A 3D array. The first two dimensions must be the same size as the first two - dimensions of the image. The third dimension represented the x and y displacement. - - Returns: Warped image. The border has values np.NaN. - """ - # Adapted from https://github.com/opencv/opencv/issues/11068 - height, width = flow.shape[:2] - remap = -flow.copy() - remap[..., 0] += np.arange(width) # map_x - remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y - return cv2.remap( - src=image, - map1=remap, - map2=None, - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=np.NaN, - ) - - def _dataset_to_data_source_output(output: xr.Dataset) -> OpticalFlow: - return OpticalFlow(output) - - -def crop_center(img, cropx, cropy): - """ - Crop center of numpy image - - Args: - img: Image to crop - cropx: Size in x direction - cropy: Size in y direction - - Returns: - The cropped image - """ - y, x = img.shape - startx = x // 2 - (cropx // 2) - starty = y // 2 - (cropy // 2) - return img[starty : starty + cropy, startx : startx + cropx] diff --git a/nowcasting_dataset/data_sources/transforms/__init__.py b/nowcasting_dataset/data_sources/transforms/__init__.py new file mode 100644 index 00000000..b5e5438f --- /dev/null +++ b/nowcasting_dataset/data_sources/transforms/__init__.py @@ -0,0 +1 @@ +"""Set of transforms for creating derived data sources from other data sources""" diff --git a/nowcasting_dataset/data_sources/transforms/base.py b/nowcasting_dataset/data_sources/transforms/base.py new file mode 100644 index 00000000..1d4f2dd9 --- /dev/null +++ b/nowcasting_dataset/data_sources/transforms/base.py @@ -0,0 +1,30 @@ +"""Generic Transform class""" + +from dataclasses import dataclass +from typing import List + +from nowcasting_dataset.data_sources.data_source import DataSource +from nowcasting_dataset.dataset.batch import Batch + + +@dataclass +class Transform: + """Abstract base class. + + Attributes: + data_sources: List of data sources that this transform will be applied to + """ + + data_sources: List[DataSource] + + def apply_transform(self, batch: Batch) -> Batch: + """ + Apply transform to the Batch, returning the Batch with added/transformed data + + Args: + batch: Batch consisting of the data to transform + + Returns: + Batch with the transformed data + """ + return batch diff --git a/nowcasting_dataset/data_sources/transforms/optical_flow.py b/nowcasting_dataset/data_sources/transforms/optical_flow.py new file mode 100644 index 00000000..1335f7a7 --- /dev/null +++ b/nowcasting_dataset/data_sources/transforms/optical_flow.py @@ -0,0 +1,279 @@ +"""Functions for computing the optical flow on the fly for satellite images""" +import logging +from typing import Optional + +import cv2 +import numpy as np +import pandas as pd +import xarray as xr + +from nowcasting_dataset.data_sources.transforms.transform import Transform +from nowcasting_dataset.dataset.batch import Batch + +_LOG = logging.getLogger("nowcasting_dataset") + + +class OpticalFlowTransform(Transform): + """ + Optical Flow Transform that adds optical flow images + + """ + + final_image_size_pixels: Optional[int] = None + + def apply_transform(self, batch: Batch) -> Batch: + """ + Calculate optical flow for the batch, and add to Batch + + Args: + batch: Batch containing satellite data for optical flow + + Returns: + Batch with optical flow added + """ + batch.optical_flow = compute_optical_flow_for_batch(batch) + return batch + + +def compute_optical_flow_for_batch( + batch: Batch, final_image_size_pixels: Optional[int] = None +) -> xr.DataArray: + """ + Computes the optical flow for satellite images in the batch + + Assumes metadata is also in Batch, for getting t0 + + Args: + batch: Batch containing at least metadata and satellite data + + Returns: + Tensor containing the Optical Flow predictions + """ + + assert ( + batch.satellite is not None + ), "Satellite data does not exist in batch, required for optical flow" + assert batch.metadata is not None, "Metadata does not exist in batch, required for optical flow" + + if final_image_size_pixels is None: + final_image_size_pixels = len(batch.satellite.x_index) + + # Only do optical flow for satellite data + optical_flow_predictions = [] + for i in range(batch.batch_size): + satellite_data: xr.DataArray = batch.satellite.sel(example=i) + t0_dt = batch.metadata.t0_dt.values[i] + optical_flow_predictions.append( + _compute_and_return_optical_flow( + satellite_data, t0_dt=t0_dt, final_image_size_pixels=final_image_size_pixels + ) + ) + # Concatenate all the DataArrays + dataarray = xr.concat(optical_flow_predictions, dim="example") + return dataarray + + +def _update_dataarray_with_predictions( + satellite_data: xr.DataArray, + predictions: np.ndarray, + t0_dt: pd.Timestamp, + final_image_size_pixels: int, +) -> xr.DataArray: + """ + Updates the dataarray with predictions + + Additionally, changes the temporal size to t0+1 to forecast horizon + + Args: + satellite_data: Satellite data + predictions: Predictions from the optical flow + + Returns: + The Xarray DataArray with the optical flow predictions + """ + + # Combine all channels for a single timestep + satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) + # Make sure its the correct size + buffer = satellite_data.sizes["x"] - final_image_size_pixels // 2 + satellite_data = satellite_data.isel( + x=slice(buffer, satellite_data.sizes["x"] - buffer), + y=slice(buffer, satellite_data.sizes["y"] - buffer), + ) + dataarray = xr.DataArray( + data=predictions, + dims=satellite_data.dims, + coords=satellite_data.coords, + ) + + return dataarray + + +def _get_previous_timesteps( + satellite_data: xr.DataArray, + t0_dt: pd.Timestamp, +) -> xr.DataArray: + """ + Get timestamp of previous + + Args: + satellite_data: Satellite data to use + t0_dt: Timestamp + + Returns: + The previous timesteps + """ + satellite_data = satellite_data.where(satellite_data.time <= t0_dt, drop=True) + return satellite_data + + +def _get_number_future_timesteps(satellite_data: xr.DataArray, t0_dt: pd.Timestamp) -> int: + """ + Get number of future timestamps + + Args: + satellite_data: Satellite data to use + t0_dt: The timestamp of the t0 image + + Returns: + The number of future timesteps + """ + satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) + return len(satellite_data.coords["time_index"]) + + +def _compute_and_return_optical_flow( + satellite_data: xr.DataArray, + t0_dt: pd.Timestamp, + final_image_size_pixels: int, +) -> xr.DataArray: + """ + Compute and return optical flow predictions for the example + + Args: + satellite_data: Satellite DataArray + t0_dt: t0 timestamp + + Returns: + The Tensor with the optical flow predictions for t0 to forecast horizon + """ + + # Get the previous timestamp + future_timesteps = _get_number_future_timesteps(satellite_data, t0_dt) + satellite_data: xr.DataArray = _get_previous_timesteps( + satellite_data, + t0_dt=t0_dt, + ) + prediction_block = np.zeros( + ( + future_timesteps, + final_image_size_pixels, + final_image_size_pixels, + satellite_data.sizes["channels_index"], + ) + ) + for prediction_timestep in range(future_timesteps): + for channel in range(0, len(satellite_data.coords["channels_index"]), 4): + # Optical Flow works with RGB images, so chunking channels for it to be faster + channel_images = satellite_data.sel(channels_index=slice(channel, channel + 3)) + # Extra 1 in shape from time dimension, so removing that dimension + t0_image = channel_images.isel( + time_index=len(satellite_data.time_index) - 1 + ).data.values + previous_image = channel_images.isel( + time_index=len(satellite_data.time_index) - 2 + ).data.values + optical_flow = _compute_optical_flow(t0_image, previous_image) + # Do predictions now + flow = optical_flow * prediction_timestep + 1 # Otherwise first prediction would be 0 + warped_image = _remap_image(t0_image, flow) + warped_image = crop_center( + warped_image, + final_image_size_pixels, + final_image_size_pixels, + ) + prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image + # Convert to correct C, T, H, W order + prediction_block = np.permute(prediction_block, [3, 0, 1, 2]) + dataarray = _update_dataarray_with_predictions( + satellite_data=satellite_data, predictions=prediction_block, t0_dt=t0_dt + ) + return dataarray + + +def _compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: + """ + Compute the optical flow for a set of images + + Args: + t0_image: t0 image + previous_image: previous image to compute optical flow with + + Returns: + Optical Flow field + """ + # Input images have to be single channel and between 0 and 1 + image_min = np.min([t0_image, previous_image]) + image_max = np.max([t0_image, previous_image]) + t0_image -= image_min + t0_image /= image_max + previous_image -= image_min + previous_image /= image_max + t0_image = cv2.cvtColor(t0_image.astype(np.float32), cv2.COLOR_RGBA2GRAY) + previous_image = cv2.cvtColor(previous_image.astype(np.float32), cv2.COLOR_RGBA2GRAY) + return cv2.calcOpticalFlowFarneback( + prev=previous_image, + next=t0_image, + flow=None, + pyr_scale=0.5, + levels=2, + winsize=40, + iterations=3, + poly_n=5, + poly_sigma=0.7, + flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, + ) + + +def _remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: + """ + Takes an image and warps it forwards in time according to the flow field. + + Args: + image: The grayscale image to warp. + flow: A 3D array. The first two dimensions must be the same size as the first two + dimensions of the image. The third dimension represented the x and y displacement. + + Returns: Warped image. The border has values np.NaN. + """ + # Adapted from https://github.com/opencv/opencv/issues/11068 + height, width = flow.shape[:2] + remap = -flow.copy() + remap[..., 0] += np.arange(width) # map_x + remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y + return cv2.remap( + src=image, + map1=remap, + map2=None, + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=np.NaN, + ) + + +def crop_center(image, x_size, y_size): + """ + Crop center of numpy image + + Args: + image: Image to crop + x_size: Size in x direction + y_size: Size in y direction + + Returns: + The cropped image + """ + y, x, channels = image.shape + startx = x // 2 - (x_size // 2) + starty = y // 2 - (y_size // 2) + return image[starty : starty + y_size, startx : startx + x_size] From 00af8b2e7d8706c4939f3596abdd93bf14226c89 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Nov 2021 10:39:20 +0000 Subject: [PATCH 029/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/data_sources/fake.py | 2 +- .../data_sources/optical_flow/optical_flow_model.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index de41e69a..b070c5fc 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -126,7 +126,7 @@ def optical_flow_fake( satellite_image_size_pixels=64, number_satellite_channels=7, ) -> OpticalFlow: - """ Create fake data """ + """Create fake data""" # make batch of arrays xr_arrays = [ create_image_array( diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py index 58e504f4..9cf7f2df 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py @@ -7,14 +7,14 @@ class OpticalFlow(DataSourceOutput): - """ Class to store optical flow data as a xr.Dataset with some validation """ + """Class to store optical flow data as a xr.Dataset with some validation""" __slots__ = () _expected_dimensions = ("time", "x", "y", "channels") @classmethod def model_validation(cls, v): - """ Check that all values are not NaN, Infinite, or -1.""" + """Check that all values are not NaN, Infinite, or -1.""" assert (~isnan(v.data)).all(), "Some optical flow data values are NaNs" assert (~isinf(v.data)).all(), "Some optical flow data values are Infinite" assert (v.data != -1).all(), "Some optical flow data values are -1's" From 35968b709cec88868b081e3f0002302e1cfda830 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Tue, 9 Nov 2021 15:38:52 +0000 Subject: [PATCH 030/197] Readd OpticalFlowDataSource New plan is to have a set of DerivedDataSources that are computed using the pre-made batches. So the pre-made batch is then read and a new batch for the new data source is then added --- .../data_sources/data_source.py | 5 + .../optical_flow/optical_flow_data_source.py | 384 ++++++++++++++++++ .../data_sources/transforms/base.py | 31 +- .../data_sources/transforms/optical_flow.py | 5 +- nowcasting_dataset/dataset/batch.py | 3 +- nowcasting_dataset/manager.py | 68 ++++ requirements.txt | 1 - scripts/get_raw_eumetsat_data.py | 86 ---- scripts/prepare_ml_data.py | 1 + 9 files changed, 489 insertions(+), 95 deletions(-) create mode 100644 nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py delete mode 100644 scripts/get_raw_eumetsat_data.py diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 4d1e336e..e3769cb6 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -16,6 +16,7 @@ from nowcasting_dataset import square from nowcasting_dataset.consts import SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput +from nowcasting_dataset.data_sources.transforms.base import Transform from nowcasting_dataset.dataset.xr_utils import join_list_dataset_to_batch_dataset, make_dim_index logger = logging.getLogger(__name__) @@ -42,6 +43,7 @@ class DataSource: history_minutes: int forecast_minutes: int + transform: Transform def __post_init__(self): """Post Init""" @@ -202,6 +204,9 @@ def create_batches( y_locations=locations_for_batch.y_center_OSGB, ) + # Run transforms on batch + batch: DataSourceOutput = self.transform.apply_transforms(batch) + # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) batch.to_netcdf(netcdf_filename) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py new file mode 100644 index 00000000..b5f0b801 --- /dev/null +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -0,0 +1,384 @@ +""" Optical Flow Data Source """ +import logging +from concurrent import futures +from dataclasses import InitVar, dataclass +from numbers import Number +from typing import Iterable, Optional + +import cv2 +import numpy as np +import pandas as pd +import xarray as xr + +import nowcasting_dataset.time as nd_time +from nowcasting_dataset.data_sources.data_source import ZarrDataSource +from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput +from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow +from nowcasting_dataset.dataset.xr_utils import join_list_data_array_to_batch_dataset + +_LOG = logging.getLogger("nowcasting_dataset") + + +@dataclass +class OpticalFlowDataSource(ZarrDataSource): + """ + Optical Flow Data Source, computing flow between Satellite data + + zarr_path: Must start with 'gs://' if on GCP. + """ + + zarr_path: str = None + previous_timestep_for_flow: int = 1 + image_size_pixels: InitVar[int] = 128 + meters_per_pixel: InitVar[int] = 2_000 + + def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): + """Post Init""" + super().__post_init__(image_size_pixels, meters_per_pixel) + self._cache = {} + self._shape_of_example = ( + self._total_seq_length, + image_size_pixels, + image_size_pixels, + 2, + ) + + def open(self) -> None: + """ + Open Satellite data + + We don't want to open_sat_data in __init__. + If we did that, then we couldn't copy SatelliteDataSource + instances into separate processes. Instead, + call open() _after_ creating separate processes. + """ + self._data = self._open_data() + self._data = self._data.sel(variable=list(self.channels)) + + def _open_data(self) -> xr.DataArray: + return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) + + def get_batch( + self, + t0_datetimes: pd.DatetimeIndex, + x_locations: Iterable[Number], + y_locations: Iterable[Number], + ) -> OpticalFlow: + """ + Get batch data + + Load the first _n_timesteps_per_batch concurrently. This + loads the timesteps from disk concurrently, and fills the + cache. If we try loading all examples + concurrently, then SatelliteDataSource will try reading from + empty caches, and things are much slower! + + Args: + t0_datetimes: list of timestamps for the datetime of the batches. The batch will also + include data for historic and future depending on `history_minutes` and + `future_minutes`. + x_locations: x center batch locations + y_locations: y center batch locations + + Returns: Batch data + + """ + # Load the first _n_timesteps_per_batch concurrently. This + # loads the timesteps from disk concurrently, and fills the + # cache. If we try loading all examples + # concurrently, then SatelliteDataSource will try reading from + # empty caches, and things are much slower! + zipped = list(zip(t0_datetimes, x_locations, y_locations)) + batch_size = len(t0_datetimes) + + with futures.ThreadPoolExecutor(max_workers=batch_size) as executor: + future_examples = [] + for coords in zipped[: self.n_timesteps_per_batch]: + t0_datetime, x_location, y_location = coords + future_example = executor.submit( + self.get_example, t0_datetime, x_location, y_location + ) + future_examples.append(future_example) + examples = [future_example.result() for future_example in future_examples] + + # Load the remaining examples. This should hit the DataSource caches. + for coords in zipped[self.n_timesteps_per_batch :]: + t0_datetime, x_location, y_location = coords + example = self.get_example(t0_datetime, x_location, y_location) + examples.append(example) + + output = join_list_data_array_to_batch_dataset(examples) + + self._cache = {} + + return OpticalFlow(output) + + def get_example( + self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number + ) -> DataSourceOutput: + """ + Get Optical Flow Example data + + Args: + t0_dt: list of timestamps for the datetime of the batches. The batch will also include + data for historic and future depending on `history_minutes` and `future_minutes`. + x_meters_center: x center batch locations + y_meters_center: y center batch locations + + Returns: Example Data + + """ + selected_data = self._get_time_slice(t0_dt) + bounding_box = self._square.bounding_box_centered_on( + x_meters_center=x_meters_center, y_meters_center=y_meters_center + ) + selected_data = selected_data.sel( + x=slice(bounding_box.left, bounding_box.right), + y=slice(bounding_box.top, bounding_box.bottom), + ) + + # selected_sat_data is likely to have 1 too many pixels in x and y + # because sel(x=slice(a, b)) is [a, b], not [a, b). So trim: + selected_data = selected_data.isel( + x=slice(0, self._square.size_pixels), y=slice(0, self._square.size_pixels) + ) + + selected_data = self._post_process_example(selected_data, t0_dt) + + if selected_data.shape != self._shape_of_example: + raise RuntimeError( + "Example is wrong shape! " + f"x_meters_center={x_meters_center}\n" + f"y_meters_center={y_meters_center}\n" + f"t0_dt={t0_dt}\n" + f"times are {selected_data.time}\n" + f"expected shape={self._shape_of_example}\n" + f"actual shape {selected_data.shape}" + ) + + # rename 'variable' to 'channels' + selected_data = selected_data.rename({"variable": "channels"}) + + # Compute optical flow for the timesteps + # Get Optical Flow for the pre-t0 time, and applying the t0-previous_timesteps_per_flow to + # t0 optical flow for forecast steps in the future + # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions + # for all future timesteps for each of them + # Compute optical flow per channel, as it might be different + selected_data = self._compute_and_return_optical_flow(selected_data, t0_dt=t0_dt) + + return selected_data + + def _update_dataarray_with_predictions( + self, + satellite_data: xr.DataArray, + predictions: np.ndarray, + t0_dt: pd.Timestamp, + final_image_size_pixels: int, + ) -> xr.DataArray: + """ + Updates the dataarray with predictions + + Additionally, changes the temporal size to t0+1 to forecast horizon + + Args: + satellite_data: Satellite data + predictions: Predictions from the optical flow + + Returns: + The Xarray DataArray with the optical flow predictions + """ + + # Combine all channels for a single timestep + satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) + # Make sure its the correct size + buffer = satellite_data.sizes["x"] - final_image_size_pixels // 2 + satellite_data = satellite_data.isel( + x=slice(buffer, satellite_data.sizes["x"] - buffer), + y=slice(buffer, satellite_data.sizes["y"] - buffer), + ) + dataarray = xr.DataArray( + data=predictions, + dims=satellite_data.dims, + coords=satellite_data.coords, + ) + + return dataarray + + def _get_previous_timesteps( + self, + satellite_data: xr.DataArray, + t0_dt: pd.Timestamp, + ) -> xr.DataArray: + """ + Get timestamp of previous + + Args: + satellite_data: Satellite data to use + t0_dt: Timestamp + + Returns: + The previous timesteps + """ + satellite_data = satellite_data.where(satellite_data.time <= t0_dt, drop=True) + return satellite_data + + def _get_number_future_timesteps( + self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp + ) -> int: + """ + Get number of future timestamps + + Args: + satellite_data: Satellite data to use + t0_dt: The timestamp of the t0 image + + Returns: + The number of future timesteps + """ + satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) + return len(satellite_data.coords["time_index"]) + + def _compute_and_return_optical_flow( + self, + satellite_data: xr.DataArray, + t0_dt: pd.Timestamp, + final_image_size_pixels: int, + ) -> xr.DataArray: + """ + Compute and return optical flow predictions for the example + + Args: + satellite_data: Satellite DataArray + t0_dt: t0 timestamp + + Returns: + The Tensor with the optical flow predictions for t0 to forecast horizon + """ + + # Get the previous timestamp + future_timesteps = _get_number_future_timesteps(satellite_data, t0_dt) + satellite_data: xr.DataArray = _get_previous_timesteps( + satellite_data, + t0_dt=t0_dt, + ) + prediction_block = np.zeros( + ( + future_timesteps, + final_image_size_pixels, + final_image_size_pixels, + satellite_data.sizes["channels_index"], + ) + ) + for prediction_timestep in range(future_timesteps): + for channel in range(0, len(satellite_data.coords["channels_index"]), 4): + # Optical Flow works with RGB images, so chunking channels for it to be faster + channel_images = satellite_data.sel(channels_index=slice(channel, channel + 3)) + # Extra 1 in shape from time dimension, so removing that dimension + t0_image = channel_images.isel( + time_index=len(satellite_data.time_index) - 1 + ).data.values + previous_image = channel_images.isel( + time_index=len(satellite_data.time_index) - 2 + ).data.values + optical_flow = _compute_optical_flow(t0_image, previous_image) + # Do predictions now + flow = ( + optical_flow * prediction_timestep + 1 + ) # Otherwise first prediction would be 0 + warped_image = _remap_image(t0_image, flow) + warped_image = crop_center( + warped_image, + final_image_size_pixels, + final_image_size_pixels, + ) + prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image + # Convert to correct C, T, H, W order + prediction_block = np.permute(prediction_block, [3, 0, 1, 2]) + dataarray = _update_dataarray_with_predictions( + satellite_data=satellite_data, predictions=prediction_block, t0_dt=t0_dt + ) + return dataarray + + def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: + """ + Compute the optical flow for a set of images + + Args: + t0_image: t0 image + previous_image: previous image to compute optical flow with + + Returns: + Optical Flow field + """ + # Input images have to be single channel and between 0 and 1 + image_min = np.min([t0_image, previous_image]) + image_max = np.max([t0_image, previous_image]) + t0_image -= image_min + t0_image /= image_max + previous_image -= image_min + previous_image /= image_max + t0_image = cv2.cvtColor(t0_image.astype(np.float32), cv2.COLOR_RGBA2GRAY) + previous_image = cv2.cvtColor(previous_image.astype(np.float32), cv2.COLOR_RGBA2GRAY) + return cv2.calcOpticalFlowFarneback( + prev=previous_image, + next=t0_image, + flow=None, + pyr_scale=0.5, + levels=2, + winsize=40, + iterations=3, + poly_n=5, + poly_sigma=0.7, + flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, + ) + + def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: + """ + Takes an image and warps it forwards in time according to the flow field. + + Args: + image: The grayscale image to warp. + flow: A 3D array. The first two dimensions must be the same size as the first two + dimensions of the image. The third dimension represented the x and y displacement. + + Returns: Warped image. The border has values np.NaN. + """ + # Adapted from https://github.com/opencv/opencv/issues/11068 + height, width = flow.shape[:2] + remap = -flow.copy() + remap[..., 0] += np.arange(width) # map_x + remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y + return cv2.remap( + src=image, + map1=remap, + map2=None, + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=np.NaN, + ) + + def crop_center(self, image, x_size, y_size): + """ + Crop center of numpy image + + Args: + image: Image to crop + x_size: Size in x direction + y_size: Size in y direction + + Returns: + The cropped image + """ + y, x, channels = image.shape + startx = x // 2 - (x_size // 2) + starty = y // 2 - (y_size // 2) + return image[starty : starty + y_size, startx : startx + x_size] + + def _post_process_example( + self, selected_data: xr.DataArray, t0_dt: pd.Timestamp + ) -> xr.DataArray: + + selected_data.data = selected_data.data.astype(np.float32) + + return selected_data diff --git a/nowcasting_dataset/data_sources/transforms/base.py b/nowcasting_dataset/data_sources/transforms/base.py index 1d4f2dd9..30dfd2ff 100644 --- a/nowcasting_dataset/data_sources/transforms/base.py +++ b/nowcasting_dataset/data_sources/transforms/base.py @@ -3,8 +3,7 @@ from dataclasses import dataclass from typing import List -from nowcasting_dataset.data_sources.data_source import DataSource -from nowcasting_dataset.dataset.batch import Batch +from nowcasting_dataset.data_sources.data_source import DataSource, DataSourceOutput @dataclass @@ -12,12 +11,12 @@ class Transform: """Abstract base class. Attributes: - data_sources: List of data sources that this transform will be applied to + data_sources: Data source that this transform will use """ data_sources: List[DataSource] - def apply_transform(self, batch: Batch) -> Batch: + def apply_transforms(self, batch: DataSourceOutput) -> DataSourceOutput: """ Apply transform to the Batch, returning the Batch with added/transformed data @@ -25,6 +24,28 @@ def apply_transform(self, batch: Batch) -> Batch: batch: Batch consisting of the data to transform Returns: - Batch with the transformed data + Datasource with the transformed data """ + return NotImplementedError + + +class Compose(Transform): + """Applies list of transforms in order""" + + transforms: List[Transform] + + def apply_transforms(self, batch: DataSourceOutput) -> DataSourceOutput: + """ + Apply list of transforms + + Args: + batch: Batch containing data to be transformed + + Returns: + Transformed data + """ + + for transform in self.transforms: + batch = transform.apply_transforms(batch) + return batch diff --git a/nowcasting_dataset/data_sources/transforms/optical_flow.py b/nowcasting_dataset/data_sources/transforms/optical_flow.py index 1335f7a7..cb65442b 100644 --- a/nowcasting_dataset/data_sources/transforms/optical_flow.py +++ b/nowcasting_dataset/data_sources/transforms/optical_flow.py @@ -7,7 +7,8 @@ import pandas as pd import xarray as xr -from nowcasting_dataset.data_sources.transforms.transform import Transform +from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput +from nowcasting_dataset.data_sources.transforms.base import Transform from nowcasting_dataset.dataset.batch import Batch _LOG = logging.getLogger("nowcasting_dataset") @@ -21,7 +22,7 @@ class OpticalFlowTransform(Transform): final_image_size_pixels: Optional[int] = None - def apply_transform(self, batch: Batch) -> Batch: + def apply_transforms(self, batch: Batch) -> DataSourceOutput: """ Calculate optical flow for the batch, and add to Batch diff --git a/nowcasting_dataset/dataset/batch.py b/nowcasting_dataset/dataset/batch.py index 4cb71e30..cfa90f4d 100644 --- a/nowcasting_dataset/dataset/batch.py +++ b/nowcasting_dataset/dataset/batch.py @@ -139,7 +139,6 @@ def save_netcdf(self, batch_i: int, path: Path): path: the path where it will be saved. This can be local or in the cloud. """ - with futures.ThreadPoolExecutor() as executor: # Submit tasks to the executor. for data_source in self.data_sources: @@ -195,6 +194,7 @@ class Example(BaseModel): metadata: Optional[Metadata] satellite: Optional[Satellite] topographic: Optional[Topographic] + optical_flow: Optional[OpticalFlow] pv: Optional[PV] sun: Optional[Sun] gsp: Optional[GSP] @@ -205,6 +205,7 @@ def data_sources(self): """The different data sources""" return [ self.satellite, + self.optical_flow, self.topographic, self.pv, self.sun, diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 9383a73e..5e8f26f7 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -316,6 +316,74 @@ def _find_splits_which_need_more_batches( splits_which_need_more_batches.append(split_name) return splits_which_need_more_batches + def create_derived_batches(self, overwrite_batches: bool) -> None: + """ + Create batches of derived data sources + + This loads previously created batches + + Args: + overwrite_batches: If True then start from batch 0, regardless of which batches have + previously been written to disk. If False then check which batches have previously been + written to disk, and only create any batches which have not yet been written to disk. + + """ + first_batches_to_create = self._get_first_batches_to_create(overwrite_batches) + + # Check if there's any work to do. + if overwrite_batches: + splits_which_need_more_batches = [split_name for split_name in split.SplitName] + else: + splits_which_need_more_batches = self._find_splits_which_need_more_batches( + first_batches_to_create + ) + if len(splits_which_need_more_batches) == 0: + logger.info("All batches have already been created! No work to do!") + return + + with futures.ProcessPoolExecutor(max_workers=n_data_sources) as executor: + future_create_batches_jobs = [] + for worker_id, (data_source_name, data_source) in enumerate(self.data_sources.items()): + # Get indexes of first batch and example. And subset locations_for_split. + idx_of_first_batch = first_batches_to_create[split_name][data_source_name] + idx_of_first_example = idx_of_first_batch * self.config.process.batch_size + + # Get paths. + dst_path = self.config.output_data.filepath / split_name.value / data_source_name + local_temp_path = ( + self.local_temp_path + / split_name.value + / data_source_name + / f"worker_{worker_id}" + ) + + # Make folders. + nd_fs_utils.makedirs(dst_path, exist_ok=True) + if self.save_batches_locally_and_upload: + nd_fs_utils.makedirs(local_temp_path, exist_ok=True) + + # Submit data_source.create_batches task to the worker process. + future = executor.submit( + data_source.create_batches, + idx_of_first_batch=idx_of_first_batch, + batch_size=self.config.process.batch_size, + dst_path=dst_path, + local_temp_path=local_temp_path, + upload_every_n_batches=self.config.process.upload_every_n_batches, + ) + future_create_batches_jobs.append(future) + + # Wait for all futures to finish: + for future, data_source_name in zip( + future_create_batches_jobs, self.data_sources.keys() + ): + # Call exception() to propagate any exceptions raised by the worker process into + # the main process, and to wait for the worker to finish. + exception = future.exception() + if exception is not None: + logger.exception(f"Worker process {data_source_name} raised exception!") + raise exception + def create_batches(self, overwrite_batches: bool) -> None: """Create batches (if necessary). diff --git a/requirements.txt b/requirements.txt index 0a0f2eef..6cc95c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,5 +28,4 @@ pre-commit s3fs fsspec pathy -satip>=2.0.2 opencv-contrib-python-headless diff --git a/scripts/get_raw_eumetsat_data.py b/scripts/get_raw_eumetsat_data.py deleted file mode 100644 index cd5ae8a8..00000000 --- a/scripts/get_raw_eumetsat_data.py +++ /dev/null @@ -1,86 +0,0 @@ -############ -# Pull raw satellite data from EUMetSat -# -# 2021-09-28 -# Jacob Bieker -# -############ -from datetime import datetime - -import click -import pandas as pd -import satip.download - -NATIVE_FILESIZE_MB = 102.210123 -CLOUD_FILESIZE_MB = 3.445185 -RSS_ID = "EO:EUM:DAT:MSG:MSG15-RSS" -CLOUD_ID = "EO:EUM:DAT:MSG:RSS-CLM" - -format_dt_str = lambda dt: pd.to_datetime(dt).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def validate_date(ctx, param, value): - try: - return format_dt_str(value) - except ValueError: - raise click.BadParameter("Date must be in format accepted by pd.to_datetime()") - - -@click.command() -@click.option( - "--download_directory", - "--dir", - default="/storage/", - help="Where to download the data to. Also where the script searches for previously downloaded data.", -) -@click.option( - "--start_date", - "--start", - default="2010-01-01", - prompt="Starting date to download data, in format accepted by pd.to_datetime()", - callback=validate_date, -) -@click.option( - "--end_date", - "--end", - default=datetime.now().strftime("%Y-%m-%d"), - prompt="Ending date to download data, in format accepted by pd.to_datetime()", - callback=validate_date, -) -@click.option( - "--backfill", - "-b", - default=False, - prompt="Whether to download any missing data from the start date of the data on disk to the end date", - is_flag=True, -) -@click.option( - "--user_key", - "--key", - default=None, - help="The User Key for EUMETSAT access. Alternatively, the user key can be set using an auth file.", -) -@click.option( - "--user_secret", - "--secret", - default=None, - help="The User secret for EUMETSAT access. Alternatively, the user secret can be set using an auth file.", -) -@click.option( - "--auth_filename", - default="auth.yaml", - help="The auth file containing the user key and access key for EUMETSAT access", -) -@click.option( - "--bandwidth_limit", - "--bw_limit", - default=0.0, - prompt="Bandwidth limit, in MB/sec, currently ignored", - type=float, -) -def download_sat_files(*args, **kwargs): - satip.download.download_eumetsat_data(*args, **kwargs) - - -if __name__ == "__main__": - download_sat_files() diff --git a/scripts/prepare_ml_data.py b/scripts/prepare_ml_data.py index 818052e8..b5ff8d4d 100755 --- a/scripts/prepare_ml_data.py +++ b/scripts/prepare_ml_data.py @@ -66,6 +66,7 @@ def main(config_filename: str, data_source: list[str], overwrite_batches: bool): # of data_sources is passed in at the command line. manager.create_files_specifying_spatial_and_temporal_locations_of_each_example_if_necessary() manager.create_batches(overwrite_batches) + manager.create_derived_batches(overwrite_batches) manager.save_yaml_configuration() # TODO: Issue #317: Validate ML data. logger.info("Done!") From c5e2b1bb08aef4f6667635e6468d9805db059bf4 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Tue, 9 Nov 2021 16:37:56 +0000 Subject: [PATCH 031/197] Start adding DerivedDataSource DerivedDataSource would be for ones like Optical Flow and anything else that is derived from other data sources. --- .../data_sources/data_source.py | 54 +++++++- .../optical_flow/optical_flow_data_source.py | 130 +++--------------- 2 files changed, 67 insertions(+), 117 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index e3769cb6..726a09ca 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -43,7 +43,6 @@ class DataSource: history_minutes: int forecast_minutes: int - transform: Transform def __post_init__(self): """Post Init""" @@ -204,9 +203,6 @@ def create_batches( y_locations=locations_for_batch.y_center_OSGB, ) - # Run transforms on batch - batch: DataSourceOutput = self.transform.apply_transforms(batch) - # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) batch.to_netcdf(netcdf_filename) @@ -461,3 +457,53 @@ def open(self) -> None: def _open_data(self) -> xr.DataArray: raise NotImplementedError() + + +@dataclass +class DerivedDataSource(DataSource): + """ + Base class for data sources derived from other data sources + """ + + def datetime_index(self): + """The datetime index of this datasource""" + return NotImplementedError( + "DerivedDataSources only use other, pre-computed batches, so no datetime_index is " + "needed" + ) + + def get_batch(self, t0_datetimes: pd.DatetimeIndex, **kwargs) -> DataSourceOutput: + """ + Get Batch of data Data + + Args: + **kwargs: + t0_datetimes: list of timestamps for the datetime of the batches. The batch will also + include data for historic and future depending on `history_minutes` and + `future_minutes`. The batch size is given by the length of the t0_datetimes. + x_locations: x center batch locations + y_locations: y center batch locations + + Returns: Batch data. + """ + zipped = list(t0_datetimes) + batch_size = len(t0_datetimes) + + with futures.ThreadPoolExecutor(max_workers=batch_size) as executor: + future_examples = [] + for coords in zipped: + t0_datetime = coords + future_example = executor.submit( + self.get_example, t0_datetime + ) + future_examples.append(future_example) + examples = [future_example.result() for future_example in future_examples] + + # Get the DataSource class, this could be one of the data sources like Sun + cls = examples[0].__class__ + + # Set the coords to be indices before joining into a batch + examples = [make_dim_index(example) for example in examples] + + # join the examples together, and cast them to the cls, so that validation can occur + return cls(join_list_dataset_to_batch_dataset(examples)) \ No newline at end of file diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index b5f0b801..148fe825 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -1,17 +1,18 @@ """ Optical Flow Data Source """ import logging from concurrent import futures -from dataclasses import InitVar, dataclass +from dataclasses import dataclass from numbers import Number -from typing import Iterable, Optional +from typing import Iterable +from pathlib import Path +from typing import Union import cv2 import numpy as np import pandas as pd import xarray as xr -import nowcasting_dataset.time as nd_time -from nowcasting_dataset.data_sources.data_source import ZarrDataSource +from nowcasting_dataset.data_sources.data_source import DerivedDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow from nowcasting_dataset.dataset.xr_utils import join_list_data_array_to_batch_dataset @@ -20,98 +21,25 @@ @dataclass -class OpticalFlowDataSource(ZarrDataSource): +class OpticalFlowDataSource(DerivedDataSource): """ Optical Flow Data Source, computing flow between Satellite data zarr_path: Must start with 'gs://' if on GCP. """ - zarr_path: str = None + netcdf_path: Union[str, Path] previous_timestep_for_flow: int = 1 - image_size_pixels: InitVar[int] = 128 - meters_per_pixel: InitVar[int] = 2_000 - def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): + def __post_init__(self): """Post Init""" - super().__post_init__(image_size_pixels, meters_per_pixel) - self._cache = {} - self._shape_of_example = ( - self._total_seq_length, - image_size_pixels, - image_size_pixels, - 2, - ) + self.open() def open(self) -> None: """ Open Satellite data - - We don't want to open_sat_data in __init__. - If we did that, then we couldn't copy SatelliteDataSource - instances into separate processes. Instead, - call open() _after_ creating separate processes. - """ - self._data = self._open_data() - self._data = self._data.sel(variable=list(self.channels)) - - def _open_data(self) -> xr.DataArray: - return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) - - def get_batch( - self, - t0_datetimes: pd.DatetimeIndex, - x_locations: Iterable[Number], - y_locations: Iterable[Number], - ) -> OpticalFlow: """ - Get batch data - - Load the first _n_timesteps_per_batch concurrently. This - loads the timesteps from disk concurrently, and fills the - cache. If we try loading all examples - concurrently, then SatelliteDataSource will try reading from - empty caches, and things are much slower! - - Args: - t0_datetimes: list of timestamps for the datetime of the batches. The batch will also - include data for historic and future depending on `history_minutes` and - `future_minutes`. - x_locations: x center batch locations - y_locations: y center batch locations - - Returns: Batch data - - """ - # Load the first _n_timesteps_per_batch concurrently. This - # loads the timesteps from disk concurrently, and fills the - # cache. If we try loading all examples - # concurrently, then SatelliteDataSource will try reading from - # empty caches, and things are much slower! - zipped = list(zip(t0_datetimes, x_locations, y_locations)) - batch_size = len(t0_datetimes) - - with futures.ThreadPoolExecutor(max_workers=batch_size) as executor: - future_examples = [] - for coords in zipped[: self.n_timesteps_per_batch]: - t0_datetime, x_location, y_location = coords - future_example = executor.submit( - self.get_example, t0_datetime, x_location, y_location - ) - future_examples.append(future_example) - examples = [future_example.result() for future_example in future_examples] - - # Load the remaining examples. This should hit the DataSource caches. - for coords in zipped[self.n_timesteps_per_batch :]: - t0_datetime, x_location, y_location = coords - example = self.get_example(t0_datetime, x_location, y_location) - examples.append(example) - - output = join_list_data_array_to_batch_dataset(examples) - - self._cache = {} - - return OpticalFlow(output) + self._data = xr.load_dataset(self.netcdf_path) def get_example( self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number @@ -128,20 +56,7 @@ def get_example( Returns: Example Data """ - selected_data = self._get_time_slice(t0_dt) - bounding_box = self._square.bounding_box_centered_on( - x_meters_center=x_meters_center, y_meters_center=y_meters_center - ) - selected_data = selected_data.sel( - x=slice(bounding_box.left, bounding_box.right), - y=slice(bounding_box.top, bounding_box.bottom), - ) - - # selected_sat_data is likely to have 1 too many pixels in x and y - # because sel(x=slice(a, b)) is [a, b], not [a, b). So trim: - selected_data = selected_data.isel( - x=slice(0, self._square.size_pixels), y=slice(0, self._square.size_pixels) - ) + selected_data = self._compute_and_return_optical_flow(self._data, t0_dt = t0_dt) selected_data = self._post_process_example(selected_data, t0_dt) @@ -156,17 +71,6 @@ def get_example( f"actual shape {selected_data.shape}" ) - # rename 'variable' to 'channels' - selected_data = selected_data.rename({"variable": "channels"}) - - # Compute optical flow for the timesteps - # Get Optical Flow for the pre-t0 time, and applying the t0-previous_timesteps_per_flow to - # t0 optical flow for forecast steps in the future - # Creates a pyramid of optical flows for all timesteps up to t0, and apply predictions - # for all future timesteps for each of them - # Compute optical flow per channel, as it might be different - selected_data = self._compute_and_return_optical_flow(selected_data, t0_dt=t0_dt) - return selected_data def _update_dataarray_with_predictions( @@ -257,8 +161,8 @@ def _compute_and_return_optical_flow( """ # Get the previous timestamp - future_timesteps = _get_number_future_timesteps(satellite_data, t0_dt) - satellite_data: xr.DataArray = _get_previous_timesteps( + future_timesteps = self._get_number_future_timesteps(satellite_data, t0_dt) + satellite_data: xr.DataArray = self._get_previous_timesteps( satellite_data, t0_dt=t0_dt, ) @@ -281,13 +185,13 @@ def _compute_and_return_optical_flow( previous_image = channel_images.isel( time_index=len(satellite_data.time_index) - 2 ).data.values - optical_flow = _compute_optical_flow(t0_image, previous_image) + optical_flow = self._compute_optical_flow(t0_image, previous_image) # Do predictions now flow = ( optical_flow * prediction_timestep + 1 ) # Otherwise first prediction would be 0 - warped_image = _remap_image(t0_image, flow) - warped_image = crop_center( + warped_image = self._remap_image(t0_image, flow) + warped_image = self.crop_center( warped_image, final_image_size_pixels, final_image_size_pixels, @@ -295,7 +199,7 @@ def _compute_and_return_optical_flow( prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image # Convert to correct C, T, H, W order prediction_block = np.permute(prediction_block, [3, 0, 1, 2]) - dataarray = _update_dataarray_with_predictions( + dataarray = self._update_dataarray_with_predictions( satellite_data=satellite_data, predictions=prediction_block, t0_dt=t0_dt ) return dataarray From 322897e79052ea13931885c5d339e78d26b61c11 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 08:56:38 +0000 Subject: [PATCH 032/197] Simplify Optical Flow data source a bit Also try to get around circular importing of Batch and DataSource --- .../data_sources/data_source.py | 24 +++---- .../optical_flow/optical_flow_data_source.py | 62 +++++-------------- 2 files changed, 26 insertions(+), 60 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 726a09ca..d91c62dc 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -16,8 +16,8 @@ from nowcasting_dataset import square from nowcasting_dataset.consts import SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput -from nowcasting_dataset.data_sources.transforms.base import Transform from nowcasting_dataset.dataset.xr_utils import join_list_dataset_to_batch_dataset, make_dim_index +import nowcasting_dataset.dataset.batch logger = logging.getLogger(__name__) @@ -472,29 +472,23 @@ def datetime_index(self): "needed" ) - def get_batch(self, t0_datetimes: pd.DatetimeIndex, **kwargs) -> DataSourceOutput: + def get_batch(self, net_cdf_path: Union[str, Path], batch_idx: int, **kwargs) -> \ + DataSourceOutput: """ - Get Batch of data Data + Get Batch of derived data Args: **kwargs: - t0_datetimes: list of timestamps for the datetime of the batches. The batch will also - include data for historic and future depending on `history_minutes` and - `future_minutes`. The batch size is given by the length of the t0_datetimes. - x_locations: x center batch locations - y_locations: y center batch locations + net_cdf_path: PAth to the NetCDF files of the Batch to load Returns: Batch data. """ - zipped = list(t0_datetimes) - batch_size = len(t0_datetimes) - - with futures.ThreadPoolExecutor(max_workers=batch_size) as executor: + batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(net_cdf_path, batch_idx = batch_idx) + with futures.ThreadPoolExecutor(max_workers=batch.batch_size) as executor: future_examples = [] - for coords in zipped: - t0_datetime = coords + for example_idx in range(batch.batch_size): future_example = executor.submit( - self.get_example, t0_datetime + self.get_example, batch, example_idx ) future_examples.append(future_example) examples = [future_example.result() for future_example in future_examples] diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 148fe825..aa174838 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -1,11 +1,8 @@ """ Optical Flow Data Source """ import logging -from concurrent import futures from dataclasses import dataclass -from numbers import Number -from typing import Iterable from pathlib import Path -from typing import Union +from typing import Union, Optional import cv2 import numpy as np @@ -14,8 +11,8 @@ from nowcasting_dataset.data_sources.data_source import DerivedDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput -from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow from nowcasting_dataset.dataset.xr_utils import join_list_data_array_to_batch_dataset +import nowcasting_dataset.dataset.batch _LOG = logging.getLogger("nowcasting_dataset") @@ -30,19 +27,11 @@ class OpticalFlowDataSource(DerivedDataSource): netcdf_path: Union[str, Path] previous_timestep_for_flow: int = 1 + final_image_size_pixels: Optional[int] = None - def __post_init__(self): - """Post Init""" - self.open() - - def open(self) -> None: - """ - Open Satellite data - """ - self._data = xr.load_dataset(self.netcdf_path) def get_example( - self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number + self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, **kwargs ) -> DataSourceOutput: """ Get Optical Flow Example data @@ -50,26 +39,19 @@ def get_example( Args: t0_dt: list of timestamps for the datetime of the batches. The batch will also include data for historic and future depending on `history_minutes` and `future_minutes`. - x_meters_center: x center batch locations - y_meters_center: y center batch locations Returns: Example Data """ - selected_data = self._compute_and_return_optical_flow(self._data, t0_dt = t0_dt) - selected_data = self._post_process_example(selected_data, t0_dt) - - if selected_data.shape != self._shape_of_example: - raise RuntimeError( - "Example is wrong shape! " - f"x_meters_center={x_meters_center}\n" - f"y_meters_center={y_meters_center}\n" - f"t0_dt={t0_dt}\n" - f"times are {selected_data.time}\n" - f"expected shape={self._shape_of_example}\n" - f"actual shape {selected_data.shape}" - ) + if self.final_image_size_pixels is None: + self.final_image_size_pixels = len(batch.satellite.x_index) + + # Only do optical flow for satellite data + self._data: xr.DataArray = batch.satellite.sel(example=example_idx) + t0_dt = batch.metadata.t0_dt.values[example_idx] + + selected_data = self._compute_and_return_optical_flow(self._data, t0_dt = t0_dt) return selected_data @@ -78,7 +60,6 @@ def _update_dataarray_with_predictions( satellite_data: xr.DataArray, predictions: np.ndarray, t0_dt: pd.Timestamp, - final_image_size_pixels: int, ) -> xr.DataArray: """ Updates the dataarray with predictions @@ -96,7 +77,7 @@ def _update_dataarray_with_predictions( # Combine all channels for a single timestep satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) # Make sure its the correct size - buffer = satellite_data.sizes["x"] - final_image_size_pixels // 2 + buffer = satellite_data.sizes["x"] - self.final_image_size_pixels // 2 satellite_data = satellite_data.isel( x=slice(buffer, satellite_data.sizes["x"] - buffer), y=slice(buffer, satellite_data.sizes["y"] - buffer), @@ -147,7 +128,6 @@ def _compute_and_return_optical_flow( self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp, - final_image_size_pixels: int, ) -> xr.DataArray: """ Compute and return optical flow predictions for the example @@ -169,8 +149,8 @@ def _compute_and_return_optical_flow( prediction_block = np.zeros( ( future_timesteps, - final_image_size_pixels, - final_image_size_pixels, + self.final_image_size_pixels, + self.final_image_size_pixels, satellite_data.sizes["channels_index"], ) ) @@ -193,8 +173,8 @@ def _compute_and_return_optical_flow( warped_image = self._remap_image(t0_image, flow) warped_image = self.crop_center( warped_image, - final_image_size_pixels, - final_image_size_pixels, + self.final_image_size_pixels, + self.final_image_size_pixels, ) prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image # Convert to correct C, T, H, W order @@ -278,11 +258,3 @@ def crop_center(self, image, x_size, y_size): startx = x // 2 - (x_size // 2) starty = y // 2 - (y_size // 2) return image[starty : starty + y_size, startx : startx + x_size] - - def _post_process_example( - self, selected_data: xr.DataArray, t0_dt: pd.Timestamp - ) -> xr.DataArray: - - selected_data.data = selected_data.data.astype(np.float32) - - return selected_data From 41dcc64e6ada0cfc9b1a56cd59fb84e6f88a3437 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 09:02:07 +0000 Subject: [PATCH 033/197] Remove making example dim in derived sources --- nowcasting_dataset/data_sources/data_source.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index d91c62dc..1e49d6c4 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -496,8 +496,5 @@ def get_batch(self, net_cdf_path: Union[str, Path], batch_idx: int, **kwargs) -> # Get the DataSource class, this could be one of the data sources like Sun cls = examples[0].__class__ - # Set the coords to be indices before joining into a batch - examples = [make_dim_index(example) for example in examples] - # join the examples together, and cast them to the cls, so that validation can occur return cls(join_list_dataset_to_batch_dataset(examples)) \ No newline at end of file From 54b4d4dc83516513997ceeae12f1fb5d0982de5e Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 09:17:45 +0000 Subject: [PATCH 034/197] Remove tests --- .../test_optical_flow_data_source.py | 57 ------------------- 1 file changed, 57 deletions(-) delete mode 100644 tests/data_sources/optical_flow/test_optical_flow_data_source.py diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py deleted file mode 100644 index 5d9a7f4d..00000000 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Test OpticalFlowDataSource.""" -import numpy as np -import pandas as pd -import pytest - - -def test_satellite_data_source_init(optical_flow_data_source): # noqa: D103 - pass - - -def test_open(optical_flow_data_source): # noqa: D103 - optical_flow_data_source.open() - assert optical_flow_data_source.data is not None - - -def test_datetime_index(optical_flow_data_source): # noqa: D103 - datetimes = optical_flow_data_source.datetime_index() - assert isinstance(datetimes, pd.DatetimeIndex) - assert len(datetimes) > 0 - assert len(np.unique(datetimes)) == len(datetimes) - assert np.all(np.diff(datetimes.view(int)) > 0) - - -@pytest.mark.parametrize( - "x, y, left, right, top, bottom", - [ - (0, 0, -128_000, 126_000, 128_000, -126_000), - (10, 0, -126_000, 128_000, 128_000, -126_000), - (30, 0, -126_000, 128_000, 128_000, -126_000), - (1000, 0, -126_000, 128_000, 128_000, -126_000), - (0, 1000, -128_000, 126_000, 128_000, -126_000), - (1000, 1000, -126_000, 128_000, 128_000, -126_000), - (2000, 2000, -126_000, 128_000, 130_000, -124_000), - (2000, 1000, -126_000, 128_000, 128_000, -126_000), - (2001, 2001, -124_000, 130_000, 130_000, -124_000), - ], -) -def test_get_example(optical_flow_data_source, x, y, left, right, top, bottom): # noqa: D103 - optical_flow_data_source.open() - t0_dt = pd.Timestamp("2019-01-01T13:00") - sat_data = optical_flow_data_source.get_example( - t0_dt=t0_dt, x_meters_center=x, y_meters_center=y - ) - - assert left == sat_data.x.values[0] - assert right == sat_data.x.values[-1] - # sat_data.y is top-to-bottom. - assert top == sat_data.y.values[0] - assert bottom == sat_data.y.values[-1] - assert len(sat_data.x) == pytest.IMAGE_SIZE_PIXELS - assert len(sat_data.y) == pytest.IMAGE_SIZE_PIXELS - - -def test_geospatial_border(optical_flow_data_source): # noqa: D103 - border = optical_flow_data_source.geospatial_border() - correct_border = [(-110000, 1094000), (-110000, -58000), (730000, 1094000), (730000, -58000)] - np.testing.assert_array_equal(border, correct_border) From 539ea330b702958105a038012408307a65c19fa7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 09:23:06 +0000 Subject: [PATCH 035/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/data_source.py | 19 ++++++++++--------- .../optical_flow/optical_flow_data_source.py | 7 +++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 1e49d6c4..35df8de2 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -10,6 +10,7 @@ import pandas as pd import xarray as xr +import nowcasting_dataset.dataset.batch import nowcasting_dataset.filesystem.utils as nd_fs_utils import nowcasting_dataset.time as nd_time import nowcasting_dataset.utils as nd_utils @@ -17,7 +18,6 @@ from nowcasting_dataset.consts import SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.dataset.xr_utils import join_list_dataset_to_batch_dataset, make_dim_index -import nowcasting_dataset.dataset.batch logger = logging.getLogger(__name__) @@ -470,10 +470,11 @@ def datetime_index(self): return NotImplementedError( "DerivedDataSources only use other, pre-computed batches, so no datetime_index is " "needed" - ) + ) - def get_batch(self, net_cdf_path: Union[str, Path], batch_idx: int, **kwargs) -> \ - DataSourceOutput: + def get_batch( + self, net_cdf_path: Union[str, Path], batch_idx: int, **kwargs + ) -> DataSourceOutput: """ Get Batch of derived data @@ -483,13 +484,13 @@ def get_batch(self, net_cdf_path: Union[str, Path], batch_idx: int, **kwargs) -> Returns: Batch data. """ - batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(net_cdf_path, batch_idx = batch_idx) + batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf( + net_cdf_path, batch_idx=batch_idx + ) with futures.ThreadPoolExecutor(max_workers=batch.batch_size) as executor: future_examples = [] for example_idx in range(batch.batch_size): - future_example = executor.submit( - self.get_example, batch, example_idx - ) + future_example = executor.submit(self.get_example, batch, example_idx) future_examples.append(future_example) examples = [future_example.result() for future_example in future_examples] @@ -497,4 +498,4 @@ def get_batch(self, net_cdf_path: Union[str, Path], batch_idx: int, **kwargs) -> cls = examples[0].__class__ # join the examples together, and cast them to the cls, so that validation can occur - return cls(join_list_dataset_to_batch_dataset(examples)) \ No newline at end of file + return cls(join_list_dataset_to_batch_dataset(examples)) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index aa174838..d33462a9 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -2,17 +2,17 @@ import logging from dataclasses import dataclass from pathlib import Path -from typing import Union, Optional +from typing import Optional, Union import cv2 import numpy as np import pandas as pd import xarray as xr +import nowcasting_dataset.dataset.batch from nowcasting_dataset.data_sources.data_source import DerivedDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.dataset.xr_utils import join_list_data_array_to_batch_dataset -import nowcasting_dataset.dataset.batch _LOG = logging.getLogger("nowcasting_dataset") @@ -29,7 +29,6 @@ class OpticalFlowDataSource(DerivedDataSource): previous_timestep_for_flow: int = 1 final_image_size_pixels: Optional[int] = None - def get_example( self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, **kwargs ) -> DataSourceOutput: @@ -51,7 +50,7 @@ def get_example( self._data: xr.DataArray = batch.satellite.sel(example=example_idx) t0_dt = batch.metadata.t0_dt.values[example_idx] - selected_data = self._compute_and_return_optical_flow(self._data, t0_dt = t0_dt) + selected_data = self._compute_and_return_optical_flow(self._data, t0_dt=t0_dt) return selected_data From b8844960b5f21f3a4087c4af2ff657777b72906e Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 09:28:30 +0000 Subject: [PATCH 036/197] Update docstrings --- nowcasting_dataset/data_sources/data_source.py | 11 ++++++----- .../optical_flow/optical_flow_data_source.py | 5 ++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 35df8de2..2e19d501 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -473,19 +473,20 @@ def datetime_index(self): ) def get_batch( - self, net_cdf_path: Union[str, Path], batch_idx: int, **kwargs + self, netcdf_path: Union[str, Path], batch_idx: int, **kwargs ) -> DataSourceOutput: """ Get Batch of derived data Args: - **kwargs: - net_cdf_path: PAth to the NetCDF files of the Batch to load + netcdf_path: Path to the NetCDF files of the Batch to load + batch_idx: The batch ID to load from those in teh path - Returns: Batch data. + Returns: + Batch of the derived data source """ batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf( - net_cdf_path, batch_idx=batch_idx + netcdf_path, batch_idx=batch_idx ) with futures.ThreadPoolExecutor(max_workers=batch.batch_size) as executor: future_examples = [] diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index d33462a9..84935b16 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -12,7 +12,6 @@ import nowcasting_dataset.dataset.batch from nowcasting_dataset.data_sources.data_source import DerivedDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput -from nowcasting_dataset.dataset.xr_utils import join_list_data_array_to_batch_dataset _LOG = logging.getLogger("nowcasting_dataset") @@ -36,8 +35,8 @@ def get_example( Get Optical Flow Example data Args: - t0_dt: list of timestamps for the datetime of the batches. The batch will also include - data for historic and future depending on `history_minutes` and `future_minutes`. + batch: Batch containing satellite and metadata at least + example_idx: The example to load and use Returns: Example Data From ee8a5ad767b0def7a634064af791c0083eafc13b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 09:28:50 +0000 Subject: [PATCH 037/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/data_sources/data_source.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 2e19d501..88a46580 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -485,9 +485,7 @@ def get_batch( Returns: Batch of the derived data source """ - batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf( - netcdf_path, batch_idx=batch_idx - ) + batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(netcdf_path, batch_idx=batch_idx) with futures.ThreadPoolExecutor(max_workers=batch.batch_size) as executor: future_examples = [] for example_idx in range(batch.batch_size): From d188a9cf76df1a4721e2f1e8ef6cab446a495cea Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 10:30:15 +0000 Subject: [PATCH 038/197] Add getting which batches are needed for derived data sources --- .../data_sources/data_source.py | 59 +++++++++++++ nowcasting_dataset/manager.py | 85 ++++++++++--------- 2 files changed, 104 insertions(+), 40 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 88a46580..da8fc8b6 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -472,6 +472,65 @@ def datetime_index(self): "needed" ) + def create_batches( + self, batch_path: Path, total_number_batches: int, idx_of_first_batch: int, dst_path: Path, local_temp_path: Path, + upload_every_n_batches: int, **kwargs + ) -> None: + """Create multiple batches and save them to disk. + + Safe to call from worker processes. + + Args: + batch_path: Path to where the netcdf batches are stored + total_number_batches: The total number of batches to make + idx_of_first_batch: The batch number of the first batch to create. + dst_path: The final destination path for the batches. Must exist. + local_temp_path: The local temporary path. This is only required when dst_path is a + cloud storage bucket, so files must first be created on the VM's local disk in temp_path + and then uploaded to dst_path every upload_every_n_batches. Must exist. Will be emptied. + upload_every_n_batches: Upload the contents of temp_path to dst_path after this number + of batches have been created. If 0 then will write directly to dst_path. + """ + # Sanity checks: + assert idx_of_first_batch >= 0 + assert upload_every_n_batches >= 0 + assert total_number_batches >= 0 + + self.open() + + # Figure out where to write batches to: + save_batches_locally_and_upload = upload_every_n_batches > 0 + if save_batches_locally_and_upload: + nd_fs_utils.delete_all_files_in_temp_path(local_temp_path) + path_to_write_to = local_temp_path if save_batches_locally_and_upload else dst_path + + # Loop round each batch: + n_batches_processed = 0 + for batch_idx in range(idx_of_first_batch, total_number_batches): + logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") + + # Generate batch. + batch = self.get_batch( + netcdf_path=batch_path, + batch_idx = batch_idx + ) + + # Save batch to disk. + netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) + batch.to_netcdf(netcdf_filename) + n_batches_processed += 1 + # Upload if necessary. + if ( + save_batches_locally_and_upload + and n_batches_processed > 0 + and n_batches_processed % upload_every_n_batches == 0 + ): + nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) + + # Upload last few batches, if necessary: + if save_batches_locally_and_upload: + nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) + def get_batch( self, netcdf_path: Union[str, Path], batch_idx: int, **kwargs ) -> DataSourceOutput: diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 5e8f26f7..76f6e0cf 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -39,6 +39,7 @@ class Manager: def __init__(self) -> None: # noqa: D107 self.config = None self.data_sources = {} + self.derived_data_sources = {} self.data_source_which_defines_geospatial_locations = None def load_yaml_configuration(self, filename: str) -> None: @@ -340,49 +341,53 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: if len(splits_which_need_more_batches) == 0: logger.info("All batches have already been created! No work to do!") return + n_data_sources = len(self.derived_data_sources) + nd_utils.set_fsspec_for_multiprocess() + for split_name in splits_which_need_more_batches: + with futures.ProcessPoolExecutor(max_workers=n_data_sources) as executor: + future_create_batches_jobs = [] + for worker_id, (data_source_name, data_source) in enumerate( + self.derived_data_sources.items()): + # Get indexes of first batch and example. And subset locations_for_split. + idx_of_first_batch = first_batches_to_create[split_name][data_source_name] - with futures.ProcessPoolExecutor(max_workers=n_data_sources) as executor: - future_create_batches_jobs = [] - for worker_id, (data_source_name, data_source) in enumerate(self.data_sources.items()): - # Get indexes of first batch and example. And subset locations_for_split. - idx_of_first_batch = first_batches_to_create[split_name][data_source_name] - idx_of_first_example = idx_of_first_batch * self.config.process.batch_size - - # Get paths. - dst_path = self.config.output_data.filepath / split_name.value / data_source_name - local_temp_path = ( - self.local_temp_path - / split_name.value - / data_source_name - / f"worker_{worker_id}" - ) + # Get paths. + dst_path = self.config.output_data.filepath / split_name.value / data_source_name + local_temp_path = ( + self.local_temp_path + / split_name.value + / data_source_name + / f"worker_{worker_id}" + ) - # Make folders. - nd_fs_utils.makedirs(dst_path, exist_ok=True) - if self.save_batches_locally_and_upload: - nd_fs_utils.makedirs(local_temp_path, exist_ok=True) - - # Submit data_source.create_batches task to the worker process. - future = executor.submit( - data_source.create_batches, - idx_of_first_batch=idx_of_first_batch, - batch_size=self.config.process.batch_size, - dst_path=dst_path, - local_temp_path=local_temp_path, - upload_every_n_batches=self.config.process.upload_every_n_batches, - ) - future_create_batches_jobs.append(future) + # Make folders. + nd_fs_utils.makedirs(dst_path, exist_ok=True) + if self.save_batches_locally_and_upload: + nd_fs_utils.makedirs(local_temp_path, exist_ok=True) - # Wait for all futures to finish: - for future, data_source_name in zip( - future_create_batches_jobs, self.data_sources.keys() - ): - # Call exception() to propagate any exceptions raised by the worker process into - # the main process, and to wait for the worker to finish. - exception = future.exception() - if exception is not None: - logger.exception(f"Worker process {data_source_name} raised exception!") - raise exception + # Submit data_source.create_batches task to the worker process. + future = executor.submit( + data_source.create_batches, + batch_path="", + total_number_batches = 0, + idx_of_first_batch=idx_of_first_batch, + batch_size=self.config.process.batch_size, + dst_path=dst_path, + local_temp_path=local_temp_path, + upload_every_n_batches=self.config.process.upload_every_n_batches, + ) + future_create_batches_jobs.append(future) + + # Wait for all futures to finish: + for future, data_source_name in zip( + future_create_batches_jobs, self.data_sources.keys() + ): + # Call exception() to propagate any exceptions raised by the worker process into + # the main process, and to wait for the worker to finish. + exception = future.exception() + if exception is not None: + logger.exception(f"Worker process {data_source_name} raised exception!") + raise exception def create_batches(self, overwrite_batches: bool) -> None: """Create batches (if necessary). From 944322ebce6ce1481913c4a0fe62142a8cd30eaa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 10:30:33 +0000 Subject: [PATCH 039/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/data_source.py | 23 +++++++++++-------- nowcasting_dataset/manager.py | 9 +++++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index da8fc8b6..452d922c 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -473,9 +473,15 @@ def datetime_index(self): ) def create_batches( - self, batch_path: Path, total_number_batches: int, idx_of_first_batch: int, dst_path: Path, local_temp_path: Path, - upload_every_n_batches: int, **kwargs - ) -> None: + self, + batch_path: Path, + total_number_batches: int, + idx_of_first_batch: int, + dst_path: Path, + local_temp_path: Path, + upload_every_n_batches: int, + **kwargs, + ) -> None: """Create multiple batches and save them to disk. Safe to call from worker processes. @@ -510,10 +516,7 @@ def create_batches( logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") # Generate batch. - batch = self.get_batch( - netcdf_path=batch_path, - batch_idx = batch_idx - ) + batch = self.get_batch(netcdf_path=batch_path, batch_idx=batch_idx) # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) @@ -521,9 +524,9 @@ def create_batches( n_batches_processed += 1 # Upload if necessary. if ( - save_batches_locally_and_upload - and n_batches_processed > 0 - and n_batches_processed % upload_every_n_batches == 0 + save_batches_locally_and_upload + and n_batches_processed > 0 + and n_batches_processed % upload_every_n_batches == 0 ): nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 76f6e0cf..5aa711de 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -347,12 +347,15 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: with futures.ProcessPoolExecutor(max_workers=n_data_sources) as executor: future_create_batches_jobs = [] for worker_id, (data_source_name, data_source) in enumerate( - self.derived_data_sources.items()): + self.derived_data_sources.items() + ): # Get indexes of first batch and example. And subset locations_for_split. idx_of_first_batch = first_batches_to_create[split_name][data_source_name] # Get paths. - dst_path = self.config.output_data.filepath / split_name.value / data_source_name + dst_path = ( + self.config.output_data.filepath / split_name.value / data_source_name + ) local_temp_path = ( self.local_temp_path / split_name.value @@ -369,7 +372,7 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: future = executor.submit( data_source.create_batches, batch_path="", - total_number_batches = 0, + total_number_batches=0, idx_of_first_batch=idx_of_first_batch, batch_size=self.config.process.batch_size, dst_path=dst_path, From d49781ffaf8e64581c327faca068d188a812d500 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 10:44:49 +0000 Subject: [PATCH 040/197] Auto stash before merge of "jacob/optical-flow-datasource" and "origin/jacob/optical-flow-datasource" --- nowcasting_dataset/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 5aa711de..29de481e 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -372,7 +372,7 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: future = executor.submit( data_source.create_batches, batch_path="", - total_number_batches=0, + total_number_batches = self._get_n_batches_for_split_name(split_name.value), idx_of_first_batch=idx_of_first_batch, batch_size=self.config.process.batch_size, dst_path=dst_path, From e8aea849eb15ab39dd513c2c55addd8c0eec66b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 10:45:15 +0000 Subject: [PATCH 041/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 29de481e..4ff7b3ec 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -372,7 +372,7 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: future = executor.submit( data_source.create_batches, batch_path="", - total_number_batches = self._get_n_batches_for_split_name(split_name.value), + total_number_batches=self._get_n_batches_for_split_name(split_name.value), idx_of_first_batch=idx_of_first_batch, batch_size=self.config.process.batch_size, dst_path=dst_path, From 221ce7e0ae82935c7d0ddafdd56cee41627ad955 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 10:46:49 +0000 Subject: [PATCH 042/197] Auto stash before merge of "jacob/optical-flow-datasource" and "origin/jacob/optical-flow-datasource" --- nowcasting_dataset/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 4ff7b3ec..623808a3 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -371,8 +371,8 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: # Submit data_source.create_batches task to the worker process. future = executor.submit( data_source.create_batches, - batch_path="", - total_number_batches=self._get_n_batches_for_split_name(split_name.value), + batch_path=self.config.output_data.filepath / split_name.value, + total_number_batches = self._get_n_batches_for_split_name(split_name.value), idx_of_first_batch=idx_of_first_batch, batch_size=self.config.process.batch_size, dst_path=dst_path, From ef1eee371c65e5fff6a27340bf7a55fc0abe3446 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 10:47:06 +0000 Subject: [PATCH 043/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 623808a3..bdc1b282 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -372,7 +372,7 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: future = executor.submit( data_source.create_batches, batch_path=self.config.output_data.filepath / split_name.value, - total_number_batches = self._get_n_batches_for_split_name(split_name.value), + total_number_batches=self._get_n_batches_for_split_name(split_name.value), idx_of_first_batch=idx_of_first_batch, batch_size=self.config.process.batch_size, dst_path=dst_path, From 9e0d0e33c9b510120b82025bbd41ec43b177e729 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 10:59:21 +0000 Subject: [PATCH 044/197] Add adding to derived data source dict --- nowcasting_dataset/data_sources/__init__.py | 1 + nowcasting_dataset/manager.py | 11 ++++++++--- scripts/prepare_ml_data.py | 2 +- tests/test_manager.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py index 6fc93310..7f6c3be2 100644 --- a/nowcasting_dataset/data_sources/__init__.py +++ b/nowcasting_dataset/data_sources/__init__.py @@ -15,6 +15,7 @@ MAP_DATA_SOURCE_NAME_TO_CLASS = { "pv": PVDataSource, "satellite": SatelliteDataSource, + "optical_flow": OpticalFlowDataSource, "nwp": NWPDataSource, "gsp": GSPDataSource, "topographic": TopographicDataSource, diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index bdc1b282..d6b5823d 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -17,6 +17,7 @@ SPATIAL_AND_TEMPORAL_LOCATIONS_OF_EACH_EXAMPLE_FILENAME, ) from nowcasting_dataset.data_sources import ALL_DATA_SOURCE_NAMES, MAP_DATA_SOURCE_NAME_TO_CLASS +from nowcasting_dataset.data_sources.data_source import DerivedDataSource from nowcasting_dataset.dataset.split import split from nowcasting_dataset.filesystem import utils as nd_fs_utils @@ -29,6 +30,7 @@ class Manager: Attrs: config: Configuration object. data_sources: dict[str, DataSource] + derived_data_sources: dict[str, DerivedDataSource] data_source_which_defines_geospatial_locations: DataSource: The DataSource used to compute the geospatial locations of each example. save_batches_locally_and_upload: bool: Set to True by `load_yaml_configuration()` if @@ -57,10 +59,10 @@ def save_yaml_configuration(self): """Save configuration to the 'output_data' location""" config.save_yaml_configuration(configuration=self.config) - def initialise_data_sources( + def initialize_data_sources( self, names_of_selected_data_sources: Optional[list[str]] = ALL_DATA_SOURCE_NAMES ) -> None: - """Initialise DataSources specified in the InputData configuration. + """Initialize DataSources specified in the InputData configuration. For each key in each DataSource's configuration object, the string `_` is removed from the key before passing to the DataSource constructor. This allows us to @@ -86,7 +88,10 @@ def initialise_data_sources( except Exception: logger.exception(f"Exception whilst instantiating {data_source_name}!") raise - self.data_sources[data_source_name] = data_source + if isinstance(data_source, DerivedDataSource): + self.derived_data_sources[data_source_name] = data_source + else: + self.data_sources[data_source_name] = data_source # Set data_source_which_defines_geospatial_locations: try: diff --git a/scripts/prepare_ml_data.py b/scripts/prepare_ml_data.py index b5ff8d4d..bd984abf 100755 --- a/scripts/prepare_ml_data.py +++ b/scripts/prepare_ml_data.py @@ -60,7 +60,7 @@ def main(config_filename: str, data_source: list[str], overwrite_batches: bool): """Generate pre-prepared batches of data.""" manager = Manager() manager.load_yaml_configuration(config_filename) - manager.initialise_data_sources(names_of_selected_data_sources=data_source) + manager.initialize_data_sources(names_of_selected_data_sources=data_source) # TODO: Issue 323: maybe don't allow # create_files_specifying_spatial_and_temporal_locations_of_each_example to be run if a subset # of data_sources is passed in at the command line. diff --git a/tests/test_manager.py b/tests/test_manager.py index 81daf75e..21898774 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -43,7 +43,7 @@ def test_load_yaml_configuration(): # noqa: D103 local_path = Path(nowcasting_dataset.__file__).parent.parent filename = local_path / "tests" / "config" / "test.yaml" manager.load_yaml_configuration(filename=filename) - manager.initialise_data_sources() + manager.initialize_data_sources() assert len(manager.data_sources) == 6 assert isinstance(manager.data_source_which_defines_geospatial_locations, GSPDataSource) From 877cf8044accbaa940be8947c22eeb5400b308b2 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 12:02:40 +0000 Subject: [PATCH 045/197] Try to get around circular import --- nowcasting_dataset/data_sources/data_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 452d922c..ab7dfb7d 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -10,7 +10,6 @@ import pandas as pd import xarray as xr -import nowcasting_dataset.dataset.batch import nowcasting_dataset.filesystem.utils as nd_fs_utils import nowcasting_dataset.time as nd_time import nowcasting_dataset.utils as nd_utils @@ -547,6 +546,9 @@ def get_batch( Returns: Batch of the derived data source """ + # To get around circular imports + import nowcasting_dataset.dataset.batch + batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(netcdf_path, batch_idx=batch_idx) with futures.ThreadPoolExecutor(max_workers=batch.batch_size) as executor: future_examples = [] From 8a3934efba210dc80ad489c4ee71637eb6932fa1 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 12:31:23 +0000 Subject: [PATCH 046/197] Simplify OF model --- nowcasting_dataset/config/model.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 016f44d1..9fc4d10b 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -115,18 +115,10 @@ class Satellite(DataSourceMixin): class OpticalFlow(DataSourceMixin): - """Satellite configuration model""" + """Optical Flow configuration model""" - satellite_zarr_path: str = Field( - "gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr", # noqa: E501 - description="The path which holds the satellite zarr.", - ) - satellite_channels: tuple = Field( - SAT_VARIABLE_NAMES, description="the satellite channels that are used" - ) - satellite_image_size_pixels: int = IMAGE_SIZE_PIXELS_FIELD - satellite_meters_per_pixel: int = METERS_PER_PIXEL_FIELD previous_timestep_to_use: int = 1 + final_image_size_pixels: int = IMAGE_SIZE_PIXELS_FIELD class NWP(DataSourceMixin): From 5a6953e83b1b94350a2cedf55a51d27a7a2e8521 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 12:32:42 +0000 Subject: [PATCH 047/197] Remove init config netcdf --- .../data_sources/optical_flow/optical_flow_data_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 84935b16..029db2d4 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -24,7 +24,6 @@ class OpticalFlowDataSource(DerivedDataSource): zarr_path: Must start with 'gs://' if on GCP. """ - netcdf_path: Union[str, Path] previous_timestep_for_flow: int = 1 final_image_size_pixels: Optional[int] = None From 2c6f814f93235e5ea8defccaf1da8c02d5fa5340 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 12:52:11 +0000 Subject: [PATCH 048/197] Change name --- nowcasting_dataset/dataset/batch.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/dataset/batch.py b/nowcasting_dataset/dataset/batch.py index cfa90f4d..f8349d30 100644 --- a/nowcasting_dataset/dataset/batch.py +++ b/nowcasting_dataset/dataset/batch.py @@ -59,7 +59,7 @@ class Batch(BaseModel): metadata: Optional[Metadata] satellite: Optional[Satellite] topographic: Optional[Topographic] - optical_flow: Optional[OpticalFlow] + opticalflow: Optional[OpticalFlow] pv: Optional[PV] sun: Optional[Sun] gsp: Optional[GSP] @@ -71,7 +71,7 @@ def data_sources(self): return [ self.satellite, self.topographic, - self.optical_flow, + self.opticalflow, self.pv, self.sun, self.gsp, @@ -96,7 +96,7 @@ def fake(configuration: Configuration): configuration.input_data.satellite.satellite_channels ), ), - optical_flow=optical_flow_fake( + opticalflow=optical_flow_fake( batch_size=batch_size, seq_length_5=configuration.input_data.satellite.seq_length_5_minutes, satellite_image_size_pixels=satellite_image_size_pixels, @@ -194,7 +194,7 @@ class Example(BaseModel): metadata: Optional[Metadata] satellite: Optional[Satellite] topographic: Optional[Topographic] - optical_flow: Optional[OpticalFlow] + opticalflow: Optional[OpticalFlow] pv: Optional[PV] sun: Optional[Sun] gsp: Optional[GSP] @@ -205,7 +205,7 @@ def data_sources(self): """The different data sources""" return [ self.satellite, - self.optical_flow, + self.opticalflow, self.topographic, self.pv, self.sun, From a45e38674048a55de3250a726385d8dd735abdbe Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 12:54:31 +0000 Subject: [PATCH 049/197] Fix linting error --- .../data_sources/optical_flow/optical_flow_data_source.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 029db2d4..c21a786f 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -1,8 +1,7 @@ """ Optical Flow Data Source """ import logging from dataclasses import dataclass -from pathlib import Path -from typing import Optional, Union +from typing import Optional import cv2 import numpy as np From 6ea1a4854641b9eb895807d0cdbcd8c18b65a6ef Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 13:35:13 +0000 Subject: [PATCH 050/197] Add in tests for OpticalFlow Data Source --- conftest.py | 11 - nowcasting_dataset/data_sources/__init__.py | 2 +- .../data_sources/data_source.py | 6 +- .../data_sources/transforms/__init__.py | 1 - .../data_sources/transforms/base.py | 51 ---- .../data_sources/transforms/optical_flow.py | 280 ------------------ .../test_optical_flow_data_source.py | 30 ++ 7 files changed, 36 insertions(+), 345 deletions(-) delete mode 100644 nowcasting_dataset/data_sources/transforms/__init__.py delete mode 100644 nowcasting_dataset/data_sources/transforms/base.py delete mode 100644 nowcasting_dataset/data_sources/transforms/optical_flow.py create mode 100644 tests/data_sources/optical_flow/test_optical_flow_data_source.py diff --git a/conftest.py b/conftest.py index a95ee4c0..042d281d 100644 --- a/conftest.py +++ b/conftest.py @@ -49,17 +49,6 @@ def sat_data_source(sat_filename: Path): # noqa: D103 ) -@pytest.fixture -def optical_flow_data_source(sat_filename: Path): # noqa: D103 - return OpticalFlowDataSource( - image_size_pixels=pytest.IMAGE_SIZE_PIXELS, - zarr_path=sat_filename, - history_minutes=0, - forecast_minutes=5, - channels=("HRV",), - ) - - @pytest.fixture def general_data_source(): # noqa: D103 return MetadataDataSource(history_minutes=0, forecast_minutes=5, object_at_center="GSP") diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py index 7f6c3be2..fc61b881 100644 --- a/nowcasting_dataset/data_sources/__init__.py +++ b/nowcasting_dataset/data_sources/__init__.py @@ -15,7 +15,7 @@ MAP_DATA_SOURCE_NAME_TO_CLASS = { "pv": PVDataSource, "satellite": SatelliteDataSource, - "optical_flow": OpticalFlowDataSource, + "opticalflow": OpticalFlowDataSource, "nwp": NWPDataSource, "gsp": GSPDataSource, "topographic": TopographicDataSource, diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index ab7dfb7d..94fe5ca2 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -5,7 +5,7 @@ from dataclasses import InitVar, dataclass from numbers import Number from pathlib import Path -from typing import Iterable, List, Tuple, Union +from typing import Iterable, List, Tuple, Union, Optional import pandas as pd import xarray as xr @@ -464,6 +464,10 @@ class DerivedDataSource(DataSource): Base class for data sources derived from other data sources """ + history_minutes: int = 0 + forecast_minutes: int = 0 + + def datetime_index(self): """The datetime index of this datasource""" return NotImplementedError( diff --git a/nowcasting_dataset/data_sources/transforms/__init__.py b/nowcasting_dataset/data_sources/transforms/__init__.py deleted file mode 100644 index b5e5438f..00000000 --- a/nowcasting_dataset/data_sources/transforms/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Set of transforms for creating derived data sources from other data sources""" diff --git a/nowcasting_dataset/data_sources/transforms/base.py b/nowcasting_dataset/data_sources/transforms/base.py deleted file mode 100644 index 30dfd2ff..00000000 --- a/nowcasting_dataset/data_sources/transforms/base.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Generic Transform class""" - -from dataclasses import dataclass -from typing import List - -from nowcasting_dataset.data_sources.data_source import DataSource, DataSourceOutput - - -@dataclass -class Transform: - """Abstract base class. - - Attributes: - data_sources: Data source that this transform will use - """ - - data_sources: List[DataSource] - - def apply_transforms(self, batch: DataSourceOutput) -> DataSourceOutput: - """ - Apply transform to the Batch, returning the Batch with added/transformed data - - Args: - batch: Batch consisting of the data to transform - - Returns: - Datasource with the transformed data - """ - return NotImplementedError - - -class Compose(Transform): - """Applies list of transforms in order""" - - transforms: List[Transform] - - def apply_transforms(self, batch: DataSourceOutput) -> DataSourceOutput: - """ - Apply list of transforms - - Args: - batch: Batch containing data to be transformed - - Returns: - Transformed data - """ - - for transform in self.transforms: - batch = transform.apply_transforms(batch) - - return batch diff --git a/nowcasting_dataset/data_sources/transforms/optical_flow.py b/nowcasting_dataset/data_sources/transforms/optical_flow.py deleted file mode 100644 index cb65442b..00000000 --- a/nowcasting_dataset/data_sources/transforms/optical_flow.py +++ /dev/null @@ -1,280 +0,0 @@ -"""Functions for computing the optical flow on the fly for satellite images""" -import logging -from typing import Optional - -import cv2 -import numpy as np -import pandas as pd -import xarray as xr - -from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput -from nowcasting_dataset.data_sources.transforms.base import Transform -from nowcasting_dataset.dataset.batch import Batch - -_LOG = logging.getLogger("nowcasting_dataset") - - -class OpticalFlowTransform(Transform): - """ - Optical Flow Transform that adds optical flow images - - """ - - final_image_size_pixels: Optional[int] = None - - def apply_transforms(self, batch: Batch) -> DataSourceOutput: - """ - Calculate optical flow for the batch, and add to Batch - - Args: - batch: Batch containing satellite data for optical flow - - Returns: - Batch with optical flow added - """ - batch.optical_flow = compute_optical_flow_for_batch(batch) - return batch - - -def compute_optical_flow_for_batch( - batch: Batch, final_image_size_pixels: Optional[int] = None -) -> xr.DataArray: - """ - Computes the optical flow for satellite images in the batch - - Assumes metadata is also in Batch, for getting t0 - - Args: - batch: Batch containing at least metadata and satellite data - - Returns: - Tensor containing the Optical Flow predictions - """ - - assert ( - batch.satellite is not None - ), "Satellite data does not exist in batch, required for optical flow" - assert batch.metadata is not None, "Metadata does not exist in batch, required for optical flow" - - if final_image_size_pixels is None: - final_image_size_pixels = len(batch.satellite.x_index) - - # Only do optical flow for satellite data - optical_flow_predictions = [] - for i in range(batch.batch_size): - satellite_data: xr.DataArray = batch.satellite.sel(example=i) - t0_dt = batch.metadata.t0_dt.values[i] - optical_flow_predictions.append( - _compute_and_return_optical_flow( - satellite_data, t0_dt=t0_dt, final_image_size_pixels=final_image_size_pixels - ) - ) - # Concatenate all the DataArrays - dataarray = xr.concat(optical_flow_predictions, dim="example") - return dataarray - - -def _update_dataarray_with_predictions( - satellite_data: xr.DataArray, - predictions: np.ndarray, - t0_dt: pd.Timestamp, - final_image_size_pixels: int, -) -> xr.DataArray: - """ - Updates the dataarray with predictions - - Additionally, changes the temporal size to t0+1 to forecast horizon - - Args: - satellite_data: Satellite data - predictions: Predictions from the optical flow - - Returns: - The Xarray DataArray with the optical flow predictions - """ - - # Combine all channels for a single timestep - satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) - # Make sure its the correct size - buffer = satellite_data.sizes["x"] - final_image_size_pixels // 2 - satellite_data = satellite_data.isel( - x=slice(buffer, satellite_data.sizes["x"] - buffer), - y=slice(buffer, satellite_data.sizes["y"] - buffer), - ) - dataarray = xr.DataArray( - data=predictions, - dims=satellite_data.dims, - coords=satellite_data.coords, - ) - - return dataarray - - -def _get_previous_timesteps( - satellite_data: xr.DataArray, - t0_dt: pd.Timestamp, -) -> xr.DataArray: - """ - Get timestamp of previous - - Args: - satellite_data: Satellite data to use - t0_dt: Timestamp - - Returns: - The previous timesteps - """ - satellite_data = satellite_data.where(satellite_data.time <= t0_dt, drop=True) - return satellite_data - - -def _get_number_future_timesteps(satellite_data: xr.DataArray, t0_dt: pd.Timestamp) -> int: - """ - Get number of future timestamps - - Args: - satellite_data: Satellite data to use - t0_dt: The timestamp of the t0 image - - Returns: - The number of future timesteps - """ - satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) - return len(satellite_data.coords["time_index"]) - - -def _compute_and_return_optical_flow( - satellite_data: xr.DataArray, - t0_dt: pd.Timestamp, - final_image_size_pixels: int, -) -> xr.DataArray: - """ - Compute and return optical flow predictions for the example - - Args: - satellite_data: Satellite DataArray - t0_dt: t0 timestamp - - Returns: - The Tensor with the optical flow predictions for t0 to forecast horizon - """ - - # Get the previous timestamp - future_timesteps = _get_number_future_timesteps(satellite_data, t0_dt) - satellite_data: xr.DataArray = _get_previous_timesteps( - satellite_data, - t0_dt=t0_dt, - ) - prediction_block = np.zeros( - ( - future_timesteps, - final_image_size_pixels, - final_image_size_pixels, - satellite_data.sizes["channels_index"], - ) - ) - for prediction_timestep in range(future_timesteps): - for channel in range(0, len(satellite_data.coords["channels_index"]), 4): - # Optical Flow works with RGB images, so chunking channels for it to be faster - channel_images = satellite_data.sel(channels_index=slice(channel, channel + 3)) - # Extra 1 in shape from time dimension, so removing that dimension - t0_image = channel_images.isel( - time_index=len(satellite_data.time_index) - 1 - ).data.values - previous_image = channel_images.isel( - time_index=len(satellite_data.time_index) - 2 - ).data.values - optical_flow = _compute_optical_flow(t0_image, previous_image) - # Do predictions now - flow = optical_flow * prediction_timestep + 1 # Otherwise first prediction would be 0 - warped_image = _remap_image(t0_image, flow) - warped_image = crop_center( - warped_image, - final_image_size_pixels, - final_image_size_pixels, - ) - prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image - # Convert to correct C, T, H, W order - prediction_block = np.permute(prediction_block, [3, 0, 1, 2]) - dataarray = _update_dataarray_with_predictions( - satellite_data=satellite_data, predictions=prediction_block, t0_dt=t0_dt - ) - return dataarray - - -def _compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: - """ - Compute the optical flow for a set of images - - Args: - t0_image: t0 image - previous_image: previous image to compute optical flow with - - Returns: - Optical Flow field - """ - # Input images have to be single channel and between 0 and 1 - image_min = np.min([t0_image, previous_image]) - image_max = np.max([t0_image, previous_image]) - t0_image -= image_min - t0_image /= image_max - previous_image -= image_min - previous_image /= image_max - t0_image = cv2.cvtColor(t0_image.astype(np.float32), cv2.COLOR_RGBA2GRAY) - previous_image = cv2.cvtColor(previous_image.astype(np.float32), cv2.COLOR_RGBA2GRAY) - return cv2.calcOpticalFlowFarneback( - prev=previous_image, - next=t0_image, - flow=None, - pyr_scale=0.5, - levels=2, - winsize=40, - iterations=3, - poly_n=5, - poly_sigma=0.7, - flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, - ) - - -def _remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: - """ - Takes an image and warps it forwards in time according to the flow field. - - Args: - image: The grayscale image to warp. - flow: A 3D array. The first two dimensions must be the same size as the first two - dimensions of the image. The third dimension represented the x and y displacement. - - Returns: Warped image. The border has values np.NaN. - """ - # Adapted from https://github.com/opencv/opencv/issues/11068 - height, width = flow.shape[:2] - remap = -flow.copy() - remap[..., 0] += np.arange(width) # map_x - remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y - return cv2.remap( - src=image, - map1=remap, - map2=None, - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=np.NaN, - ) - - -def crop_center(image, x_size, y_size): - """ - Crop center of numpy image - - Args: - image: Image to crop - x_size: Size in x direction - y_size: Size in y direction - - Returns: - The cropped image - """ - y, x, channels = image.shape - startx = x // 2 - (x_size // 2) - starty = y // 2 - (y_size // 2) - return image[starty : starty + y_size, startx : startx + x_size] diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py new file mode 100644 index 00000000..7d07cc41 --- /dev/null +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -0,0 +1,30 @@ +"""Test Optical Flow Data Source""" +import pytest +import tempfile + +import pytest + +from nowcasting_dataset.config.model import Configuration, InputData +from nowcasting_dataset.dataset.batch import Batch + +from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( + OpticalFlowDataSource, + ) + + +@pytest.fixture +def configuration(): # noqa: D103 + con = Configuration() + con.input_data = InputData.set_all_to_defaults() + con.process.batch_size = 4 + return con + + +def test_optical_flow_data_source_get_batch(configuration): # noqa: D103 + optical_flow_datasource = OpticalFlowDataSource(previous_timestep_for_flow = 1, + final_image_size_pixels = 64) + with tempfile.TemporaryDirectory() as dirpath: + Batch.fake(configuration=configuration).save_netcdf(path=dirpath, batch_i=0) + + optical_flow = optical_flow_datasource.get_batch(netcdf_path = dirpath, batch_idx = 0) + print(optical_flow) From e4b2f7c1eac3d1cacf59b6a60d22fae5e865adae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 13:35:30 +0000 Subject: [PATCH 051/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/data_sources/data_source.py | 3 +-- .../optical_flow/test_optical_flow_data_source.py | 13 ++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 94fe5ca2..b54209f5 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -5,7 +5,7 @@ from dataclasses import InitVar, dataclass from numbers import Number from pathlib import Path -from typing import Iterable, List, Tuple, Union, Optional +from typing import Iterable, List, Optional, Tuple, Union import pandas as pd import xarray as xr @@ -467,7 +467,6 @@ class DerivedDataSource(DataSource): history_minutes: int = 0 forecast_minutes: int = 0 - def datetime_index(self): """The datetime index of this datasource""" return NotImplementedError( diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 7d07cc41..2486474c 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -1,15 +1,13 @@ """Test Optical Flow Data Source""" -import pytest import tempfile import pytest from nowcasting_dataset.config.model import Configuration, InputData -from nowcasting_dataset.dataset.batch import Batch - from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( OpticalFlowDataSource, - ) +) +from nowcasting_dataset.dataset.batch import Batch @pytest.fixture @@ -21,10 +19,11 @@ def configuration(): # noqa: D103 def test_optical_flow_data_source_get_batch(configuration): # noqa: D103 - optical_flow_datasource = OpticalFlowDataSource(previous_timestep_for_flow = 1, - final_image_size_pixels = 64) + optical_flow_datasource = OpticalFlowDataSource( + previous_timestep_for_flow=1, final_image_size_pixels=64 + ) with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=configuration).save_netcdf(path=dirpath, batch_i=0) - optical_flow = optical_flow_datasource.get_batch(netcdf_path = dirpath, batch_idx = 0) + optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) print(optical_flow) From dc41d1034f53b9cfe521d91ab602caa12d669c4b Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 13:36:33 +0000 Subject: [PATCH 052/197] Fix lint --- nowcasting_dataset/data_sources/data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index b54209f5..3734dbad 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -5,7 +5,7 @@ from dataclasses import InitVar, dataclass from numbers import Number from pathlib import Path -from typing import Iterable, List, Optional, Tuple, Union +from typing import Iterable, List, Tuple, Union import pandas as pd import xarray as xr From 3a6f7a34a96159446101b9043e6fcd829fcc7b1b Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 13:42:54 +0000 Subject: [PATCH 053/197] Fix names --- nowcasting_dataset/config/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 9fc4d10b..b5fca9a5 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -185,7 +185,7 @@ class InputData(BaseModel): pv: Optional[PV] = None satellite: Optional[Satellite] = None - optical_flow: Optional[OpticalFlow] = None + opticalflow: Optional[OpticalFlow] = None nwp: Optional[NWP] = None gsp: Optional[GSP] = None topographic: Optional[Topographic] = None @@ -232,7 +232,7 @@ def set_forecast_and_history_minutes(cls, values): "gsp", "topographic", "sun", - "optical_flow", + "opticalflow", ) enabled_data_sources = [ data_source_name From 8c03f10c285da2031171019d7f3c5854bc45fd0e Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 13:59:31 +0000 Subject: [PATCH 054/197] Fix outside temp directory --- .../optical_flow/test_optical_flow_data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 2486474c..42189fc1 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -25,5 +25,5 @@ def test_optical_flow_data_source_get_batch(configuration): # noqa: D103 with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=configuration).save_netcdf(path=dirpath, batch_i=0) - optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) - print(optical_flow) + optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) + print(optical_flow) From 58ee6ef40864ea1178cd5a7a7b5a19263347ac13 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 14:05:38 +0000 Subject: [PATCH 055/197] Fix numpy --- .../data_sources/optical_flow/optical_flow_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index c21a786f..d7829232 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -174,7 +174,7 @@ def _compute_and_return_optical_flow( ) prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image # Convert to correct C, T, H, W order - prediction_block = np.permute(prediction_block, [3, 0, 1, 2]) + prediction_block = np.transpose(prediction_block, [3, 0, 1, 2]) dataarray = self._update_dataarray_with_predictions( satellite_data=satellite_data, predictions=prediction_block, t0_dt=t0_dt ) From d17dcfbb75d5097e49b59254d5d3cee3bd85e957 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 14:14:56 +0000 Subject: [PATCH 056/197] Change to use x_index --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 +++--- .../optical_flow/test_optical_flow_data_source.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index d7829232..38d52b6e 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -73,10 +73,10 @@ def _update_dataarray_with_predictions( # Combine all channels for a single timestep satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) # Make sure its the correct size - buffer = satellite_data.sizes["x"] - self.final_image_size_pixels // 2 + buffer = satellite_data.sizes["x_index"] - self.final_image_size_pixels // 2 satellite_data = satellite_data.isel( - x=slice(buffer, satellite_data.sizes["x"] - buffer), - y=slice(buffer, satellite_data.sizes["y"] - buffer), + x=slice(buffer, satellite_data.sizes["x_index"] - buffer), + y=slice(buffer, satellite_data.sizes["y_index"] - buffer), ) dataarray = xr.DataArray( data=predictions, diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 42189fc1..05f956c8 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -24,6 +24,6 @@ def test_optical_flow_data_source_get_batch(configuration): # noqa: D103 ) with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=configuration).save_netcdf(path=dirpath, batch_i=0) - + print(Batch.fake(configuration = configuration)) optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) print(optical_flow) From 94f5d91d19581bfeab749a05ba6a037dfa314e4c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 14:15:16 +0000 Subject: [PATCH 057/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/optical_flow/test_optical_flow_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 05f956c8..b54d11ae 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -24,6 +24,6 @@ def test_optical_flow_data_source_get_batch(configuration): # noqa: D103 ) with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=configuration).save_netcdf(path=dirpath, batch_i=0) - print(Batch.fake(configuration = configuration)) + print(Batch.fake(configuration=configuration)) optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) print(optical_flow) From a80d24c211f70a84694e3eb536d3a66e47e79444 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 14:23:03 +0000 Subject: [PATCH 058/197] Update index name --- .../data_sources/optical_flow/optical_flow_data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 38d52b6e..f74e54e4 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -75,8 +75,8 @@ def _update_dataarray_with_predictions( # Make sure its the correct size buffer = satellite_data.sizes["x_index"] - self.final_image_size_pixels // 2 satellite_data = satellite_data.isel( - x=slice(buffer, satellite_data.sizes["x_index"] - buffer), - y=slice(buffer, satellite_data.sizes["y_index"] - buffer), + x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), + y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), ) dataarray = xr.DataArray( data=predictions, From 0cb7efdc6769f734646b7ad28bc65c17528cc56b Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 14:29:49 +0000 Subject: [PATCH 059/197] Add more history to minutes --- .../optical_flow/test_optical_flow_data_source.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index b54d11ae..c2174f18 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -11,19 +11,21 @@ @pytest.fixture -def configuration(): # noqa: D103 +def optical_flow_configuration(): # noqa: D103 con = Configuration() con.input_data = InputData.set_all_to_defaults() con.process.batch_size = 4 + con.input_data.satellite.forecast_minutes = 60 + con.input_data.satellite.history_minutes = 30 return con -def test_optical_flow_data_source_get_batch(configuration): # noqa: D103 +def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( previous_timestep_for_flow=1, final_image_size_pixels=64 ) with tempfile.TemporaryDirectory() as dirpath: - Batch.fake(configuration=configuration).save_netcdf(path=dirpath, batch_i=0) - print(Batch.fake(configuration=configuration)) + Batch.fake(configuration=optical_flow_configuration).save_netcdf(path=dirpath, batch_i=0) + print(Batch.fake(configuration=optical_flow_configuration)) optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) print(optical_flow) From cb20097f04b7002d46a4464968f4dc851fcb2c4f Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 14:36:32 +0000 Subject: [PATCH 060/197] Fix time index --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index f74e54e4..be885be6 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -71,9 +71,11 @@ def _update_dataarray_with_predictions( """ # Combine all channels for a single timestep - satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) + satellite_data = satellite_data.isel(time_index=slice(satellite_data.sizes["time_index"] + - predictions.shape[1], + satellite_data.sizes["time_index"])) # Make sure its the correct size - buffer = satellite_data.sizes["x_index"] - self.final_image_size_pixels // 2 + buffer = (satellite_data.sizes["x_index"] - self.final_image_size_pixels) // 2 satellite_data = satellite_data.isel( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), From e7f3694963042b328813f30671c9110b7ce6d402 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 14:36:48 +0000 Subject: [PATCH 061/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../optical_flow/optical_flow_data_source.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index be885be6..c4f264b4 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -71,9 +71,12 @@ def _update_dataarray_with_predictions( """ # Combine all channels for a single timestep - satellite_data = satellite_data.isel(time_index=slice(satellite_data.sizes["time_index"] - - predictions.shape[1], - satellite_data.sizes["time_index"])) + satellite_data = satellite_data.isel( + time_index=slice( + satellite_data.sizes["time_index"] - predictions.shape[1], + satellite_data.sizes["time_index"], + ) + ) # Make sure its the correct size buffer = (satellite_data.sizes["x_index"] - self.final_image_size_pixels) // 2 satellite_data = satellite_data.isel( From 8aa083578e41c8282757bac9b657adb52e13086a Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 14:46:02 +0000 Subject: [PATCH 062/197] Reshape --- .../data_sources/optical_flow/optical_flow_data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index c4f264b4..40424ac4 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -73,7 +73,7 @@ def _update_dataarray_with_predictions( # Combine all channels for a single timestep satellite_data = satellite_data.isel( time_index=slice( - satellite_data.sizes["time_index"] - predictions.shape[1], + satellite_data.sizes["time_index"] - predictions.shape[0], satellite_data.sizes["time_index"], ) ) @@ -179,7 +179,7 @@ def _compute_and_return_optical_flow( ) prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image # Convert to correct C, T, H, W order - prediction_block = np.transpose(prediction_block, [3, 0, 1, 2]) + prediction_block = np.transpose(prediction_block, [0, 3, 1, 2]) dataarray = self._update_dataarray_with_predictions( satellite_data=satellite_data, predictions=prediction_block, t0_dt=t0_dt ) From 955ed74890234684d26d06cce1fe5679a99c4968 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:00:48 +0000 Subject: [PATCH 063/197] Fix shapes --- .../optical_flow/optical_flow_data_source.py | 13 ++++--------- .../optical_flow/test_optical_flow_data_source.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 40424ac4..3b48bfc5 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -71,12 +71,9 @@ def _update_dataarray_with_predictions( """ # Combine all channels for a single timestep - satellite_data = satellite_data.isel( - time_index=slice( - satellite_data.sizes["time_index"] - predictions.shape[0], - satellite_data.sizes["time_index"], - ) - ) + satellite_data = satellite_data.isel(time_index=slice(satellite_data.sizes["time_index"] + - predictions.shape[0], + satellite_data.sizes["time_index"])) # Make sure its the correct size buffer = (satellite_data.sizes["x_index"] - self.final_image_size_pixels) // 2 satellite_data = satellite_data.isel( @@ -178,10 +175,8 @@ def _compute_and_return_optical_flow( self.final_image_size_pixels, ) prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image - # Convert to correct C, T, H, W order - prediction_block = np.transpose(prediction_block, [0, 3, 1, 2]) dataarray = self._update_dataarray_with_predictions( - satellite_data=satellite_data, predictions=prediction_block, t0_dt=t0_dt + satellite_data=self._data, predictions=prediction_block, t0_dt=t0_dt ) return dataarray diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index c2174f18..d67f9a37 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -20,12 +20,19 @@ def optical_flow_configuration(): # noqa: D103 return con +def test_optical_flow_get_example(optical_flow_configuration): + optical_flow_datasource = OpticalFlowDataSource( + previous_timestep_for_flow=1, final_image_size_pixels=32 + ) + batch = Batch.fake(configuration=optical_flow_configuration) + example = optical_flow_datasource.get_example(batch=batch, example_idx = 0) + assert example.values.shape == (12, 32, 32, 12) + + def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( previous_timestep_for_flow=1, final_image_size_pixels=64 ) with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=optical_flow_configuration).save_netcdf(path=dirpath, batch_i=0) - print(Batch.fake(configuration=optical_flow_configuration)) optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) - print(optical_flow) From 4daf634f460dca79dd9e957abec4bf95cd47ac89 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 15:01:06 +0000 Subject: [PATCH 064/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../optical_flow/optical_flow_data_source.py | 9 ++++++--- .../optical_flow/test_optical_flow_data_source.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 3b48bfc5..7aabee41 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -71,9 +71,12 @@ def _update_dataarray_with_predictions( """ # Combine all channels for a single timestep - satellite_data = satellite_data.isel(time_index=slice(satellite_data.sizes["time_index"] - - predictions.shape[0], - satellite_data.sizes["time_index"])) + satellite_data = satellite_data.isel( + time_index=slice( + satellite_data.sizes["time_index"] - predictions.shape[0], + satellite_data.sizes["time_index"], + ) + ) # Make sure its the correct size buffer = (satellite_data.sizes["x_index"] - self.final_image_size_pixels) // 2 satellite_data = satellite_data.isel( diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index d67f9a37..afaa486b 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -23,9 +23,9 @@ def optical_flow_configuration(): # noqa: D103 def test_optical_flow_get_example(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( previous_timestep_for_flow=1, final_image_size_pixels=32 - ) + ) batch = Batch.fake(configuration=optical_flow_configuration) - example = optical_flow_datasource.get_example(batch=batch, example_idx = 0) + example = optical_flow_datasource.get_example(batch=batch, example_idx=0) assert example.values.shape == (12, 32, 32, 12) From 17bd0fbcfc6b0f8526acf50b03906c1d044c8f5a Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:04:46 +0000 Subject: [PATCH 065/197] Add to on premise config --- nowcasting_dataset/config/on_premises.yaml | 5 ++++ .../optical_flow/optical_flow_data_source.py | 23 +++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index b254943c..b871ebf1 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -56,6 +56,11 @@ input_data: topographic: topographic_filename: /mnt/storage_b/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/Topographic/europe_dem_1km_osgb.tif + # ------------------------- Optical Flow --------------- + opticalflow: + previous_timestep_for_flow: 1 + opticalflow_image_size_pixels: 64 + output_data: filepath: /mnt/storage_b/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/prepared_ML_training_data/v10/ process: diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 7aabee41..b402e76d 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -24,7 +24,7 @@ class OpticalFlowDataSource(DerivedDataSource): """ previous_timestep_for_flow: int = 1 - final_image_size_pixels: Optional[int] = None + opticalflow_image_size_pixels: Optional[int] = None def get_example( self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, **kwargs @@ -40,8 +40,8 @@ def get_example( """ - if self.final_image_size_pixels is None: - self.final_image_size_pixels = len(batch.satellite.x_index) + if self.opticalflow_image_size_pixels is None: + self.opticalflow_image_size_pixels = len(batch.satellite.x_index) # Only do optical flow for satellite data self._data: xr.DataArray = batch.satellite.sel(example=example_idx) @@ -55,7 +55,6 @@ def _update_dataarray_with_predictions( self, satellite_data: xr.DataArray, predictions: np.ndarray, - t0_dt: pd.Timestamp, ) -> xr.DataArray: """ Updates the dataarray with predictions @@ -78,7 +77,7 @@ def _update_dataarray_with_predictions( ) ) # Make sure its the correct size - buffer = (satellite_data.sizes["x_index"] - self.final_image_size_pixels) // 2 + buffer = (satellite_data.sizes["x_index"] - self.opticalflow_image_size_pixels) // 2 satellite_data = satellite_data.isel( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), @@ -150,8 +149,8 @@ def _compute_and_return_optical_flow( prediction_block = np.zeros( ( future_timesteps, - self.final_image_size_pixels, - self.final_image_size_pixels, + self.opticalflow_image_size_pixels, + self.opticalflow_image_size_pixels, satellite_data.sizes["channels_index"], ) ) @@ -172,14 +171,14 @@ def _compute_and_return_optical_flow( optical_flow * prediction_timestep + 1 ) # Otherwise first prediction would be 0 warped_image = self._remap_image(t0_image, flow) - warped_image = self.crop_center( + warped_image = self._crop_center( warped_image, - self.final_image_size_pixels, - self.final_image_size_pixels, + self.opticalflow_image_size_pixels, + self.opticalflow_image_size_pixels, ) prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image dataarray = self._update_dataarray_with_predictions( - satellite_data=self._data, predictions=prediction_block, t0_dt=t0_dt + satellite_data=self._data, predictions=prediction_block ) return dataarray @@ -241,7 +240,7 @@ def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: borderValue=np.NaN, ) - def crop_center(self, image, x_size, y_size): + def _crop_center(self, image: np.ndarray, x_size: int, y_size: int) -> np.ndarray: """ Crop center of numpy image From f2203a3260f75d2e6f6ca4759820a794d1b1aaee Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:07:06 +0000 Subject: [PATCH 066/197] Fix name --- .../optical_flow/test_optical_flow_data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index afaa486b..1db48949 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -22,7 +22,7 @@ def optical_flow_configuration(): # noqa: D103 def test_optical_flow_get_example(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( - previous_timestep_for_flow=1, final_image_size_pixels=32 + previous_timestep_for_flow=1, opticalflow_image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example(batch=batch, example_idx=0) @@ -31,7 +31,7 @@ def test_optical_flow_get_example(optical_flow_configuration): def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( - previous_timestep_for_flow=1, final_image_size_pixels=64 + previous_timestep_for_flow=1, opticalflow_image_size_pixels=64 ) with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=optical_flow_configuration).save_netcdf(path=dirpath, batch_i=0) From 01e637623fe7ae951871aebed3d363632471e5c0 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:17:56 +0000 Subject: [PATCH 067/197] Add assert --- .../data_sources/optical_flow/test_optical_flow_data_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 1db48949..c0142eac 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -31,8 +31,9 @@ def test_optical_flow_get_example(optical_flow_configuration): def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( - previous_timestep_for_flow=1, opticalflow_image_size_pixels=64 + previous_timestep_for_flow=1, opticalflow_image_size_pixels=32 ) with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=optical_flow_configuration).save_netcdf(path=dirpath, batch_i=0) optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) + assert optical_flow.values.shape == (4, 12, 32, 32, 12) From 658f4e201308858f669477ca9984a92300e03979 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:38:45 +0000 Subject: [PATCH 068/197] Add prints --- .../optical_flow/optical_flow_data_source.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index b402e76d..fd6a9584 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -46,6 +46,7 @@ def get_example( # Only do optical flow for satellite data self._data: xr.DataArray = batch.satellite.sel(example=example_idx) t0_dt = batch.metadata.t0_dt.values[example_idx] + print(self._data) selected_data = self._compute_and_return_optical_flow(self._data, t0_dt=t0_dt) @@ -68,7 +69,8 @@ def _update_dataarray_with_predictions( Returns: The Xarray DataArray with the optical flow predictions """ - + print("Satellite Update One") + print(satellite_data) # Combine all channels for a single timestep satellite_data = satellite_data.isel( time_index=slice( @@ -76,12 +78,16 @@ def _update_dataarray_with_predictions( satellite_data.sizes["time_index"], ) ) + print("Satellite Update Two") + print(satellite_data) # Make sure its the correct size buffer = (satellite_data.sizes["x_index"] - self.opticalflow_image_size_pixels) // 2 satellite_data = satellite_data.isel( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), ) + print("Satellite Update Three") + print(satellite_data) dataarray = xr.DataArray( data=predictions, dims=satellite_data.dims, @@ -154,6 +160,7 @@ def _compute_and_return_optical_flow( satellite_data.sizes["channels_index"], ) ) + print(f"Prediction Shape: {prediction_block.shape} Future Timestep: {future_timesteps}") for prediction_timestep in range(future_timesteps): for channel in range(0, len(satellite_data.coords["channels_index"]), 4): # Optical Flow works with RGB images, so chunking channels for it to be faster From 69c2ad1f8a32005c9d30768b35e4df6de61450ce Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:41:46 +0000 Subject: [PATCH 069/197] More debug --- .../data_sources/optical_flow/optical_flow_data_source.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index fd6a9584..41afd634 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -93,6 +93,8 @@ def _update_dataarray_with_predictions( dims=satellite_data.dims, coords=satellite_data.coords, ) + print("Satellite Update Four") + print(dataarray) return dataarray From 76b046cd16b5f4b8d5311906453c23d7c9c1c584 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:43:03 +0000 Subject: [PATCH 070/197] More debug --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 41afd634..05241935 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -69,8 +69,6 @@ def _update_dataarray_with_predictions( Returns: The Xarray DataArray with the optical flow predictions """ - print("Satellite Update One") - print(satellite_data) # Combine all channels for a single timestep satellite_data = satellite_data.isel( time_index=slice( @@ -78,16 +76,14 @@ def _update_dataarray_with_predictions( satellite_data.sizes["time_index"], ) ) - print("Satellite Update Two") - print(satellite_data) # Make sure its the correct size buffer = (satellite_data.sizes["x_index"] - self.opticalflow_image_size_pixels) // 2 satellite_data = satellite_data.isel( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), ) - print("Satellite Update Three") print(satellite_data) + print(predictions) dataarray = xr.DataArray( data=predictions, dims=satellite_data.dims, From d61633d8f7661693b59963a76f99eeda6ead1c7d Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:44:01 +0000 Subject: [PATCH 071/197] More debug --- .../data_sources/optical_flow/optical_flow_data_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 05241935..e585fc8a 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -90,7 +90,9 @@ def _update_dataarray_with_predictions( coords=satellite_data.coords, ) print("Satellite Update Four") - print(dataarray) + print(dataarray.values.shape) + print(dataarray.dims) + print(dataarray.coords) return dataarray From ee6ba3c0095a0c446b11d78dcbf49ac40e8ca79b Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:45:18 +0000 Subject: [PATCH 072/197] More debug --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index e585fc8a..242c2112 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -46,7 +46,6 @@ def get_example( # Only do optical flow for satellite data self._data: xr.DataArray = batch.satellite.sel(example=example_idx) t0_dt = batch.metadata.t0_dt.values[example_idx] - print(self._data) selected_data = self._compute_and_return_optical_flow(self._data, t0_dt=t0_dt) @@ -91,8 +90,8 @@ def _update_dataarray_with_predictions( ) print("Satellite Update Four") print(dataarray.values.shape) - print(dataarray.dims) - print(dataarray.coords) + #print(dataarray.dims) + #print(dataarray.coords) return dataarray @@ -160,7 +159,6 @@ def _compute_and_return_optical_flow( satellite_data.sizes["channels_index"], ) ) - print(f"Prediction Shape: {prediction_block.shape} Future Timestep: {future_timesteps}") for prediction_timestep in range(future_timesteps): for channel in range(0, len(satellite_data.coords["channels_index"]), 4): # Optical Flow works with RGB images, so chunking channels for it to be faster From 98930adaafab1d9728091da14619a3b604fad04f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 15:45:44 +0000 Subject: [PATCH 073/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/optical_flow/optical_flow_data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 242c2112..6e241f7e 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -90,8 +90,8 @@ def _update_dataarray_with_predictions( ) print("Satellite Update Four") print(dataarray.values.shape) - #print(dataarray.dims) - #print(dataarray.coords) + # print(dataarray.dims) + # print(dataarray.coords) return dataarray From 9b00926c986fd1608292189e2a9694091295fa28 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:46:04 +0000 Subject: [PATCH 074/197] More debug --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 242c2112..909de8d9 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -82,7 +82,7 @@ def _update_dataarray_with_predictions( y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), ) print(satellite_data) - print(predictions) + print(predictions.shape) dataarray = xr.DataArray( data=predictions, dims=satellite_data.dims, @@ -90,8 +90,8 @@ def _update_dataarray_with_predictions( ) print("Satellite Update Four") print(dataarray.values.shape) - #print(dataarray.dims) - #print(dataarray.coords) + print(dataarray.dims) + print(dataarray.coords) return dataarray From 0341ae1daf77c0549a9a6159d5ab01f8ed3d8b12 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:56:16 +0000 Subject: [PATCH 075/197] More debug --- .../data_sources/optical_flow/optical_flow_data_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 1964e4b1..1bf15d54 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -83,8 +83,9 @@ def _update_dataarray_with_predictions( ) print(satellite_data) print(predictions.shape) - print(satellite_data.dims) print(satellite_data.coords) + print(satellite_data.dims) + print(satellite_data.transpose({"time_index", "x_index", "y_index", "channels_index"}).dims) dataarray = xr.DataArray( data=predictions, dims=satellite_data.dims, From 2e2c2b52b9b7151d46361f85801df76b276299d8 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 15:58:57 +0000 Subject: [PATCH 076/197] More debug --- .../optical_flow/optical_flow_data_source.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 1bf15d54..34ca8e84 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -85,11 +85,16 @@ def _update_dataarray_with_predictions( print(predictions.shape) print(satellite_data.coords) print(satellite_data.dims) - print(satellite_data.transpose({"time_index", "x_index", "y_index", "channels_index"}).dims) dataarray = xr.DataArray( data=predictions, - dims=satellite_data.dims, - coords=satellite_data.coords, + dims={"time_index": satellite_data.dims["time_index"], + "x_index": satellite_data.dims["x_index"], + "y_index": satellite_data.dims["y_index"], + "channels_index": satellite_data.dims["channels_index"]}, + coords={"time_index": satellite_data.coords["time_index"], + "x_index": satellite_data.coords["x_index"], + "y_index": satellite_data.coords["y_index"], + "channels_index": satellite_data.coords["channels_index"]}, ) print("Satellite Update Four") print(dataarray.values.shape) From 094d21538d4d4b11ec1de8b413fdc8a10a65fc0d Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Wed, 10 Nov 2021 16:01:13 +0000 Subject: [PATCH 077/197] Remove deubg statements --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 34ca8e84..e8918e27 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -81,10 +81,6 @@ def _update_dataarray_with_predictions( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), ) - print(satellite_data) - print(predictions.shape) - print(satellite_data.coords) - print(satellite_data.dims) dataarray = xr.DataArray( data=predictions, dims={"time_index": satellite_data.dims["time_index"], @@ -96,8 +92,6 @@ def _update_dataarray_with_predictions( "y_index": satellite_data.coords["y_index"], "channels_index": satellite_data.coords["channels_index"]}, ) - print("Satellite Update Four") - print(dataarray.values.shape) return dataarray From 65b655a56857cb6e42f0ad23cf03c54ca5abc116 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 16:01:34 +0000 Subject: [PATCH 078/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../optical_flow/optical_flow_data_source.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index e8918e27..b5d7408e 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -83,14 +83,18 @@ def _update_dataarray_with_predictions( ) dataarray = xr.DataArray( data=predictions, - dims={"time_index": satellite_data.dims["time_index"], + dims={ + "time_index": satellite_data.dims["time_index"], "x_index": satellite_data.dims["x_index"], "y_index": satellite_data.dims["y_index"], - "channels_index": satellite_data.dims["channels_index"]}, - coords={"time_index": satellite_data.coords["time_index"], - "x_index": satellite_data.coords["x_index"], - "y_index": satellite_data.coords["y_index"], - "channels_index": satellite_data.coords["channels_index"]}, + "channels_index": satellite_data.dims["channels_index"], + }, + coords={ + "time_index": satellite_data.coords["time_index"], + "x_index": satellite_data.coords["x_index"], + "y_index": satellite_data.coords["y_index"], + "channels_index": satellite_data.coords["channels_index"], + }, ) return dataarray From e3c742ae0b0deed59cab6a08d60fbfb5db72fdb1 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 10:02:12 +0000 Subject: [PATCH 079/197] Update testing of OpticalFlowDataSource in Manager --- .../optical_flow/optical_flow_data_source.py | 16 ++--- tests/test_manager.py | 60 +++++++++++++++++++ 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index b5d7408e..c84721e9 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -24,7 +24,7 @@ class OpticalFlowDataSource(DerivedDataSource): """ previous_timestep_for_flow: int = 1 - opticalflow_image_size_pixels: Optional[int] = None + image_size_pixels: Optional[int] = None def get_example( self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, **kwargs @@ -40,8 +40,8 @@ def get_example( """ - if self.opticalflow_image_size_pixels is None: - self.opticalflow_image_size_pixels = len(batch.satellite.x_index) + if self.image_size_pixels is None: + self.image_size_pixels = len(batch.satellite.x_index) # Only do optical flow for satellite data self._data: xr.DataArray = batch.satellite.sel(example=example_idx) @@ -76,7 +76,7 @@ def _update_dataarray_with_predictions( ) ) # Make sure its the correct size - buffer = (satellite_data.sizes["x_index"] - self.opticalflow_image_size_pixels) // 2 + buffer = (satellite_data.sizes["x_index"] - self.image_size_pixels) // 2 satellite_data = satellite_data.isel( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), @@ -158,8 +158,8 @@ def _compute_and_return_optical_flow( prediction_block = np.zeros( ( future_timesteps, - self.opticalflow_image_size_pixels, - self.opticalflow_image_size_pixels, + self.image_size_pixels, + self.image_size_pixels, satellite_data.sizes["channels_index"], ) ) @@ -182,8 +182,8 @@ def _compute_and_return_optical_flow( warped_image = self._remap_image(t0_image, flow) warped_image = self._crop_center( warped_image, - self.opticalflow_image_size_pixels, - self.opticalflow_image_size_pixels, + self.image_size_pixels, + self.image_size_pixels, ) prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image dataarray = self._update_dataarray_with_predictions( diff --git a/tests/test_manager.py b/tests/test_manager.py index 2c86fccb..7fad22b2 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -8,6 +8,7 @@ import pandas as pd import nowcasting_dataset +from nowcasting_dataset.data_sources import OpticalFlowDataSource from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource from nowcasting_dataset.manager import Manager @@ -135,6 +136,65 @@ def test_batches(): assert os.path.exists(f"{dst_path}/train/gsp/000001.nc") assert os.path.exists(f"{dst_path}/train/sat/000001.nc") +def test_derived_batches(): + """Test that derived batches can be made""" + filename = Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "sat_data.zarr" + + sat = SatelliteDataSource( + zarr_path=filename, + history_minutes=30, + forecast_minutes=60, + image_size_pixels=64, + meters_per_pixel=2000, + channels=("HRV",), + ) + + filename = ( + Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "gsp" / "test.zarr" + ) + + gsp = GSPDataSource( + zarr_path=filename, + start_dt=datetime(2019, 1, 1), + end_dt=datetime(2019, 1, 2), + history_minutes=30, + forecast_minutes=60, + image_size_pixels=64, + meters_per_pixel=2000, + ) + + of = OpticalFlowDataSource( + history_minutes=30, + forecast_minutes=60, + image_size_pixels=32, + ) + + manager = Manager() + + # load config + local_path = Path(nowcasting_dataset.__file__).parent.parent + filename = local_path / "tests" / "config" / "test.yaml" + manager.load_yaml_configuration(filename=filename) + + with tempfile.TemporaryDirectory() as local_temp_path, tempfile.TemporaryDirectory() as dst_path: # noqa 101 + + # set local temp path, and dst path + manager.config.output_data.filepath = Path(dst_path) + manager.local_temp_path = Path(local_temp_path) + + # just set satellite as data source + manager.data_sources = {"gsp": gsp, "sat": sat} + manager.derived_data_sources = {"opticalflow": of} + manager.data_source_which_defines_geospatial_locations = gsp + + # make file for locations + manager.create_files_specifying_spatial_and_temporal_locations_of_each_example_if_necessary() # noqa 101 + + # make batches + manager.create_batches(overwrite_batches=True) + + # make derived batches + manager.create_derived_batches(overwrite_batches = True) def test_save_config(): """Test that configuration file is saved""" From 9adaf4beeb43c22f9d587ccf4c87212092126261 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 10:02:32 +0000 Subject: [PATCH 080/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_manager.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_manager.py b/tests/test_manager.py index 7fad22b2..fb69dfcb 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -136,6 +136,7 @@ def test_batches(): assert os.path.exists(f"{dst_path}/train/gsp/000001.nc") assert os.path.exists(f"{dst_path}/train/sat/000001.nc") + def test_derived_batches(): """Test that derived batches can be made""" filename = Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "sat_data.zarr" @@ -147,10 +148,10 @@ def test_derived_batches(): image_size_pixels=64, meters_per_pixel=2000, channels=("HRV",), - ) + ) filename = ( - Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "gsp" / "test.zarr" + Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "gsp" / "test.zarr" ) gsp = GSPDataSource( @@ -161,13 +162,13 @@ def test_derived_batches(): forecast_minutes=60, image_size_pixels=64, meters_per_pixel=2000, - ) + ) of = OpticalFlowDataSource( history_minutes=30, forecast_minutes=60, image_size_pixels=32, - ) + ) manager = Manager() @@ -194,7 +195,8 @@ def test_derived_batches(): manager.create_batches(overwrite_batches=True) # make derived batches - manager.create_derived_batches(overwrite_batches = True) + manager.create_derived_batches(overwrite_batches=True) + def test_save_config(): """Test that configuration file is saved""" From 3a8875f654d03729043d0809bed3dd25e4a8fbb1 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 14:30:01 +0000 Subject: [PATCH 081/197] Do Optical Flow per channel --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index c84721e9..7d41c781 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -164,9 +164,9 @@ def _compute_and_return_optical_flow( ) ) for prediction_timestep in range(future_timesteps): - for channel in range(0, len(satellite_data.coords["channels_index"]), 4): + for channel in range(0, len(satellite_data.coords["channels_index"])): # Optical Flow works with RGB images, so chunking channels for it to be faster - channel_images = satellite_data.sel(channels_index=slice(channel, channel + 3)) + channel_images = satellite_data.sel(channels_index=channel) # Extra 1 in shape from time dimension, so removing that dimension t0_image = channel_images.isel( time_index=len(satellite_data.time_index) - 1 @@ -209,8 +209,6 @@ def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray t0_image /= image_max previous_image -= image_min previous_image /= image_max - t0_image = cv2.cvtColor(t0_image.astype(np.float32), cv2.COLOR_RGBA2GRAY) - previous_image = cv2.cvtColor(previous_image.astype(np.float32), cv2.COLOR_RGBA2GRAY) return cv2.calcOpticalFlowFarneback( prev=previous_image, next=t0_image, From 5b7bbeb8b42091cc6eaada1d8464765f5abca1e2 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 14:31:05 +0000 Subject: [PATCH 082/197] Update nowcasting_dataset/data_sources/data_source.py Co-authored-by: Jack Kelly --- nowcasting_dataset/data_sources/data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 3734dbad..fe0e1ef4 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -544,7 +544,7 @@ def get_batch( Args: netcdf_path: Path to the NetCDF files of the Batch to load - batch_idx: The batch ID to load from those in teh path + batch_idx: The batch ID to load from those in the path Returns: Batch of the derived data source From 87a5127f4ec531a4fed9952016d42cf5fe24a269 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 14:41:51 +0000 Subject: [PATCH 083/197] Switch to ProcessPoolExecuter --- nowcasting_dataset/data_sources/data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index fe0e1ef4..d2584ebe 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -553,7 +553,7 @@ def get_batch( import nowcasting_dataset.dataset.batch batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(netcdf_path, batch_idx=batch_idx) - with futures.ThreadPoolExecutor(max_workers=batch.batch_size) as executor: + with futures.ProcessPoolExecutor(max_workers=batch.batch_size) as executor: future_examples = [] for example_idx in range(batch.batch_size): future_example = executor.submit(self.get_example, batch, example_idx) From 609736bb511c9a61bd47ddc327377554849057a2 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 16:12:56 +0000 Subject: [PATCH 084/197] Address some PR comments --- nowcasting_dataset/config/model.py | 2 +- .../optical_flow/optical_flow_data_source.py | 175 +++++++++--------- .../satellite/satellite_data_source.py | 2 +- 3 files changed, 89 insertions(+), 90 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index b5fca9a5..9319fdd1 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -118,7 +118,7 @@ class OpticalFlow(DataSourceMixin): """Optical Flow configuration model""" previous_timestep_to_use: int = 1 - final_image_size_pixels: int = IMAGE_SIZE_PIXELS_FIELD + opticalflow_image_size_pixels: int = IMAGE_SIZE_PIXELS_FIELD class NWP(DataSourceMixin): diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 7d41c781..e0466014 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -19,12 +19,10 @@ class OpticalFlowDataSource(DerivedDataSource): """ Optical Flow Data Source, computing flow between Satellite data - - zarr_path: Must start with 'gs://' if on GCP. """ - previous_timestep_for_flow: int = 1 - image_size_pixels: Optional[int] = None + previous_timestep_to_use: int = 1 + opticalflow_image_size_pixels: Optional[int] = None def get_example( self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, **kwargs @@ -40,8 +38,8 @@ def get_example( """ - if self.image_size_pixels is None: - self.image_size_pixels = len(batch.satellite.x_index) + if self.opticalflow_image_size_pixels is None: + self.opticalflow_image_size_pixels = len(batch.satellite.x_index) # Only do optical flow for satellite data self._data: xr.DataArray = batch.satellite.sel(example=example_idx) @@ -76,7 +74,7 @@ def _update_dataarray_with_predictions( ) ) # Make sure its the correct size - buffer = (satellite_data.sizes["x_index"] - self.image_size_pixels) // 2 + buffer = (satellite_data.sizes["x_index"] - self.opticalflow_image_size_pixels) // 2 satellite_data = satellite_data.isel( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), @@ -158,108 +156,109 @@ def _compute_and_return_optical_flow( prediction_block = np.zeros( ( future_timesteps, - self.image_size_pixels, - self.image_size_pixels, + self.opticalflow_image_size_pixels, + self.opticalflow_image_size_pixels, satellite_data.sizes["channels_index"], ) ) for prediction_timestep in range(future_timesteps): + t0 = satellite_data.isel( + time_index=len(satellite_data.time_index) - 1 + ).data.values + previous = satellite_data.isel( + time_index=len(satellite_data.time_index) - 2 + ).data.values for channel in range(0, len(satellite_data.coords["channels_index"])): # Optical Flow works with RGB images, so chunking channels for it to be faster - channel_images = satellite_data.sel(channels_index=channel) + t0_image = t0.sel(channels_index=channel) + previous_image = previous.sel(channels_index=channel) # Extra 1 in shape from time dimension, so removing that dimension - t0_image = channel_images.isel( - time_index=len(satellite_data.time_index) - 1 - ).data.values - previous_image = channel_images.isel( - time_index=len(satellite_data.time_index) - 2 - ).data.values - optical_flow = self._compute_optical_flow(t0_image, previous_image) + optical_flow = compute_optical_flow(t0_image, previous_image) # Do predictions now flow = ( optical_flow * prediction_timestep + 1 ) # Otherwise first prediction would be 0 - warped_image = self._remap_image(t0_image, flow) - warped_image = self._crop_center( + warped_image = remap_image(t0_image, flow) + warped_image = crop_center( warped_image, - self.image_size_pixels, - self.image_size_pixels, + self.opticalflow_image_size_pixels, + self.opticalflow_image_size_pixels, ) - prediction_block[prediction_timestep, :, :, channel : channel + 4] = warped_image + prediction_block[prediction_timestep, :, :, channel] = warped_image dataarray = self._update_dataarray_with_predictions( satellite_data=self._data, predictions=prediction_block ) return dataarray - def _compute_optical_flow(self, t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: - """ - Compute the optical flow for a set of images - - Args: - t0_image: t0 image - previous_image: previous image to compute optical flow with - - Returns: - Optical Flow field - """ - # Input images have to be single channel and between 0 and 1 - image_min = np.min([t0_image, previous_image]) - image_max = np.max([t0_image, previous_image]) - t0_image -= image_min - t0_image /= image_max - previous_image -= image_min - previous_image /= image_max - return cv2.calcOpticalFlowFarneback( - prev=previous_image, - next=t0_image, - flow=None, - pyr_scale=0.5, - levels=2, - winsize=40, - iterations=3, - poly_n=5, - poly_sigma=0.7, - flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, - ) +def compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: + """ + Compute the optical flow for a set of images - def _remap_image(self, image: np.ndarray, flow: np.ndarray) -> np.ndarray: - """ - Takes an image and warps it forwards in time according to the flow field. + Args: + t0_image: t0 image + previous_image: previous image to compute optical flow with - Args: - image: The grayscale image to warp. - flow: A 3D array. The first two dimensions must be the same size as the first two - dimensions of the image. The third dimension represented the x and y displacement. + Returns: + Optical Flow field + """ + # Input images have to be single channel and between 0 and 1 + image_min = np.min([t0_image, previous_image]) + image_max = np.max([t0_image, previous_image]) + t0_image -= image_min + t0_image /= image_max + previous_image -= image_min + previous_image /= image_max + return cv2.calcOpticalFlowFarneback( + prev=previous_image, + next=t0_image, + flow=None, + pyr_scale=0.5, + levels=2, + winsize=40, + iterations=3, + poly_n=5, + poly_sigma=0.7, + flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, + ) + +def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: + """ + Takes an image and warps it forwards in time according to the flow field. - Returns: Warped image. The border has values np.NaN. - """ - # Adapted from https://github.com/opencv/opencv/issues/11068 - height, width = flow.shape[:2] - remap = -flow.copy() - remap[..., 0] += np.arange(width) # map_x - remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y - return cv2.remap( - src=image, - map1=remap, - map2=None, - interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, - borderValue=np.NaN, - ) + Args: + image: The grayscale image to warp. + flow: A 3D array. The first two dimensions must be the same size as the first two + dimensions of the image. The third dimension represented the x and y displacement. - def _crop_center(self, image: np.ndarray, x_size: int, y_size: int) -> np.ndarray: - """ - Crop center of numpy image + Returns: Warped image. The border has values np.NaN. + """ + # Adapted from https://github.com/opencv/opencv/issues/11068 + height, width = flow.shape[:2] + remap = -flow.copy() + remap[..., 0] += np.arange(width) # map_x + remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y + return cv2.remap( + src=image, + map1=remap, + map2=None, + interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=np.NaN, + ) + +def crop_center(image: np.ndarray, x_size: int, y_size: int) -> np.ndarray: + """ + Crop center of numpy image - Args: - image: Image to crop - x_size: Size in x direction - y_size: Size in y direction + Args: + image: Image to crop + x_size: Size in x direction + y_size: Size in y direction - Returns: - The cropped image - """ - y, x, channels = image.shape - startx = x // 2 - (x_size // 2) - starty = y // 2 - (y_size // 2) - return image[starty : starty + y_size, startx : startx + x_size] + Returns: + The cropped image + """ + y, x, channels = image.shape + startx = x // 2 - (x_size // 2) + starty = y // 2 - (y_size // 2) + return image[starty : starty + y_size, startx : startx + x_size] diff --git a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py index afdc4260..0ffddb33 100644 --- a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py +++ b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py @@ -147,5 +147,5 @@ def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: # the time dimension. # TODO Remove this as new Zarr already has the time fixed # See https://github.com/openclimatefix/nowcasting_dataset/issues/313 - data_array["time"] = data_array.time + pd.Timedelta("1 minute") + data_array["time"] = pd.DatetimeIndex(data_array.time).round('5 min') return data_array From c5233e58fa4d9502a285673081b9b153dc77001c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:13:19 +0000 Subject: [PATCH 085/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../optical_flow/optical_flow_data_source.py | 9 +++++---- .../data_sources/satellite/satellite_data_source.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index e0466014..ba007ed2 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -162,12 +162,10 @@ def _compute_and_return_optical_flow( ) ) for prediction_timestep in range(future_timesteps): - t0 = satellite_data.isel( - time_index=len(satellite_data.time_index) - 1 - ).data.values + t0 = satellite_data.isel(time_index=len(satellite_data.time_index) - 1).data.values previous = satellite_data.isel( time_index=len(satellite_data.time_index) - 2 - ).data.values + ).data.values for channel in range(0, len(satellite_data.coords["channels_index"])): # Optical Flow works with RGB images, so chunking channels for it to be faster t0_image = t0.sel(channels_index=channel) @@ -190,6 +188,7 @@ def _compute_and_return_optical_flow( ) return dataarray + def compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: """ Compute the optical flow for a set of images @@ -221,6 +220,7 @@ def compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, ) + def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: """ Takes an image and warps it forwards in time according to the flow field. @@ -246,6 +246,7 @@ def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: borderValue=np.NaN, ) + def crop_center(image: np.ndarray, x_size: int, y_size: int) -> np.ndarray: """ Crop center of numpy image diff --git a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py index 0ffddb33..2b7e1d84 100644 --- a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py +++ b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py @@ -147,5 +147,5 @@ def open_sat_data(zarr_path: str, consolidated: bool) -> xr.DataArray: # the time dimension. # TODO Remove this as new Zarr already has the time fixed # See https://github.com/openclimatefix/nowcasting_dataset/issues/313 - data_array["time"] = pd.DatetimeIndex(data_array.time).round('5 min') + data_array["time"] = pd.DatetimeIndex(data_array.time).round("5 min") return data_array From 81d5b4084d13c83c3c96c8d43723c7e8c6381eec Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 16:24:14 +0000 Subject: [PATCH 086/197] Address more PR comments --- .../optical_flow/optical_flow_data_source.py | 31 +++++-------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index ba007ed2..db2e4451 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -79,21 +79,8 @@ def _update_dataarray_with_predictions( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), ) - dataarray = xr.DataArray( - data=predictions, - dims={ - "time_index": satellite_data.dims["time_index"], - "x_index": satellite_data.dims["x_index"], - "y_index": satellite_data.dims["y_index"], - "channels_index": satellite_data.dims["channels_index"], - }, - coords={ - "time_index": satellite_data.coords["time_index"], - "x_index": satellite_data.coords["x_index"], - "y_index": satellite_data.coords["y_index"], - "channels_index": satellite_data.coords["channels_index"], - }, - ) + dataarray = xr.full_like(satellite_data, fill_value=np.NaN) + dataarray[:] = predictions return dataarray @@ -149,7 +136,7 @@ def _compute_and_return_optical_flow( # Get the previous timestamp future_timesteps = self._get_number_future_timesteps(satellite_data, t0_dt) - satellite_data: xr.DataArray = self._get_previous_timesteps( + historical_satellite_data: xr.DataArray = self._get_previous_timesteps( satellite_data, t0_dt=t0_dt, ) @@ -162,19 +149,17 @@ def _compute_and_return_optical_flow( ) ) for prediction_timestep in range(future_timesteps): - t0 = satellite_data.isel(time_index=len(satellite_data.time_index) - 1).data.values - previous = satellite_data.isel( - time_index=len(satellite_data.time_index) - 2 + t0 = historical_satellite_data.isel(time_index=len(historical_satellite_data.time_index) - 1).data.values + previous = historical_satellite_data.isel( + time_index=len(historical_satellite_data.time_index) - 2 ).data.values - for channel in range(0, len(satellite_data.coords["channels_index"])): - # Optical Flow works with RGB images, so chunking channels for it to be faster + for channel in range(0, len(historical_satellite_data.coords["channels_index"])): t0_image = t0.sel(channels_index=channel) previous_image = previous.sel(channels_index=channel) - # Extra 1 in shape from time dimension, so removing that dimension optical_flow = compute_optical_flow(t0_image, previous_image) # Do predictions now flow = ( - optical_flow * prediction_timestep + 1 + optical_flow * (prediction_timestep + 1) ) # Otherwise first prediction would be 0 warped_image = remap_image(t0_image, flow) warped_image = crop_center( From b11e0646660b5df5c5a8a5d5dc8d95b60f95ca7b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:24:32 +0000 Subject: [PATCH 087/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/optical_flow/optical_flow_data_source.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index db2e4451..e2d52007 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -149,7 +149,9 @@ def _compute_and_return_optical_flow( ) ) for prediction_timestep in range(future_timesteps): - t0 = historical_satellite_data.isel(time_index=len(historical_satellite_data.time_index) - 1).data.values + t0 = historical_satellite_data.isel( + time_index=len(historical_satellite_data.time_index) - 1 + ).data.values previous = historical_satellite_data.isel( time_index=len(historical_satellite_data.time_index) - 2 ).data.values @@ -158,8 +160,8 @@ def _compute_and_return_optical_flow( previous_image = previous.sel(channels_index=channel) optical_flow = compute_optical_flow(t0_image, previous_image) # Do predictions now - flow = ( - optical_flow * (prediction_timestep + 1) + flow = optical_flow * ( + prediction_timestep + 1 ) # Otherwise first prediction would be 0 warped_image = remap_image(t0_image, flow) warped_image = crop_center( From e2860343b3dbcc673f38980130cab6acbc81f887 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 16:40:08 +0000 Subject: [PATCH 088/197] Address more PR comments --- .../optical_flow/optical_flow_data_source.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index e2d52007..34206263 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -150,19 +150,17 @@ def _compute_and_return_optical_flow( ) for prediction_timestep in range(future_timesteps): t0 = historical_satellite_data.isel( - time_index=len(historical_satellite_data.time_index) - 1 - ).data.values + time_index=-1 + ) previous = historical_satellite_data.isel( - time_index=len(historical_satellite_data.time_index) - 2 - ).data.values + time_index=-2 + ) for channel in range(0, len(historical_satellite_data.coords["channels_index"])): - t0_image = t0.sel(channels_index=channel) - previous_image = previous.sel(channels_index=channel) + t0_image = t0.sel(channels_index=channel).data.values + previous_image = previous.sel(channels_index=channel).data.values optical_flow = compute_optical_flow(t0_image, previous_image) # Do predictions now - flow = optical_flow * ( - prediction_timestep + 1 - ) # Otherwise first prediction would be 0 + flow = optical_flow * (prediction_timestep + 1) warped_image = remap_image(t0_image, flow) warped_image = crop_center( warped_image, From 36f2bf558eb45818ba19faacfa24b27dfa642fcf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:41:02 +0000 Subject: [PATCH 089/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/optical_flow/optical_flow_data_source.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 34206263..abfc88bc 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -149,12 +149,8 @@ def _compute_and_return_optical_flow( ) ) for prediction_timestep in range(future_timesteps): - t0 = historical_satellite_data.isel( - time_index=-1 - ) - previous = historical_satellite_data.isel( - time_index=-2 - ) + t0 = historical_satellite_data.isel(time_index=-1) + previous = historical_satellite_data.isel(time_index=-2) for channel in range(0, len(historical_satellite_data.coords["channels_index"])): t0_image = t0.sel(channels_index=channel).data.values previous_image = previous.sel(channels_index=channel).data.values From f8465a872cbbc26ea67150b2b33b648944fe6d84 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:25:30 +0000 Subject: [PATCH 090/197] Add multi-timestep fixes --- .../optical_flow/optical_flow_data_source.py | 37 +++++++++++++++---- .../test_optical_flow_data_source.py | 12 +++++- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index abfc88bc..4e79f15c 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -79,8 +79,21 @@ def _update_dataarray_with_predictions( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), ) - dataarray = xr.full_like(satellite_data, fill_value=np.NaN) - dataarray[:] = predictions + dataarray = xr.DataArray( + data=predictions, + dims={ + "time_index": satellite_data.dims["time_index"], + "x_index": satellite_data.dims["x_index"], + "y_index": satellite_data.dims["y_index"], + "channels_index": satellite_data.dims["channels_index"], + }, + coords={ + "time_index": satellite_data.coords["time_index"], + "x_index": satellite_data.coords["x_index"], + "y_index": satellite_data.coords["y_index"], + "channels_index": satellite_data.coords["channels_index"], + }, + ) return dataarray @@ -149,13 +162,21 @@ def _compute_and_return_optical_flow( ) ) for prediction_timestep in range(future_timesteps): - t0 = historical_satellite_data.isel(time_index=-1) - previous = historical_satellite_data.isel(time_index=-2) for channel in range(0, len(historical_satellite_data.coords["channels_index"])): - t0_image = t0.sel(channels_index=channel).data.values - previous_image = previous.sel(channels_index=channel).data.values - optical_flow = compute_optical_flow(t0_image, previous_image) + t0 = historical_satellite_data.sel(channels_index=channel) + previous = historical_satellite_data.sel(channels_index=channel) + optical_flows = [] + for i in range(len(historical_satellite_data.coords[ + "time_index"])-1, len(historical_satellite_data.coords[ + "time_index"])-self.previous_timestep_to_use-1, -1): + t0_image = t0.isel(time_index=i).data.values + previous_image = previous.isel(time_index=i-1).data.values + optical_flow = compute_optical_flow(t0_image, previous_image) + optical_flows.append(optical_flow) + # Average predictions + optical_flow = np.mean(optical_flows, axis = 0) # Do predictions now + t0_image = t0.isel(time_index=-1).data.values flow = optical_flow * (prediction_timestep + 1) warped_image = remap_image(t0_image, flow) warped_image = crop_center( @@ -240,7 +261,7 @@ def crop_center(image: np.ndarray, x_size: int, y_size: int) -> np.ndarray: Returns: The cropped image """ - y, x, channels = image.shape + y, x = image.shape startx = x // 2 - (x_size // 2) starty = y // 2 - (y_size // 2) return image[starty : starty + y_size, startx : startx + x_size] diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index c0142eac..f41e9f82 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -1,5 +1,6 @@ """Test Optical Flow Data Source""" import tempfile +import numpy as np import pytest @@ -22,13 +23,22 @@ def optical_flow_configuration(): # noqa: D103 def test_optical_flow_get_example(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( - previous_timestep_for_flow=1, opticalflow_image_size_pixels=32 + previous_timestep_to_use=1, opticalflow_image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example(batch=batch, example_idx=0) assert example.values.shape == (12, 32, 32, 12) +def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): + optical_flow_datasource = OpticalFlowDataSource( + previous_timestep_to_use=3, opticalflow_image_size_pixels=32 + ) + batch = Batch.fake(configuration=optical_flow_configuration) + example = optical_flow_datasource.get_example(batch=batch, example_idx=0) + assert example.values.shape == (12, 32, 32, 12) + + def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( previous_timestep_for_flow=1, opticalflow_image_size_pixels=32 From 24fb0cf9dd8695e15a50e74c9a1d16225228cd72 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 17:25:53 +0000 Subject: [PATCH 091/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../optical_flow/optical_flow_data_source.py | 20 +++++++++++-------- .../test_optical_flow_data_source.py | 4 ++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 4e79f15c..cf53afe2 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -86,14 +86,14 @@ def _update_dataarray_with_predictions( "x_index": satellite_data.dims["x_index"], "y_index": satellite_data.dims["y_index"], "channels_index": satellite_data.dims["channels_index"], - }, + }, coords={ "time_index": satellite_data.coords["time_index"], "x_index": satellite_data.coords["x_index"], "y_index": satellite_data.coords["y_index"], "channels_index": satellite_data.coords["channels_index"], - }, - ) + }, + ) return dataarray @@ -166,15 +166,19 @@ def _compute_and_return_optical_flow( t0 = historical_satellite_data.sel(channels_index=channel) previous = historical_satellite_data.sel(channels_index=channel) optical_flows = [] - for i in range(len(historical_satellite_data.coords[ - "time_index"])-1, len(historical_satellite_data.coords[ - "time_index"])-self.previous_timestep_to_use-1, -1): + for i in range( + len(historical_satellite_data.coords["time_index"]) - 1, + len(historical_satellite_data.coords["time_index"]) + - self.previous_timestep_to_use + - 1, + -1, + ): t0_image = t0.isel(time_index=i).data.values - previous_image = previous.isel(time_index=i-1).data.values + previous_image = previous.isel(time_index=i - 1).data.values optical_flow = compute_optical_flow(t0_image, previous_image) optical_flows.append(optical_flow) # Average predictions - optical_flow = np.mean(optical_flows, axis = 0) + optical_flow = np.mean(optical_flows, axis=0) # Do predictions now t0_image = t0.isel(time_index=-1).data.values flow = optical_flow * (prediction_timestep + 1) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index f41e9f82..2ee4b038 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -1,7 +1,7 @@ """Test Optical Flow Data Source""" import tempfile -import numpy as np +import numpy as np import pytest from nowcasting_dataset.config.model import Configuration, InputData @@ -33,7 +33,7 @@ def test_optical_flow_get_example(optical_flow_configuration): def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( previous_timestep_to_use=3, opticalflow_image_size_pixels=32 - ) + ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example(batch=batch, example_idx=0) assert example.values.shape == (12, 32, 32, 12) From 0cc39c876a21f84a8e09c4984bca26d80d7843e8 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:28:38 +0000 Subject: [PATCH 092/197] Update docstring --- .../data_sources/optical_flow/optical_flow_data_source.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 4e79f15c..8f42d82a 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -19,9 +19,13 @@ class OpticalFlowDataSource(DerivedDataSource): """ Optical Flow Data Source, computing flow between Satellite data + + number_previous_timesteps_to_use: Number of previous timesteps to use, i.e. if 1, only uses the + flow between t-1 and t0 images, if 3, computes the flow between (t-3,t-2),(t-2,t-1), + and (t-1,t0) image pairs and uses the mean optical flow for future timesteps. """ - previous_timestep_to_use: int = 1 + number_previous_timesteps_to_use: int = 1 opticalflow_image_size_pixels: Optional[int] = None def get_example( @@ -168,7 +172,7 @@ def _compute_and_return_optical_flow( optical_flows = [] for i in range(len(historical_satellite_data.coords[ "time_index"])-1, len(historical_satellite_data.coords[ - "time_index"])-self.previous_timestep_to_use-1, -1): + "time_index"]) - self.number_previous_timesteps_to_use - 1, -1): t0_image = t0.isel(time_index=i).data.values previous_image = previous.isel(time_index=i-1).data.values optical_flow = compute_optical_flow(t0_image, previous_image) From 147c5ee7e569fe1a08e4863fbcd5b5d8634cb96d Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:29:38 +0000 Subject: [PATCH 093/197] Update model --- nowcasting_dataset/config/model.py | 2 +- nowcasting_dataset/config/on_premises.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 9319fdd1..ae37e7df 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -117,7 +117,7 @@ class Satellite(DataSourceMixin): class OpticalFlow(DataSourceMixin): """Optical Flow configuration model""" - previous_timestep_to_use: int = 1 + number_previous_timesteps_to_use: int = 1 opticalflow_image_size_pixels: int = IMAGE_SIZE_PIXELS_FIELD diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index b871ebf1..21f6dbb0 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -58,7 +58,7 @@ input_data: # ------------------------- Optical Flow --------------- opticalflow: - previous_timestep_for_flow: 1 + number_previous_timesteps_to_use: 1 opticalflow_image_size_pixels: 64 output_data: From 3eb38a78fdaac5bcb11a69121c06e7d1e5d38ba5 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:33:36 +0000 Subject: [PATCH 094/197] Update nowcasting_dataset/data_sources/data_source.py Co-authored-by: Jack Kelly --- nowcasting_dataset/data_sources/data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index d2584ebe..3f287075 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -489,7 +489,7 @@ def create_batches( Safe to call from worker processes. Args: - batch_path: Path to where the netcdf batches are stored + batch_path: Path to where the netcdf batches are stored (these will fed into the `DerivedDataSource`) total_number_batches: The total number of batches to make idx_of_first_batch: The batch number of the first batch to create. dst_path: The final destination path for the batches. Must exist. From 3980f019cab09dc4191c02374073a32dc612e591 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:34:55 +0000 Subject: [PATCH 095/197] Make docstring more descriptive --- nowcasting_dataset/data_sources/data_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 3f287075..34e40d8d 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -489,7 +489,9 @@ def create_batches( Safe to call from worker processes. Args: - batch_path: Path to where the netcdf batches are stored (these will fed into the `DerivedDataSource`) + batch_path: Path to where the netcdf batches are stored + (these will fed into the `DerivedDataSource`). This is the path to the top level path, + such as `foo/v10/train/` total_number_batches: The total number of batches to make idx_of_first_batch: The batch number of the first batch to create. dst_path: The final destination path for the batches. Must exist. From f2e65919124708750c10a306f1322f68de55b5fc Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:41:05 +0000 Subject: [PATCH 096/197] Add links to #367 #minor --- nowcasting_dataset/data_sources/data_source.py | 2 ++ nowcasting_dataset/manager.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 34e40d8d..d54036dc 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -135,6 +135,7 @@ def check_input_paths_exist(self) -> None: pass # TODO: Issue #319: Standardise parameter names. + # TODO: Reduce duplication: https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_batches( self, spatial_and_temporal_locations_of_each_example: pd.DataFrame, @@ -474,6 +475,7 @@ def datetime_index(self): "needed" ) + # TODO Reduce duplication https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_batches( self, batch_path: Path, diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 1a591d3a..5c056142 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -322,6 +322,7 @@ def _find_splits_which_need_more_batches( splits_which_need_more_batches.append(split_name) return splits_which_need_more_batches + # TODO: Reduce duplication: https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_derived_batches(self, overwrite_batches: bool) -> None: """ Create batches of derived data sources @@ -397,6 +398,7 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: logger.exception(f"Worker process {data_source_name} raised exception!") raise exception + # TODO: Reduce duplication: https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_batches(self, overwrite_batches: bool) -> None: """Create batches (if necessary). From 77169eaafda86eca3eae9d15de977ad51fe7aad0 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:43:19 +0000 Subject: [PATCH 097/197] Fix tests --- .../optical_flow/test_optical_flow_data_source.py | 6 +++--- tests/test_manager.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 2ee4b038..d317d1db 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -23,7 +23,7 @@ def optical_flow_configuration(): # noqa: D103 def test_optical_flow_get_example(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( - previous_timestep_to_use=1, opticalflow_image_size_pixels=32 + number_previous_timesteps_to_use=1, opticalflow_image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example(batch=batch, example_idx=0) @@ -32,7 +32,7 @@ def test_optical_flow_get_example(optical_flow_configuration): def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( - previous_timestep_to_use=3, opticalflow_image_size_pixels=32 + number_previous_timesteps_to_use=3, opticalflow_image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example(batch=batch, example_idx=0) @@ -41,7 +41,7 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( - previous_timestep_for_flow=1, opticalflow_image_size_pixels=32 + number_previous_timesteps_to_use=1, opticalflow_image_size_pixels=32 ) with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=optical_flow_configuration).save_netcdf(path=dirpath, batch_i=0) diff --git a/tests/test_manager.py b/tests/test_manager.py index fb69dfcb..2bdff114 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -167,7 +167,7 @@ def test_derived_batches(): of = OpticalFlowDataSource( history_minutes=30, forecast_minutes=60, - image_size_pixels=32, + opticalflow_image_size_pixels=32, ) manager = Manager() From 0768c4029fd0cee5df9bfdf05827382dd327947d Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:52:15 +0000 Subject: [PATCH 098/197] Add assert, fix error --- .../data_sources/optical_flow/optical_flow_data_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index a1396c90..fdb3dcbc 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -157,6 +157,7 @@ def _compute_and_return_optical_flow( satellite_data, t0_dt=t0_dt, ) + assert len(historical_satellite_data.coords["time_index"])-self.number_previous_timesteps_to_use- 1, ValueError("Trying to compute flow further back than the number of historical timesteps") prediction_block = np.zeros( ( future_timesteps, @@ -173,7 +174,7 @@ def _compute_and_return_optical_flow( for i in range( len(historical_satellite_data.coords["time_index"]) - 1, len(historical_satellite_data.coords["time_index"]) - - self.previous_timestep_to_use + - self.number_previous_timesteps_to_use - 1, -1, ): From f5165caaca3a16eb5aa2aa26af73c210a7693257 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 17:52:35 +0000 Subject: [PATCH 099/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index fdb3dcbc..ececd5c6 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -157,7 +157,11 @@ def _compute_and_return_optical_flow( satellite_data, t0_dt=t0_dt, ) - assert len(historical_satellite_data.coords["time_index"])-self.number_previous_timesteps_to_use- 1, ValueError("Trying to compute flow further back than the number of historical timesteps") + assert ( + len(historical_satellite_data.coords["time_index"]) + - self.number_previous_timesteps_to_use + - 1 + ), ValueError("Trying to compute flow further back than the number of historical timesteps") prediction_block = np.zeros( ( future_timesteps, From 5e800b76de7d8815705bc4fd788a093a6277b6fd Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:53:11 +0000 Subject: [PATCH 100/197] Fix assert --- .../data_sources/optical_flow/optical_flow_data_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index ececd5c6..5d2502a3 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -161,7 +161,8 @@ def _compute_and_return_optical_flow( len(historical_satellite_data.coords["time_index"]) - self.number_previous_timesteps_to_use - 1 - ), ValueError("Trying to compute flow further back than the number of historical timesteps") + ) >= 0, ValueError("Trying to compute flow further back than the number of historical " + "timesteps") prediction_block = np.zeros( ( future_timesteps, From df661f36bae4b5475d1adad2fab452c54a353dce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 17:53:30 +0000 Subject: [PATCH 101/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/optical_flow/optical_flow_data_source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 5d2502a3..3e09f293 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -161,8 +161,9 @@ def _compute_and_return_optical_flow( len(historical_satellite_data.coords["time_index"]) - self.number_previous_timesteps_to_use - 1 - ) >= 0, ValueError("Trying to compute flow further back than the number of historical " - "timesteps") + ) >= 0, ValueError( + "Trying to compute flow further back than the number of historical " "timesteps" + ) prediction_block = np.zeros( ( future_timesteps, From 39aacd98b6395846aa434783e33ddf39cc74db70 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 17:55:02 +0000 Subject: [PATCH 102/197] Add test for assert --- .../data_sources/optical_flow/optical_flow_data_source.py | 2 +- .../optical_flow/test_optical_flow_data_source.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 3e09f293..0933a457 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -162,7 +162,7 @@ def _compute_and_return_optical_flow( - self.number_previous_timesteps_to_use - 1 ) >= 0, ValueError( - "Trying to compute flow further back than the number of historical " "timesteps" + "Trying to compute flow further back than the number of historical timesteps" ) prediction_block = np.zeros( ( diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index d317d1db..fea032e3 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -38,6 +38,14 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): example = optical_flow_datasource.get_example(batch=batch, example_idx=0) assert example.values.shape == (12, 32, 32, 12) +def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration): + optical_flow_datasource = OpticalFlowDataSource( + number_previous_timesteps_to_use=300, opticalflow_image_size_pixels=32 + ) + batch = Batch.fake(configuration=optical_flow_configuration) + with pytest.raises(ValueError): + example = optical_flow_datasource.get_example(batch=batch, example_idx=0) + def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( From dae02dbe36284de267f5ab6d478083528f97fbb6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 17:55:20 +0000 Subject: [PATCH 103/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/optical_flow/test_optical_flow_data_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index fea032e3..8b7d0946 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -38,10 +38,11 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): example = optical_flow_datasource.get_example(batch=batch, example_idx=0) assert example.values.shape == (12, 32, 32, 12) + def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( number_previous_timesteps_to_use=300, opticalflow_image_size_pixels=32 - ) + ) batch = Batch.fake(configuration=optical_flow_configuration) with pytest.raises(ValueError): example = optical_flow_datasource.get_example(batch=batch, example_idx=0) From d523d27f9e66d4247f136a8a91c1c847a665e2b1 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 18:06:05 +0000 Subject: [PATCH 104/197] Add giving different data sources --- .../optical_flow/optical_flow_data_source.py | 4 +--- nowcasting_dataset/manager.py | 10 +++++----- .../optical_flow/test_optical_flow_data_source.py | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 0933a457..3c7a831c 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -161,9 +161,7 @@ def _compute_and_return_optical_flow( len(historical_satellite_data.coords["time_index"]) - self.number_previous_timesteps_to_use - 1 - ) >= 0, ValueError( - "Trying to compute flow further back than the number of historical timesteps" - ) + ) >= 0, "Trying to compute flow further back than the number of historical timesteps" prediction_block = np.zeros( ( future_timesteps, diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 5c056142..d1415592 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -267,7 +267,7 @@ def sample_spatial_and_temporal_locations_for_examples( ) def _get_first_batches_to_create( - self, overwrite_batches: bool + self, overwrite_batches: bool, data_sources: dict, ) -> dict[split.SplitName, dict[str, int]]: """For each SplitName & for each DataSource name, return the first batch ID to create. @@ -278,7 +278,7 @@ def _get_first_batches_to_create( first_batches_to_create: dict[split.SplitName, dict[str, int]] = {} for split_name in split.SplitName: first_batches_to_create[split_name] = { - data_source_name: 0 for data_source_name in self.data_sources + data_source_name: 0 for data_source_name in data_sources } if overwrite_batches: @@ -286,7 +286,7 @@ def _get_first_batches_to_create( # If we're not overwriting batches then find the last batch on disk. for split_name in split.SplitName: - for data_source_name in self.data_sources: + for data_source_name in data_sources: path = ( self.config.output_data.filepath / split_name.value / data_source_name / "*.nc" ) @@ -335,7 +335,7 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: written to disk, and only create any batches which have not yet been written to disk. """ - first_batches_to_create = self._get_first_batches_to_create(overwrite_batches) + first_batches_to_create = self._get_first_batches_to_create(overwrite_batches, self.derived_data_sources) # Check if there's any work to do. if overwrite_batches: @@ -411,7 +411,7 @@ def create_batches(self, overwrite_batches: bool) -> None: previously been written to disk. If False then check which batches have previously been written to disk, and only create any batches which have not yet been written to disk. """ - first_batches_to_create = self._get_first_batches_to_create(overwrite_batches) + first_batches_to_create = self._get_first_batches_to_create(overwrite_batches, self.data_sources) # Check if there's any work to do. if overwrite_batches: diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 8b7d0946..7d1f5f6e 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -44,8 +44,8 @@ def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration) number_previous_timesteps_to_use=300, opticalflow_image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) - with pytest.raises(ValueError): - example = optical_flow_datasource.get_example(batch=batch, example_idx=0) + with pytest.raises(AssertionError): + optical_flow_datasource.get_example(batch=batch, example_idx=0) def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 From 2f44b8cf86e6a169cdb531aeccb91ad2377b90a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 18:06:26 +0000 Subject: [PATCH 105/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/manager.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index d1415592..381f0dbd 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -267,7 +267,9 @@ def sample_spatial_and_temporal_locations_for_examples( ) def _get_first_batches_to_create( - self, overwrite_batches: bool, data_sources: dict, + self, + overwrite_batches: bool, + data_sources: dict, ) -> dict[split.SplitName, dict[str, int]]: """For each SplitName & for each DataSource name, return the first batch ID to create. @@ -335,7 +337,9 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: written to disk, and only create any batches which have not yet been written to disk. """ - first_batches_to_create = self._get_first_batches_to_create(overwrite_batches, self.derived_data_sources) + first_batches_to_create = self._get_first_batches_to_create( + overwrite_batches, self.derived_data_sources + ) # Check if there's any work to do. if overwrite_batches: @@ -411,7 +415,9 @@ def create_batches(self, overwrite_batches: bool) -> None: previously been written to disk. If False then check which batches have previously been written to disk, and only create any batches which have not yet been written to disk. """ - first_batches_to_create = self._get_first_batches_to_create(overwrite_batches, self.data_sources) + first_batches_to_create = self._get_first_batches_to_create( + overwrite_batches, self.data_sources + ) # Check if there's any work to do. if overwrite_batches: From 83ebe159c9c92c5e7dd65656441f8819d217f986 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Thu, 11 Nov 2021 18:20:08 +0000 Subject: [PATCH 106/197] Fix error --- nowcasting_dataset/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 381f0dbd..4d5db1e4 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -393,7 +393,7 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: # Wait for all futures to finish: for future, data_source_name in zip( - future_create_batches_jobs, self.data_sources.keys() + future_create_batches_jobs, self.derived_data_sources.keys() ): # Call exception() to propagate any exceptions raised by the worker process into # the main process, and to wait for the worker to finish. From 44dfc699054ac3e6b24c932a412818909b82e35c Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 08:08:35 +0000 Subject: [PATCH 107/197] Update test configs --- tests/config/nwp_size_test.yaml | 3 +++ tests/config/test.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/config/nwp_size_test.yaml b/tests/config/nwp_size_test.yaml index 176a08a5..dd3fff75 100644 --- a/tests/config/nwp_size_test.yaml +++ b/tests/config/nwp_size_test.yaml @@ -22,6 +22,9 @@ input_data: sun_zarr_path: tests/data/sun/test.zarr topographic: topographic_filename: tests/data/europe_dem_2km_osgb.tif + opticalflow: + number_previous_timesteps_to_use: 1 + opticalflow_image_size_pixels: 32 output_data: filepath: not used by unittests! process: diff --git a/tests/config/test.yaml b/tests/config/test.yaml index 37f846cc..8565da87 100644 --- a/tests/config/test.yaml +++ b/tests/config/test.yaml @@ -23,6 +23,9 @@ input_data: sun_zarr_path: tests/data/sun/test.zarr topographic: topographic_filename: tests/data/europe_dem_2km_osgb.tif + opticalflow: + number_previous_timesteps_to_use: 1 + opticalflow_image_size_pixels: 32 output_data: filepath: not used by unittests! process: From a2e30145b0bd4d5144d99550b5a09c5fdd66c4e9 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 08:15:52 +0000 Subject: [PATCH 108/197] Fix name --- .../optical_flow/optical_flow_data_source.py | 16 ++++++++-------- .../test_optical_flow_data_source.py | 8 ++++---- tests/test_manager.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 3c7a831c..40b0356b 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -26,7 +26,7 @@ class OpticalFlowDataSource(DerivedDataSource): """ number_previous_timesteps_to_use: int = 1 - opticalflow_image_size_pixels: Optional[int] = None + image_size_pixels: Optional[int] = None def get_example( self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, **kwargs @@ -42,8 +42,8 @@ def get_example( """ - if self.opticalflow_image_size_pixels is None: - self.opticalflow_image_size_pixels = len(batch.satellite.x_index) + if self.image_size_pixels is None: + self.image_size_pixels = len(batch.satellite.x_index) # Only do optical flow for satellite data self._data: xr.DataArray = batch.satellite.sel(example=example_idx) @@ -78,7 +78,7 @@ def _update_dataarray_with_predictions( ) ) # Make sure its the correct size - buffer = (satellite_data.sizes["x_index"] - self.opticalflow_image_size_pixels) // 2 + buffer = (satellite_data.sizes["x_index"] - self.image_size_pixels) // 2 satellite_data = satellite_data.isel( x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), @@ -165,8 +165,8 @@ def _compute_and_return_optical_flow( prediction_block = np.zeros( ( future_timesteps, - self.opticalflow_image_size_pixels, - self.opticalflow_image_size_pixels, + self.image_size_pixels, + self.image_size_pixels, satellite_data.sizes["channels_index"], ) ) @@ -194,8 +194,8 @@ def _compute_and_return_optical_flow( warped_image = remap_image(t0_image, flow) warped_image = crop_center( warped_image, - self.opticalflow_image_size_pixels, - self.opticalflow_image_size_pixels, + self.image_size_pixels, + self.image_size_pixels, ) prediction_block[prediction_timestep, :, :, channel] = warped_image dataarray = self._update_dataarray_with_predictions( diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 7d1f5f6e..dbad8cb8 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -23,7 +23,7 @@ def optical_flow_configuration(): # noqa: D103 def test_optical_flow_get_example(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( - number_previous_timesteps_to_use=1, opticalflow_image_size_pixels=32 + number_previous_timesteps_to_use=1, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example(batch=batch, example_idx=0) @@ -32,7 +32,7 @@ def test_optical_flow_get_example(optical_flow_configuration): def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( - number_previous_timesteps_to_use=3, opticalflow_image_size_pixels=32 + number_previous_timesteps_to_use=3, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example(batch=batch, example_idx=0) @@ -41,7 +41,7 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration): optical_flow_datasource = OpticalFlowDataSource( - number_previous_timesteps_to_use=300, opticalflow_image_size_pixels=32 + number_previous_timesteps_to_use=300, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) with pytest.raises(AssertionError): @@ -50,7 +50,7 @@ def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration) def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( - number_previous_timesteps_to_use=1, opticalflow_image_size_pixels=32 + number_previous_timesteps_to_use=1, image_size_pixels=32 ) with tempfile.TemporaryDirectory() as dirpath: Batch.fake(configuration=optical_flow_configuration).save_netcdf(path=dirpath, batch_i=0) diff --git a/tests/test_manager.py b/tests/test_manager.py index 2bdff114..a8ff961f 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -167,7 +167,7 @@ def test_derived_batches(): of = OpticalFlowDataSource( history_minutes=30, forecast_minutes=60, - opticalflow_image_size_pixels=32, + oimage_size_pixels=32, ) manager = Manager() From e25462abf9fdc8c2bdc364739325a6113fa91544 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 08:19:38 +0000 Subject: [PATCH 109/197] Fix name --- tests/test_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_manager.py b/tests/test_manager.py index a8ff961f..fb69dfcb 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -167,7 +167,7 @@ def test_derived_batches(): of = OpticalFlowDataSource( history_minutes=30, forecast_minutes=60, - oimage_size_pixels=32, + image_size_pixels=32, ) manager = Manager() From d50a73da731ae2f1592297447e471963a1464d6a Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 08:33:38 +0000 Subject: [PATCH 110/197] Add assert --- tests/test_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_manager.py b/tests/test_manager.py index fb69dfcb..e43e7367 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -130,6 +130,7 @@ def test_batches(): manager.create_batches(overwrite_batches=True) assert os.path.exists(f"{dst_path}/train") + assert os.path.exists(f"{dst_path}/train/metadata/000000.nc") assert os.path.exists(f"{dst_path}/train/gsp") assert os.path.exists(f"{dst_path}/train/gsp/000000.nc") assert os.path.exists(f"{dst_path}/train/sat/000000.nc") @@ -186,6 +187,7 @@ def test_derived_batches(): # just set satellite as data source manager.data_sources = {"gsp": gsp, "sat": sat} manager.derived_data_sources = {"opticalflow": of} + print(manager.derived_data_sources) manager.data_source_which_defines_geospatial_locations = gsp # make file for locations From a24fc54a206e5b641e94845d1c0c677613b19b9c Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 08:47:32 +0000 Subject: [PATCH 111/197] Try different way of getting metadata file --- tests/config/derived_datasource_test.yaml | 33 +++++++++++++++++++++++ tests/test_manager.py | 18 +++++++------ 2 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 tests/config/derived_datasource_test.yaml diff --git a/tests/config/derived_datasource_test.yaml b/tests/config/derived_datasource_test.yaml new file mode 100644 index 00000000..22829dba --- /dev/null +++ b/tests/config/derived_datasource_test.yaml @@ -0,0 +1,33 @@ +general: + description: example configuration + name: example +git: null +input_data: + gsp: + gsp_zarr_path: tests/data/gsp/test.zarr + nwp: + nwp_channels: + - t + nwp_image_size_pixels: 2 + nwp_zarr_path: tests/data/nwp_data/test.zarr + history_minutes: 60 + satellite: + satellite_channels: + - HRV + satellite_image_size_pixels: 64 + satellite_zarr_path: tests/data/sat_data.zarr + topographic: + topographic_filename: tests/data/europe_dem_2km_osgb.tif + opticalflow: + number_previous_timesteps_to_use: 1 + opticalflow_image_size_pixels: 32 +output_data: + filepath: not used by unittests! +process: + batch_size: 32 + local_temp_path: ~/temp/ + seed: 1234 + upload_every_n_batches: 16 + n_train_batches: 2 + n_validation_batches: 0 + n_test_batches: 0 diff --git a/tests/test_manager.py b/tests/test_manager.py index e43e7367..1dd194f5 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -11,6 +11,7 @@ from nowcasting_dataset.data_sources import OpticalFlowDataSource from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource +from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource from nowcasting_dataset.manager import Manager @@ -130,7 +131,6 @@ def test_batches(): manager.create_batches(overwrite_batches=True) assert os.path.exists(f"{dst_path}/train") - assert os.path.exists(f"{dst_path}/train/metadata/000000.nc") assert os.path.exists(f"{dst_path}/train/gsp") assert os.path.exists(f"{dst_path}/train/gsp/000000.nc") assert os.path.exists(f"{dst_path}/train/sat/000000.nc") @@ -165,6 +165,8 @@ def test_derived_batches(): meters_per_pixel=2000, ) + meta = MetadataDataSource(history_minutes=30, forecast_minutes=60, object_at_center="GSP") + of = OpticalFlowDataSource( history_minutes=30, forecast_minutes=60, @@ -172,23 +174,23 @@ def test_derived_batches(): ) manager = Manager() + from nowcasting_dataset.data_sources import ALL_DATA_SOURCE_NAMES + # load config local_path = Path(nowcasting_dataset.__file__).parent.parent - filename = local_path / "tests" / "config" / "test.yaml" + filename = local_path / "tests" / "config" / "derived_datasource_test.yaml" manager.load_yaml_configuration(filename=filename) - + manager.initialize_data_sources(names_of_selected_data_sources=ALL_DATA_SOURCE_NAMES) with tempfile.TemporaryDirectory() as local_temp_path, tempfile.TemporaryDirectory() as dst_path: # noqa 101 # set local temp path, and dst path manager.config.output_data.filepath = Path(dst_path) manager.local_temp_path = Path(local_temp_path) - # just set satellite as data source - manager.data_sources = {"gsp": gsp, "sat": sat} - manager.derived_data_sources = {"opticalflow": of} - print(manager.derived_data_sources) - manager.data_source_which_defines_geospatial_locations = gsp + #manager.data_sources = {"gsp": gsp, "sat": sat, 'meta': meta} + #manager.derived_data_sources = {"opticalflow": of} + #manager.data_source_which_defines_geospatial_locations = gsp # make file for locations manager.create_files_specifying_spatial_and_temporal_locations_of_each_example_if_necessary() # noqa 101 From 7a6c1eb8c955d745f6f5b4a50ea0088cd699d011 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 08:47:54 +0000 Subject: [PATCH 112/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_manager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_manager.py b/tests/test_manager.py index 1dd194f5..0df01068 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -10,8 +10,8 @@ import nowcasting_dataset from nowcasting_dataset.data_sources import OpticalFlowDataSource from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource -from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource +from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource from nowcasting_dataset.manager import Manager @@ -176,7 +176,6 @@ def test_derived_batches(): manager = Manager() from nowcasting_dataset.data_sources import ALL_DATA_SOURCE_NAMES - # load config local_path = Path(nowcasting_dataset.__file__).parent.parent filename = local_path / "tests" / "config" / "derived_datasource_test.yaml" @@ -188,9 +187,9 @@ def test_derived_batches(): manager.config.output_data.filepath = Path(dst_path) manager.local_temp_path = Path(local_temp_path) # just set satellite as data source - #manager.data_sources = {"gsp": gsp, "sat": sat, 'meta': meta} - #manager.derived_data_sources = {"opticalflow": of} - #manager.data_source_which_defines_geospatial_locations = gsp + # manager.data_sources = {"gsp": gsp, "sat": sat, 'meta': meta} + # manager.derived_data_sources = {"opticalflow": of} + # manager.data_source_which_defines_geospatial_locations = gsp # make file for locations manager.create_files_specifying_spatial_and_temporal_locations_of_each_example_if_necessary() # noqa 101 From 03979121aca0ec551c7961d74748d46912198615 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 08:52:58 +0000 Subject: [PATCH 113/197] Remove NWP for now --- tests/config/derived_datasource_test.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/config/derived_datasource_test.yaml b/tests/config/derived_datasource_test.yaml index 22829dba..02fe4ae2 100644 --- a/tests/config/derived_datasource_test.yaml +++ b/tests/config/derived_datasource_test.yaml @@ -5,12 +5,6 @@ git: null input_data: gsp: gsp_zarr_path: tests/data/gsp/test.zarr - nwp: - nwp_channels: - - t - nwp_image_size_pixels: 2 - nwp_zarr_path: tests/data/nwp_data/test.zarr - history_minutes: 60 satellite: satellite_channels: - HRV From 008853a054b8eecac34c69eff339709ced73448c Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 08:58:42 +0000 Subject: [PATCH 114/197] Try metadata more --- tests/test_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_manager.py b/tests/test_manager.py index 0df01068..b16ca958 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -180,16 +180,16 @@ def test_derived_batches(): local_path = Path(nowcasting_dataset.__file__).parent.parent filename = local_path / "tests" / "config" / "derived_datasource_test.yaml" manager.load_yaml_configuration(filename=filename) - manager.initialize_data_sources(names_of_selected_data_sources=ALL_DATA_SOURCE_NAMES) + # manager.initialize_data_sources(names_of_selected_data_sources=ALL_DATA_SOURCE_NAMES) with tempfile.TemporaryDirectory() as local_temp_path, tempfile.TemporaryDirectory() as dst_path: # noqa 101 # set local temp path, and dst path manager.config.output_data.filepath = Path(dst_path) manager.local_temp_path = Path(local_temp_path) # just set satellite as data source - # manager.data_sources = {"gsp": gsp, "sat": sat, 'meta': meta} - # manager.derived_data_sources = {"opticalflow": of} - # manager.data_source_which_defines_geospatial_locations = gsp + manager.data_sources = {"gsp": gsp, "sat": sat, 'meta': meta} + manager.derived_data_sources = {"opticalflow": of} + manager.data_source_which_defines_geospatial_locations = gsp # make file for locations manager.create_files_specifying_spatial_and_temporal_locations_of_each_example_if_necessary() # noqa 101 From 2f232e0d21e9d347c274652edee2a1716bad6ca1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 08:59:07 +0000 Subject: [PATCH 115/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_manager.py b/tests/test_manager.py index b16ca958..d944333c 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -187,7 +187,7 @@ def test_derived_batches(): manager.config.output_data.filepath = Path(dst_path) manager.local_temp_path = Path(local_temp_path) # just set satellite as data source - manager.data_sources = {"gsp": gsp, "sat": sat, 'meta': meta} + manager.data_sources = {"gsp": gsp, "sat": sat, "meta": meta} manager.derived_data_sources = {"opticalflow": of} manager.data_source_which_defines_geospatial_locations = gsp From 7fdfb97700de8df42c294eb63f675d042a7a05c2 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 09:40:18 +0000 Subject: [PATCH 116/197] Use CSV instead of MetadataDataSource --- .../data_sources/data_source.py | 43 +++++++++++++------ .../data_sources/metadata/__init__.py | 1 - .../optical_flow/optical_flow_data_source.py | 6 +-- nowcasting_dataset/manager.py | 25 +++++++++++ tests/config/derived_datasource_test.yaml | 27 ------------ tests/test_manager.py | 5 +-- 6 files changed, 58 insertions(+), 49 deletions(-) delete mode 100644 nowcasting_dataset/data_sources/metadata/__init__.py delete mode 100644 tests/config/derived_datasource_test.yaml diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index d54036dc..7e87f0ac 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -477,15 +477,10 @@ def datetime_index(self): # TODO Reduce duplication https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_batches( - self, - batch_path: Path, - total_number_batches: int, - idx_of_first_batch: int, - dst_path: Path, - local_temp_path: Path, - upload_every_n_batches: int, - **kwargs, - ) -> None: + self, batch_path: Path, spatial_and_temporal_locations_of_each_example: pd.DataFrame, + total_number_batches: int, idx_of_first_batch: int, dst_path: Path, + local_temp_path: Path, upload_every_n_batches: int, **kwargs + ) -> None: """Create multiple batches and save them to disk. Safe to call from worker processes. @@ -494,6 +489,10 @@ def create_batches( batch_path: Path to where the netcdf batches are stored (these will fed into the `DerivedDataSource`). This is the path to the top level path, such as `foo/v10/train/` + spatial_and_temporal_locations_of_each_example: A DataFrame where each row specifies + the spatial and temporal location of an example. The number of rows must be + an exact multiple of `batch_size`. + Columns are: t0_datetime_UTC, x_center_OSGB, y_center_OSGB. total_number_batches: The total number of batches to make idx_of_first_batch: The batch number of the first batch to create. dst_path: The final destination path for the batches. Must exist. @@ -516,13 +515,25 @@ def create_batches( nd_fs_utils.delete_all_files_in_temp_path(local_temp_path) path_to_write_to = local_temp_path if save_batches_locally_and_upload else dst_path + # Split locations per example into batches: + batch_size = len(spatial_and_temporal_locations_of_each_example) // total_number_batches + locations_for_batches = [] + for batch_idx in range(total_number_batches): + start_example_idx = batch_idx * batch_size + end_example_idx = (batch_idx + 1) * batch_size + locations_for_batch = spatial_and_temporal_locations_of_each_example.iloc[ + start_example_idx:end_example_idx + ] + locations_for_batches.append(locations_for_batch) + # Loop round each batch: - n_batches_processed = 0 - for batch_idx in range(idx_of_first_batch, total_number_batches): + for n_batches_processed, locations_for_batch in enumerate(locations_for_batches): + batch_idx = idx_of_first_batch + n_batches_processed logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") # Generate batch. - batch = self.get_batch(netcdf_path=batch_path, batch_idx=batch_idx) + batch = self.get_batch(netcdf_path=batch_path, batch_idx=batch_idx, + t0_datetimes=locations_for_batch.t0_datetime_UTC,) # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) @@ -541,7 +552,7 @@ def create_batches( nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) def get_batch( - self, netcdf_path: Union[str, Path], batch_idx: int, **kwargs + self, netcdf_path: Union[str, Path], batch_idx: int, t0_datetimes: pd.DatetimeIndex, **kwargs ) -> DataSourceOutput: """ Get Batch of derived data @@ -549,6 +560,9 @@ def get_batch( Args: netcdf_path: Path to the NetCDF files of the Batch to load batch_idx: The batch ID to load from those in the path + t0_datetimes: list of timestamps for the datetime of the batches. The batch will also + include data for historic and future depending on `history_minutes` and + `future_minutes`. The batch size is given by the length of the t0_datetimes. Returns: Batch of the derived data source @@ -560,7 +574,8 @@ def get_batch( with futures.ProcessPoolExecutor(max_workers=batch.batch_size) as executor: future_examples = [] for example_idx in range(batch.batch_size): - future_example = executor.submit(self.get_example, batch, example_idx) + future_example = executor.submit(self.get_example, batch, example_idx, + t0_datetimes[example_idx]) future_examples.append(future_example) examples = [future_example.result() for future_example in future_examples] diff --git a/nowcasting_dataset/data_sources/metadata/__init__.py b/nowcasting_dataset/data_sources/metadata/__init__.py deleted file mode 100644 index d95ccd84..00000000 --- a/nowcasting_dataset/data_sources/metadata/__init__.py +++ /dev/null @@ -1 +0,0 @@ -""" Metadata data sources and functions """ diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 40b0356b..04d04b78 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -29,7 +29,7 @@ class OpticalFlowDataSource(DerivedDataSource): image_size_pixels: Optional[int] = None def get_example( - self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, **kwargs + self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, t0_datetime: pd.Timestamp, **kwargs ) -> DataSourceOutput: """ Get Optical Flow Example data @@ -37,6 +37,7 @@ def get_example( Args: batch: Batch containing satellite and metadata at least example_idx: The example to load and use + t0_datetime: t0 datetime for the example Returns: Example Data @@ -47,9 +48,8 @@ def get_example( # Only do optical flow for satellite data self._data: xr.DataArray = batch.satellite.sel(example=example_idx) - t0_dt = batch.metadata.t0_dt.values[example_idx] - selected_data = self._compute_and_return_optical_flow(self._data, t0_dt=t0_dt) + selected_data = self._compute_and_return_optical_flow(self._data, t0_dt=t0_datetime) return selected_data diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 4d5db1e4..dde1d943 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -351,16 +351,40 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: if len(splits_which_need_more_batches) == 0: logger.info("All batches have already been created! No work to do!") return + + # Load locations for each example off disk. + locations_for_each_example_of_each_split: dict[split.SplitName, pd.DataFrame] = {} + for split_name in splits_which_need_more_batches: + filename = self._filename_of_locations_csv_file(split_name.value) + logger.info(f"Loading {filename}.") + locations_for_each_example = pd.read_csv(filename, index_col=0) + assert locations_for_each_example.columns.to_list() == list( + SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES + ) + # Converting to datetimes is much faster using `pd.to_datetime()` than + # passing `parse_datetimes` into `pd.read_csv()`. + locations_for_each_example["t0_datetime_UTC"] = pd.to_datetime( + locations_for_each_example["t0_datetime_UTC"] + ) + locations_for_each_example_of_each_split[split_name] = locations_for_each_example + n_data_sources = len(self.derived_data_sources) nd_utils.set_fsspec_for_multiprocess() for split_name in splits_which_need_more_batches: + locations_for_split = locations_for_each_example_of_each_split[split_name] with futures.ProcessPoolExecutor(max_workers=n_data_sources) as executor: future_create_batches_jobs = [] for worker_id, (data_source_name, data_source) in enumerate( self.derived_data_sources.items() ): + + if len(locations_for_split) == 0: + break + # Get indexes of first batch and example. And subset locations_for_split. idx_of_first_batch = first_batches_to_create[split_name][data_source_name] + idx_of_first_example = idx_of_first_batch * self.config.process.batch_size + locations = locations_for_split.loc[idx_of_first_example:] # Get paths. dst_path = ( @@ -382,6 +406,7 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: future = executor.submit( data_source.create_batches, batch_path=self.config.output_data.filepath / split_name.value, + spatial_and_temporal_locations_of_each_example=locations, total_number_batches=self._get_n_batches_for_split_name(split_name.value), idx_of_first_batch=idx_of_first_batch, batch_size=self.config.process.batch_size, diff --git a/tests/config/derived_datasource_test.yaml b/tests/config/derived_datasource_test.yaml deleted file mode 100644 index 02fe4ae2..00000000 --- a/tests/config/derived_datasource_test.yaml +++ /dev/null @@ -1,27 +0,0 @@ -general: - description: example configuration - name: example -git: null -input_data: - gsp: - gsp_zarr_path: tests/data/gsp/test.zarr - satellite: - satellite_channels: - - HRV - satellite_image_size_pixels: 64 - satellite_zarr_path: tests/data/sat_data.zarr - topographic: - topographic_filename: tests/data/europe_dem_2km_osgb.tif - opticalflow: - number_previous_timesteps_to_use: 1 - opticalflow_image_size_pixels: 32 -output_data: - filepath: not used by unittests! -process: - batch_size: 32 - local_temp_path: ~/temp/ - seed: 1234 - upload_every_n_batches: 16 - n_train_batches: 2 - n_validation_batches: 0 - n_test_batches: 0 diff --git a/tests/test_manager.py b/tests/test_manager.py index d944333c..6c6acd1e 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -10,7 +10,6 @@ import nowcasting_dataset from nowcasting_dataset.data_sources import OpticalFlowDataSource from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource -from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource from nowcasting_dataset.manager import Manager @@ -165,7 +164,6 @@ def test_derived_batches(): meters_per_pixel=2000, ) - meta = MetadataDataSource(history_minutes=30, forecast_minutes=60, object_at_center="GSP") of = OpticalFlowDataSource( history_minutes=30, @@ -180,14 +178,13 @@ def test_derived_batches(): local_path = Path(nowcasting_dataset.__file__).parent.parent filename = local_path / "tests" / "config" / "derived_datasource_test.yaml" manager.load_yaml_configuration(filename=filename) - # manager.initialize_data_sources(names_of_selected_data_sources=ALL_DATA_SOURCE_NAMES) with tempfile.TemporaryDirectory() as local_temp_path, tempfile.TemporaryDirectory() as dst_path: # noqa 101 # set local temp path, and dst path manager.config.output_data.filepath = Path(dst_path) manager.local_temp_path = Path(local_temp_path) # just set satellite as data source - manager.data_sources = {"gsp": gsp, "sat": sat, "meta": meta} + manager.data_sources = {"gsp": gsp, "sat": sat} manager.derived_data_sources = {"opticalflow": of} manager.data_source_which_defines_geospatial_locations = gsp From 1c9504e79fb6ae17f8f557aabec1aaab736c1e8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 09:40:41 +0000 Subject: [PATCH 117/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/data_source.py | 36 +++++++++++++------ .../optical_flow/optical_flow_data_source.py | 6 +++- nowcasting_dataset/manager.py | 4 +-- tests/test_manager.py | 1 - 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 7e87f0ac..72b32e77 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -477,10 +477,16 @@ def datetime_index(self): # TODO Reduce duplication https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_batches( - self, batch_path: Path, spatial_and_temporal_locations_of_each_example: pd.DataFrame, - total_number_batches: int, idx_of_first_batch: int, dst_path: Path, - local_temp_path: Path, upload_every_n_batches: int, **kwargs - ) -> None: + self, + batch_path: Path, + spatial_and_temporal_locations_of_each_example: pd.DataFrame, + total_number_batches: int, + idx_of_first_batch: int, + dst_path: Path, + local_temp_path: Path, + upload_every_n_batches: int, + **kwargs, + ) -> None: """Create multiple batches and save them to disk. Safe to call from worker processes. @@ -522,8 +528,8 @@ def create_batches( start_example_idx = batch_idx * batch_size end_example_idx = (batch_idx + 1) * batch_size locations_for_batch = spatial_and_temporal_locations_of_each_example.iloc[ - start_example_idx:end_example_idx - ] + start_example_idx:end_example_idx + ] locations_for_batches.append(locations_for_batch) # Loop round each batch: @@ -532,8 +538,11 @@ def create_batches( logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") # Generate batch. - batch = self.get_batch(netcdf_path=batch_path, batch_idx=batch_idx, - t0_datetimes=locations_for_batch.t0_datetime_UTC,) + batch = self.get_batch( + netcdf_path=batch_path, + batch_idx=batch_idx, + t0_datetimes=locations_for_batch.t0_datetime_UTC, + ) # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) @@ -552,7 +561,11 @@ def create_batches( nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) def get_batch( - self, netcdf_path: Union[str, Path], batch_idx: int, t0_datetimes: pd.DatetimeIndex, **kwargs + self, + netcdf_path: Union[str, Path], + batch_idx: int, + t0_datetimes: pd.DatetimeIndex, + **kwargs, ) -> DataSourceOutput: """ Get Batch of derived data @@ -574,8 +587,9 @@ def get_batch( with futures.ProcessPoolExecutor(max_workers=batch.batch_size) as executor: future_examples = [] for example_idx in range(batch.batch_size): - future_example = executor.submit(self.get_example, batch, example_idx, - t0_datetimes[example_idx]) + future_example = executor.submit( + self.get_example, batch, example_idx, t0_datetimes[example_idx] + ) future_examples.append(future_example) examples = [future_example.result() for future_example in future_examples] diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 04d04b78..8e8c1b6c 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -29,7 +29,11 @@ class OpticalFlowDataSource(DerivedDataSource): image_size_pixels: Optional[int] = None def get_example( - self, batch: nowcasting_dataset.dataset.batch.Batch, example_idx: int, t0_datetime: pd.Timestamp, **kwargs + self, + batch: nowcasting_dataset.dataset.batch.Batch, + example_idx: int, + t0_datetime: pd.Timestamp, + **kwargs ) -> DataSourceOutput: """ Get Optical Flow Example data diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index dde1d943..88e00594 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -360,12 +360,12 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: locations_for_each_example = pd.read_csv(filename, index_col=0) assert locations_for_each_example.columns.to_list() == list( SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES - ) + ) # Converting to datetimes is much faster using `pd.to_datetime()` than # passing `parse_datetimes` into `pd.read_csv()`. locations_for_each_example["t0_datetime_UTC"] = pd.to_datetime( locations_for_each_example["t0_datetime_UTC"] - ) + ) locations_for_each_example_of_each_split[split_name] = locations_for_each_example n_data_sources = len(self.derived_data_sources) diff --git a/tests/test_manager.py b/tests/test_manager.py index 6c6acd1e..f21598a4 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -164,7 +164,6 @@ def test_derived_batches(): meters_per_pixel=2000, ) - of = OpticalFlowDataSource( history_minutes=30, forecast_minutes=60, From e8cabf168be9c1f1148ef2f011e30731c11017e0 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 09:52:24 +0000 Subject: [PATCH 118/197] Update tests --- .../optical_flow/test_optical_flow_data_source.py | 13 ++++++++----- tests/test_manager.py | 3 +-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index dbad8cb8..0ba3a579 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -26,7 +26,8 @@ def test_optical_flow_get_example(optical_flow_configuration): number_previous_timesteps_to_use=1, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) - example = optical_flow_datasource.get_example(batch=batch, example_idx=0) + example = optical_flow_datasource.get_example(batch=batch, example_idx=0, + t0_datetime=batch.metadata.t0_dt.values[0]) assert example.values.shape == (12, 32, 32, 12) @@ -35,7 +36,7 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): number_previous_timesteps_to_use=3, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) - example = optical_flow_datasource.get_example(batch=batch, example_idx=0) + example = optical_flow_datasource.get_example(batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0]) assert example.values.shape == (12, 32, 32, 12) @@ -45,7 +46,7 @@ def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration) ) batch = Batch.fake(configuration=optical_flow_configuration) with pytest.raises(AssertionError): - optical_flow_datasource.get_example(batch=batch, example_idx=0) + optical_flow_datasource.get_example(batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0]) def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 @@ -53,6 +54,8 @@ def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa number_previous_timesteps_to_use=1, image_size_pixels=32 ) with tempfile.TemporaryDirectory() as dirpath: - Batch.fake(configuration=optical_flow_configuration).save_netcdf(path=dirpath, batch_i=0) - optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0) + batch = Batch.fake(configuration=optical_flow_configuration) + batch.save_netcdf(path=dirpath, batch_i=0) + optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0, + t0_datetimes = batch.metadata.t0_dt.values) assert optical_flow.values.shape == (4, 12, 32, 32, 12) diff --git a/tests/test_manager.py b/tests/test_manager.py index f21598a4..83de05ce 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -171,11 +171,10 @@ def test_derived_batches(): ) manager = Manager() - from nowcasting_dataset.data_sources import ALL_DATA_SOURCE_NAMES # load config local_path = Path(nowcasting_dataset.__file__).parent.parent - filename = local_path / "tests" / "config" / "derived_datasource_test.yaml" + filename = local_path / "tests" / "config" / "test.yaml" manager.load_yaml_configuration(filename=filename) with tempfile.TemporaryDirectory() as local_temp_path, tempfile.TemporaryDirectory() as dst_path: # noqa 101 From 854c281d6c4f0d3fcd0817183f2dbfa9c1f2e228 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 09:52:45 +0000 Subject: [PATCH 119/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../test_optical_flow_data_source.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 0ba3a579..666e0445 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -26,8 +26,9 @@ def test_optical_flow_get_example(optical_flow_configuration): number_previous_timesteps_to_use=1, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) - example = optical_flow_datasource.get_example(batch=batch, example_idx=0, - t0_datetime=batch.metadata.t0_dt.values[0]) + example = optical_flow_datasource.get_example( + batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0] + ) assert example.values.shape == (12, 32, 32, 12) @@ -36,7 +37,9 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): number_previous_timesteps_to_use=3, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) - example = optical_flow_datasource.get_example(batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0]) + example = optical_flow_datasource.get_example( + batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0] + ) assert example.values.shape == (12, 32, 32, 12) @@ -46,7 +49,9 @@ def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration) ) batch = Batch.fake(configuration=optical_flow_configuration) with pytest.raises(AssertionError): - optical_flow_datasource.get_example(batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0]) + optical_flow_datasource.get_example( + batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0] + ) def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 @@ -56,6 +61,7 @@ def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa with tempfile.TemporaryDirectory() as dirpath: batch = Batch.fake(configuration=optical_flow_configuration) batch.save_netcdf(path=dirpath, batch_i=0) - optical_flow = optical_flow_datasource.get_batch(netcdf_path=dirpath, batch_idx=0, - t0_datetimes = batch.metadata.t0_dt.values) + optical_flow = optical_flow_datasource.get_batch( + netcdf_path=dirpath, batch_idx=0, t0_datetimes=batch.metadata.t0_dt.values + ) assert optical_flow.values.shape == (4, 12, 32, 32, 12) From e14126a0c9da0f74b639db092badd36ae6ca60d6 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 10:18:42 +0000 Subject: [PATCH 120/197] Readd trying metadata --- nowcasting_dataset/data_sources/__init__.py | 2 ++ nowcasting_dataset/dataset/batch.py | 15 ++++++++------- tests/test_manager.py | 5 ++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py index fc61b881..6d2ea6b8 100644 --- a/nowcasting_dataset/data_sources/__init__.py +++ b/nowcasting_dataset/data_sources/__init__.py @@ -1,6 +1,7 @@ """ Various DataSources """ from nowcasting_dataset.data_sources.data_source import DataSource # noqa: F401 from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource +from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource from nowcasting_dataset.data_sources.nwp.nwp_data_source import NWPDataSource from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( OpticalFlowDataSource, @@ -13,6 +14,7 @@ ) MAP_DATA_SOURCE_NAME_TO_CLASS = { + "metadata": MetadataDataSource, "pv": PVDataSource, "satellite": SatelliteDataSource, "opticalflow": OpticalFlowDataSource, diff --git a/nowcasting_dataset/dataset/batch.py b/nowcasting_dataset/dataset/batch.py index f8349d30..9c8f2347 100644 --- a/nowcasting_dataset/dataset/batch.py +++ b/nowcasting_dataset/dataset/batch.py @@ -165,13 +165,14 @@ def load_netcdf(local_netcdf_path: Union[Path, str], batch_idx: int): local_netcdf_filename = os.path.join( local_netcdf_path, data_source_name, get_netcdf_filename(batch_idx) ) - - # submit task - future_examples = executor.submit( - xr.load_dataset, - filename_or_obj=local_netcdf_filename, - ) - future_examples_per_source.append([data_source_name, future_examples]) + # If the file exists, load it, otherwise data source isn't used + if os.path.isfile(local_netcdf_filename): + # submit task + future_examples = executor.submit( + xr.load_dataset, + filename_or_obj=local_netcdf_filename, + ) + future_examples_per_source.append([data_source_name, future_examples]) # Collect results from each thread. for data_source_name, future_examples in future_examples_per_source: diff --git a/tests/test_manager.py b/tests/test_manager.py index 83de05ce..643f2a13 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -10,6 +10,7 @@ import nowcasting_dataset from nowcasting_dataset.data_sources import OpticalFlowDataSource from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource +from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource from nowcasting_dataset.manager import Manager @@ -170,6 +171,8 @@ def test_derived_batches(): image_size_pixels=32, ) + meta = MetadataDataSource(forecast_minutes = 60, history_minutes = 30) + manager = Manager() # load config @@ -182,7 +185,7 @@ def test_derived_batches(): manager.config.output_data.filepath = Path(dst_path) manager.local_temp_path = Path(local_temp_path) # just set satellite as data source - manager.data_sources = {"gsp": gsp, "sat": sat} + manager.data_sources = {"gsp": gsp, "sat": sat, "metadata": meta} manager.derived_data_sources = {"opticalflow": of} manager.data_source_which_defines_geospatial_locations = gsp From 35447eb29a1b542c0694bf390ef38020175cc0d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 10:18:59 +0000 Subject: [PATCH 121/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_manager.py b/tests/test_manager.py index 643f2a13..c927f4fc 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -171,7 +171,7 @@ def test_derived_batches(): image_size_pixels=32, ) - meta = MetadataDataSource(forecast_minutes = 60, history_minutes = 30) + meta = MetadataDataSource(forecast_minutes=60, history_minutes=30) manager = Manager() From 4317c3474c0faa4c819078541f2c58765d183a35 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 10:22:05 +0000 Subject: [PATCH 122/197] Change dim name in metadat --- .../data_sources/metadata/metadata_data_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py index de4bdbf1..d1535db3 100644 --- a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py +++ b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py @@ -50,9 +50,9 @@ def get_example( d_all = { "t0_dt": {"dims": ("t0_dt"), "data": [t0_dt]}, - "x_meters_center": {"dims": ("t0_dt_index"), "data": [x_meters_center]}, - "y_meters_center": {"dims": ("t0_dt_index"), "data": [y_meters_center]}, - "object_at_center_label": {"dims": ("t0_dt_index"), "data": [object_at_center_label]}, + "x_meters_center": {"dims": ("t0_dt"), "data": [x_meters_center]}, + "y_meters_center": {"dims": ("t0_dt"), "data": [y_meters_center]}, + "object_at_center_label": {"dims": ("t0_dt"), "data": [object_at_center_label]}, } data = convert_data_array_to_dataset(xr.DataArray.from_dict(d_all["t0_dt"])) From 8a3c2879d9271fb211c36e8721f9c6868b17cb28 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 10:47:43 +0000 Subject: [PATCH 123/197] Add metadata test --- nowcasting_dataset/data_sources/__init__.py | 2 -- .../metadata/metadata_data_source.py | 27 ++++++++----------- tests/data_sources/test_metadata.py | 13 +++++++++ 3 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 tests/data_sources/test_metadata.py diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py index 6d2ea6b8..fc61b881 100644 --- a/nowcasting_dataset/data_sources/__init__.py +++ b/nowcasting_dataset/data_sources/__init__.py @@ -1,7 +1,6 @@ """ Various DataSources """ from nowcasting_dataset.data_sources.data_source import DataSource # noqa: F401 from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource -from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource from nowcasting_dataset.data_sources.nwp.nwp_data_source import NWPDataSource from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( OpticalFlowDataSource, @@ -14,7 +13,6 @@ ) MAP_DATA_SOURCE_NAME_TO_CLASS = { - "metadata": MetadataDataSource, "pv": PVDataSource, "satellite": SatelliteDataSource, "opticalflow": OpticalFlowDataSource, diff --git a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py index d1535db3..008f6474 100644 --- a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py +++ b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py @@ -42,26 +42,21 @@ def get_example( # TODO: data_dict is unused in this function. Is that a bug? # https://github.com/openclimatefix/nowcasting_dataset/issues/279 data_dict = dict( # noqa: F841 - t0_dt=to_numpy(t0_dt), #: Shape: [batch_size,] - x_meters_center=np.array(x_meters_center), - y_meters_center=np.array(y_meters_center), - object_at_center_label=object_at_center_label, + t0_dt=t0_dt, #: Shape: [batch_size,] + x_meters_center=np.array([x_meters_center]), + y_meters_center=np.array([y_meters_center]), + object_at_center_label=np.array([object_at_center_label]), ) + d = { + "dims": ("t0_dt",), + "data": data_dict["t0_dt"], + } - d_all = { - "t0_dt": {"dims": ("t0_dt"), "data": [t0_dt]}, - "x_meters_center": {"dims": ("t0_dt"), "data": [x_meters_center]}, - "y_meters_center": {"dims": ("t0_dt"), "data": [y_meters_center]}, - "object_at_center_label": {"dims": ("t0_dt"), "data": [object_at_center_label]}, - } - - data = convert_data_array_to_dataset(xr.DataArray.from_dict(d_all["t0_dt"])) + data = convert_data_array_to_dataset(xr.DataArray.from_dict(d)) for v in ["x_meters_center", "y_meters_center", "object_at_center_label"]: - d: dict = d_all[v] - d: xr.Dataset = convert_data_array_to_dataset(xr.DataArray.from_dict(d)).rename( - {"data": v} - ) + d: dict = {"dims": ("t0_dt",), "data": data_dict[v]} + d: xr.Dataset = convert_data_array_to_dataset(xr.DataArray.from_dict(d)).rename({"data": v}) data[v] = getattr(d, v) return Metadata(data) diff --git a/tests/data_sources/test_metadata.py b/tests/data_sources/test_metadata.py new file mode 100644 index 00000000..9d5db7b6 --- /dev/null +++ b/tests/data_sources/test_metadata.py @@ -0,0 +1,13 @@ +import pytest +from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource +import pandas as pd + + +def test_metadata_example(): + data_source = MetadataDataSource(history_minutes=0, forecast_minutes=5, object_at_center="GSP") + t0 = pd.date_range("2021-01-01", freq="5T", periods=1) + pd.Timedelta("30T") + x_meters_center = 1000 + y_meters_center = 1000 + example = data_source.get_example(t0_dt = t0, x_meters_center = x_meters_center, + y_meters_center = y_meters_center) + assert "t0_dt_index" in example.coords From 792267fba777ff42861a7e99fe2d5d9b78b031c2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 10:48:01 +0000 Subject: [PATCH 124/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/metadata/metadata_data_source.py | 6 ++++-- tests/data_sources/test_metadata.py | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py index 008f6474..177c0570 100644 --- a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py +++ b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py @@ -50,13 +50,15 @@ def get_example( d = { "dims": ("t0_dt",), "data": data_dict["t0_dt"], - } + } data = convert_data_array_to_dataset(xr.DataArray.from_dict(d)) for v in ["x_meters_center", "y_meters_center", "object_at_center_label"]: d: dict = {"dims": ("t0_dt",), "data": data_dict[v]} - d: xr.Dataset = convert_data_array_to_dataset(xr.DataArray.from_dict(d)).rename({"data": v}) + d: xr.Dataset = convert_data_array_to_dataset(xr.DataArray.from_dict(d)).rename( + {"data": v} + ) data[v] = getattr(d, v) return Metadata(data) diff --git a/tests/data_sources/test_metadata.py b/tests/data_sources/test_metadata.py index 9d5db7b6..b5031469 100644 --- a/tests/data_sources/test_metadata.py +++ b/tests/data_sources/test_metadata.py @@ -1,6 +1,7 @@ +import pandas as pd import pytest + from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource -import pandas as pd def test_metadata_example(): @@ -8,6 +9,7 @@ def test_metadata_example(): t0 = pd.date_range("2021-01-01", freq="5T", periods=1) + pd.Timedelta("30T") x_meters_center = 1000 y_meters_center = 1000 - example = data_source.get_example(t0_dt = t0, x_meters_center = x_meters_center, - y_meters_center = y_meters_center) + example = data_source.get_example( + t0_dt=t0, x_meters_center=x_meters_center, y_meters_center=y_meters_center + ) assert "t0_dt_index" in example.coords From ab8693dc8b766d1805caee5f2ce13fe4199750f0 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 10:49:16 +0000 Subject: [PATCH 125/197] Fix linter error --- nowcasting_dataset/data_sources/metadata/metadata_data_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py index 177c0570..8d46e6c8 100644 --- a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py +++ b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py @@ -9,7 +9,6 @@ from nowcasting_dataset.data_sources.data_source import DataSource from nowcasting_dataset.data_sources.metadata.metadata_model import Metadata from nowcasting_dataset.dataset.xr_utils import convert_data_array_to_dataset -from nowcasting_dataset.utils import to_numpy @dataclass From c3ff1cc5d0b3c9afaa1659bbea3d752319c69277 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 10:59:37 +0000 Subject: [PATCH 126/197] Rearrange metadata time --- .../data_sources/metadata/metadata_data_source.py | 10 ++++++---- tests/data_sources/test_metadata.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py index 8d46e6c8..a7ad31cc 100644 --- a/nowcasting_dataset/data_sources/metadata/metadata_data_source.py +++ b/nowcasting_dataset/data_sources/metadata/metadata_data_source.py @@ -41,10 +41,10 @@ def get_example( # TODO: data_dict is unused in this function. Is that a bug? # https://github.com/openclimatefix/nowcasting_dataset/issues/279 data_dict = dict( # noqa: F841 - t0_dt=t0_dt, #: Shape: [batch_size,] - x_meters_center=np.array([x_meters_center]), - y_meters_center=np.array([y_meters_center]), - object_at_center_label=np.array([object_at_center_label]), + t0_dt=[t0_dt], #: Shape: [batch_size,] + x_meters_center=[x_meters_center], + y_meters_center=[y_meters_center], + object_at_center_label=[object_at_center_label], ) d = { "dims": ("t0_dt",), @@ -59,5 +59,7 @@ def get_example( {"data": v} ) data[v] = getattr(d, v) + data = data.drop_vars("t0_dt") + data = data.rename({"data": "t0_dt"}) return Metadata(data) diff --git a/tests/data_sources/test_metadata.py b/tests/data_sources/test_metadata.py index b5031469..053a96f4 100644 --- a/tests/data_sources/test_metadata.py +++ b/tests/data_sources/test_metadata.py @@ -6,7 +6,7 @@ def test_metadata_example(): data_source = MetadataDataSource(history_minutes=0, forecast_minutes=5, object_at_center="GSP") - t0 = pd.date_range("2021-01-01", freq="5T", periods=1) + pd.Timedelta("30T") + t0 = pd.Timestamp('2021-01-01') x_meters_center = 1000 y_meters_center = 1000 example = data_source.get_example( From a350da2de964e49cac066c3fef97b9ab0afddc2c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 11:00:14 +0000 Subject: [PATCH 127/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/data_sources/test_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data_sources/test_metadata.py b/tests/data_sources/test_metadata.py index 053a96f4..912f592f 100644 --- a/tests/data_sources/test_metadata.py +++ b/tests/data_sources/test_metadata.py @@ -6,7 +6,7 @@ def test_metadata_example(): data_source = MetadataDataSource(history_minutes=0, forecast_minutes=5, object_at_center="GSP") - t0 = pd.Timestamp('2021-01-01') + t0 = pd.Timestamp("2021-01-01") x_meters_center = 1000 y_meters_center = 1000 example = data_source.get_example( From f091295ccc7a82c436b333d1a60d702d28f305c8 Mon Sep 17 00:00:00 2001 From: Jacob Bieker Date: Fri, 12 Nov 2021 11:57:05 +0000 Subject: [PATCH 128/197] Remove adding dim index a second time --- .../data_sources/data_source.py | 2 +- tests/data_sources/test_metadata.py | 10 +++++++ .../test_topographic_data_source.py | 28 +++++++++++++++++++ tests/test_manager.py | 5 ++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 72b32e77..d08f875d 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -261,7 +261,7 @@ def get_batch( cls = examples[0].__class__ # Set the coords to be indices before joining into a batch - examples = [make_dim_index(example) for example in examples] + # examples = [make_dim_index(example) for example in examples] # join the examples together, and cast them to the cls, so that validation can occur return cls(join_list_dataset_to_batch_dataset(examples)) diff --git a/tests/data_sources/test_metadata.py b/tests/data_sources/test_metadata.py index 912f592f..d351fff4 100644 --- a/tests/data_sources/test_metadata.py +++ b/tests/data_sources/test_metadata.py @@ -1,4 +1,5 @@ import pandas as pd +import numpy as np import pytest from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource @@ -13,3 +14,12 @@ def test_metadata_example(): t0_dt=t0, x_meters_center=x_meters_center, y_meters_center=y_meters_center ) assert "t0_dt_index" in example.coords + +def test_metadata_batch(): + data_source = MetadataDataSource(history_minutes=0, forecast_minutes=5, object_at_center="GSP") + t0_datetimes = pd.date_range("2021-01-01", freq="5T", periods=32) + pd.Timedelta("30T") + x_meters_centers = np.random.random(32) + y_meters_centers = np.random.random(32) + batch = data_source.get_batch(t0_datetimes = t0_datetimes, x_locations = x_meters_centers, + y_locations = y_meters_centers) + assert "t0_dt_index" in batch.coords diff --git a/tests/data_sources/test_topographic_data_source.py b/tests/data_sources/test_topographic_data_source.py index 109328b6..0701b0fa 100644 --- a/tests/data_sources/test_topographic_data_source.py +++ b/tests/data_sources/test_topographic_data_source.py @@ -40,6 +40,34 @@ def test_get_example_2km(x, y, left, right, top, bottom): assert np.isclose(top, topo_data.y.values[0], atol=size) assert np.isclose(bottom, topo_data.y.values[-1], atol=size) +@pytest.mark.parametrize( + "x, y, left, right, top, bottom", + [ + (0, 0, -128_000, 126_000, 128_000, -126_000), + (10, 0, -126_000, 128_000, 128_000, -126_000), + (30, 0, -126_000, 128_000, 128_000, -126_000), + (1000, 0, -126_000, 128_000, 128_000, -126_000), + (0, 1000, -128_000, 126_000, 128_000, -126_000), + (1000, 1000, -126_000, 128_000, 128_000, -126_000), + (2000, 2000, -126_000, 128_000, 130_000, -124_000), + (2000, 1000, -126_000, 128_000, 128_000, -126_000), + (2001, 2001, -124_000, 130_000, 130_000, -124_000), + ], + ) +def test_get_batch_2km(x, y, left, right, top, bottom): + size = 2000 # meters + topo_source = TopographicDataSource( + filename="tests/data/europe_dem_2km_osgb.tif", + image_size_pixels=128, + meters_per_pixel=size, + forecast_minutes=300, + history_minutes=10, + ) + x = np.array([x]*32) + y = np.array([y]*32) + t0_datetimes = pd.date_range("2021-01-01", freq="5T", periods=32) + pd.Timedelta("30T") + topo_data = topo_source.get_batch(t0_datetimes=t0_datetimes, x_locations=x, y_locations=y) + assert "x_index_index" not in topo_data.dims @pytest.mark.skip("CD does not have access to GCS") def test_get_example_gcs(): diff --git a/tests/test_manager.py b/tests/test_manager.py index c927f4fc..61b315ad 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -194,6 +194,11 @@ def test_derived_batches(): # make batches manager.create_batches(overwrite_batches=True) + import glob + print(list(glob.glob(os.path.join(dst_path, "train", "*")))) + # Load batch + from nowcasting_dataset.dataset.batch import Batch + batch = Batch.load_netcdf(os.path.join(dst_path, "train"), batch_idx = 0) # make derived batches manager.create_derived_batches(overwrite_batches=True) From 82d8cc90a696a187dff84a0efc83d6a0305e42b0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 11:57:25 +0000 Subject: [PATCH 129/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/data_sources/test_metadata.py | 8 +++++--- tests/data_sources/test_topographic_data_source.py | 12 +++++++----- tests/test_manager.py | 4 +++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/data_sources/test_metadata.py b/tests/data_sources/test_metadata.py index d351fff4..0b995f64 100644 --- a/tests/data_sources/test_metadata.py +++ b/tests/data_sources/test_metadata.py @@ -1,5 +1,5 @@ -import pandas as pd import numpy as np +import pandas as pd import pytest from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource @@ -15,11 +15,13 @@ def test_metadata_example(): ) assert "t0_dt_index" in example.coords + def test_metadata_batch(): data_source = MetadataDataSource(history_minutes=0, forecast_minutes=5, object_at_center="GSP") t0_datetimes = pd.date_range("2021-01-01", freq="5T", periods=32) + pd.Timedelta("30T") x_meters_centers = np.random.random(32) y_meters_centers = np.random.random(32) - batch = data_source.get_batch(t0_datetimes = t0_datetimes, x_locations = x_meters_centers, - y_locations = y_meters_centers) + batch = data_source.get_batch( + t0_datetimes=t0_datetimes, x_locations=x_meters_centers, y_locations=y_meters_centers + ) assert "t0_dt_index" in batch.coords diff --git a/tests/data_sources/test_topographic_data_source.py b/tests/data_sources/test_topographic_data_source.py index 0701b0fa..348486f8 100644 --- a/tests/data_sources/test_topographic_data_source.py +++ b/tests/data_sources/test_topographic_data_source.py @@ -40,6 +40,7 @@ def test_get_example_2km(x, y, left, right, top, bottom): assert np.isclose(top, topo_data.y.values[0], atol=size) assert np.isclose(bottom, topo_data.y.values[-1], atol=size) + @pytest.mark.parametrize( "x, y, left, right, top, bottom", [ @@ -52,8 +53,8 @@ def test_get_example_2km(x, y, left, right, top, bottom): (2000, 2000, -126_000, 128_000, 130_000, -124_000), (2000, 1000, -126_000, 128_000, 128_000, -126_000), (2001, 2001, -124_000, 130_000, 130_000, -124_000), - ], - ) + ], +) def test_get_batch_2km(x, y, left, right, top, bottom): size = 2000 # meters topo_source = TopographicDataSource( @@ -62,13 +63,14 @@ def test_get_batch_2km(x, y, left, right, top, bottom): meters_per_pixel=size, forecast_minutes=300, history_minutes=10, - ) - x = np.array([x]*32) - y = np.array([y]*32) + ) + x = np.array([x] * 32) + y = np.array([y] * 32) t0_datetimes = pd.date_range("2021-01-01", freq="5T", periods=32) + pd.Timedelta("30T") topo_data = topo_source.get_batch(t0_datetimes=t0_datetimes, x_locations=x, y_locations=y) assert "x_index_index" not in topo_data.dims + @pytest.mark.skip("CD does not have access to GCS") def test_get_example_gcs(): """Note this test takes ~5 seconds as the topo data has to be downloaded locally""" diff --git a/tests/test_manager.py b/tests/test_manager.py index 61b315ad..fe8c5b7d 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -195,10 +195,12 @@ def test_derived_batches(): # make batches manager.create_batches(overwrite_batches=True) import glob + print(list(glob.glob(os.path.join(dst_path, "train", "*")))) # Load batch from nowcasting_dataset.dataset.batch import Batch - batch = Batch.load_netcdf(os.path.join(dst_path, "train"), batch_idx = 0) + + batch = Batch.load_netcdf(os.path.join(dst_path, "train"), batch_idx=0) # make derived batches manager.create_derived_batches(overwrite_batches=True) From 6bd6de660e8311f6d5846a2b74c735f9b230f239 Mon Sep 17 00:00:00 2001 From: Nasser Benabderrazik Date: Tue, 23 Nov 2021 09:53:23 +0100 Subject: [PATCH 130/197] Refactor 'create_batches' between 'DataSource' and 'DerivedDataSource' (#470) * Refactor 'create_batches' between 'DataSource' and 'DerivedDataSource' * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add text in assert statements and missing info in docstring * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/data_source.py | 176 +++++++----------- 1 file changed, 69 insertions(+), 107 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 42970573..5ed33937 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -148,32 +148,54 @@ def create_batches( dst_path: Path, local_temp_path: Path, upload_every_n_batches: int, + total_number_batches: int = None, + **kwargs, ) -> None: """Create multiple batches and save them to disk. Safe to call from worker processes. Args: - spatial_and_temporal_locations_of_each_example: A DataFrame where each row specifies - the spatial and temporal location of an example. The number of rows must be - an exact multiple of `batch_size`. - Columns are: t0_datetime_UTC, x_center_OSGB, y_center_OSGB. - idx_of_first_batch: The batch number of the first batch to create. - batch_size: The number of examples per batch. - dst_path: The final destination path for the batches. Must exist. - local_temp_path: The local temporary path. This is only required when dst_path is a - cloud storage bucket, so files must first be created on the VM's local disk in temp_path - and then uploaded to dst_path every upload_every_n_batches. Must exist. Will be emptied. - upload_every_n_batches: Upload the contents of temp_path to dst_path after this number - of batches have been created. If 0 then will write directly to dst_path. + spatial_and_temporal_locations_of_each_example (pd.DataFrame): A DataFrame where each + row specifies the spatial and temporal location of an example. The number of rows + must be an exact multiple of `batch_size`. + Columns are: t0_datetime_UTC, x_center_OSGB, y_center_OSGB. + idx_of_first_batch (int): The batch number of the first batch to create. + batch_size (int): The number of examples per batch. + dst_path (Path): The final destination path for the batches. Must exist. + local_temp_path (Path): The local temporary path. This is only required when dst_path + is a cloud storage bucket, so files must first be created on the VM's local disk in + temp_path and then uploaded to dst_path every `upload_every_n_batches`. Must exist. + Will be emptied. + upload_every_n_batches (int): Upload the contents of temp_path to dst_path after this + number of batches have been created. If 0 then will write directly to `dst_path`. + total_number_batches (int, optional): If specified it will be used to compute the batch + size (`batch_size` will not be used in that case). + **kwargs: Arguments specific to the `_get_batch` method. """ # Sanity checks: - assert idx_of_first_batch >= 0 - assert batch_size > 0 - assert len(spatial_and_temporal_locations_of_each_example) % batch_size == 0 - assert upload_every_n_batches >= 0 - assert spatial_and_temporal_locations_of_each_example.columns.to_list() == list( + assert idx_of_first_batch >= 0, ( + "The batch number of the first batch to create should be" " greater than 0" + ) + + if total_number_batches is None: + assert batch_size > 0, ( + "The batch size should be strictly greater than 0. Otherwise," + " you should specify 'total_number_batches' to compute the batch size from" + " 'spatial_and_temporal_locations_of_each_example'" + ) + assert len(spatial_and_temporal_locations_of_each_example) % batch_size == 0 + + assert upload_every_n_batches >= 0, "'upload_every_n_batches' should be greater than 0" + + spatial_and_temporal_locations_of_each_example_columns = ( + spatial_and_temporal_locations_of_each_example.columns.to_list() + ) + assert spatial_and_temporal_locations_of_each_example_columns == list( SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES + ), ( + f"The provided data columns ({spatial_and_temporal_locations_of_each_example_columns})" + f"do not match {SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES}" ) self.open() @@ -185,9 +207,13 @@ def create_batches( path_to_write_to = local_temp_path if save_batches_locally_and_upload else dst_path # Split locations per example into batches: - n_batches = len(spatial_and_temporal_locations_of_each_example) // batch_size + if total_number_batches is not None: + batch_size = len(spatial_and_temporal_locations_of_each_example) // total_number_batches + else: + total_number_batches = len(spatial_and_temporal_locations_of_each_example) // batch_size + locations_for_batches = [] - for batch_idx in range(n_batches): + for batch_idx in range(total_number_batches): start_example_idx = batch_idx * batch_size end_example_idx = (batch_idx + 1) * batch_size locations_for_batch = spatial_and_temporal_locations_of_each_example.iloc[ @@ -201,11 +227,7 @@ def create_batches( logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") # Generate batch. - batch = self.get_batch( - t0_datetimes=locations_for_batch.t0_datetime_UTC, - x_locations=locations_for_batch.x_center_OSGB, - y_locations=locations_for_batch.y_center_OSGB, - ) + batch = self._get_batch(locations_for_batch, **kwargs) # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) @@ -223,6 +245,20 @@ def create_batches( if save_batches_locally_and_upload: nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) + def _get_batch(self, locations_for_batch, **kwargs): + """Get the batch for the given datasource. This, along with `get_batch`, should be + implemented in the child classes if needed. + + `_get_batch` is used internally here and has a specific signature, because it is called in + `create_batches` which can be common to different classes inheriting from `DataSource` + (e.g. `DerivedDataSource`). + """ + return self.get_batch( + t0_datetimes=locations_for_batch.t0_datetime_UTC, + x_locations=locations_for_batch.x_center_OSGB, + y_locations=locations_for_batch.y_center_OSGB, + ) + # TODO: Issue #319: Standardise parameter names. def get_batch( self, @@ -483,90 +519,16 @@ def datetime_index(self): "needed" ) - # TODO Reduce duplication https://github.com/openclimatefix/nowcasting_dataset/issues/367 - def create_batches( - self, - batch_path: Path, - spatial_and_temporal_locations_of_each_example: pd.DataFrame, - total_number_batches: int, - idx_of_first_batch: int, - dst_path: Path, - local_temp_path: Path, - upload_every_n_batches: int, - **kwargs, - ) -> None: - """Create multiple batches and save them to disk. - - Safe to call from worker processes. - - Args: - batch_path: Path to where the netcdf batches are stored - (these will fed into the `DerivedDataSource`). This is the path to the top level path, - such as `foo/v10/train/` - spatial_and_temporal_locations_of_each_example: A DataFrame where each row specifies - the spatial and temporal location of an example. The number of rows must be - an exact multiple of `batch_size`. - Columns are: t0_datetime_UTC, x_center_OSGB, y_center_OSGB. - total_number_batches: The total number of batches to make - idx_of_first_batch: The batch number of the first batch to create. - dst_path: The final destination path for the batches. Must exist. - local_temp_path: The local temporary path. This is only required when dst_path is a - cloud storage bucket, so files must first be created on the VM's local disk in temp_path - and then uploaded to dst_path every upload_every_n_batches. Must exist. Will be emptied. - upload_every_n_batches: Upload the contents of temp_path to dst_path after this number - of batches have been created. If 0 then will write directly to dst_path. - """ - # Sanity checks: - assert idx_of_first_batch >= 0 - assert upload_every_n_batches >= 0 - assert total_number_batches >= 0 - - self.open() - - # Figure out where to write batches to: - save_batches_locally_and_upload = upload_every_n_batches > 0 - if save_batches_locally_and_upload: - nd_fs_utils.delete_all_files_in_temp_path(local_temp_path) - path_to_write_to = local_temp_path if save_batches_locally_and_upload else dst_path - - # Split locations per example into batches: - batch_size = len(spatial_and_temporal_locations_of_each_example) // total_number_batches - locations_for_batches = [] - for batch_idx in range(total_number_batches): - start_example_idx = batch_idx * batch_size - end_example_idx = (batch_idx + 1) * batch_size - locations_for_batch = spatial_and_temporal_locations_of_each_example.iloc[ - start_example_idx:end_example_idx - ] - locations_for_batches.append(locations_for_batch) - - # Loop round each batch: - for n_batches_processed, locations_for_batch in enumerate(locations_for_batches): - batch_idx = idx_of_first_batch + n_batches_processed - logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") + def _get_batch(self, locations_for_batch, **kwargs): + if not all(key in kwargs for key in ["batch_path", "batch_idx"]): + raise ValueError("Missing arguments 'batch_path' and 'batch_idx'") - # Generate batch. - batch = self.get_batch( - netcdf_path=batch_path, - batch_idx=batch_idx, - t0_datetimes=locations_for_batch.t0_datetime_UTC, - ) - - # Save batch to disk. - netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) - batch.to_netcdf(netcdf_filename) - n_batches_processed += 1 - # Upload if necessary. - if ( - save_batches_locally_and_upload - and n_batches_processed > 0 - and n_batches_processed % upload_every_n_batches == 0 - ): - nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) - - # Upload last few batches, if necessary: - if save_batches_locally_and_upload: - nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) + batch = self.get_batch( + netcdf_path=kwargs["batch_path"], + batch_idx=kwargs["batch_idx"], + t0_datetimes=locations_for_batch.t0_datetime_UTC, + ) + return batch def get_batch( self, From 1a84aa7cf7709cfed9986e30ab4a76e050db4b38 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 25 Nov 2021 15:44:06 +0000 Subject: [PATCH 131/197] add fastai::opencv-python-headless to environment.yml --- environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index ec3eb3dd..8c00dce4 100644 --- a/environment.yml +++ b/environment.yml @@ -2,6 +2,7 @@ name: nowcasting_dataset channels: - pvlib - conda-forge + - fastai dependencies: - python>=3.9 - pip @@ -16,6 +17,7 @@ dependencies: - xarray - ipykernel - h5netcdf # For opening NetCDF files from cloud buckets. + - fastai::opencv-python-headless # Cloud & distributed compute - gcsfs From aa89b888318f27dff10af86b453faadc7fad98aa Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 25 Nov 2021 16:13:01 +0000 Subject: [PATCH 132/197] fix circular import of Batch --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 8e8c1b6c..0459c794 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -8,7 +8,6 @@ import pandas as pd import xarray as xr -import nowcasting_dataset.dataset.batch from nowcasting_dataset.data_sources.data_source import DerivedDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput @@ -30,7 +29,8 @@ class OpticalFlowDataSource(DerivedDataSource): def get_example( self, - batch: nowcasting_dataset.dataset.batch.Batch, + batch, # Of type nowcasting_dataset.dataset.batch.Batch. But we can't use + # an "actual" type hint here otherwise we get a circular import error! example_idx: int, t0_datetime: pd.Timestamp, **kwargs @@ -39,7 +39,7 @@ def get_example( Get Optical Flow Example data Args: - batch: Batch containing satellite and metadata at least + batch: nowcasting_dataset.dataset.batch.Batch containing satellite and metadata at least example_idx: The example to load and use t0_datetime: t0 datetime for the example From 80dfc96624bfc86ddf3de6a56ea774257f76cf17 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 25 Nov 2021 17:14:25 +0000 Subject: [PATCH 133/197] fixed all but one test failure in test_batch --- nowcasting_dataset/data_sources/fake.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index 5707717b..6757cb51 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -175,7 +175,8 @@ def optical_flow_fake( # make batch of arrays xr_arrays = [ create_image_array( - seq_length_5=seq_length_5, + seq_length=seq_length_5, + freq="5T", image_size_pixels=satellite_image_size_pixels, channels=SAT_VARIABLE_NAMES[0:number_satellite_channels], ) From 778053fd5c67968aaea0b032b85272e2a199d45a Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 25 Nov 2021 17:30:13 +0000 Subject: [PATCH 134/197] all tests in test_batch pass now --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 0459c794..14943667 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -10,6 +10,7 @@ from nowcasting_dataset.data_sources.data_source import DerivedDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput +from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow _LOG = logging.getLogger("nowcasting_dataset") @@ -57,6 +58,11 @@ def get_example( return selected_data + @staticmethod + def get_data_model_for_batch(): + """Get the model that is used in the batch""" + return OpticalFlow + def _update_dataarray_with_predictions( self, satellite_data: xr.DataArray, From ec92942c13e927a2c0734e771952c665f7556346 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 25 Nov 2021 18:03:06 +0000 Subject: [PATCH 135/197] metadata.t0_datetime_utc should be pd.Timestamp --- nowcasting_dataset/data_sources/data_source.py | 4 ++-- nowcasting_dataset/data_sources/fake.py | 2 ++ .../data_sources/metadata/metadata_model.py | 3 +-- .../optical_flow/test_optical_flow_data_source.py | 15 +++++++-------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index c33a3e10..a7330d0a 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -564,9 +564,9 @@ def get_batch( import nowcasting_dataset.dataset.batch batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(netcdf_path, batch_idx=batch_idx) - with futures.ProcessPoolExecutor(max_workers=batch.batch_size) as executor: + with futures.ProcessPoolExecutor(max_workers=batch.metadata.batch_size) as executor: future_examples = [] - for example_idx in range(batch.batch_size): + for example_idx in range(batch.metadata.batch_size): future_example = executor.submit( self.get_example, batch, example_idx, t0_datetimes[example_idx] ) diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index 6757cb51..543733d0 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -61,6 +61,8 @@ def metadata_fake(batch_size): # get random times all_datetimes = pd.date_range("2021-01-01", "2021-02-01", freq="5T") t0_datetimes_utc = np.random.choice(all_datetimes, batch_size, replace=False) + # np.random.choice turns the pd.Timestamp objects into datetime.datetime objects. + t0_datetimes_utc = pd.to_datetime(t0_datetimes_utc) metadata_dict = {} metadata_dict["batch_size"] = batch_size diff --git a/nowcasting_dataset/data_sources/metadata/metadata_model.py b/nowcasting_dataset/data_sources/metadata/metadata_model.py index 22d2bcdb..99678702 100644 --- a/nowcasting_dataset/data_sources/metadata/metadata_model.py +++ b/nowcasting_dataset/data_sources/metadata/metadata_model.py @@ -1,6 +1,5 @@ """ Model for output of general/metadata data, useful for a batch """ -from datetime import datetime from typing import List import pandas as pd @@ -21,7 +20,7 @@ class Metadata(BaseModel): "then this item stores one data item", ) - t0_datetime_utc: List[datetime] = Field( + t0_datetime_utc: List[pd.Timestamp] = Field( ..., description="The t0s of each example ", ) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 666e0445..f043ca1d 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -1,7 +1,6 @@ """Test Optical Flow Data Source""" import tempfile -import numpy as np import pytest from nowcasting_dataset.config.model import Configuration, InputData @@ -21,36 +20,36 @@ def optical_flow_configuration(): # noqa: D103 return con -def test_optical_flow_get_example(optical_flow_configuration): +def test_optical_flow_get_example(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( number_previous_timesteps_to_use=1, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0] + batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_datetime_utc[0] ) assert example.values.shape == (12, 32, 32, 12) -def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): +def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( number_previous_timesteps_to_use=3, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0] + batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_datetime_utc[0] ) assert example.values.shape == (12, 32, 32, 12) -def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration): +def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration): # noqa: D103 optical_flow_datasource = OpticalFlowDataSource( number_previous_timesteps_to_use=300, image_size_pixels=32 ) batch = Batch.fake(configuration=optical_flow_configuration) with pytest.raises(AssertionError): optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_dt.values[0] + batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_datetime_utc[0] ) @@ -62,6 +61,6 @@ def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa batch = Batch.fake(configuration=optical_flow_configuration) batch.save_netcdf(path=dirpath, batch_i=0) optical_flow = optical_flow_datasource.get_batch( - netcdf_path=dirpath, batch_idx=0, t0_datetimes=batch.metadata.t0_dt.values + netcdf_path=dirpath, batch_idx=0, t0_datetimes=batch.metadata.t0_datetime_utc ) assert optical_flow.values.shape == (4, 12, 32, 32, 12) From 986b6cfb9b3442a33c239be00515d26aca80846c Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Mon, 29 Nov 2021 14:19:15 +0000 Subject: [PATCH 136/197] fix number of channels in test. The number of timesteps is still wrong --- .../optical_flow/test_optical_flow_data_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index f043ca1d..dd419442 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -28,7 +28,7 @@ def test_optical_flow_get_example(optical_flow_configuration): # noqa: D103 example = optical_flow_datasource.get_example( batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_datetime_utc[0] ) - assert example.values.shape == (12, 32, 32, 12) + assert example.values.shape == (12, 32, 32, 10) # timesteps, height, width, channels def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): # noqa: D103 @@ -39,7 +39,7 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): example = optical_flow_datasource.get_example( batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_datetime_utc[0] ) - assert example.values.shape == (12, 32, 32, 12) + assert example.values.shape == (12, 32, 32, 10) def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration): # noqa: D103 @@ -63,4 +63,4 @@ def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa optical_flow = optical_flow_datasource.get_batch( netcdf_path=dirpath, batch_idx=0, t0_datetimes=batch.metadata.t0_datetime_utc ) - assert optical_flow.values.shape == (4, 12, 32, 32, 12) + assert optical_flow.values.shape == (4, 12, 32, 32, 10) From 795ac368077a60b0de88bb5acbf2c68c2261b8b4 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Mon, 29 Nov 2021 17:22:12 +0000 Subject: [PATCH 137/197] all tests pass! --- .../data_sources/data_source.py | 24 ++++++-- .../optical_flow/optical_flow_data_source.py | 26 ++++---- nowcasting_dataset/manager.py | 10 +++- .../test_optical_flow_data_source.py | 22 ++++--- tests/test_manager.py | 59 +++++++++++-------- 5 files changed, 88 insertions(+), 53 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index a7330d0a..86ccbe49 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -140,7 +140,7 @@ def check_input_paths_exist(self) -> None: pass # TODO: Issue #319: Standardise parameter names. - # TODO: Reduce duplication: https://github.com/openclimatefix/nowcasting_dataset/issues/367 + # TODO: Issue #367: Reduce duplication. def create_batches( self, spatial_and_temporal_locations_of_each_example: pd.DataFrame, @@ -230,7 +230,9 @@ def create_batches( logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") # Generate batch. - batch = self._get_batch(locations_for_batch, **kwargs) + batch = self._get_batch( + locations_for_batch=locations_for_batch, batch_idx=batch_idx, **kwargs + ) # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) @@ -530,13 +532,15 @@ def datetime_index(self): ) def _get_batch(self, locations_for_batch, **kwargs): - if not all(key in kwargs for key in ["batch_path", "batch_idx"]): - raise ValueError("Missing arguments 'batch_path' and 'batch_idx'") + # Sanity check: + for key in ["batch_path", "batch_idx"]: + if key not in kwargs: + raise ValueError(f"Argument {key} is missing! ") batch = self.get_batch( netcdf_path=kwargs["batch_path"], batch_idx=kwargs["batch_idx"], - t0_datetimes=locations_for_batch.t0_datetime_UTC, + t0_datetimes=pd.DatetimeIndex(locations_for_batch.t0_datetime_UTC), ) return batch @@ -564,11 +568,19 @@ def get_batch( import nowcasting_dataset.dataset.batch batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(netcdf_path, batch_idx=batch_idx) + + # Sanity check + assert len(t0_datetimes) == batch.metadata.batch_size + assert isinstance(t0_datetimes, pd.DatetimeIndex) + with futures.ProcessPoolExecutor(max_workers=batch.metadata.batch_size) as executor: future_examples = [] for example_idx in range(batch.metadata.batch_size): future_example = executor.submit( - self.get_example, batch, example_idx, t0_datetimes[example_idx] + self.get_example, + batch=batch, + example_idx=example_idx, + t0_dt=t0_datetimes[example_idx], ) future_examples.append(future_example) examples = [future_example.result() for future_example in future_examples] diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 14943667..c057c7c6 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -33,7 +33,7 @@ def get_example( batch, # Of type nowcasting_dataset.dataset.batch.Batch. But we can't use # an "actual" type hint here otherwise we get a circular import error! example_idx: int, - t0_datetime: pd.Timestamp, + t0_dt: pd.Timestamp, **kwargs ) -> DataSourceOutput: """ @@ -42,7 +42,7 @@ def get_example( Args: batch: nowcasting_dataset.dataset.batch.Batch containing satellite and metadata at least example_idx: The example to load and use - t0_datetime: t0 datetime for the example + t0_dt: t0 datetime for the example Returns: Example Data @@ -54,7 +54,7 @@ def get_example( # Only do optical flow for satellite data self._data: xr.DataArray = batch.satellite.sel(example=example_idx) - selected_data = self._compute_and_return_optical_flow(self._data, t0_dt=t0_datetime) + selected_data = self._compute_and_return_optical_flow(self._data, t0_datetime_utc=t0_dt) return selected_data @@ -114,58 +114,58 @@ def _update_dataarray_with_predictions( def _get_previous_timesteps( self, satellite_data: xr.DataArray, - t0_dt: pd.Timestamp, + t0_datetime_utc: pd.Timestamp, ) -> xr.DataArray: """ Get timestamp of previous Args: satellite_data: Satellite data to use - t0_dt: Timestamp + t0_datetime_utc: Timestamp Returns: The previous timesteps """ - satellite_data = satellite_data.where(satellite_data.time <= t0_dt, drop=True) + satellite_data = satellite_data.where(satellite_data.time <= t0_datetime_utc, drop=True) return satellite_data def _get_number_future_timesteps( - self, satellite_data: xr.DataArray, t0_dt: pd.Timestamp + self, satellite_data: xr.DataArray, t0_datetime_utc: pd.Timestamp ) -> int: """ Get number of future timestamps Args: satellite_data: Satellite data to use - t0_dt: The timestamp of the t0 image + t0_datetime_utc: The timestamp of the t0 image Returns: The number of future timesteps """ - satellite_data = satellite_data.where(satellite_data.time > t0_dt, drop=True) + satellite_data = satellite_data.where(satellite_data.time > t0_datetime_utc, drop=True) return len(satellite_data.coords["time_index"]) def _compute_and_return_optical_flow( self, satellite_data: xr.DataArray, - t0_dt: pd.Timestamp, + t0_datetime_utc: pd.Timestamp, ) -> xr.DataArray: """ Compute and return optical flow predictions for the example Args: satellite_data: Satellite DataArray - t0_dt: t0 timestamp + t0_datetime_utc: t0 timestamp Returns: The Tensor with the optical flow predictions for t0 to forecast horizon """ # Get the previous timestamp - future_timesteps = self._get_number_future_timesteps(satellite_data, t0_dt) + future_timesteps = self._get_number_future_timesteps(satellite_data, t0_datetime_utc) historical_satellite_data: xr.DataArray = self._get_previous_timesteps( satellite_data, - t0_dt=t0_dt, + t0_datetime_utc=t0_datetime_utc, ) assert ( len(historical_satellite_data.coords["time_index"]) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index b8319f45..67339d5f 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -411,7 +411,11 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: # Check if there's any work to do. if overwrite_batches: - splits_which_need_more_batches = [split_name for split_name in split.SplitName] + splits_which_need_more_batches = [ + split_name + for split_name in split.SplitName + if self._get_n_batches_requested_for_split_name(split_name.value) > 0 + ] else: splits_which_need_more_batches = self._find_splits_which_need_more_batches( first_batches_to_create @@ -476,7 +480,9 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: data_source.create_batches, batch_path=self.config.output_data.filepath / split_name.value, spatial_and_temporal_locations_of_each_example=locations, - total_number_batches=self._get_n_batches_for_split_name(split_name.value), + total_number_batches=self._get_n_batches_requested_for_split_name( + split_name.value + ), idx_of_first_batch=idx_of_first_batch, batch_size=self.config.process.batch_size, dst_path=dst_path, diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index dd419442..8abd1d2a 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -1,6 +1,7 @@ """Test Optical Flow Data Source""" import tempfile +import pandas as pd import pytest from nowcasting_dataset.config.model import Configuration, InputData @@ -26,9 +27,11 @@ def test_optical_flow_get_example(optical_flow_configuration): # noqa: D103 ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_datetime_utc[0] + batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] ) - assert example.values.shape == (12, 32, 32, 10) # timesteps, height, width, channels + # As a nasty hack to get round #511, the number of timesteps is set to 0 for now. + # TODO: Issue #513: Set the number of timesteps back to 12! + assert example.values.shape == (0, 32, 32, 10) # timesteps, height, width, channels def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): # noqa: D103 @@ -37,9 +40,11 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_datetime_utc[0] + batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] ) - assert example.values.shape == (12, 32, 32, 10) + # As a nasty hack to get round #511, the number of timesteps is set to 0 for now. + # TODO: Issue #513: Set the number of timesteps back to 12! + assert example.values.shape == (0, 32, 32, 10) # timesteps, height, width, channels def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration): # noqa: D103 @@ -49,7 +54,7 @@ def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration) batch = Batch.fake(configuration=optical_flow_configuration) with pytest.raises(AssertionError): optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_datetime=batch.metadata.t0_datetime_utc[0] + batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] ) @@ -60,7 +65,10 @@ def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa with tempfile.TemporaryDirectory() as dirpath: batch = Batch.fake(configuration=optical_flow_configuration) batch.save_netcdf(path=dirpath, batch_i=0) + t0_datetime_utc = pd.DatetimeIndex(batch.metadata.t0_datetime_utc) optical_flow = optical_flow_datasource.get_batch( - netcdf_path=dirpath, batch_idx=0, t0_datetimes=batch.metadata.t0_datetime_utc + netcdf_path=dirpath, batch_idx=0, t0_datetimes=t0_datetime_utc ) - assert optical_flow.values.shape == (4, 12, 32, 32, 10) + # As a nasty hack to get round #511, the number of timesteps is set to 0 for now. + # TODO: Issue #513: Set the number of timesteps back to 12! + assert optical_flow.values.shape == (4, 0, 32, 32, 10) # ?, timesteps, height, width, chans diff --git a/tests/test_manager.py b/tests/test_manager.py index 607de335..e667f4d4 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -10,7 +10,6 @@ import nowcasting_dataset from nowcasting_dataset.data_sources import OpticalFlowDataSource from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource -from nowcasting_dataset.data_sources.metadata.metadata_data_source import MetadataDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource from nowcasting_dataset.data_sources.sun.sun_data_source import SunDataSource from nowcasting_dataset.manager import Manager @@ -108,10 +107,12 @@ def test_get_daylight_datetime_index(): def test_batches(): """Test that batches can be made""" - filename = Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "sat_data.zarr" + sat_filename = ( + Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "sat_data.zarr" + ) sat = SatelliteDataSource( - zarr_path=filename, + zarr_path=sat_filename, history_minutes=30, forecast_minutes=60, image_size_pixels=24, @@ -119,11 +120,11 @@ def test_batches(): channels=("IR_016",), ) - filename = ( + hrv_filename = ( Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "hrv_sat_data.zarr" ) hrvsat = SatelliteDataSource( - zarr_path=filename, + zarr_path=hrv_filename, history_minutes=30, forecast_minutes=60, image_size_pixels=64, @@ -131,12 +132,12 @@ def test_batches(): channels=("HRV",), ) - filename = ( + gsp_filename = ( Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "gsp" / "test.zarr" ) gsp = GSPDataSource( - zarr_path=filename, + zarr_path=gsp_filename, start_dt=datetime(2020, 4, 1), end_dt=datetime(2020, 4, 2), history_minutes=30, @@ -158,8 +159,8 @@ def test_batches(): manager.config.output_data.filepath = Path(dst_path) manager.local_temp_path = Path(local_temp_path) - # just set satellite as data source - manager.data_sources = {"gsp": gsp, "sat": sat, "hrvsat": hrvsat} + # Set data sources + manager.data_sources = {"gsp": gsp, "satellite": sat, "hrvsatellite": hrvsat} manager.data_source_which_defines_geospatial_locations = gsp # make file for locations @@ -171,19 +172,22 @@ def test_batches(): assert os.path.exists(f"{dst_path}/train") assert os.path.exists(f"{dst_path}/train/gsp") assert os.path.exists(f"{dst_path}/train/gsp/000000.nc") - assert os.path.exists(f"{dst_path}/train/sat/000000.nc") assert os.path.exists(f"{dst_path}/train/gsp/000001.nc") - assert os.path.exists(f"{dst_path}/train/sat/000001.nc") - assert os.path.exists(f"{dst_path}/train/hrvsat/000001.nc") - assert os.path.exists(f"{dst_path}/train/hrvsat/000000.nc") + assert os.path.exists(f"{dst_path}/train/satellite/000000.nc") + assert os.path.exists(f"{dst_path}/train/satellite/000001.nc") + assert os.path.exists(f"{dst_path}/train/hrvsatellite/000001.nc") + assert os.path.exists(f"{dst_path}/train/hrvsatellite/000000.nc") def test_derived_batches(): """Test that derived batches can be made""" - filename = Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "sat_data.zarr" + sat_filename = ( + Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "hrv_sat_data.zarr" + ) + # TODO: Reduce duplication between here and test_batches() sat = SatelliteDataSource( - zarr_path=filename, + zarr_path=sat_filename, history_minutes=30, forecast_minutes=60, image_size_pixels=64, @@ -191,14 +195,14 @@ def test_derived_batches(): channels=("HRV",), ) - filename = ( + gsp_filename = ( Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "gsp" / "test.zarr" ) gsp = GSPDataSource( - zarr_path=filename, - start_dt=datetime(2019, 1, 1), - end_dt=datetime(2019, 1, 2), + zarr_path=gsp_filename, + start_dt=datetime(2020, 4, 1), + end_dt=datetime(2020, 4, 2), history_minutes=30, forecast_minutes=60, image_size_pixels=64, @@ -211,8 +215,6 @@ def test_derived_batches(): image_size_pixels=32, ) - meta = MetadataDataSource(forecast_minutes=60, history_minutes=30) - manager = Manager() # load config @@ -224,8 +226,8 @@ def test_derived_batches(): # set local temp path, and dst path manager.config.output_data.filepath = Path(dst_path) manager.local_temp_path = Path(local_temp_path) - # just set satellite as data source - manager.data_sources = {"gsp": gsp, "sat": sat, "metadata": meta} + # Set data sources + manager.data_sources = {"gsp": gsp, "satellite": sat} manager.derived_data_sources = {"opticalflow": of} manager.data_source_which_defines_geospatial_locations = gsp @@ -236,11 +238,18 @@ def test_derived_batches(): manager.create_batches(overwrite_batches=True) import glob - print(list(glob.glob(os.path.join(dst_path, "train", "*")))) + print("glob(dst_path / train / *)", list(glob.glob(os.path.join(dst_path, "train", "*")))) + print( + "glob(dst_path / train / satellite / *)", + list(glob.glob(os.path.join(dst_path, "train", "satellite", "*"))), + ) + # Load batch from nowcasting_dataset.dataset.batch import Batch - _ = Batch.load_netcdf(os.path.join(dst_path, "train"), batch_idx=0) + _ = Batch.load_netcdf( + os.path.join(dst_path, "train"), batch_idx=0, data_sources_names=["satellite"] + ) # make derived batches manager.create_derived_batches(overwrite_batches=True) From 8ce5495f38f258e4a069983ede8e836fc326e245 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 10:34:46 +0000 Subject: [PATCH 138/197] Fix duplicate entries in log output. Fixes #446 --- nowcasting_dataset/data_sources/data_source.py | 2 +- nowcasting_dataset/data_sources/gsp/eso.py | 2 +- .../data_sources/optical_flow/optical_flow_data_source.py | 2 +- .../data_sources/satellite/satellite_data_source.py | 2 +- nowcasting_dataset/filesystem/utils.py | 2 +- nowcasting_dataset/manager.py | 3 ++- nowcasting_dataset/utils.py | 2 +- scripts/prepare_ml_data.py | 1 - 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 86ccbe49..dccf0f62 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -116,7 +116,7 @@ def sample_period_minutes(self) -> int: This functions may be overwritten if the sample period of the data source is not 5 minutes. """ - logging.debug( + logger.debug( "Getting sample_period_minutes default of 5 minutes. " "This means the data is spaced 5 minutes apart" ) diff --git a/nowcasting_dataset/data_sources/gsp/eso.py b/nowcasting_dataset/data_sources/gsp/eso.py index 4b8b1d9b..32b17c01 100644 --- a/nowcasting_dataset/data_sources/gsp/eso.py +++ b/nowcasting_dataset/data_sources/gsp/eso.py @@ -226,7 +226,7 @@ def get_list_of_gsp_ids(maximum_number_of_gsp: Optional[int] = None) -> List[int if maximum_number_of_gsp is None: maximum_number_of_gsp = len(metadata) if maximum_number_of_gsp > len(metadata): - logging.warning(f"Only {len(metadata)} gsp available to load") + logger.warning(f"Only {len(metadata)} gsp available to load") if maximum_number_of_gsp < len(metadata): gsp_ids = gsp_ids[0:maximum_number_of_gsp] diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index c057c7c6..39ef98f6 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -12,7 +12,7 @@ from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow -_LOG = logging.getLogger("nowcasting_dataset") +_LOG = logging.getLogger(__name__) @dataclass diff --git a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py index bf9caf5c..2211f177 100644 --- a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py +++ b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py @@ -14,7 +14,7 @@ from nowcasting_dataset.data_sources.data_source import ZarrDataSource from nowcasting_dataset.data_sources.satellite.satellite_model import Satellite -_LOG = logging.getLogger("nowcasting_dataset") +_LOG = logging.getLogger(__name__) @dataclass diff --git a/nowcasting_dataset/filesystem/utils.py b/nowcasting_dataset/filesystem/utils.py index c563b4e1..37dd9c2e 100644 --- a/nowcasting_dataset/filesystem/utils.py +++ b/nowcasting_dataset/filesystem/utils.py @@ -7,7 +7,7 @@ import numpy as np from pathy import Pathy -_LOG = logging.getLogger("nowcasting_dataset") +_LOG = logging.getLogger(__name__) def upload_and_delete_local_files(dst_path: Union[str, Path], local_path: Union[str, Path]): diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 67339d5f..6b008b36 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -92,6 +92,7 @@ def configure_loggers( log_filename = self.config.output_data.filepath / f"{data_source_name}.log" nd_utils.configure_logger( log_level=log_level, + # TODO: Fix bug #467: satellite.log file is not being appended to. logger_name=f"nowcasting_dataset.data_sources.{data_source_name}", handlers=[logging.FileHandler(log_filename, mode="a")], ) @@ -242,7 +243,7 @@ def _locations_csv_file_exists(self) -> bool: try: nd_fs_utils.check_path_exists(filename) except FileNotFoundError: - logging.info(f"{filename} does not exist!") + logger.info(f"{filename} does not exist!") return False else: logger.info(f"{filename} exists!") diff --git a/nowcasting_dataset/utils.py b/nowcasting_dataset/utils.py index 00a630c0..acc2ba97 100644 --- a/nowcasting_dataset/utils.py +++ b/nowcasting_dataset/utils.py @@ -169,7 +169,7 @@ def configure_logger(log_level: str, logger_name: str, handlers=list[logging.Han log_level = getattr(logging, log_level) # Convert string to int. formatter = logging.Formatter( - "%(asctime)s %(levelname)s processID=%(process)d %(message)s | %(pathname)s#L%(lineno)d" + "%(asctime)s:%(levelname)s:%(module)s#L%(lineno)d:PID=%(process)d:%(message)s" ) local_logger = logging.getLogger(logger_name) diff --git a/scripts/prepare_ml_data.py b/scripts/prepare_ml_data.py index bf44aa98..8d1b0a88 100755 --- a/scripts/prepare_ml_data.py +++ b/scripts/prepare_ml_data.py @@ -65,7 +65,6 @@ def main(config_filename: str, data_source: list[str], overwrite_batches: bool, manager.load_yaml_configuration(config_filename) manager.configure_loggers(log_level=log_level, names_of_selected_data_sources=data_source) manager.initialise_data_sources(names_of_selected_data_sources=data_source) - manager.initialize_data_sources(names_of_selected_data_sources=data_source) # TODO: Issue 323: maybe don't allow # create_files_specifying_spatial_and_temporal_locations_of_each_example to be run if a subset # of data_sources is passed in at the command line. From 506b5145b40d778f58dfe5f52cef462f2663cb28 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 11:04:04 +0000 Subject: [PATCH 139/197] Fixed bug where Manager thought all DerivedDataSources were complete. New ValueError bug --- nowcasting_dataset/manager.py | 38 +++++++++++++++++++++-------------- nowcasting_dataset/utils.py | 4 +--- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 6b008b36..c3503c92 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -18,7 +18,7 @@ SPATIAL_AND_TEMPORAL_LOCATIONS_OF_EACH_EXAMPLE_FILENAME, ) from nowcasting_dataset.data_sources import ALL_DATA_SOURCE_NAMES, MAP_DATA_SOURCE_NAME_TO_CLASS -from nowcasting_dataset.data_sources.data_source import DerivedDataSource +from nowcasting_dataset.data_sources.data_source import DataSource, DerivedDataSource from nowcasting_dataset.dataset.split import split from nowcasting_dataset.filesystem import utils as nd_fs_utils @@ -338,7 +338,7 @@ def sample_spatial_and_temporal_locations_for_examples( def _get_first_batches_to_create( self, overwrite_batches: bool, - data_sources: dict, + data_sources: dict[str, DataSource], ) -> dict[split.SplitName, dict[str, int]]: """For each SplitName & for each DataSource name, return the first batch ID to create. @@ -373,25 +373,30 @@ def _check_if_more_batches_are_required_for_split( self, split_name: split.SplitName, first_batches_to_create: dict[split.SplitName, dict[str, int]], + data_sources: dict[str, DataSource], ) -> bool: """Returns True if batches still need to be created for any DataSource.""" n_batches_requested = self._get_n_batches_requested_for_split_name(split_name.value) - for data_source_name in self.data_sources: + for data_source_name in data_sources: if first_batches_to_create[split_name][data_source_name] < n_batches_requested: return True return False def _find_splits_which_need_more_batches( - self, first_batches_to_create: dict[split.SplitName, dict[str, int]] + self, + first_batches_to_create: dict[split.SplitName, dict[str, int]], + data_sources: dict[str, DataSource], ) -> list[split.SplitName]: """Returns list of SplitNames which need more batches to be produced.""" - splits_which_need_more_batches = [] - for split_name in split.SplitName: + return [ + split_name + for split_name in split.SplitName if self._check_if_more_batches_are_required_for_split( - split_name, first_batches_to_create - ): - splits_which_need_more_batches.append(split_name) - return splits_which_need_more_batches + split_name=split_name, + first_batches_to_create=first_batches_to_create, + data_sources=data_sources, + ) + ] # TODO: Reduce duplication: https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_derived_batches(self, overwrite_batches: bool) -> None: @@ -406,8 +411,9 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: written to disk, and only create any batches which have not yet been written to disk. """ + logger.debug("Entering Manager.create_derived_batches...") first_batches_to_create = self._get_first_batches_to_create( - overwrite_batches, self.derived_data_sources + overwrite_batches=overwrite_batches, data_sources=self.derived_data_sources ) # Check if there's any work to do. @@ -419,10 +425,11 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: ] else: splits_which_need_more_batches = self._find_splits_which_need_more_batches( - first_batches_to_create + first_batches_to_create=first_batches_to_create, + data_sources=self.derived_data_sources, ) if len(splits_which_need_more_batches) == 0: - logger.info("All batches have already been created! No work to do!") + logger.info("All derived batches have already been created! No work to do!") return # Load locations for each example off disk. @@ -516,8 +523,9 @@ def create_batches(self, overwrite_batches: bool) -> None: previously been written to disk. If False then check which batches have previously been written to disk, and only create any batches which have not yet been written to disk. """ + logger.debug("Entering Manager.create_batches...") first_batches_to_create = self._get_first_batches_to_create( - overwrite_batches, self.data_sources + overwrite_batches=overwrite_batches, data_sources=self.data_sources ) # Check if there's any work to do. @@ -529,7 +537,7 @@ def create_batches(self, overwrite_batches: bool) -> None: ] else: splits_which_need_more_batches = self._find_splits_which_need_more_batches( - first_batches_to_create + first_batches_to_create=first_batches_to_create, data_sources=self.data_sources ) if len(splits_which_need_more_batches) == 0: logger.info("All batches have already been created! No work to do!") diff --git a/nowcasting_dataset/utils.py b/nowcasting_dataset/utils.py index acc2ba97..d28d497b 100644 --- a/nowcasting_dataset/utils.py +++ b/nowcasting_dataset/utils.py @@ -149,9 +149,7 @@ def arg_logger(func): # Adapted from https://stackoverflow.com/a/23983263/732596 @wraps(func) def inner_func(*args, **kwargs): - logger.debug( - f"Arguments passed into function `{func.__name__}`:" f" args={args}; kwargs={kwargs}" - ) + logger.debug(f"Arguments passed into function `{func.__name__}`: {args=}; {kwargs=}") return func(*args, **kwargs) return inner_func From 0303b3e3c8fc492c9d1c36251fc8ee3a38a207f7 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 11:20:03 +0000 Subject: [PATCH 140/197] It is now creating OpticalFlow batches! --- nowcasting_dataset/config/on_premises.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index b2a66716..a23a6f0d 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -65,7 +65,7 @@ input_data: # ------------------------- Optical Flow --------------- opticalflow: number_previous_timesteps_to_use: 1 - opticalflow_image_size_pixels: 64 + opticalflow_image_size_pixels: 20 output_data: filepath: /mnt/storage_ssd_4tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/prepared_ML_training_data/v15 From ca614dc3e0ac7eb692792d553ba7aafb5e2814ec Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 13:18:42 +0000 Subject: [PATCH 141/197] a number of small fixes. But the opt flow predictions are still not changing --- .../data_sources/data_source.py | 11 +- .../optical_flow/optical_flow_data_source.py | 103 +++++++++--------- nowcasting_dataset/manager.py | 3 +- nowcasting_dataset/utils.py | 32 ++++++ 4 files changed, 91 insertions(+), 58 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index dccf0f62..5b6496cc 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -20,7 +20,7 @@ convert_coordinates_to_indexes_for_list_datasets, join_list_dataset_to_batch_dataset, ) -from nowcasting_dataset.utils import get_start_and_end_example_index +from nowcasting_dataset.utils import DummyExecutor, get_start_and_end_example_index logger = logging.getLogger(__name__) @@ -557,7 +557,7 @@ def get_batch( Args: netcdf_path: Path to the NetCDF files of the Batch to load batch_idx: The batch ID to load from those in the path - t0_datetimes: list of timestamps for the datetime of the batches. The batch will also + t0_datetimes: t0 datetimes for each example in the batch. The batch will also include data for historic and future depending on `history_minutes` and `future_minutes`. The batch size is given by the length of the t0_datetimes. @@ -570,10 +570,13 @@ def get_batch( batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(netcdf_path, batch_idx=batch_idx) # Sanity check - assert len(t0_datetimes) == batch.metadata.batch_size + assert ( + len(t0_datetimes) == batch.metadata.batch_size + ), f"{len(t0_datetimes)=} != {batch.metadata.batch_size=}" assert isinstance(t0_datetimes, pd.DatetimeIndex) - with futures.ProcessPoolExecutor(max_workers=batch.metadata.batch_size) as executor: + # with futures.ProcessPoolExecutor(max_workers=batch.metadata.batch_size) as executor: + with DummyExecutor(max_workers=batch.metadata.batch_size) as executor: future_examples = [] for example_idx in range(batch.metadata.batch_size): future_example = executor.submit( diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 39ef98f6..18078ecb 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -52,18 +52,16 @@ def get_example( self.image_size_pixels = len(batch.satellite.x_index) # Only do optical flow for satellite data - self._data: xr.DataArray = batch.satellite.sel(example=example_idx) - - selected_data = self._compute_and_return_optical_flow(self._data, t0_datetime_utc=t0_dt) - - return selected_data + # TODO: Enable this to work with hrvsatellite too. + satellite_data: xr.DataArray = batch.satellite.sel(example=example_idx) + return self._compute_and_return_optical_flow(satellite_data, t0_datetime_utc=t0_dt) @staticmethod def get_data_model_for_batch(): """Get the model that is used in the batch""" return OpticalFlow - def _update_dataarray_with_predictions( + def _put_predictions_into_data_array( self, satellite_data: xr.DataArray, predictions: np.ndarray, @@ -80,37 +78,30 @@ def _update_dataarray_with_predictions( Returns: The Xarray DataArray with the optical flow predictions """ - # Combine all channels for a single timestep + # Select the timesteps for the optical flow predictions. satellite_data = satellite_data.isel( time_index=slice( satellite_data.sizes["time_index"] - predictions.shape[0], satellite_data.sizes["time_index"], ) ) - # Make sure its the correct size - buffer = (satellite_data.sizes["x_index"] - self.image_size_pixels) // 2 + # Select the center crop. + border = (satellite_data.sizes["x_index"] - self.image_size_pixels) // 2 satellite_data = satellite_data.isel( - x_index=slice(buffer, satellite_data.sizes["x_index"] - buffer), - y_index=slice(buffer, satellite_data.sizes["y_index"] - buffer), + x_index=slice(border, satellite_data.sizes["x_index"] - border), + y_index=slice(border, satellite_data.sizes["y_index"] - border), ) - dataarray = xr.DataArray( + return xr.DataArray( data=predictions, - dims={ - "time_index": satellite_data.dims["time_index"], - "x_index": satellite_data.dims["x_index"], - "y_index": satellite_data.dims["y_index"], - "channels_index": satellite_data.dims["channels_index"], - }, - coords={ - "time_index": satellite_data.coords["time_index"], - "x_index": satellite_data.coords["x_index"], - "y_index": satellite_data.coords["y_index"], - "channels_index": satellite_data.coords["channels_index"], - }, + coords=( + ("time_index", satellite_data.coords["time_index"].values), + ("x_index", satellite_data.coords["x_index"].values), + ("y_index", satellite_data.coords["y_index"].values), + ("channels_index", satellite_data.coords["channels_index"].values), + ), + name="data", ) - return dataarray - def _get_previous_timesteps( self, satellite_data: xr.DataArray, @@ -172,34 +163,39 @@ def _compute_and_return_optical_flow( - self.number_previous_timesteps_to_use - 1 ) >= 0, "Trying to compute flow further back than the number of historical timesteps" - prediction_block = np.zeros( - ( + + # TODO: Use the correct dtype. + n_channels = satellite_data.sizes["channels_index"] + prediction_block = np.full( + shape=( future_timesteps, self.image_size_pixels, self.image_size_pixels, - satellite_data.sizes["channels_index"], - ) + n_channels, + ), + fill_value=np.NaN, ) - for prediction_timestep in range(future_timesteps): - for channel in range(0, len(historical_satellite_data.coords["channels_index"])): - t0 = historical_satellite_data.sel(channels_index=channel) - previous = historical_satellite_data.sel(channels_index=channel) - optical_flows = [] - for i in range( - len(historical_satellite_data.coords["time_index"]) - 1, - len(historical_satellite_data.coords["time_index"]) - - self.number_previous_timesteps_to_use - - 1, - -1, - ): - t0_image = t0.isel(time_index=i).data.values - previous_image = previous.isel(time_index=i - 1).data.values - optical_flow = compute_optical_flow(t0_image, previous_image) - optical_flows.append(optical_flow) - # Average predictions - optical_flow = np.mean(optical_flows, axis=0) - # Do predictions now - t0_image = t0.isel(time_index=-1).data.values + + for channel in range(n_channels): + # Compute optical flow field: + historical_sat_data_for_chan = historical_satellite_data.isel(channels_index=channel) + + # Loop through pairs of historical images to compute optical flow fields: + optical_flows = [] + n_historical_timesteps = len(historical_satellite_data.coords["time_index"]) + end_time_i = n_historical_timesteps + start_time_i = end_time_i - self.number_previous_timesteps_to_use + for time_i in range(start_time_i, end_time_i): + image_0 = historical_sat_data_for_chan.isel(time_index=time_i - 1).data.values + image_1 = historical_sat_data_for_chan.isel(time_index=time_i).data.values + optical_flow = compute_optical_flow(image_1, image_0) + optical_flows.append(optical_flow) + # Average predictions + optical_flow = np.mean(optical_flows, axis=0) + + # Compute predicted images. + t0_image = historical_sat_data_for_chan.isel(time_index=-1).data.values + for prediction_timestep in range(future_timesteps): flow = optical_flow * (prediction_timestep + 1) warped_image = remap_image(t0_image, flow) warped_image = crop_center( @@ -208,10 +204,11 @@ def _compute_and_return_optical_flow( self.image_size_pixels, ) prediction_block[prediction_timestep, :, :, channel] = warped_image - dataarray = self._update_dataarray_with_predictions( - satellite_data=self._data, predictions=prediction_block + + data_array = self._put_predictions_into_data_array( + satellite_data=satellite_data, predictions=prediction_block ) - return dataarray + return data_array def compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index c3503c92..923a83c8 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -453,7 +453,8 @@ def create_derived_batches(self, overwrite_batches: bool) -> None: for split_name in splits_which_need_more_batches: locations_for_split = locations_for_each_example_of_each_split[split_name] # TODO: Maybe use multiprocessing.Pool instead of ProcessPoolExecutor? - with futures.ProcessPoolExecutor(max_workers=n_data_sources) as executor: + # with futures.ProcessPoolExecutor(max_workers=n_data_sources) as executor: + with nd_utils.DummyExecutor(max_workers=n_data_sources) as executor: future_create_batches_jobs = [] for worker_id, (data_source_name, data_source) in enumerate( self.derived_data_sources.items() diff --git a/nowcasting_dataset/utils.py b/nowcasting_dataset/utils.py index d28d497b..59d18062 100644 --- a/nowcasting_dataset/utils.py +++ b/nowcasting_dataset/utils.py @@ -3,6 +3,8 @@ import os import re import tempfile +import threading +from concurrent import futures from functools import wraps import fsspec.asyn @@ -194,3 +196,33 @@ def get_start_and_end_example_index(batch_idx: int, batch_size: int) -> (int, in end_example_idx = (batch_idx + 1) * batch_size return start_example_idx, end_example_idx + + +class DummyExecutor(futures.Executor): + """Drop-in replacement for ThreadPoolExecutor or ProcessPoolExecutor for easy debugging. + + Adapted from https://stackoverflow.com/a/10436851/732596 + """ + + def __init__(self, *args, **kwargs): + self._shutdown = False + self._shutdownLock = threading.Lock() + + def submit(self, fn, *args, **kwargs): + with self._shutdownLock: + if self._shutdown: + raise RuntimeError("cannot schedule new futures after shutdown") + + f = futures.Future() + try: + result = fn(*args, **kwargs) + except BaseException as e: + f.set_exception(e) + else: + f.set_result(result) + + return f + + def shutdown(self, wait=True): + with self._shutdownLock: + self._shutdown = True From 01a4be579af6b644e39a8629e9dd6cc51c70ac5d Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 13:52:20 +0000 Subject: [PATCH 142/197] optical flow now moves image forwards. But now need much larger input images! --- .../optical_flow/optical_flow_data_source.py | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 18078ecb..cfa4cb88 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -186,9 +186,9 @@ def _compute_and_return_optical_flow( end_time_i = n_historical_timesteps start_time_i = end_time_i - self.number_previous_timesteps_to_use for time_i in range(start_time_i, end_time_i): - image_0 = historical_sat_data_for_chan.isel(time_index=time_i - 1).data.values - image_1 = historical_sat_data_for_chan.isel(time_index=time_i).data.values - optical_flow = compute_optical_flow(image_1, image_0) + prev_image = historical_sat_data_for_chan.isel(time_index=time_i - 1).data.values + next_image = historical_sat_data_for_chan.isel(time_index=time_i).data.values + optical_flow = compute_optical_flow(prev_image, next_image) optical_flows.append(optical_flow) # Average predictions optical_flow = np.mean(optical_flows, axis=0) @@ -197,7 +197,7 @@ def _compute_and_return_optical_flow( t0_image = historical_sat_data_for_chan.isel(time_index=-1).data.values for prediction_timestep in range(future_timesteps): flow = optical_flow * (prediction_timestep + 1) - warped_image = remap_image(t0_image, flow) + warped_image = remap_image(image=t0_image, flow=flow) warped_image = crop_center( warped_image, self.image_size_pixels, @@ -211,7 +211,7 @@ def _compute_and_return_optical_flow( return data_array -def compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np.ndarray: +def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.ndarray: """ Compute the optical flow for a set of images @@ -222,16 +222,23 @@ def compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np Returns: Optical Flow field """ - # Input images have to be single channel and between 0 and 1 - image_min = np.min([t0_image, previous_image]) - image_max = np.max([t0_image, previous_image]) - t0_image -= image_min - t0_image /= image_max - previous_image -= image_min - previous_image /= image_max - return cv2.calcOpticalFlowFarneback( - prev=previous_image, - next=t0_image, + # Input images have to be single channel and uint8. + # TODO: Refactor this! + image_min = np.min([prev_image, next_image]) + image_max = np.max([prev_image, next_image]) + prev_image = prev_image - image_min + prev_image = prev_image / (image_max - image_min) + prev_image = prev_image * 255 + prev_image = prev_image.astype(np.uint8) + next_image = next_image - image_min + next_image = next_image / (image_max - image_min) + next_image = next_image * 255 + next_image = next_image.astype(np.uint8) + + # Docs: https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af # nopa + flow = cv2.calcOpticalFlowFarneback( + prev=prev_image, + next=next_image, flow=None, pyr_scale=0.5, levels=2, @@ -241,6 +248,7 @@ def compute_optical_flow(t0_image: np.ndarray, previous_image: np.ndarray) -> np poly_sigma=0.7, flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, ) + return flow def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: @@ -259,7 +267,9 @@ def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: remap = -flow.copy() remap[..., 0] += np.arange(width) # map_x remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y - return cv2.remap( + # remap docs: https://docs.opencv.org/4.5.4/da/d54/group__imgproc__transform.html#gab75ef31ce5cdfb5c44b6da5f3b908ea4 # noqa + # TODO: Maybe use integer remap: docs say that might be faster? + remapped_image = cv2.remap( src=image, map1=remap, map2=None, @@ -267,6 +277,7 @@ def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: borderMode=cv2.BORDER_CONSTANT, borderValue=np.NaN, ) + return remapped_image def crop_center(image: np.ndarray, x_size: int, y_size: int) -> np.ndarray: From aa542dbe03af581f54d20a239d24feb502461132 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 15:07:57 +0000 Subject: [PATCH 143/197] making a start on removing DerivedDataSource --- .../data_sources/data_source.py | 80 ------------ .../optical_flow/optical_flow_data_source.py | 4 +- nowcasting_dataset/manager.py | 114 ------------------ 3 files changed, 2 insertions(+), 196 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 5b6496cc..15a77cd4 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -513,83 +513,3 @@ def open(self) -> None: def _open_data(self) -> xr.DataArray: raise NotImplementedError() - - -@dataclass -class DerivedDataSource(DataSource): - """ - Base class for data sources derived from other data sources - """ - - history_minutes: int = 0 - forecast_minutes: int = 0 - - def datetime_index(self): - """The datetime index of this datasource""" - return NotImplementedError( - "DerivedDataSources only use other, pre-computed batches, so no datetime_index is " - "needed" - ) - - def _get_batch(self, locations_for_batch, **kwargs): - # Sanity check: - for key in ["batch_path", "batch_idx"]: - if key not in kwargs: - raise ValueError(f"Argument {key} is missing! ") - - batch = self.get_batch( - netcdf_path=kwargs["batch_path"], - batch_idx=kwargs["batch_idx"], - t0_datetimes=pd.DatetimeIndex(locations_for_batch.t0_datetime_UTC), - ) - return batch - - def get_batch( - self, - netcdf_path: Union[str, Path], - batch_idx: int, - t0_datetimes: pd.DatetimeIndex, - **kwargs, - ) -> DataSourceOutput: - """ - Get Batch of derived data - - Args: - netcdf_path: Path to the NetCDF files of the Batch to load - batch_idx: The batch ID to load from those in the path - t0_datetimes: t0 datetimes for each example in the batch. The batch will also - include data for historic and future depending on `history_minutes` and - `future_minutes`. The batch size is given by the length of the t0_datetimes. - - Returns: - Batch of the derived data source - """ - # To get around circular imports - import nowcasting_dataset.dataset.batch - - batch = nowcasting_dataset.dataset.batch.Batch.load_netcdf(netcdf_path, batch_idx=batch_idx) - - # Sanity check - assert ( - len(t0_datetimes) == batch.metadata.batch_size - ), f"{len(t0_datetimes)=} != {batch.metadata.batch_size=}" - assert isinstance(t0_datetimes, pd.DatetimeIndex) - - # with futures.ProcessPoolExecutor(max_workers=batch.metadata.batch_size) as executor: - with DummyExecutor(max_workers=batch.metadata.batch_size) as executor: - future_examples = [] - for example_idx in range(batch.metadata.batch_size): - future_example = executor.submit( - self.get_example, - batch=batch, - example_idx=example_idx, - t0_dt=t0_datetimes[example_idx], - ) - future_examples.append(future_example) - examples = [future_example.result() for future_example in future_examples] - - # Get the DataSource class, this could be one of the data sources like Sun - cls = examples[0].__class__ - - # join the examples together, and cast them to the cls, so that validation can occur - return cls(join_list_dataset_to_batch_dataset(examples)) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index cfa4cb88..184b9245 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -8,7 +8,7 @@ import pandas as pd import xarray as xr -from nowcasting_dataset.data_sources.data_source import DerivedDataSource +from nowcasting_dataset.data_sources.data_source.satellite import SatelliteDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow @@ -16,7 +16,7 @@ @dataclass -class OpticalFlowDataSource(DerivedDataSource): +class OpticalFlowDataSource(SatelliteDataSource): """ Optical Flow Data Source, computing flow between Satellite data diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 923a83c8..3952c998 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -2,7 +2,6 @@ import logging import multiprocessing -from concurrent import futures from pathlib import Path from typing import Optional, Union @@ -398,119 +397,6 @@ def _find_splits_which_need_more_batches( ) ] - # TODO: Reduce duplication: https://github.com/openclimatefix/nowcasting_dataset/issues/367 - def create_derived_batches(self, overwrite_batches: bool) -> None: - """ - Create batches of derived data sources - - This loads previously created batches - - Args: - overwrite_batches: If True then start from batch 0, regardless of which batches have - previously been written to disk. If False then check which batches have previously been - written to disk, and only create any batches which have not yet been written to disk. - - """ - logger.debug("Entering Manager.create_derived_batches...") - first_batches_to_create = self._get_first_batches_to_create( - overwrite_batches=overwrite_batches, data_sources=self.derived_data_sources - ) - - # Check if there's any work to do. - if overwrite_batches: - splits_which_need_more_batches = [ - split_name - for split_name in split.SplitName - if self._get_n_batches_requested_for_split_name(split_name.value) > 0 - ] - else: - splits_which_need_more_batches = self._find_splits_which_need_more_batches( - first_batches_to_create=first_batches_to_create, - data_sources=self.derived_data_sources, - ) - if len(splits_which_need_more_batches) == 0: - logger.info("All derived batches have already been created! No work to do!") - return - - # Load locations for each example off disk. - locations_for_each_example_of_each_split: dict[split.SplitName, pd.DataFrame] = {} - for split_name in splits_which_need_more_batches: - filename = self._filename_of_locations_csv_file(split_name.value) - logger.info(f"Loading {filename}.") - locations_for_each_example = pd.read_csv(filename, index_col=0) - assert locations_for_each_example.columns.to_list() == list( - SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES - ) - # Converting to datetimes is much faster using `pd.to_datetime()` than - # passing `parse_datetimes` into `pd.read_csv()`. - locations_for_each_example["t0_datetime_UTC"] = pd.to_datetime( - locations_for_each_example["t0_datetime_UTC"] - ) - locations_for_each_example_of_each_split[split_name] = locations_for_each_example - - n_data_sources = len(self.derived_data_sources) - nd_utils.set_fsspec_for_multiprocess() - for split_name in splits_which_need_more_batches: - locations_for_split = locations_for_each_example_of_each_split[split_name] - # TODO: Maybe use multiprocessing.Pool instead of ProcessPoolExecutor? - # with futures.ProcessPoolExecutor(max_workers=n_data_sources) as executor: - with nd_utils.DummyExecutor(max_workers=n_data_sources) as executor: - future_create_batches_jobs = [] - for worker_id, (data_source_name, data_source) in enumerate( - self.derived_data_sources.items() - ): - - if len(locations_for_split) == 0: - break - - # Get indexes of first batch and example. And subset locations_for_split. - idx_of_first_batch = first_batches_to_create[split_name][data_source_name] - idx_of_first_example = idx_of_first_batch * self.config.process.batch_size - locations = locations_for_split.loc[idx_of_first_example:] - - # Get paths. - dst_path = ( - self.config.output_data.filepath / split_name.value / data_source_name - ) - local_temp_path = ( - self.local_temp_path - / split_name.value - / data_source_name - / f"worker_{worker_id}" - ) - - # Make folders. - nd_fs_utils.makedirs(dst_path, exist_ok=True) - if self.save_batches_locally_and_upload: - nd_fs_utils.makedirs(local_temp_path, exist_ok=True) - - # Submit data_source.create_batches task to the worker process. - future = executor.submit( - data_source.create_batches, - batch_path=self.config.output_data.filepath / split_name.value, - spatial_and_temporal_locations_of_each_example=locations, - total_number_batches=self._get_n_batches_requested_for_split_name( - split_name.value - ), - idx_of_first_batch=idx_of_first_batch, - batch_size=self.config.process.batch_size, - dst_path=dst_path, - local_temp_path=local_temp_path, - upload_every_n_batches=self.config.process.upload_every_n_batches, - ) - future_create_batches_jobs.append(future) - - # Wait for all futures to finish: - for future, data_source_name in zip( - future_create_batches_jobs, self.derived_data_sources.keys() - ): - # Call exception() to propagate any exceptions raised by the worker process into - # the main process, and to wait for the worker to finish. - exception = future.exception() - if exception is not None: - logger.exception(f"Worker process {data_source_name} raised exception!") - raise exception - # TODO: Reduce duplication: https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_batches(self, overwrite_batches: bool) -> None: """Create batches (if necessary). From d95fdde2c6bbe9e74443b08b2077c7005f05a101 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 15:20:37 +0000 Subject: [PATCH 144/197] finish removing DerivedDataSource --- .../data_sources/data_source.py | 21 +---- nowcasting_dataset/manager.py | 29 ++----- scripts/prepare_ml_data.py | 1 - tests/test_manager.py | 77 ------------------- 4 files changed, 9 insertions(+), 119 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 15a77cd4..743e03b6 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -150,7 +150,6 @@ def create_batches( local_temp_path: Path, upload_every_n_batches: int, total_number_batches: int = None, - **kwargs, ) -> None: """Create multiple batches and save them to disk. @@ -172,7 +171,6 @@ def create_batches( number of batches have been created. If 0 then will write directly to `dst_path`. total_number_batches (int, optional): If specified it will be used to compute the batch size (`batch_size` will not be used in that case). - **kwargs: Arguments specific to the `_get_batch` method. """ # Sanity checks: assert idx_of_first_batch >= 0, ( @@ -230,9 +228,7 @@ def create_batches( logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") # Generate batch. - batch = self._get_batch( - locations_for_batch=locations_for_batch, batch_idx=batch_idx, **kwargs - ) + batch = self.get_batch(locations_for_batch=locations_for_batch, batch_idx=batch_idx) # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) @@ -250,21 +246,6 @@ def create_batches( if save_batches_locally_and_upload: nd_fs_utils.upload_and_delete_local_files(dst_path, path_to_write_to) - def _get_batch(self, locations_for_batch, **kwargs): - """Get the batch for the given datasource. - - This, along with `get_batch`, should be implemented in the child classes if needed. - - `_get_batch` is used internally here and has a specific signature, because it is called in - `create_batches` which can be common to different classes inheriting from `DataSource` - (e.g. `DerivedDataSource`). - """ - return self.get_batch( - t0_datetimes=locations_for_batch.t0_datetime_UTC, - x_locations=locations_for_batch.x_center_OSGB, - y_locations=locations_for_batch.y_center_OSGB, - ) - # TODO: Issue #319: Standardise parameter names. def get_batch( self, diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 3952c998..5565ddaa 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -17,7 +17,7 @@ SPATIAL_AND_TEMPORAL_LOCATIONS_OF_EACH_EXAMPLE_FILENAME, ) from nowcasting_dataset.data_sources import ALL_DATA_SOURCE_NAMES, MAP_DATA_SOURCE_NAME_TO_CLASS -from nowcasting_dataset.data_sources.data_source import DataSource, DerivedDataSource +from nowcasting_dataset.data_sources.data_source import DataSource from nowcasting_dataset.dataset.split import split from nowcasting_dataset.filesystem import utils as nd_fs_utils @@ -30,7 +30,6 @@ class Manager: Attrs: config: Configuration object. data_sources: dict[str, DataSource] - derived_data_sources: dict[str, DerivedDataSource] data_source_which_defines_geospatial_locations: DataSource: The DataSource used to compute the geospatial locations of each example. save_batches_locally_and_upload: bool: Set to True by `load_yaml_configuration()` if @@ -41,7 +40,6 @@ class Manager: def __init__(self) -> None: # noqa: D107 self.config = None self.data_sources = {} - self.derived_data_sources = {} self.data_source_which_defines_geospatial_locations = None def load_yaml_configuration(self, filename: str) -> None: @@ -125,10 +123,7 @@ def initialise_data_sources( except Exception: logger.exception(f"Exception whilst instantiating {data_source_name}!") raise - if isinstance(data_source, DerivedDataSource): - self.derived_data_sources[data_source_name] = data_source - else: - self.data_sources[data_source_name] = data_source + self.data_sources[data_source_name] = data_source # Set data_source_which_defines_geospatial_locations: try: @@ -335,9 +330,7 @@ def sample_spatial_and_temporal_locations_for_examples( ) def _get_first_batches_to_create( - self, - overwrite_batches: bool, - data_sources: dict[str, DataSource], + self, overwrite_batches: bool ) -> dict[split.SplitName, dict[str, int]]: """For each SplitName & for each DataSource name, return the first batch ID to create. @@ -348,7 +341,7 @@ def _get_first_batches_to_create( first_batches_to_create: dict[split.SplitName, dict[str, int]] = {} for split_name in split.SplitName: first_batches_to_create[split_name] = { - data_source_name: 0 for data_source_name in data_sources + data_source_name: 0 for data_source_name in self.data_sources } if overwrite_batches: @@ -356,7 +349,7 @@ def _get_first_batches_to_create( # If we're not overwriting batches then find the last batch on disk. for split_name in split.SplitName: - for data_source_name in data_sources: + for data_source_name in self.data_sources: path = ( self.config.output_data.filepath / split_name.value / data_source_name / "*.nc" ) @@ -372,11 +365,10 @@ def _check_if_more_batches_are_required_for_split( self, split_name: split.SplitName, first_batches_to_create: dict[split.SplitName, dict[str, int]], - data_sources: dict[str, DataSource], ) -> bool: """Returns True if batches still need to be created for any DataSource.""" n_batches_requested = self._get_n_batches_requested_for_split_name(split_name.value) - for data_source_name in data_sources: + for data_source_name in self.data_sources: if first_batches_to_create[split_name][data_source_name] < n_batches_requested: return True return False @@ -384,7 +376,6 @@ def _check_if_more_batches_are_required_for_split( def _find_splits_which_need_more_batches( self, first_batches_to_create: dict[split.SplitName, dict[str, int]], - data_sources: dict[str, DataSource], ) -> list[split.SplitName]: """Returns list of SplitNames which need more batches to be produced.""" return [ @@ -393,11 +384,9 @@ def _find_splits_which_need_more_batches( if self._check_if_more_batches_are_required_for_split( split_name=split_name, first_batches_to_create=first_batches_to_create, - data_sources=data_sources, ) ] - # TODO: Reduce duplication: https://github.com/openclimatefix/nowcasting_dataset/issues/367 def create_batches(self, overwrite_batches: bool) -> None: """Create batches (if necessary). @@ -411,9 +400,7 @@ def create_batches(self, overwrite_batches: bool) -> None: written to disk, and only create any batches which have not yet been written to disk. """ logger.debug("Entering Manager.create_batches...") - first_batches_to_create = self._get_first_batches_to_create( - overwrite_batches=overwrite_batches, data_sources=self.data_sources - ) + first_batches_to_create = self._get_first_batches_to_create(overwrite_batches) # Check if there's any work to do. if overwrite_batches: @@ -424,7 +411,7 @@ def create_batches(self, overwrite_batches: bool) -> None: ] else: splits_which_need_more_batches = self._find_splits_which_need_more_batches( - first_batches_to_create=first_batches_to_create, data_sources=self.data_sources + first_batches_to_create=first_batches_to_create ) if len(splits_which_need_more_batches) == 0: logger.info("All batches have already been created! No work to do!") diff --git a/scripts/prepare_ml_data.py b/scripts/prepare_ml_data.py index 8d1b0a88..7c60bfe8 100755 --- a/scripts/prepare_ml_data.py +++ b/scripts/prepare_ml_data.py @@ -70,7 +70,6 @@ def main(config_filename: str, data_source: list[str], overwrite_batches: bool, # of data_sources is passed in at the command line. manager.create_files_specifying_spatial_and_temporal_locations_of_each_example_if_necessary() manager.create_batches(overwrite_batches) - manager.create_derived_batches(overwrite_batches) manager.save_yaml_configuration() # TODO: Issue #317: Validate ML data. logger.info("Done!") diff --git a/tests/test_manager.py b/tests/test_manager.py index e667f4d4..dcbaaf8d 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -8,7 +8,6 @@ import pandas as pd import nowcasting_dataset -from nowcasting_dataset.data_sources import OpticalFlowDataSource from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import SatelliteDataSource from nowcasting_dataset.data_sources.sun.sun_data_source import SunDataSource @@ -179,82 +178,6 @@ def test_batches(): assert os.path.exists(f"{dst_path}/train/hrvsatellite/000000.nc") -def test_derived_batches(): - """Test that derived batches can be made""" - sat_filename = ( - Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "hrv_sat_data.zarr" - ) - - # TODO: Reduce duplication between here and test_batches() - sat = SatelliteDataSource( - zarr_path=sat_filename, - history_minutes=30, - forecast_minutes=60, - image_size_pixels=64, - meters_per_pixel=2000, - channels=("HRV",), - ) - - gsp_filename = ( - Path(nowcasting_dataset.__file__).parent.parent / "tests" / "data" / "gsp" / "test.zarr" - ) - - gsp = GSPDataSource( - zarr_path=gsp_filename, - start_dt=datetime(2020, 4, 1), - end_dt=datetime(2020, 4, 2), - history_minutes=30, - forecast_minutes=60, - image_size_pixels=64, - meters_per_pixel=2000, - ) - - of = OpticalFlowDataSource( - history_minutes=30, - forecast_minutes=60, - image_size_pixels=32, - ) - - manager = Manager() - - # load config - local_path = Path(nowcasting_dataset.__file__).parent.parent - filename = local_path / "tests" / "config" / "test.yaml" - manager.load_yaml_configuration(filename=filename) - with tempfile.TemporaryDirectory() as local_temp_path, tempfile.TemporaryDirectory() as dst_path: # noqa 101 - - # set local temp path, and dst path - manager.config.output_data.filepath = Path(dst_path) - manager.local_temp_path = Path(local_temp_path) - # Set data sources - manager.data_sources = {"gsp": gsp, "satellite": sat} - manager.derived_data_sources = {"opticalflow": of} - manager.data_source_which_defines_geospatial_locations = gsp - - # make file for locations - manager.create_files_specifying_spatial_and_temporal_locations_of_each_example_if_necessary() # noqa 101 - - # make batches - manager.create_batches(overwrite_batches=True) - import glob - - print("glob(dst_path / train / *)", list(glob.glob(os.path.join(dst_path, "train", "*")))) - print( - "glob(dst_path / train / satellite / *)", - list(glob.glob(os.path.join(dst_path, "train", "satellite", "*"))), - ) - - # Load batch - from nowcasting_dataset.dataset.batch import Batch - - _ = Batch.load_netcdf( - os.path.join(dst_path, "train"), batch_idx=0, data_sources_names=["satellite"] - ) - - # make derived batches - manager.create_derived_batches(overwrite_batches=True) - - def test_save_config(): """Test that configuration file is saved""" From 8a4c213f774919d2de34210382f3dbadcb89e4e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Nov 2021 15:20:58 +0000 Subject: [PATCH 145/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 5565ddaa..a462b84f 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -123,7 +123,7 @@ def initialise_data_sources( except Exception: logger.exception(f"Exception whilst instantiating {data_source_name}!") raise - self.data_sources[data_source_name] = data_source + self.data_sources[data_source_name] = data_source # Set data_source_which_defines_geospatial_locations: try: @@ -330,7 +330,7 @@ def sample_spatial_and_temporal_locations_for_examples( ) def _get_first_batches_to_create( - self, overwrite_batches: bool + self, overwrite_batches: bool ) -> dict[split.SplitName, dict[str, int]]: """For each SplitName & for each DataSource name, return the first batch ID to create. From 86ec62070d23886375d4930a7b56abc1d3a61acd Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 16:17:44 +0000 Subject: [PATCH 146/197] Tests run again. But they do not pass! --- nowcasting_dataset/data_sources/__init__.py | 8 +++++--- .../data_sources/optical_flow/optical_flow_data_source.py | 2 +- nowcasting_dataset/manager.py | 1 - 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py index f116b794..3b069e17 100644 --- a/nowcasting_dataset/data_sources/__init__.py +++ b/nowcasting_dataset/data_sources/__init__.py @@ -2,14 +2,16 @@ from nowcasting_dataset.data_sources.data_source import DataSource # noqa: F401 from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource from nowcasting_dataset.data_sources.nwp.nwp_data_source import NWPDataSource -from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( - OpticalFlowDataSource, -) from nowcasting_dataset.data_sources.pv.pv_data_source import PVDataSource from nowcasting_dataset.data_sources.satellite.satellite_data_source import ( HRVSatelliteDataSource, SatelliteDataSource, ) +# We must import OpticalFlowDataSource *after* SatelliteDataSource, +# otherwise we get circular import errors! +from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( + OpticalFlowDataSource, +) from nowcasting_dataset.data_sources.sun.sun_data_source import SunDataSource from nowcasting_dataset.data_sources.topographic.topographic_data_source import ( TopographicDataSource, diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 184b9245..f3851776 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -8,7 +8,7 @@ import pandas as pd import xarray as xr -from nowcasting_dataset.data_sources.data_source.satellite import SatelliteDataSource +from nowcasting_dataset.data_sources import SatelliteDataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 5565ddaa..767eac58 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -17,7 +17,6 @@ SPATIAL_AND_TEMPORAL_LOCATIONS_OF_EACH_EXAMPLE_FILENAME, ) from nowcasting_dataset.data_sources import ALL_DATA_SOURCE_NAMES, MAP_DATA_SOURCE_NAME_TO_CLASS -from nowcasting_dataset.data_sources.data_source import DataSource from nowcasting_dataset.dataset.split import split from nowcasting_dataset.filesystem import utils as nd_fs_utils From 76c1cb9958c7e8e1e51143194de944295326b9df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Nov 2021 16:18:39 +0000 Subject: [PATCH 147/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/data_sources/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/__init__.py b/nowcasting_dataset/data_sources/__init__.py index 3b069e17..56c0a510 100644 --- a/nowcasting_dataset/data_sources/__init__.py +++ b/nowcasting_dataset/data_sources/__init__.py @@ -2,16 +2,17 @@ from nowcasting_dataset.data_sources.data_source import DataSource # noqa: F401 from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource from nowcasting_dataset.data_sources.nwp.nwp_data_source import NWPDataSource -from nowcasting_dataset.data_sources.pv.pv_data_source import PVDataSource -from nowcasting_dataset.data_sources.satellite.satellite_data_source import ( - HRVSatelliteDataSource, - SatelliteDataSource, -) + # We must import OpticalFlowDataSource *after* SatelliteDataSource, # otherwise we get circular import errors! from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( OpticalFlowDataSource, ) +from nowcasting_dataset.data_sources.pv.pv_data_source import PVDataSource +from nowcasting_dataset.data_sources.satellite.satellite_data_source import ( + HRVSatelliteDataSource, + SatelliteDataSource, +) from nowcasting_dataset.data_sources.sun.sun_data_source import SunDataSource from nowcasting_dataset.data_sources.topographic.topographic_data_source import ( TopographicDataSource, From 148fe1e6f9b22e184c9bae330790d42729b259d9 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 19:19:26 +0000 Subject: [PATCH 148/197] test_optical_flow_get_example passes! --- nowcasting_dataset/config/model.py | 30 ++++++- nowcasting_dataset/config/on_premises.yaml | 18 ++++- .../optical_flow/optical_flow_data_source.py | 78 +++++++++---------- .../test_optical_flow_data_source.py | 62 +++++++-------- 4 files changed, 108 insertions(+), 80 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index a8eb30d8..f9d2bfc8 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -28,7 +28,9 @@ ) from nowcasting_dataset.dataset.split import split -IMAGE_SIZE_PIXELS_FIELD = Field(64, description="The number of pixels of the region of interest.") +IMAGE_SIZE_PIXELS = 64 +IMAGE_SIZE_PIXELS_FIELD = Field( + IMAGE_SIZE_PIXELS, description="The number of pixels of the region of interest.") METERS_PER_PIXEL_FIELD = Field(2000, description="The number of meters per pixel.") @@ -153,8 +155,30 @@ class HRVSatellite(DataSourceMixin): class OpticalFlow(DataSourceMixin): """Optical Flow configuration model""" - number_previous_timesteps_to_use: int = 1 - opticalflow_image_size_pixels: int = IMAGE_SIZE_PIXELS_FIELD + opticalflow_zarr_path: str = Field( + "", + description=( + "The satellite Zarr data to use. If in doubt, use the same value as" + " satellite.satellite_zarr_path.") + ) + opticalflow_meters_per_pixels: int = METERS_PER_PIXEL_FIELD + opticalflow_number_previous_timesteps_to_use: int = Field( + 1, + description=( + "Number of previous timesteps to use, i.e. if 1, only uses the" + " flow between t-1 and t0 images, if 3, computes the flow between (t-3,t-2),(t-2,t-1)," + " and (t-1,t0) image pairs and uses the mean optical flow for future timesteps.") + ) + opticalflow_image_size_pixels: int = Field( + IMAGE_SIZE_PIXELS * 2, + description="The size of the *input* images (i.e. the size of the images to load off disk)") + opticalflow_output_image_size_pixels: int = Field( + IMAGE_SIZE_PIXELS, + description="The size of the images after optical flow has been applied." + ) + opticalflow_channels: tuple = Field( + SAT_VARIABLE_NAMES[1:], description="the satellite channels that are used" + ) class NWP(DataSourceMixin): diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index a23a6f0d..9a4580e6 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -64,8 +64,22 @@ input_data: # ------------------------- Optical Flow --------------- opticalflow: - number_previous_timesteps_to_use: 1 - opticalflow_image_size_pixels: 20 + opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* + opticalflow_number_previous_timesteps_to_use: 1 + opticalflow_image_size_pixels: 64 + opticalflow_output_image_size_pixels: 24 + opticalflow_channels: + - IR_016 + - IR_039 + - IR_087 + - IR_097 + - IR_108 + - IR_120 + - IR_134 + - VIS006 + - VIS008 + - WV_062 + - WV_073 output_data: filepath: /mnt/storage_ssd_4tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/prepared_ML_training_data/v15 diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index f3851776..7c4c57db 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -1,7 +1,7 @@ """ Optical Flow Data Source """ import logging -from dataclasses import dataclass -from typing import Optional +from dataclasses import InitVar, dataclass +from numbers import Number import cv2 import numpy as np @@ -23,37 +23,31 @@ class OpticalFlowDataSource(SatelliteDataSource): number_previous_timesteps_to_use: Number of previous timesteps to use, i.e. if 1, only uses the flow between t-1 and t0 images, if 3, computes the flow between (t-3,t-2),(t-2,t-1), and (t-1,t0) image pairs and uses the mean optical flow for future timesteps. + image_size_pixels: The *input* image size (i.e. the image size to load off disk). + output_image_size_pixels: The size of the output image. """ number_previous_timesteps_to_use: int = 1 - image_size_pixels: Optional[int] = None + output_image_size_pixels: int = 64 def get_example( - self, - batch, # Of type nowcasting_dataset.dataset.batch.Batch. But we can't use - # an "actual" type hint here otherwise we get a circular import error! - example_idx: int, - t0_dt: pd.Timestamp, - **kwargs + self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number ) -> DataSourceOutput: """ Get Optical Flow Example data Args: - batch: nowcasting_dataset.dataset.batch.Batch containing satellite and metadata at least - example_idx: The example to load and use - t0_dt: t0 datetime for the example + t0_dt: list of timestamps for the datetime of the batches. The batch will also include + data for historic and future depending on `history_minutes` and `future_minutes`. + x_meters_center: x center batch locations + y_meters_center: y center batch locations Returns: Example Data """ - - if self.image_size_pixels is None: - self.image_size_pixels = len(batch.satellite.x_index) - - # Only do optical flow for satellite data - # TODO: Enable this to work with hrvsatellite too. - satellite_data: xr.DataArray = batch.satellite.sel(example=example_idx) + satellite_data: xr.Dataset = super().get_example( + t0_dt=t0_dt, x_meters_center=x_meters_center, y_meters_center=y_meters_center) + satellite_data = satellite_data["data"] return self._compute_and_return_optical_flow(satellite_data, t0_datetime_utc=t0_dt) @staticmethod @@ -80,24 +74,24 @@ def _put_predictions_into_data_array( """ # Select the timesteps for the optical flow predictions. satellite_data = satellite_data.isel( - time_index=slice( - satellite_data.sizes["time_index"] - predictions.shape[0], - satellite_data.sizes["time_index"], + time=slice( + satellite_data.sizes["time"] - predictions.shape[0], + satellite_data.sizes["time"], ) ) # Select the center crop. - border = (satellite_data.sizes["x_index"] - self.image_size_pixels) // 2 + border = (satellite_data.sizes["x"] - self.output_image_size_pixels) // 2 satellite_data = satellite_data.isel( - x_index=slice(border, satellite_data.sizes["x_index"] - border), - y_index=slice(border, satellite_data.sizes["y_index"] - border), + x=slice(border, satellite_data.sizes["x"] - border), + y=slice(border, satellite_data.sizes["y"] - border), ) return xr.DataArray( data=predictions, coords=( - ("time_index", satellite_data.coords["time_index"].values), - ("x_index", satellite_data.coords["x_index"].values), - ("y_index", satellite_data.coords["y_index"].values), - ("channels_index", satellite_data.coords["channels_index"].values), + ("time", satellite_data.coords["time"].values), + ("x", satellite_data.coords["x"].values), + ("y", satellite_data.coords["y"].values), + ("channels", satellite_data.coords["channels"].values), ), name="data", ) @@ -134,7 +128,7 @@ def _get_number_future_timesteps( The number of future timesteps """ satellite_data = satellite_data.where(satellite_data.time > t0_datetime_utc, drop=True) - return len(satellite_data.coords["time_index"]) + return len(satellite_data.coords["time"]) def _compute_and_return_optical_flow( self, @@ -159,18 +153,18 @@ def _compute_and_return_optical_flow( t0_datetime_utc=t0_datetime_utc, ) assert ( - len(historical_satellite_data.coords["time_index"]) + len(historical_satellite_data.coords["time"]) - self.number_previous_timesteps_to_use - 1 ) >= 0, "Trying to compute flow further back than the number of historical timesteps" # TODO: Use the correct dtype. - n_channels = satellite_data.sizes["channels_index"] + n_channels = satellite_data.sizes["channels"] prediction_block = np.full( shape=( future_timesteps, - self.image_size_pixels, - self.image_size_pixels, + self.output_image_size_pixels, + self.output_image_size_pixels, n_channels, ), fill_value=np.NaN, @@ -178,30 +172,30 @@ def _compute_and_return_optical_flow( for channel in range(n_channels): # Compute optical flow field: - historical_sat_data_for_chan = historical_satellite_data.isel(channels_index=channel) + historical_sat_data_for_chan = historical_satellite_data.isel(channels=channel) # Loop through pairs of historical images to compute optical flow fields: optical_flows = [] - n_historical_timesteps = len(historical_satellite_data.coords["time_index"]) + n_historical_timesteps = len(historical_satellite_data.coords["time"]) end_time_i = n_historical_timesteps start_time_i = end_time_i - self.number_previous_timesteps_to_use for time_i in range(start_time_i, end_time_i): - prev_image = historical_sat_data_for_chan.isel(time_index=time_i - 1).data.values - next_image = historical_sat_data_for_chan.isel(time_index=time_i).data.values + prev_image = historical_sat_data_for_chan.isel(time=time_i - 1).data + next_image = historical_sat_data_for_chan.isel(time=time_i).data optical_flow = compute_optical_flow(prev_image, next_image) optical_flows.append(optical_flow) # Average predictions optical_flow = np.mean(optical_flows, axis=0) # Compute predicted images. - t0_image = historical_sat_data_for_chan.isel(time_index=-1).data.values + t0_image = historical_sat_data_for_chan.isel(time=-1).data for prediction_timestep in range(future_timesteps): flow = optical_flow * (prediction_timestep + 1) warped_image = remap_image(image=t0_image, flow=flow) warped_image = crop_center( warped_image, - self.image_size_pixels, - self.image_size_pixels, + self.output_image_size_pixels, + self.output_image_size_pixels, ) prediction_block[prediction_timestep, :, :, channel] = warped_image @@ -235,7 +229,7 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n next_image = next_image * 255 next_image = next_image.astype(np.uint8) - # Docs: https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af # nopa + # Docs: https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af # noqa flow = cv2.calcOpticalFlowFarneback( prev=prev_image, next=next_image, diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 8abd1d2a..e3d860c8 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -1,5 +1,6 @@ """Test Optical Flow Data Source""" import tempfile +from pathlib import Path import pandas as pd import pytest @@ -21,23 +22,34 @@ def optical_flow_configuration(): # noqa: D103 return con -def test_optical_flow_get_example(optical_flow_configuration): # noqa: D103 - optical_flow_datasource = OpticalFlowDataSource( - number_previous_timesteps_to_use=1, image_size_pixels=32 +def _get_optical_flow_data_source( + sat_filename: Path, + number_previous_timesteps_to_use: int = 1, +) -> OpticalFlowDataSource: + return OpticalFlowDataSource( + zarr_path=sat_filename, + number_previous_timesteps_to_use=number_previous_timesteps_to_use, + image_size_pixels=64, + output_image_size_pixels=32, + history_minutes=30, + forecast_minutes=120, + channels=("IR_016",), ) - batch = Batch.fake(configuration=optical_flow_configuration) + + +def test_optical_flow_get_example(optical_flow_configuration, sat_filename: Path): # noqa: D103 + optical_flow_datasource = _get_optical_flow_data_source(sat_filename=sat_filename) + optical_flow_datasource.open() + t0_dt = pd.Timestamp("2020-04-01T13:00") example = optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] - ) - # As a nasty hack to get round #511, the number of timesteps is set to 0 for now. - # TODO: Issue #513: Set the number of timesteps back to 12! - assert example.values.shape == (0, 32, 32, 10) # timesteps, height, width, channels + t0_dt=t0_dt, x_meters_center=10_000, y_meters_center=10_000) + assert example.values.shape == (24, 32, 32, 1) # timesteps, height, width, channels -def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): # noqa: D103 - optical_flow_datasource = OpticalFlowDataSource( - number_previous_timesteps_to_use=3, image_size_pixels=32 - ) +def test_optical_flow_get_example_multi_timesteps( + optical_flow_configuration, sat_filename: Path): # noqa: D103 + optical_flow_datasource = _get_optical_flow_data_source( + number_previous_timesteps_to_use=3, sat_filename=sat_filename) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example( batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] @@ -47,28 +59,12 @@ def test_optical_flow_get_example_multi_timesteps(optical_flow_configuration): assert example.values.shape == (0, 32, 32, 10) # timesteps, height, width, channels -def test_optical_flow_get_example_too_many_timesteps(optical_flow_configuration): # noqa: D103 - optical_flow_datasource = OpticalFlowDataSource( - number_previous_timesteps_to_use=300, image_size_pixels=32 - ) +def test_optical_flow_get_example_too_many_timesteps( + optical_flow_configuration, sat_filename: Path): # noqa: D103 + optical_flow_datasource = _get_optical_flow_data_source( + number_previous_timesteps_to_use=300, sat_filename=sat_filename) batch = Batch.fake(configuration=optical_flow_configuration) with pytest.raises(AssertionError): optical_flow_datasource.get_example( batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] ) - - -def test_optical_flow_data_source_get_batch(optical_flow_configuration): # noqa: D103 - optical_flow_datasource = OpticalFlowDataSource( - number_previous_timesteps_to_use=1, image_size_pixels=32 - ) - with tempfile.TemporaryDirectory() as dirpath: - batch = Batch.fake(configuration=optical_flow_configuration) - batch.save_netcdf(path=dirpath, batch_i=0) - t0_datetime_utc = pd.DatetimeIndex(batch.metadata.t0_datetime_utc) - optical_flow = optical_flow_datasource.get_batch( - netcdf_path=dirpath, batch_idx=0, t0_datetimes=t0_datetime_utc - ) - # As a nasty hack to get round #511, the number of timesteps is set to 0 for now. - # TODO: Issue #513: Set the number of timesteps back to 12! - assert optical_flow.values.shape == (4, 0, 32, 32, 10) # ?, timesteps, height, width, chans From 9545dbb2a9f258ddc9cd9fae05c0eb656abdac3b Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Tue, 30 Nov 2021 19:20:06 +0000 Subject: [PATCH 149/197] tiny update --- .../data_sources/optical_flow/optical_flow_data_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 7c4c57db..13df5719 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -20,6 +20,7 @@ class OpticalFlowDataSource(SatelliteDataSource): """ Optical Flow Data Source, computing flow between Satellite data + TODO: This is redundant: Use history_minutes instead. number_previous_timesteps_to_use: Number of previous timesteps to use, i.e. if 1, only uses the flow between t-1 and t0 images, if 3, computes the flow between (t-3,t-2),(t-2,t-1), and (t-1,t0) image pairs and uses the mean optical flow for future timesteps. @@ -46,7 +47,8 @@ def get_example( """ satellite_data: xr.Dataset = super().get_example( - t0_dt=t0_dt, x_meters_center=x_meters_center, y_meters_center=y_meters_center) + t0_dt=t0_dt, x_meters_center=x_meters_center, y_meters_center=y_meters_center + ) satellite_data = satellite_data["data"] return self._compute_and_return_optical_flow(satellite_data, t0_datetime_utc=t0_dt) From 1339424d7ac729a044de5caec2fbfaed7ef3261b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Nov 2021 19:20:01 +0000 Subject: [PATCH 150/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/config/model.py | 15 +++++++++------ .../optical_flow/optical_flow_data_source.py | 3 ++- .../test_optical_flow_data_source.py | 19 ++++++++++++------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index f9d2bfc8..96883427 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -30,7 +30,8 @@ IMAGE_SIZE_PIXELS = 64 IMAGE_SIZE_PIXELS_FIELD = Field( - IMAGE_SIZE_PIXELS, description="The number of pixels of the region of interest.") + IMAGE_SIZE_PIXELS, description="The number of pixels of the region of interest." +) METERS_PER_PIXEL_FIELD = Field(2000, description="The number of meters per pixel.") @@ -159,7 +160,8 @@ class OpticalFlow(DataSourceMixin): "", description=( "The satellite Zarr data to use. If in doubt, use the same value as" - " satellite.satellite_zarr_path.") + " satellite.satellite_zarr_path." + ), ) opticalflow_meters_per_pixels: int = METERS_PER_PIXEL_FIELD opticalflow_number_previous_timesteps_to_use: int = Field( @@ -167,14 +169,15 @@ class OpticalFlow(DataSourceMixin): description=( "Number of previous timesteps to use, i.e. if 1, only uses the" " flow between t-1 and t0 images, if 3, computes the flow between (t-3,t-2),(t-2,t-1)," - " and (t-1,t0) image pairs and uses the mean optical flow for future timesteps.") + " and (t-1,t0) image pairs and uses the mean optical flow for future timesteps." + ), ) opticalflow_image_size_pixels: int = Field( IMAGE_SIZE_PIXELS * 2, - description="The size of the *input* images (i.e. the size of the images to load off disk)") + description="The size of the *input* images (i.e. the size of the images to load off disk)", + ) opticalflow_output_image_size_pixels: int = Field( - IMAGE_SIZE_PIXELS, - description="The size of the images after optical flow has been applied." + IMAGE_SIZE_PIXELS, description="The size of the images after optical flow has been applied." ) opticalflow_channels: tuple = Field( SAT_VARIABLE_NAMES[1:], description="the satellite channels that are used" diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 7c4c57db..9a180e6a 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -46,7 +46,8 @@ def get_example( """ satellite_data: xr.Dataset = super().get_example( - t0_dt=t0_dt, x_meters_center=x_meters_center, y_meters_center=y_meters_center) + t0_dt=t0_dt, x_meters_center=x_meters_center, y_meters_center=y_meters_center + ) satellite_data = satellite_data["data"] return self._compute_and_return_optical_flow(satellite_data, t0_datetime_utc=t0_dt) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index e3d860c8..1974c752 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -23,8 +23,8 @@ def optical_flow_configuration(): # noqa: D103 def _get_optical_flow_data_source( - sat_filename: Path, - number_previous_timesteps_to_use: int = 1, + sat_filename: Path, + number_previous_timesteps_to_use: int = 1, ) -> OpticalFlowDataSource: return OpticalFlowDataSource( zarr_path=sat_filename, @@ -42,14 +42,17 @@ def test_optical_flow_get_example(optical_flow_configuration, sat_filename: Path optical_flow_datasource.open() t0_dt = pd.Timestamp("2020-04-01T13:00") example = optical_flow_datasource.get_example( - t0_dt=t0_dt, x_meters_center=10_000, y_meters_center=10_000) + t0_dt=t0_dt, x_meters_center=10_000, y_meters_center=10_000 + ) assert example.values.shape == (24, 32, 32, 1) # timesteps, height, width, channels def test_optical_flow_get_example_multi_timesteps( - optical_flow_configuration, sat_filename: Path): # noqa: D103 + optical_flow_configuration, sat_filename: Path +): # noqa: D103 optical_flow_datasource = _get_optical_flow_data_source( - number_previous_timesteps_to_use=3, sat_filename=sat_filename) + number_previous_timesteps_to_use=3, sat_filename=sat_filename + ) batch = Batch.fake(configuration=optical_flow_configuration) example = optical_flow_datasource.get_example( batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] @@ -60,9 +63,11 @@ def test_optical_flow_get_example_multi_timesteps( def test_optical_flow_get_example_too_many_timesteps( - optical_flow_configuration, sat_filename: Path): # noqa: D103 + optical_flow_configuration, sat_filename: Path +): # noqa: D103 optical_flow_datasource = _get_optical_flow_data_source( - number_previous_timesteps_to_use=300, sat_filename=sat_filename) + number_previous_timesteps_to_use=300, sat_filename=sat_filename + ) batch = Batch.fake(configuration=optical_flow_configuration) with pytest.raises(AssertionError): optical_flow_datasource.get_example( From bdaa060333962c2d6f18229714bebea8f4bb62c4 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Wed, 1 Dec 2021 17:50:54 +0000 Subject: [PATCH 151/197] Slight redesign: OpticalFlowDataSource now inherits from DataSource (not SatelliteDataSource) and the SatelliteDataSource is a member attribute. Also replace number_of_previous_timesteps_to_use with history_minutes. And do not load future satellite data off disk. All test_optical_flow_data_source tests pass --- nowcasting_dataset/config/model.py | 36 ++-- nowcasting_dataset/config/on_premises.yaml | 4 +- .../data_sources/data_source.py | 4 +- .../optical_flow/optical_flow_data_source.py | 159 +++++++++--------- .../test_optical_flow_data_source.py | 50 ++---- 5 files changed, 119 insertions(+), 134 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 96883427..5dd66efa 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -163,25 +163,41 @@ class OpticalFlow(DataSourceMixin): " satellite.satellite_zarr_path." ), ) - opticalflow_meters_per_pixels: int = METERS_PER_PIXEL_FIELD - opticalflow_number_previous_timesteps_to_use: int = Field( - 1, + opticalflow_history_minutes: int = Field( + 5, description=( - "Number of previous timesteps to use, i.e. if 1, only uses the" - " flow between t-1 and t0 images, if 3, computes the flow between (t-3,t-2),(t-2,t-1)," - " and (t-1,t0) image pairs and uses the mean optical flow for future timesteps." - ), + "Duration of historical data to use when computing the optical flow field." + " For example, set to 5 to use just two images: the t-1 and t0 images. Set to 10 to" + " compute the optical flow field separately for the image pairs (t-2, t-1), and" + " (t-1, t0) and to use the mean optical flow field." + ) ) - opticalflow_image_size_pixels: int = Field( + opticalflow_forecast_minutes: int = Field( + 120, description="Duration of the optical flow predictions.") + opticalflow_meters_per_pixels: int = METERS_PER_PIXEL_FIELD + opticalflow_input_image_size_pixels: int = Field( IMAGE_SIZE_PIXELS * 2, - description="The size of the *input* images (i.e. the size of the images to load off disk)", + description=( + "The *input* image size (i.e. the image size to load off disk)." + " This should be larger than output_image_size_pixels to provide sufficient border to" + " mean that, even after the image has been flowed, all edges of the output image are" + " real pixels values, and not NaNs."), ) opticalflow_output_image_size_pixels: int = Field( - IMAGE_SIZE_PIXELS, description="The size of the images after optical flow has been applied." + IMAGE_SIZE_PIXELS, + description=( + "The size of the images after optical flow has been applied. The output image is a" + " center-crop of the input image, after it has been flowed.") ) opticalflow_channels: tuple = Field( SAT_VARIABLE_NAMES[1:], description="the satellite channels that are used" ) + opticalflow_source_data_source_class_name: str = Field( + "SatelliteDataSource", + description=( + "Either SatelliteDataSource or HRVSatelliteDataSource." + " The name of the DataSource that will load the satellite images."), + ) class NWP(DataSourceMixin): diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index 9a4580e6..93c70f07 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -65,9 +65,11 @@ input_data: # ------------------------- Optical Flow --------------- opticalflow: opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* - opticalflow_number_previous_timesteps_to_use: 1 + opticalflow_history_minutes: 5 + opticalflow_forecast_minutes: 120 opticalflow_image_size_pixels: 64 opticalflow_output_image_size_pixels: 24 + opticalflow_source_data_source_class_name: SatelliteDataSource opticalflow_channels: - IR_016 - IR_039 diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 743e03b6..e52def4d 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -20,7 +20,7 @@ convert_coordinates_to_indexes_for_list_datasets, join_list_dataset_to_batch_dataset, ) -from nowcasting_dataset.utils import DummyExecutor, get_start_and_end_example_index +from nowcasting_dataset.utils import get_start_and_end_example_index logger = logging.getLogger(__name__) @@ -343,7 +343,7 @@ def get_locations(self, t0_datetimes: pd.DatetimeIndex) -> Tuple[List[Number], L # ****************** METHODS THAT MUST BE OVERRIDDEN ********************** # TODO: Issue #319: Standardise parameter names. def _get_time_slice(self, t0_dt: pd.Timestamp): - """Get a single timestep of data. Must be overridden.""" + """Get a single timestep of data. Must be overridden if get_example is not overridden.""" raise NotImplementedError() # TODO: Issue #319: Standardise parameter names. diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 13df5719..d7690682 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -1,14 +1,16 @@ """ Optical Flow Data Source """ import logging -from dataclasses import InitVar, dataclass +from dataclasses import dataclass from numbers import Number +from pathlib import Path +from typing import Iterable, Union import cv2 import numpy as np import pandas as pd import xarray as xr -from nowcasting_dataset.data_sources import SatelliteDataSource +from nowcasting_dataset.data_sources import DataSource from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow @@ -16,20 +18,56 @@ @dataclass -class OpticalFlowDataSource(SatelliteDataSource): +class OpticalFlowDataSource(DataSource): """ Optical Flow Data Source, computing flow between Satellite data - TODO: This is redundant: Use history_minutes instead. - number_previous_timesteps_to_use: Number of previous timesteps to use, i.e. if 1, only uses the - flow between t-1 and t0 images, if 3, computes the flow between (t-3,t-2),(t-2,t-1), - and (t-1,t0) image pairs and uses the mean optical flow for future timesteps. - image_size_pixels: The *input* image size (i.e. the image size to load off disk). - output_image_size_pixels: The size of the output image. + history_minutes: Duration of historical data to use when computing the optical flow field. + For example, set to 5 to use just two images: the t-1 and t0 images. Set to 10 to compute + the optical flow field separately for the image pairs (t-2, t-1), and (t-1, t0) and to + use the mean optical flow field. + forecast_minutes: Duration of the optical flow predictions. + zarr_path: The location of the intermediate satellite data to compute optical flows with. + input_image_size_pixels: The *input* image size (i.e. the image size to load off disk). + This should be larger than output_image_size_pixels to provide sufficient border to mean + that, even after the image has been "flowed", all edges of the output image are + "real" pixels values, and not NaNs. + output_image_size_pixels: The size of the output image. The output image is a center-crop of + the input image, after it has been "flowed". + source_data_source_class: Either HRVSatelliteDataSource or SatelliteDataSource. + channels: The satellite channels to compute optical flow for. """ - number_previous_timesteps_to_use: int = 1 - output_image_size_pixels: int = 64 + zarr_path: Union[Path, str] + channels: Iterable[str] + input_image_size_pixels: int = 64 + meters_per_pixel: int = 2000 + output_image_size_pixels: int = 32 + source_data_source_class_name: str = "SatelliteDataSource" + + def __post_init__(self): + super().__post_init__() + + # Get round circular import problem + from nowcasting_dataset.data_sources import SatelliteDataSource, HRVSatelliteDataSource + _MAP_SATELLITE_DATA_SOURCE_NAME_TO_CLASS = { + "HRVSatelliteDataSource": HRVSatelliteDataSource, + "SatelliteDataSource": SatelliteDataSource, + } + + source_data_source_class = _MAP_SATELLITE_DATA_SOURCE_NAME_TO_CLASS[ + self.source_data_source_class_name] + self.source_data_source = source_data_source_class( + zarr_path=self.zarr_path, + image_size_pixels=self.input_image_size_pixels, + history_minutes=self.history_minutes, + forecast_minutes=0, + channels=self.channels, + meters_per_pixel=self.meters_per_pixel + ) + + def open(self): + self.source_data_source.open() def get_example( self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number @@ -46,11 +84,11 @@ def get_example( Returns: Example Data """ - satellite_data: xr.Dataset = super().get_example( + satellite_data: xr.Dataset = self.source_data_source.get_example( t0_dt=t0_dt, x_meters_center=x_meters_center, y_meters_center=y_meters_center ) satellite_data = satellite_data["data"] - return self._compute_and_return_optical_flow(satellite_data, t0_datetime_utc=t0_dt) + return self._compute_and_return_optical_flow(satellite_data) @staticmethod def get_data_model_for_batch(): @@ -75,13 +113,15 @@ def _put_predictions_into_data_array( The Xarray DataArray with the optical flow predictions """ # Select the timesteps for the optical flow predictions. - satellite_data = satellite_data.isel( - time=slice( - satellite_data.sizes["time"] - predictions.shape[0], - satellite_data.sizes["time"], - ) + t0_datetime_utc = satellite_data.isel(time=-1)["time"].values + datetime_index_of_predictions = pd.date_range( + t0_datetime_utc, + periods=self.forecast_length, + freq=self.sample_period_duration ) + # Select the center crop. + # TODO: Generalise crop_center and use again here: border = (satellite_data.sizes["x"] - self.output_image_size_pixels) // 2 satellite_data = satellite_data.isel( x=slice(border, satellite_data.sizes["x"] - border), @@ -90,7 +130,7 @@ def _put_predictions_into_data_array( return xr.DataArray( data=predictions, coords=( - ("time", satellite_data.coords["time"].values), + ("time", datetime_index_of_predictions), ("x", satellite_data.coords["x"].values), ("y", satellite_data.coords["y"].values), ("channels", satellite_data.coords["channels"].values), @@ -98,73 +138,28 @@ def _put_predictions_into_data_array( name="data", ) - def _get_previous_timesteps( - self, - satellite_data: xr.DataArray, - t0_datetime_utc: pd.Timestamp, - ) -> xr.DataArray: - """ - Get timestamp of previous - - Args: - satellite_data: Satellite data to use - t0_datetime_utc: Timestamp - - Returns: - The previous timesteps - """ - satellite_data = satellite_data.where(satellite_data.time <= t0_datetime_utc, drop=True) - return satellite_data - - def _get_number_future_timesteps( - self, satellite_data: xr.DataArray, t0_datetime_utc: pd.Timestamp - ) -> int: - """ - Get number of future timestamps - - Args: - satellite_data: Satellite data to use - t0_datetime_utc: The timestamp of the t0 image - - Returns: - The number of future timesteps - """ - satellite_data = satellite_data.where(satellite_data.time > t0_datetime_utc, drop=True) - return len(satellite_data.coords["time"]) - - def _compute_and_return_optical_flow( - self, - satellite_data: xr.DataArray, - t0_datetime_utc: pd.Timestamp, - ) -> xr.DataArray: + def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.DataArray: """ Compute and return optical flow predictions for the example Args: - satellite_data: Satellite DataArray - t0_datetime_utc: t0 timestamp + satellite_data: Satellite DataArray of historical satellite images. Returns: The Tensor with the optical flow predictions for t0 to forecast horizon """ + n_channels = satellite_data.sizes["channels"] - # Get the previous timestamp - future_timesteps = self._get_number_future_timesteps(satellite_data, t0_datetime_utc) - historical_satellite_data: xr.DataArray = self._get_previous_timesteps( - satellite_data, - t0_datetime_utc=t0_datetime_utc, + # Sanity check + assert len(satellite_data.coords["time"]) == self.history_length+1, ( + f"{len(satellite_data.coords['time'])=} != {self.history_length+1=}" ) - assert ( - len(historical_satellite_data.coords["time"]) - - self.number_previous_timesteps_to_use - - 1 - ) >= 0, "Trying to compute flow further back than the number of historical timesteps" + assert n_channels == len(self.channels), f"{n_channels=} != {len(self.channels)=}" # TODO: Use the correct dtype. - n_channels = satellite_data.sizes["channels"] prediction_block = np.full( shape=( - future_timesteps, + self.forecast_length, self.output_image_size_pixels, self.output_image_size_pixels, n_channels, @@ -172,26 +167,24 @@ def _compute_and_return_optical_flow( fill_value=np.NaN, ) - for channel in range(n_channels): + for channel_i in range(n_channels): # Compute optical flow field: - historical_sat_data_for_chan = historical_satellite_data.isel(channels=channel) + sat_data_for_chan = satellite_data.isel(channels=channel_i) # Loop through pairs of historical images to compute optical flow fields: optical_flows = [] - n_historical_timesteps = len(historical_satellite_data.coords["time"]) - end_time_i = n_historical_timesteps - start_time_i = end_time_i - self.number_previous_timesteps_to_use - for time_i in range(start_time_i, end_time_i): - prev_image = historical_sat_data_for_chan.isel(time=time_i - 1).data - next_image = historical_sat_data_for_chan.isel(time=time_i).data + # self.history_length does not include t0. + for history_timestep in range(self.history_length): + prev_image = sat_data_for_chan.isel(time=history_timestep).data + next_image = sat_data_for_chan.isel(time=history_timestep+1).data optical_flow = compute_optical_flow(prev_image, next_image) optical_flows.append(optical_flow) # Average predictions optical_flow = np.mean(optical_flows, axis=0) # Compute predicted images. - t0_image = historical_sat_data_for_chan.isel(time=-1).data - for prediction_timestep in range(future_timesteps): + t0_image = sat_data_for_chan.isel(time=-1).data + for prediction_timestep in range(self.forecast_length): flow = optical_flow * (prediction_timestep + 1) warped_image = remap_image(image=t0_image, flow=flow) warped_image = crop_center( @@ -199,7 +192,7 @@ def _compute_and_return_optical_flow( self.output_image_size_pixels, self.output_image_size_pixels, ) - prediction_block[prediction_timestep, :, :, channel] = warped_image + prediction_block[prediction_timestep, :, :, channel_i] = warped_image data_array = self._put_predictions_into_data_array( satellite_data=satellite_data, predictions=prediction_block diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index 1974c752..cf9735ae 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -1,5 +1,4 @@ """Test Optical Flow Data Source""" -import tempfile from pathlib import Path import pandas as pd @@ -9,7 +8,6 @@ from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import ( OpticalFlowDataSource, ) -from nowcasting_dataset.dataset.batch import Batch @pytest.fixture @@ -24,52 +22,28 @@ def optical_flow_configuration(): # noqa: D103 def _get_optical_flow_data_source( sat_filename: Path, - number_previous_timesteps_to_use: int = 1, + history_minutes: int = 5 ) -> OpticalFlowDataSource: return OpticalFlowDataSource( zarr_path=sat_filename, - number_previous_timesteps_to_use=number_previous_timesteps_to_use, - image_size_pixels=64, - output_image_size_pixels=32, - history_minutes=30, - forecast_minutes=120, channels=("IR_016",), + history_minutes=history_minutes, + forecast_minutes=120, + input_image_size_pixels=64, + output_image_size_pixels=32, ) -def test_optical_flow_get_example(optical_flow_configuration, sat_filename: Path): # noqa: D103 - optical_flow_datasource = _get_optical_flow_data_source(sat_filename=sat_filename) +@pytest.mark.parametrize("history_minutes", [5, 15]) +def test_optical_flow_get_example( + optical_flow_configuration, + sat_filename: Path, + history_minutes: int): # noqa: D103 + optical_flow_datasource = _get_optical_flow_data_source( + sat_filename=sat_filename, history_minutes=history_minutes) optical_flow_datasource.open() t0_dt = pd.Timestamp("2020-04-01T13:00") example = optical_flow_datasource.get_example( t0_dt=t0_dt, x_meters_center=10_000, y_meters_center=10_000 ) assert example.values.shape == (24, 32, 32, 1) # timesteps, height, width, channels - - -def test_optical_flow_get_example_multi_timesteps( - optical_flow_configuration, sat_filename: Path -): # noqa: D103 - optical_flow_datasource = _get_optical_flow_data_source( - number_previous_timesteps_to_use=3, sat_filename=sat_filename - ) - batch = Batch.fake(configuration=optical_flow_configuration) - example = optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] - ) - # As a nasty hack to get round #511, the number of timesteps is set to 0 for now. - # TODO: Issue #513: Set the number of timesteps back to 12! - assert example.values.shape == (0, 32, 32, 10) # timesteps, height, width, channels - - -def test_optical_flow_get_example_too_many_timesteps( - optical_flow_configuration, sat_filename: Path -): # noqa: D103 - optical_flow_datasource = _get_optical_flow_data_source( - number_previous_timesteps_to_use=300, sat_filename=sat_filename - ) - batch = Batch.fake(configuration=optical_flow_configuration) - with pytest.raises(AssertionError): - optical_flow_datasource.get_example( - batch=batch, example_idx=0, t0_dt=batch.metadata.t0_datetime_utc[0] - ) From c1c5423a4f0ef57c23db2770b4fcea03f1473b85 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 1 Dec 2021 17:51:16 +0000 Subject: [PATCH 152/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/config/model.py | 14 ++++++++----- .../optical_flow/optical_flow_data_source.py | 20 +++++++++---------- .../test_optical_flow_data_source.py | 11 +++++----- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 5dd66efa..ffc3f314 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -170,10 +170,11 @@ class OpticalFlow(DataSourceMixin): " For example, set to 5 to use just two images: the t-1 and t0 images. Set to 10 to" " compute the optical flow field separately for the image pairs (t-2, t-1), and" " (t-1, t0) and to use the mean optical flow field." - ) + ), ) opticalflow_forecast_minutes: int = Field( - 120, description="Duration of the optical flow predictions.") + 120, description="Duration of the optical flow predictions." + ) opticalflow_meters_per_pixels: int = METERS_PER_PIXEL_FIELD opticalflow_input_image_size_pixels: int = Field( IMAGE_SIZE_PIXELS * 2, @@ -181,13 +182,15 @@ class OpticalFlow(DataSourceMixin): "The *input* image size (i.e. the image size to load off disk)." " This should be larger than output_image_size_pixels to provide sufficient border to" " mean that, even after the image has been flowed, all edges of the output image are" - " real pixels values, and not NaNs."), + " real pixels values, and not NaNs." + ), ) opticalflow_output_image_size_pixels: int = Field( IMAGE_SIZE_PIXELS, description=( "The size of the images after optical flow has been applied. The output image is a" - " center-crop of the input image, after it has been flowed.") + " center-crop of the input image, after it has been flowed." + ), ) opticalflow_channels: tuple = Field( SAT_VARIABLE_NAMES[1:], description="the satellite channels that are used" @@ -196,7 +199,8 @@ class OpticalFlow(DataSourceMixin): "SatelliteDataSource", description=( "Either SatelliteDataSource or HRVSatelliteDataSource." - " The name of the DataSource that will load the satellite images."), + " The name of the DataSource that will load the satellite images." + ), ) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index d7690682..7ff4bd31 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -49,21 +49,23 @@ def __post_init__(self): super().__post_init__() # Get round circular import problem - from nowcasting_dataset.data_sources import SatelliteDataSource, HRVSatelliteDataSource + from nowcasting_dataset.data_sources import HRVSatelliteDataSource, SatelliteDataSource + _MAP_SATELLITE_DATA_SOURCE_NAME_TO_CLASS = { "HRVSatelliteDataSource": HRVSatelliteDataSource, "SatelliteDataSource": SatelliteDataSource, } source_data_source_class = _MAP_SATELLITE_DATA_SOURCE_NAME_TO_CLASS[ - self.source_data_source_class_name] + self.source_data_source_class_name + ] self.source_data_source = source_data_source_class( zarr_path=self.zarr_path, image_size_pixels=self.input_image_size_pixels, history_minutes=self.history_minutes, forecast_minutes=0, channels=self.channels, - meters_per_pixel=self.meters_per_pixel + meters_per_pixel=self.meters_per_pixel, ) def open(self): @@ -115,9 +117,7 @@ def _put_predictions_into_data_array( # Select the timesteps for the optical flow predictions. t0_datetime_utc = satellite_data.isel(time=-1)["time"].values datetime_index_of_predictions = pd.date_range( - t0_datetime_utc, - periods=self.forecast_length, - freq=self.sample_period_duration + t0_datetime_utc, periods=self.forecast_length, freq=self.sample_period_duration ) # Select the center crop. @@ -151,9 +151,9 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D n_channels = satellite_data.sizes["channels"] # Sanity check - assert len(satellite_data.coords["time"]) == self.history_length+1, ( - f"{len(satellite_data.coords['time'])=} != {self.history_length+1=}" - ) + assert ( + len(satellite_data.coords["time"]) == self.history_length + 1 + ), f"{len(satellite_data.coords['time'])=} != {self.history_length+1=}" assert n_channels == len(self.channels), f"{n_channels=} != {len(self.channels)=}" # TODO: Use the correct dtype. @@ -176,7 +176,7 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D # self.history_length does not include t0. for history_timestep in range(self.history_length): prev_image = sat_data_for_chan.isel(time=history_timestep).data - next_image = sat_data_for_chan.isel(time=history_timestep+1).data + next_image = sat_data_for_chan.isel(time=history_timestep + 1).data optical_flow = compute_optical_flow(prev_image, next_image) optical_flows.append(optical_flow) # Average predictions diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index cf9735ae..b0a0e2df 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -21,8 +21,7 @@ def optical_flow_configuration(): # noqa: D103 def _get_optical_flow_data_source( - sat_filename: Path, - history_minutes: int = 5 + sat_filename: Path, history_minutes: int = 5 ) -> OpticalFlowDataSource: return OpticalFlowDataSource( zarr_path=sat_filename, @@ -36,11 +35,11 @@ def _get_optical_flow_data_source( @pytest.mark.parametrize("history_minutes", [5, 15]) def test_optical_flow_get_example( - optical_flow_configuration, - sat_filename: Path, - history_minutes: int): # noqa: D103 + optical_flow_configuration, sat_filename: Path, history_minutes: int +): # noqa: D103 optical_flow_datasource = _get_optical_flow_data_source( - sat_filename=sat_filename, history_minutes=history_minutes) + sat_filename=sat_filename, history_minutes=history_minutes + ) optical_flow_datasource.open() t0_dt = pd.Timestamp("2020-04-01T13:00") example = optical_flow_datasource.get_example( From 83c511856649ed4eaaf8f48d38e3f81332697a29 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Wed, 1 Dec 2021 18:01:32 +0000 Subject: [PATCH 153/197] manager tests pass --- nowcasting_dataset/config/model.py | 2 +- nowcasting_dataset/data_sources/data_source.py | 6 +++++- tests/config/test.yaml | 3 +-- tests/test_manager.py | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 5dd66efa..ed5ea845 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -174,7 +174,7 @@ class OpticalFlow(DataSourceMixin): ) opticalflow_forecast_minutes: int = Field( 120, description="Duration of the optical flow predictions.") - opticalflow_meters_per_pixels: int = METERS_PER_PIXEL_FIELD + opticalflow_meters_per_pixel: int = METERS_PER_PIXEL_FIELD opticalflow_input_image_size_pixels: int = Field( IMAGE_SIZE_PIXELS * 2, description=( diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index e52def4d..5f8fb10f 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -228,7 +228,11 @@ def create_batches( logger.debug(f"{self.__class__.__name__} creating batch {batch_idx}!") # Generate batch. - batch = self.get_batch(locations_for_batch=locations_for_batch, batch_idx=batch_idx) + batch = self.get_batch( + t0_datetimes=locations_for_batch.t0_datetime_UTC, + x_locations=locations_for_batch.x_center_OSGB, + y_locations=locations_for_batch.y_center_OSGB, + ) # Save batch to disk. netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) diff --git a/tests/config/test.yaml b/tests/config/test.yaml index 7b001802..af7dc0b5 100644 --- a/tests/config/test.yaml +++ b/tests/config/test.yaml @@ -33,8 +33,7 @@ input_data: topographic: topographic_filename: tests/data/europe_dem_2km_osgb.tif opticalflow: - number_previous_timesteps_to_use: 1 - opticalflow_image_size_pixels: 32 + opticalflow_input_image_size_pixels: 32 output_data: filepath: not used by unittests! process: diff --git a/tests/test_manager.py b/tests/test_manager.py index dcbaaf8d..b3729c0a 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -72,7 +72,7 @@ def test_load_yaml_configuration(): # noqa: D103 filename = local_path / "tests" / "config" / "test.yaml" manager.load_yaml_configuration(filename=filename) manager.initialise_data_sources() - assert len(manager.data_sources) == 7 + assert len(manager.data_sources) == 8 assert isinstance(manager.data_source_which_defines_geospatial_locations, GSPDataSource) From 76bff4ee988080342b04bef1e8cfe11df63d5d45 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 1 Dec 2021 18:02:41 +0000 Subject: [PATCH 154/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/config/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index 2aa576a9..da558217 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -173,7 +173,8 @@ class OpticalFlow(DataSourceMixin): ), ) opticalflow_forecast_minutes: int = Field( - 120, description="Duration of the optical flow predictions.") + 120, description="Duration of the optical flow predictions." + ) opticalflow_meters_per_pixel: int = METERS_PER_PIXEL_FIELD opticalflow_input_image_size_pixels: int = Field( IMAGE_SIZE_PIXELS * 2, From e468deab8529b58865ba171e3e29f0ce35e505e4 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Wed, 1 Dec 2021 18:10:57 +0000 Subject: [PATCH 155/197] prepare_ml_data.py runs again --- nowcasting_dataset/config/on_premises.yaml | 2 +- .../data_sources/optical_flow/optical_flow_data_source.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index 93c70f07..fe5c7078 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -67,7 +67,7 @@ input_data: opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* opticalflow_history_minutes: 5 opticalflow_forecast_minutes: 120 - opticalflow_image_size_pixels: 64 + opticalflow_input_image_size_pixels: 64 opticalflow_output_image_size_pixels: 24 opticalflow_source_data_source_class_name: SatelliteDataSource opticalflow_channels: diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 7ff4bd31..2e121391 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -90,7 +90,8 @@ def get_example( t0_dt=t0_dt, x_meters_center=x_meters_center, y_meters_center=y_meters_center ) satellite_data = satellite_data["data"] - return self._compute_and_return_optical_flow(satellite_data) + optical_flow_data_array = self._compute_and_return_optical_flow(satellite_data) + return optical_flow_data_array.to_dataset() @staticmethod def get_data_model_for_batch(): From 83e1e164d16c05b5f193e8a6dc07b731566e6426 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Wed, 1 Dec 2021 18:15:21 +0000 Subject: [PATCH 156/197] fix test_optical_flow_data_source --- .../data_sources/optical_flow/test_optical_flow_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data_sources/optical_flow/test_optical_flow_data_source.py b/tests/data_sources/optical_flow/test_optical_flow_data_source.py index b0a0e2df..ac89ae04 100644 --- a/tests/data_sources/optical_flow/test_optical_flow_data_source.py +++ b/tests/data_sources/optical_flow/test_optical_flow_data_source.py @@ -45,4 +45,4 @@ def test_optical_flow_get_example( example = optical_flow_datasource.get_example( t0_dt=t0_dt, x_meters_center=10_000, y_meters_center=10_000 ) - assert example.values.shape == (24, 32, 32, 1) # timesteps, height, width, channels + assert example["data"].shape == (24, 32, 32, 1) # timesteps, height, width, channels From 39a15f0f33d24e420a33299225c101559c3dcdfd Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Wed, 1 Dec 2021 18:25:30 +0000 Subject: [PATCH 157/197] border values should be -1 --- nowcasting_dataset/config/on_premises.yaml | 2 +- .../data_sources/optical_flow/optical_flow_data_source.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index fe5c7078..73c110b6 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -67,7 +67,7 @@ input_data: opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* opticalflow_history_minutes: 5 opticalflow_forecast_minutes: 120 - opticalflow_input_image_size_pixels: 64 + opticalflow_input_image_size_pixels: 200 opticalflow_output_image_size_pixels: 24 opticalflow_source_data_source_class_name: SatelliteDataSource opticalflow_channels: diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 2e121391..837e14ba 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -265,7 +265,7 @@ def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: map2=None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, - borderValue=np.NaN, + borderValue=-1, ) return remapped_image From fd66db542b653ed6cdd1597a38b5604d2a43e2f9 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 2 Dec 2021 13:33:13 +0000 Subject: [PATCH 158/197] improve logging when exception occurs --- .../data_sources/data_source.py | 11 +++++++++- nowcasting_dataset/manager.py | 22 +++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 5f8fb10f..a39950a4 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -286,7 +286,16 @@ def get_batch( self.get_example, t0_datetime, x_location, y_location ) future_examples.append(future_example) - examples = [future_example.result() for future_example in future_examples] + + # Get the examples back. future_example.result() will raise an exception + # if the worker thread raised an exception. + examples = [] + for example_i, future_example in enumerate(future_examples): + try: + examples.append(future_example.result()) + except Exception: + logger.error(f"Exception when processing {example_i=}!") + raise # Get the DataSource class, this could be one of the data sources like Sun cls = self.get_data_model_for_batch() diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index ba9a9262..e4331bc6 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -4,6 +4,7 @@ import multiprocessing from pathlib import Path from typing import Optional, Union +from functools import partial import numpy as np import pandas as pd @@ -441,6 +442,7 @@ def create_batches(self, overwrite_batches: bool) -> None: for split_name, locations_for_split in locations_for_each_example_of_each_split.items(): with multiprocessing.Pool(processes=n_data_sources) as pool: async_results_from_create_batches = [] + an_error_has_occured = multiprocessing.Event() for worker_id, (data_source_name, data_source) in enumerate( self.data_sources.items() ): @@ -482,10 +484,14 @@ def create_batches(self, overwrite_batches: bool) -> None: callback_msg = ( f"{data_source_name} has finished created batches for {split_name}!" ) - error_callback_msg = ( - f"Exception raised by {data_source_name} whilst creating batches for" - f" {split_name}:\n" - ) + + def _error_callback(exception): + error_callback_msg = ( + f"Exception raised by {data_source_name} whilst creating batches for" + f" {split_name}:\n" + ) + logger.error(error_callback_msg + str(exception)) + an_error_has_occured.set() # Submit data_source.create_batches task to the worker process. logger.debug( @@ -495,14 +501,16 @@ def create_batches(self, overwrite_batches: bool) -> None: data_source.create_batches, kwds=kwargs_for_create_batches, callback=lambda result: logger.info(callback_msg), - error_callback=lambda exception: logger.error( - error_callback_msg + str(exception) - ), + error_callback=_error_callback, ) async_results_from_create_batches.append(async_result) # Wait for all async_results to finish: for async_result in async_results_from_create_batches: async_result.wait() + if an_error_has_occured.is_set(): + raise RuntimeError( + f"Worker process {data_source_name} raised an exception" + f" whilst working on {split_name}!") logger.info(f"Finished creating batches for {split_name}!") From 4ba7d6306cadb71c787e8db75086602248cb6781 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 2 Dec 2021 13:35:06 +0000 Subject: [PATCH 159/197] improve logging when exception occurs --- nowcasting_dataset/config/on_premises.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index 73c110b6..e4b7fcfa 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -67,7 +67,7 @@ input_data: opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* opticalflow_history_minutes: 5 opticalflow_forecast_minutes: 120 - opticalflow_input_image_size_pixels: 200 + opticalflow_input_image_size_pixels: 118 opticalflow_output_image_size_pixels: 24 opticalflow_source_data_source_class_name: SatelliteDataSource opticalflow_channels: From 78ea3153d6c0ba16f7f0cfacc3ba73a67928698c Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 2 Dec 2021 15:25:19 +0000 Subject: [PATCH 160/197] More informative logging when requested region of interest steps outside of available Zarr data --- nowcasting_dataset/config/on_premises.yaml | 2 +- .../satellite/satellite_data_source.py | 51 +++++++++++++------ nowcasting_dataset/manager.py | 2 +- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index e4b7fcfa..2f479c86 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -67,7 +67,7 @@ input_data: opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* opticalflow_history_minutes: 5 opticalflow_forecast_minutes: 120 - opticalflow_input_image_size_pixels: 118 + opticalflow_input_image_size_pixels: 112 opticalflow_output_image_size_pixels: 24 opticalflow_source_data_source_class_name: SatelliteDataSource opticalflow_channels: diff --git a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py index 2211f177..cc1b6cb2 100644 --- a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py +++ b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py @@ -86,23 +86,44 @@ def get_spatial_region_of_interest( Returns: The selected data around the center """ - x_index = ( - np.searchsorted(data_array.x.values, x_center_osgb) - 1 - ) # To have the center fall within the pixel - y_index = np.searchsorted(data_array.y.values, y_center_osgb) - 1 - min_y = y_index - (self._square.size_pixels // 2) - min_x = x_index - (self._square.size_pixels // 2) - assert min_y >= 0, ( - f"Y location must be at least {(self._square.size_pixels // 2)} " - f"pixels from the edge of the area, but is {y_index} for y center of {y_center_osgb}" - ) - assert min_x >= 0, ( - f"X location must be at least {(self._square.size_pixels // 2)}" - f" pixels from the edge of the area, but is {x_index} for x center of {x_center_osgb}" + # Get the index into x and y nearest to x_center_osgb and y_center_osgb: + x_index_at_center = np.searchsorted(data_array.x.values, x_center_osgb) - 1 + y_index_at_center = np.searchsorted(data_array.y.values, y_center_osgb) - 1 + x_and_y_index_at_center = pd.Series({"x": x_index_at_center, "y": y_index_at_center}) + half_image_size_pixels = self._square.size_pixels // 2 + min_x_and_y_index = x_and_y_index_at_center - half_image_size_pixels + max_x_and_y_index = x_and_y_index_at_center + half_image_size_pixels + + # Check whether the requested region of interest steps outside of the available data: + suggested_reduction_of_image_size_pixels = ( + max( + (-min_x_and_y_index.min() if (min_x_and_y_index < 0).any() else 0), + (max_x_and_y_index.x - len(data_array.x)), + (max_x_and_y_index.y - len(data_array.y)), + ) + * 2 ) + if suggested_reduction_of_image_size_pixels > 0: + new_suggested_image_size_pixels = ( + self._square.size_pixels - suggested_reduction_of_image_size_pixels + ) + raise RuntimeError( + "Requested region of interest of satellite data steps outside of the available" + " geographical extent of the Zarr data. The requested region of interest extends" + f" from pixel indicies" + f" x={min_x_and_y_index.x} to x={max_x_and_y_index.x}," + f" y={min_x_and_y_index.y} to y={max_x_and_y_index.y}. In the Zarr data," + f" len(x)={len(data_array.x)}, len(y)={len(data_array.y)}. Try reducing" + f" image_size_pixels from {self._square.size_pixels} to" + f" {new_suggested_image_size_pixels} pixels." + ) + + # Select the geographical region of interest. + # Note that isel is *exclusive* of the end of the slice. + # e.g. isel(x=slice(0, 3)) will return the first, second, and third values. data_array = data_array.isel( - x=slice(min_x, min_x + self._square.size_pixels), - y=slice(min_y, min_y + self._square.size_pixels), + x=slice(min_x_and_y_index.x, max_x_and_y_index.x), + y=slice(min_x_and_y_index.y, max_x_and_y_index.y), ) return data_array diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index e4331bc6..e75e9d63 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -2,9 +2,9 @@ import logging import multiprocessing +from functools import partial from pathlib import Path from typing import Optional, Union -from functools import partial import numpy as np import pandas as pd From 5e6a09ad1f10616edb8ccf5f9635b8b8a1cc37ea Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 2 Dec 2021 15:38:39 +0000 Subject: [PATCH 161/197] prepare_ml_data.py runs! Now using input_image_size_pixels=106 and borderMode=cv2.BORDER_REPLICATE --- nowcasting_dataset/config/on_premises.yaml | 2 +- .../optical_flow/optical_flow_data_source.py | 19 ++++++++++++++----- .../optical_flow/optical_flow_model.py | 8 ++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index 2f479c86..7cc50c32 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -67,7 +67,7 @@ input_data: opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* opticalflow_history_minutes: 5 opticalflow_forecast_minutes: 120 - opticalflow_input_image_size_pixels: 112 + opticalflow_input_image_size_pixels: 106 opticalflow_output_image_size_pixels: 24 opticalflow_source_data_source_class_name: SatelliteDataSource opticalflow_channels: diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 837e14ba..345aa644 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -225,7 +225,8 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n next_image = next_image * 255 next_image = next_image.astype(np.uint8) - # Docs: https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af # noqa + # Docs: + # https://docs.opencv.org/4.5.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af flow = cv2.calcOpticalFlowFarneback( prev=prev_image, next=next_image, @@ -241,7 +242,11 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n return flow -def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: +def remap_image( + image: np.ndarray, + flow: np.ndarray, + border_mode: int = cv2.BORDER_REPLICATE, +) -> np.ndarray: """ Takes an image and warps it forwards in time according to the flow field. @@ -249,22 +254,26 @@ def remap_image(image: np.ndarray, flow: np.ndarray) -> np.ndarray: image: The grayscale image to warp. flow: A 3D array. The first two dimensions must be the same size as the first two dimensions of the image. The third dimension represented the x and y displacement. + border_mode: One of cv2's BorderTypes such as cv2.BORDER_CONSTANT or cv2.BORDER_REPLICATE. + If border_mode=cv2.BORDER_CONSTANT then the border will be set to -1. + docs.opencv.org/4.5.4/d2/de8/group__core__array.html#ga209f2f4869e304c82d07739337eae7c5 - Returns: Warped image. The border has values np.NaN. + Returns: Warped image. """ # Adapted from https://github.com/opencv/opencv/issues/11068 height, width = flow.shape[:2] remap = -flow.copy() remap[..., 0] += np.arange(width) # map_x remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y - # remap docs: https://docs.opencv.org/4.5.4/da/d54/group__imgproc__transform.html#gab75ef31ce5cdfb5c44b6da5f3b908ea4 # noqa + # remap docs: + # docs.opencv.org/4.5.4/da/d54/group__imgproc__transform.html#gab75ef31ce5cdfb5c44b6da5f3b908ea4 # TODO: Maybe use integer remap: docs say that might be faster? remapped_image = cv2.remap( src=image, map1=remap, map2=None, interpolation=cv2.INTER_LINEAR, - borderMode=cv2.BORDER_CONSTANT, + borderMode=border_mode, borderValue=-1, ) return remapped_image diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py index 9cf7f2df..538a7428 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py @@ -1,7 +1,7 @@ """ Model for output of Optical Flow data """ from __future__ import annotations -from xarray.ufuncs import isinf, isnan +import numpy as np from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput @@ -15,7 +15,7 @@ class OpticalFlow(DataSourceOutput): @classmethod def model_validation(cls, v): """Check that all values are not NaN, Infinite, or -1.""" - assert (~isnan(v.data)).all(), "Some optical flow data values are NaNs" - assert (~isinf(v.data)).all(), "Some optical flow data values are Infinite" - assert (v.data != -1).all(), "Some optical flow data values are -1's" + assert (~np.isnan(v.data)).all(), "Some optical flow data values are NaNs" + assert (~np.isinf(v.data)).all(), "Some optical flow data values are Infinite" + assert (v.data != -1).all(), "Some optical flow data values are -1" return v From 742d389967f2ff5400b910167da25247aa5f16c1 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 2 Dec 2021 18:11:12 +0000 Subject: [PATCH 162/197] fix compression and dtype --- nowcasting_dataset/config/on_premises.yaml | 2 +- nowcasting_dataset/data_sources/data_source.py | 4 +++- .../optical_flow/optical_flow_data_source.py | 18 +++++++++--------- .../optical_flow/optical_flow_model.py | 1 + 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index 7cc50c32..3e72efed 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -67,7 +67,7 @@ input_data: opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* opticalflow_history_minutes: 5 opticalflow_forecast_minutes: 120 - opticalflow_input_image_size_pixels: 106 + opticalflow_input_image_size_pixels: 102 opticalflow_output_image_size_pixels: 24 opticalflow_source_data_source_class_name: SatelliteDataSource opticalflow_channels: diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index a39950a4..03d24615 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -235,8 +235,10 @@ def create_batches( ) # Save batch to disk. + # TODO: Use DataSourceOutput.save_netcdf netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) - batch.to_netcdf(netcdf_filename, engine="h5netcdf") + encoding = {name: {"compression": "lzf"} for name in batch.data_vars} + batch.to_netcdf(netcdf_filename, engine="h5netcdf", encoding=encoding) # Upload if necessary. if ( diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 345aa644..6c95e652 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -11,7 +11,6 @@ import xarray as xr from nowcasting_dataset.data_sources import DataSource -from nowcasting_dataset.data_sources.datasource_output import DataSourceOutput from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow _LOG = logging.getLogger(__name__) @@ -34,7 +33,7 @@ class OpticalFlowDataSource(DataSource): "real" pixels values, and not NaNs. output_image_size_pixels: The size of the output image. The output image is a center-crop of the input image, after it has been "flowed". - source_data_source_class: Either HRVSatelliteDataSource or SatelliteDataSource. + source_data_source_class_name: Either HRVSatelliteDataSource or SatelliteDataSource. channels: The satellite channels to compute optical flow for. """ @@ -73,7 +72,7 @@ def open(self): def get_example( self, t0_dt: pd.Timestamp, x_meters_center: Number, y_meters_center: Number - ) -> DataSourceOutput: + ) -> xr.Dataset: """ Get Optical Flow Example data @@ -165,7 +164,8 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D self.output_image_size_pixels, n_channels, ), - fill_value=np.NaN, + fill_value=-1, + dtype=np.int16, ) for channel_i in range(n_channels): @@ -243,9 +243,9 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n def remap_image( - image: np.ndarray, - flow: np.ndarray, - border_mode: int = cv2.BORDER_REPLICATE, + image: np.ndarray, + flow: np.ndarray, + border_mode: int = cv2.BORDER_REPLICATE, ) -> np.ndarray: """ Takes an image and warps it forwards in time according to the flow field. @@ -292,6 +292,6 @@ def crop_center(image: np.ndarray, x_size: int, y_size: int) -> np.ndarray: The cropped image """ y, x = image.shape - startx = x // 2 - (x_size // 2) - starty = y // 2 - (y_size // 2) + startx = (x // 2) - (x_size // 2) + starty = (y // 2) - (y_size // 2) return image[starty : starty + y_size, startx : startx + x_size] diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py index 538a7428..1f410e05 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_model.py @@ -11,6 +11,7 @@ class OpticalFlow(DataSourceOutput): __slots__ = () _expected_dimensions = ("time", "x", "y", "channels") + _expected_data_vars = ("data",) @classmethod def model_validation(cls, v): From 445705c55a2d41ca381a7b4ca91acfbe54f96b51 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 09:19:05 +0000 Subject: [PATCH 163/197] update docstring and tidy crop_center --- .../optical_flow/optical_flow_data_source.py | 61 +++++++++++++------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 6c95e652..98e0108f 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -19,20 +19,37 @@ @dataclass class OpticalFlowDataSource(DataSource): """ - Optical Flow Data Source, computing flow between Satellite data + Optical Flow Data Source. + + Predicts future satellite imagery by computing the "flow" between consecutive pairs of + satellite images and using that flow to "warp" the most recent satellite image (the "t0 image") + to predict future satellite images. + + Optical flow is surprisingly effective at predicting future satellite images over time horizons + out to about 2 hours. After 2 hours the predictions start to go a bit crazy. There are some + notable problems with optical flow predictions: + + 1) Optical flow doesn't understand that clouds grow, shrink, appear from "nothing", and disappear + into "nothing". Optical flow just moves pixels around. + 2) Optical flow doesn't understand that satellite images tend to get brighter as the sun rises + and darker as the sun sets. + + Arguments for the OpticalFlowDataSource constructor: history_minutes: Duration of historical data to use when computing the optical flow field. For example, set to 5 to use just two images: the t-1 and t0 images. Set to 10 to compute - the optical flow field separately for the image pairs (t-2, t-1), and (t-1, t0) and to + the optical flow field separately for the image pairs (t-2, t-1) and (t-1, t0) and to use the mean optical flow field. forecast_minutes: Duration of the optical flow predictions. zarr_path: The location of the intermediate satellite data to compute optical flows with. input_image_size_pixels: The *input* image size (i.e. the image size to load off disk). - This should be larger than output_image_size_pixels to provide sufficient border to mean - that, even after the image has been "flowed", all edges of the output image are - "real" pixels values, and not NaNs. + This should be significantly larger than output_image_size_pixels to provide sufficient + border so that, even after the image has been "flowed", all edges of the output image are + "real" pixels values, and not NaNs. For a forecast horizon of 120 minutes, and an output + image size of 24 pixels, we have found that the input image size needs to be at least + 128 pixels. output_image_size_pixels: The size of the output image. The output image is a center-crop of - the input image, after it has been "flowed". + the input image after it has been "flowed". source_data_source_class_name: Either HRVSatelliteDataSource or SatelliteDataSource. channels: The satellite channels to compute optical flow for. """ @@ -206,13 +223,13 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n Compute the optical flow for a set of images Args: - t0_image: t0 image + t0_image: t0 image. Can be any dtype. previous_image: previous image to compute optical flow with Returns: Optical Flow field """ - # Input images have to be single channel and uint8. + # cv2.calcOpticalFlowFarneback expects images to be uint8: # TODO: Refactor this! image_min = np.min([prev_image, next_image]) image_max = np.max([prev_image, next_image]) @@ -225,7 +242,7 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n next_image = next_image * 255 next_image = next_image.astype(np.uint8) - # Docs: + # Docs for cv2.calcOpticalFlowFarneback: # https://docs.opencv.org/4.5.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af flow = cv2.calcOpticalFlowFarneback( prev=prev_image, @@ -256,6 +273,7 @@ def remap_image( dimensions of the image. The third dimension represented the x and y displacement. border_mode: One of cv2's BorderTypes such as cv2.BORDER_CONSTANT or cv2.BORDER_REPLICATE. If border_mode=cv2.BORDER_CONSTANT then the border will be set to -1. + For details of other border_mode settings, see the Open CV docs here: docs.opencv.org/4.5.4/d2/de8/group__core__array.html#ga209f2f4869e304c82d07739337eae7c5 Returns: Warped image. @@ -279,19 +297,22 @@ def remap_image( return remapped_image -def crop_center(image: np.ndarray, x_size: int, y_size: int) -> np.ndarray: +def crop_center(image: np.ndarray, output_image_size_pixels: int) -> np.ndarray: """ - Crop center of numpy image + Crop center of a 2D numpy image. Args: - image: Image to crop - x_size: Size in x direction - y_size: Size in y direction - + image: The input image to crop. + output_image_size_pixels: The requested size of the output image. Returns: - The cropped image + The cropped image, of size output_image_size_pixels x output_image_size_pixels """ - y, x = image.shape - startx = (x // 2) - (x_size // 2) - starty = (y // 2) - (y_size // 2) - return image[starty : starty + y_size, startx : startx + x_size] + input_size_y, input_size_x = image.shape + assert input_size_x >= output_image_size_pixels + assert input_size_y >= output_image_size_pixels + half_output_image_size_pixels = output_image_size_pixels // 2 + start_x = (input_size_x // 2) - half_output_image_size_pixels + start_y = (input_size_y // 2) - half_output_image_size_pixels + end_x = start_x + output_image_size_pixels + end_y = start_y + output_image_size_pixels + return image[start_y:end_y, start_x:end_x] From 58a86a277a10675bfe870dbb0783aef37368ad8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Dec 2021 09:19:26 +0000 Subject: [PATCH 164/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nowcasting_dataset/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index e75e9d63..578c894e 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -511,6 +511,7 @@ def _error_callback(exception): if an_error_has_occured.is_set(): raise RuntimeError( f"Worker process {data_source_name} raised an exception" - f" whilst working on {split_name}!") + f" whilst working on {split_name}!" + ) logger.info(f"Finished creating batches for {split_name}!") From e6e1dd43dec8242c68e43f524ea7bd3d67cdef7d Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 09:20:19 +0000 Subject: [PATCH 165/197] fix passing crop_center too many args --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 98e0108f..33e8e1ba 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -205,11 +205,7 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D for prediction_timestep in range(self.forecast_length): flow = optical_flow * (prediction_timestep + 1) warped_image = remap_image(image=t0_image, flow=flow) - warped_image = crop_center( - warped_image, - self.output_image_size_pixels, - self.output_image_size_pixels, - ) + warped_image = crop_center(warped_image, self.output_image_size_pixels) prediction_block[prediction_timestep, :, :, channel_i] = warped_image data_array = self._put_predictions_into_data_array( From 92f8b602e39bde9312a82e8fbf0106c3c832c105 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 10:33:50 +0000 Subject: [PATCH 166/197] refactor --- .../optical_flow/optical_flow_data_source.py | 63 +++++++++++-------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 33e8e1ba..23bae0a1 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -120,9 +120,7 @@ def _put_predictions_into_data_array( predictions: np.ndarray, ) -> xr.DataArray: """ - Updates the dataarray with predictions - - Additionally, changes the temporal size to t0+1 to forecast horizon + Puts optical flow predictions into an xr.DataArray. Args: satellite_data: Satellite data @@ -131,25 +129,24 @@ def _put_predictions_into_data_array( Returns: The Xarray DataArray with the optical flow predictions """ - # Select the timesteps for the optical flow predictions. + # Generate a pd.DatetimeIndex for the optical flow predictions. t0_datetime_utc = satellite_data.isel(time=-1)["time"].values + t1_datetime_utc = t0_datetime_utc + self.sample_period_duration datetime_index_of_predictions = pd.date_range( - t0_datetime_utc, periods=self.forecast_length, freq=self.sample_period_duration + t1_datetime_utc, periods=self.forecast_length, freq=self.sample_period_duration ) # Select the center crop. - # TODO: Generalise crop_center and use again here: - border = (satellite_data.sizes["x"] - self.output_image_size_pixels) // 2 - satellite_data = satellite_data.isel( - x=slice(border, satellite_data.sizes["x"] - border), - y=slice(border, satellite_data.sizes["y"] - border), - ) + satellite_data_cropped = satellite_data.isel(time_index=0, channels_index=0) + satellite_data_cropped = crop_center(satellite_data_cropped, self.output_image_size_pixels) + + # Put into DataArray return xr.DataArray( data=predictions, coords=( ("time", datetime_index_of_predictions), - ("x", satellite_data.coords["x"].values), - ("y", satellite_data.coords["y"].values), + ("x", satellite_data_cropped.coords["x"].values), + ("y", satellite_data_cropped.coords["y"].values), ("channels", satellite_data.coords["channels"].values), ), name="data", @@ -214,29 +211,41 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D return data_array +def _convert_arrays_to_uint8(*arrays: tuple[np.ndarray]) -> tuple[np.ndarray]: + """Convert multiple arrays to uint8, using the same min and max to scale all arrays. + """ + # First, stack into a single numpy array so we can work on all images at the same time: + stacked = np.stack(arrays) + + # Rescale pixel values to be in the range [0, 1]: + stacked -= stacked.min() + stacked /= stacked.max() + + # Convert to uint8 (uint8 can represent integers in the range [0, 255]): + stacked *= 255 + stacked = stacked.astype(np.uint8) + + return tuple(stacked) + + def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.ndarray: """ Compute the optical flow for a set of images Args: - t0_image: t0 image. Can be any dtype. - previous_image: previous image to compute optical flow with + prev_image, next_image: A pair of images representing two timesteps. This algorithm + will estimate the "movement" across these two timesteps. Both images must be the + same dtype. Returns: - Optical Flow field + Dense optical flow field: A 3D array. The first two dimension are the same size as the + input images. The third dimension is of size 2 and represents the + displacement in x and y. """ + assert prev_image.dtype == next_image.dtype + # cv2.calcOpticalFlowFarneback expects images to be uint8: - # TODO: Refactor this! - image_min = np.min([prev_image, next_image]) - image_max = np.max([prev_image, next_image]) - prev_image = prev_image - image_min - prev_image = prev_image / (image_max - image_min) - prev_image = prev_image * 255 - prev_image = prev_image.astype(np.uint8) - next_image = next_image - image_min - next_image = next_image / (image_max - image_min) - next_image = next_image * 255 - next_image = next_image.astype(np.uint8) + prev_image, next_image = _convert_arrays_to_uint8(prev_image, next_image) # Docs for cv2.calcOpticalFlowFarneback: # https://docs.opencv.org/4.5.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af From ae6aa75a72b20e518d79a3a3c60974ebd0ea5fe0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Dec 2021 10:34:09 +0000 Subject: [PATCH 167/197] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../data_sources/optical_flow/optical_flow_data_source.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 23bae0a1..40c67f79 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -212,8 +212,7 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D def _convert_arrays_to_uint8(*arrays: tuple[np.ndarray]) -> tuple[np.ndarray]: - """Convert multiple arrays to uint8, using the same min and max to scale all arrays. - """ + """Convert multiple arrays to uint8, using the same min and max to scale all arrays.""" # First, stack into a single numpy array so we can work on all images at the same time: stacked = np.stack(arrays) From e772d3b68d050bbef8b1247a59a9b7613430d442 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 10:41:08 +0000 Subject: [PATCH 168/197] fix bug with _convert_array_to_uint8 --- .../data_sources/optical_flow/optical_flow_data_source.py | 6 +++++- nowcasting_dataset/manager.py | 5 ++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 23bae0a1..930f5125 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -140,7 +140,7 @@ def _put_predictions_into_data_array( satellite_data_cropped = satellite_data.isel(time_index=0, channels_index=0) satellite_data_cropped = crop_center(satellite_data_cropped, self.output_image_size_pixels) - # Put into DataArray + # Put into DataArray: return xr.DataArray( data=predictions, coords=( @@ -217,12 +217,16 @@ def _convert_arrays_to_uint8(*arrays: tuple[np.ndarray]) -> tuple[np.ndarray]: # First, stack into a single numpy array so we can work on all images at the same time: stacked = np.stack(arrays) + # Convert to float64 for normalisation: + stacked = stacked.astype(np.float64) + # Rescale pixel values to be in the range [0, 1]: stacked -= stacked.min() stacked /= stacked.max() # Convert to uint8 (uint8 can represent integers in the range [0, 255]): stacked *= 255 + stacked = stacked.round() stacked = stacked.astype(np.uint8) return tuple(stacked) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 578c894e..cd934926 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -486,11 +486,10 @@ def create_batches(self, overwrite_batches: bool) -> None: ) def _error_callback(exception): - error_callback_msg = ( + logger.error( f"Exception raised by {data_source_name} whilst creating batches for" - f" {split_name}:\n" + f" {split_name}:\n{exception}" ) - logger.error(error_callback_msg + str(exception)) an_error_has_occured.set() # Submit data_source.create_batches task to the worker process. From 6175d3fbe355dd32f8d504ed22b6170dc14d224d Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 10:43:10 +0000 Subject: [PATCH 169/197] fix bug with _put_predictions_into_data_array --- .../data_sources/optical_flow/optical_flow_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 64136585..8aa78cf7 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -137,7 +137,7 @@ def _put_predictions_into_data_array( ) # Select the center crop. - satellite_data_cropped = satellite_data.isel(time_index=0, channels_index=0) + satellite_data_cropped = satellite_data.isel(time=0, channels=0) satellite_data_cropped = crop_center(satellite_data_cropped, self.output_image_size_pixels) # Put into DataArray: From 34d5c22cbd066e94a7ae7436c7b7ec1d0bba0b5e Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 11:13:50 +0000 Subject: [PATCH 170/197] update docs --- .../optical_flow/optical_flow_data_source.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 8aa78cf7..534aee7f 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -157,10 +157,10 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D Compute and return optical flow predictions for the example Args: - satellite_data: Satellite DataArray of historical satellite images. + satellite_data: Satellite DataArray of historical satellite images, up to and include t0 Returns: - The Tensor with the optical flow predictions for t0 to forecast horizon + DataArray with the optical flow predictions from t1 to the forecast horizon. """ n_channels = satellite_data.sizes["channels"] @@ -170,7 +170,7 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D ), f"{len(satellite_data.coords['time'])=} != {self.history_length+1=}" assert n_channels == len(self.channels), f"{n_channels=} != {len(self.channels)=}" - # TODO: Use the correct dtype. + # Pre-allocate an array, into which our optical flow prediction will be placed. prediction_block = np.full( shape=( self.forecast_length, @@ -182,11 +182,15 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D dtype=np.int16, ) + # Compute flow fields and optical flow predictions separately for each satellite channel + # because the different channels represent different physical phenomena and so, + # in principle, could move in different directions (e.g. water vapour vs high clouds). for channel_i in range(n_channels): # Compute optical flow field: sat_data_for_chan = satellite_data.isel(channels=channel_i) - # Loop through pairs of historical images to compute optical flow fields: + # Loop through pairs of historical images to compute optical flow fields for each + # pair of consecutive satellite images, and then compute the mean of those flow fields. optical_flows = [] # self.history_length does not include t0. for history_timestep in range(self.history_length): @@ -194,7 +198,6 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D next_image = sat_data_for_chan.isel(time=history_timestep + 1).data optical_flow = compute_optical_flow(prev_image, next_image) optical_flows.append(optical_flow) - # Average predictions optical_flow = np.mean(optical_flows, axis=0) # Compute predicted images. From 8523aa2636e8405e57a43caf6ad4d944677c1e25 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 11:24:17 +0000 Subject: [PATCH 171/197] fix linter errors in optical_flow_data_source.py --- .../data_sources/optical_flow/optical_flow_data_source.py | 8 +++++--- nowcasting_dataset/manager.py | 1 - nowcasting_dataset/utils.py | 5 ++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 534aee7f..de875fde 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -29,8 +29,8 @@ class OpticalFlowDataSource(DataSource): out to about 2 hours. After 2 hours the predictions start to go a bit crazy. There are some notable problems with optical flow predictions: - 1) Optical flow doesn't understand that clouds grow, shrink, appear from "nothing", and disappear - into "nothing". Optical flow just moves pixels around. + 1) Optical flow doesn't understand that clouds grow, shrink, appear from "nothing", and + disappear into "nothing". Optical flow just moves pixels around. 2) Optical flow doesn't understand that satellite images tend to get brighter as the sun rises and darker as the sun sets. @@ -61,7 +61,7 @@ class OpticalFlowDataSource(DataSource): output_image_size_pixels: int = 32 source_data_source_class_name: str = "SatelliteDataSource" - def __post_init__(self): + def __post_init__(self): # noqa super().__post_init__() # Get round circular import problem @@ -85,6 +85,7 @@ def __post_init__(self): ) def open(self): + """Open the underlying self.source_data_source.""" self.source_data_source.open() def get_example( @@ -315,6 +316,7 @@ def crop_center(image: np.ndarray, output_image_size_pixels: int) -> np.ndarray: Args: image: The input image to crop. output_image_size_pixels: The requested size of the output image. + Returns: The cropped image, of size output_image_size_pixels x output_image_size_pixels """ diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index cd934926..da45f59e 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -2,7 +2,6 @@ import logging import multiprocessing -from functools import partial from pathlib import Path from typing import Optional, Union diff --git a/nowcasting_dataset/utils.py b/nowcasting_dataset/utils.py index 59d18062..ab4af4e1 100644 --- a/nowcasting_dataset/utils.py +++ b/nowcasting_dataset/utils.py @@ -199,16 +199,18 @@ def get_start_and_end_example_index(batch_idx: int, batch_size: int) -> (int, in class DummyExecutor(futures.Executor): - """Drop-in replacement for ThreadPoolExecutor or ProcessPoolExecutor for easy debugging. + """Drop-in replacement for ThreadPoolExecutor or ProcessPoolExecutor to make debugging easier. Adapted from https://stackoverflow.com/a/10436851/732596 """ def __init__(self, *args, **kwargs): + """Initialise DummyExecutor.""" self._shutdown = False self._shutdownLock = threading.Lock() def submit(self, fn, *args, **kwargs): + """Submit task to DummyExecutor.""" with self._shutdownLock: if self._shutdown: raise RuntimeError("cannot schedule new futures after shutdown") @@ -224,5 +226,6 @@ def submit(self, fn, *args, **kwargs): return f def shutdown(self, wait=True): + """Shutdown dummy executor.""" with self._shutdownLock: self._shutdown = True From 322bb724a3cad15ddfca4354b4653b6b96dfe8ff Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 12:46:55 +0000 Subject: [PATCH 172/197] tests/test_manager.py passes --- .../data_sources/data_source.py | 14 ++++++------- .../optical_flow/optical_flow_data_source.py | 20 ++++++++++++++++--- .../satellite/satellite_data_source.py | 11 +++++++++- .../data_sources/sun/raw_data_load_save.py | 7 +------ nowcasting_dataset/filesystem/utils.py | 2 ++ nowcasting_dataset/manager.py | 19 ++++++++++-------- tests/config/test.yaml | 6 ++++++ 7 files changed, 54 insertions(+), 25 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 03d24615..9fd8820a 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -132,13 +132,6 @@ def open(self): """ pass - def check_input_paths_exist(self) -> None: - """Check any input paths exist. Raise FileNotFoundError if not. - - Can be overridden by child classes. - """ - pass - # TODO: Issue #319: Standardise parameter names. # TODO: Issue #367: Reduce duplication. def create_batches( @@ -371,6 +364,13 @@ def get_example( """Must be overridden by child classes.""" raise NotImplementedError() + def check_input_paths_exist(self) -> None: + """Check any input paths exist. Raise FileNotFoundError if not. + + Must be overridden by child classes. + """ + raise NotImplementedError() + @dataclass class ImageDataSource(DataSource): diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index de875fde..2e6b179d 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -10,6 +10,7 @@ import pandas as pd import xarray as xr +import nowcasting_dataset.filesystem.utils as nd_fs_utils from nowcasting_dataset.data_sources import DataSource from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow @@ -62,6 +63,11 @@ class OpticalFlowDataSource(DataSource): source_data_source_class_name: str = "SatelliteDataSource" def __post_init__(self): # noqa + assert self.output_image_size_pixels <= self.input_image_size_pixels, ( + "output_image_size_pixels must be equal to or smaller than input_image_size_pixels" + f" {self.output_image_size_pixels=}, {self.input_image_size_pixels=}" + ) + super().__post_init__() # Get round circular import problem @@ -214,6 +220,10 @@ def _compute_and_return_optical_flow(self, satellite_data: xr.DataArray) -> xr.D ) return data_array + def check_input_paths_exist(self) -> None: + """Check input paths exist. If not, raise a FileNotFoundError.""" + nd_fs_utils.check_path_exists(self.zarr_path) + def _convert_arrays_to_uint8(*arrays: tuple[np.ndarray]) -> tuple[np.ndarray]: """Convert multiple arrays to uint8, using the same min and max to scale all arrays.""" @@ -249,7 +259,7 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n input images. The third dimension is of size 2 and represents the displacement in x and y. """ - assert prev_image.dtype == next_image.dtype + assert prev_image.dtype == next_image.dtype, "Images must be the same dtype!" # cv2.calcOpticalFlowFarneback expects images to be uint8: prev_image, next_image = _convert_arrays_to_uint8(prev_image, next_image) @@ -321,8 +331,12 @@ def crop_center(image: np.ndarray, output_image_size_pixels: int) -> np.ndarray: The cropped image, of size output_image_size_pixels x output_image_size_pixels """ input_size_y, input_size_x = image.shape - assert input_size_x >= output_image_size_pixels - assert input_size_y >= output_image_size_pixels + assert ( + input_size_x >= output_image_size_pixels + ), "output_image_size_pixels is larger than the input image!" + assert ( + input_size_y >= output_image_size_pixels + ), "output_image_size_pixels is larger than the input image!" half_output_image_size_pixels = output_image_size_pixels // 2 start_x = (input_size_x // 2) - half_output_image_size_pixels start_y = (input_size_y // 2) - half_output_image_size_pixels diff --git a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py index cc1b6cb2..e47a8b63 100644 --- a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py +++ b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py @@ -27,6 +27,9 @@ class SatelliteDataSource(ZarrDataSource): def __post_init__(self, image_size_pixels: int, meters_per_pixel: int): """Post Init""" + assert len(self.channels) > 0, "channels cannot be empty!" + assert image_size_pixels > 0, "image_size_pixels cannot be <= 0!" + assert meters_per_pixel > 0, "meters_per_pixel cannot be <= 0!" super().__post_init__(image_size_pixels, meters_per_pixel) n_channels = len(self.channels) self._shape_of_example = ( @@ -46,9 +49,15 @@ def open(self) -> None: call open() _after_ creating separate processes. """ self._data = self._open_data() - self._data = self._data.sel(variable=list(self.channels)) if "variable" in self._data.dims: self._data = self._data.rename({"variable": "channels"}) + if not set(self.channels).issubset(self._data.channels.values): + raise RuntimeError( + f"One or more requested channels are not available in {self.zarr_path}!" + f" Requested channels={self.channels}." + f" Available channels={self._data.channels.values}" + ) + self._data = self._data.sel(channels=list(self.channels)) def _open_data(self) -> xr.DataArray: return open_sat_data(zarr_path=self.zarr_path, consolidated=self.consolidated) diff --git a/nowcasting_dataset/data_sources/sun/raw_data_load_save.py b/nowcasting_dataset/data_sources/sun/raw_data_load_save.py index 355a8732..f3ed3b30 100644 --- a/nowcasting_dataset/data_sources/sun/raw_data_load_save.py +++ b/nowcasting_dataset/data_sources/sun/raw_data_load_save.py @@ -141,13 +141,8 @@ def load_from_zarr( The index is timestamps, and the columns are the x and y coordinates """ - logger.debug("Loading sun data") + logger.debug(f"Loading sun data from {zarr_path}") - # It is possible to simplify the code below and do - # xr.open_dataset(file, engine='h5netcdf') - # in the first 'with' block, and delete the second 'with' block. - # But that takes 1 minute to load the data, where as loading into memory - # first and then loading from memory takes 23 seconds! sun = xr.open_dataset(zarr_path, engine="zarr") if (start_dt is not None) and (end_dt is not None): diff --git a/nowcasting_dataset/filesystem/utils.py b/nowcasting_dataset/filesystem/utils.py index 37dd9c2e..bbccccf4 100644 --- a/nowcasting_dataset/filesystem/utils.py +++ b/nowcasting_dataset/filesystem/utils.py @@ -97,6 +97,8 @@ def check_path_exists(path: Union[str, Path]): `path` can include wildcards. """ + if not bool(path): + raise FileNotFoundError("Not a valid path!") filesystem = get_filesystem(path) if not filesystem.exists(path): # Now try using `glob`. Maybe `path` includes a wildcard? diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index da45f59e..4cee7e75 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -480,14 +480,15 @@ def create_batches(self, overwrite_batches: bool) -> None: ) # Logger messages for callbacks: - callback_msg = ( - f"{data_source_name} has finished created batches for {split_name}!" - ) + def _callback(result): + logger.info( + f"{data_source_name} has finished created batches for {split_name}!" + ) def _error_callback(exception): - logger.error( + logger.exception( f"Exception raised by {data_source_name} whilst creating batches for" - f" {split_name}:\n{exception}" + f" {split_name}:\n{exception.__class__.__name__}: {exception}" ) an_error_has_occured.set() @@ -498,7 +499,7 @@ def _error_callback(exception): async_result = pool.apply_async( data_source.create_batches, kwds=kwargs_for_create_batches, - callback=lambda result: logger.info(callback_msg), + callback=_callback, error_callback=_error_callback, ) async_results_from_create_batches.append(async_result) @@ -507,9 +508,11 @@ def _error_callback(exception): for async_result in async_results_from_create_batches: async_result.wait() if an_error_has_occured.is_set(): + # An error has occurred but, at this point in the code, we don't know which + # worker process raised the exception. But, with luck, the worker process + # will have logged an informative exception via the _error_callback func. raise RuntimeError( - f"Worker process {data_source_name} raised an exception" - f" whilst working on {split_name}!" + f"A worker process raised an exception whilst working on {split_name}!" ) logger.info(f"Finished creating batches for {split_name}!") diff --git a/tests/config/test.yaml b/tests/config/test.yaml index af7dc0b5..3af1bc16 100644 --- a/tests/config/test.yaml +++ b/tests/config/test.yaml @@ -33,7 +33,13 @@ input_data: topographic: topographic_filename: tests/data/europe_dem_2km_osgb.tif opticalflow: + history_minutes: 5 + forecast_minutes: 30 + opticalflow_zarr_path: tests/data/sat_data.zarr opticalflow_input_image_size_pixels: 32 + opticalflow_output_image_size_pixels: 8 + opticalflow_channels: + - IR_016 output_data: filepath: not used by unittests! process: From d96137768b3cf312791ea68c36062e14a3c2c565 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 13:04:19 +0000 Subject: [PATCH 173/197] log the correct data_source_name if an exception occurs! --- nowcasting_dataset/manager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 4cee7e75..01568a85 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -2,6 +2,7 @@ import logging import multiprocessing +from functools import partial from pathlib import Path from typing import Optional, Union @@ -485,10 +486,13 @@ def _callback(result): f"{data_source_name} has finished created batches for {split_name}!" ) - def _error_callback(exception): + def _error_callback(exception, data_source_name): + # Need to pass in data_source_name rather than rely on data_source_name + # in the outer scope, because otherwise the error message will contain + # the wrong data_source_name (due to stuff happening concurrently!) logger.exception( f"Exception raised by {data_source_name} whilst creating batches for" - f" {split_name}:\n{exception.__class__.__name__}: {exception}" + f" {split_name.value}\n{exception.__class__.__name__}: {exception}" ) an_error_has_occured.set() @@ -500,7 +504,7 @@ def _error_callback(exception): data_source.create_batches, kwds=kwargs_for_create_batches, callback=_callback, - error_callback=_error_callback, + error_callback=partial(_error_callback, data_source_name=data_source_name), ) async_results_from_create_batches.append(async_result) From 6ecaa3dd202420662ebe11283cdbe7fe9af70a24 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 13:13:03 +0000 Subject: [PATCH 174/197] All tests pass! --- nowcasting_dataset/data_sources/data_source.py | 2 ++ tests/data_sources/test_data_source.py | 10 ---------- 2 files changed, 2 insertions(+), 10 deletions(-) delete mode 100644 tests/data_sources/test_data_source.py diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 9fd8820a..8eb40588 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -377,6 +377,8 @@ class ImageDataSource(DataSource): """ Image Data source + Note that this is an abstract class. + Args: image_size_pixels: Size of the width and height of the image crop returned by get_sample(). diff --git a/tests/data_sources/test_data_source.py b/tests/data_sources/test_data_source.py deleted file mode 100644 index e4a51f16..00000000 --- a/tests/data_sources/test_data_source.py +++ /dev/null @@ -1,10 +0,0 @@ -from nowcasting_dataset.data_sources.data_source import ImageDataSource - - -def test_image_data_source(): - _ = ImageDataSource( - image_size_pixels=64, - meters_per_pixel=2000, - history_minutes=30, - forecast_minutes=60, - ) From 3cedd239519de9d0ec053dca4dfc41738cf60974 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 13:26:15 +0000 Subject: [PATCH 175/197] add notebook to plot optical flow batches --- notebooks/plot_optical_flow_batches.ipynb | 19727 ++++++++++++++++++++ 1 file changed, 19727 insertions(+) create mode 100644 notebooks/plot_optical_flow_batches.ipynb diff --git a/notebooks/plot_optical_flow_batches.ipynb b/notebooks/plot_optical_flow_batches.ipynb new file mode 100644 index 00000000..280654ee --- /dev/null +++ b/notebooks/plot_optical_flow_batches.ipynb @@ -0,0 +1,19727 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "cb2858bd-479d-43a8-a644-7fc4306f2798", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import xarray as xr\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1de03cdc-6187-4bae-9b93-f2623c48d1b6", + "metadata": {}, + "outputs": [], + "source": [ + "BASE_PATH = Path(\"/mnt/storage_ssd_4tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/prepared_ML_training_data/v15/test\")\n", + "SATELLITE_PATH = BASE_PATH / \"satellite\"\n", + "OPT_FLOW_PATH = BASE_PATH / \"opticalflow\"\n", + "BATCH_FILENAME = \"000170.nc\"\n", + "\n", + "assert SATELLITE_PATH.exists()\n", + "assert OPT_FLOW_PATH.exists()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "baa010e7-fba7-4e03-a427-e52e346a4373", + "metadata": {}, + "outputs": [], + "source": [ + "satellite_filename = SATELLITE_PATH / BATCH_FILENAME\n", + "opt_flow_filename = OPT_FLOW_PATH / BATCH_FILENAME\n", + "\n", + "assert satellite_filename.exists()\n", + "assert opt_flow_filename.exists()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4fcc6b66-f866-4c90-a07c-7eb953d0613a", + "metadata": {}, + "outputs": [], + "source": [ + "sat_batch = xr.load_dataset(satellite_filename, mode=\"r\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7f59a386-f28e-4c56-b60e-2fa05493c3a3", + "metadata": {}, + "outputs": [], + "source": [ + "opt_flow_batch = xr.load_dataset(opt_flow_filename, mode=\"r\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2ca3b5c0-b638-45c7-9a51-c73e6a8963e5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:         (example: 32, channels_index: 11, time_index: 31, x_index: 24, y_index: 24)\n",
+       "Coordinates:\n",
+       "  * channels_index  (channels_index) int64 0 1 2 3 4 5 6 7 8 9 10\n",
+       "  * example         (example) int64 0 1 2 3 4 5 6 7 ... 24 25 26 27 28 29 30 31\n",
+       "  * time_index      (time_index) int64 0 1 2 3 4 5 6 7 ... 24 25 26 27 28 29 30\n",
+       "  * x_index         (x_index) int64 0 1 2 3 4 5 6 7 ... 16 17 18 19 20 21 22 23\n",
+       "  * y_index         (y_index) int64 0 1 2 3 4 5 6 7 ... 16 17 18 19 20 21 22 23\n",
+       "Data variables:\n",
+       "    channels        (example, channels_index) object 'IR_016' ... 'WV_073'\n",
+       "    data            (example, time_index, x_index, y_index, channels_index) int16 ...\n",
+       "    time            (example, time_index) datetime64[ns] 2021-03-22T13:20:00 ...\n",
+       "    x               (example, x_index) float64 3.368e+05 3.401e+05 ... 5.419e+05\n",
+       "    y               (example, y_index) float64 6.92e+05 6.992e+05 ... 2.449e+05
" + ], + "text/plain": [ + "\n", + "Dimensions: (example: 32, channels_index: 11, time_index: 31, x_index: 24, y_index: 24)\n", + "Coordinates:\n", + " * channels_index (channels_index) int64 0 1 2 3 4 5 6 7 8 9 10\n", + " * example (example) int64 0 1 2 3 4 5 6 7 ... 24 25 26 27 28 29 30 31\n", + " * time_index (time_index) int64 0 1 2 3 4 5 6 7 ... 24 25 26 27 28 29 30\n", + " * x_index (x_index) int64 0 1 2 3 4 5 6 7 ... 16 17 18 19 20 21 22 23\n", + " * y_index (y_index) int64 0 1 2 3 4 5 6 7 ... 16 17 18 19 20 21 22 23\n", + "Data variables:\n", + " channels (example, channels_index) object 'IR_016' ... 'WV_073'\n", + " data (example, time_index, x_index, y_index, channels_index) int16 ...\n", + " time (example, time_index) datetime64[ns] 2021-03-22T13:20:00 ...\n", + " x (example, x_index) float64 3.368e+05 3.401e+05 ... 5.419e+05\n", + " y (example, y_index) float64 6.92e+05 6.992e+05 ... 2.449e+05" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sat_batch" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "dbb924d9-8591-47f8-9eea-239a1bf0dc63", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'zlib': False,\n", + " 'shuffle': False,\n", + " 'complevel': 0,\n", + " 'fletcher32': False,\n", + " 'contiguous': True,\n", + " 'chunksizes': None,\n", + " 'source': '/mnt/storage_ssd_4tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/prepared_ML_training_data/v15/test/satellite/000170.nc',\n", + " 'original_shape': (32, 31, 24, 24, 11),\n", + " 'dtype': dtype('int16')}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sat_batch[\"data\"].encoding" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "69c76de6-5fa5-4fd6-8496-0c6869079f67", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:         (example: 32, channels_index: 11, time_index: 24, x_index: 24, y_index: 24)\n",
+       "Coordinates:\n",
+       "  * channels_index  (channels_index) int64 0 1 2 3 4 5 6 7 8 9 10\n",
+       "  * example         (example) int64 0 1 2 3 4 5 6 7 ... 24 25 26 27 28 29 30 31\n",
+       "  * time_index      (time_index) int64 0 1 2 3 4 5 6 7 ... 17 18 19 20 21 22 23\n",
+       "  * x_index         (x_index) int64 0 1 2 3 4 5 6 7 ... 16 17 18 19 20 21 22 23\n",
+       "  * y_index         (y_index) int64 0 1 2 3 4 5 6 7 ... 16 17 18 19 20 21 22 23\n",
+       "Data variables:\n",
+       "    channels        (example, channels_index) object 'IR_016' ... 'WV_073'\n",
+       "    data            (example, time_index, x_index, y_index, channels_index) int16 ...\n",
+       "    time            (example, time_index) datetime64[ns] 2021-03-22T13:55:00 ...\n",
+       "    x               (example, x_index) float64 3.368e+05 3.401e+05 ... 5.419e+05\n",
+       "    y               (example, y_index) float64 6.92e+05 6.992e+05 ... 2.449e+05
" + ], + "text/plain": [ + "\n", + "Dimensions: (example: 32, channels_index: 11, time_index: 24, x_index: 24, y_index: 24)\n", + "Coordinates:\n", + " * channels_index (channels_index) int64 0 1 2 3 4 5 6 7 8 9 10\n", + " * example (example) int64 0 1 2 3 4 5 6 7 ... 24 25 26 27 28 29 30 31\n", + " * time_index (time_index) int64 0 1 2 3 4 5 6 7 ... 17 18 19 20 21 22 23\n", + " * x_index (x_index) int64 0 1 2 3 4 5 6 7 ... 16 17 18 19 20 21 22 23\n", + " * y_index (y_index) int64 0 1 2 3 4 5 6 7 ... 16 17 18 19 20 21 22 23\n", + "Data variables:\n", + " channels (example, channels_index) object 'IR_016' ... 'WV_073'\n", + " data (example, time_index, x_index, y_index, channels_index) int16 ...\n", + " time (example, time_index) datetime64[ns] 2021-03-22T13:55:00 ...\n", + " x (example, x_index) float64 3.368e+05 3.401e+05 ... 5.419e+05\n", + " y (example, y_index) float64 6.92e+05 6.992e+05 ... 2.449e+05" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "opt_flow_batch" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "56368a1b-3ca6-4d95-855a-aa8ecd92cc48", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'zlib': False,\n", + " 'shuffle': False,\n", + " 'complevel': 0,\n", + " 'fletcher32': False,\n", + " 'contiguous': False,\n", + " 'chunksizes': (8, 6, 6, 12, 6),\n", + " 'source': '/mnt/storage_ssd_4tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/prepared_ML_training_data/v15/test/opticalflow/000170.nc',\n", + " 'original_shape': (32, 24, 24, 24, 11),\n", + " 'dtype': dtype('int16')}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "opt_flow_batch[\"data\"].encoding" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "31f40bed-3faa-41ed-a207-9930c1522feb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAJcCAYAAAC8DwN/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABIMklEQVR4nO3de5hlVX3g/e+vqvoO3XRzE7tRUNAZYCIjiCQZ3xgxgMaIMTLpjMaO+gZlSIxmMolMJmI0vKPGPCa+GZkhkUc0RkBGA+OEwVZfk8kMFxvES6OEVgQakFtfaPpS3VX1e/84q+RQqa5zulln167u7+d59lPnrL3X76y169SpdX577b0jM5EkSVI7DM12AyRJkvQUB2eSJEkt4uBMkiSpRRycSZIktYiDM0mSpBZxcCZJktQiDs5URUT8WkT8w2y3Y6qI+GRE/NEzqL8+Il4+G6+tfRMRP4yIV852O/ZXRLwxIr7U8Gv6/pZayMGZNIPMPDkzvzbb7aghIs6KiO9FxI6I+P8i4rmz3aa5LiL+TUTcGxHbI+JvImJFn/WOi4iMiJHJssz8TGaePbjW/lO+v6V2cnAmHQQi4gjg88AfACuAdcDVs9qoOS4iTgb+K/CrwNHADuDjs9qog5Tvbx1oHJxpn0TEsRHx+Yh4NCIej4g/n7L+IxGxOSLuiYhXdZW/JSK+GxHbIuIHEfH2rnUvj4iNEfHvIuKRiHgoIt7Stf6TEfGfI+J/lPq3RMTzu9b/s4hYGxGbIuKuiPjXe2n7ERHxxYjYUrb9XxEx499A96GyiHhfRFwTEZ8q7VgfEad3bfsvI+L2su5qYOGUWK+JiDvK6/+fiPiJUv7LZZ8sLc9fFRE/iogjZ/xl7JvXA+sz83OZuQt4H/CiiPhn/VTe2z6OiOeXsheX58+OiMcmD5X1+Xv/3a7f++si4tUR8Y8l7n/o2v59EXFtRFxd4t0eES/aS3uHIuI9EfH98j69pt+s1j54I/DfM/PvM/NJOgOD10fEoX3U/fvyc0tEPBkRPxlTpgaUzNq/jYi7S38/UPb3TRHxROnT/K7tp31/zcT3t9RSmeni0tcCDAPfBD4KLKHz4fyvyrpfA/YAv162uxB4EIiy/ueB5wMB/AydLMOLy7qXA2PA+4F5wKvL+uVl/SeBTcAZwAjwGeCqsm4JcD/wlrLuxcBjwMlddf+oPP5PwH8przEPeNlk+2bo8w+BV5bH7wN2lfYNl3g3l3XzgXuBd5fYbyj7Y/K1Xww8Ary01F1TYi8o6z9T2np42W+vmaFNW2ZY3rOXOn8GXDal7DvAL/Xxe++1j38d+C6wGLgR+EhX3X5+7+8t++zXgUeBvwYOBU4u+/t5Xft/T9m384DfAe4B5k3zu3oXcDOwClhAJ8P12b307zk99um/2Uu964Dfm1L2JHBaH/v0OCCBka6yXwP+oet5AtcDS8u+GAW+AjwPWAbcCazp5/3l+9vFZW4ts94Al7mzAD9J55/nyDTrfg3Y0PV8cfnn8qy9xPob4LfK45cDO6f8o3oEOLM8/iTwl13rXg18rzz+ZeB/TYn9X4FLuupO/gN5P51/qCfsQ5+n/vP6cte6k4Cd5fH/RddgtJT9n67Xvgz4wJTYdwE/Ux4fBtwHfBv4rwP43X0C+OCUsv8N/FofdWfcx+X59aXt32KGAcFefu/D5fmh5T3z0q7tbwNe17X/b+5aNwQ8BLxsmt/Vd4GzurY9hs5g4p+8d5/BPv0K8I4pZQ8AL++j7nH0Nzj76Sn74ve6nv8J8Kf9vL98f7u4zK3Fw5raF8cC92bm2F7W/2jyQWbuKA8PgR8fyri5HKraQmeAdURX3cenxN0xWXdq7Cnrngu8tBxK2VJivxF41jTt+2NgA/ClcpjlPXvv6l5NbcfC6EzqfjbwQGZm1/p7ux4/F/h3U9p5bKlHZm4BPgecQuefbm1P0snAdFsKbOujbj/7+C/otP3/zczRycI+f+/j5fHO8vPhrvU7efr74P7JB5k5AWyk7MNp2vyFrvZ+FxinMzeslmeyT/s1dV/sbd/M+P7aBwfj+1tqHQdn2hf3A8+JrjPM+hERC4D/BnwEODozDwP+ls6hrhpt+rvMPKxrOSQzL5y6YWZuy8x/l5nPA34B+O2IOKtCG6CTwVkZEd19es6Udl46pZ2LM/OzABFxKvBW4LPAx2Z6oTJHaW/Lf9hLtfXAi7piLKFzuHF9H32bcR9HxCHAn9LJXrxvcm7XgH7vx3b1YYjOYcsH99LmV01p88LMfGDqhhHxnB779I17acvUffo8OodQ/7GPfmTvTfbJjO+vCg7k97fUOg7OtC9upfMh/cGIWBIRCyPip/uoN5/OP61HgbHonChQ65IBXwReEBG/GhHzyvKSiPjnUzcsE5ZPKP9gnqCTSRmfut1+uonO/Kl3RsRIRLyezhy5SX8BvCMiXhodSyLi5yPi0IhYCPwV8B/ozOtaGRH/dm8vVAZGe1v+n71U+wJwSkT8Unm99wLfyszvwY+vU/fDvdTttY//DLgtM/9v4H/QmdcHg/m9nxYRry9fEN5FZx7WzdNs91+AS6NcTiEijoyI86YLmJn39dinn9lLWz4D/EJEvKwMBt4PfD4zt5XXfF9EfG0vdR8FJujMH6thr++vSvHn9PtbmmscnKlv5fDTLwAn0Jk/spHOfKRe9bYB7wSuATYD/4bOHKUabdpG5x/+ajoZlB8BH6IzKJjqRODLdA6B3AR8PCtd4ykzd9M5Y+zX6PTxl+mc2j+5fh2dCe9/XtZvKNtCZ+L1xsy8rBwSfBPwRxFxYo22ldd/FPgl4NLy+i+ls88mHUtnjs50dfe6j8uA51zgHWXz3wZeHBFvHNDv/To6+3YznUtYvD4z90yz3Z+V1/pSRGyjM4B76TN87afJzPV0+v0ZOnMkDwW6Bx0z7dMddH4X/7scBjzzGbZlpvfXM3YAvL+lOWXyTDpJB7HoXJn+tzLzu7Pdlr2JiPfROZnjTbPdln5ExB10Tkp4fLbbImlu2ae5Q5IOTNnwlekPBpl56my3QdLc5OBMB7WIeA6d60VN56TMvK/J9kg1+f6W5iYPa0qSJLWIJwRIkiS1iIc1i0XLF+SyY5ZUiTUUE1XijFSK04lV54oRI9RrU0S9rO141vmeMVYpDsBwxf7Ni71d93ffTFTs30SVy9TBUMX31K6c33ujPk1knf4tHhrtvVHf6rQJ6n0mzK/XJLaO1/n91XpvAhw5PN3JwPtnNOvs8/tG690m9ol/fOSxzKx5n9MZnfOzS/LxTbWuYDSz2741emNmntvIi1Xm4KxYdswS3vjXr6wSa/Hw7ipxjhh5skocgCNHnqgSZ8VwvTYtHKr3obdp/JDeG/Vhy/jiKnEADh3aVS3Ws0a2VImzfWK6K4zsnx1ZJ9ahQzt7b9Sn743u6wXx9+7J8YW9N+rD6Yt/UCUO1PsSAnBUpb/lVSN1vjgA/O3251aJU3OQfsGy6a5xvH++v6fOPv/NH/zrKnEAbnz5x+7tvVU9j28a59Ybn9N7wwqGj7n7iN5btZODM0mS1IgEJipmyw9UzjmTJElqETNnkiSpIcl4mjnrxcyZJElSizg4kyRJahEPa0qSpEZ0Tgjw4ve9mDmTJElqETNnkiSpMV5KozczZ5IkSS1i5kySJDUiScbTOWe9mDmTJElqETNnkiSpMZ6t2ZuZM0mSpBYxcyZJkhqRwLiZs54GmjmLiHdHxPqI+E5EfDYiFpby34yIu8q6D3dtf3FEbCjrzukqPy0ivl3WfSwiopQviIirS/ktEXFcV501EXF3WdYMsp+SJEm1DCxzFhErgXcCJ2Xmzoi4BlgdEfcC5wE/kZmjEXFU2f4kYDVwMvBs4MsR8YLMHAcuAy4Abgb+FjgXuAF4G7A5M0+IiNXAh4BfjogVwCXA6XQG6rdFxPWZuXlQ/ZUkSb0556y3Qc85GwEWRcQIsBh4ELgQ+GBmjgJk5iNl2/OAqzJzNDPvATYAZ0TEMcDSzLwpMxP4FPC6rjpXlsfXAmeVrNo5wNrM3FQGZGvpDOgkSZJabWCDs8x8APgIcB/wELA1M78EvAB4WTkM+XcR8ZJSZSVwf1eIjaVsZXk8tfxpdTJzDNgKHD5DrKeJiAsiYl1ErNuxZfSZdFeSJPWQwHhmI0s/IuKHZdrUHRGxrpStiIi1ZVrU2ohY3rX9tNOvahvY4Kx05jzgeDqHKZdExJvoZNOWA2cC/x64pmS7YpowOUM5+1nnqYLMyzPz9Mw8ffFhC3r0SJIkHYB+NjNPzczTy/P3AF/JzBOBr5TnU6dfnQt8PCKGB9GgQR7WfCVwT2Y+mpl7gM8DP0Uni/X57LgVmACOKOXHdtVfRecw6MbyeGo53XXKodNlwKYZYkmSpFk00dDyDHRPmbqSp0+l+ifTr57ZS01vkIOz+4AzI2JxyYydBXwX+BvgFQAR8QJgPvAYcD2dEwYWRMTxwInArZn5ELAtIs4scd4MXFde43pg8kzMNwBfLfPSbgTOjojlJYN3dimTJEkHhyMmpy6V5YJptkngSxFxW9f6o8vYg/LzqFLe15SpGgZ2tmZm3hIR1wK3A2PAN4DL6eyIKyLiO8BuYE0ZUK0vZ3TeWba/qJypCZ2TCD4JLKJzluYNpfwTwKcjYgOdjNnq8tqbIuIDwNfLdu/PzE0ztXfe0DjHzN/6zDsOzIvx3hv14ciRJ6rEAVgx/GSVOAuH9lSJA7BlfHHrYu2amFclDsDurPfndejQzipxNo0fUiUOwOOVYh03f6xKHIDhZ/p9ucshw7uqxHnR/Hp/x8PTztjYP3fuWVglzraJep8JSyvt82seeEnvjfo0kfX2+fPmP9J7oz784NHDq8Q5CDzWdahyb346Mx8sV45YGxHfm2HbvqZM1TDQi9Bm5iV0Lmkx1Zv2sv2lwKXTlK8DTpmmfBdw/l5iXQFcsS/tlSRJg5Nkqy5Cm5kPlp+PRMQX6BymfDgijsnMh8oVIyZH1Y1NmfL2TZIk6aATEUsi4tDJx3SmQH2Hp0+ZWsPTp1L9k+lXg2ibt2+SJEnNSBhvT+LsaOAL5aZDI8BfZ+b/jIiv07mSxNvozJ8/HyAzZ5p+VZWDM0mSdNDJzB8AL5qm/HE6JzFOV2fa6Ve1OTiTJEmNSJ7xZS4OCs45kyRJahEzZ5IkqSHBeMVLwhyozJxJkiS1iJkzSZLUiAQm2nO2ZmuZOZMkSWoRM2eSJKkxzjnrzcyZJElSi5g5kyRJjUjMnPXDzJkkSVKLmDmTJEmNmUgzZ72YOZMkSWoRB2eSJEkt4mFNSZLUCE8I6I+ZM0mSpBYxcyZJkhqRBOPmhXpyD0mSJLWImbNihHGOGHmiSqznzNtUJc7uHK4SB2DLxOIqcXaMLagSB+BvH/sX1WItm7erSpznLnq8ShyAmzcdXy3WqYdtrBLnhIUPV4kDsGNifpU4hw3tqBIH4F8t3Vwt1oY9df7+fjBWZz8BHDeyu1qs9aOrqsT50fCTVeIAHF4p1uM763zeAVz/8IuqxXpg67IqcUZuO7RKnNnipTR6M3MmSZLUImbOJElSIzxbsz9mziRJklrEzJkkSWpIMJ7mhXpxD0mSJLWImTNJktSIBCbMC/XkHpIkSWoRM2eSJKkxnq3Zm5kzSZKkFjFzJkmSGpHp2Zr9cA9JkiS1iIMzSZKkFvGwpiRJasyEJwT0ZOZMkiSpRcycSZKkRnRufG5eqBf3kCRJUouYOZMkSQ3xUhr9cA9JkiS1iJkzSZLUCG983h/3kCRJUouYOZMkSY0ZT69z1ouZM0mSpBYxc1YMxwSHDe+oEmtFpThffOJFVeIA3Lr5uCpxls7fWSUOwG0/fE61WENDWSXOcUevqBIH4Pv3H1Ut1obFR1SJc96J364SB+CY+VuqxPnLR36mShyAJcO7q8V61oKtVeIMUee9CfDChQ9Vi/Xlx/95lTiP7Di0ShyAlxxxb5U4jzy2tEocgEeiXqzFS0arxImJKmFmRRJe56wP7iFJkqQWMXMmSZIaM+F1znpyD0mSJLWImTNJktQI763ZH/eQJElSizg4kyRJahEPa0qSpEYk4UVo+2DmTJIkqUXMnEmSpMZ44/PeBrqHIuLdEbE+Ir4TEZ+NiIVd634nIjIijugquzgiNkTEXRFxTlf5aRHx7bLuYxERpXxBRFxdym+JiOO66qyJiLvLsmaQ/ZQkSaplYIOziFgJvBM4PTNPAYaB1WXdscDPAfd1bX9SWX8ycC7w8YgYLqsvAy4ATizLuaX8bcDmzDwB+CjwoRJrBXAJ8FLgDOCSiFg+qL5KkqTeMmE8hxpZ5rJBt34EWBQRI8Bi4MFS/lHgd+FpN507D7gqM0cz8x5gA3BGRBwDLM3MmzIzgU8Br+uqc2V5fC1wVsmqnQOszcxNmbkZWMtTAzpJkqTWGtics8x8ICI+Qic7thP4UmZ+KSJeCzyQmd8sRycnrQRu7nq+sZTtKY+nlk/Wub+83lhEbAUO7y6fps6PRcQFdDJyHPnsefvZU0mS1J9gAs/W7GWQhzWX08lsHQ88G1gSEW8Gfh9473RVpinLGcr3t85TBZmXZ+bpmXn60hWeGyFJkmbfIEckrwTuycxHASLi88Bb6AzWJrNmq4DbI+IMOtmtY7vqr6JzGHRjeTy1nK46G8uh02XAplL+8il1vlava5IkaV8lzPn5YE0Y5B66DzgzIhaXeWBnAZ/PzKMy87jMPI7OIOrFmfkj4HpgdTkD83g6E/9vzcyHgG0RcWaJ82bguvIa1wOTZ2K+AfhqmZd2I3B2RCwvGbyzS5kkSVKrDXLO2S0RcS1wOzAGfAO4fIbt10fENcCdZfuLMnO8rL4Q+CSwCLihLACfAD4dERvoZMxWl1ibIuIDwNfLdu/PzE0VuydJkvaDNz7vbaATrTLzEjqXtNjb+uOmPL8UuHSa7dYBp0xTvgs4fy+xrwCu2LcWS5IkzS5nwUuSpEYkwYT31uzJ3KIkSVKLmDmTJEmNcc5Zb+4hSZKkFjFzVoznEFvGF1eJ9YNKY97vbT+6ShyAezavqBJn/sh47436NG/+WLVYUWkKw8ZNh9UJBCw8ZLRarJFK+33d48+pEgdgybxnVYnzvQfrvc/Hx+t931xx2PY6cRbtqBIH4EfLllaLtXx+nXa94JBHqsQBeONht1SJ85KX/qBKHID1O1f13qhPy4Z3Vonz9ytOrBIH4M4PVwulihycSZKkRiQw4UVoe3IPSZIktYiZM0mS1JBg3Buf92TmTJIkqUXMnEmSpEY456w/7iFJkqQWMXMmSZIa45yz3sycSZIktYiZM0mS1IjMcM5ZH9xDkiRJLWLmTJIkNWbczFlP7iFJkqQWMXMmSZIakcCEZ2v2ZOZMkiSpRcycSZKkhoRzzvrgHpIkSWoRM2eSJKkRnXtrOuesFzNnkiRJLeLgTJIkqUU8rClJkhozbl6oJ/eQJElSi5g5K8YYYtP4IVVi3bf7iCpxfvjE4VXiAOwZG64SZ/uOBVXiAExsqhcr9tSZYDqyvd5E1bHFWS3W7mftqhLnwT31/uT37K4Ta3xXnfcmwMKlo9Vibd81v0qcJ3fWe58/un1JtVijld4LQ0P13uffO+LoKnGeu3hTlTgAC4bGqsX64Z46n+lL5++sEmc2JOEJAX0wcyZJktQiZs4kSVJjJswL9eQekiRJahEzZ5IkqRGZMO6cs57MnEmSJLWImTNJktQYz9bszcyZJElSi5g5kyRJjehc58y8UC/uIUmSpBYxcyZJkhozjnPOejFzJkmS1CJmziRJUiMSz9bsh5kzSZKkFnFwJkmS1CIe1pQkSQ3xUhr9cA9JkqSDUkQMR8Q3IuKL5fn7IuKBiLijLK/u2vbiiNgQEXdFxDmDbJeZM0mS1JiJdl1K47eA7wJLu8o+mpkf6d4oIk4CVgMnA88GvhwRL8jM8UE0ysyZJEk66ETEKuDngb/sY/PzgKsyczQz7wE2AGcMqm0OziRJUiMyYTyjkQU4IiLWdS0XTGnOnwK/C0xMKf+NiPhWRFwREctL2Urg/q5tNpaygXBwJkmSDkSPZebpXcvlkysi4jXAI5l525Q6lwHPB04FHgL+ZLLKNPFzAG0GnHMmSZIa1JKzNX8aeG2Z8L8QWBoRf5WZb5rcICL+AvhieboROLar/irgwUE1rhV7SJIkqSmZeXFmrsrM4+hM9P9qZr4pIo7p2uwXge+Ux9cDqyNiQUQcD5wI3Dqo9pk5K7bsXszfPHBqlViL5+2uEueRrYdUiQMwumlRlThDO+uN55dtqBdr3vY62eXRZfXOItr5rGqh2LN9XpU44yPDVeIAzFs4ViXO4sN3VYkDcPiSHdVizR+qcxLW5l11/vYAtmyrF2tkfZ3Pl6i3y7njxQuqxNm9qt6/tvu3HFYtVq3bFs0fGcgJgo1Iou23b/pwRJxK55DlD4G3A2Tm+oi4BrgTGAMuGtSZmuDgTJIkHcQy82vA18rjX51hu0uBS5tok4MzSZLUmJZd56yVBjrnLCLeHRHrI+I7EfHZiFgYEX8cEd8rp6l+ISIO69p+2qvvRsRpEfHtsu5jERGlfEFEXF3Kb4mI47rqrImIu8uyZpD9lCRJqmVgg7OIWAm8Ezg9M08BhulMulsLnJKZPwH8I3Bx2b776rvnAh+PiMkJMpcBF9CZgHdiWQ/wNmBzZp4AfBT4UIm1ArgEeCmdi8Rd0nWtEkmSNAuSzty7Jpa5bNBna44AiyJiBFgMPJiZX8rMyZnEN9M5HRX2cvXdcubE0sy8KTMT+BTwuq46V5bH1wJnlazaOcDazNyUmZvpDAgnB3SSJEmtNbA5Z5n5QER8BLgP2Al8KTO/NGWztwJXl8cr6QzWJk1efXdPeTy1fLLO/eX1xiJiK3A4fV7Jt1wt+AKABUcduo89lCRJ+6ol1zlrtUEe1lxOJ7N1PJ2bhC6JiO6Lu/0+ndNRPzNZNE2YnKF8f+s8VZB5+eSVg+ctW7y3rkiSJDVmkMPXVwL3ZOajmbkH+DzwU9CZrA+8BnhjOVQJe7/67kaeOvTZXf60OuXQ6TJg0wyxJEmSWm2Qg7P7gDMjYnGZB3YW8N2IOBf4PeC1mdl9+cJpr76bmQ8B2yLizBLnzcB1XXUmz8R8A50r/CZwI3B2RCwvGbyzS5kkSZotDZ0MMNdPCBjknLNbIuJa4HY6hy+/AVwOrAcWAGvLFTFuzsx39Lj67oXAJ4FFwA1lAfgE8OmI2EAnY7a6vPamiPgA8PWy3fszc9Og+ipJklTLQC9Cm5mX0LmkRbcTZth+2qvvZuY64JRpyncB5+8l1hXAFfvSXkmSNDiJF6Hth6dMSJIktYi3b5IkSY2Z6/PBmmDmTJIkqUXMnEmSpEZM3r5JMzNzJkmS1CJmziRJUmPMnPVm5kySJKlFzJxJkqRGJHP/6v1NMHMmSZLUImbOJElSY7xDQG9mziRJklrEzFmxe+c87v/Os6rEmlg2ViXO8KZ5VeIALNpa55vK2OKsEgdg59H1Ym1dMVElTizbXSVOJ1i9UAvm13lPRcU2zZtXp00jQ3V+dwAbH11eLdbEeJ2dtWhJvffUIYtHq8XackKdj//cWe/fyPyR8SpxNm5dViUOwNaN9WLFnjrvqVxR8XOqaenZmv0wcyZJktQiDs4kSZJaxMOakiSpEd6+qT9mziRJklrEzJkkSWqMmbPezJxJkiS1iJkzSZLUCG/f1B8zZ5IkSS1i5kySJDUmzZz1ZOZMkiSpRcycSZKkxnjj897MnEmSJLWImTNJktSI9MbnfTFzJkmS1CJmziRJUmM8W7M3M2eSJEktYuZMkiQ1xDsE9MPMmSRJUos4OJMkSWoRD2tKkqTGeEJAb2bOJEmSWsTMWREJw7vrjOZj07wqcYb2VAkDwO5lWSXO+OH1GjVvUb1Yhy3ZVS1WLWMT9b77jAxNVImzc3R+lTgAT25ZXCfQ9nofQwsfHK4Wa2xJnb+ZeSfvrBIH4KQjHq4Wa+LwOp93O8bqvace3nFIlTiPb6kTByD2tC/LE0N13puzIfEitP0wcyZJktQiZs4kSVIzsnMLJ83MzJkkSVKLmDmTJEmNmcA5Z72YOZMkSWoRM2eSJKkRidc564eZM0mSpBYxcyZJkhrijc/7YeZMkiSpRcycSZKkxnids97MnEmSJLWImTNJktQYz9bszcyZJElSizg4kyRJahEPa0qSpEZkelizH2bOJEmSWsTMmSRJaowXoe3NzJkkSVKLDHRwFhHvjoj1EfGdiPhsRCyMiBURsTYi7i4/l3dtf3FEbIiIuyLinK7y0yLi22XdxyIiSvmCiLi6lN8SEcd11VlTXuPuiFgzyH5KkqT+dOadDX6ZywY2OIuIlcA7gdMz8xRgGFgNvAf4SmaeCHylPCciTirrTwbOBT4eEcMl3GXABcCJZTm3lL8N2JyZJwAfBT5UYq0ALgFeCpwBXNI9CJQkSWqrQR/WHAEWRcQIsBh4EDgPuLKsvxJ4XXl8HnBVZo5m5j3ABuCMiDgGWJqZN2VmAp+aUmcy1rXAWSWrdg6wNjM3ZeZmYC1PDegkSdIsyYxGlrlsYIOzzHwA+AhwH/AQsDUzvwQcnZkPlW0eAo4qVVYC93eF2FjKVpbHU8ufViczx4CtwOEzxHqaiLggItZFxLrx7dv3v7OSJEmVDOxszXIY8TzgeGAL8LmIeNNMVaYpyxnK97fOUwWZlwOXAyw4blXuOXLPDM3bB3vqjHnHo95B8+FDxqrEWbJod5U4AAvmVdrfwJL5dWINVdzno+PDvTfq09bti6rE2bV5YZU4AEzU+WY6vHy0ShyAPGq8Wqxa76mjD91WJQ7AjrF51WLNH66zr0aG6u3z3WN1/mZqZk1yeb3PvByv879hLp/Jl8z9rFYTBvk7fiVwT2Y+mpl7gM8DPwU8XA5VUn4+UrbfCBzbVX8VncOgG8vjqeVPq1MOnS4DNs0QS5IkqdUGOTi7DzgzIhaXeWBnAd8Frgcmz55cA1xXHl8PrC5nYB5PZ+L/reXQ57aIOLPEefOUOpOx3gB8tcxLuxE4OyKWlwze2aVMkiTNomxomcsGdlgzM2+JiGuB24Ex4Bt0DiEeAlwTEW+jM4A7v2y/PiKuAe4s21+UmZP58guBTwKLgBvKAvAJ4NMRsYFOxmx1ibUpIj4AfL1s9/7M3DSovkqSJNUy0DsEZOYldC5p0W2UThZtuu0vBS6dpnwdcMo05bsog7tp1l0BXLGPTZYkSYPivTX7MpfnFUqSJB1wvLemJElqzlyfENYAM2eSJEkt4uBMkiSpRTysKUmSGuMJAb2ZOZMkSWoRM2eSJKkx6QkBPZk5kyRJahEzZ5IkqRGJc876YeZMkiSpRcycSZKkZiRg5qwnM2eSJEktYuZMkiQ1xrM1ezNzJkmS1CJmziRJUnPMnPVk5kySJKlFzJxJkqSGhNc564OZM0mSpBYxczZpIognK+2OZXuqhMk99cbO46PDVeLsinlV4gCMjdXr3+6xOr+7oag3GWLXaMV9tbvO7y8WjFeJAzAyv06sZx++tUocgGcvqRdrLOu8P3eN13sfbN29qFqs8Yk6/du8o16btm1ZXCdQxb/jBYvrfJ4DDA1NVImzZ/cc/9ftnLOezJxJkiS1iIMzSZKkFpnjuVFJkjRnpDc+74eZM0mSpBYxcyZJkprjCQE9mTmTJElqETNnkiSpQc4568XMmSRJUouYOZMkSc1xzllPZs4kSZJaxMyZJElqjpmznsycSZKkg1JEDEfENyLii+X5iohYGxF3l5/Lu7a9OCI2RMRdEXHOINvl4EySJDUjgYxmlv78FvDdrufvAb6SmScCXynPiYiTgNXAycC5wMcjYrjWbpnKwZkkSTroRMQq4OeBv+wqPg+4sjy+EnhdV/lVmTmamfcAG4AzBtU255xJkqTGZHNzzo6IiHVdzy/PzMu7nv8p8LvAoV1lR2fmQwCZ+VBEHFXKVwI3d223sZQNhIMzSZJ0IHosM0+fbkVEvAZ4JDNvi4iX9xFruuOkAxtmOjiTJEnNacfZmj8NvDYiXg0sBJZGxF8BD0fEMSVrdgzwSNl+I3BsV/1VwIODapxzziRJ0kElMy/OzFWZeRydif5fzcw3AdcDa8pma4DryuPrgdURsSAijgdOBG4dVPvMnEmSJHV8ELgmIt4G3AecD5CZ6yPiGuBOYAy4KDPHB9UIB2eSJKk5/V/mohGZ+TXga+Xx48BZe9nuUuDSJtrkYU1JkqQWMXNWDO2BRT+qM1bdOVxntw7vrDh2jjozMHOo3ltmrOKk0NHFE3UCza8UB2C84rfDiTqxYvFYlTgAhx6ys0qcFQu3V4kDMFTpfQ7w5O4FVeJMVMwSHHfIpmqx9mSdz5exiSOqxAHYPr/OPs9Kfy8A8+bV+5uZN1znKFi0K/G0zyr+mR6wzJxJkiS1iJkzSZLUjKQtl9JoNTNnkiRJLWLmTJIkNWSfbkp+0DJzJkmS1CJmziRJUnOcc9aTmTNJkqQWMXMmSZKaY+asJzNnkiRJLWLmTJIkNcfMWU9mziRJklpkYIOziHhhRNzRtTwREe+KiFMj4uZSti4izuiqc3FEbIiIuyLinK7y0yLi22XdxyI6dxaLiAURcXUpvyUijuuqsyYi7i7LmkH1U5Ik9SnpXOesiWUOG9jgLDPvysxTM/NU4DRgB/AF4MPAH5by95bnRMRJwGrgZOBc4OMRMVzCXQZcAJxYlnNL+duAzZl5AvBR4EMl1grgEuClwBnAJRGxfFB9lSRJqqWpw5pnAd/PzHvpjJuXlvJlwIPl8XnAVZk5mpn3ABuAMyLiGGBpZt6UmQl8CnhdV50ry+NrgbNKVu0cYG1mbsrMzcBanhrQSZIktVZTJwSsBj5bHr8LuDEiPkJncPhTpXwlcHNXnY2lbE95PLV8ss79AJk5FhFbgcO7y6ep82MRcQGdjBwjS02sSZI0aHGQnBAQEQvpHOE7GVg4WZ6Zb+1Vd+CZs4iYD7wW+FwpuhB4d2YeC7wb+MTkptNUzxnK97fOUwWZl2fm6Zl5+sjiJXvvhCRJ0r75NPAsOkfz/g5YBWzrp2IThzVfBdyemQ+X52uAz5fHn6MzJww62a1ju+qtonPIc2N5PLX8aXUiYoTOYdJNM8SSJEmzKRtaZt8JmfkHwPbMvBL4eeBf9FOxicHZr/DUIU3oDJJ+pjx+BXB3eXw9sLqcgXk8nYn/t2bmQ8C2iDizzCd7M3BdV53JMzHfAHy1zEu7ETg7IpaXEwHOLmWSJElN2FN+bomIU+gkkI7rp+JA55xFxGLg54C3dxX/OvBnJdO1izLnKzPXR8Q1wJ3AGHBRZo6XOhcCnwQWATeUBTqHRD8dERvoZMxWl1ibIuIDwNfLdu/PzE0D6aQkSdI/dXlJEP1HOsmkQ4A/6KfiQAdnmbmDzgT97rJ/oHNpjem2vxS4dJrydcAp05TvAs7fS6wrgCv2vdWSJEnP2FfKFSP+HngeQDky2JN3CJAkSY2JbGZpgf82Tdm1/VTsmTmLiN8APlNGf5IkSdqLiPhndC6fsSwiXt+1aildl9SYST+HNZ8FfD0ibqdzmPDGMun+gJJDMLGgUrBKd40YP2J3nUDAvIVjVeKMj9VLto5vm1ctVq19zni9W37EnoqJ6fHem/QjK85k2Dpc5/Iz39ne12dV48ZG6+yr+Yv29N6oT0NH1vvo/dG2Q6vE2bKl3mWIstLnS2wf7r1Rn7ZV/JwaXlrnvTDn/wPP8Vsr9eGFwGuAw4Bf6CrfRmfefU89P30y8z9GxB/QOePxLcCfl4n7n8jM7+9riyVJkg5UmXkdcF1E/GRm3rQ/Mfr6apiZGRE/An5E50zK5cC1EbE2M393f15YkiQdZNpzDbImfCMiLmIQdwiIiHdGxG10blD+v4F/kZkX0jnj8pf2u8mSJEkHrv2+Q0A/mbMjgNeXm5b/WGZORMRr9rGhkiTpYHbwZM5OyMzzI+K8zLwyIv6aPi+I38+cs/fOsO67+9BISZKkg8XUOwT8iDbcIUCSJKlbS65B1oR23iFAkiTpYBIRv9319C3l538uP/u69oyDM0mS1JwDP3M2eRHBFwIvoZM1g841z/6+nwAOziRJkirJzD8EiIgvAS/OzG3l+fuAz/UTw3trSpIk1fccoPtWP7vxhABJktQ6B/5hzUmfBm6NiC/Q6fUvAlf2U9HBmSRJUmWZeWlE3AC8rBS9JTO/0U9dB2eSJKkRkQfVpTTIzNuB2/e1nnPOJEmSWsTMmSRJak7GbLeg9cycSZIktYiZM0mS1JyDaM7Z/jJzJkmS1CJmziRJUmMOprM195eZM0mSpBYxcyZJkppj5qwnM2eSJEktYuZs0sIJxl+4vU6oeeNV4kTFA/NR6bIye4aG6wQCxg6tFoqJPXXaFU/W69/IjnrffYZ31fkFRp23JgB7Dq2zr/YcvqdKHIBDVuyoFmvRwjrt2jNW7z31g0cPrxZr98OLq8RZ9FC9/s3bVifO8O56n53j8+tdk2v3sjr/cnedMFolzqw4yO4QsL/MnEmSJLWImTNJktQcM2c9mTmTJElqEQdnkiRJLeJhTUmS1BwPa/Zk5kySJKlFzJxJkqTGeCmN3sycSZIktYiDM0mSpBZxcCZJktQizjmTJEnNcc5ZT2bOJEmSWsTMmSRJaoY3Pu+LmTNJkqQWMXMmSZKaY+asJzNnkiRJLWLmTJIkNcfMWU9mziRJklrEzJkkSWpE4Nma/TBzJkmS1CIOziRJklrEw5rFvJFxjj1yc5VYe8aHq8TZvntelTgAO0fnV4kztrtO34BOfruS4QXjVeJMPFmvf8M76nVw0aN1jgMMj1YJA8ATz6sTZ8Gh9Rr1/BWPV4u1fazO38y9j6yoEgdg7NGF1WItfrDOe33B5nrHqBY9NlElzvytY1XiAEzMq5fDGD2szj4fX1jnvTlrPKzZk5kzSZKkFjFzJkmSmuHtm/pi5kySJKlFzJxJkqTmmDnraWCZs4h4YUTc0bU8ERHvKut+MyLuioj1EfHhrjoXR8SGsu6crvLTIuLbZd3HIiJK+YKIuLqU3xIRx3XVWRMRd5dlzaD6KUmSVNPAMmeZeRdwKkBEDAMPAF+IiJ8FzgN+IjNHI+Koss1JwGrgZODZwJcj4gWZOQ5cBlwA3Az8LXAucAPwNmBzZp4QEauBDwG/HBErgEuA0+mM0W+LiOszs87pmJIkaf+YOeupqTlnZwHfz8x7gQuBD2bmKEBmPlK2OQ+4KjNHM/MeYANwRkQcAyzNzJsyM4FPAa/rqnNleXwtcFbJqp0DrM3MTWVAtpbOgE6SJKnVmhqcrQY+Wx6/AHhZOQz5dxHxklK+Eri/q87GUrayPJ5a/rQ6mTkGbAUOnyHW00TEBRGxLiLW7dmy4xl0T5Ik9SOymWUuG/jgLCLmA68FPleKRoDlwJnAvweuKdmu6a7YmTOUs591nirIvDwzT8/M0+cdtnjGfkiSJDWhiczZq4DbM/Ph8nwj8PnsuBWYAI4o5cd21VsFPFjKV01TTnediBgBlgGbZoglSZJmUza0zGFNDM5+hacOaQL8DfAKgIh4ATAfeAy4HlhdzsA8HjgRuDUzHwK2RcSZJcP2ZuC6Eut6YPJMzDcAXy3z0m4Ezo6I5RGxHDi7lEmSJLXaQK9zFhGLgZ8D3t5VfAVwRUR8B9gNrCkDqvURcQ1wJzAGXFTO1ITOSQSfBBbROUvzhlL+CeDTEbGBTsZsNUBmboqIDwBfL9u9PzM3DaaXkiSpLwdAVqsJAx2cZeYOOhP0u8t2A2/ay/aXApdOU74OOGWa8l3A+XuJdQWdgaAkSdKc4R0CJElSY+b6mZRN8N6akiRJLeLgTJIkqUU8rClJkprjYc2ezJxJkiS1iJkzSZLUGE8I6M3MmSRJUouYOZMkSc0xc9aTmTNJkqQWMXNWzBsa58hFT1aJ9eSeBVXiDA9NVIkDMBR14oyN1RvPDw/X+/q0asWWKnH2HDlcJQ7AvfcfUS1W5PwqcRY+Vm+fz99S5021895DqsQB+Oa2On97AIzV6V/sqfc3M29rvVi13gvDu6uEAWDX8jr92350vfdB1vtIYN72Ovt8+XfncOrJ2zf1xcyZJElSi5g5kyRJjYiyaGZmziRJklrEzJkkSWqOc856MnMmSZLUImbOJElSY7xDQG9mziRJklrEzJkkSWqOmbOezJxJkqSDTkQsjIhbI+KbEbE+Iv6wlL8vIh6IiDvK8uquOhdHxIaIuCsizhlU28ycSZKkg9Eo8IrMfDIi5gH/EBE3lHUfzcyPdG8cEScBq4GTgWcDX46IF2TmeO2GmTmTJEnNyYaWXs3omLxv47yyzFTzPOCqzBzNzHuADcAZ/XV63zg4kyRJB6IjImJd13LB1A0iYjgi7gAeAdZm5i1l1W9ExLci4oqIWF7KVgL3d1XfWMqq87CmJElqRjZ6KY3HMvP0mTYohyRPjYjDgC9ExCnAZcAH6GTRPgD8CfBWpr/z1EB6Y+ZMkiQd1DJzC/A14NzMfDgzxzNzAvgLnjp0uRE4tqvaKuDBQbTHwZkkSWpOS+acRcSRJWNGRCwCXgl8LyKO6drsF4HvlMfXA6sjYkFEHA+cCNy67zugNw9rSpKkg9ExwJURMUwnWXVNZn4xIj4dEafSGeL9EHg7QGauj4hrgDuBMeCiQZypCQ7OJElSg9py+6bM/BbwL6cp/9UZ6lwKXDrIdoGHNSVJklrFzJkkSWpOSzJnbWbmTJIkqUXMnEmSpMa0Zc5Zm5k5kyRJahEzZ12GWnYgfP5QvTN0ly7cVSXOgpGxKnFqx1q1ZEuVOFt3L6wSB+DBJUurxRpdUedPdXhnve9jC7bW+XsZ2jPdRbf3z9im+dViZaVdlS39lB09rE6cih9TTFTaVzlcJw5A1nt7MrakTrDh0YqNalqf1yA72Jk5kyRJapGWfqeTJEkHJDNnPZk5kyRJahEHZ5IkSS3iYU1JktSIwEtp9MPMmSRJUouYOZMkSc0xc9aTmTNJkqQWMXMmSZIaE2nqrBczZ5IkSS1i5kySJDXD2zf1xcyZJElSi5g5kyRJjfE6Z72ZOZMkSWoRM2eSJKk5Zs56MnMmSZLUImbOJElSY5xz1puZM0mSpBYxcyZJkppj5qyngWXOIuKFEXFH1/JERLyra/3vRERGxBFdZRdHxIaIuCsizukqPy0ivl3WfSwiopQviIirS/ktEXFcV501EXF3WdYMqp+SJEk1DWxwlpl3ZeapmXkqcBqwA/gCQEQcC/wccN/k9hFxErAaOBk4F/h4RAyX1ZcBFwAnluXcUv42YHNmngB8FPhQibUCuAR4KXAGcElELB9UXyVJkmpp6rDmWcD3M/Pe8vyjwO8C13Vtcx5wVWaOAvdExAbgjIj4IbA0M28CiIhPAa8Dbih13lfqXwv8ecmqnQOszcxNpc5aOgO6z+6tgXvGh3lw+7Jn3lPgsAU7q8SZP2+sShyAkaGJKnEOHRmtEgdgT9b7bvDwzkOrxPnBo4dXiQOwZ/PCarGGx+vEGV9UJw7AruGoEieHe2/Tf7B6oeZtrxNnaHedOADU2eUADI3V2VlDe6qEAWBsYZ0Oji2uEgaAqLjPo9Lfcc193rj0hIB+NHVCwGrKwCgiXgs8kJnfnLLNSuD+rucbS9nK8nhq+dPqZOYYsBU4fIZYTxMRF0TEuohYt2frjv3rmSRJUkUDz5xFxHzgtcDFEbEY+H3g7Ok2naYsZyjf3zpPFWReDlwOcOgLnuVYXpKkQfO/bU9NZM5eBdyemQ8DzweOB75ZDleuAm6PiGfRyW4d21VvFfBgKV81TTnddSJiBFgGbJohliRJUqs1MTj7Fcohzcz8dmYelZnHZeZxdAZRL87MHwHXA6vLGZjH05n4f2tmPgRsi4gzy3yyN/PUXLXrgckzMd8AfDUzE7gRODsilpcTAc4uZZIkaZYEnTlnTSxz2UAPa5bDmD8HvL3Xtpm5PiKuAe4ExoCLMnNy+uSFwCeBRXROBLihlH8C+HQ5eWATnbltZOamiPgA8PWy3fsnTw6QJElqs4EOzjJzB50J+ntbf9yU55cCl06z3TrglGnKdwHn7yX2FcAV+9ZiSZI0UDnH01oN8PZNkiRJLeLtmyRJUmPm+nywJpg5kyRJahEzZ5IkqRmJ1znrg5kzSZKkFjFzJkmSGhN1bvV8QDNzJkmS1CJmziRJUnOcc9aTmTNJkqQWcXAmSZLUIh7WlCRJjfEitL2ZOZMkSWoRM2eSJKkZiTc+74OZM0mSpBYxcyZJkhrjnLPezJxJkiS1iJmzYs/4MA9tXlol1lErt1WJs3B4T5U4ABNZZxy+bWxBlTgA92w5vFqsxx49tEqc2FHvTyLGol6siTqxxpbU+8o6HHXatGBLlTAlVr3+zd9W5x4z4/PrvQ92HlHv+3SljwRGdrVvn+9ZXG8/jS2qFoocqvdemNPMnPVk5kySJKlFzJxJkqRGBM4564eZM0mSpBYxcyZJkpqR6XXO+mDmTJIkqUXMnEmSpMY456w3M2eSJEktYuZMkiQ1x8xZT2bOJEmSWsTBmSRJUot4WFOSJDXGEwJ6M3MmSZLUImbOJElSMxKYMHXWi5kzSZKkFjFzJkmSmmPirCczZ5IkSS1i5kySJDXGszV7M3MmSZLUImbOJElSc9LUWS9mziRJklrEzJkkSWqMc856M3MmSZLUImbOikwYH68zVt09Xme37hqfVyUOwI+ePLRKnE1bDqkSByAeWFgt1uLNUSVODlcJA0BM1Is1sr1OnPnb6n1lXbh5vEqcedvrxAEY3jFWLdbQWJ1f4PaVi6rEAaDO2xyAiZE6wXbX+WgBYHi0Tpui3luKkV31YtW6wNfEXP7PnXidsz6YOZMkSWqRuTz+liRJc0gA4dmaPZk5kyRJahEHZ5IkSS3iYU1JktSciidLHajMnEmSJLWImTNJktQYTwjozcyZJElSi5g5kyRJzfAitH0xcyZJktQiZs4kSVJDsnO/RM3IzJkkSVKLDGxwFhEvjIg7upYnIuJdEfHHEfG9iPhWRHwhIg7rqnNxRGyIiLsi4pyu8tMi4ttl3cciIkr5goi4upTfEhHHddVZExF3l2XNoPopSZL6F9nMMpcNbHCWmXdl5qmZeSpwGrAD+AKwFjglM38C+EfgYoCIOAlYDZwMnAt8PCKGS7jLgAuAE8tybil/G7A5M08APgp8qMRaAVwCvBQ4A7gkIpYPqq+SJEm1NHVY8yzg+5l5b2Z+KTPHSvnNwKry+Dzgqswczcx7gA3AGRFxDLA0M2/KzAQ+Bbyuq86V5fG1wFklq3YOsDYzN2XmZjoDwskBnSRJmi2ZzSxzWFODs9XAZ6cpfytwQ3m8Eri/a93GUrayPJ5a/rQ6ZcC3FTh8hlhPExEXRMS6iFg3vm37PnZJkiSpvoGfrRkR84HXUg5fdpX/PjAGfGayaJrqOUP5/tZ5qiDzcuBygAXPWzm3h9mSJLVdQnhvzZ6ayJy9Crg9Mx+eLCgT9F8DvLEcqoROduvYrnqrgAdL+appyp9WJyJGgGXAphliSZIktVoTg7NfoeuQZkScC/we8NrM3NG13fXA6nIG5vF0Jv7fmpkPAdsi4swyn+zNwHVddSbPxHwD8NUy2LsRODsilpcTAc4uZZIkaTY556yngR7WjIjFwM8Bb+8q/nNgAbC2XBHj5sx8R2auj4hrgDvpHO68KDPHS50LgU8Ci+jMUZucp/YJ4NMRsYFOxmw1QGZuiogPAF8v270/MzcNppeSJEn1DHRwVjJjh08pO2GG7S8FLp2mfB1wyjTlu4Dz9xLrCuCKfWyyJEkapLmd1GqEt2+aNDbExGMLqoRazzFV4mROd17DfsZ6pE7fFv2o3pHwhZvq/YWO7GjfDNOhsd7b9Gv+k+O9N+rDgkd3VYkDMLy50hnOI8O9t+nTxOL51WLtWLWkSpwnn12xf/OqhSLqvKUYW1jvc2q80q9vZFe9z5ah3dVC1VNvl6ulvH2TJElSi5g5kyRJjYk5Plm/CWbOJEmSWsTMmSRJao6Zs57MnEmSJLWIgzNJktSMBCYaWnqIiIURcWtEfDMi1kfEH5byFRGxNiLuLj+Xd9W5OCI2RMRdEXHOM90de+PgTJIkHYxGgVdk5ouAU4FzI+JM4D3AVzLzROAr5TkRcRKdi92fDJwLfDwi6l0rp4uDM0mS1IggiWxm6SU7nixP55UlgfOAK0v5lcDryuPzgKsyczQz7wE2AGdU3D0/5uBMkiQdiI6IiHVdywVTN4iI4Yi4A3gEWJuZtwBHl/t6U34eVTZfCdzfVX1jKavOszUlSVJzmjtb87HMPH3mpuQ4cGpEHAZ8ISL+ya0iu0x3b4aBdMbMmSRJOqhl5hbga3Tmkj0cEccAlJ+PlM02Asd2VVsFPDiI9jg4kyRJzclsZukhIo4sGTMiYhHwSuB7wPXAmrLZGuC68vh6YHVELIiI44ETgVvr7pwOD2tKkqSD0THAleWMyyHgmsz8YkTcBFwTEW8D7gPOB8jM9RFxDXAnMAZcVA6LVufgTJIkNWPyOmctkJnfAv7lNOWPA2ftpc6lwKUDbpqHNSVJktrEzJkkSWpMP9cgO9iZOZMkSWoRB2eSJEkt4mFNSZLUHA9r9mTmTJIkqUXMnEmSpIb0d4HYg52ZM0mSpBYxc1YM7YYl9w1XiTX+6OIqcWKsShgADnmgzjeVQx4YrRKntqGxSlc1rHhxxBivF2x4++46gcbrfWOdWLqoSpwcrvcdceez67QJYPOJdT4eR5fX2+fDu6e77/L+Gar0lhraUycOQFS61vrognr7Kap+JtSLNWclZs76YOZMkiSpRcycSZKk5rTk9k1tZuZMkiSpRcycSZKkxnj7pt7MnEmSJLWImTNJktQcM2c9mTmTJElqETNnkiSpGQlMmDnrxcyZJElSi5g5kyRJDfHemv0wcyZJktQiDs4kSZJaxMOakiSpOR7W7MnMmSRJUouYOZMkSc0xc9aTmTNJkqQWMXMmSZKa4UVo+2LmTJIkqUXMnEmSpIYk5MRsN6L1zJxJkiS1iJkzSZLUHM/W7MnMmSRJUouYOSsiYXh3nVgjO+vEWfxovePyy775WJU4sb1S54BcuqRerKE63zOGtm2vEgdg4rFN1WLFooVV4oyfsLJKHICdz6rTpt1L6n1HfPLYerF2HVnn7y+jSphOrIqf2EOVYg2P1uxgpTAV91MO1cvyxFidfTU0ViXM7PBszb6YOZMkSWoRM2eSJKk5zjnrycyZJElSi5g5kyRJzTFz1pOZM0mSpBYZ2OAsIl4YEXd0LU9ExLsiYkVErI2Iu8vP5V11Lo6IDRFxV0Sc01V+WkR8u6z7WEREKV8QEVeX8lsi4riuOmvKa9wdEWsG1U9JkqSaBjY4y8y7MvPUzDwVOA3YAXwBeA/wlcw8EfhKeU5EnASsBk4GzgU+HhHDJdxlwAXAiWU5t5S/DdicmScAHwU+VGKtAC4BXgqcAVzSPQiUJEmzITuHNZtY5rCmDmueBXw/M+8FzgOuLOVXAq8rj88DrsrM0cy8B9gAnBERxwBLM/OmzEzgU1PqTMa6FjirZNXOAdZm5qbM3Ays5akBnSRJUms1dULAauCz5fHRmfkQQGY+FBFHlfKVwM1ddTaWsj3l8dTyyTr3l1hjEbEVOLy7fJo6PxYRF9DJyDHvUBNrkiQNVAIT3vi8l4FnziJiPvBa4HO9Np2mLGco3986TxVkXp6Zp2fm6SOL6l2tXpIkaX81cVjzVcDtmflwef5wOVRJ+flIKd8IHNtVbxXwYClfNU350+pExAiwDNg0QyxJkjSbnHPWUxODs1/hqUOaANcDk2dPrgGu6ypfXc7APJ7OxP9byyHQbRFxZplP9uYpdSZjvQH4apmXdiNwdkQsLycCnF3KJEmSWm2gc84iYjHwc8Dbu4o/CFwTEW8D7gPOB8jM9RFxDXAnMAZclJnjpc6FwCeBRcANZQH4BPDpiNhAJ2O2usTaFBEfAL5etnt/Zta7C7UkSdo/czyr1YSBDs4ycwedCfrdZY/TOXtzuu0vBS6dpnwdcMo05bsog7tp1l0BXLHvrZYkSZo93r5JkiQ1JGHCzFkv3r5JkiSpRcycSZKkZiRkep2zXsycSZIktYiZM0mS1BznnPVk5kySJKlFzJxJkqTmeJ2znhycFQnkdHfk3A8Lnqjzxlt619YqcQDywYd7b9SPY4+pEweYWLKgWqzhx7dViTPx6ONV4gBM7NhRLdbQic+tEmfr8xdXiQOw55A6fzC14gCMrqj3oZ+VjisMj9brX4xVC8XQ7jrtivHe2/RrYl6lOMP13gdDYxV/f5X21bwn68RRe3lYU5IkqUXMnEmSpGZkwoSX0ujFzJkkSVKLmDmTJEnN8YSAnsycSZIktYiZM0mS1Jh0zllPZs4kSZJaxMyZJElqSDrnrA9mziRJklrEzJkkSWpG4o3P+2DmTJIkqUXMnEmSpOakZ2v2YuZMkiSpRcycSZKkRiSQzjnrycyZJElSi5g5kyRJzch0zlkfzJxJkiS1iIMzSZKkFvGwpiRJaownBPRm5kySJKlFzJxJkqTmeEJAT2bOJEmSWiQyPfYLEBGPAvc2+JJHAI81+HqzwT7OfQd6/8A+Hijs4/55bmYeWTnmXkXE/6TTjyY8lpnnNvRaVTk4myURsS4zT5/tdgySfZz7DvT+gX08UNhHHUg8rClJktQiDs4kSZJaxMHZ7Ll8thvQAPs49x3o/QP7eKCwjzpgOOdMkiSpRcycSZIktYiDM0mSpBZxcLYPImJhRNwaEd+MiPUR8YdT1v9ORGREHNFVdnFEbIiIuyLinK7y0yLi22XdxyIiSvmCiLi6lN8SEcd11VkTEXeXZU2TfYyI90XEAxFxR1lefaD1saz7zdKP9RHx4QOtj6VNk7/DH0bEHQdgH0+NiJtLH9dFxBkHYB9fFBE3lTb/94hYOlf7WF5nOCK+ERFfLM9XRMTa8rprI2L5XO7fXvp4fvmdTkTE6VO2nZN9VEWZ6dLnAgRwSHk8D7gFOLM8Pxa4kc6FbI8oZScB3wQWAMcD3weGy7pbgZ8sMW8AXlXK/y3wX8rj1cDV5fEK4Afl5/LyeHlTfQTeB/zONNsfSH38WeDLwIKy7qgDrY9TtvkT4L0HWh+BL3W18dXA1w7APn4d+JlS/lbgA3O1j+W1fhv4a+CL5fmHgfeUx+8BPjSX+7eXPv5z4IXA14DTu7abs310qbeYOdsH2fFkeTqvLJNnVHwU+N2u5wDnAVdl5mhm3gNsAM6IiGOApZl5U3b+gj4FvK6rzpXl8bXAWeXb0TnA2szclJmbgbVA9Ssf9+jjdA6kPl4IfDAzR8t2jxyAfQSgtOVfA589APuYwGQmaRnw4AHYxxcCf1/K1wK/NFf7GBGrgJ8H/rKruLtNV05p65zqH0zfx8z8bmbeNc3mc7KPqsvB2T4qqek7gEfovOlviYjXAg9k5jenbL4SuL/r+cZStrI8nlr+tDqZOQZsBQ6fIVZ10/WxrPqNiPhWRFzRdZjhQOrjC4CXlcMCfxcRL5na3intmot9nPQy4OHMvHtqe6e0ay728V3AH0fE/cBHgIuntndKu+ZiH78DvLZscj6dzP3T2julXW3u45/S+WLbfTfsozPzodKmh4CjprZ1Spva3D+Yvo97M1f7qIocnO2jzBzPzFOBVXS+zfwE8PvAe6fZPKYLMUP5/tapapo+ngJcBjwfOBV4iM4hMWZo11zs4wid1P+ZwL8HrinfPg+kPk76FZ7KmjFDu+ZiHy8E3p2ZxwLvBj5RNj+Q+vhW4KKIuA04FNhdNp9TfYyI1wCPZOZt/VaZpqy1/YODo4+qz8HZfsrMLXTmCpxHZ17ANyPih3Q+QG+PiGfR+ZZybFe1VXQOsWwsj6eW010nIkboHJbZNEOsgenq47mZ+XD5JzEB/AUwOcn6gOljef3Pl0NJt9L5lnvEDO2ai32cbM/rgau7NjuQ+rgG+HxZ9TkOwPdqZn4vM8/OzNPoDLK/P7W9U9rV1j7+NPDa8tl5FfCKiPgr4OFyGI/yc3KKwVzrH+y9j3szF/uo2rIFE9/mygIcCRxWHi8C/hfwminb/JCnTgg4madP7PwBT03s/DqdDM3kxM5Xl/KLePrEzmvK4xXAPXQyO8vL4xVN9RE4pmubd9OZE3Gg9fEdwPtL+QvoHA6IA6mP5fm5wN9N2f6A6SPwXeDlpfws4LYDsI+TJ6sM0Zl79Na52seuvr6cpybL/zFPPyHgw3O9f1P72FX2NZ5+QsCc7qNLpffKbDdgLi3ATwDfAL5FZ87He6fZ5oeUwVl5/vt0vtXeRTmzppSfXmJ8H/hznrpbw0I63/Y30Dkz53lddd5ayjcAb2myj8CngW+X8ut5+mDtQOnjfOCvStntwCsOtD6WdZ8E3jFNnQOij8C/Am6j8w/uFuC0A7CPvwX8Y1k+ONneudjHrtd6OU8Nzg4HvgLcXX6umOv9m6aPv0gnszUKPAzceCD00aXO4u2bJEmSWsQ5Z5IkSS3i4EySJKlFHJxJkiS1iIMzSZKkFnFwJkmS1CIOziRJklrEwZkkSVKLODiTNCsi4iUR8a2IWBgRSyJi/ZT7f0rSQcmL0EqaNRHxR3Subr4I2JiZ/2mWmyRJs87BmaRZExHz6dwvcBfwU5k5PstNkqRZ52FNSbNpBXAIcCidDJokHfTMnEmaNRFxPXAVcDxwTGb+xiw3SZJm3chsN0DSwSki3gyMZeZfR8Qw8H8i4hWZ+dXZbpskzSYzZ5IkSS3inDNJkqQWcXAmSZLUIg7OJEmSWsTBmSRJUos4OJMkSWoRB2eSJEkt4uBMkiSpRf5/hapoYZLiiZEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "EXAMPLE_I = 0\n", + "\n", + "opt_flow_batch[\"data\"].sel(example=EXAMPLE_I, channels_index=0, time_index=0).assign_coords(\n", + " x=(\n", + " \"x_index\", \n", + " opt_flow_batch[\"x\"].sel(example=EXAMPLE_I).data\n", + " ),\n", + " y=(\n", + " \"y_index\", \n", + " opt_flow_batch[\"y\"].sel(example=EXAMPLE_I).data\n", + " )\n", + ").swap_dims(\n", + " {\n", + " \"x_index\": \"x\",\n", + " \"y_index\": \"y\"\n", + " }\n", + ").plot.imshow(x=\"x\", y=\"y\", figsize=(10, 10))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "3cf20dca-6508-49e1-8e4e-793c3f9aa029", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAJcCAYAAAC8DwN/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABIbElEQVR4nO3de5xlVXng/d/TVd3VNxq6QbDpxoABzQBRIgTJxTdGDKAxYIxOOqOxo7zBGDOJZjKJTCZiJMxoYl4T34xkSOQjGiMgo5FxwmCLr2aScBEQxRYJrdwakFs3TTd9q8vz/nFWhdNldZ3TsM6uXd2/7+ezP3XO2ns9Z69dp06t86y1947MRJIkSe0wb7Z3QJIkSU+zcyZJktQids4kSZJaxM6ZJElSi9g5kyRJahE7Z5IkSS1i50xVRMSvRMQ/zvZ+TBURH4uIP3oW9ddHxMtn47W1byLinoh45WzvxzMVEW+MiC80/Jq+v6UWsnMmzSAzT8jML8/2ftQQEadHxLcjYntE/H8R8QOzvU9zXUT8u4i4NyKeioi/i4gVfdY7OiIyIoYnyzLzk5l5xuD29vvtL+/v0rHd1rVsL8f35NneN+mZsHMmHQAi4jDgM8AfACuAm4ErZnWn5riIOAH478AvA0cA24GPzOpOHaBKx3bp5AL8OvBd4NZZ3jXpGbFzpn0SEUdFxGci4tGIeDwi/mLK+g9GxOaIuDsiXtVV/paIuCMitkbEdyPibV3rXh4RGyPiP0TEIxHxUES8pWv9xyLiv0XE/yr1b4yIH+xa/0MRsS4iNkXEnRHxb/ey74dFxOcj4omy7f+JiBn/BrqHyiLivRFxZUR8vOzH+og4pWvbH4mIW8u6K4CFU2K9JiJuK6//zxHxolL+i+WYLCvPXxUR34uI58z4y9g3rwPWZ+anM3Mn8F7gxRHxQ/1U3tsxjogfLGUvKc+PjIjHJofK+vy9/27X7/21EfHqiPiXEvc/dW3/3oi4KiKuKPFujYgX72V/50XEuyPiO+V9emW/Wa198Ebgf2bmP2TmNjod39dFxEF91P2H8vOJkun5sZgyNaBkfn49Iu4q7b2wHO/rI+LJ0qYFXdtP+/6ayX70/p5qLfDx9BY4mqsy08WlrwUYAr4OfAhYQufD+SfLul8BRoFfLdu9HXgQiLL+Z4EfBAL4KTpZhpeUdS8HxoD3AfOBV5f1y8v6jwGbgFOBYeCTwOVl3RLgfuAtZd1LgMeAE7rq/lF5/F+BvyyvMR942eT+zdDme4BXlsfvBXaW/Rsq8W4o6xYA9wLvKrFfX47H5Gu/BHgEeGmpu7bEHinrP1n29dBy3F4zwz49McPy7r3U+XPg4ill3wR+oY/fe69j/KvAHcBi4Frgg111+/m9v6ccs18FHgX+FjgIOKEc7+d3Hf/RcmznA78D3A3Mn+Z39U7gBmA1MEInw/WpvbTveT2O6b/bS73PAb83pWwbcHIfx/RoIIHhrrJfAf6x63kCVwPLyrHYBVwHPB84GPgWsLaf99f+/v6eUv8HgHHgmNn4nHRxqbHM+g64zJ0F+DE6/zyHp1n3K8CGrueLyz+X5+4l1t8Bv1UevxzYMeUf1SPAaeXxx4C/7lr3auDb5fEvAv9nSuz/DlzQVXfyH8j76PxDPXYf2jz1n9cXu9YdD+woj/8vujqjpeyfu177YuDCKbHvBH6qPD4EuA+4HfjvA/jdfRR4/5SyfwJ+pY+6Mx7j8vzqsu/fYIYOwV5+70Pl+UHlPfPSru1vAV7bdfxv6Fo3D3gIeNk0v6s7gNO7tl1JpzPxfe/dZ3FMrwN+bUrZA8DL+6h7NP11zn5iyrH4va7nfwr8WT/vr/39/T3ldf8A+PIgX8PFZdCLw5raF0cB92bm2F7Wf2/yQWZuLw+Xwr8OZdxQhqqeoNPBOqyr7uNT4m6frDs19pR1PwC8tAylPFFivxF47jT79yfABuALZZjl3Xtv6l5N3Y+F0ZnUfSTwQGZ2D6Pc2/X4B4D/MGU/jyr1yMwngE8DJ9L5p1vbNjoZmG7LgK191O3nGP8VnX3/fzNz12Rhn7/38fJ4R/n5cNf6Hez5Prh/8kFmTgAbKcdwmn3+bNf+3kEnm3JEH+3t17M5pv2aeiz2dmxmfH/tg7n6/u72ZuCyAb+GNFB2zrQv7geeF11nmPUjIkaA/wF8EDgiMw8B/p7OUFeNffpKZh7StSzNzLdP3TAzt2bmf8jM5wM/B/x2RJxeYR+gk8FZFRHdbXrelP28aMp+Ls7MTwFExEnAW4FPAR+e6YViz7PSpi7/aS/V1gMv7oqxhM5w4/o+2jbjMY6IpcCf0cnOvXdybteAfu9HdbVhHp1hywf3ss+vmrLPCzPzgakbRsTzehzTN+5lX6Ye0+fTGUL9lz7aUXsu1Izvrwra/v6erPsTdDqEV+17E6X2sHOmfXETnQ/p90fEkohYWD4Me1lA55/Wo8BYdE4UqHXJgM8DL4iIX46I+WX50Yj4N1M3LBOWjy3/YJ6kk0kZn7rdM3Q9nflTvxkRwxHxOjpz5Cb9FfBrEfHS6FgSET8bEQdFxELgb4D/RGde16qI+PW9vVB2nZU2zfJf9lLts8CJEfEL5fXeA3wjM78N/3qdunv2UrfXMf5z4JbM/L+B/0VnXh8M5vd+ckS8rnxBeCedeVg3TLPdXwIXRblcSEQ8JyLOmS5gZt7X45h+ci/78kng5yLiZaWz+z7gM5m5tbzmeyPiy3up+ygwQWf+WA17fX9Vit/29/ektcD/mPwdSHOVnTP1rQw//RxwLJ35IxvpzEfqVW8r8JvAlcBm4N/RmaNUY5+20vmHv4ZOBuV7wAfodAqmOg74Ip3hqOuBj2Slazxl5m46Z0T+Cp02/iKdS1dMrr+ZzoT3vyjrN5RtoTPxemNmXlyGBN8E/FFEHFdj38rrPwr8AnBRef2X0jlmk46iMwdturp7Pcalw3MW8Gtl898GXhIRbxzQ7/1zdI7tZjqXsHhdZo5Os92fl9f6QkRspdOBe+mzfO09ZOZ6Ou3+JJ05kgfRuYTDpJmO6XY6v4t/KsOApz3LfZnp/fWstf39DVA6gf8WhzS1H5g8k07SASw6V6b/rcy8Y7b3ZW8i4r10TuZ402zvSz8i4jY6JyU8Ptv7Imlu2ae5Q5L2T9nwlekPBJl50mzvg6S5yc6ZDmgR8Tw614uazvGZeV+T+yPV5Ptbmpsc1pQkSWoRTwiQJElqEYc1i0XLR/LglUuqxIqok42cH7Wu8lDPcMV9mlf9Uk/PXla59FrHWLbvu0/N91RU+v3Nq/T3ArBjYkHvjfpU6/e3cN50J5M+MxNZ7/1Za79GKv7+Hh5d2nujftQ7TKwe3t57oz5tz4kqce7ZcWiVOAA7Nnzvscwc5H1O93DmTy/Jxzc187/tlm/sujYzz2rkxSqzc1YcvHIJb/zbV1aJNTJvbxfQ3zdHzH+yShyAeVHnQ+HQoW1V4kDdf1q1jFfsUD0+XukfTUWHD9d7T82POu/zg+btrBIH4PadR/XeqE+Pjda5RNgLFz5UJQ7AzpxfLdYPLaizX8+fX+/39yeP/mSVODU7/B844rZqsW7btav3Rn0495u/XCUOwG2v+S/39t6qnsc3jXPTtc/rvWEFQyvvOqz3Vu1k50ySJDUigQnqJAv2Z+0bd5EkSTqAmTmTJEkNScYrzb3bn5k5kyRJahE7Z5IkSS3isKYkSWpE54SA9l1GqW3MnEmSJLWImTNJktQYL6XRm5kzSZKkFjFzJkmSGpEk4+mcs17MnEmSJLWImTNJktQYz9bszcyZJElSi5g5kyRJjUhg3MxZTwPNnEXEuyJifUR8MyI+FRELS/m/j4g7y7o/7tr+/IjYUNad2VV+ckTcXtZ9OCKilI9ExBWl/MaIOLqrztqIuKssawfZTkmSpFoGljmLiFXAbwLHZ+aOiLgSWBMR9wLnAC/KzF0RcXjZ/nhgDXACcCTwxYh4QWaOAxcD5wE3AH8PnAVcA5wLbM7MYyNiDfAB4BcjYgVwAXAKnY76LRFxdWZuHlR7JUlSb845623Qc86GgUURMQwsBh4E3g68PzN3AWTmI2Xbc4DLM3NXZt4NbABOjYiVwLLMvD4zE/g48NquOpeVx1cBp5es2pnAuszcVDpk6+h06CRJklptYJ2zzHwA+CBwH/AQsCUzvwC8AHhZGYb8SkT8aKmyCri/K8TGUraqPJ5avkedzBwDtgCHzhBrDxFxXkTcHBE3b39i17NpriRJ6iGB8cxGlrlsYJ2ziFhOJ7N1DJ1hyiUR8SY62bTlwGnAfwSuLNmumCZMzlDOM6zzdEHmJZl5SmaesviQkR4tkiRJGrxBDmu+Erg7Mx/NzFHgM8CP08lifSY7bgImgMNK+VFd9VfTGQbdWB5PLae7Thk6PRjYNEMsSZI0iyYaWuayQXbO7gNOi4jFJTN2OnAH8HfAKwAi4gXAAuAx4Go6JwyMRMQxwHHATZn5ELA1Ik4rcd4MfK68xtXA5JmYrwe+VOalXQucERHLSwbvjFImSZLUagM7WzMzb4yIq4BbgTHga8AldIYXL42IbwK7gbWlQ7W+nNH5rbL9O8qZmtA5ieBjwCI6Z2leU8o/CnwiIjbQyZitKa+9KSIuBL5atntfZm6aaX+HYoLlw9uffcNLrBrmVYoD8NzhLVXijOZQlTgA9+8+tFqskXmj1WLVcteOI6rFOnrhY1Xi7JyYXyUOwPyhsSpxar6nah7zWp8HP7Xo/t4b9anmu/yesaVV4iyOer+/lQvqfE599M4fqxIH4Ixlt1eL9e1dx1aJs3V9vc9OtdNAL0KbmRfQuaTFVG/ay/YXARdNU34zcOI05TuBN+wl1qXApfuyv5IkaXCS9CK0ffD2TZIk6YAUEfeUi9zfFhE3l7IVEbGuXMR+XZkeNbn9tBfLr83OmSRJakbCeEPLPvjpzDwpM08pz98NXJeZxwHXledTL5Z/FvCRiIrj+l3snEmSJD2t+wL3l7Hnhe+/72L5g9gBO2eSJKkRSaOX0jhs8kLzZTlvL7v0hYi4pWv9EeVKEZSfh5fyvi5wX8NATwiQJEmaJY91DVXuzU9k5oPlPt/rIuLbM2zb1wXua7BzJkmSGhKMT9vHmR2Z+WD5+UhEfJbOMOXDEbEyMx8q9/eevAd4Yxe4d1hTkiQdcCJiSUQcNPmYzgXrv8meF7hfy54Xvv++i+UPYt/MnEmSpEYkMNGey5wdAXy2c/MhhoG/zcz/HRFfpXPf73Pp3O3oDQCZOdPF8quycyZJkg44mfld4MXTlD9O55aT09WZ9mL5tdk5kyRJjWnTnLO2cs6ZJElSi5g5kyRJjUjMnPXDzJkkSVKLmDmTJEmNmUgzZ72YOZMkSWoRO2eSJEkt4rCmJElqhCcE9MfMmSRJUouYOZMkSY1IgnHzQj15hCRJklrEzFkRJEMxUSXWiYvurxJnYYxWiQNw6LztVeLctuuoKnEArnnkhGqx3nnUuipx1j15YpU4AJ/51knVYp334n+sEmfV/M1V4gA8MLqiSpwXjdT5ewF4yZJ7qsXaMr6kSpxNE0NV4gAcM1wv1t9se2GVOKNLNlSJA/CKJXdUifOXu19WJQ7A5Y+dVi3WP288ukqcg++sEmbWeCmN3sycSZIktYiZM0mS1AjP1uyPmTNJkqQWMXMmSZIaEoyneaFePEKSJEktYuZMkiQ1IoEJ80I9eYQkSZJaxMyZJElqjGdr9mbmTJIkqUXMnEmSpEZkerZmPzxCkiRJLWLnTJIkqUUc1pQkSY2Z8ISAnsycSZIktYiZM0mS1IjOjc/NC/XiEZIkSWoRM2eSJKkhXkqjHx4hSZKkFjFzJkmSGuGNz/vjEZIkSWoRM2eSJKkx4+l1znoxcyZJktQiZs6KhfNGeeHCh6rEOmjezipxPvzAK6vEAfjx5d+tEufvHnhxlTgAm760slqsS17zU1XiPLDt4CpxABauX1Qt1scXnVolzh+c+PdV4gAsiLEqcd5//6urxAF4dMeSarF+5NAHqsQ5aGhHlTgAWxfU+YwC+Mqjx7UqDsDZK79eJc7olpEqcQBuePAHqsVauGC0SpzdB8/dzFMSXuesDx4hSZKkFjFzJkmSGjPhdc568ghJkiS1iJkzSZLUCO+t2R+PkCRJUovYOZMkSWoRhzUlSVIjkvAitH0wcyZJktQiZs4kSVJjvPF5bwM9QhHxrohYHxHfjIhPRcTCrnW/ExEZEYd1lZ0fERsi4s6IOLOr/OSIuL2s+3BERCkfiYgrSvmNEXF0V521EXFXWdYOsp2SJEm1DKxzFhGrgN8ETsnME4EhYE1ZdxTwM8B9XdsfX9afAJwFfCQihsrqi4HzgOPKclYpPxfYnJnHAh8CPlBirQAuAF4KnApcEBHLB9VWSZLUWyaM57xGlrls0Hs/DCyKiGFgMfBgKf8Q8Lt0Lnky6Rzg8szclZl3AxuAUyNiJbAsM6/PzAQ+Dry2q85l5fFVwOklq3YmsC4zN2XmZmAdT3foJEmSWmtgc84y84GI+CCd7NgO4AuZ+YWIOBt4IDO/XkYnJ60Cbuh6vrGUjZbHU8sn69xfXm8sIrYAh3aXT1PnX0XEeXQychx25Pxn2FJJktSfYALP1uxlkMOay+lkto4BjgSWRMSbgd8H3jNdlWnKcobyZ1rn6YLMSzLzlMw85eAVnhshSZJm3yB7JK8E7s7MRwEi4jPAW+h01iazZquBWyPiVDrZraO66q+mMwy6sTyeWk5XnY1l6PRgYFMpf/mUOl+u1zRJkrSvEub8fLAmDPII3QecFhGLyzyw04HPZObhmXl0Zh5NpxP1ksz8HnA1sKacgXkMnYn/N2XmQ8DWiDitxHkz8LnyGlcDk2divh74UpmXdi1wRkQsLxm8M0qZJElSqw1yztmNEXEVcCswBnwNuGSG7ddHxJXAt8r278jM8bL67cDHgEXANWUB+CjwiYjYQCdjtqbE2hQRFwJfLdu9LzM3VWyeJEl6BrzxeW8DnWiVmRfQuaTF3tYfPeX5RcBF02x3M3DiNOU7gTfsJfalwKX7tseSJEmzy1nwkiSpEUkw4b01ezK3KEmS1CJmziRJUmOcc9abR0iSJKlFzJwVwzHOc4e2VIl1y86jq8R5YNvBVeIA3Da8uvdGfXhqd707KTz1Q7uqxfrWw8+tEmdsrN73lSWn1TtBeN68iSpxrnvi+CpxAObPG++9UR++s+nQKnEAtm8bqRbrn3YtqBJneHWd41TbDy57rEqcJUP1/o5PXnhPlTgvP+mOKnEAhqPO3x7AMYvqHPOv/NxxVeIAfPND1UKpIjtnkiSpEQlMeBHanjxCkiRJLWLmTJIkNSQY98bnPZk5kyRJahEzZ5IkqRHOOeuPR0iSJKlFzJxJkqTGOOesNzNnkiRJLWLmTJIkNSIznHPWB4+QJElSi5g5kyRJjRk3c9aTR0iSJKlFzJxJkqRGJDDh2Zo9mTmTJElqETNnkiSpIeGcsz54hCRJklrEzJkkSWpE596azjnrxcyZJElSi9g5kyRJahGHNSVJUmPGzQv15BGSJElqETNnxc6J+Xx798oqsf7Xwz9cJc7usaEqcQA27VpSJc6C4fEqcQDmzZ+oFmv3PUurxBl5vN73la2H1mvfgqO3VYlz6yOrqsQB2PrUwipxxiu+z4fm13t/PrVjQZU4t28+skocgE27F1eL9eTuRdVi1XLtUJ3PzlULn6gSB+CbW+r9/kbmjVWJ84rD76wSB+CL1SL1JwlPCOiDmTNJkqQWMXMmSZIaM2FeqCePkCRJUouYOZMkSY3IhHHnnPVk5kySJKlFzJxJkqTGeLZmb2bOJEmSWsTMmSRJakTnOmfmhXrxCEmSJLWImTNJktSYcZxz1ouZM0mSpBYxcyZJkhqReLZmP8ycSZIktYidM0mSpBZxWFOSJDXES2n0wyMkSZLUInbOJElSYyaIRpZ+RMRQRHwtIj5fnr83Ih6IiNvK8uqubc+PiA0RcWdEnDmgwwM4rClJkg5cvwXcASzrKvtQZn6we6OIOB5YA5wAHAl8MSJekJnjg9gpM2eSJKkRmTCe0cjSS0SsBn4W+Os+dv0c4PLM3JWZdwMbgFOf1cGYgZ0zSZK0PzosIm7uWs6bsv7PgN8FJqaU/0ZEfCMiLo2I5aVsFXB/1zYbS9lAOKwpSZIa0+DZmo9l5inTrYiI1wCPZOYtEfHyrlUXAxfSuV7uhcCfAm+FaSexZdW97WLnTJIkHWh+Aji7TPhfCCyLiL/JzDdNbhARfwV8vjzdCBzVVX818OCgds7OWbFpdAmfeqDO8PG9jy/vvVEfdm0bqRKnpq1PLawWa8WX6sVads/uKnE2HV/vmG8/amqm/JkbG6vzTXPrWL1jPrp9QZU4i5btrBIH4AXPebRarAVDY1XibBut9566/ZEjq8Xa/o06n1MxUe9WPJtOW1wlzuqlT1SJA3D7rcdUi/Wdo5+sEueHD69znGZDEq24fVNmng+cD1AyZ7+TmW+KiJWZ+VDZ7OeBb5bHVwN/GxH/D50TAo4DbhrU/tk5kyRJ6vjjiDiJzpDlPcDbADJzfURcCXwLGAPeMagzNcHOmSRJalC/1yBrSmZ+GfhyefzLM2x3EXBRE/s00Fl5EfGuiFgfEd+MiE9FxMKI+JOI+HY5E+KzEXFI1/bTXuAtIk6OiNvLug9HRJTykYi4opTfGBFHd9VZGxF3lWXtINspSZJUy8A6ZxGxCvhN4JTMPBEYonMBt3XAiZn5IuBfeHrMt/sCb2cBH4mIoRLuYuA8OmO8x5X1AOcCmzPzWOBDwAdKrBXABcBL6VyH5IKu02ElSdIsSGAio5FlLhv0+azDwKKIGAYWAw9m5hcyc3Km7Q10zniAvVzgLSJWAssy8/rMTODjwGu76lxWHl8FnF6yamcC6zJzU2ZuptMhnOzQSZIktdbA5pxl5gMR8UHgPmAH8IXM/MKUzd4KXFEer6LTWZs0eYG30fJ4avlknfvL641FxBbgUPq8WFy5IN15AAuPOGgfWyhJkvZVg9c5m7MGOay5nE5m6xg6p50uiYju64f8Pp0zHj45WTRNmJyh/JnWebog85LMPCUzT5l/8KK9NUWSJKkxg+y+vhK4OzMfzcxR4DPAj0Nnsj7wGuCNZagS9n6Bt408PfTZXb5HnTJ0ejCwaYZYkiRJrTbIztl9wGkRsbjMAzsduCMizgJ+Dzg7M7d3bX81sKacgXkM5QJv5WJwWyPitBLnzcDnuupMnon5euBLpbN3LXBGRCwvGbwzSpkkSZotDZ0MMNdPCBjknLMbI+Iq4FY6w5dfAy4B1gMjwLpyRYwbMvPXelzg7e3Ax4BFwDVlAfgo8ImI2EAnY7amvPamiLgQ+GrZ7n2ZuWlQbZUkSaploBehzcwL6FzSotuxM2w/7QXeMvNm4MRpyncCb9hLrEuBS/dlfyVJ0uAk7bsIbRt5yoQkSVKLePsmSZLUmLk+H6wJZs4kSZJaxMyZJElqxOTtmzQzM2eSJEktYuZMkiQ1xsxZb2bOJEmSWsTMmSRJakQy96/e3wQzZ5IkSS1i5kySJDXGOwT0ZuZMkiSpRcycFaPjQ2x84uAqsXY/urhKnKEd9b5dbNm0okqcGK+3T0+8sFooHvvJOvu1bMXmKnEAllacVzE6OlQlzqKR0SpxABYfsq1KnKHIKnEAdk/UOU4Aj+5YUiXO9x6r87kCkI+PVIs1XOlQja4YrxMImD9UJ9aTowurxAFY+Ei9HMa2QxZViXP3wjqf57MiPVuzH2bOJEmSWsTOmSRJUos4rClJkhrh7Zv6Y+ZMkiSpRcycSZKkxpg5683MmSRJUouYOZMkSY3w9k39MXMmSZLUImbOJElSY9LMWU9mziRJklrEzJkkSWqMNz7vzcyZJElSi5g5kyRJjUhvfN4XM2eSJEktYuZMkiQ1xrM1ezNzJkmS1CJmziRJUkO8Q0A/zJxJkiS1iJ0zSZKkFnFYU5IkNcYTAnozcyZJktQiZs6KibF5bN+8qEqsmKgShon5deIA5KLxesFqGc5qoUaW7qoSZ17U26fhoXrHfPniHVXi7Byr9yf/6BNLq8SZmKj3HXHBgrFqscbG6uzX4iV13psAyw7bUi3WqqV1Yh0+sq1KHIBvbDqySpz7HzukShyA8edU+kAHqJQx2vJUnf9VsyHxIrT9MHMmSZLUImbOJElSM7JzCyfNzMyZJElSi5g5kyRJjZnAOWe9mDmTJElqETNnkiSpEYnXOeuHmTNJkqQWMXMmSZIa4o3P+2HmTJIkqUXMnEmSpMZ4nbPezJxJkiS1iJkzSZLUGM/W7M3MmSRJUovYOZMkSWoRhzUlSVIjMh3W7IeZM0mSpBYxcyZJkhrjRWh7M3MmSZLUIgPtnEXEuyJifUR8MyI+FRELI2JFRKyLiLvKz+Vd258fERsi4s6IOLOr/OSIuL2s+3BERCkfiYgrSvmNEXF0V5215TXuioi1g2ynJEnqT2fe2eCXuWxgnbOIWAX8JnBKZp4IDAFrgHcD12XmccB15TkRcXxZfwJwFvCRiBgq4S4GzgOOK8tZpfxcYHNmHgt8CPhAibUCuAB4KXAqcEF3J1CSJKmtBj2sOQwsiohhYDHwIHAOcFlZfxnw2vL4HODyzNyVmXcDG4BTI2IlsCwzr8/MBD4+pc5krKuA00tW7UxgXWZuyszNwDqe7tBJkqRZkhmNLHPZwDpnmfkA8EHgPuAhYEtmfgE4IjMfKts8BBxeqqwC7u8KsbGUrSqPp5bvUSczx4AtwKEzxNpDRJwXETdHxM3j25565o2VJEmqZGBna5ZhxHOAY4AngE9HxJtmqjJNWc5Q/kzrPF2QeQlwCcDI81flvIXjM+xe/ybGavV56w2az1s8VidQxXH8hYt3V4v1Ays2V4kzL+o1cDgmqsV6dMeSKnGeeHJxlTgAo5tH6gRaVOfvDuDg5Vurxdo9NtR7oz4ctWxLlTgAxx30SLVYT4zWeS88smtplTg1TUzUyzsMrdxRLdbCBXU+h0dH67w3Z0My97NaTRjksOYrgbsz89HMHAU+A/w48HAZqqT8nPy02Qgc1VV/NZ1h0I3l8dTyPeqUodODgU0zxJIkSWq1QXbO7gNOi4jFZR7Y6cAdwNXA5NmTa4HPlcdXA2vKGZjH0Jn4f1MZ+twaEaeVOG+eUmcy1uuBL5V5adcCZ0TE8pLBO6OUSZKkWZQNLXPZwIY1M/PGiLgKuBUYA75GZwhxKXBlRJxLpwP3hrL9+oi4EvhW2f4dmTk53vF24GPAIuCasgB8FPhERGygkzFbU2JtiogLga+W7d6XmZsG1VZJkqRaBnqHgMy8gM4lLbrtopNFm277i4CLpim/GThxmvKdlM7dNOsuBS7dx12WJEmD4r01++IdAiRJklrEe2tKkqTmzPUJYQ0wcyZJktQids4kSZJaxGFNSZLUGE8I6M3MmSRJUouYOZMkSY1JTwjoycyZJElSi5g5kyRJjUicc9YPM2eSJEktYuZMkiQ1IwEzZz2ZOZMkSWoRM2eSJKkxnq3Zm5kzSZKkFjFzJkmSmmPmrCczZ5IkSS1i5kySJDUkvM5ZH8ycSZIktYiZs0kZTOweqhJq6RHbqsQZH6/Xd969e//+Vd/z+IoqceYPj1eJAzAyf6xarCefWlgtVi1LV9Z5nx+57MkqcQCOXLKlWqw2und7nfc5wAPbDq4S59EnllaJAzC6Y36dQBUzMzm/3mfCQUtHq8RZuKBOnFnjnLOezJxJkiS1iJ0zSZKkFtm/x7okSVJ7pDc+74eZM0mSpBYxcyZJkprjCQE9mTmTJElqETNnkiSpQc4568XMmSRJUovYOZMkSc3JhpY+RMRQRHwtIj5fnq+IiHURcVf5ubxr2/MjYkNE3BkRZz6rY9CDnTNJknSg+i3gjq7n7wauy8zjgOvKcyLieGANcAJwFvCRiKhzW6Fp2DmTJEnNaUnmLCJWAz8L/HVX8TnAZeXxZcBru8ovz8xdmXk3sAE4dd8a3j87Z5IkaX90WETc3LWcN2X9nwG/C0x0lR2RmQ8BlJ+Hl/JVwP1d220sZQPh2ZqSJKkZSdUb0/fwWGaeMt2KiHgN8Ehm3hIRL+8j1nQ7PbArttk5kyRJB5qfAM6OiFcDC4FlEfE3wMMRsTIzH4qIlcAjZfuNwFFd9VcDDw5q5xzWlCRJjclsZpl5H/L8zFydmUfTmej/pcx8E3A1sLZsthb4XHl8NbAmIkYi4hjgOOCmARwewMyZJEnSpPcDV0bEucB9wBsAMnN9RFwJfAsYA96RmeOD2gk7Z5IkqTktu7dmZn4Z+HJ5/Dhw+l62uwi4qIl9clhTkiSpReycSZIktYjDmpIkqTnNXUpjzjJzJkmS1CJmzop5O4KldyyoEuup1XUOa0z03qZf80bb901ldy6qFmt8YZ0Zprufs7NKHICxkXrffXY9OVInUMWvYwct3VElztC8em/08YrfyJ/cXef9uWNsfpU4AMcte7RarCMXbakS519GDu+9UZ8e276kSpwnttb7bBnbVe/f5FM76vwdL1m0q0qc2RItOyGgjcycSZIktYiZM0mS1Iw+b0p+oDNzJkmS1CJmziRJUkPCszX7YOZMkiSpRcycSZKk5jjnrCczZ5IkSS1i5kySJDXHzFlPZs4kSZJaxMyZJElqjpmznsycSZIktcjAOmcR8cKIuK1reTIi3hkRJ0XEDaXs5og4tavO+RGxISLujIgzu8pPjojby7oPR0SU8pGIuKKU3xgRR3fVWRsRd5Vl7aDaKUmS+pR0rnPWxDKHDaxzlpl3ZuZJmXkScDKwHfgs8MfAH5by95TnRMTxwBrgBOAs4CMRMVTCXQycBxxXlrNK+bnA5sw8FvgQ8IESawVwAfBS4FTggohYPqi2SpIk1dLUsObpwHcy8146/eZlpfxg4MHy+Bzg8szclZl3AxuAUyNiJbAsM6/PzAQ+Dry2q85l5fFVwOklq3YmsC4zN2XmZmAdT3foJEmSWqupEwLWAJ8qj98JXBsRH6TTOfzxUr4KuKGrzsZSNloeTy2frHM/QGaORcQW4NDu8mnq/KuIOI9ORo75y0ysSZI0aHGAnBAQEQvpjPCdACycLM/Mt/aqO/DMWUQsAM4GPl2K3g68KzOPAt4FfHRy02mq5wzlz7TO0wWZl2TmKZl5ytCiJXtvhCRJ0r75BPBcOqN5XwFWA1v7qdjEsOargFsz8+HyfC3wmfL403TmhEEnu3VUV73VdIY8N5bHU8v3qBMRw3SGSTfNEEuSJM2mbGiZfcdm5h8AT2XmZcDPAj/cT8UmOme/xNNDmtDpJP1UefwK4K7y+GpgTTkD8xg6E/9vysyHgK0RcVqZT/Zm4HNddSbPxHw98KUyL+1a4IyIWF5OBDijlEmSJDVhtPx8IiJOpJNAOrqfigOdcxYRi4GfAd7WVfyrwJ+XTNdOypyvzFwfEVcC3wLGgHdk5nip83bgY8Ai4JqyQGdI9BMRsYFOxmxNibUpIi4Evlq2e19mbhpIIyVJkr7fJSVB9J/pJJOWAn/QT8WBds4yczudCfrdZf9I59Ia021/EXDRNOU3AydOU74TeMNeYl0KXLrvey1JkvSsXVeuGPEPwPMByshgT94hQJIkNSaymaUF/sc0ZVf1U7Fn5iwifgP4ZOn9SZIkaS8i4ofoXD7j4Ih4XdeqZXRdUmMm/QxrPhf4akTcSmeY8Noy6X6/kvNgfEGlYPPqHJ6RVU9ViQOwdNGuKnHGJ+rdEuOpHSPVYk3sauqSff3btWt+vWDjtY57vT/dJ7bUufxMrTgA9yxcUS1WLbX+9gA2LVxcLda9W+tc2/HJHX39r+nLrkp/x6Nb6n22ML/e38zo7jrt27yj4mfLbJjjt1bqwwuB1wCHAD/XVb6Vzrz7nnq+UzLzP0fEH9A54/EtwF+Uifsfzczv7OseS5Ik7a8y83PA5yLixzLz+mcSo69ufGZmRHwP+B6dMymXA1dFxLrM/N1n8sKSJOkA055rkDXhaxHxDgZxh4CI+M2IuIXODcr/CfjhzHw7nTMuf+EZ77IkSdL+6xnfIaCfzNlhwOvKTcv/VWZORMRr9nFHJUnSgezAyZwdm5lviIhzMvOyiPhb+rwgfj9zzt4zw7o79mEnJUmSDhRT7xDwPdpwhwBJkqRuLbkGWRPaeYcASZKkA0lE/HbX07eUn/+t/Ozr2kF2ziRJUnP2/8zZQeXnC4EfpZM1g841z/6hnwB2ziRJkirJzD8EiIgvAC/JzK3l+XuBT/cTw3trSpIk1fc8YHfX8914QoAkSWqd/X9Yc9IngJsi4rN0Wv3zwGX9VLRzJkmSVFlmXhQR1wAvK0Vvycyv9VPXzpkkSWpE5AF1KQ0y81bg1n2t55wzSZKkFjFzJkmSmpMx23vQembOJEmSWsTMmSRJas4BNOfsmTJzJkmS1CJmziRJUmMOpLM1nykzZ5IkSS1i5kySJDXHzFlPZs4kSZJaxMxZMW/JOCOnbqoS6+AFo1XiLBweqxIH4JEnl1aJM7q73ltm3tBEtVhUumzO+M567Rt+dH69WLUuC1TxG+vE/ErH6jm76sQBRubX+5tZsWR7lTjDUe99fsdjR1SLtfWuQ6rEmb+t3nf8Wh95Bz9Y740+vLNerB2HLagS58njxqvEmRUH2B0CnikzZ5IkSS1i5kySJDXHzFlPZs4kSZJaxM6ZJElSizisKUmSmuOwZk9mziRJklrEzJkkSWqMl9LozcyZJElSi9g5kyRJahE7Z5IkSS3inDNJktQc55z1ZOZMkiSpRcycSZKkZnjj876YOZMkSWoRM2eSJKk5Zs56MnMmSZLUImbOJElSc8yc9WTmTJIkqUXMnEmSpEYEnq3ZDzNnkiRJLWLnTJIkqUUc1iyG5k1wyOIdVWItmDdeJc7uiaEqcQAWDNfZp9Hd9d4yY2P12jeycLRKnF1V0+3z60XaElXiLNhaJQwAu1bU2afhY3ZViQNw3KGPVou1bXSkSpzvPnpolTgAux5bVC3W0ofrfDc/6P6JKnEAYrzOH+Dwznp/yAsfrvN/AWDpfXU+83YftLhKnFnjsGZPZs4kSZJaxMyZJElqhrdv6ouZM0mSpBYxcyZJkppj5qyngWXOIuKFEXFb1/JkRLyzrPv3EXFnRKyPiD/uqnN+RGwo687sKj85Im4v6z4cEVHKRyLiilJ+Y0Qc3VVnbUTcVZa1g2qnJElSTQPLnGXmncBJABExBDwAfDYifho4B3hRZu6KiMPLNscDa4ATgCOBL0bECzJzHLgYOA+4Afh74CzgGuBcYHNmHhsRa4APAL8YESuAC4BT6PTRb4mIqzNz86DaK0mS+mDmrKem5pydDnwnM+8F3g68PzN3AWTmI2Wbc4DLM3NXZt4NbABOjYiVwLLMvD4zE/g48NquOpeVx1cBp5es2pnAuszcVDpk6+h06CRJklqtqc7ZGuBT5fELgJeVYcivRMSPlvJVwP1ddTaWslXl8dTyPepk5hiwBTh0hlh7iIjzIuLmiLh59Intz6J5kiSpH5HNLHPZwDtnEbEAOBv4dCkaBpYDpwH/EbiyZLumu6JlzlDOM6zzdEHmJZl5SmaeMv+QOX5RP0mStF9oInP2KuDWzHy4PN8IfCY7bgImgMNK+VFd9VYDD5by1dOU010nIoaBg4FNM8SSJEmzKRta5rAmOme/xNNDmgB/B7wCICJeACwAHgOuBtaUMzCPAY4DbsrMh4CtEXFaybC9GfhciXU1MHkm5uuBL5V5adcCZ0TE8ohYDpxRyiRJklptoNc5i4jFwM8Ab+sqvhS4NCK+CewG1pYO1fqIuBL4FjAGvKOcqQmdkwg+Biyic5bmNaX8o8AnImIDnYzZGoDM3BQRFwJfLdu9LzM3DaaVkiSpL/tBVqsJA+2cZeZ2OhP0u8t2A2/ay/YXARdNU34zcOI05TuBN+wl1qV0OoKSJElzhncIkCRJjZnrZ1I2wXtrSpIktYidM0mSpBZxWFOSJDXHYc2ezJxJkiS1iJkzSZLUGE8I6M3MmSRJUouYOZMkSc0xc9aTmTNJkqQWMXNWzJ83zsrFT1aJ9eTowipxdozNrxIHYOGC0Spxxiba2Z8/6pAnqsRZPLy7ShyA7yw7rFqsJx4+qEqc8Ufr/cnHWFSJ89R9y6rEAbj5yTp/ewA5Xum9PlrnOAEMbx2qFmtkc530xcgT47036tPE/ErvqSPqHadtK5dWi7VgW51jvuTBOZx68vZNfWnnf1pJkqQDlJ0zSZLUiGhw6bkvEQsj4qaI+HpErI+IPyzl742IByLitrK8uqvO+RGxISLujIgzn+Xh2CuHNSVJ0oFoF/CKzNwWEfOBf4yIa8q6D2XmB7s3jojjgTXACcCRwBcj4gWZWW9svzBzJkmSmpMNLb12o2NbeTq/LDPVPAe4PDN3ZebdwAbg1P4avW/snEmSpP3RYRFxc9dy3tQNImIoIm4DHgHWZeaNZdVvRMQ3IuLSiFheylYB93dV31jKqnNYU5IkNabBOwQ8lpmnzLRBGZI8KSIOAT4bEScCFwMX0smiXQj8KfBWpp/KNpDWmDmTJEkHtMx8AvgycFZmPpyZ45k5AfwVTw9dbgSO6qq2GnhwEPtj50ySJDWnJXPOIuI5JWNGRCwCXgl8OyJWdm3288A3y+OrgTURMRIRxwDHATft+wHozWFNSZJ0IFoJXBYRQ3SSVVdm5ucj4hMRcRKdLt49wNsAMnN9RFwJfAsYA94xiDM1wc6ZJEk6AGXmN4Afmab8l2eocxFw0SD3C+ycSZKkJnn7pp6ccyZJktQiZs4kSVIzstFLacxZZs4kSZJaxMyZJElqjpmznsycSZIktYiZM0mS1BjnnPVm5kySJKlFzJxJkqTmmDnrycyZJElSi5g5kyRJjXHOWW9mziRJklrEzFmRCWNZp6+6cGi0SpzNuahKHICFw2NV4ixbtqtKnNqOWLS1SpwlQ7urxAHYvGRxtVhbRurEmphf709+pM4hZ3h7ve+IE48urBZrXp0/GSYqfsrW2ieAifl10hdbjqnXwLFFUSXOxIIqYTqxhurF2rW8TvuGd1QJMzsS55z1wcyZJElSi5g5kyRJzTFz1pOZM0mSpBaxcyZJktQiDmtKkqRGBF5Kox9mziRJklrEzJkkSWqOmbOezJxJkiS1iJkzSZLUmEhTZ72YOZMkSWoRM2eSJKkZ3r6pL2bOJEmSWsTMmSRJaozXOevNzJkkSVKLmDmTJEnNMXPWk5kzSZKkFjFzJkmSGuOcs97MnEmSJLWImTNJktQcM2c9DSxzFhEvjIjbupYnI+KdXet/JyIyIg7rKjs/IjZExJ0RcWZX+ckRcXtZ9+GIiFI+EhFXlPIbI+LorjprI+KusqwdVDslSZJqGljnLDPvzMyTMvMk4GRgO/BZgIg4CvgZ4L7J7SPieGANcAJwFvCRiBgqqy8GzgOOK8tZpfxcYHNmHgt8CPhAibUCuAB4KXAqcEFELB9UWyVJkmppaljzdOA7mXlvef4h4HeBz3Vtcw5weWbuAu6OiA3AqRFxD7AsM68HiIiPA68Fril13lvqXwX8RcmqnQmsy8xNpc46Oh26T+1tB3eOzWf9w8999i0FVh+ypUqc5yx6qkocgGULdlSJc8TI1ipxart3+4oqcb7x6MoqcQCe2LKkWqwcrfM9KufVG08YXxDVYtUyMb9e+4Z31GnfyKZ2juHERLviAIxsqRNsdHG99+au5fVixXidODmXZ4unJwT0o6lf8RpKxygizgYeyMyvT9lmFXB/1/ONpWxVeTy1fI86mTkGbAEOnSHWHiLivIi4OSJuHnty+zNrmSRJUkUDz5xFxALgbOD8iFgM/D5wxnSbTlOWM5Q/0zpPF2ReAlwCsOjYI+3LS5I0aP637amJzNmrgFsz82HgB4FjgK+X4crVwK0R8Vw62a2juuqtBh4s5aunKae7TkQMAwcDm2aIJUmS1GpNdM5+iTKkmZm3Z+bhmXl0Zh5NpxP1ksz8HnA1sKacgXkMnYn/N2XmQ8DWiDitzCd7M0/PVbsamDwT8/XAlzIzgWuBMyJieTkR4IxSJkmSZknQmXPWxDKXDXRYswxj/gzwtl7bZub6iLgS+BYwBrwjMyenT74d+BiwiM6JANeU8o8CnygnD2yiM7eNzNwUERcCXy3bvW/y5ABJkqQ2G2jnLDO305mgv7f1R095fhFw0TTb3QycOE35TuANe4l9KXDpvu2xJEkaqJzjaa0GzOUTciVJkvY73r5JkiQ1Zq7PB2uCmTNJkqQWMXMmSZKakXidsz6YOZMkSWoRM2eSJKkxNe/Hur8ycyZJktQiZs4kSVJznHPWk5kzSZKkFrFzJkmS1CIOa0qSpMZ4EdrezJxJkiS1iJkzSZLUjMQbn/fBzJkkSVKLmDmTJEmNcc5Zb2bOJEmSWsTMWTGxa4jd31lWJdbmE3ZXifO8wzZXiQOwZHhXlTijOVQlDsA92w6tFuubD6ysEmd82/wqcQCYX/EeJUN1vmqOLxuvEgcgJqJKnAWb68QBWPRotVAseqzOsap5q5rdS+t9n56o9FYf3lkvDTJ/e52DNT6/3ufUvNFqoarGmtPMnPVk5kySJKlFzJxJkqRGBM4564eZM0mSpBYxcyZJkpqR6XXO+mDmTJIkqUXMnEmSpMY456w3M2eSJEktYuZMkiQ1x8xZT2bOJEmSWsTOmSRJUos4rClJkhrjCQG9mTmTJElqETNnkiSpGQlMmDrrxcyZJElSi5g5kyRJzTFx1pOZM0mSpBYxcyZJkhrj2Zq9mTmTJElqETNnkiSpOWnqrBczZ5IkSS1i5kySJDXGOWe9mTmTJElqETNnk+YlY0vHq4RaODxWJc6uiaEqcQDue+K5VeI8+OSyKnEAntp4ULVYI4/XOVZD8+t9pYuxqBZreEelONvrxAFY9NhElTgLttb5uwNY8ORotViM13kvPLVqYZU4AFnx63StWDuX19up7YfXiRV13poALHiy3mfCvEpvz/GROnFmReJ1zvpg5kySJKlFzJxJkqRGBBCerdmTmTNJkqQWsXMmSZLUIg5rSpKk5lQ8YWN/ZeZMkiSpRcycSZKkxnhCQG9mziRJklrEzJkkSWqGF6Hti5kzSZKkFjFzJkmSGpLgnLOezJxJkiS1yMA6ZxHxwoi4rWt5MiLeGRF/EhHfjohvRMRnI+KQrjrnR8SGiLgzIs7sKj85Im4v6z4cEVHKRyLiilJ+Y0Qc3VVnbUTcVZa1g2qnJEnqX2Qzy1w2sM5ZZt6ZmSdl5knAycB24LPAOuDEzHwR8C/A+QARcTywBjgBOAv4SEQMlXAXA+cBx5XlrFJ+LrA5M48FPgR8oMRaAVwAvBQ4FbggIpYPqq2SJEm1NDWseTrwncy8NzO/kJljpfwGYHV5fA5weWbuysy7gQ3AqRGxEliWmddnZgIfB17bVeey8vgq4PSSVTsTWJeZmzJzM50O4WSHTpIkzZbMZpY5rKnO2RrgU9OUvxW4pjxeBdzftW5jKVtVHk8t36NO6fBtAQ6dIdYeIuK8iLg5Im4e3/bUPjZJkiSpvoF3ziJiAXA28Okp5b8PjAGfnCyapnrOUP5M6zxdkHlJZp6SmacMLV0yfQMkSVIdCTHRzNJLRCyMiJsi4usRsT4i/rCUr4iIdWXO+rruaVF7mxtfWxOZs1cBt2bmw5MFZYL+a4A3lqFK6GS3juqqtxp4sJSvnqZ8jzoRMQwcDGyaIZYkSRLALuAVmfli4CTgrIg4DXg3cF1mHgdcV573mhtfVROds1+ia0gzIs4Cfg84OzO3d213NbCmnIF5DJ2J/zdl5kPA1og4rcwnezPwua46k2divh74UunsXQucERHLS4/3jFImSZJmU0vmnGXHtvJ0flmSPeezX8ae89y/b258xSPzrwZ6EdqIWAz8DPC2ruK/AEaAdeWKGDdk5q9l5vqIuBL4Fp3hzndk5nip83bgY8AiOnPUJuepfRT4RERsoJMxWwOQmZsi4kLgq2W792XmpsG0UpIktdBhEXFz1/NLMvOS7g1K5usW4Fjgv2XmjRFxREkMkZkPRcThZfNVdE5knDTtfPYaBto5K5mxQ6eUHTvD9hcBF01TfjNw4jTlO4E37CXWpcCl+7jLkiRpkJo7kfKxzDxlpg1KEuikcs3Vz0bE9/U1uvQ1n70Gb980aSIY2l5nlPf+Bw7tvVEfHnz84CpxAMa3zq8SZ+FDdeIAHPxYtVDM213n72PeaJUwAMREvb/Z4V11Yi16pF4DF3730TqBdu6qEwcYX3VYtVhPHntQlTjbVtWbPdLPJOd+DVV6T42PVAkDwESlj5eaf8e19gkgxntv04/52+f2ZSLaKDOfiIgv05lL9nBErCxZs5XAI2Wzxuaze/smSZJ0wImI50zepSgiFgGvBL7NnvPZ17LnPPfvmxs/iH0zcyZJkhoT7blA7ErgsjLvbB5wZWZ+PiKuB66MiHOB+yjTp3rMja/KzpkkSTrgZOY3gB+ZpvxxOnc2mq7OtHPja7NzJkmSmtOezFlrOedMkiSpRcycSZKkZiRQ8azj/ZWZM0mSpBYxcyZJkhoRZJvO1mwtM2eSJEktYuZMkiQ1x8xZT2bOJEmSWsTMmSRJao6Zs57MnEmSJLWImTNJktQMr3PWFzNnkiRJLWLmTJIkNcbrnPVm5kySJKlF7JxJkiS1iMOakiSpOQ5r9mTmTJIkqUXMnEmSpIakmbM+mDmTJElqETNnxbxRWPRwnb5qPragShyoFQcWbKnzTeXgu0erxAGIiXrfnsZH6vzu5o3W26d5u+tdabHWsYqxevuUC+bXCTQ8VCcO8ORxB1WL9fgJUSXOxEi991RUvHhnVPpTnjdeJw7A0M56sWrZvazO+wBgaKROnPnb6sSZFYmZsz6YOZMkSWoRM2eSJKk53r6pJzNnkiRJLWLmTJIkNcbbN/Vm5kySJKlFzJxJkqTmmDnrycyZJElSi5g5kyRJzUig4jUu91dmziRJklrEzJkkSWqI99bsh5kzSZKkFrFzJkmS1CIOa0qSpOY4rNmTmTNJkqQWMXMmSZKaY+asJzNnkiRJLWLmTJIkNcOL0PbFzJkkSVKLmDmTJEkNSciJ2d6J1jNzJkmS1CJmziRJUnM8W7MnM2eSJEktYuasS1QaBp83WifOwsfrfbtYdt/OKnGGN++oEgcg5w/VizVU53vG0BPbq8QB4JHHqoXKo1dVibPjyCVV4gDsOuzQKnF2HlzvfbD1mKgWa3TFeJ1AFafXxGi99s2LOrHmPVUlTEelj7yxRXXiAGS9tyfjI7Xi1HsfNM6zNfti5kySJKlFzJxJkqTmOOesJzNnkiRJLWLmTJIkNcfMWU9mziRJklpkYJ2ziHhhRNzWtTwZEe+MiBURsS4i7io/l3fVOT8iNkTEnRFxZlf5yRFxe1n34YjOaUYRMRIRV5TyGyPi6K46a8tr3BURawfVTkmSpJoG1jnLzDsz86TMPAk4GdgOfBZ4N3BdZh4HXFeeExHHA2uAE4CzgI9ExORJzBcD5wHHleWsUn4usDkzjwU+BHygxFoBXAC8FDgVuKC7EyhJkmZDdoY1m1jmsKaGNU8HvpOZ9wLnAJeV8suA15bH5wCXZ+auzLwb2ACcGhErgWWZeX1mJvDxKXUmY10FnF6yamcC6zJzU2ZuBtbxdIdOkiSptZo6IWAN8Kny+IjMfAggMx+KiMNL+Srghq46G0vZaHk8tXyyzv0l1lhEbAEO7S6fps6/iojz6GTkmH+QiTVJkgYqgQlvfN7LwDNnEbEAOBv4dK9NpynLGcqfaZ2nCzIvycxTMvOUocX1rpwuSZL0TDUxrPkq4NbMfLg8f7gMVVJ+PlLKNwJHddVbDTxYyldPU75HnYgYBg4GNs0QS5IkzSbnnPXUROfsl3h6SBPgamDy7Mm1wOe6yteUMzCPoTPx/6YyBLo1Ik4r88nePKXOZKzXA18q89KuBc6IiOXlRIAzSpkkSVKrDXTOWUQsBn4GeFtX8fuBKyPiXOA+4A0Ambk+Iq4EvgWMAe/IzMk7D78d+BiwCLimLAAfBT4RERvoZMzWlFibIuJC4Ktlu/dl5qaBNFKSJPVvjme1mjDQzllmbqczQb+77HE6Z29Ot/1FwEXTlN8MnDhN+U5K526adZcCl+77XkuSJM0eb98kSZIakjBh5qwXb98kSZLUImbOJElSMxIyvc5ZL2bOJEmSWsTMmSRJao5zznoycyZJktQiZs4kSVJzvM5ZT3bOBmBoV5033pKHdleJAzD/oSerxBk/ZHGVOABDW3ZUi5X317k71/j27VXiAAwtX14t1q7DFlWJs3vZUJU4AOPzp7uF7b7bdlSdOACjS+tNNB7aVmdgIYfa+Y9o+Kk6x32o3scU43Xe5kxUPOZDu+q9P2O89zb9GK73MaWWclhTkiSpRcycSZKkZmTChJfS6MXMmSRJUouYOZMkSc3xhICezJxJkiS1iJkzSZLUmHTOWU9mziRJklrEzJkkSWpIOuesD2bOJEmSWsTMmSRJakbijc/7YOZMkiSpRcycSZKk5qRna/Zi5kySJKlFzJxJkqRGJJDOOevJzJkkSVKLmDmTJEnNyHTOWR/MnEmSJLWInTNJkqQWcVhTkiQ1xhMCejNzJkmS1CJmziRJUnM8IaAnM2eSJEktEpmO/QJExKPAvQ2+5GHAYw2+3mywjXPf/t4+sI37C9v4zPxAZj6ncsy9ioj/TacdTXgsM89q6LWqsnM2SyLi5sw8Zbb3Y5Bs49y3v7cPbOP+wjZqf+KwpiRJUovYOZMkSWoRO2ez55LZ3oEG2Ma5b39vH9jG/YVt1H7DOWeSJEktYuZMkiSpReycSZIktYids30QEQsj4qaI+HpErI+IP5yy/nciIiPisK6y8yNiQ0TcGRFndpWfHBG3l3Ufjogo5SMRcUUpvzEiju6qszYi7irL2ibbGBHvjYgHIuK2srx6f2tjWffvSzvWR8Qf729tLPs0+Tu8JyJu2w/beFJE3FDaeHNEnLoftvHFEXF92ef/GRHL5moby+sMRcTXIuLz5fmKiFhXXnddRCyfy+3bSxvfUH6nExFxypRt52QbVVFmuvS5AAEsLY/nAzcCp5XnRwHX0rmQ7WGl7Hjg68AIcAzwHWCorLsJ+LES8xrgVaX814G/LI/XAFeUxyuA75afy8vj5U21EXgv8DvTbL8/tfGngS8CI2Xd4ftbG6ds86fAe/a3NgJf6NrHVwNf3g/b+FXgp0r5W4EL52oby2v9NvC3wOfL8z8G3l0evxv4wFxu317a+G+AFwJfBk7p2m7OttGl3mLmbB9kx7bydH5ZJs+o+BDwu13PAc4BLs/MXZl5N7ABODUiVgLLMvP67PwFfRx4bVedy8rjq4DTy7ejM4F1mbkpMzcD64DqVz7u0cbp7E9tfDvw/szcVbZ7ZD9sIwBlX/4t8Kn9sI0JTGaSDgYe3A/b+ELgH0r5OuAX5mobI2I18LPAX3cVd+/TZVP2dU61D6ZvY2bekZl3TrP5nGyj6rJzto9Kavo24BE6b/obI+Js4IHM/PqUzVcB93c931jKVpXHU8v3qJOZY8AW4NAZYlU3XRvLqt+IiG9ExKVdwwz7UxtfALysDAt8JSJ+dOr+TtmvudjGSS8DHs7Mu6bu75T9mottfCfwJxFxP/BB4Pyp+ztlv+ZiG78JnF02eQOdzP0e+ztlv9rcxj+j88W2+27YR2TmQ2WfHgIOn7qvU/apze2D6du4N3O1jarIztk+yszxzDwJWE3n28yLgN8H3jPN5jFdiBnKn2mdqqZp44nAxcAPAicBD9EZEmOG/ZqLbRymk/o/DfiPwJXl2+f+1MZJv8TTWTNm2K+52Ma3A+/KzKOAdwEfLZvvT218K/COiLgFOAjYXTafU22MiNcAj2TmLf1Wmaaste2DA6ONqs/O2TOUmU/QmStwDp15AV+PiHvofIDeGhHPpfMt5aiuaqvpDLFsLI+nltNdJyKG6QzLbJoh1sB0tfGszHy4/JOYAP4KmJxkvd+0sbz+Z8pQ0k10vuUeNsN+zcU2Tu7P64Arujbbn9q4FvhMWfVp9sP3amZ+OzPPyMyT6XSyvzN1f6fsV1vb+BPA2eWz83LgFRHxN8DDZRiP8nNyisFcax/svY17MxfbqNqyBRPf5soCPAc4pDxeBPwf4DVTtrmHp08IOIE9J3Z+l6cndn6VToZmcmLnq0v5O9hzYueV5fEK4G46mZ3l5fGKptoIrOza5l105kTsb238NeB9pfwFdIYDYn9qY3l+FvCVKdvvN20E7gBeXspPB27ZD9s4ebLKPDpzj946V9vY1daX8/Rk+T9hzxMC/niut29qG7vKvsyeJwTM6Ta6VHqvzPYOzKUFeBHwNeAbdOZ8vGeabe6hdM7K89+n8632TsqZNaX8lBLjO8Bf8PTdGhbS+ba/gc6ZOc/vqvPWUr4BeEuTbQQ+Adxeyq9mz87a/tLGBcDflLJbgVfsb20s6z4G/No0dfaLNgI/CdxC5x/cjcDJ+2Ebfwv4l7K8f3J/52Ibu17r5TzdOTsUuA64q/xcMdfbN00bf55OZmsX8DBw7f7QRpc6i7dvkiRJahHnnEmSJLWInTNJkqQWsXMmSZLUInbOJEmSWsTOmSRJUovYOZMkSWoRO2eSJEktYudM0qyIiB+NiG9ExMKIWBIR66fc/1OSDkhehFbSrImIP6JzdfNFwMbM/K+zvEuSNOvsnEmaNRGxgM79AncCP56Z47O8S5I06xzWlDSbVgBLgYPoZNAk6YBn5kzSrImIq4HLgWOAlZn5G7O8S5I064ZnewckHZgi4s3AWGb+bUQMAf8cEa/IzC/N9r5J0mwycyZJktQizjmTJElqETtnkiRJLWLnTJIkqUXsnEmSJLWInTNJkqQWsXMmSZLUInbOJEmSWuT/B39buwCoYIQcAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "sat_batch[\"data\"].sel(example=EXAMPLE_I, channels_index=0, time_index=7).assign_coords(\n", + " x=(\n", + " \"x_index\", \n", + " sat_batch[\"x\"].sel(example=EXAMPLE_I).data\n", + " ),\n", + " y=(\n", + " \"y_index\", \n", + " sat_batch[\"y\"].sel(example=EXAMPLE_I).data\n", + " )\n", + ").swap_dims(\n", + " {\n", + " \"x_index\": \"x\",\n", + " \"y_index\": \"y\"\n", + " }\n", + ").plot.imshow(x=\"x\", y=\"y\", figsize=(10, 10))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "892ae298-046c-479f-a639-500b4fd9491b", + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import HTML\n", + "from matplotlib.animation import FuncAnimation\n", + "import numpy as np\n", + "import pandas as pd\n", + "from nowcasting_dataset.data_sources.optical_flow.optical_flow_data_source import crop_center" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "33fcdc69-a3c7-49f4-ae3a-3dfea70fbbf3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABQgAAAI4CAYAAAAmvQRNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABM+klEQVR4nO3deZylZ1kn/N9VS+9Nks5GEkLCLqAQFAFFATcERwXXcUPwdQYc9VVnUF91RkVHZ9QZ9w1xQFBxYRSRURQQRWDGQSIi+04gIfvSnfTeVXW/fzxPQ6Xp6q4kfVd11/P9fj716a5z6tzXfc5zznmu83uWU621AAAAAADTNLPeEwAAAAAA1o+AEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAKe5qrpvVe2tqtl7OM7rq+rfrHBdVdXvVNVtVfWPVfWkqrrmntQ7XVTV86vqR09w/Y9U1f9Y4zm9uKqetZY1AVYiIITjOF2aoapqVfXA8f8vrqqfGv//+VX1vo51P1Grt/Gxfv1a1GLtVdWzqurFd/N2b+owJQBYE+O67B1Vtb+qrq+q36yqs+/C7a+qqi8++ntr7WOttR2ttcUuEx58XpIvSXKf1tpjOtZZc62172it/efk+L1+a+2/tNaOG5yuh6p6cFX9eVXdVFW3VtWrq+ohx/zNvx+fW3uq6kVVtXm8fHNVvbCqPlpVd1TVP1fVU5fdblNV/cn4HGtV9aRVzOeLquq94/P576rqsmOu/8yqesMYYt9QVd97grGqqn62qm4Zf36uqmrZ9VdU1RvH+3VNVf3YCcb69PGxubmq2nGu31VVf1ZV+8bH45tOMNbDqurKMSC/rar+pqoetuz651XVkfE+Hv25/4keNziTCAg5o40rtQPjm/P1Y7C1Y73ndTxVdfm4Ap67p2O11t7YWvtEg3BsA7mWTrQV+hSN/9+r6gNjc/PeqvrWY66/oqr+aWxW/qmqrlh23TPHy24fm4ufW/74V9V3j03AodWEWCdrMKpqW1X9xtig7KmqN5xkvBUbrbGxe/7YYN1aVf+rqi45wVgvqKr3VdVSHWdL9EoN5Apj/X5VXTc+bu9fvnyXPY+XN0Yrbo3fqMZG9CePc/nTxsd5ro4J2qvq28flfce4XP+yqnaO11WduFG+fHyO7B/H+OJj6p5fVX9QVbvHhvalPe8/ACdWVc9N8rNJfiDJWUkel+SyJK+tqk3rObeTuCzJVa21fes9keM5FX30GeTsJK9M8pAkFyb5xyR/fvTKqvrSJD+U5IuSXJ7k/kl+Yrx6LsnVSZ6Y4fn3o0leVlWXLxv/TUm+Jcn1J5tIVZ2X5OXjOLuSXJnkj4+5/q+T/FaSc5M8MMlrTjDks5M8PckjkzwiyZcnec6y6/8gyRvGWk9M8u+q6itXGOtIkpcl+fYVrv/1JIczPIbfnOQ3q+rhK/zttUm+dqx7XobH/4+O+Zs/HoP6oz8fXulOwplGQMhG8BWttR1JrkjyqCQ/vL7T4RTbl+QrMjQ3z0zyy1X1ucmw9TNDo/T7Sc5J8pIkf76s8d6W5PsyrOAfm6GB+v5lY1+b5KeSvGiVczlZg/GCDA3FQ8d///1KA52s0UryvUk+J0PTdHGS3Ul+9QRz+5ck35nkrcepdaIG8nj+a5LLW2v3SvKVSX6qqj7rmL85e1lj9J9PMNZG9eIkz1ge4o2ekeSlrbWF5RdW1ROT/Jck39ha25nhOfKyZX9yskb5D5P8c4am+z8m+ZOqOn/Z9S/P0OBfluSCJP/9Htw3AO6BqrpXhvXs/9ta++vW2pHW2lVJvj7D+/S3jH/3vBr24vrjcePRW6vqkeN1v5fkvkn+17gx7gfrmI3N44bL36mqa8eNQ68YLz+nqv6ihj3Pbhv/f59VzPvbk/yPJJ8z1vyUXqGqHlrDxuHdVfWuo6FNVd1vvGxm/P1/VNWNy273+1X1fSvUvaqqfriq3j3O93eqast43ZNq2Mj7/1XV9Ul+p6pmquqHqupD40a1l1XVrmXjfV5V/Z9xPlfXuOH06Ia7qtqe5K+SXLxsY+fF4/L4/WXjfOV4H3eP9/mhx8z5+6vq7TVsfP3jZXM+b3zMd9ewkfeNRx+Xu6K19o+ttRe21m5trR1J8otJHlJV545/8swkL2ytvau1dluS/5zkWeNt97XWntdau6q1ttRa+4skH0nyWeP1h1trv9Rae1OS1eyR+tVJ3tVa+5+ttYNJnpfkkVX1aeP1/yHJq1trL22tHWqt3dFae88Jxntmkp9vrV3TWvt4kp8/OvfR5Rn6qcXW2ocyhJnHDfVaa+9rrb0wybuOvW5c1l+T5Edba3vH+/vKDP3a8cbaPT5mLUlleGweeIL7ARuKgJANo7V2fZJXZwgKkyRV9bhlDcK/1LLd56vq26rqPWND9uGqes6nDHocNfjFqrpxbAjeXlWfPl73r2rYhf/2sSF53rKbHt2bbPfYiHzOeJv/Z5zHbTXslXRZTqKWHRZxvAbyZPf9OOM9qoam9I6q+uMkW5Zdt2KTWVU/neTzk/zaWPvXxst/ebz/t9ewB9/nr+axPZ7W2o+31t47NjdvTvLGDMFZkjwpwxbSXxqbkV/JsDL/wvG2vznubXl4bD5emuTxy8Z+eWvtFUluOdk8TtZg1HDIx1cmeXZr7aaxofmnEwx5skbrfhkarRvG6/8oKzRG43359dba65IcPM7VKzaQK4z1rtbaoaO/jj8POMF9uduq6tKqevn4/Lrl6HNo2fX/fXzefaTufGjMiq/fZR8mnju+Tq+rqm87xVN/RYZg9xPP7ao6J0Ow97vH+fvPTvIPrbV/TpKx2X9Ja+2O8foVG+WqenCSz0zy4621A621P03yjgzPx1TVk5NcmuQHWmt7xg+i/3yK7y8Aq/e5GXqply+/sLW2N0Mw9SXLLn5akv+ZYZ3yB0leUVXzrbVnJPlYxg3hrbWfO06d38uwMfThGTYO/eJ4+UyS38kQRt43yYEkv3ac29/JGLJ8R4b11Y7W2o8vv76q5pP8rwx7hl2Q5P9N8tKqekhr7SNJbs+wsT4Z1o97l4VqT0jy9yco/81JvjRDv/HgJP9p2XX3zvD4XJZhg9r3ZNio9sQMG1Fvy7ARN1V13wyP8a8mOT/D54K3HXM/9yV5apJrl23svPaY+/rgDBvnvm8c51UZeu3le39+fZKnZOjZHpFP9lbPTXLNeLsLk/xIhl7qnnpCkutba0f71odn2EB81L8kuXBZgLj8/lyY4XH9lBBtle5Ua3wMP5RP9qaPS3Lr+NnjxhqOfLnvascb/7+8z/2lJN9aVfNjj/05Sf7mbsz7wUkWW2vvX6nW+Fnp85bfqKp2Z+ipfzXDBt7lvmIMft9VVf/ubswJTlsCQjaMMbR6apIPjr9fkuQvM+whtivDnmN/Wp/c6+bGDB/m75Xk25L8YlV95ipKPTnDCvrBGXb9/9f5ZMC0L8m3jpf/qwy7wz99vO4J479H97z6h/G6H8kQFp2fIfz6w7tyv4/XQK7ivn/C2Oi8IkOTuStDk/o1y/5kxSaztfYfxzl/91j7u8fbvCVDQ3a02f2fNW5VvSeqamuGoOVoc/PwJG8ft/Id9fasHKQ9IXe/MTpZg/HYJB9N8hM1HGL8jqr6mmMHWeZkjdYLkzy+hi3a2zI0zn91N+d+wgayhsOif2P5DcbL9id5b5LrMjTGy310DOF+p4a9Ie+yGk6y/hcZHrfLk1ySOx/G8dgk78uwB+jPJXlh1Sf22DvZ6/feGfY6vSTDISe/PgZ4x5vHb4zN4fF+3n6827TWDmTYA3D5Ie9fn+S9rbV/Oc5N3pzkS6vqJ6rq8fWph3ifqFF+eJIPLwsTj73+cRkep5eMIetbathjEYD1cV6Sm4/dm3x03Xj9Uf/UWvuTcQ+xX8gQLD7uZAWq6qIMfe93tNZuGzcO/X2StNZuaa39aWtt/7ju+OkMYdo99bgkO5L8zLjx9W8zrMe/cbz+75M8saruPf7+J+Pv98uwvj7e+vGoX2utXd1au3Wc7zcuu24pw0ayQ+P69zlJ/uO4Ue1Qho2sX1vDnpXfnORvWmt/OD4mt7TW3nY37uu/TvKXrbXXjsvmvyfZmiH8PepXWmvXjnP+X/nkTgpHklyU5LJxDm88ple9y8bPOb+eYU+9o3Yk2bPs96P/33nMbeczbCR/SWvtvXdzCsfWOlrvaK37ZNjY+b0ZPi98JCf+THO8ue9Y1uf9RYZDfQ9k6EVf2Fp7S4d5p7V29rjhP8svy9BHfneGIziOelmGo0DOT/Jvk/xYVS1/rsIZTUDIRvCKqrojw3k2bkxydGvntyR5VWvtVePeZ6/NcBjnlyVJa+0vW2sfaoO/z7A1dDV7uh3JsFL5tCTVWntPa+26cczXt9beMdZ7e4YV44kasuck+a/jGAsZtlBdUavYi/AkTnjfj/G4JPMZ9sI70lr7kwwBX8b7dJebzNba74+3W2it/XySzRnOn3JPPT9Dc/nq8feTrvSPGvcge3Tu/qGXq2mMPn287OIMDcVLlm05v6vjvT9D8PvxDFvkH5rkU855dzfnfqcGsrX2na2171x+g/H3nRleEy9PcnSPwpszhLSXZThMZWeGpvPueEyGx+oH2nAozMFjGrSPttZ+uw0nY39Jhmb7wnF+J3v9Hknyk+Nz+lVJ9maF5+B4/89e4ecRJ5j/S5J83RhcJ0NY+JIVarwxw4aAz8wQ3t9SVb9Qn/wmyhM1yqt57j05yd9lCEZ/PsOh9ncruAXgHrs5yXl1/PPlXTRef9TVR//TWlvKsOfZxauocWmSW9twZMCd1HBO5N+q4XzJt2c4iuXsuofffjzO6+pxnkd9NMPGuGQICJ+UYYPsG5K8PkPP+MQkbzzmdse6etn/P5o7PwY3teFoiqMuS/JnRzfmJXlPhkNBL8zwuHzoLt2r47t4nEeSTyybq/PJ+5rc+dx9+zOsr5Pkv2XYYeE141EOP3RPJjJu4H9Nkt9orS0P3fZmCF6POvr/O5bddibDTgCHM/Smq6l39Nuy91bV3hVqHa13tNaBJH/WWnvLuKx+IsnnVtVZNXw79NHxnn+Cue9trbUaDhf/6wx975YMy/RLq+pOveoqnWzeKxo33j8/ye9W1QXjZe8eQ+HF1tr/SfLLGYJM2BAEhGwET2/DOb2elCG0O/qh+LIMH953L2sgPi9DY5aqempV/d9xF/HdGcKzk36gHreW/lqGrXg31PDlEPcax3xsDV8kcFNV7clwmMaJxrwswzn1js7v1gyHyK74ZRSrdML7foyLk3z8mC2bn2iI7k6TWcOhne+p4RDs3Rm2wN2jsKKq/luGAO7rl811VSv9cU/Nn0ny1NbazVmFqvqrZc3MN6+i1oEModRPjVvV/z5DYPPku9lo/WaGpujcJNszhHR3dw/CkzaQxzM2P2/KEED9u/Gyva21K8fw94YMzeaTj74G7qJLM4SAx9vDIlnWeLfW9o//3ZGs6vV7yzHjLm/cT4nxsbkpydNq+Aa7z86wx+xKf/9XrbWvyLBn7dMyHIp09AtgVmyUj3Pd0euXP/euasN5io601v4ow4eYxweA9fAPGTasffXyC2s4XclTk7xu2cWXLrt+JsM69+jhrifa6+zqJLvq+N+K/NwMG8Ue24bzCR89iuXY8+beVdcmubTufD69+2bYmJkMAeHnZ+jJ/z7DeeMenyEgPNHhxcmyx2Ecc/khv8c+Dldn6OmWb9Db0oZTdFyd1Z0W5WR79F2boZ9OMpxiaJzjx1e8xdGBh/PvPbe1dv8M59H+D1X1RauY06cYj354TZJXttZ++pir35Xh3MVHPTLJDW08BHmc8wszBKdfM+4JeVLtk9+WvaMN53n/lFrjc/kB+eSROW/PnR/To/+vNnw79NHxvuMEcz861v0zHLXzu2O/eU2GI0yOt6PDybw/yVxVPWiFWiczk+Ew/pU+mx09VyFsCAJCNowxkHlxPrmH2NVJfu+Y5mF7a+1nxsP7/nT82wvbsBv5q7LKN/jW2q+01j4rwyF+D87wDXXJEA68MsmlrbWzMmx1Ojrm8RqRq5M855g5bh23SN0Vx2ucjnvfj3Pb65JcsmyX/mRozI46WZN5p9o1nG/w/8twuOU542O7J/dg5VnDSbKfmuTJrbXbl131riSPOGbuj8iylX5VPSXJb2c4BPsdq63ZWnvqsmbmpTl5g3HcQ1HHse5Oo/XIJC9uw7nqDmU4B8pj7uZeYSdsIFdhLis3259oAO/GvK5Oct8V9rBY0T19/R5nvOfXnb+VefnPyRrI382w5+AzkrxmDE1PqA179b4uyd9mCL2TEzfK70py/xq/8fg41x/blAOwjlprezLsQfWrVfWUGs6jdnmG07hck2GPrqM+q6q+elwXfl+GYPH/jtfdkCEsOV6N6zJsOPyNGs4XPV9VR3u0nRk2Hu0e98b68eONcTe8OcPpdH5wrPekDAHYH41z+sBY91uSvGHs2W7IcOqakwWE31VV9xnn+yO58xe3Hev5SX66xiNuqur8qnraeN1Lk3xxVX19Vc1V1blVdcVxxrghyblVddYKNV6W5F9V1RfVcIjuczMsm5P26FX15VX1wLE/vT3D3o2r+SKQY8e5V4ajZv53a+14eyH+bpJvr6qHjUHif8rwWeio38xwBMpXtOHQ7GPH31yfPAXQpqrackxPvdyfJfn0qvqa8TY/luE0P0cPWf6dJF9VVVeMj9ePJnlTa233CuP9bobg9JKqujjD43t07u8fplffVMMX0tw7wyHfxz1EvQZbkmwaf98y9opH9wJ8eZKfrKrtVfX4DBtpf2+Fsb6khnOzz46P/y9kOMfle8brnza+3qqqHpPhfJh/fryx4EwkIGSj+aUkXzI2Ar+f4SSyXzq+yW+p4csL7pNhBbI5w94/CzV8+cGTV1Ogqj67hj0F5zM0SQfzyZX+zgyHexwcVxrftOymN2U4h8ryRu/5SX64xm/CrWE3/K+7G/f72AbyRPf9WP+QZCHJ94yN1FdnOPTzqJM1mcfW3jmOd1OGQO3H8ql7P61aVf1whsfxS44TaL0+w2P/PWOTc/TQib8db/uFGRrFr2mt/eNxxp4bG4rZJEcfp+OGVatoMN6Q4ZDgHx7HfXyGLeivPt54OXmj9ZYMJ2c+a3yufWeGk2kfdw/Iqto0jlNJ5sf7cvQ9/mQN5PJxLqiqb6iqHeNz50sznAfo6GP62Kp6yNiwnZvkV5K8fvwwdFf9Y4aA+mfGx3TL+LidzN1+/R5Pa+07lge4x/ys+MUwo99N8sUZzkNz3MOLk080lN9wTFP5xHzyQ+CKjXIbznv5tiQ/Pj5GX5UhCP/T8bZ/luScqnrmuMy+NsOW7v991x8NAE6FNnypyI9k2Jh1e4Zw7eokX9Q++UVgyRAu/OsMIcQzknz1sj29/muS/1TD0SDff5wyz8hw9MJ7M5xm5/vGy38pw/nybs6wnvnrU3SfDmf4QranjmP/RpJvbXc+r93fZ9iL/2PLfq/c+Txux/MHGfaU+/D481Mn+NtfzrBB/jU1nGbo/2Y4b3HGul+WYT16a4b15yOPHWCc8x8m+fD4+F58zPXvyxB0/up4X78iQ9B2+CT3I0kelOELNfZm6LN/o7X2+lXc7lhfleHohG+rO2+8vO84x7/OcI7mv8tw9M9HM/bpNYSnz8lwXsTr685HxRz1vgw9/iUZ+tUDWbbX5HKttZsyBL0/neG5+tgk37Ds+r/N8Hz/ywzPxQfmzp+DjvVbGc7b+I4k7xxv91vjWLdn2Pv234+13jb+zbF7UB512Tj3oxtOD4z37ajvzPB6uDHDMv93rbXlOxPsrU9+oeLZ49/syXCo+gOTPKV98hD3b8hw+PgdGXq3n22trdj/wRmntebHzxn7k+SqJF98zGW/meRPx/8/NkNjcmuGMOEvk9x3vO67MoRbuzOEPH+U4fDQZAh2rlmh5hdl2GNnb4aG4aVJdozXfW2GlfMdGU6u+2tJfn/ZbX9ynMfuJI8bL3tGhpXj7Rkaxxct+/uW5IHj/1+80vwyBFUfG8f9/pPd9+Pcp0dnaNzuyLDF9o+X1bo4QxC3N8MWveeM85obr/+c8fLbMoRFsxkOZ7g9Q/jzg8dbTstqPylDwLTSMm4ZttjuXfbzI8uuf1SSf8rQDLw1yaOWXfd3GcLK5bf9q2XXPy+f/Jbeoz/PO8FcdmX4Qpd94+P9Tcdc//AMjeC+JO9O8lUnef5+cYam/sD4GF++7Lpzx+fWjeNyfVOSx5xgrNcf5748adn1/yHD8/32DFt5Ny+77vlJnj/+//zxebN7/Nt3JPm3y/72GzOceHrfuHx/N8m9TzCvZ2XYE3Kl6+87Pqa3ZHg9/cqy273pOM+Fo6+Hu/T6PdFz8BS8D70+w/N/8zGXv3jZnJ6Q4ZCymzO8zt6f5AeX/W1laPJvHX9+LsNhOUevv3ysc7TpPfZ97/PHZbU3w/lGP7/HffXjx48fP6fuJ0Mf8vvrPY/1/um5jj7df8Ze4VnrPQ8/fvz4aa0NHz4A1ksNh6c8r7X2pPWdCT1U1bMyBJXPWuepAMBppaqel2HD17es91zWU1VdleTftNb+Zr3nstaq6sUZNpS/eJ2nApC7dN4nAAAA4JR4RYY9KAHWnT0IgXVVw0m7n2TL6cY0ng/08tbaK9Z5KgAAAKxAQAgAAAAAE3bGHGJ83q7Zdvml811r7G9LXcdPkmsPn929xlo4a+5A9xrnzx45+R/dA4vpv7yvObKze43Z6h/yz9Xiyf/oHrrpjrv9RcertgZ3I3Pb+j5vk+TIwmz3GnO39/+S+9lD/V+Di5vX4H6cu5ovNLz7Ns8udB0/SXbOHjz5H93TGjP9XxsLa7DNcy79i/SucM01i7n11qXqXOaU2rlrrp1/yeauNXYvbOs6fpLMVf/3vc1r8FqbX4MV6lz61liL/mlmDd4vFtP/pXyk9f/IeHCp7+e8JLl9YUv3Gof39b8fc/1X2Zk92P+9qhbXoA/c0r9nPrKj72uwNvV/nOwzdnqpNejQDn342ptba+cfe/kZExBeful8/vHVl3at8fbD/d9tf+yjT+teYy182QXv6F7j2Wdd23X8vUv9l/cPXvek7jV2rkGXcN78Hd1r/Obrv7h7jbm9/cOi86+4oXuNG245q3uNXa/t30Sf9cH+Gxr2PGBr9xq7nvmxruNftuPWruMnyRec9d7uNb5w2zXda9y02P81vmumfzDSu8KXf9nNnSuceudfsjk/9fKHd63xipsf1XX8JDlv077uNR64rf966N5ze7rXOH/u9q7jnz3Tfx20ZQ2C1N1LfYPzJLl+4ezuNd538KLuNV5340O61/jYlZd0r7Hr3d1L5Oz37u1eY3ZP/9fg3k/b1b3GtU/o23vM3md/1/GTZHENdj7IGmyUWQtrsXV1br7/uuP9X/vjHz3e5f07aQAAAADgtCUgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwITNrfcEpmamlrrXeN9NF3Sv8aF79a9xzfb3dx3/Xw6f13X8JNk+e6h7jaVWa1BjDbYl9L8bWdjZ//W3Z//W7jW2vGMNauxe6F5j3322dK9x82P6L/NH7bi16/iHl/qvqi+fv7l7jX1LrXuNLbXYvcZ89X+z2t/5seq/JE69s2aO5Knbr+1a42Gbr+s6fpLctLi9e42Dbb57jd1rcD+uXzi76/hnbzrQdfwk2b20uXuNXTMHu9fYsgbriNfteVj3Gh/+yIXda2y9o/86opb69za1BuvstnVT9xq3Pah/D7V41uGu4y/d0L/vnz3Q/7Pe4pY16D7O6bsskmTz1iPda2zb0j9DWIk9CAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABM2Nx6T2C1jrTFXLewt3ONTV3HT5LP2fXh7jWu23ev7jUunL+9e40jre/4dyxu7VsgyVlzB7rXuPnIju41PnzgvO41svNI9xJtof82kcPv7//6O++qpe41Dp41273G7k/rXiKP+PSPdK9x2+FtXce/bNutXcdPkkdt7v+c2lz936v2LPV/z71hsf9jddnc5q7jz9eZt314JpUd1fdxmc2hruMnyWLr/9jvXtzevcZNCzu71zh/7o6u41+/2H99ff5s3/uQJNtn+r8n/c0dD+pe49Xve2j3GvM39f/oO7PQvURm+rfMqYP978gtjz6ne419ly92rzG7tW+NxcP91xvV+XP3WhXZvvNg9xpnbe1f4+wt/fvZt61w+ZnXIQIAAAAAp4yAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACZtb7wms1v42l7cePm+9p3GP7V/c3L3GTzzold1rXDJ3e/cav3bzE7qOf+uR7V3HT5JLtuzuXmPzzEL3Gu/afVH3GjvudaB7jZ1bDnWvcd0dF3SvcevDqnuNxS3dS2Tufnu71/j0s67tXuOtt13adfxz5vZ3HT9JNtd89xobxc5q3Wv0Xh4z6f8ecqotpOWWpb7riasXdnUdP0neffCS7jWuOnhu9xp3LPRfSTxo241dxz97276u4yfJkTbbvcZf73tw9xrPf+/nda8x95H+z6ktt555733HM3u4/3ro1s88p3uNmz9rqXuNrRf17zVnZvouj0Ozi13HT5Kc0/+1cfb2g91rXLCj//LeMd//8+SW2f6f71diD0IAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwroGhFV1aVX9XVW9p6reVVXfO16+q6peW1UfGP89p+c8AABYW/pAAIAzR+89CBeSPLe19tAkj0vyXVX1sCQ/lOR1rbUHJXnd+DsAABuHPhAA4AzRNSBsrV3XWnvr+P87krwnySVJnpbkJeOfvSTJ03vOAwCAtaUPBAA4c6zZOQir6vIkj0ry5iQXttauS4bmMckFK9zm2VV1ZVVduefWxbWaKgAAp9A97QNvuWVpzeYKADBFaxIQVtWOJH+a5Ptaa7ev9nattRe01h7dWnv0Wbtm+00QAIAuTkUfeO65vlcPAKCn7t1WVc1naApf2lp7+XjxDVV10Xj9RUlu7D0PAADWlj4QAODM0PtbjCvJC5O8p7X2C8uuemWSZ47/f2aSP+85DwAA1pY+EADgzDHXefzHJ3lGkndU1dvGy34kyc8keVlVfXuSjyX5us7zAABgbekDAQDOEF0Dwtbam5LUCld/Uc/aAACsH30gAMCZwxmfAQAAAGDCBIQAAAAAMGECQgAAAACYMAEhAAAAAEyYgBAAAAAAJqzrtxifSptqMZfP3da1xh1Lm7qOnyRLK36Z36nz5G1Hutf4y/3ndK/x8is/q+v4m84+1HX8JHn6g9/evcb9Nt/UvcY7Zi7uXuPc7fu719hzYEv3GrOH+r/GD116uHuNiy/q+36bJI87/6ruNXbOHuxe48DCfNfxL5zf03X8jeSsma1rUKN7CY7jcKtctdC3T7tlcUfX8ZPk5iP9a+xb2Ny9xvbZ/uuh8+bv6Dr+2TP9+443739g9xpvuOVB3Wvsv71//7TlSP/+afZA615jaVP/+3HrQ2e71zhw4VL3GvMXHOheY/P8QvcaC0t9G4MtW/p/tj9/x77uNS7c2vc9PUl2zvfv+xeW1uD1t9j3s8WJaHMBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATNjcek9gtVqSI61vnnn70pau4yfJN5/9j91rJNu7V/jHfQ/oXmP+rENdx5+bW+w6fpLsW9jcvcbZ2/Z3r/GZ51zdvcarr3lo9xp737Wre42q7iVyr3P6L/NHnfvx7jW+8Kx3d69x5b77d6+x7/CmruNvmTnSdXw4E8xlKefP9O0Ltm++puv4STJfC91rvHmpf4929YFzute4efPOruO/aeEhXcdPko8fOrt7jaX0bzy23etg9xoHzp3vXmPuwBrsG7MGfeChXUvda8ye33+Zb9ncv785sjjbvcbCQt8aa/GZdfNs/3XTTLXuNWbXoMbSGtSYm+m/zFdiD0IAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEza33hNYrT2L2/KqOx7RtcafXHVF1/GT5Fc//Q+71zh7Zl/3GjtmD3av8Z2f8Yau479r7yVdx0+S7XOHutfYOXuge41HbPtY9xqvm31w9xpHzlrsXmN+V//Xxv3OubV7jYdsu757jZ0z/R+rpVT3GnsPbO46/pE223V8OBNUkvnOL+eDS/1faweX5rvX+NDe87vXmKml7jX2LvZ9bz1nfi365f594JbZI91rzM32X96Zbd1LLG7pXiKLm9bgfpyz0L3GOTv6f76Ynen/WO0/1P8998jhvpHK0lL/XvbQYv9YaP9C/2Wx0Prv/za3Buu/tVjHrlh73SoDAAAAAOtOQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwITNrfcEVmu+FnPh/J6uNW679qyu4yfJ7odt617jA0e6l8iu2X3da3zp9g92Hf+zt3646/hJ8t5DF3evMZ/F7jUevun67jUedd7Hu9d4e7XuNdbC+Vv2dq9x/803dK8xm6XuNT6y/9zuNQ7u39R1/PPnbu86PpwJ9rVNefPBvuvUI2226/hJcs3h/u9Ja7GO2D53qHuNizb17fvPWoNedu/ilu41Di/2/zh3ZKH/a2MNWoIsbOvfBy5u7V9j047D3Wvca0v/1/jBhf7P3cXF/vtDLe7rez9m7tX/s95Sq+41Di7Od6+x1Pov702zC91rLCz1f6xWYg9CAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABM2t94TWK1tM4fy2Vuv6lrj4Z92ddfxk+TJW/d1r3Hd4oHuNe4/98HuNS6a29F1/PuswbP/sjV4nP7l8Hnda2ypxe41Ltx0e/ca2+f7P1bnb93bvcZTznlH9xqP3XxL9xp/se9+3Wu848aLu9fYtuNQ1/E/ew2WRbJ9DWqwWofaka7jL6V1Hb+HI2021y+c1b1Gb0up7jV2berfa86vQV9wx+KWruPP10LX8ZPktiPbute4+UD/9++Dezd3r7EWFnYuda8xc87h7jXOO6t/r7ltvv/9OLjQ/8PYwkL/9/XM9F2n7tjWt89Mks2z/d8Pd8z3vx9z1f81fvuRvuumJLnujp3da6zEHoQAAAAAMGECQgAAAACYMAEhAAAAAEyYgBAAAAAAJkxACAAAAAATJiAEAAAAgAkTEAIAAADAhHUNCKvqRVV1Y1W9c9llz6uqj1fV28afL+s5BwAA1odeEADgzNB7D8IXJ3nKcS7/xdbaFePPqzrPAQCA9fHi6AUBAE57XQPC1tobktzaswYAAKcnvSAAwJlhvc5B+N1V9fbxsJNzVvqjqnp2VV1ZVVfuvnVpLecHAEA/J+0Fl/eBe287vNbzAwCYlPUICH8zyQOSXJHkuiQ/v9IfttZe0Fp7dGvt0Wfv8n0qAAAbwKp6weV94I5zNq3h9AAApmfNU7fW2g2ttcXW2lKS307ymLWeAwAA60MvCABw+lnzgLCqLlr261cleedKfwsAwMaiFwQAOP3M9Ry8qv4wyZOSnFdV1yT58SRPqqorkrQkVyV5Ts85AACwPvSCAABnhq4BYWvtG49z8Qt71gQA4PSgFwQAODP45g8AAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAE9b1S0pOpW01k0ds2tK1xm/c/392HT9J5mtH9xr3netfg9W5z5osi5u7V/jowrbuNb5g57u717hwfk/3Gp+/7YPdazx809buNfYuzXav8bZ99+1eY//BTd1rfMn939d1/Atmt3cdn9PPnqXDXcdfTOs6fg+H21w+dujcrjV2zh7sOn6S7FiDGttm+j5/kmS+FrvXmKmlruOvxX1YC2vyap7pX6Vt7bu8k6Q291/mm7f0f/1V9wrJocX+McHhhf695ubNR7rX2LS97/v6js39n1ObZvu/NrbP9b8ftxzq3zNfe/u9utdYi88vK7EHIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJm1vvCZxO7ju3o3uNjxzZ273G/eb73w9OH/dZg+ftmw6c073GN+y8rXuNB829r3uNi9ZgeayFl9z+oO41rtp7bvcaWzYf6V7j35z3hs4VNncen9PNBbPbu44/n1u7jt/DXBaza25f1xofP3R21/GT5I7ZLd1rbJs53L3GRZt2d69x9mzf5X3H4tau4yfJ1tn+66Cdmw51r7FnR/8are/bXpJk09xi9xp37O3/vLppYbZ7jaq2BjW6l8jSUv8i2zr3mtvm+7+n75jr/xq/aPOe7jUOLM53r7EWZmb6v/5WrL1ulQEAAACAdScgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwubWewJTc7/5Hes9BbjLzp7dv95TOCXed+Re3WscbLd3r/HWQxd3r3Hl7Zd3r3H9vp3da1x0r/7L44rNm7vXgKmbqZYdswe71rj5UP8ebffhrd1rXL7j1u41zpu/o3uNWxf6Lo/9S/3fu7fMHOleY9fm/j3anq1butc4dKT/x9KDh+e711gLiwuz/Wvc3v+xqqXqXqNtXexeY9NZe7uOv2V2oev4SbJ97nD3GotrsG/a7kP917Fr8V61uLh++/HZgxAAAAAAJkxACAAAAAATJiAEAAAAgAkTEAIAAADAhAkIAQAAAGDCVhUQVtXvVdVZy36/rKpe129aAACcLvSCAAAb22r3IHxTkjdX1ZdV1b9N8tokv9RtVgAAnE70ggAAG9jcav6otfZbVfWuJH+X5OYkj2qtXd91ZgAAnBb0ggAAG9tqDzF+RpIXJfnWJC9O8qqqemTHeQEAcJrQCwIAbGyr2oMwydck+bzW2o1J/rCq/izJS5Jc0WtiAACcNvSCAAAb2GoPMX56klTV9tbavtbaP1bVY7rODACA04JeEABgY1vtIcafU1XvTvKe8fdHxompAQAmQS8IALCxrfZbjH8pyZcmuSVJWmv/kuQJneYEAMDp5ZeiFwQA2LBWGxCmtXb1MRctnuK5AABwmtILAgBsXKv9kpKrq+pzk7Sq2pTkezIeYgIAwIanFwQA2MBWuwfhdyT5riSXJLkmwzfWfVenOQEAcHrRCwIAbGCr/Rbjm5N8c+e5AABwGtILAgBsbCcMCKvqV5O0la5vrX3PKZ8RAACnBb0gAMA0nOwQ4yuT/FOSLUk+M8kHxp8r4sTUAAAbnV4QAGACTrgHYWvtJUlSVc9K8gWttSPj789P8pruswMAYN3oBQEApmG1X1JycZKdy37fMV4GAMDGpxcEANjAVvUlJUl+Jsk/V9Xfjb8/McnzuswIOO08Zduh7jXef2Rf9xoH267uNe5oq31bvfvec7D/Z/IP7D6/e40brjmne42HPuKG7jUW21LX8Wdrtdvy2Cj2LB3oOv5i7tZzdl17wU21kEvmb+ta4/Jtt3QdP0mumTm7e4218PFD/d+/F1e9H8PdM3v3Xgd3yeaZhe41ZmrFU4SeMocXZrvX2LtvS/caiwv916fbdvTvmQ/s39y9xtye/st87kB1r3HoAUe615if6fteMjezMc7mcfX+/uuNQ4v9P4dt2dT/OdXm+687VrLabzH+nar6qySPHS/6odba9f2mBQDA6UIvCACwsd2VzSizSW5KcluSB1fVE/pMCQCA05BeEABgg1rVHoRV9bNJ/nWSdyWf2B+/JXlDp3kBAHCa0AsCAGxsqz1I++lJHtJa639SBQAATjdPj14QAGDDWu0hxh9OMt9zIgAAnLb0ggAAG9hq9yDcn+RtVfW6JJ/Yctxa+54uswIA4HSiFwQA2MBWGxC+cvwBAGB69IIAABvYqgLC1tpLek8EAIDTk14QAGBjO2FAWFUva619fVW9I8M31d1Ja+0R3WYGAMC60gsCAEzDyfYg/N7x3y/vPREAAE47ekEAgAk4YUDYWrtu/PejJ/q7qvqH1trnnMqJAQCwvvSCAADTMHOKxtlyisYBAODMoxcEADiDnaqA8FPOSQMAwGToBQEAzmCnKiAEAAAAAM5AqwoIq+q7q+qcE/3JKZoPAACnGb0gAMDGtto9CO+d5C1V9bKqekpVHdsEPuMUzwsAgNOHXhAAYANbVUDYWvtPSR6U5IVJnpXkA1X1X6rqAeP17+w2QwAA1pVeEABgY1v1OQhbay3J9ePPQpJzkvxJVf1cp7kBAHCa0AsCAGxcc6v5o6r6niTPTHJzkv+R5Adaa0eqaibJB5L8YL8pAgCwnvSCAAAb26oCwiTnJfnq1tpHl1/YWluqqi8/9dMCAOA0ohcEANjAVhUQttZ+7ATXvefUTYczxf6lw91rbJvZ1L0Gp4837n9A9xqfteWjJ/+je2j/0nz3GlcfPNEXiZ4a17/jwu415he7l8jDdlzbvcZ1i/u7jn+fuR1dx99IPnRkb/ca82vwXb27l1a7/fbuOdTu+m3WuxfcPrOQz91yQ9caZ8/0fS0nyTs2Xdq9xp7Frf1rLPSvsW9xc9fxd84d7Dp+ktxw+F7da7zv1gu617j1hv73IwurPvPV3TZ/dv9lPj/bv7nZt9h/RTQz271EWv9Fnnakf5HbD/Z9r9q1ZV/X8ZNkppbWoMbdaD7uorM29X+NLy71f07d0fk5dSJr8LIEAAAAAE5XAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJ6xoQVtWLqurGqnrnsst2VdVrq+oD47/n9JwDAADrQy8IAHBm6L0H4YuTPOWYy34oyetaaw9K8rrxdwAANp4XRy8IAHDa6xoQttbekOTWYy5+WpKXjP9/SZKn95wDAADrQy8IAHBmWI9zEF7YWrsuScZ/L1jpD6vq2VV1ZVVdedMti2s2QQAAullVL7i8D7zllqU1nSAAwNSc1l9S0lp7QWvt0a21R59/7ux6TwcAgDWyvA8899zTumUFADjjrUe3dUNVXZQk4783rsMcAABYH3pBAIDTzHoEhK9M8szx/89M8ufrMAcAANaHXhAA4DTTNSCsqj9M8g9JHlJV11TVtyf5mSRfUlUfSPIl4+8AAGwwekEAgDPDXM/BW2vfuMJVX9SzLgAA608vCABwZnDGZwAAAACYMAEhAAAAAEyYgBAAAAAAJkxACAAAAAATJiAEAAAAgAnr+i3GbFzzNbveU2CD+fxtH+pe4+LZ/s/bX7/t07rXeP1HHtS9xs6PVvca+x6/r3uNL9vxzu417jO3o3sNVufqhXt1r7GljnSv8a5Dl3Qdf//S7q7j9zCbyo6a71pj1+z+ruMnycXzt3WvsXmm/3N0x+zB7jUOLfVd3mtxH96zcHH3GvsP9X2ckqQO9e+f2txS/xpL/feN2b17e/cabX//j/Czh7uXSC32rzGzp/9jdfvWrV3HP7iz/2t838Lm7jUOLvZfFkvp//llfrb/E3fb5jV4Aa7AHoQAAAAAMGECQgAAAACYMAEhAAAAAEyYgBAAAAAAJkxACAAAAAATJiAEAAAAgAkTEAIAAADAhAkIAQAAAGDCBIQAAAAAMGECQgAAAACYMAEhAAAAAEyYgBAAAAAAJkxACAAAAAATJiAEAAAAgAkTEAIAAADAhAkIAQAAAGDCBIQAAAAAMGECQgAAAACYMAEhAAAAAEyYgBAAAAAAJmxuvSdwOtmzdKB7jbNmtnavcaQtboga8zXbvQanj4tn+y/vfzm8qXuNf7j1/t1rtA9v715j/0Wte43vfeTfdq/xwHmruSl55Ka93Wt8eKH/c2rP4rau4y+egduHF9pSbl063LXG/qW16NH6r+sOLc13r7EW9+OcuX1dx5/NUtfxk2TzzJHuNbZv6fu6SJID2xe618iR/u9LC7v794G1VN1rzB7sX2PLzf1rrMFbVRYW+t+PhX1978jVu8/uOn6S3Lal//pvpvp/ttgx3//9cPsa1Ng5f6h7jZWceR0iAAAAAHDKCAgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJiwufWewGq1tBxpi11rHGpLXcdPktsW93evMVPVvcZSa91rbMum7jVYnbV43t6xBq+/l97ypO413nP9hd1rzB7o/xq/zxOv7l7jm3a+r3uNxbYGq7n+i2NDWFyD1/g5s9u61zh7aW/3Goudn1T91+Cn3uHM5JqFrV1r3Li4s+v4SXLL4o7uNT52aFf3GkfabPcas5v7PlMPLs13HT9Jbj2yvXuN1vqvhGq2//t3278Gz6n9/feNWdra/x12ZqH/Ml/Y0r1EDp/V/7FavPBQ9xpbtx/uOn5V/8dpz76+69ckWVzs//qbP+f27jUWWv/7cXBh/WI6exACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmLC59Z7AalUq8zXbtcYFs9u7jg9nqnNmt3WvsfvI3u413n7Lxd1rLF3V/31k4Zyl7jV+4LK/7l5jLZ5Xe5cOdq/B6iylda/Rt0sY3Gduc/caR5b6tmetVdfxezjc5vKxhV1da+xf6r9s9y5u6V7jSOv/SlhY6l/j4NJ81/Fnq/+6dOvske41zt56oHuNvQf6vzYOHuz/sXRpU//1UBb6v7/O7u9fY2Fb/8dq8eJD3Wvc58LbutfYtWV/1/H3LWzqOn6S3Ly3/+eXvYf7r/9u2df/s8Va9FAzM/3XTyvWXrfKAAAAAMC6ExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhc+s9AYAkefPBS7vX+PhHz+1e417XV/caO7/0hu41vmDrwe41ktnuFfa3xe41dnSvsDHMV//lvRbm1uB5u2tub9fx56r/6+JUO7Q0nw8evHfXGltmjnQdP0kWW/9t80ut/3pobqb/c2h+ZqHr+Dtn+q/nHri1/+O0Fq/nA0fmu9e4/kj/99bFmf4ffTdfs6l7jbn93Utk32VL3WtcdMHu7jV2bDrUv8Z83xoXb9vTdfwk2T53uHuND7dd3WscPND/9Te/qe+6KUl2bun/vF2JPQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACZtbr8JVdVWSO5IsJllorT16veYCAMDa0gsCAJw+1i0gHH1Ba+3mdZ4DAADrQy8IAHAacIgxAAAAAEzYegaELclrquqfqurZx/uDqnp2VV1ZVVfedMviGk8PAICOTtgLLu8D9912eB2mBwAwHet5iPHjW2vXVtUFSV5bVe9trb1h+R+01l6Q5AVJ8uhHbmnrMUkAALo4YS+4vA+85OFn6wMBADpatz0IW2vXjv/emOTPkjxmveYCAMDa0gsCAJw+1iUgrKrtVbXz6P+TPDnJO9djLgAArC29IADA6WW9DjG+MMmfVdXROfxBa+2v12kuAACsLb0gAMBpZF0Cwtbah5M8cj1qAwCwvvSCAACnl/X8FmMAAAAAYJ0JCAEAAABgwgSEAAAAADBhAkIAAAAAmLD1+hZjOKnrFvZ2Hf/DC9u6jp8kD5k/0L3GebPbu9dYC6+69TO619j6sfnuNZZmu5fIt132f7rXmK/+d+RjnV/jSbJl+IZUOGXee+RQ9xoHW9/3qqV4XRzPwaX+64hbjvRfZ9+xsKV7jfM29X//3jlzsOv4957f03X8JJnJUvcaR9oarK+37upe4/Ztm7vX2H9d/9fGzo+27jX2XdT/PXz+gv6fYbbNH+le45o9Z3WvcdVC39fH/c+7pev4STJX/d+rdm3r/5y69mD/9fjcbP/Haj27NHsQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATNjcek8AVvK3By7rOv77D9676/hJ8qhz/7l7jbXwtkOHutf4lxsu6V5j823dS2T/Ra17jU/bfG33Gov970buO7ejfxEm5bbF/d1r3LTY/3m7f3Fz1/GX2pm3fXiuFrNrbm/XGnsWt3UdP0lmqv+b60Wb93Svcd/Nt3SvcemmvjV2zhzsOn6S3LG0pXuNxTV4PS+lute449bt3Wtc+NbuJbIWb69Hzur/PrJl00L3GgeOzHevccfN/Z9XOdJ3ob//yGzX8ZPk3rtu715j69yR7jW2bz3cvcbMzFL3Gout/3vuSs68DhEAAAAAOGUEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACRMQAgAAAMCECQgBAAAAYMIEhAAAAAAwYQJCAAAAAJgwASEAAAAATNjcek8AVrJz5kDX8e+76Zau4yfJtplN3WushZfe9rjuNW6/cUf3Gmd3r5C0B+3rXuNh8we715itbd1rMC03L/Z/bVyz0L+tuX7hrO415mcWuo5f1bqO38NMteyc7fvet2ex//veRZv2dK9x/803dq9x8dxt3WscabNdx79p8V5dx0+Sqw6f173GRw/u6l7j/Tec373Grv87373GOe/c3b3GLVec3b3Gwo7F7jXWYj1x8+3bu9eY336ke42Fw33fqxYO9e9tDq5B/zSzBs+p7ZsPd6+xFhZbrVttexACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmDABIQAAAABMmIAQAAAAACZMQAgAAAAAEyYgBAAAAIAJExACAAAAwIQJCAEAAABgwgSEAAAAADBhAkIAAAAAmLC59Z4ArOQrt+/vXKH3+Gtj79LB7jXeeP0DuteY2T/bvcahc7qXyJc96N3da5wzu617DU4fh9qR7jWuXTjUvcaepfnuNfa1/jW2zfR/rC6dv7Xr+Jtqoev4PcykZUv1fS1snun/Wts503+dvXPmQPcaR1r/dfaHDl/Qdfx37L+06/hJ8vEDZ3ev8d5b+j5OSZL37Oxe4t6v/Xj3GmthYdvZ/YtsXexeYmmp/35ECwv930d2nbWve42Fxb6P1aEjGyOyObTYf3nPVOteY35mqXuN9Vzi9iAEAAAAgAkTEAIAAADAhAkIAQAAAGDCBIQAAAAAMGECQgAAAACYMAEhAAAAAEyYgBAAAAAAJkxACAAAAAATJiAEAAAAgAkTEAIAAADAhAkIAQAAAGDCBIQAAAAAMGECQgAAAACYMAEhAAAAAEyYgBAAAAAAJkxACAAAAAATJiAEAAAAgAkTEAIAAADAhAkIAQAAAGDCBIQAAAAAMGFz6z0B4J75+4Nnd6+x//B89xpLOxe61zg4O9u9xmdsu6Z7jT1LB7rXOGtma/caG0Xv5XFN/5dGPr5wTvcaS2uwTXIx1b3GA+Zv6V7jrJnFruPvqMNdx+9hJkvZPnOoa43ZtK7jJ8nB1n99ev3C2d1r7F7c1r3GBw5c2HX8q/ae23X8JLl+387uNXZ/9OzuNS7/3/3fMxY+8tHuNZae+KjuNfZf1L1Eaqb/e9W+2/r3gZt29H9eXbB9b/caO+b7rptuObi96/hJcsehzd1rLC717wNb9X9tzK5Bjc1za9D8r8AehAAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAmTEAIAAAAABMmIAQAAACACVu3gLCqnlJV76uqD1bVD63XPAAAWHt6QQCA08e6BIRVNZvk15M8NcnDknxjVT1sPeYCAMDa0gsCAJxe1msPwsck+WBr7cOttcNJ/ijJ09ZpLgAArC29IADAaWS9AsJLkly97PdrxsvupKqeXVVXVtWVN92yuGaTAwCgq5P2gsv7wD236gMBAHpar4CwjnNZ+5QLWntBa+3RrbVHn3/u7BpMCwCANXDSXnB5H3jWLn0gAEBP6xUQXpPk0mW/3yfJtes0FwAA1pZeEADgNLJeAeFbkjyoqu5XVZuSfEOSV67TXAAAWFt6QQCA08jcehRtrS1U1XcneXWS2SQvaq29az3mAgDA2tILAgCcXtYlIEyS1tqrkrxqveoDALB+9IIAAKeP9TrEGAAAAAA4DQgIAQAAAGDCBIQAAAAAMGECQgAAAACYMAEhAAAAAExYtdbWew6rUlU3JfnoXbjJeUlu7jQdTk+W+bRY3tNjmU+L5d3PZa2189d7EnfF3egDE8+hqbG8p8cynxbLe3os836O2wueMQHhXVVVV7bWHr3e82DtWObTYnlPj2U+LZY395Tn0LRY3tNjmU+L5T09lvnac4gxAAAAAEyYgBAAAAAAJmwjB4QvWO8JsOYs82mxvKfHMp8Wy5t7ynNoWizv6bHMp8Xynh7LfI1t2HMQAgAAAAAnt5H3IAQAAAAATkJACAAAAAATtiEDwqp6SlW9r6o+WFU/tN7zoa+quqqq3lFVb6uqK9d7Ppx6VfWiqrqxqt657LJdVfXaqvrA+O856zlHTp0Vlvfzqurj4+v8bVX1Zes5R06dqrq0qv6uqt5TVe+qqu8dL/ca527RB06PXnBj0wdOj15wWvSCp48NFxBW1WySX0/y1CQPS/KNVfWw9Z0Va+ALWmtXtNYevd4ToYsXJ3nKMZf9UJLXtdYelOR14+9sDC/Opy7vJPnF8XV+RWvtVWs8J/pZSPLc1tpDkzwuyXeN622vce4yfeCk6QU3rhdHHzg1L45ecEr0gqeJDRcQJnlMkg+21j7cWjuc5I+SPG2d5wTcA621NyS59ZiLn5bkJeP/X5Lk6Ws5J/pZYXmzQbXWrmutvXX8/x1J3pPkkniNc/foA2GD0QdOj15wWvSCp4+NGBBekuTqZb9fM17GxtWSvKaq/qmqnr3ek2HNXNhauy4ZVipJLljn+dDfd1fV28fDThxisAFV1eVJHpXkzfEa5+7RB06TXnB6rCOmSS+4wekF19dGDAjrOJe1NZ8Fa+nxrbXPzHA40XdV1RPWe0LAKfebSR6Q5Iok1yX5+XWdDadcVe1I8qdJvq+1dvt6z4czlj5wmvSCsPHpBTc4veD624gB4TVJLl32+32SXLtOc2ENtNauHf+9McmfZTi8iI3vhqq6KEnGf29c5/nQUWvthtbaYmttKclvx+t8Q6mq+QwN4Utbay8fL/Ya5+7QB06QXnCSrCMmRi+4sekFTw8bMSB8S5IHVdX9qmpTkm9I8sp1nhOdVNX2qtp59P9JnpzknSe+FRvEK5M8c/z/M5P8+TrOhc6ONgejr4rX+YZRVZXkhUne01r7hWVXeY1zd+gDJ0YvOFnWEROjF9y49IKnj2pt4x11MX7l+S8lmU3yotbaT6/vjOilqu6fYUtxkswl+QPLe+Opqj9M8qQk5yW5IcmPJ3lFkpcluW+SjyX5utaakxlvACss7ydlOKSkJbkqyXOOnpOEM1tVfV6SNyZ5R5Kl8eIfyXDuGa9x7jJ94LToBTc+feD06AWnRS94+tiQASEAAAAAsDob8RBjAAAAAGCVBIQAAAAAMGECQgAAAACYMAEhAAAAAEyYgBAAAAAAJkxACAAAAAATJiAEWKWquriq/uQu3uZZVfVrveYEAEB/+kBgo5tb7wkAnClaa9cm+dr1ngcAAGtLHwhsdPYgBCavqj67qt5eVVuqantVvauqPv04f3d5Vb1z/P+zqurlVfXXVfWBqvq5ZX/3bVX1/qr6+ySPX3b5+VX1p1X1lvHn8ePlf15V3zr+/zlV9dLudxoAAH0gwMgehMDktdbeUlWvTPJTSbYm+f3W2jtXcdMrkjwqyaEk76uqX02ykOQnknxWkj1J/i7JP49//8tJfrG19qaqum+SVyd5aJJnJ/nfVfWRJM9N8rhTdd8AAFiZPhBgICAEGPxkkrckOZjke1Z5m9e11vYkSVW9O8llSc5L8vrW2k3j5X+c5MHj339xkodV1dHb36uqdrbWbqiqH8vQRH5Va+3WU3GHAABYFX0gMHkCQoDBriQ7kswn2ZJk3ypuc2jZ/xfzyffUtsLfzyT5nNbageNc9xlJbkly8apmCwDAqaIPBCbPOQgBBi9I8qNJXprkZ+/BOG9O8qSqOreq5pN83bLrXpPku4/+UlVXjP8+JslTMxym8v1Vdb97UB8AgLtGHwhMnoAQmLzxxNALrbU/SPIzST67qr7w7ozVWrsuyfOS/EOSv0ny1mVXf0+SR48nwn53ku+oqs1JfjvJ/zN+O95zk7yolh1/AgBAH/pAgEG1ttIe0AAAAADARmcPQgAAAACYMF9SAnCMqvqMJL93zMWHWmuPXY/5AACwNvSBwFQ5xBgAAAAAJswhxgAAAAAwYQJCAAAAAJgwASEAAAAATJiAEAAAAAAm7P8HS27Q0WJQNwkAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "EXAMPLE_I = 6\n", + "CHANNEL_I = 7\n", + "HISTORY_LENGTH = 7\n", + "sat_data = sat_batch[\"data\"].sel(example=EXAMPLE_I, channels_index=CHANNEL_I)\n", + "channel_name = sat_batch[\"channels\"].sel(example=EXAMPLE_I, channels_index=CHANNEL_I).values\n", + "opt_flow_data = opt_flow_batch[\"data\"].sel(example=EXAMPLE_I, channels_index=CHANNEL_I)\n", + "min_pixel_val = min(sat_data.min(), opt_flow_data.min())\n", + "max_pixel_val = min(sat_data.max(), opt_flow_data.max())\n", + "imshow_kwargs = dict(x='x_index', y='y_index', add_colorbar=False, vmin=min_pixel_val, vmax=max_pixel_val)\n", + "\n", + "fig, axes = plt.subplots(figsize=(18, 8), ncols=2)\n", + "\n", + "ax = axes[0]\n", + "sat_img = sat_data.isel(time_index=0).plot.imshow(ax=ax, **imshow_kwargs)\n", + "\n", + "ax = axes[1]\n", + "opt_flow_img = opt_flow_data.isel(time_index=0).plot.imshow(ax=ax, **imshow_kwargs)\n", + "OPT_FLOW_TITLE = \"Optical flow precitions\"\n", + "ax.set_title(OPT_FLOW_TITLE)\n", + "\n", + "\n", + "def format_date(dt: np.datetime64) -> str:\n", + " return pd.Timestamp(dt).strftime(\"%Y-%m-%d %H:%M\")\n", + "\n", + "plt.tight_layout()\n", + "\n", + "def init():\n", + " sat_img.set_data(sat_data.isel(time_index=0))\n", + " axes[1].set_title(OPT_FLOW_TITLE)\n", + " opt_flow_img.set_data(np.full(shape=opt_flow_data.isel(time_index=0).shape, fill_value=np.NaN))\n", + " return sat_img, opt_flow_img\n", + "\n", + "def update(i):\n", + " # SAT DATA\n", + " sat_img.set_data(sat_data.isel(time_index=i))\n", + " datetime = sat_batch[\"time\"].isel(example=EXAMPLE_I, time_index=i).values\n", + " axes[0].set_title(\"Real satellite data | \" + format_date(datetime) + \" | chan = \" + channel_name)\n", + " \n", + " # OPTICAL FLOW PREDICTIONS\n", + " if i > HISTORY_LENGTH:\n", + " opt_flow_datetime = opt_flow_batch[\"time\"].isel(example=EXAMPLE_I, time_index=i-HISTORY_LENGTH).values\n", + " axes[1].set_title(OPT_FLOW_TITLE + \" | \" + format_date(opt_flow_datetime))\n", + " new_opt_flow_data = opt_flow_data.isel(time_index=i-HISTORY_LENGTH).values.copy()\n", + " opt_flow_img.set_data(new_opt_flow_data)\n", + " return sat_img, opt_flow_img\n", + "\n", + "anim = FuncAnimation(fig, func=update, frames=np.arange(30), init_func=init, interval=250, blit=True)\n", + "#anim.save('optical_flow.gif', writer='imagemagick')\n", + "html = anim.to_html5_video()\n", + "HTML(html)" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "id": "e63ccdad-819f-45cd-83cf-0f9fdadf57e0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 126, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABQcAAAHwCAYAAAAb5dmEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAADePklEQVR4nOzdd5wk21kf/N9T1bl74ua9UTkRBFyTBEJ+BZgsESSDMUgYW/AajANBAhsTXjAyBhtsTJAJuiSBDAbJJkmWJZJBSCAJhSvpSvfqps07eTpXnfePqp1znmemq2e6du/O3vl9P5/9bNdUOnXqVFV3dfXviHMOREREREREREREdPREN7sAREREREREREREdHPw5iAREREREREREdERxZuDRERERERERERERxRvDhIRERERERERER1RvDlIRERERERERER0RPHmIBERERERERER0RHFm4NERER0ICJyp4hsiUhccjlvE5F/PGGciMgviciqiPyViLxARB4ts77DQkR+VkS+t2D894jIzz+eZcrX+zoRefHjvd7rSUSciDw1f/0fReSbb3aZiIgOitfZcnidvf4ez/YhIl8mIr/xeKyLPN4cvMEOy0nWvFl+rYj8UP76s0XkQzdwvTvrutHyun7b47EuevyJyMtF5LUzzvdnN6BIRLeM/Dh4r4h0ReSCiPyMiCweYP6PicjnXht2zj3snOs455IbUuDMZwH4PAC3O+c+9Qau53HnnPtm59z/B+z9PsE59++cc3t+mLtRROQTAHwigDc8nuu9wf4DgH8tIrW9RorI00XkDSJyWURWROSPROQZZpp/mR8z6yLyiyJSz/9eF5FfEJGHRGRTRN4lIl9o5n2hiHwwP+7eKiJ3FRW2aHoR+YP8RsG1f0MReW/BskRE/r2IXM3//aiISD7uZP4B9Vy+XX8uIp9WsKyPy+vmioi4PcbfLSK/n99guCAiPyUilQnLujt/Txpuy/cG4yeWm+gw43X2cOF19tYRXBd2rhvOuTcC+Li8zuhxwpuDufyE3MvfpFzIb2p1bna59rLXATQr59yfOud23gjbC9PjSQq+2bpOy/8xEbk/fxP/QRH5ejP+uSLy1/lF/a9F5LnBuJflf9sQkUfzN6uVYPyyiPyOiGznHxT+wZSyTJxeRL7WvGnu5vv7UwqWV/SB4jtF5H35dj8oIt85pWyvEZEPiUgqIi8340REfkhEHss/ULxNRJ5TsKy3iUg/2JYPmfEH+uD0RJR/4PrBPf7+ovxcVBFzk11EvjGvt00RuSgivycic/m4wg9W+fnjrXmdf9Ae7yJyQkR+XUTWJPug92s3cvvpxhORbwfw7wF8J4AFAJ8O4C4Ab5YJN00OibsAfMw5t32zC7KX63ENPmS+CcCvOed23fwpcpjrwTl3HsAHAXzZhEkWAbwRwDMAnALwVwg+tInI3wPwKgAvBHA3gCcD+IF8dAXAIwA+B9lx9b0AXi8id+fzHgfwP/K/LwN4J4DfnFTWadM7574wv1HQcc51APxfAP+9YPNfAeDFyD6IfgKAL0G2jwGgA+AdAD4lX9e9AH6v4H3vCMDrAXzjhPE/DeASgDMAnousTv5pQdkAYDHYnv9vn+UmOpR4nb0xDvP1ZUYzXWePsNchuybQ48U5x3/Z8fkxAJ+bvz4N4D0Afvg6LPcFAB69zmW9G4ADUDnAPA7AU/PXrwXwQ9Pq4TqVdeK69pj2bQD+ccm6flvB+B8A8ExkN8U/DcAqgM/Mx9UAPATgXwKoA/i2fLiWj/9/AXx2Pt1tAP4awKuCZb8O2Zv4DrJv4dYBPKegLPueHsDLAXwUgEwYfzyf/yUAGsielPjLYPx3AfhkZB9knpFv11cXlO1bkH0QeieAl5txLwVwDtkHpBjAjwD4m1n26bRyT6iH187QLl4O4M+uV5u+3v8AfA2AB+3+BfBbAH48f71zHCH70HURwCflw8sAXgZgLh/+JgAfAnB73lY/AOCbg+X+BYD/CKAJ4CsBrAE4EYz/03z8AoDqtfXw3635D8A8gC0ALzV/7yD7MP+P8uHvz9vcbwLYBPA3AD4xH/crAFIAvXxZ3wVzHcrb4S/l54dVAL+b/30JwP8CcDn/+/9C9oTCtXLseY5AdhOiDyDJ1/kDMNdTAM/K518D8H4AX5b//Un536J8+OcBXArm+1UA/2JCfX0MwHfnx81qvk2NfNwLADwK4JUALuT1EiG7efRRAFeR3UBZDpb3Wchu4qwhu5H08vzvrwXwQwDaeb2m+XZuATib749fDZbzZfk2ruXb/CxT5u8A8LfIzqm/GZT5eF7nawBWkB3f0YRtfwDAZ5m//RMA9yFrEx8A8MnBOl+Zr3OA7PoyrYzT6vV7AFzJp/3aYN46gB8D8DCyc9/PAmgG478TwHlkbe8fIXi/k4//1wB+aZ/Hy3I+/7F8+NcB/Ltg/AsBXCiY/28BfGX++hUA/m8w7tq+fuaEefc9PbLjLwHwpIKy/F8ArzDHVNE1dgPAp0ypn6cCcHv8/T4AXxQM/wcAPzdhGXej4D3sQcvNf/x3s/+B11mA19mZrrPI7jdsBf8cgBfsozx77pdgu38awB/ky/xzZPc2fiKv7w8ieG+f18VvI2s/DwL4tmBcM1/ear6/vhMF9zUAfCayL57W8/8/09Th5wbDO/WP7Prugnr4jPzvzwPw4M0+xo/Sv5tegMPyb48G+6MAfi8Y/vTgxPOeawduPu4b4N88PwDgm4JxL5h0EAEQAP8J2YVjPT/hfFw+7osBvAvZm7VHAHx/MN+kA+gf5eVYBfBHAO4K5tl5swx9o2GnfNjjwjRt2/fYpk9CdrHbRHbi/I1gXRMvXgB+GNnFqZ+v+6fyv/9kvv0byG7IfXbBul+AgpuDe0z/RgDfnr/+fACPIbhBk9fzF0yY918B+J/56zaAIYCnB+N/BcCrJ8x70OnfCuD7CrbjoB9A/jOA/7KP+vkz7L45+EoArw+GnwOgX7CMt2HyzcGDlvvlKLg5COAOZE9dXEb25uGngvn+DNmHy1VkF74vPMjxC+DbkR2n5wF8w37b2D7bYRPZ8f/84G9L+bHwiW73MfsdyN8QTljexA9WAJ6O7IP8XDD+T5HfPMyPg48BiK/nNvLfzfsH4AsAjLHHB3FkTwu9Ln/9/cieDvoqZDeFvyM/Vqr5+I9BXyPvhv7Q8nvIzvlL+fyfk//9GLKb0C0Ac8iedPrdYDlF54iXI7ixD329qgL4CLIbSjUA/09+DD8jH/8w8hsdyG6WP4D8jXU+bs+b3vl2vg/Z+WQZ2Zvq8Ho5RvZ0SD0/dv8FgL9EdjO+DuDngjq9My/T1+TlPQbgufm412KP63BQju+Hf9P8dADbyH76VUX2ofEj8F9efQzZE29n8zLfB39M/wiym2nV/N9nY48vmpCdfx30FwUvQXZd/DvI3q88Ffn7inyd787rqbnPMk6r1/+Y1+Hn5Mu6ti9/Atn1ehlZG/qfAH4kaN8XAXxcvg2/jt03B78CBV9imXp4MYDzwfB7APz9YPg4gpuHZt5TyM7bz8yHfxLAz5hp3of85uEe8+97egD/FlPe7yC7rnxaMHwPgM0J0z43L/vClGVOujn4zQB+Gdlxflte7i8Pxq8h/0AMf+54DNn19ZcAHJ+l3PzHf4fhH3idBXidnek6a8a/AtmNu/mi8uxjv7wW2Rdtn4Ls4Yv/g6ydfT2yBzt+CMBb82kjZJ+v/22+rCfn+/Hv5eNfjexzwnK+v95n6zEo/zKyz1lfh+wLw6/Jh48FdTjp5uDd2ONLI/gv7OZv9nF+VP7xZ8V7EJHbAXwhsgMPInIbshPyDyFrpN8B4LdF5EQ+yyVkP3uYR3aj4T+JyCfvY1WfD+D5yE4AiwD+PrKbGkB2Qvj6/O9fDOD/FR9e+vz8/0WX/RzjL/Jx34PsTfAJZAfy6w6y3c65r0N2Mv/SfLk/uo9t35E/Nv+7yG50LSO7OH1lMEmE7E3gXchO5j0AP5Wv+1/nZf7WfN3fms/zDmRvWpeRven/7yLSOMh27UVEmsg+8Lw//9NzAPyty89Eub/N/76X5wfzPh1A4pz7cDD+PQXz7nv6/Ge2z0f2pnuS5+TzAwBc9tOAj05YniC7aL3fjtun3wDwVMlymqrInlj7w2D5Py0iP23m+RHJcor+XEReMEu5p5EsrPl/IXsq8m5kH07CENtPQ/bG5TiyG/+/EPzUdtrxexrZU3S3IbvR9l9FZGlCOX46/znuXv/+dq95nHM9ZN+Chj9zfymADzrn3rPHLG8H8PdE5AdE5HmS518FVL1Ct63nAHjAObc5YfynI6une/OfJL9DRD5nr3LTLeM4gCvOufEe487n46/5a+fcbznnRshu1DSQtYlCInIG2TXzm51zq865kXPujwHAOXfVOffbzrlu3u5+GNnNn7I+HdlTGa92zg2dc/8H2Tnga/Lxfwzgc0TkdD78W/nwk5Ad63sdW9f8lHPuEefcSl7erwnGpci+rBnkx+43AfjXzrlHnXMDZG92vyr/KdTXAvjfzrnX5XVy1Tn37hm29e8j+7Lyzfm++TFkH5g+M5jmPzvnzuVl/p/IrptA9kH0DLKbeiOXRYmE17lrFvP/w3PDPwbwo865d7jMR5xzD5l1PpLXw37KWFSvAPC9eb3+MbL3HS/Nz9P/BMC/dM6t5G3o3wH46nyelyJ7KvB9+TXk+/fYts1g+ybK3/v9V2Rf/l3TQXaz6pprr+fMvFUAvwbgXufcByfMe23+OeztINN/PbIPgEX2KnsnjJnIyz6P7H3bDzjn7Pr364+RXUc2kN3weyey94MAAOfconPuWvbvFWTvv+5C9uF1DlndHajcRIcIr7MZXmcPfp0FAIjIZyH7rP1lzrmNKeWZtl8A4Hecc3/tnOsD+B1kD3L8ssvyK38T2cM8QHYuPuGc+8F8WQ8A+G/Q19gfzq+/jyB7wGSSLwZwv3PuV5xzY+fc65Dd7PzSgnmmuVZXiyWWQQfAm4Pa74rIJrIn1S4B+L787/8QwO87537fOZc6596M7I3PFwGAc+73nHMfzd88/zGANyG7ATPNCNmbomci+4bhPpfl48A59zbn3Hvz9f0tsht9RSf6b0L2Tfp9+cXp3wF47nXIcCvcduPTkX2b8RP5yfG3kN3cQ75NB754Oed+NZ9v7Jz7cWTfGD2jaJ59+llkF60/yof3/aZcRL4B2TfZP3bQeWeY/usB/Klz7sEJyzro8r4f/ibtLM4ju4n7IWQ3d1+C7KfYAADn3D91zoU5Q69E9i3UbQBeA+B/ishTZij3NJ+K7Nu873TObTvn+sEHEQB4yDn33/KL4r3ILuKn8jJPO35HAH4wb9O/j+zJ1j3bYL79ixP+FQXq3gvgJflNayDb7/dOWMefIvsS4JORfYC+KlmPnNd6syv6YDWtzm9H9qXFW5HdFP1xAG/Is7Do1nQFwPEJuT1n8vHXPHLthXMuRfYh/+w+1nEHgBXn3KodISItEfk5ybJVNwD8CYBFKdn7Yl6uR/JyXvMQsnMNkH1oeQGyL1f+BNmTE5+T//tTM5/1SPD6Ieg6uJy/2b7mLgC/c+1LAGRPEyTIzi93IPvCo6yzeTkA7OybR+C3Fch+fnVNF9mxDmQ/7/wIgDeJyAMi8qoJ61jL/w/Pv9PKH9bTfspYVK+rTmdeXRt/AtnTMH8d1PEf5n+/tl67XGsOfvv2lH/h+SYAP51/oLlmC9mH3Guuvd4M5o2Q3VwbAvjWYFo777X5N8X3QrolIlvTpjdl/Sxk5+ffCv72PcHyfrag7Fvhh9b8mvM/kT1d/iOYQb79f4Tsyf02shshS8ie/NnFObflnHtn/r7uIrI6+/z8JuW+yk10yPA6y+vsrNdZiMgdyB4SeJnzD44UlWfafgGyJ+qv6e0xfK3sdwE4Gz7MgOxho1NBOaZdY685u8d4W66DulZXayWWQQfAm4Pai51zc8hOdM+E/6bnLmQf3MMD57OQnfAhIl8oIn8pWU93a8hunE39MJ3f6f8pZN9UX5SsI4j5fJmfJlmnAZdFZB3ZTzaKlnkXgJ8MyreC7GdAZQ7Ia8uduO3GWQCPmTdwOyeJWS5eIvLtInKfZJ1frCF7gqvUjQoR+Q/IfoL00qCs+31T/mJkj1h/oXPu2sW+cF7RPQx+7X7XlVM3iUp+oPjWfHlfnH/zNovvQ/Yt0x3Ivu38AQD/R0Rae03snHu7c27TZd/+3YvspwPXbiwfpB6muQPZDcC9vrUFggu6c66bv+wA+zp+r5rlhm8Irov8RuZlAC8SkScjq+NfL5j+D5xzX4rsidoXIftZyLXOfIo+WE2r8x6yYOpfyG+G/gayNwXPK7F5dHP9BbKfkn9F+EcRaSN7CuEtwZ/vCMZHyG4Wn8v/VPTB/BEAy7J3r4zfjuxm+qc55+bhn3wv+xTQOQB35OW85k5kP1UEsg8tn43sev7HyKIFnofsQ8sfT1n2HcHrO+HrANhdD48gux6EXwQ0nHOP5eOegumm3fQ4h+xaDGDnCfA74Ld18oKz8++3O+eejOzb+38lIi/cY7prT24/PfjztPKH5d5PGYvqdSlvk3b8FWTnpecE9bvgsg45gOwLK7tc61koeIJFsifB3wTgjc65Hzaj34+sY4xrPhHARefc1WA7fwHZB6mvdNkTHnvOm2/fUwC83/leSDvBtkyc3pTpZQD+h3Pu2nsAuKzXzWvL++aCsu8sS7Knzn8X2T4q0+HHtZ+b/VR+rb+K7AvIvb5E3su1dnTtnFBYbqJDiNdZXmdnus7mX9D8LrIHa/5gn+WZtl8O4hFkmX5h3c45566dv/dzjd2zzHuUaxvZl33XnA5eT9o/z0L2uWSjaCPo+uHNwT3kTw+9Fv7JsEcA/Io5cNrOuVfnb65+O5/2lHNuEcDvY58nZOfcf3bOfQqyn2M8HVnQJ5DdGHgjgDuccwvInnS7tsy9DqBHkGWlhWVsOuf+78G2fs8T8p7bvse85wHclp/ArglPItMuXmrdIvLZyJ48eymApbxu11HiYiciP4DsQv355kTzfgCfYMr+CdBvpL8A2aPWX+qce28w3YcBVETkacHfdt7MOt3D4K9Nmz5Y3/OQ3XDdeTpg1g8UIvKPkPe46Jx7dHINTfWJAH7TZY/2j51zr0X2hMCz9zm/w4QPAAUfhPbjEQB3TvjWdqKyx+8ey/tZ0T1Nh/+mbdcvI7t5+3UA3pQ/UVHIZU/zvgVZnsjH5X8u+mD1fgBPlrxn4z3G/y2mv4GiW4jLfib4AwD+i4h8gYhUJetN9b8je2LhV4LJP0VEviI/jv4Fsg87f5mPu4jsKeC91nEeWfD1T4vIUr6Oa+f3OWQ3d9ZEZBn+ifyy3o7sjeZ35et7AbI35b+Rl+n+fL3/EMCf5Of7i8iiLqZ9aPkWEbk9L+/3oKCHWWTX5h+W/Cl9yXr7flE+7tcAfK6IvFSyXsePichz91jGRQDHRGRhwjpeD+CLJevdvYrsWjpAli9aSES+RESeml/bNpA9bZFMmPz3oZ/m/3kA3yEinyKZp8rkXyPsp4zT6vUHRKSWX/u/BMB/z5+M+G/I4h5O5tt0m2S9CF9b78tF5Nn5l1R7ta/PQdY+d5HsC9k/AvDnzrm9nvb4ZQDfmC9/CcC/gf45788g+/DypS77+VvodwB8nIh8pWRxKP8WWXzJB7G3qdPnHyRfguk/Kb5W9n+V19dZZPvktflyqsjeW/QAfP2UJ3yQ7/8GskwqiEgjv34i/6L0QWTxN5X85sXLMOGGrGRffj9DRCIROYbsZ2pvc/4nzRPLTXQY8TrL62yJ6+wvIosR+tEDlKdwvxzQXwHYEJFXikhTRGIR+TgR+TtBOb47b3O3A/hnBcv6fQBPF5F/kO+Lv4/s8+H/yse/G8BX52W+B1n25jWXkf2c3Lb/iddvujF4c3CynwDwefkJ5lcBfKmI/L38oGmIyAvyg6SG7KeulwGMReQLkf0sbyoR+Tv5m6QqsoP8Wo9RQHaiX3HO9UXkUwH8g2DWvQ6gn0V28D4nX/aCiLxkhu22F6aibbf+AlmI7LflJ4WvQPZzz2umXbzsuufy5V1GdjPt32L3U0/7JiLfjaweP+/at/6BtyGr+28TkbpkT9kB2U0XiMj/g+wC9JXOub8KZ8y/CfofAH5QRNqS3dR7EfSbgVmmfxmA33Y6H24vhR8oJHta8d/l2/3AlGUh/3DWQHaDrJrv82vnincge5L0VP7G/uvgg3HtchbzdtPI28PXIrshfO2n3Af94FTkr5DdnH51XqeNvF6nmfn43Ytz7pvDm7fm37QsxV8G8LnI8rX2/EkxAIjIi0Tkq/MLteTnh8+Bf3M58YOVy36u8G4A35fX0Zcjuwn+2/m8v4PsCZ6X5cf7VyF7+vjPD14bdFjkbzq/B9lN8A1kbywfQfZlQfgU8RuQ5dxcC5T+iuBJqB8B8G8ke4L8O/ZYzdch+wn+B5HFcvyL/O8/gSwn5wqyNvqHe8w7yzYNkfXk94X5sn8a2U2O8Pzxx8ie/H04GBZknX0V+XVkT5I9kP/7oYJpfxLZF3lvkiyW5C+RZZwiX+8XITsGV5Ade59oF5CX+XUAHsjr96wZ/yFkH77+S76tX4rsZtRwynYAwNMA/G9kTw3/BbKfzb5twrSvAfC1+QccOOf+O7L4j19H9nTx7yJ7SmyXfZaxqF4vIGt355Bda7852JevRHaN+UvJfnXwv5FHO+RPWvwEsmv1R/L/d0iW0/VsBPl3xpcje1L7G0R/mXNnvvw/RJZT+1Zkv4R4CPl7l/yD6jchy526IPoXAnDOXUb2IfmH8237NPgcp73qcD/TvxjZl6RvnbScwM8h+8nwe5GFyP9e/jcgy636EmTXu7Wg7JMice5C9v7t2hdJPWTxItd8BbJOGS4j2w9jBJEjZtlPRnYe2MzLNYDOyioqN9GhxOssr7OY4TqL7Bz/5eb689lF5dnnftkXl8UtfSmy69iD+fJ+Htkv9YDspvdD+bg3YcJn23xZV5FdV74dWR8K3wXgS5z/pd33InsIZDVf7q8H83aRXfv+PN8/13I4vwY8/z++3CHoFeUw/IPpQSf/288guzkDZCehP0Z24rmM7M3Knfm4b0F2Y2sN2UET9tD7Akzu1eeFyJ7U2UJ2MP4agE4+7quQHYybyO64/xR0d+s/mJdjDcCn53/7OmRvpjaQXZB+MZjeYUpvxfnwi5B1SrIG4Dumbfse23QPsgvCJrJvgX4zWNdZZDfhtpA9PfdN0L1wfUb+91Vk3yTHyH6us4Hsxs937bWfgnW/AAW99+XrGkB3Gf89wfhPQtZjUw9Zj8thN+9vRfZmN5z3D4Lxy8g+fGzn9fcPprS3wumR/WR3Ddmbiv20389F9mahl9fx3cG4B5G9mQjL/rMFy3pbXlfhvxcE5fqv+f7YyOvpC4J5f/baspFlQr0jbwtryC7mn7ffcu9RrpejuLfiO/M6vYrsePrPwXx/ZqYNj4cDHb9FbfA6nIfehqz9183fXxuU6fnIfqJyJa/bDyPvWTwfL8g+zK7k/34Uuhfuu/P1XPtwZ897n43sPLKFLF90Yg/h/PfE+Yeg17ij/O9GHt+3wj9kb9Zf/HjW617n2eu43h8H8E9vdr3yH//xH//xOrtTD7zO3oDr7BPtH7Kblq+/2eU4av8kr3yiW17+WPX3O+decHNLQjeCiLwc2U3Kl9/kohA94YjI9yO7Yf4Pb3ZZbiYR+RiAf+yc+983uyxPJEX1ml+7f9U5t9cvEoiInhB4nc3wOkt0ePFnxUREREREREREREcUnxykJwzJwn9f4LJOMugJJs//vNs597s3uShERERERERETxi8OUhERERERERERHREVW52AfarFjddszqp9/F92OkU6PHnopK/3o7Lld2VWH1aKbfutGwLK7P6kve9peT87uY1uXL1hlu77GX3eyllgxrScrOXbbM3U//Co1eccydudjmul0qr7aoLe3bsui9ljkFXLdcQpHLrNqTS33emN/HkF5UrfLWSlJp/NChxwS5Zb1Ku6Kj0y80f92cvgCv5/nLcKXfhSBqzz1urj6ZPVGCh1is1f13GpeYfl3iD20+rpdZ96b7VJ8w1q7nYcHNn2zvD1eCAvLLdUdPWL+s3KjIKjx1zDrOntMgcKxLsP3sYmeMqrflpxbxX0mWAvhAk6eRxewnX6+y8xbMWL/cAywnLYOusaFoAKPjM6WI9Lq36Yft5La3ZeYOBiq0XXYbquh6Ohn764aIuw9LClhruRP5kfnk0p8YNr9b1etaCE386pVLDejH1sKuNpGY4mNfV9XkjaehlJUERVZ0Bu67xEk8us7PXVDMcd/3r6rZp/7b8QRsZzusdXVnU14BW7Dtn3h7rRjAa2/cItvx+PdI3baKrp436BecN04ZdcB/EmXsikkw+54zmzHLMbrf7J6r5MqVjPXFktidcj/3sFY30H8J7KdMuO5F9KxLsytgs154Hi2xuPrbv69Utc3OwWV3AZ97+dTPPbw/mAyl5cy9t1aZPVGDcLvcGZtSZfTf3jtsz28EMlsq9aU7q06eZJB6UWjWicu+ZMS7xhn3XyeFxXDcAuLJnhhJvoG7musvOn9bLrTzulvyAXfLmYrmVl5v9gz/8rx66PgU5HKoLy3jSN/yrmecv87m1d1u5D9y1pXJ3WqKSN7nSEjeaxsNy16x0UG7+MjfJ4k65i86p5Y1S85978PjM80a9cu+T6lfLzb90f7mT3/wH12aet9T7SwAXPn1u+kQFNp45+xuGO592sdS6v/jse0vN/+Ta5VLzXx7PXncf7J0pte7/8sm/8YS5Zs2dbeOrfuULd4ZP19d3Xv/CX362mvYZP9NVw/HlNT8w1tce++s0aeg39a4efD4yN7lcXb8Z7N3h93Wlq9t87cKmGsYoKMfmth43nHKerQTrHegPErt+bRfehLHjzPZI7K8tLim4mQlA6r6ebJ3tWk9df8Z0ndbe5QOQzOkPB93b/LTdE/ocvHWHXs1o3q9Xjul6Sc119+wf6OHOI/5LhAe/rK3GvfSL/kwNf0bn/p3Xv3BOt72H732qGj75hg/vvHaDoRoHU8cy529yS7ulxrlt3abdpr5hGc6bPOm0Grf2dL09a0/zr8M6AwDX1sdHtR20RXOHadw315VtfTws/43fXyfevqrLa+ui6ud97PP1dX75ix9Tw5+47IffcflONe78pUU1HJmbm2ni23z9I7qtnXqnuQn5kRVf3rHeV2lHzzteau68Hs7peqmt6W0Nbxae/2y9b8Z6t2OwpI+Pubv8eW/jip638yF9nIX3COK+rofOBb092yf98dA7Zc5z5q1PfUUPV3p+2e2LernVrf2/13/r//mefV+v2CEJERERERERERHREcWbg0REREREREREREfULfOzYiIiIiIioqNCRH4RwJcAuOSc+7j8b8sAfhPA3QA+BuClzrnVfNx3A/hGAAmAb3PO/dHUdcChEuTZfFzz0Z3XrWP6J5e7ctDDn29W9U/+pKJ/Xmoz79RPb81PYKWvf4YYZnv2j+n1VC/r9Ugw7a6fkMJsj82qCzP+YrNc81NVlSlfkFMHmJ8S258G2/XUC+KoKvqjuzPTpk1fN9Iv/tlhmFXXNIkao47+Q5hBGFV1Pdi1VLd1Gxku+p9Gj9vmp6gmr/DCaHHn9fvO6Z/+nz1n1hT8jF3sz9JtnQZt00ZBiI3FqNgARl/mpKHHDRZNNmZ9chidzXOu1nz501SXYWzzCQcmczA8POyjXjavM4g3G87rSRdqOkZmnPp6G5qMQWd+2hxf0XXc3PDrbV0w27phfs4fHAPOnidqej1pxW9gNNLHWWVT/8S9f9r/HHiwZMsghcOba/5cUb2s20hkfqkdRmBV7SnS/Mx4uODX07vT1EOiyxAN9bYvftS3kfoVva+Sli6jzWOcFZ8cJCIiIiIiOnxeC+ALzN9eBeAtzrmnAXhLPgwReTaArwbwnHyenxaRkkGqRER0VPDmIBERERER0SHjnPsTACamHi8CcG/++l4ALw7+/hvOuYFz7kEAHwHwqY9HOYmI6NbHm4NERERERES3hlPOufMAkP9/Mv/7bQAeCaZ7NP/bLiLyChF5p4i8s7c62GsSIiI6YnhzkIiIiIiI6Na2V+iU2+NvcM69xjl3j3PunuZSfa9JiIjoiGGHJERERERERLeGiyJyxjl3XkTOALiU//1RAHcE090O4Ny0hQmAqvhOJp5RvbTzut0wSfzS0MO1IBTfdsJRnfIxM+zEw3TSISPd+UR1yw8PFnQnHMmCLlO87cvsTJmioenoZGNTlynsyMJ2oBLpe69hGe16DkJsBxhS0LGA6bwhnW+qYReU0W5rVNFlbDzmt726put04645vdygM43hpukwZWTqeKw7jRgs+u2LTccaf3HlSWr46oLvUCL6UEeNaz5mf10flM+2H9NRi2sEw7Z+beclC6bXDlWnujMW2/lEbc1PO27rehnWYjPsj51GUx9nw0i3iYrp5KXaNR3gqEKZzk06kzu4ed9jutOX93T96aOyoju86FzWZZh7WJehfc53mBF39fZEZlgx+0NM+6ls+3Zsl+Oquk63bvNltnVm+oeBM2msruf/IHo3Y6QPBwwXfRl7p/R6upd0vQ2W/Yrjtj4mky09bWq+pxkshGXSI9Oq2c/NoCOmguYxDZ8cJCIiIiIiujW8EcDL8tcvA/CG4O9fLSJ1EXkSgKcB+KubUD4iIroF8clBIiIiIiKiQ0ZEXgfgBQCOi8ijAL4PwKsBvF5EvhHAwwBeAgDOufeLyOsBfADAGMC3OOeSPRdMRERk8OYgERERERHRIeOc+5oJo144YfofBvDDN65ERET0RMWbg0REREREREfQ2EVYGfqst4fGSzuvn750WU17ofNkNRyfD8KtDpi7J8nkzEEMdLZY7fK2f72oP74mDR0eFm/65Upf98TsRjrza5ckeNDSZg4WZQGmJuSrqC7scmo6d8wFyxKznGRJh58Nl3XeYnXD15vNbXRjvR7p+bpJl3R24WBR748wny1e0/Vf3dRlTCu6jsd1v73NC3rbH/qAzrw7d3ph53XnMV2GaHXLlCnIY4MmdZPP1mliEjHZmMkxXccqs8+sKNJVXEgGJiNu0++PnpnWjfW0la5ecRr74bSlMwWjLZPLF0xbMSsaP6zrpXM1yFc0h0ptXe+P9gV9bFUvbuy8FpN3adu8C7MzbV7kUFdqFIyXgR7Xu03vqzRo4ifeo6ftL+nzhM0RjHpBnZvTUVLfs1+nbFxTH/vds2Zbw3mv6nZZWzfnCXMaGbWCDMuGySfUg0grwb5LJpd3GmYOEhERERERERERHVG8OUhERERERERERHRE8eYgERERERERERHREcXMQSIiIiIioiNokFbwwOaxneG31J698/rO5oqa9tHm09Rw3WYFhmwOnxXOa/PxzLyysr7zuvWoyeir6GddpBfk7m3rkDU3NuuxeWdhNpqZFqLX45KCjqDttoe5aSZjcJdguW5BB6ONTMbgqKNz1KprwfbYOrR1XA8y707rLLRxx+T99X291Fd1plp1Uw1i1J787NHiA7oM9TVd/q2Vzs7r9kVT3qHO0lNivRxU9HDS8tvqIlP+rsmlNOMHy37ecUOPS3TcH8ZNX2/jljk2IlOnXV9Gt23KXy3OjBssBvtjXe+7+mByu2xeNu091dvTuOrHV/p62nhghrt6/0iYE5qY9m/2h8ogtOcQ007VMWlyQMM8RQBoXfbrrW6azMFFXYb6itn2oC4i3SSQmMjKJMiATJrFdZpW/HhnqiHumba4rcfXtie3g60lXRfbd/htd/HsmYO3zs1BEXUSOygb1nkQrhpPn6ho3ZWb+4BmPJxycS6ct1zZXVF47z7YsM2DkNk3GwDgyhW9VNldySMzaUyfpojMfk4BAEQF1++p6y4xL1B+v5VqNyXbe1TwPnNfqy/Z5tMS7a5svT/RpBWgf3z2HVLmHFBb6s8+M4CPP3uu1Px3tVamT3SDPNZfLDX/ua2F6RMV2BrM/l5jc2tyYPp+XFnvTJ+oQG1l9vc60bDcCcCGpB9UPCh38pMS4d3jern3iOP29GmKuOrs2z5Oy73HWx+3Ss0/Kvn+elTiRFmVkhdcIiKiJxj+rJiIiIiIiIiIiOiI4s1BIiIiIiIiIiKiI+rW+VkxERERERERXTejcYxzKz5S4Q+3nrXz+rNvf0BNO5zTPwVvR8FzJiZaRdIpP9cPM/Fsfp8ZDvP94svrelqbNxdkn7nRqHC5NpfPhdszMtPabLRwXluGInZaUyap+wy58bLOHRgs6o/ulb4uf9QPttfmvpn1pC0fC9A9rp8XSut62voVv972OZOdN9bDo1Y0cXz9is4OigY6gyke+rppXDLBb0OzL9WM5nmnyORQBm0xaeo6TNs6i0lGk6Ma+ssm886s1oX5chXTXmzzafj12Jy6qKsXHJvIpWqQRReZ8opp01EQL9a+oHP4kpq5FRSUsXlJ13elq4ejLRNhE7Y3G7MU2YoKMjhtOzVcsN9dU0e3JHW9nkrPL6t/3GSTmsOufc6sN1hUdVuPS6t6Pdun/MI2jzkzrR6urYf5inqVsWnS8w/p4y4Oju+VZ+nt2Xy63pedU1s7r6MS+WB8cpCIiIiIiIiIiOiI4s1BIiIiIiIiIiKiI4o3B4mIiIiIiIiIiI4o3hwkIiIiIiIiIiI6otghCRERERER0RHkhhGSR3znFOmKf3bkT/FkNW2rZjsaMMNqwbZDBtsjQ9B5g+lEQTq6I47hXcd2Xo9NhxKN81tqOAo6JIGY52DssBWWw3aiYMo4cycktrMGsxzXbu68HrdNhx0DXYfVDd0pgXSDTjxsRycjPa2r+u0bzRfsRwDVoIpbpqMKF+t5k4aut0rX11vcn1wGAGheCeZb2S4sU0iqpvOJZl1PELQ121FOatqT3T8uGIxGet7+8uRpayt6v45bet6k7uvC1UwHGGOzXHOcVbtBhyRjM29db08UdKxTv6x7Nqktz+kyNvx64p7eV/FVfZzJATqIseWXcdA27XFlj7uKr8febbq8m3foOm4E5y5nmrSY/oXiod4fYTu2+7l5UXe+UtvwHaOkNd1JyvZtenuSoL+byFZZcV8s6B8LOj55qt4f7ZP6+Ag7Idm4os+fB8EnB4mIiIiIiIiIiI4o3hwkIiIiIiIiIiI6onhzkIiIiIiIiIiI6Ihi5iAREREREdERFI2BxmX/vMiJ9/hgrJXuopq2fVFnliEpyOizGYOJCdgKs8ZMDpyr6CyxwaLPlNu8XX98bc0vqOH5IDMu2ujqdZrcPegYNZXT52z5Lbu9IZuhWAnKbDIGbV5esuDzH20+Xm1Th5ZVV/T2ST/YP6b8rqrrrX/SZxsO5/S00UBvW23Tj6/0dXibMxl9cT81w0GdT6vTcLzNs6yY2xbB+DCnEQDSls6BU4vZGOg/mGYpZt+1H/PDLtLrGc7rfRkPfJmal/Vyu6f0cDI/sYhwDV3HwwVdF/1Fv3+Sms5XdJEe7jzs8/KqJsexvtpSw3HbLzfu6bZmMyt3ZWcG7cuZzEEpOvYNO+940Zdx4y7dBgZLet5KcDjUN3RbG7VMHS7p9XTO+e1rnjN5l6a89a4/zk52G2rc9nk93AtyKW15Ez0pNu7U7SkJmltsTmXbF02uYLB5lbUD5KEafHKQiIiIiIiIiIjoiOLNQSIiIiIiIiIioiOKNweJiIiIiIiIiIiOKGYOEhERERERHUGSALV1n8/VuODztpahc62qayavLcwds3ly04aDfD84k9FnMsqioZ93rGPfsPIs/axLf8lnEC5+RE9cu9rTyzVZYq4bjB+bjDWTMSjhtpscQbVtAKRemzitm9O5by72y417ugzRls58lG5fDav1moy+1Kxn66wvx9hmDvZNjuBwclagyhQEIGOTJ5cE81Z0HSZ1XRejIPOuUddZjFFNDyMYThZN/pppa/Gmb7cqlxEozL8DgPHcsZ3XvWMmD09XKeJwd5gqi8a6TiXIddyVb2nnHel5o/Hk/RHZZhvuD9veY5MbGI4z+ypd7JgJdBnC/W5zKO2y1HpN9UdDvQGu5ueNdAwiGlf0cHXbl2nU1GXYvFtPC6fHN9aC/WHbacNkWAblr6zrc2JtXbfTcDVi1tk7qYe7Zybv1/qKOf9c1sPhfo/NaeEg+OQgERERERERERHREXXrPDkoAme+QTiIpDn7vGl99h5fAGDcLDd/Wpt8V38/otGUnqFuIPvtxUG5Erevk9l3OQBAyu02pJM7ypoqHkyfpoj9ZuWgJJk+TZFKb/o0E9dd/AXeVGXaTNn5y9Z7pcQ3PQCQlmzzSX36NJOMWzfvPHMYSS1FdHt3+oQTNBuzN6Znnbg487wA8LzFj5aa/67a5ekT3SDnmkvTJyrwruqdpea/b/X0zPOuXijounAfKmvl3tK1Ls/+XqPstT4qeCrl8Zi/6OmJacadcide21vhQUl99gt2uzqcPtENtJE2p090gyyUeaNCRET0BMQnB4mIiIiIiIiIiI6oW+fJQSIiIiIiIrpuxAFx8BDpuON/+hIP9JOpLjbPlVSDj5ImZw+JzZ4zy7Ljw3Fd/WRn634fLpbWTqhxFz5N/9Sne8o/CbzwoH4qOK3paaOK+ZlQsH2SmNw0mwsXZgdG5unjivkpRpg5aPPYTAZhFGT4iclf25UxOCh4+tdsW/+sDsjbfFJQhmWznAu6/EnVlzk1bSAemRzEkWkHQVajM5l3kprcumDQ7itp6ces03n/5HHvtN62+or+CVZ8cc2XoV/8Mx2Z09l660/x6+me0fuuvqrnnX/Yb7v9JVJaNft56IcHy7oe0qapJ/t0fhjj2LN5nXrSqOv3j63vuK8nHjf98TxaOODPiYJFu4qup3HLZDU2/LDNs2xcNu0pyFecf0T/qiat2OPbD3fNLzd3/brKHM/Dji9TeA4EgHHLZGN2/HDTHM/dk/r22qjtxw8XdRFGHdP+7b4LhqtbelxtQ8/buuQbSTyY/Wd4fHKQiIiIiIiIiIjoiOLNQSIiIiIiIiIioiOKNweJiIiIiIiIiIiOKN4cJCIiyonIooj8loh8UETuE5HPEJFlEXmziNyf/1+uS1wiIiIiIqJDhB2SEBEReT8J4A+dc18lIjUALQDfA+AtzrlXi8irALwKwCtvZiGJiIiuBwfABXn7q8/0HTDYThVaV3TQfWXDd+4gW109se2AZGzS9l1BaL6ZN1x244rusKCyrTujqG7717UV3flE1NMdGmBohiXYYNPJiNji2k5IwmltRydh5ys13TOC2I5cgjKJLV9fd7Sxe8W+TK6qP+Zvm44Sxmd8PTbbern9qu6QwcUF2zq2nWfoYResVsxzSdFITxv3w/LrOkyO6Y5CBsu+w4zecT1tdVsPu1FQj7YjnIqul2RZr6d3IuiMpa47gait60U1Lwb1aDq8iMa6g49uUOakqUZhMGc6vzGGc75M1Z7eN1XTyUjINXUZxKymvu7ntZ192E450qoePww63hi3Tac75jwSHkvVLV2I6pZeT9gpkm0vMrbHqF9RzSy3eVEXwnYO0j8WlKGr23+lq9db3fZlGizp43ls9+VysI6T5pxY0WWsbNhOa/z2jXSzBGD2e9fPW920PdjsH58cJCIiAiAi8wCeD+AXAMA5N3TOrQF4EYB788nuBfDim1E+IiIiIiKiG4E3B4mIiDJPBnAZwC+JyLtE5OdFpA3glHPuPADk/5+8mYUkIiIiIiK6nnhzkIiIKFMB8MkAfsY590kAtpH9hHhfROQVIvJOEXlnsrE9fQYiIiIiIqJDgJmDREREmUcBPOqce3s+/FvIbg5eFJEzzrnzInIGwKW9ZnbOvQbAawCg8ZTbigNjiIiIDqEwYy0y0VXNFT3sKv45k12pdCZzDenkLLQwK2+vecPMuMqqzjZsXNUhX2HWmPT1BkhXZxDaHMQwK9CNzTM0kcmqSydf5l1qstHUfDpXzJZJZeKNdfmdGZbK/j/KD+d1HXcWfT3WK7oeerHdNj+vmO22GYOwdRpuj8lijEw+XjVYdjQ0+WyxzcDz+yfVsW+QccFbsFjvV5sPOVps6PW0g2lNE47scD/Ii0x0Geqrer3jhh+ur5hxLdv2sG/1FZ0fKSNfSJvjGNt8QnvMqgXp+u/O6bY3WPLjh4vm+DWHd3XL/yE2zT81+zkssZjMwcpAlz8Nzkcu0rmBA9P+bXuKe/71sGPPZnoH1Db8eu3xkDT1vINjvsyyrPNSGw2dKdpr6LaXbPo63tV+dNQqtk8H453e9oPgk4NEREQAnHMXADwiIs/I//RCAB8A8EYAL8v/9jIAb7gJxSMiIiIiIroh+OQgERGR988A/FreU/EDAL4B2RdprxeRbwTwMICX3MTyERERERERXVe8OUhERJRzzr0bwD17jHrh41wUIiIiIiKixwVvDhIRERERER1RYSbYcMFnaFW3TS5X3+TNBXlmNoPP7cocNMMuzKIzH0kLMghtRl9tUy93HGR+9e6YU+Oaj+jFRkOd+WUz/XSZTBpXEmx7rLPcbFYgqkEoXl9nwhXWkzN5flPYrMNQoqMZsdDw5egOdEZZ3DcZa0GOY2xyHFGwTgBA5PeHzXmzQZVhppz09L6RiskKDKq/0tN1WNkwQXbhfjaZgzZnr7qu90/zkq+bUVsXuNrVmXfRZrBes1xp6WDExoqvR0l0+0lNDqLNVAz3R5h/BwCwGZCjYD0mD7K6ZeopKLNr6DaxK1vS1dVgNPbHcKWr62k0p4fT4HAfmXy/4YI5loLdVTH1DZM5GAVltJmPjVWTV2jaeP9YcN6Yt+cfPdi66OvURTZ7UU/cuntj5/VzTl5Q49YG+qC8v39Sr1Z8XUQ6rhDVTT0cjfx6h3OzJwcyc5CIiIiIiIiIiOiI4s1BIiIiIiIiIiKiI+qW+VlxWokwPNacPuEESb3EfdCCXr33Y1d35Ac0atrutA9GDvZEujKevcoBAK7k7WcpeLp/6rwlthsoX3b7qPyBZi1Z9upWufklmT7NjZrfdgl/8PlLzY60NvuOS0ueUaNhyZNNmUaHcm0+aZQtO4Xq1RInv5JGLp4+UYG+q06fqEBaoiFupo1S6x6n5ba9Nyqx7Um547eyXW7+xsrsx3Dp62VJUVLu/OPsTwIPIK2Uq/ey585qY/ZzRbMymj5RgdSV2/atpNzxmpS45q2XfYNLRET0BHPL3BwkIiIiIiKi60iA8DuhpDn5hvWu747CDDObs5cc4Jtim7tnMwfD4Uh/GzFu6WkHC364saandXWzAXY9QZmloTPV3MjkEw59CNiu29Sx/qJI5QoOD3BTPiq+Ab4rIzH8osPUU6I3B3M1n623tq1vlsc9vd7apl9P1DXhZybHbledhkyZnNm+OMiQi7qT8/AAoHHZZ+LV1vV+jdb0UxIubIsmx9Eleji+sqGGF+/3Fbdxt87hGzd0+V3Tj7dlqKx29fC6rwtJOmrc9hnbTjFR3NVtQAqOJbFtz+67ID9STJZkVNO3japbuoz1Kz0/70gvd7yk21fvhK8nW4c2QzHMqeye1PUfLevjLBr7bbcPhg1NXuTYDI9a/nXSMjmUW3ZaXxf2oRL7oEit4vfP1kgfhI+sLqphuajH14PzV2wOh+qWLmP4Jamt04Pgz4qJiIiIiIiIiIiOKN4cJCIiIiIiIiIiOqJ4c5CIiIiIiIiIiOiI4s1BIiIiIiIiIiKiI4odkhARERERER1BaQXoHw87/PAdESQdHXo/apvnSmI/7EwHBrs6GTEdQSANxk/r9TtYlqsU9y5e8f0ioHFFd8AQbQ1QROpBhwB2PdO2L2R7QA87gpjWUUvYScdoSm/m1ckf5cPOMQAgrZgODMQPj0amc4lts5p1X37p6zoV06mFs2UKOsQIO5cAALF9ZwyCZZvOM2yHMLXHgnFmX7ktvQG6QxLdhsUWwmxPPPL73XY2MeyYziiavoOM6KpuL7K2qWcOOrxx8Zye1jS12rouY/OK7+SiutpT48R25FLUaZBVCTbQtmHT3pPG5PFV06lLbUN3xlK95Ld9vNxW46KBKWO43jN62uGc6eAmaF9234zMsJjDMA5ODeLsvHray88N9rNZTkXvDqxcnt95vb7ZUuPiB3VHLe3LpoxhO7CnU3sanNzED+SGPjkoIneIyFtF5D4Reb+I/PP878si8mYRuT//f+lGloOIiIiIiIiIiIh2u9E/Kx4D+Hbn3LMAfDqAbxGRZwN4FYC3OOeeBuAt+TARERERERERERE9jm7ozUHn3Hnn3N/krzcB3AfgNgAvAnBvPtm9AF58I8tBREREREREREREuz1umYMicjeATwLwdgCnnHPngewGooicnDDPKwC8AgDqjcXHp6BERERERERHgIuB4UIQblXw6EhqswHD3MDUTR6313ARm3cWZMaJyaJrrOqAtnHdlzEa6kAwO++u/L8wc20w1OPMvFKUz2aXOy1nMBTUkzPziV1PQR3LUGe3tc7rfffhc6d88Taqatz8il5uvB1kDo5MJpzNYrTDweQy0o0rikz+X5BnaDMG0dNZes7uSzXO7Lswt86Wz2YkxuYASPy8Nk9uNKfrdNzy9Vgxy3E9kw0YrrKr67Rm9kd9XbeD1kM+0082TECkXW/dL0siXV6bDzm6bXHndfekzqwcN/Ryh2bb25f8sjqjRTUu2jI5iEE7tcfocLmhJw3OOWlVJo4DoEIsxw2zrWbS2rZu4/Ug19Fu2/btet7RbUH7Wtf7auF+XU9pxddj0rDT6uXWN3RdhOcymzHoZEpdzOhx6a1YRDoAfhvAv3DObUyb/hrn3Gucc/c45+6pVtvTZyAiIiIiIiIiIqJ9u+E3B0WkiuzG4K855/5H/ueLInImH38GwKUbXQ4iIiIiIiIiIiLSbnRvxQLgFwDc55z7j8GoNwJ4Wf76ZQDecCPLQURERERERERERLvd6MzB5wH4OgDvFZF353/7HgCvBvB6EflGAA8DeMkNLgcRERERERGFIoek7XPY6kF2WH1NT9q+oLPcJMjlc85kuU3J2XNBDpyMTY6dzdYLhl1N53YlJoesMggy+yKbkWjK6ExmXzC9S9KJ47IV1f3rkc0n1NuutlWKs8FszmDxxDbfz9ejbOuMu4WP6Yy+/vHmzuuoYfLXNvRwtB1kxtnMQUPEbHtwt0ESU98Dk+PY9euxmYK76iUcjqY87xTud9u2pgjzFptXdQ5f74S+lXLlE3ybWK6dVuNa7zunhl3fb2v1wroat5DYzMeBLtSVVb8cU16VhQkATV+m8XJHjRou1dXw5u3+2Bq3dTtNdBQgUlONzRU//bijj1G0zTFb9/vLmay84ZxecO94FIzTi3Vmt1d6flliKqayK2NQHzvVbd+exiYbcNzW87YX/LE1OKfbRPu8bqe1DV/IUUuXydbhsGPyIoPB6rYub31dH4fDeb+w/uLsz//d0JuDzrk/g87bDL3wRq6biIiIiIiIiIiIij0uHZIQERERERERERHR4cObg0REREREREREREcUbw4SEREREREREREdUTe6Q5LrxsXAqHOwANFQUisOfy1S6duoz4OJxuXmd/HsZQeA0dzs8ye16dMUsUGhB1XpTZ9mEhtEelDjxvRpiiTx7AXYFaB8QFFxVvD0+YflKq9s3Zda9wFynPfiohKFr5bbb2ml3Pxljzcp0W4knT7NUSLi0GiMpk84wXKzO/O8c5XB9IkKPNA7UWr+92+dLTV/OjGqeLqLvbnpExUYJDfxbVG13EE0XCw3f3959hNI61K5dZd5jwaUP3eiMvu2pyXP+7iJ585xyYvGxrg5faICIzf7+3oA6JV4k3pp0Jk+0REhiaC24vfF8ff6N1LtR/Ub8fjqlp456JAEpgMPZzr7KOqIw5lOLqSuO0oI53UV3W6Sul5uY9WXv3p1W6+nr6+PUjUdJ6RBme178UhfH8J5bWcsbmA6KClwkI427LQC23FL8NqUofnophpeWF7ceb11m15PbDpUkbBzkIN0mAKz70yHMGKeU3KjA6wnqBvbtmzbKy6gOQ8WdG5iz/eJbqZqeDinl9Ns6POVhNtnjp3quRW9YLMvXdiBj+mAxNnOfYK6WX9aW43aun3yttrLw2hO1+loUe+f3hm/nvaj+kN047KeN+zsZLCg15M09bRp1Q8nHXOcVfRwZd3XReOS3le1Lb3cSk/PG3boMW7peSNzOG8/5t9rLj6ix7XO99WwjP16+if1NXP7lD5+t8/o9cbBomrm1Bv3dP03Rn49Lpr92sgnB4mIiIiIiIiIiI4o3hwkIiIiIiIiIiI6onhzkIiIiIiIiIiI6Ii6ZTIHiYiIiIiI6PqJe8DSfT6Pa+6D6zuvo25/r1mCmYPMLJvRZ/Ll1LSWzZcb6sxeF+SqSU/nBs4/pIerV31mb2py3qLjS3o9dvvGvhzS0vlgLjbP1PSDILIpGXdFeYu7hJl3tg5tBqHNdSwYF23q/MjWJZ+7OZifklEW1Ivd1qIyAACqft850wbEZE0eJM+wsE7tuIIcwV2ZfabNuHpQZrPYxlW97Y01Pzx/v854lLHetvTE4s7r0bGWGldd0fsqurKuhm1GoVqPbSNBXdhM+LFercr7cyY7P2nrdcbzOogvCha+ZepwsKTrOMzSG5zQ9RIt6+M5XG7dZAymqd4hYYnSVZ0navP4bVbp5h1+P/ePmWPHRHm3zvlpFz+q66FyRYcDhm28vaUXlFb1+Shp6uMj7vlyVLdMXud48rmhtj57iDyfHCQiIiIiIiIiIjqieHOQiIiIiIiIiIjoiOLNQSIiIiIiIiIioiOKmYNERERERERHUJQ4NFaCrL2+z8VylSkZcVOy9vSkZlqbpxdOO9Q5XhJk0dk11h/S2WHjk/M7r/vHG2pc87FtNRylNl/O55RFVzd0GWx5x0FdpCZ3rygPr6o/ftsp7babBe9/PTa/r6fzFWsrfri2pfPZdlVysO9270eTQQiTjRa0GRmY/WqzJQuy9HYJMy4rtk7N9qiR+tkomWur4aSl20Ra8dNXN3T7X+jqZcV9X+dpzWRsLuj19G73mY+rT9XlbazW1fDSfSaz70rQNu0xaY5ZF/t6Co9zAGhe1uXfvCuY75TJ8oz1vhlt6zInQ78sGenlVrZ0O21e9m2mtq7Lu32bzvocLfk2kjR0+V2il1tZ9fXUvKTbZTzU5e8d0+vtnvbTJy09beOSnrZ5xY+vrul6sm1cHbOmvbfO62xJQG97Wp18fIvN/py0zgPik4NERERERERERERHFG8OEhERERERERERHVG8OUhERERERERERHREMXOQiIiIiIjoCEpqgs07/EfC1kNB5prJM3OxyWsLs63S4vxBMblwYbaezZrblcMX5OdF83N6nMnw27qz5acdmSxAk9M1OtlRw2G+XM1MOzzeUsO1qz4vTB46j0K1yRl4zmYDhmKT+WiGdy8sqEezWAe9L6Oezz+rben6t/Wm8iGLyrvner1dOWkmg+1AwvZmcwQbOrPPNf2wa5isPDvc1sNpkNlX3Soub1r3+0dGpk33dBbduOHLvPlUXWlbY30EONHtdOlDfnzUNTmOfV3G8ZLPsVt7st623gld/mTOl+Ps8XW9zobOx3tsfUENr13wx6XoeEuM58x+X/Gv5x/W2z73iJ53+5Q/H/VNedOKXm7zcpCvuK6XO5g32YZnTVbpcV+PEuvlJmu6fdXX/L6NN83GRnpae84MxZs6r7AZ6TINln27HZryu0i38bjvyxSNDpDdafDJQSIiIiIiIiIioiOKNweJiIiIiIiIiIiOKN4cJCIiIiIiIiIiOqJ4c5CIiIiIiOgWIiL/UkTeLyLvE5HXiUhDRJZF5M0icn/+/9LNLicREd0a2CEJERERERHRLUJEbgPwbQCe7ZzricjrAXw1gGcDeItz7tUi8ioArwLwyqJlpTVg604/fDLogCFa2dYT2441CjqnkJbuwEPqNTXsBkEYv9MB+s4uNgjqdwPdAQMaermDBT9tUtfPwWzctaiGO+f0elvnTOcCgbhrOvRY3fRlsuU3HW+gFyx3WoceQacjUp3yUd12AhMO275LTKcdGPtyxANd/rRquoQJO/gYmvq3ZTCdKoTbu6vLmnRKXRQIO3KRsd43GNuNDzoksZ26mPLKWJeyMgw6n9jQ7WO80FTDMizoCEL0eqpbQf1v6+NqvGA6KLldtwNx7Z3XzSu6Y4rqhumQpOnnFbOvEj0r4nm/b0eJrqfU6fK36qaDlZP+XNH92LwaZ/d772Rwjhnqdtm5oPfl0kd8XXTXdT2Mm7pMqmMdewge19P2T+k6jmpBO13T55T5B/Sy2g/6zlqkb46Hg0hNR0ADve2S+nKM67r8aUXvn0rf12Nle/bj6ha6OShwse276gBzz95pC8bNcg9YJrXp09zI+ceN2edNJ3eu9biISnRiFZU4VgHATekQ7EbOX3afS3P6NEXqk98b7Us8mD7NJGXrvcyxDpSr+7TsGbW4o7/pZj9FlhYNb+LKD6FKnOJYuzvz/Hd1VqZPNEGnzAEI4Hx/YfpEBc5tl5u/O5r9wrPdL3fyHO/6QHEwUTT7QRy3xtMnKpDWyp38kouzv1lwJX+HkpY879sbAAdV5v1lNC534pbZ38OXNkjKXbQuDTrTJypQKXnB3hjN3mbPb85Nn+jwqwBoisgIQAvAOQDfDeAF+fh7AbwNU24OEhERAfxZMRERERER0S3DOfcYgB8D8DCA8wDWnXNvAnDKOXc+n+Y8gJN7zS8irxCRd4rIO5Pt7b0mISKiI4Y3B4mIiIiIiG4ReZbgiwA8CcBZAG0R+Yf7nd859xrn3D3OuXvidnv6DERE9IR3C/2smIiIiIiI6Mj7XAAPOucuA4CI/A8Anwngooiccc6dF5EzAC5NW5CLgFHH/zw+mfM/15aL+qffYvL+XJj1ZrPbTMbgrqy9dPLPyp2ZVsK8s4GOzpCBDk4bLPlpt+/Uy2k+pjMMRi0TKRDkwiUtXf64p7OOXL8gwsOUX+Xjmew5VHWUhlSCj+c2v8/m+6Egl8DkIO4aDstoFjtu6OeHXCv4Cf/q2uR1ArBhhxJuX8OE3I1NPXWDOJZpkQtBrqOzmYM2Hiloa3HXjDS5js7mPAbz2sy+ii1/y29f76y+6V5b1/s5GvllNS/r/bxl2uVoXq+3vxxk9pk4FGfaTNz3Zeyc1+Wtbut5V+Bzqa6c1uOubC2rYTFxQumi3weypI+V6BF9LIXRX6OOyQJc0uttXvFlrq+b81Gq22k8COrJpoXYQ31s/hBkN849rJe7eL+OCZL1rWCl5lix7cce7yF7PJtpJfHjo0RPm8Y2gzAoQ5molJnnJCIiIiIiosfbwwA+XURakt1teiGA+wC8EcDL8mleBuANN6l8RER0i+GTg0RERERERLcI59zbReS3APwNgDGAdwF4DYAOgNeLyDciu4H4kptXSiIiupXw5iAREREREdEtxDn3fQC+z/x5gOwpQiIiogPhzUEiIiIiIqKjSBxc3Wd5bd/m8+UWH2voaU3moLRbfsDmr21t6WGT0ecSv063K0tvMptHiE5TDcZBpFx1VSdo1db1rI01k8F2fm3ntWz3zMQmLy/yy95VepOnqHIGI10mMZllqBR8PE/Hk8dZUpweJn2/L5uPbqpxdZO3KFs+c81NWe4usc+QSxd0Dl/a0NtaueCXnV5d1ctxBW3E1Lcb6naKcDjWmXa7hq0gw8/Z/DhTF4L5ndfVTd1enJl1NOfXO1wwWYYndC6iMzMPBr7NLzyotz0aT66n+prOAoz7et6k7nMR10wWpm3kSx/QZRrO+zazfbtebmzyCRsrfmFiDuekpqftHQ/2z65sTD1tVBTBaeo/7uk/1Ff88MKD+jirXjU9uge5gq5iMh/b+pzpKr6NyFAvV8bpxGkBnUtZ6emNT8zpSC+4YNwUzBwkIiIiIiIiIiI6onhzkIiIiIiIiIiI6IjizUEiIiIiIiIiIqIjipmDRERERERER5ETSJAJNpj3r9NOS096bE4Nj+d98FXtksnl2jSZg4nO14IzwwUkyOGTqvn4emVNDZ7+U599tvZsXd7Bgsk+u2wyFIMcPtRM5tpI54W5kc9v27VtVpAzKDZT0A4HGXdiM+4qJsfOZO0hyGOUXZlxOrPM9X2unZzTGXexyeFTOY+Rzd3bf8CZi/W0o3mdbRhtBe3t8tXi9RRlEO5asZ/WjU1uo82wNCSsC7vtpp6k6+uxsqG3bXBCH0u95SBzcEnvx6ecXFHD7Ypup+/ZujNYjm4/c4/pXMEwx25sMu1s5l216+vJZvKN27q++8f1+NYFP762rse5WM8brqe6rcuQVvW826d9PY1NBKozcZEuqIrm1eJjUszo+povkz0v2GM/bIs2Y7B/Wudqhqrrerlxz+wrkz8qYbs1j/Sl5rThgraZVqbkaBbgk4NERERERERERERHFG8OEhERERERERERHVG8OUhERERERERERHRE8eYgERERERERERHREcUOSYiIiIiIiI4q58Psxy3/OlnQYfvDRd3JRW3FB+zLuu6AJLUdP9hVppM7lBDT0QPCzjXsfGY9UdCpiJhJbQcGW7fr7Vu87IelNyhcj2I7BrHCzjRspxa7lhUU2nQggYbukGTXkvpBmW2nInO6Q4y05TvMkERXVLRuOpcJty+a8mzReHIHH5HpgKG6YTr0CNdT1CmKLdOuFZkyFnSasrutmdsjBe1013KDDiWSjt5XG3fpDkr6J4JOLSp6HXNV3UHMk9q6c5aVu/y+PLdxShfJ6WM07LjCdmJhO+UIh6tbetuSJ+kybc3rehs3/PYtfFQvuLGi910UtLdoWHzsjBu+HMM7itueBKupr+rjtTGn542Hevs653zbjNf1tu5qi/ArSlq6vrsndSVX+n5bK1u2oxYzbJrTuOnXO5jX5R/N2WX519Fg/50EWXxykIiIiIiIiIiI6Ii6ZZ4cdDEwbM9+L7PanfKNToF09t6gAeg73rNISs7vSuzltDp9mhvJfuN3EK64Z/rp85fc77u/ztu/tF5iwwGMyh7Z9quLA4oH06eZpMw+B4BKt9wCyhxv49b0aYpEw+nTFM4/mj5NEfsN4kHE/XJt5okmlhRL9e7M849LXHjOjxZmnhcAHt1aLDX/pfVOqfmT8ezbHlfKnfil5AloNJq97JWSZS95ySt12q9ul6u3su+Tyl6v3bQneQrEgxInTgDV7XIX7N5w9o2/stUute7uqNybxCQt94xCdzj7+jdXym07ERHREw2fHCQiIiIiIiIiIjqibpknB4mIiIiIiOj6kTHQuBJkpQVRaRt360y+1kX9E4V42/9UxDV0pppU9MdMN5z88wixT+86/USuC7L07HPKUtNPkMrIZ41Vt/Vyuif0czHbp/Vw++TczuvaI6a8Bbl1u5iMMpVrZ+rFTlvEtfT+CHMDAUCCTD8xmXzjxaYa7p0OlmUqtf2w3tZoffIvIcRm8lXM9oTlGJr2s2nqNNh3sPt1bKat6W1XTPsJcwOdM+W19W/3c7g9pp3aNp4u+CeSt+7Q+6p7Vs87avtySFM/+79U66nhk7VNNXz3vM8gfPTkkhq39gyddVjZDrINpzS18Jdf46YZZ37dcMedK2r4wvy8L0M8p8ad/is9b2XNt4NoaLI8Tf3PPeLraVzXdRrmowJA67JfTzTQ62xd0cNuVa+2cdnnDNpjB7bNBJmWLtbnEGcevYvGwX42i0nruv2M5nSb3z7lx/dO6W21+ydcdqybz4HwyUEiIiIiIiIiIqIjijcHiYiIiIiIiIiIjijeHCQiIiIiIiIiIjqimDlIRERERER0FInOIhsHHTmP2jrnqtLV+WBh5l1lq6/HmYxBZ7Pp9jkumyDIqhP9bEsU6xyywZ3LO6+3zuiQtUpPr0dMtFjSCj4aR+YZGps7lqQTpxWTQ4a6z4ETu1zDhXlnJtMumdN5cv3jetsrvck5fMMFvaywl/j6qs5ji7p630k/GJ6WvWjrKRi2+YS79npYN8uLelxVl1+6QXszWYbODIcZhLtKb/Mhq/vvBd1mQPbu8Fl7G3fr/Zyauy5hRpwb6mkf6y6o4VP1DTX8sY1jE+dNGqaOgw2u9Ezu4VxqhoP5TD6hvWn0icuPqeE7Oj7E751ypxp3uT+vhk+/3a836utzipi8yDg45zTW9Ljthi5kUvd1MZrX+zEa6nkrfd3mZeSHnT2e7caHx4A5nOOhOcckfjit6omHi3rBm7fr4e4pP2/S1OVPa3o9cS/IQTxIPqrBJweJiIiIiIiIiIiOKN4cJCIiIiIiIiIiOqJ4c5CIiIiIiIiIiOiIYuYgERERERHREeQiYNzaO/Mv0nFguzLjKg9d2nmdXFnRy010phdMlpjKDiwaZ4YlMnlaJjOuuu6z6NqX9Efd3rLJKzTbl1b8sl3D5PdtdfVwmGNXkDEIABJkgKlMQQCweYvh9plpbRZgpTc5Vy0amCw3s5645eut0jP7KjFlPAiTDagyCG0eocl2Gx/r7LzundZ5fsOOnnbu0YFf5VW9b6RvMgeLMtgquv2ksZ5Wer7OxdRLOt9Uw/0lv6ztO03jMk2kdtVPW72i6+zDjVNq+HhjWw1fWAky/EZ6wc6sZ7zo9+1YRxlC2rqMlaqfdrSq2/DItLUrg44a/vg5n0G4dqKlxr33qXpfbj/slzU/MHmXQ3ve8G2mum3PE3rfbZ/yG1/t6P3YumSWu23yU2t+WWLaAJL9P09nMwfDHMTuKX1O2T6rl9s7qecdz/vtdRWz7U6X0QXrHc1NyXAtwCcHiYiIiIiIiIiIjijeHCQiIiIiIiIiIjqieHOQiIiIiIiIiIjoiOLNQSIiIiIiIiIioiOKHZIQEREREREdQZIA1fW9O2xorOkQ/8qFNTWcrvrhXR2QTGM7Idkv21mJWW+06TskgdOdJgyW9XYmus8FxAPfKUHLdsphy1sJPkabTlHEdLSxqyMOVWDTAYadN5z06oYarncHeoKwM5Cx2XdXdPlrTb/xrqrLD9PBigumLdwWAKjpjitcJViW2ba0oW9FDJd8hw1h5xIAMFjS9TQOytRa0uusrenOJqKCDlZkZDp9McNhmVNTT4NjuqONpObLGG/raZPlkZk27BDGFGpTb89j27onkU7bt/EN0zFFOtb11mr7NtKo6TLYvn0GY18m2yGJG+rlPrixrIZva67tvD5R31LjxHSmEXb8k9R1PZnNgSS+ve3qEKmrh0ftYLmmP6HhvC5/pWeO2XDZpmKc3h1IgzY9nNMjR03TyU7QZ03/uF7uYNnUS11vj+qEpGKOO3O6TZt+vIvZIQkREREREREREREdEG8OEhERERERERERHVG3zM+KXQwMFvd+5H0/xs3Z74NG4+nTFElL1nLZ+e3juQchsz+VCgBIaiUXILMXPo2nT1OkzCO5ABD3y1R8qVXDReXKnlanT1PE/kzjIOLR9GmKlau8cWv2edOy7b3sji/bbkrMHw3LrfuJJhaHVmX2xlyJDvjzrMBDaydnnhcArqx3pk9UIBmXO/nGldm3vdUo1xBrJdYNAKubs59ARsNyF/t0VO773nqJ01elN+PPA3PxsFzZk2q5k19am73N7voJ2gHV1stdN7Y3Z79grw3KHavr1XLbXvqalcy+gOrFkm90iIiInmBumZuDREREREREdB1FQNLa+yZ1mKEGAK5is+n8sJinKVw65cZ3mF035WEACTLAxGTaSaupJ+76PLbO/etq1GBhSQ1vn9XrrQZfdMjIPB1SMR+bw5zB1OYTmuywgpy+XRmD4XJNnqIbmi/A+n09HAX7w+QGoq5D2CSoJxlO+TIzyDJ0VVMPRRmDgNq3NtswNXlzSd3P62KTD6nj/TBuBsu13y+Z5uSCMojZF7YMlgTlGBzTTz+s3623fRDE8MVDs56tybdd7EMZrq73e9N82fzUU1d8GZZ1xWyNdBlHwdMy41RX1DDR276xHSzL7sa+/sPVjbYa/mj7+M7rS905Nc5t622PwhxBe6xUTd5lUDep+SLQPtRQ6fllxSaOs7+olxsPdJmaA1/n447eIcN5XU/9Bb+sUcdkPpp9mQSnp10Zgw1z3rAPJgVfwIl98Ce25+bwfMrMQSIiIiIiIiIiIjog3hwkIiIiIiIiIiI6onhzkIiIiIiIiIiI6Ihi5iAREREREdERFA2B9iN7j2ufN8FdNreu4fPN0p7OvwtzAgHAjQtysGTK8yphtmFHZ525zuSOoKL1LTVc6S/q1aa6jLX1IGfQZg5WbTDc/nO9JMxUNPmKtvyu7XPfoo2uXlBf7w83tr1m+mw6Wzqb7SZNv55dy7EZhNHk8ovJYhRTT67h20x6THd4Npwz0wbRbrUNXd6krtfbWHHBtDqjL+6Z/Msgx66/bNqwqZfmBT1v1PU5j7Kg57UdMA4XfUZcsqzrsNHReZGVoAO01LTD2+Z1u31y56oafmrr4s7rkdN5eLHZ893Ul/nKSNf/o91FNTwKMghXEn1MJn29n+fb+nh/aN0HLl59UGd7LnxYl7G26etYpmSTjpt+vaO2Xs5gSddb84oLXuv9OGrr8o9aevtqLT9++7Rul91Tej2DZb+etGbzRvVgmEHo2rpMYjr1EpM5GHa6ZTMHd2UQBjmDZfr64pODREREORH5mIi8V0TeLSLvzP/2/SLyWP63d4vIF93schIREREREV0vfHKQiIhI+7vOuSvmb//JOfdjN6U0RERERERENxCfHCQiIiIiIiIiIjqi+OQgERGR5wC8SUQcgJ9zzr0m//u3isjXA3gngG93zq3aGUXkFQBeAQCt0x07moiI6PARIK3tnVI1WNTZW/GW/ugYBflzEus8MDiTxSVlkrACic6XQ21yFqAzmYPxQOd0VfVoVK8GGX82UzA2z9SMzfaFirbVZPS5ps6xS2tBvmLV5Pm1dT6hzctzgyCT0OYG2lzBQZClF+lt25UCZ/MXw/KarEkrzDaURZ0XuX3aZMgt+nqbf0jv55Pv0tsT9f34aKSnTeu63tJg3zmzG+Oe3tqob3Lh+n69lb5ej5iKcqd8/Z89vq7GnWhu6+GGb3zLNT3usd6iGh6ZQq+OfT22Ip1l2DDDC3Fv53VVTD053U7Hqd8fzozr1vVx1hvq4e1zczuvOw/q/dq+qI+V6qYvhzPZpDaDMNz0UUtPmzShpw1GR4leTmNND2+f1nU66vjt6S/r9QyXdPmTxSAzsWLKOy449k2mYGQyByPToEYj345tHmG1pttpmGHZqE4+Xqfhk4NERETe85xznwzgCwF8i4g8H8DPAHgKgOcCOA/gx/ea0Tn3GufcPc65exqLjb0mISIiIiIiOnR4c5CIiCjnnDuX/38JwO8A+FTn3EXnXOKcSwH8NwCfejPLSEREREREdD3x5iAREREAEWmLyNy11wA+H8D7RORMMNmXA3jfzSgfERERERHRjcDMQSIioswpAL8jWVZQBcCvO+f+UER+RUSeiyyG52MAvummlZCIiIiIiOg6481BIiIiAM65BwB84h5//7qbUBwiIqIbLqkBW3cFwfhp0MlIqj8qzv3VVT3v+ub+VyTmB2u2w5KiWcMOPmzHGqbjDUn8cm3HGpWu7pChcdV0hhB2bGE7FUlMeZOCaa2w45CR6Vjj8poeDjt2MZ2guIbuvMR1dI8MMvDLljW9b9yc7gxkvOyHB8t1Na62rstYfeiyX86W6cXF9vBhBdueVvS0wwVdb71Tftr6qp62fU7X/3DR18VgSXeAMWrq5cbB5lR6ejmS7Op+RQvKLyPTBsysC/O+Q5sz7Q01bmuk6/i9V/2PUs52dOcll7pzangw1sfhYtN3MhJ2bAIA7YrukCTsdOTh7SU1rjvSnYokqa/zZlW3gXGi98fa+Xk13DwXdGZimoTt/GOwELRjU4fNq7qO46EfTk3fQ2lVzzwMOrTZMhPHIz3tQFcFRnNBO13U217vDNRwNWhew4HpYMhufBJMbDp5Scd62jQ155FRML6i6yXsgAQA2nW/3yvR/s+tFn9WTEREREREREREdETx5iAREREREREREdERxZuDRERERERERERER9QtkznoImDUmX1+m2lwEJJMn6Zw/ilRBtO42Yteev0HiAPZk80GOKhxe/YClK13ScpVfG21xLpLt5lyZY/G06cpkpY5s5Rsc6XnL1H30aBkvY+mT1Oo5LnCldlvJc+TTzT9cQUfWjk58/zHWtszz3vhysLM8wJAOir3vWFzbjB9ogJPPn51+kQTNOJyB9H6sDl9ogJb/fr0iSYYbsw+LwDEa+Xe0kmJc2dSL3m93Cx34h52yrXZ4cLsdVfbKHfBrM5+qAMA6lfi6RNNVGZeYFzucEHaKLffo/Hs7a5xueQF8wmk1hzh9o+/sDM8CnLHBh8+paZ1213oPwT70GYKGhLpOnfp5OnttEqq36hJX2eshVmAUtcZfdFYz9u6ZK4Zo4LjOTFvdIrea0dm29IgBzG12YVmeBysx9SDmOWmVX0Mu9hPny7pfdc/oa8x4efb3nG9nHFTr2f+ciNYUF8vZ8rpU6rBh0FTZfZ9r6v4/bPyCXpc72Rr8rw2ztIMx0GR62smx9HUcaWr66m67WeOt/T7m9q6LtPVVZ/jOFzQOYI2B6439PWyOtDLGZtjozvQH6gHY7+/Ngdmv5oPkf1gPWtXzc0U0/Tilm//jYbeOYO+LkPU1WUM30MMF8wxaj9bBTc30poeOW7qtlhfD85H5n7OaE5vQHjfIY3Nfu2ZY8lsu6hTmS5TMo7NsC+TG5rGZgal5hccxaYibMWYmz5hHKYzeYTbGw01PGz4A7HdnP19OJ8cJCIiIiIiIiIiOqJ4c5CIiIiIiIiIiOiI4s1BIiIiIiIiIiKiI+qWyRwkIiIiIiKi60fEoR77rLH+2GdZzT+o8/zcrty94DkTmxNoMwULgswlnpJ/GY6308Y2YC4skx4X93SmYGSGpShzsGaC1NOCkOqKKWOQMyg2Y9CydTxhOQAQrW7p0Ys+8279qTrHblcO38CXv7all1tfNfXg9h/ILXWdgefmgxy+RZ0BmehBVUbX0PUw7pgMuS0/HJkqs3n9o7lgOS09srZmhjf07ZFqsO3Rhs7crG/ojOfost+gtdM6lPWTjz+ihp86d9mvw3RwsDJsq+GPbhxXw1e2/PjNnq7vscnHG6758ZVVc+vHtIlxkKW33TfT2mbb1H8YLgb7w8SA2kz24XKQwzcyWYBmX/aX/fj+CdMOT+hsvbD8samXXW3E5P+F5XCr+lhPYfIWg2U5e6gv6qzGZtuXsRLrOqtVigPb4yCnMjHnU5stWTTtQfDJQSIiIiIiIiIioiPqht4cFJFfFJFLIvK+4G/fLyKPici7839fdCPLQERERERERERERHu70U8OvhbAF+zx9//knHtu/u/3b3AZiIiIiIiIiIiIaA839Oagc+5PAKzcyHUQERERERERERHRbG5WhyTfKiJfD+CdAL7dObe610Qi8goArwCAyvzS41g8IiIiIiKiJ7ZREuPRNd+xQr/nO1V4Sreggw7LdtBR0AEJYDohMZ2ZiMjkaSv646ur6mEJylE0DgCiru7QAGFnIWZeZzoZCTsWseNgyh92MuKWdEcVLjbbOvDTiu2cxHSwMl5oqOHt2/WyQ4sf1p1pDJZ9hw2S6HqpXdrWZRoH5WjoddqOWkZnFtXw1h1++t5xXf7U9PFSDToZSQe6/uO+qadgVyVmOUlLb89oPpg41cuJxrpMu/ZH2CbGtkMeMxgsOzIdRtxW17c7zlbXdl5XRR9nl8fzanhkepO5vOk7JBmNTLs0ZQp7Z3FVXSZJ9MTxtl9POjAdYJjDOW2ZDmMW/HDzMb3vmlf0ehsrfr2jti5D2HkMAIzbft7xojkfDW1vIP7lro5CzLDtDCet+/XIeHJbA0zHORUz0tRpeD6NTQckSU3Xcb2qOzNZrPvzU9hpFAC0KrrXl6WaP77HZmP/Gvt3Mzok+RkATwHwXADnAfz4pAmdc69xzt3jnLun0mpPmoyIiIiIiIiIiIhm8LjfHHTOXXTOJc65FMB/A/Cpj3cZiIiIiIiIiIiI6CbcHBSRM8HglwN436RpiYiIiIiIiIiI6Ma5oZmDIvI6AC8AcFxEHgXwfQBeICLPBeAAfAzAN93IMhAREREREdFu6TBG/2Ef9BUNgoyyqKemlUZdDbtxkINl8/Gk+BkUiYPxJktvV3BakEmo5gPgzLRhZpyrT/moa8scLsdk6dlcxF05g6GRzgdzbZ+71z/VMsvRy21cDOpcx4rB1fT29E/q/bF9ytdNdVvnvKV1Xd5R208bD01epKG2tanXmczrDMLeKT2+e9KvZ7Col5s0TQZeEN9WWzNtwBSxfzKY+LTOjnQ2/jLMpjOZcOOWbk9Jw+znMHvStL2kZqaNgtw6kzm4HOscx9OVNb8ckym4KTo7sh7pdhrmDFZM5t1SR2dLbtR8WxwOdfsZm7zCNKwnk7tnW0hzWZ8b+ts+W0/Gej2pyXHs3uaXNlrQ2yYjm/kYDJjMRAx1vUU9Pxzp+D5EJq7QVCnGc8EE9tRl1oOgzqWm699mTYZ1mg50fdfrel8tNvtquFnxGxGZPbAx1Mfd5V5n53WSzv783w29Oeic+5o9/vwLN3KdREREREREREREtD83o0MSIiIiIiIiIiIiOgR4c5CIiIiIiIiIiOiIuqE/KyYiIiIiIqLDKe4DCx/yOV9hVper6OdIpKmz0KTvs95cQX4fsDsrUOUM2ozBg7ABc8FyXVVnfMlY54NJoodVppwtrxXMKyZj0BoFuXzDeV2m6pYpU1BGV9e5h2nV7A9T5WHOYGo+5W+fqanhNMg6rAx0GYan2ma9PpOyv6TLP/+6v1TDOlFx9/CtpqhVz3/kQT3865OnfT1Om7/Y4SI6r1B+078+PqfH3Tm3qoa7Ld+GbB5ed6zbxErP763uULe9RlW38Z4Z74I8ve6detp+RwcALi/5Mg/Huj1trutzDGJf5mqs2+l4W+fuhXmpuzIHzXDcn3zOWTy2pYZ7A72tSVDmuKJbiM2AHA4n50Me7+h9VxE9/rGN+Z3X2z2d5TnX0jmby02fX1itmrDSA7hlbg66GBjNF4elFjF1fcCVl5gXQDQqccG7DuzBcBBxiXkBwPXKbXta/D6jeN0FOcH7YS+4BxUPpk8ziQ1NPaikPn2aQiXbfKlVlzwrReNyha+UaLOuxD4HgHj2czkAIKlNn6ZImWMmqd/ERnMYCRBHs194Lm+3p090g1Sb5U78ty2tl5r/THNj5nk/trVcat2bg3InT/sZ9fFU6n1OSVtny11w5x8ud9FLK+V+CNM9OXv5e8dKvtkoqX519mtWpV+uwfaPlXuPN+qU22+V7dnXX93iNYuIiCjEnxUTEREREREREREdUbw5SEREREREREREdETdMj8rJiIiIiIiousnGjl0zvssHRWrY3657Ro6v0Ri/7P6XT/Ujqb87DvIGQyXA+yRX5gGS5+WT5iGWYB2OZOnBQBX9R+NXVT8DI3KKzSZEmlb56aN5vxyxUYkjnQZ0kYwrS2/ea5HUr2wMJbIRs2Mmrre6pt+3nFdL7d7XO+PpOHntcud14N45Hs/Uw33bvOFqi31dXkjU2+pX894qMuQDkyEQzBtbDLtTi3reJJzDx736+zpba1f1cNL9+v9Mf/BtZ3XNgPywqfPqeGNZ/r9defTLqpxX3z2vWr4ybXLO68vj/VyPtg7o4bv+xQdvfGUk1d2XtsolmZBJljd5FZtjXSUSpj/Z4+yaqzboqvqKZbu8mX65OOPqHEdk7X1jqt37bz+yLkTapyYNnHbibWd15fWO7pQNsa04Ny1fZtpa3pXItryx916TSdlVuu63sKcwXpNj6ubbMY48svt9fTB89C5Y2rYbnul5tfTaujsqfmGPpZOtXw7WBvOnvTJJweJiIiIiIiIiIiOKN4cJCIiIiIiIiIiOqJ4c5CIiIiIiIiIiOiI4s1BIiIiIiIiIiKiI4odkhARERERER1B0dihvja5EwPFdBxSbsX7f0bFBR1+iOn8Q3UMYtkOPYqmBYCgQxJUppQv8T0e2A5I0o7ueMAFnbPEA1N+00FJOG0yrzuMGHX0R/dx05QxWJbpewJJXffQkPb8xM503jBc0H/onvH1ljZ3dT2jDI7pOg47Ifn4s+fUuLtaKxOX81h/UQ2f21pQw1sDX8ebW7r+r5iOK2orvt1GQ71tlZ5ebzzQ5Zck7LhFt/9xW8/rqn7ecar3zfpYdxIxqvpljZzer1WxHdFoy/XuzuvU9LyxMdZt5mLPd3bSHeleOLb6eto0aAjNmj4nzNV0pyLLc6tq+Lbm2sTyvnvtdjX84CXfEUc60nVa7+j1PHZ50U9rOqWx+7J+1Q9Xerqd9k7anoDMYLCs8ZbeH2lFt4lmy5exVtH7ynbcUguGW3XdqchgpNczHOvhKAqOO3OQXtnSjW+j39h5vdzsYlZ8cpCIiIiIiIiIiOiI4s1BIiIiIiIiIiKiI4o3B4mIiIiIiG4hIrIoIr8lIh8UkftE5DNEZFlE3iwi9+f/L93schIR0a2BmYNERERERES3lp8E8IfOua8SkRqAFoDvAfAW59yrReRVAF4F4JVTlzQhim9Xnl9FZ365JMjXEv3MiYgJsrMZg6lftrPjkuLMtUnL2WWkg/ckLc7LK8xBNFmHLshfdA39kTrMDQSASs9vT1rV64jGk8ufmIw7ZyIfbVZgGD/nbByh+dTfPekn2BVxZ6spWK4sDFGkc/e6Gn7WiYs7r5+3+FE17q7a5YnLOdfU97XfVb1TDd+3enrn9eqFeTWusqY3tnXZb4DNYoyGrnDYxUEWXUdn9iUNNQip+4psV4vraSNtThy3YIMQoVe0NfJZgVd6Onvu6qYe7m8GuYK2qZn2U237Mp+e31Tj7urofMjT9Q013E18BuQ7rtylxj306HG9oqFve51TW3rU0BxLYz9traPrdBTrfZVe8Nsa60hExAO9seN5XRnjBZ+xWKnrAyLMGASAepAzmKR6uaNEH6Th+DjS5bV5hXbeNMitrFVMwzWGYz/vo2sLBVMW45ODREREREREtwgRmQfwfAC/AADOuaFzbg3AiwDcm092L4AX34zyERHRrYc3B4mIiIiIiG4dTwZwGcAvici7ROTnRaQN4JRz7jwA5P+f3GtmEXmFiLxTRN45HG0/fqUmIqJDizcHiYiIiIiIbh0VAJ8M4Gecc58EYBvZT4j3xTn3GufcPc65e2rV9vQZiIjoCY+Zg0RERERERLeORwE86px7ez78W8huDl4UkTPOufMicgbApalLEoGr+FwsF2QFRiZnb1eGX5C7h3hKTqDNIAyXbTIGnck6VOuxyylapc1MtPmENoNw6HPHZGy2x+YtVk0AYLhekyMY9/2ybMZgZV3nmbmKf3YnGppctKb+6J7UTI5aEGOXVvU4m483Cu4JJ3WTs1cxw8Gi0m7x7YN6dXI22siEJvadzvBLg6DEzVQXeJzqeXujYN7EZDxu6+HGit8em8VoRcnkbMm0YutUT1tt+G1vVkZqXGoCIreCHZKY8L/1sc0j1Ov56Mqxndfbm2bHrtbUYG3DLztpmYw+U/5ozg8/ee6KGvfUlj6VtCKd/3cJPvfxeFPnCD6U6sxBGfidsHVJfzkR9cxx1vDHwNDk+8VXdftpXQj2szk8d7Xxpj6+2wt9X/6OfppaRM9biSbnhNr93hv7Ml7a7OgypSZ/1Cw3CtY7HOvjrrtVV8MuPC6nRKsW4ZODREREREREtwjn3AUAj4jIM/I/vRDABwC8EcDL8r+9DMAbbkLxiIjoFnTrPDnoACnupKVQNN7/t0zXW5lyZwsoOX+Zu8ej6dMUqRV31DSV7e3rQPOWbN1l91ttY/aKn/at1jT2m62DivvTpylcf236NJMMZ+9gCQAwbpXb9l09th2A7QHtwPMPSxysAJCW2/akPn2aicrM+wTUqQzw6ac+NvP8b3rwmTPP60q2g1G3xAEM4KL5ZvSg2pXZLxyPriyWWncyLnfyLTV/ycM/aRf02LkPgxIXnsFiqVWjtjn5CZz9SKvTpynSO1XimCm538rOv6tTy8dx3VKuye16uuegapvTp5kkrd28zwXX0T8D8Gt5T8UPAPgGZA9+vF5EvhHAwwBechPLR0REt5Bb5+YgERERERERwTn3bgD37DHqhY9zUYiI6AmANweJiIiIiIiOIBcB4+beT++KyRy0OXyu6fPOpKo/Vrq+ztLbJcj/czbb0IqivV/vudxwWVMyBg0JMgftemwZi549tfU2rvn6jXomt3FLP/4rdf/0vszpJ/kHC7pM/WWTgRf8isP+gmps8ubCYdcxZaroenPBU/FSLX5keLnZVcNzFd8OHuidUOPev3VWDadBrV7szalxg6TgtoUp03BRD/eXfflbl/Q4m9u469dXQQakzXG0zSs0Nk/jb5gcwTB/sZfo/XxpYH+BcVUNba22fPGu6Efn7S/fwpzBtGbzFPXwmaWNndcnajo3sGp+WrUY6/0culCfV8ONBX0uGNaDDE5ThrSvz0W1y2HeqB43/4Beb2Xgl7V9Wtd/atp4e1Efd4stP1yP9bSpOdqrUVB+k0e4PdL7crXr9/vWls6HTLf0vou6uszh0/licjVttKcEv5J10ew/C2DmIBERERERERER0RHFm4NERERERERERERHFG8OEhERERERERERHVG8OUhERERERERERHREsUMSIiIiIiKiI8iJIGns/bxI3Dc9LthOOao+UH9XBH400sOp6eQiXFaiOzuAmGD++AAdkoTLLe47AxiPJ4+LTSctsVnvyM9rOyBJ67qjgaTlP3JH4+JCjRd8pwXdU7pzg+2zulOCwbJeVloPOhkxHT2gqeu41jL7J+BSvR5p+G2t1QrqDMBdnRU13Il9ZxTn+wtq3LltPdwd+Xrb7uttH4/1/oiCThfilulAomY6Hbno69T0E4LU7OakbjqiiX1dRGPT/k2zDdkOVGwnI5Wgt4mNke6o4vym7ozluOmQJF7xy97VAUnTtMVmUBf2IDUdz4Qdcbzj6l1q3J2dVTX8nM5jangr8duwabbnxLzu3CReDDsj0m3toY2Tukxrflvjvi6+M52BDBb8srqn9bjq3FCXIZp8HPbG+vhtVfW8jdgfO7ZDktRsTxy000pVN5hhVTe+tAUt2D8V08bn53WHMBKUI0mLuksqxicHiYiIiIiIiIiIjijeHCQiIiIiIiIiIjqieHOQiIiIiIiIiIjoiGLmIBERERER0REUDVO0H/H5Va4orkoKRqa7Ugc1mytYML3YfL9qkD8XTcnTclPKEU6aTs4ds2uRkQl3C7IPnSlv2tKZZf3lMCNOly9q1NXwaN5va++YXm7vlC5vdEKHsDXrPgutXtXlrcRm3iCjrDfU5bV5bOG8jUpx5uDYhPidH/lcwUe3FtW4S+s6hy8JcgXjim4vYrLdRiM/bcVMa6MAwzZd3dbLGTf0nnYmg9AF7S0e6HqpbutbKb2hn/nKVluNC/MUASBJ/b7tmvrfXNHzHtdFQm3dzztcMLmTbVNvwyh4bbbV5FBe2vL7YzDS25aaI6KX6DKHuYnjVLfb+boJCwxsj3S2ZOOCXm/zst9fwzlzVJrTRNG5a7Suj7Px0GRYBu0rMu1/a6DLqNZpVjpK9HJ7XT/vrizPqjn/2JzQgrxCa7npz+Enm5tq3LsL5zSrPMC0RERERERERERE9ATCm4NERERERERERERHFG8OEhERERERERERHVHMHCQiIiIiIjqCZDxGfGHV/yHIWHOtxh5zeK4fZIklOj9LTD6hi8wzKeH0sc7pkmZTD4e5fHY544IsLps/aHMP7bBaqQkws+upBvVU1eUft00W3bIvczzQ46KR3tbhnJ92sGjKtDBSg3NtneUWZqXZaEabITcY+mGbm7bQ6anh463tnddJWvxsUSXS9fTQ2smd11cKMgYBnTPYagzVuJrJFVzdbO28Hg1NPt5Il7EeNINKT7fTeKinTaq6LtKaL2M00vPW1nX72t70OXxrA71t6zZfLliNS/Q6qxd1np8V7q60Ydq4zbUb+eG0qcvQmtPtp1P3dX734ooa9/EL59RwBL3ei8P5ndcf3dApiee39H4PMwl7PZ3n19xSgwgjLCOT1xkP9LTjBb+t0Whyfh8AzC3oNh7mam5s6/PecNNkDo4O8HxdsN+rbX38xhW9P8Yj3WaScbTnawBIzH5OnB+futmf/+OTg0REREREREREREcUbw4SEREREREREREdUbfMz4olAeqrU7quLxAV97peKC1ZS+4m13KJJ0uRFj/VvI+Vl5w9nj7NJJJOn6ZIpVuu8PZR54MYN6dPU6RMvQGA2J9hHFA0nv1YLVNvQPnjtcxZsdKdPk2Rsm1W0nL7Tdzs+w0lZn0iWh808UcPPGvm+dMPdaZPNEFc8rxdWyu3MwedcgV44DmlZi8lLXMMAKjUCn6iNsWwW+7k5eJyx//o+Gj6RJOk5ept40nl2kzjcrltT+qzz5/Wpk9TvICS89vfDh6AKzEvoH/uNYuy1/u4P/t+657hRYuIiCh0y9wcJCIiIiIiousoTeF6vb3H9c0d3Kq+kS+VILcO5kkMZ+58pyaTMPZPL+zKGGy31LCr+/WKzf4r+kLbrNON9/+0yK5pzXqkFtSFyUEcdfSd8+Gif93v6WltBmFS8zeux229znpb5/CFOWkA0A9yBUcmv2xXLt/Qj6+Z5VZjXce9sd/WRy8vqXFPxqNq+KGtZTV84cqCX6fJamvO6fb15ONXd143Yv2F1fpQt5Gtvs+hHG7U1bh4TW9r+OV7UtdfDNQ2dR0OO7qMwwW/rNqGbhPVbTWI+pWwzk0OpXnwI20E+ZDmoYrG5eIvL5Jm0C5s8zfZhmnwxWFrWT/F8IwTl9Tw84/dv/P67toVNe7CaEENryf6GB0F3xZtDPX+GJpsye6Kn1f6ur7tA13hAy9iDn1bp0kwnOoiIGrpBXe7eoLNyz5nsL6qy1QzX76Gy04a5hwzp9dz/MTmzuvFpj7P9kb6fDpMJn/jVjPHpD1Gwy+3H97Ux+hB8GfFRERERERERERER9S+bg6KyK+IyEIwfJeIvOXGFYuIiGg2vGYREdFhw2sTEREdZvt9cvDPALxdRL5IRP4JgDcD+IkbVioiIqLZ8ZpFRESHDa9NRER0aO0rc9A593Mi8n4AbwVwBcAnOecu3NCSERERzYDXLCIiOmx4bSIiosNsXzcHReTrAHwvgK8H8AkAfl9EvsE5954bWTgiIqKD4jWLiIgOm1vy2pTqIH43KujVXMwP0mIdri8107V30KHHrg5IaqYHcwk6aHCmx4Jkci/xu8pb1HmJJaZTCLuecLzp+GTYNh2UzPn1Jit6MeOWnnbc9MsdndTlb1b1tm/3dZ321n2nCkhM+c2mS92X+cTClhrXqemOQh5b951RjLeLe5e/vN2eOK7a1Ntz29K6Gj7T3Nh5/THTscnmQHcgUbQrpaAX+K2zul3OP6zrNK3o/dE96afvHSvuor1+1dd5xfSm3j+m98co6Piksq3HVbeK2+l4LujMZFnvq3pd13Ga+vUc6+gOSe5ZfFgNL8e+HfzpxtPVuMf6i7oMqa6nS905//rKvC6DaTNR0ClPajr0sJ2OuILfuVa65g9Boxg3dJ0mjzbUcGXTdAITVNtYT4rxgumQZN63mca8rv8T8/pYOtnyHZI0Yt3Wxg29cWGnIgAwTPytuvma7sykO9bH/jjoEGYtNT21HMB+eyv+SgCf5Zy7BOB1IvI7AO4F8NyZ10xERHRj8JpFRESHDa9NRER0aO33Z8UvBgARaTvntp1zfyUin3pDS0ZERDQDXrOIiOiw4bWJiIgOs/32VvwZIvIBAPflw58IBugSEdEhxGsWEREdNrw2ERHRYbbfnxX/BIC/B+CNAOCce4+IPP9GFYqIiKiEnwCvWUREdLj8BA7rtSkMcAuy9KSuc61cvz95vor5WGnys9DUQV7S8Blyrmrn1RlfMg6CyJK0cFo3Hk8ctztHsCCczj5CY+Z19eqerwFg+6yZNvblEFskk7HWP+7n7RzToWrjRBeqt6qzxaQ7ORPPNfSKKnVfT/N1vV+vdHVuYBKs9447r0xcBwD0BqbNpEGGYlePu7jZUcPtynDn9aMri7oM42jysKnTpK336yAIrhvoxaK2qessNZGKvVNh3qUeZ4crvcnjbA5imDNY29Tj0pppp0bzjM+1W2zrLLrEZAGubPg8z8Qck/935clqOJIn7bzeHul9NUp0Pa339PG8ccFnDtYv6WnDzE0AwIkgp2+ky+vMeSQJjrtoZI5109zDvM6KOVVFI73tqYlAHbb8sofH9LHSOaVzBI93tndet6tDNe5UQ+/MSuSXlZoAxW2bG2jGP6Vzeed1PTJZq0YrHk4c96eFc2r7enIQAJxzj5g/TU5/JSIiuol4zSIiosOG1yYiIjqs9vvk4CMi8pkAnIjUAHwb8kfiiYiIDhles4iI6LDhtYmIiA6t/T45+M0AvgXAbQAeRdar1rfcoDIRERGVwWsWEREdNrw2ERHRobXf3oqvAPjaG1wWIiKi0njNIiKiw+bQXpucA0YT8qwaNiewsfd02J1HKCY7zHVaajjM+5NJ699jWiT6l9jO5gqqQpjsNjHPxdRiM3ryczNuaV4Nj074XL7NO+pqXO+0DpgL885sxqCJGcNwwW9P0wQUbq3rOpRewXM+kamXeHI9XdicU8NrazpzsFLz+2dosud06iGQfFAvKw4y/Gpren8MOjrg74HnTCwiUpOXV6n5ihx2TVsz2zo6PgoWpJez8SRdhsZlPW9S98M2pw42sjLyy3aRybgz+XhxELsX9/U6u2eKMwcbNb89W33d9ra29DGabvjtO7eqx13ojNRwZ84fw5Fpe9tdvZ7Rhq6M2mW/gcNFXTH1Mzo7M4r8+D5M7p5pUOFud7GuFxuzN+qE2al6XNIw+7Vpdl7DD88vb6tRZ+Y31HCn6nfeUk1nPtpswF7i639zrOvQ1vHHz5/TRYr8/knMiWIuNqGKgfXEHpX7V3hzUET+C3ZHb+5wzn3bzGsmIiK6jnjNIiKiw4bXJiIiuhVM+1nxOwH8NYAGgE8GcH/+77lggC4RER0uvGYREdFhw2sTEREdeoVPDjrn7gUAEXk5gL/rnBvlwz8L4E03vHRERET7xGsWEREdNrw2ERHRrWC/HZKcBRAGCHTyvxERER02vGYREdFhw2sTEREdWvvqkATAqwG8S0Temg9/DoDvvyElmsBVgMGxgsDZKWwA6kFUtqdPU8Ttt5YnSEvOj3j6JJO4EvMCu0N2D6rMtkej6dMUsUGyBxUG2B543ka5ddsQ1oNyNsD5oKsflzlWS60aUckf6CT12bdd0tm3O5u/1OxwlXL7zYYlP17z3gA3/5o1ijB8rD19wgnq/dn3pQ3iPqhkct78/ua3gd0HnT+Z/cJRlEu/r/mTcsdQrTklVL/AuF3uIEp7Jd8slNj0xtLkYOz96KXl6j0elNv22vrs8w6WyzW68u9VZp83qU+fpkgYqD+LSnf6NEXKvEdNayVPFrO56demvQkQ712ZbqQbqO1kJOzwQ6q6YwfU9LD0Te8BYcciBR2BAIAbB+fWxPboUbAvI9PhiCkTKnq8q/sLmGvqi9nglL6mjzp+3t5xXf60Ya4FQUcoYs539n3nuOPfDNpOODAyw3a3JQV1MdJlHG357Vtd1ycDGer1jFp+3ktD3QYWzGpi8/4lfE9i31/Y9wvh9d/uVnt9Dq+39vq563oYzGqvV/b6Y68n4fXBnu/t+Ts8H9vzqz1fhuc/ey6bdn5aveq/Y3B9PXN1VQ+3Vv32hZ3dAMCwqj94bMHvoLiqj7PxUC+3sqbrKdzvw1N6uamp49HQ7/h0Sx+Tw3ldxjT4PO2qpl7se7agrdk27Cpm3pouY7Xpd+ZSS3cy0qroc9fJxpZfjOmAZHOkG3nYCckw0XX2SYuPqGHbyUgr8o2manoyOj9aVMNXRp2JZTiI/fZW/Esi8gcAPi3/06uccxdmXisREdENwmsWEREdNrw2ERHRYXaQ7xtjAJcBrAJ4uog8/8YUiYiIqDRes4iI6LDhtYmIiA6lfT05KCL/HsDfB/B+ANeewXQA/uQGlYuIiGgmvGYREdFhw2sTEREdZvsNaXkxgGc450qmixAREd1wLwavWUREdLi8GIfx2hRFkHZr73Fjk52XmnDkIKMPVfOxsq83041NVmC4bGeWW5Dh7KaEy0qYm22CqJ1Zj6QmFzHIXrTrqW7o3LHKpl9PUmuqcb3TJuswyKar9PVyuyfND/nmfL31ezqUL+qZjEQTuRZmV9uMNTF5hdF2kINostts5l37mA/Ic3alholGgwS7ebikR3bu3FDDzzx+yU9r8tkeWltSw+PUl79a0+10HOnyuyDzbjzSdTh3elMNb0YdNdx8yLeRyrbJi2ya9Ygfjk3OnmlqGASbM57T46ZlycqKX1hssvUiM9w96xuFzeyLW7reGg3fUO1hNurqtmhz0kdzwQymPQ23zbxBO7bl3VWn9WBFFb3SqK7bUxyMH20WB2DX5vTxfHzB5wje3llT407VdTutBzmDG2N97FtR0CY+buGcXm5VBx6HGYMAcKziy/TA4JQa1zWBneMg/L0ez55/vd+bgw8AqAI4XBczIiKi3Wa+ZonIxwBsAkgAjJ1z94jIMoDfBHA3gI8BeKlzbvV6FZaIiI4Efp4iIqJDa783B7sA3i0ib0FwQXPOfdsNKRUREdHsyl6z/q5z7kow/CoAb3HOvVpEXpUPv/K6lZaIiI4Cfp4iIqJDa783B9+Y/yMiIjrsrvc160UAXpC/vhfA28Cbg0REdDD8PEVERIfWvm4OOufuvdEFISIiuh5KXrMcgDeJiAPwc8651wA45Zw7ny/7vIicvB7lJCKio+PQfp5yDm7g87ckyBF085295vDThjmC3Z5ebL+vJy7IEUQcm2EzPvK5ZGKXk5iQu6J1JjqzzGGkhiUIWot6uvxRxXxsrvhCzjmbI6jrbdT2r9eepjPW+md1GcK8tsoHdRZkpasn7R/X6006wfaZ3Le4p4crwfBoTtdLfFrvy0898/DO6z998CkoMm7pMg1P+P0Tz+ltPdbWG/T0js8cTE22YWRC7s5vz++83kJdjavXJmeu2eXONfSv/Fu36zJeaQeBgOcaalxtw+QvqsxHPSrpmCy9glzBtFKcq+mOB3l5JocvFZMrGAzX67peUpOLGDbj3rauU9nWB2Vq8gudPWbVxHrQ1cIcRD1OanrialDm2Gxrraq3Z6Hpj9mtts7kGyW6gLbtLdZ9mz9e31LjlqrmwAt0U70em0A4V/VlurN+VZch1uuZi/Rxt5L488jIVHBk9nMz9m1iY6zb6UEU3hwUkdc7514qIu9F9oFJcc59wsxrJiIiuo6u0zXrec65c/kNwDeLyAcPsP5XAHgFAMRLS1OmJiKio4Cfp4iI6FYw7cnBf57//yU3uiBEREQllb5mOefO5f9fEpHfAfCpAC6KyJn8qcEzAC5NmPc1AF4DAPU77yj+2peIiI4Kfp4iIqJDr/DmYPAzqoeKphORv3DOfcb1LBgREdFBlL1miUgbQOSc28xffz6AH0SWEfUyAK/O/3/D9S47ERE9MfHzFBER3Qr22yHJNLP/sJmIiOjxNemadQrA74gIkF0ff90594ci8g4ArxeRbwTwMICXPD7FJCKiI+TmfJ6qxMDJ5Z3BpOUztJK6zrmK+zrjK3708s5rZzMHi7IArSnTSphJaPMJaya0LMwVtMudMuzGwfaJDoUTu3eCaeNNnVvXPq+Tx9ae6ss8fJqup/mOzjbsfWBx5/Xi/Tpjbes2Xaa0rn+kIKMgr3DLZPaN9PC47eeN79CZas86fVENP7B5zM93yaaqabWPW1fDZzvbE6dNUr09H9w8tfP6ar+txg1NZtw4GK7Gej+G2XMAcPfcys7r1YHOcewn+nZIJTL5i8HwWmOoxnXXdF1INyijyfNDbDMHg2GTsxe3JmcmAsCxZZ9V16zqjERbT72hPz62t3UjTk0uZRh64AbmOGvoMibmeIgawfFgt9VmG46jieOimt6Xjaavc5sP2anq/TFX8/v9pN7Nu/brQlUfh6Gq6DIcr2yq4c3Ub3wr0mVomTKF807LGOybAMbt1Oc+2mxDu96LA5/B+Vh3AbO6XjcH+fMpIiK6Vex5zXLOPQDgE/f4+1UAL7zRhSIioiONn6eIiOimKegnh4iIiIiIiIiIiJ7I9nVzUES+VUSKul6UgnFERESPG16ziIjosOG1iYiIDrP9Pjl4GsA7ROT1IvIFkgcyBb7uOpeLiIhoVrxmERHRYcNrExERHVr7yhx0zv0bEfleZD03fgOAnxKR1wP4BefcR51z77uRhSQiItovXrOIiOiwOazXprQeo3uXD7OvbvgODsSkIMbnV9Rwuh10ZHGQDkgAIE0nj4v08yuqcxNnOuGwnSqUEXRCIo26GuXapvOJXtA5gtmWSk+Xce5hP757h+50oLKgO+yor/rtqXZ1nVa6ul7mP2I6KAkWbfovQH9Zlym626/348+eU+MeWD2mhlfPBR0czBV3lrHU0p0sNCu+Pa30dC8RV1bn1PBK04/fvmJ7lNDln1/25a+YDklqkR5+WuvSzutnH3tMjfuLraeq4fP9eTU8Cjr46FX1vhs0dV0kYRFH5hksmygajjadcqRD0xmIcdeCPw4XqrrzlXdduk0Nb1zq7LyON/RyTZ87SINOR6Sl67Da1B2fJIk9Ricfh1GkNz6thJ0GmQ5JzEkntiehgJhxSzXf9k7UdCci9Ujvq1asOzcJOyFJnd62VqSnTcKdZ/pDWo718VwVv962WY7tgMQOj5zfX51Y7+dBqqftJX44dbOfE/fdIYlzzonIBQAXAIwBLAH4LRF5s3Puu2Yuwb4LAKDgGrKv+WcUD8vlA5uOaQ5sXLLvsrR2836lMG5Nn6aIafcH4komatqL6kFJmfZ6k1X65dp8mW0vs88BoLpVruz2Dd1BJNVyx5or2UVUUrLNlmnzBdfum+KmX7MAlLg2Qw74GUvNW/y+farhUomVA+jcuVFq/mcevzR9ogmGSbmD6KG1ol/8TTdOZ7/wVGvldtw4KncQ2h4DD7TuUfEHmWnmTm9On6jAZtSZPlGB5kOzX3gq2+XO+0mz5H4rcfKNS+xzoPz1elDucMN4bvo0k5R9jzjzeg/BtYmIiGgv+3oXLSLfBuBlAK4A+HkA3+mcG4lIBOB+ALyYERHRocBrFhERHTa8NhER0WG236/YjwP4CufcQ+EfnXOpiHzJ9S8WERHRzHjNIiKiw4bXJiIiOrT2mzn4bwvG3Xf9ikNERFQOr1lERHTYHNZrUzRM0Tznc7Jc0E9KtNlV07pNk5U00jlkRZzNCtzVH0ugKI/QLGfXD+vD5Zrswl3rrOqPwjLvf6s+uEvn7o06etrWwz5CY3BaRxts3q6jFqpBNVbX9biVms64O3HJb1Fa0eWtdvXWJjoWEcN5P33vrI4ImbtDR350Gj7/7L3nzqpx6UNtNSwnhzuvF5Z0ppp1aV3XRRqUaTjW256OTdZeGHVgojJkS8/ba/osnMV53U57Y5158Dfrd+y8Pl7VMRbP63xYDV9u6P3xV/GTdl5vj3T+zgZ0DqXS0PUfVXWbDnP4KlU97fJccR2HOYMf3Tiuxq09sqiG61d8vSV1034WTYxMzZfRZgxWKnra4ZauCwlyBCs1s+1mX4ZRJKk5gMXu9yA6w54xnMnvGR8gL6IVDdXw02oXfJlMn71Vk9dzeey3L4bOUFuMdVtsmPWE7HqKsg6rTtdpQ/T+aVf8tN3K7JkfNylxg4iIiIiIiIiIiG423hwkIiIiIiIiIiI6om7ozUER+UURuSQi7wv+tiwibxaR+/P/S/ZVRkRERERERERERLPYb4cks3otgJ8C8MvB314F4C3OuVeLyKvy4Vfe4HIQERERERFRKEkRrQeZg92ef72ls8/cWGdvFbH5fhLHdoL9lzHMGbR5hNEBnnUxZZCOztYb3uafWVl9hg70a12evF4Z65y02pYeHnb8tla29XYndf1xfP1p/nVaM5lkNb3caGAy1074fLM7bruqy5Dobb94dcGXYUNnlEVn+2q4EvttX394QY07CW2wonP4Htv09RjbHL6K3p5OM8hYMxl3W5sNNTwe+HrrDXX5l5s69+3Cts8R/Lm1z1LjnrZ8RQ2fbuhsxqr4bT/d1uPqFX08bA78tg5Ger+OTN5ikvh9W6vq5TQqxcfZ28/dtfO63zPZfyPTJjq+/O7kQI1rtXQeXpgN2DBlGie6LY47et5OS7eZkM0GRHBohZmCAFCN9XHWrE7ONbUZg73Et4NuquulGun2FEOvZz7y5a+KnjZsA9m8vswbqW7vsZl2LljuyOk20HfF2YDtIK9wO7Xj9L6cr/j1jGvmXHsAN/TJQefcnwBYMX9+EYB789f3AnjxjSwDERERERERERER7e1mZA6ecs6dB4D8f/uFww4ReYWIvFNE3plsF/faQ0RERERERERERAdzqDskcc69xjl3j3Punrjdnj4DERERERERERER7duNzhzcy0UROeOcOy8iZwBcugllICIiIiIiOtqSMdzV1Z1BN/Q5Vy6xOXsmRzDMDbR5flXzMbNS8LHT6dwxpGY48RlgLtF5YLvmLcontGzuYeyHO+f0ehqXdaaa9Hw91c/rXLS0qnP5ts/4/LP+0/VyPutpH1HD77l4287rjavm4ZixLq8TXefzy/6Xdmc762rc+y+dVsNJ389bXdT5ZccWt9TwxQ+f2Hldv1r8bFFlQ5fJBdmMacOMMxmEg7ZvI8ttnRtYiU0GYVdnEIYasc7Lm+/4On94Q/eF+vb3P0XPbJqT1H0biqrF7SkN9o8bmXpK9b6L276MI5Oz99iKbj93QQvzCpOertNK32QOnvRt8/RxnZk4X9P7vV31w6nJ89se6ww/dPSvOsOsvVGqyzQyeZdhVmNkKrxZ0cdSLdiXG0O9z+16+mOf4bcybKlxC5WeGl6u6DYe5gy2RLefmskRHAU5gsdivZyGjCYO16Db8JrTx7fNOmxEOtcxZPMKj1d9OWy24UHcjCcH3wjgZfnrlwF4w00oAxERERERERER0ZF3Q28OisjrAPwFgGeIyKMi8o0AXg3g80TkfgCflw8TERERERERERHR4+yG/qzYOfc1E0a98Eaul4iIiIiIiIiIiKY71B2SEBERERERERER0Y1zMzokISIiIiIiopstdaoTkpDE5jmSyAzHswffQ4JlxTJ5OkB1xCF2naaDElfUIYmddlt3UlB7eGXndXVXhyqmM426H++qelzvhJ43CfpRcF097lJvDhMNTX2bzjIg+g/b235F7xrcrmd9wHZ+4F/HT9EdU2z2TGcfsV9P/6zurMFqPE13hFINOhLp1HU7CzumAIB2xY+fr+l9cz7SnXSMg04uxomup36i67gRdHJx98KKGtes6g4kHruyqNezEtSF6exj1/5oBn9o6W1rLOg6Xp7zHXq0TBnGafHzW4Nt3zlItGHa2pxu8ydO+f3xpHm97bc119Rw2CFGN9UdkIxtJyO7Oiyp77zeGtXVuHFl8vZEpg03TIckYYclNdMpTSVKzbAfX4/0tHUpbrehE7Eu06Y5jWw7XzenK2uFywo7dmlFetvakW4T26mut2rQgUliym87HVmIfQc+3YrpPOYA+OQgERERERERERHREXXLPDkYD4DF+0sswHZzfwBJffo0RdKStRxP7sV6XyQtse21Kd/k3WAunr3skpQru+lN/HHlbvZt+9mrPZu9RNUn9XL7LR5Mn6ZImeNFStZbUvJcMW6WXH999g0oe556opEEqG7M3pbHrdn3xfBEuZNXPDeaPlGBY+3u9IkKPL1zaeZ50zInHwCRpNMnKnB+e37mebdQ7s1Gvbb/b8Wvt7L1Ptcod+Ju3V6uzV5pFzy5M825xvRpCtRKnCcAACWarCt5zUk65S56N/O9TlopecEmIiJ6grnZtyCIiIiIiIiIiIjoJrllnhwkIiIiIiKi60gEqFb9cJjLJ+bJVpM5KLVgvsqUj5Um7w8ueOzVZKzZrEMXLtsuJ9JlVEO2/FK8HvT6/nXcUqOGp3Tm3ajtyxQP9SO89tcUYVxYPK9H2ty0rS3/NLKkuvyV5b4aHvWqajhdCbLotvS2JQ39tKyc9E9st2v66etRovPM6qf8LwLCrLy9nJzb0mUKniyvmRw4MT+5GQePE3fHOjetHuun4493fDkG4+K2V5Tht1TXv3Y4drvevvFZXxc1Uwa7PdVgOJryM6wws29jqH/6c2FbP81uE+RcmMF5Qj95v7igt2eh4dvMck2Pq5qfySXBc2P1SG+rHbZ1Ggf70v4KY2h+Rlm0P/pj3abDNmHbZdPkEy5U/bbOV3RmZdWUPzb752xw0B6PO2pc1+k2fW60tPP6E+uP6eWKzSv021M1j/q3pmQOpsH+OBbrMqwl+vw0F/tt77viY7QInxwkIiIiIiIiIiI6onhzkIiIiIiIiIiI6IjizUEiIiIiIqJbjIjEIvIuEflf+fCyiLxZRO7P/1+atgwiIiKAmYNERERERES3on8O4D4A17pKfxWAtzjnXi0ir8qHX1m4BAEkzOYL8/1infElDZ2J5ebawYDJWDPDkui8LdcNMsFMjqAzsYJhrqCYbENn1xuu0+YgVvT2uLpOc5OxX3G62Fbjts/oaUctX6bFj+o8s0pXb+vgmH/9zLMX1bgPfOysGm5+yNexiSBDv67z2GAyCaNjPsOsc5fOXGuaXMGrG377TnZ0nlm7qrPQzm35vMWtvi6UTmcD1nq6B/daxdepi3V5RyZ7bhhkB9Yquk47VZ3VOBeU0eYRFglz6QCgXdHbanP4oiBDrmgcAGwlvm5WhjoTzuYKbgeZiltD3bZWN/W8Ou0SiJu+HM2WLv9SS+/3Y43J+XMbY12mQZANWDV5irVdmX3pxPFNs59T0zV9FPl66yf6GO2ZzMFBML4a6zLN1/S2LlZ9puLJ2qYad0d1RQ3fXb2ihueiybfFFiJ93jhbXQ3mM8e6OR0lMLmngWORLn830sdWOK/NSHxK7ZIa/ujw5M7r05X1ieuchk8OEhERERER3UJE5HYAXwzg54M/vwjAvfnrewG8+HEuFhER3aJ4c5CIiIiIiOjW8hMAvgtQj/Cccs6dB4D8/5N7zAcReYWIvFNE3jlM+3tNQkRERwxvDhIREREREd0iRORLAFxyzv31LPM7517jnLvHOXdPLWpMn4GIiJ7wmDlIRERERER063gegC8TkS8C0AAwLyK/CuCiiJxxzp0XkTMALhUuZS9V//FQGvrGoWvW7dSeTM7WAgBn8v6k6ZfttkwumtM5Xgiy6Ux8GSQyfwhzEm3GYE3nmaULOtvNBZswntM5cOO63r4kGN68Q9fLYFFPK2OfF/aBh8+ocfN/recNY+zWT5t6SEwd1/T4U8sbflKT51c12WiLHZ93dqWr62FF9HAczBub5VjHv/TDheNDBa1pqt70SXZsTnh9cNOeqwpz+TbMOD0cLmleT7hr2Gq3/dO+nYbOHGxWdLbkcs3n8PUSk1lpVIKcwcTptjZO9bGUmuNd5RWKaWvV3sRpx+aAtrmCYXurmRxEu9ylIHPwVFXn7p2o6Pq3ZTwX5J62Up3BqbccuC324xumHlKTDbgY+axMW6c2j3Ax6qrh4a41ezb/8rYgB3HbhpUeAJ8cJCIiIiIiukU4577bOXe7c+5uAF8N4P845/4hgDcCeFk+2csAvOEmFZGIiG4xvDlIRERERER063s1gM8TkfsBfF4+TERENBV/VkxERERERHQLcs69DcDb8tdXAbzwZpaHiIhuTXxykIiIiIiIiIiI6Ijik4NERERERERHkgBREIwf+RB8Nx7rSdd1ZwdhxyFSNyH4pvMPVzUfO4OORKRqpu3rThZULr/O4YdLdQcA4oLh2DwHYzooSeqmw5Jq0PFJrDsLqG3pDgz6x/y8m3eb4g6dGfav00iPGy7oecctP756WndQMB6ZTiG6uk7PX1jaed2c66txy8tmWenkZ4SaVb2fo6CXlGZV18vwzXep4XZ1qIY7Vb8va5FuTydquuOHk7XJ3YXUI12ms0EHDKcruvOJmm0kBbad7nhmM22q4aHzdd4Q0/6NbkFHELaTiA/2fMc0922cVuN6Y3089Md6P9fGfvtS08nFINHTXup3dl7PVfVxddzU/yjYVtt5ycZYl9926LFc850KdWK9nnC5dtlV08nIhhkOOywJO1cBgGas90cSTJuYjk5s/dvhC+PFnddPqV1U407EusOkOfHteMX0z2O7EJkLjp01s69Gpoy2A5LFSB/DRfO2xR937XhoJ983PjlIRERERERERER0RPHmIBERERERERER0RF1y/ysWFIgHrjpE94Ao7ZMn6hAWis3v3lq93GVlmwhruTtZ0lmr7uy9Xaz2hsAJI2btups/ZOfit8XF82+38q2uVGr3Pwumr3RSlquzaTVkueaesk2W2L1ZY7VJyJXdRieGk+fcILOie3pE01wtjP7vNdDUvBTpf344Oapmee92m+XWvcwsT8IOZhxifmr8f5/BrWXhebkn5/sx91zKzPPuzood+LtJ+VO/JWo3AU/LjH/WmP2n+8AQHetOX2iAtIt0WbTkuftuNw1x0Ulr1m12fdb3Jr9/ExERPREdMvcHCQiIiIiIqLrKIogbf/FihsGOV72S8+RvhnukuAGrcknFJPdBpM56IL8P6mYj6QmS0wtO7Y3xPVNYjcOhp0pf0N/+5009LJG80E5zKyDef3FV/+En8A+kNB6TN94X3+2//Ln9hNratwjWzrbLbzp3qnpOq2Z4e2xLpMb+uFKrAt1YXNODTdrvo5vn9NlstaHfl/ajLt6rMvUqug2Euaf2WnDLEMAqIqvp4VY58stmuG5qLfzOjZtoGXyCdtBRlzdfCeSoqeGE6fzC0fBN+eJzYwzP8I8N/YBkn+x/VQ1btU8vfAXF57kl2u+qGnVdPnPtDfUcCOoR/vlWs1k9t3WXPPLqZltM1mAj/SXd15vm4xBm+93wuRD3lb3GZB2uYNUt/FwP0eJbgMVSSZOO+2LwG7q8yMfHJxQ486PFgvnvaf14M7rY5FuazbfL4wytc+QVU2b6Ae5rHOm/JvmwYowNxAAFqPJX2KtmadpkqCdzk3JxizCnxUTEREREREREREdUbw5SEREREREREREdETx5iAREREREREREdERxcxBIiIiIiKio8g5uDDTb1TQyY7o50qk4oedyRxEf6CnNXl/rumHXV1nkonJCnSjghxE0wmeSDBc1ctNG3q4e0oPbzzZb89gWeeDpfMmx2vkpz31Z7peOo/pbd98ss9Cu7qls+eqi7ozqdGmr5etCx29TtuHUEWXsbbg19tp6DLcYXIFTzd8jt1CRefuXR3q9YY5g5sj3Wtip6rXEy4X0JlxNnuuFeu2dqrqM/GOxVtqXMPkqM1Fvt5OmOW0RFdUHDwPFWbAAYDtf3Jo8uUGQX7ehtNt+D29u9Twuzfv2Hn9gVXdudrqpt7vo6G/DeNM9tzC2at6uKrbSLvi67xqMvpuq6+p4dtrflk1M+1fbT1ZDadBAwuzCgFguaI7uwv3lTV0+hZTbAI8w0xCW37bOVuYI2g7vOsluj2NU7/csI4AYKGhcwS/qPM+NXxXxW/7yBVnG3Yin8FZFZ2vuJ7qY2kt8WW2mYNWS0x+p0zuNGzNDKdhuy3R1xifHCQiIiIiIiIiIjqieHOQiIiIiIiIiIjoiOLNQSIiIiIiIiIioiOKmYNERERERERHkUuBns80cybvT0l1JpZU/UdJqZiPlYnODnNdncWFYF7UdHaYLYHEwfMsicntqujMLxcuN4rMOD3tuKnDufqnfZnnbtPZed2uzpurfcRnyC3ep/PX0rquizBqrxLr8teruk5XV3ymX+2qLq+J7MP4pF5WpeLLH4uuxYroaS/05/1rzKtxYcYgAETBspbqOrttqaaHx06XeW3k89nmbQ5crNtEO/Lj5yI9zmbTJUGw2uWkpsbNRTqfsAq/7ZsmD28zrZnhphreTv1+f2h4XI27b/uMGj7f8/V4W0e3iWcvXVTDg9SXw+bjzVd0xmDPbN/J2ubOa1uHJyq63Z6M/bQfHp5W42zW5N2NK74Mpv6todnPo6BebbbkeqLrdDPxbTxsH3tpxn5f2oxBm3/5tM6lnddPqev6/juNh9Xws2o6A/J6GZi8wjg4lGziYGryLeNIH7PVIOO1m+r2v/sYLc4z3C8+OUhERERERERERHRE8eYgERERERERERHREcWbg0REREREREREREcUbw4SEREREREREREdUeyQhIiIiIiI6ChyphOSdP/B9m7kO9MIOycBoDscAYCx6cykG3SC0tSdfdh5VScjtkOSAjIcmWG9XJvhH/V9yP/WxxbUuM5D+pmauUd8BwG9M201buNuvZ76Wd8pxHJbd+DRrOgyrnX8stIN3elD0tEFrrWHarjd8MP1iq7vsen8YJj4ZXfHusMLKwnmPdbYVuPCDiOA3R2HREH3MpVIj+uazkA+FnT48YA7WTjt6sh3KLExLu7UohqstxbpeolNNxGJeXYq7ARje6zbqa3TO9qrO69tvZysbqrhOGh871i7S427b1V3HLLY0J2DjIL1Rg3dicWc6aDk3f07d143RJfpWY3H9Hpi3zZtvVxNOnp4dEwNbwWdjKyb/WH3z5Whb+NzpjOWs401NXx5OLfzepzq4+GehYf0cOuBndfPrup2ejzWx+iNspmaTkWC19upbi/2WGmYToQa4s8jV5yetm861mmIb9e2M6KD4JODRERERERERERERxRvDhIRERERERERER1RvDlIRERERERERER0RN1SmYNRMvvvp22mxEFUt2X6RAWSEuUGgLTkXkqrs5e/7LpduaqDjKdPM0mlO32aItXtcvtt1J5945P69GlupKRRbselxdElhVzJryxcVK7srkSbl/HNqzegfN1Fo+nTTFLmHPtEVK+P8Iynnpt5/u5o9sZg84sOaqXXmj5RgSurc9MnKlp/c/b1b18pV3ZUyp3355e3p080adVxMn2iArWo3PxPa12aed5nH3ts+kQF/mLrqaXmP9+fLzX/KImnTzRBr1qdPlGBQbPEGx0Apd5ijspecMvNXvoRhbTE+9vh7Pv8Cc8V7FiZXOdh/iAASMW8oTLLdQOfj2eX6hrmGhgH+8uWIdHnPgnLYTIH0dbZZ/UN/eZl+b2+UVa39TgxmV/dE37a3ildppHJBmxW/Lztqs4J3HX+2fTnlLSu66x9m86tm2/21XB/FGSUbemMtbHJO1uo+XmPmxzBasH1pB3r8u/KGDR5Z/XgzaXN87s01O8XHhz7zMGLfT3Olr8R+/0cmTehtYLraTrlw6ktf5hRaHME72peVcNfNv/undfPqeo2/Htdndn3+sufuvM63BYAuHNuVQ0PTdZeNdje2Gz7dqo/RJ6urAfz6XpZS/R7pm4w70aqj5XHhkt62kRvX1ivg1RfF0fmA8pdrZWd18v/f3t3HitLet73/fdU9Xb2u87cOwtnhtSEEi1BQ3m0ZQJB1mKtECXAMuhAiuQYoAKIkZzIkEkDSZQgQRTBtiQkipCRrJixKEuCFogWCC2mJQt2bIpDauNOhuRw9rnL3HvP2qe76s0f9/DU+z7ndnWfrntOn3P7+wEG03Wrquutt97qpU7172mlY++5nXPJ9EJWjbevW/1EMu+re88l04+14z4+nozBm2Wa8bjr9nUpq47PTfcF/1yenr9rWXqcb5bVvm+6L6h+zHejY1s0uADDnYMAAAAAAADAnOLiIAAAAAAAADCnuDgIAAAAAAAAzKlTlTkIAAAAALiLyhHBwf7fM3dfSZz/5+f5vL+FXjo9rHLWwk6avaVBuq7Fz+2349sYZRsGl3OYraf5ZisfT1ctVqtMsM2H0sy1m4+k2x2sVs9d9FyeoosRHAyqf9h0mcLPvZpmrOWb1XaK+/vJvKVumve3vZtmu233q+decMv6XNrz3aovllrpdoZuB7IoYHTBZQ4uuulll6MWuzlMM+5uDNLpfhR2H2f9SdKZzug85aHLeVtrpzlwZ6LpngvY3nH5eH0XuH82CrH/lpW/SuZ9Tc9nl1Zj/IP9tF/+1fU3J9NvWLqy/9hnAV5opdmSi1l6fNZdHmDyvJ00P7gdBfh/aPvRZN7HNy4n0/E48G3acHl5XXd84qxGP54Ws7Qv4pzEm8N0X964+HIy/dWLn95//Nc7aX/nluY4zsJVl3m65F6ebkTjadGNvTNjbtO7VlSvr+sumP6MGxOdqE/Xy+mzkLlzEAAAAAAAAJhTXBwEAAAAAAAA5hQXBwEAAAAAAIA5xcVBAAAAAAAAYE5RkAQAAAAA5lVc5GNUcZI7zcvzkfOCKxxi/TRAX92owMEwLW6g7bSoRVruw8ksnW5VX2+tmxZRCMtpAYxiJZ2/8bqqOMKNL0rbv3sm3b/OzWp+70oyS2UnbdNGtrT/+NmXlpJ57VvpsiH6dh76vuBF6r7ljWT6el7t39leWpTj0ZVryfRyXh2Pwt0v1FJaZCEuPrHWSp/XF5sYuGImL/TP7D9+tb+SzDvX2UqmL3Sr/Rm2XVEUS/u/jIqQ+CIpD3ZfS6ZXsmo8XR2mbXjNFUW5r5MWA/nOlb/Yf/yEG0/9kBaY+Icvf+3+43//0uuTeU9cfCGZHpTV/i230/Hui4GcydN+utS6uf84t5rzVdKV4er+49cGSzVLpsVYsjw96863N/3iI511y/r9icfMFy+/mMz7sna6r2fz9PicBFeLav8KudcfVwRpEI3Ti754j6VFRp4v0vF0s6wK3Ky486ztjvtm9MLh5x0Gdw4CAAAAAAAAc4qLgwAAAAAAAMCc4uIgAAAAAAAAMKfIHAQAAAAApFxu4IHMwSLKEsvr8/FCka6bZBC23FdS/1xFmlmWtsklEsbbce0vF9OMr5370gy5m49Vy2+/Ls3/UpFmi+WvVMu2N9M2DN2yiy9Vy5Z+13pu3eWq/Yvn0/y1p+7/TDK9Xab78+BilUV3rpPmvsW5gZK0XbT3H+dKj81i7jIgIztlO5m+OVxIpl9xuYKfu3V+//FCK+3ThxfSbMA4B9FbydNcvjjr7Vye7usbOq8m03Fe4cPtNHvxu1ZuJtMX89EJl3+yk46Xd1/9umT6I9cv7z9+47m0DQ/10n19bVhl6XWLtL8fbl9Ppn3mYCfKhLxWpjmCuUvofKBdbXdxOe3f57rnk+mrg+rY+SzDZdf/fTcO1qI2+mO1kqU5lV/ZrfrmcmtZqZOXMeizJV8sqpP4nDuvnnfnw8W82vcLWXq+vlSkOYLXi3Tdng2jx+lr4I7L9oyPe1af0lqLOwcBAAAAAACAOcXFQQAAAAAAAGBOcXEQAAAAAAAAmFNkDgIAAADAPDLJrMpvS9KqzA4sPlJwOVc+n9Dl/4VBlNXl8wjzBvevRPmEYTvNOstupplxO38tzTvberTKFjtz/3oy78ZLq8l0HI83WE77aTeN3VM7isTbvZT20+Bimmd2/+Ub+4+/4fInk3mv66Z5eVeH6YYWsyrDbL3oJfOuDdJsunaUYbaQp23oZel0/Fy3BmkfDkJ6rDbc/Dyrju3ZXpqd5/XL6tLEhfZGMu9cK53uWdXGi61bybzXt9Pp1yW5dum+DULa3j/aTvttvaxy4D6280Ay78pOOn7aedWnl3tpluGtYfq8cf/f306XPZen+xpnDErSZqiy63zG4GKW5gouWZRr5/IuS3efWJxtGPevJA1cxt1mmfbbUrTdR9tXk3lf2kmfaznzOYMn2/PDtE/PRN32SpHmCJ5zeYv359WYvlqmGYPXXB/GGYOS1I3GiD/PvDhn8C/6D7q5L9auGzs1FwetCGqv14TRjlG2p3+TaWfl+IVqZMNDvLHeQdlutv5gafpQyqLXbNtjxvFY2fSHXK2t6fdbkkLebN93z0y//rDBMZOkbHSe70TKTrN9L9rjlxklNHxVKrrjl6ljDU53q8/iHqts0G+S1CB/VpKUDZodd1R2hy09e+3c1Ov771iHUa42fO0aNhvIZdP3vLLB+lmzk8A2mu379kJn/EIjnFmt/9I0dtvDZi8gH7r58NTrXmivj1+oxlPLnxy/UI0rvdXxC9X40/yxqdfdHEx/zCXplhbGL3RUeg0+ZEnK2g0/Hzc8X1vt6dt/bmVz/EI1nm20NgAAJw8/KwYAAAAAAADmFBcHAQAAAAAAgDl1an5WDAAAAAA4QnE24JjcwGS+Xzb3AWc1P0MfpJlkoUzXNf9c8bI1ORw+pCK46IeNh9MlHnnsyv7jS0tpbt37r6U5adlutd0sjQrT7lr6vHH83+BM+nP4Bx+8nkx/xYXn9h+vtdLMRJ8xuOPyaF4bLO4/vhE9lg7mCp7tVHEQcVahdDBfLs4CXMjTZZcs7f/SZUr18qpzznXS+IzlPM1BirMO1/J02ZUszXI7k1fRAA+7zMEH83TfY1eLNFLgvZuPJNMvDc4m03Ee4NVBOgZe66fbubhQZQX6fXttmC57NspQvOQyB5dcPlThRnKc9+ezAeuyAv1xLdyxWsmq8dYxl3NYkzEopXmFPndvOUvzLk+654cbtfPXo9ennuuni1l6rG6U1fi/4vIJfV5k1z1XGR13PwbaSl9PP7p7af/xf9x4g2vxBzQp7hwEAAAAAAAA5hQXBwEAAAAAAIA5xcVBAAD2mFluZn9mZr+7N/0TZvaCmf353n/fPus2AgAAAMDdROYgAACVH5X0MUmr0b/9dAjhH8+oPQAAAABwpLg4CACAJDN7SNJ3SPpfJP23M24OAADHr65wiPkSHzWKNFz/QDGTWF5fgMSWqmIOwbXPdtMCDMl23fMUS2lBgO0H00oi33Tp49WyrljDn7YeTabLTtUXu2tpE3YupfseutX0Fz32SjLv8dUryXQWFfh4budc2t4iLUByrZ8WetgeVvPXumkxk/u7adGOuAhJ2xVC2Co7I5e90N6oXfZqSIt2xIVQHu6lxVe+qJv2RVzk4lbZS+ZddEVHLkUFSR5wx3k7pEVT1ovqOH9isJrMe373fDK91koLocRFX672030LIT0fHl54rWr/MG2/7+PL7Rv7j+NCINLBQhW+L+JiIXERlzutO4gKWeyEdPx4udWc+07mlo0LofRsdJGgk+rVqFDNVqh/nYuLg1zK0+PqXo30iitCElu09PWndEVH4u34AiSfGlxIpuMiJJ+8dd/IbY7Dz4oBALjtZyT9uCT/6ejtZvaXZvZLZnb24Gq3mdnbzOwZM3umuLU5ajEAAAAAOFG4OAgAmHtm9p2SXg0hfNDN+nlJb5D0hKSXJP2TUc8RQng6hPBkCOHJfHVp1GIAAAAAcKLws2IAAKSnJH3XXsGRnqRVM/vlEML3fWEBM/sFSb87qwYCAAAAwFHg4iAAYO6FEN4p6Z2SZGZfL+kfhBC+z8wuhxBe2lvseyR9eDYtBADgKJiURVlXxeglG6nJMrROmstlqyvJdHH/mf3H2VaaJ2c31pPpsFvNt7qcQ0kunk1Z9A+f3EozvbSe5rXtXKiW3X2kn8w7czaNFnlgtcrLe+NKmrPXL9Pn3Rx29x/7jMEr22nm3fXtxWS606oyzM730jZkLgcuzgr0+YobRTeZXsyrPi1cLtrL/TRwcdNlrD2yWOUMXm6/pjovDKrklgfdsl/eSXP51rLRv9DYKHeS6a1o12+Ui6pzJk8zBz+5c2n/8dWddJvLnfS4L7eq6ZvDhWTe+VY6TuMMxTivT5J2lWYoDtx0vPySpeeDPz6DUF3u8cfZLxvnFfp5dctKUjvKz1s8TDbpjFwt0vPjWlG1uePyFG+4XM0HovMhc/3yYpEeq9hhMgalNGfw2WGaavQfN74omf7UepUzeLOfZlQeBhcHAQAY7afM7And/grxOUk/NNPWAAAAAMBdxsVBAAAiIYQ/lvTHe4+/f6aNAQAAAIAjRkESAAAAAAAAYE5x5yAAAAAAzCOTlEc5WYPhyEUPiDP9fKbgmLw/a0VfQxfSjKzi4plkevtylfXWvZZ+fW357cTtd/uS7aaBir2X0+f6t1ce33/88nqaexha6f7tPlLlvp0/v5HMW2inGXL396q8ueuDNLdufZDm+5Whyh3bcZmDr+2kOXZFmWaULUbbXcjTNtwYjM7aG7gsutzlEw5CNT6e306zz24O0mP3wMLNZPpy50a1bJHu++d301zHC1Eu3xvb15J5a1matxgrQnpsCpeHtx7lOr46XE3m5S5fbsdlQL7cr5bfGqTZc0+cfz6Z3oryFtuWjrUHOmmGYpxt6PP7bpRpn7ZdEKjPGYyV7ljuhGp/SndfmN9uFmXc7Ya0H/zz5m7/4udq28m7/8xnDF4p0nOnF42D6y5j8GKe9vdiVp0Pzw7T59kJaebgisuTjNVlDErSlSgf8z9sphmDn968mC67PTqD8zBO3pEDAAAAAAAAcCxOz52DQcqGYfxyIxQL45cZZdhtVnGnbDVcvzN+mTohn377dog/Ht7J6Ho9R7/90PDS9+5qs+PWP9tgvPZGV3SbSMOdHzY4X5oqW9P3myRZs9Vl5fTHvem5WnYaNr4hazDssoavFfeasJtp+NnRf+UeZ7g6fbnIF9a74xeqkfealarMGp7Dywv98QuN0G41a/vG+vQV3iRp2J/+Y9X2bnv8QjXOLWyNX6jGy5ur4xca4f+68Z812vbj5642Wv9S79b4hWq0G7z4XVpqtu1uq9mL53p/+vO9P2j2NWAwbPYpryiafVbptKfvu17DfgcA4F7DnYMAAAAAAADAnDo9dw4CAAAAAO4ik7IRd4GGBneC25hfYmTVfOuld8D270/z8QbL1f0s5n6m0b+QrptvV3eQd19Nc8a83pV0+pPPXqrmraR3sV963fVkerlT5ZBtD3w+W7rvn1k/P3LZVlaOnO4X6XEZurttl7tpFtpSK2qTyyvsl+nX/mH0KyPf3pVWuu8bw6rPb+ymxybOU5Sk8+20zzeK6o78OJNPkpbzdDtv6r1QPW8++c9x+mHoptM+vVFWeWxXB2mW5P3tNCPxustFfGW7uqve3+m92tpJpjeKaize10775WIrvcu8E+UIboZ0X30WYC9Lc+t60U/rdkJ6XHdcVuBulIFXHMgNnP6ueb9unLG4aA1/SnUX+IzB625XO679cc6gzxhcc6+Pz0fDYMtlVK5lo3/90nd5hF2X27jljuWfbHzx/uPPbp1P5r2wsTZyO01w5yAAAAAAAAAwp7g4CAAAAAAAAMwpLg4CAAAAAAAAc2pmmYNm9jlJ65IKScMQwpOzagsAAAAAAAAwj2ZdkORvhBCuzrgNAAAAADCfshHFQ8rpCxZ41k6/dtpSVfihOLeazNs5O6JAiqSNB9Ln6Z9J2770cvXDuPaGK4xQpoUeWjuu4Eq/WnfhQlqUYLWTFhooVW3Xl23ZGaRtvLa+pFE67bTIRWbVsw3L9Ed+qwtpAYzVbjod2y1zN522KS5CsthK93XgClcsR/OX3bILWTrtC5/ECvejxbXWVjJ9Ptuu2pQtjHwer5QvQJLOf3lYFW84WEgj7f8X+meT6es7VQGWN555NZk3cAUm4qIcD3ReS+atZOmxKqLx44uI9CwtQLJkaR/vRv24q7QN/rnKaNl4m7e3Mxy5bOmOlT92mevzXrTv7RkVJImLkPgCJL7Iyw1X2CguQnIuS8fwc8P0ydbLqvDMihv/vtBJvJ1F19/e/7v1eDL9Yv9M1YaNszoO/KwYAAAAAAAAmFOzvDgYJP2BmX3QzN52pwXM7G1m9oyZPTMY1JeiBwAAAAAAAHA4s/xZ8VMhhBfN7D5Jf2hmHw8h/Em8QAjhaUlPS9LK6kP+jm0AAAAAAAAADczs4mAI4cW9/79qZr8t6ask/Un9WgAAAACAu8Iks9H5ebEQ0rnxesrT7DPz00tp7l7xwPn9xxuvW0zmDZbSbLQsimDbeDidt3uhSJctqu0uvZTmr/md211Nn8sWq0ywTit93tLltW0Nqufe3k23MyjSfS+Kat12O33eYTH6h3xdl0e43E7zzeJ8Qs9nDPr8wl5edWrL5aSdaW8n03GW3kbRTeb1y3Tfu9noXLXc5dStZNtuOu2bOkWonqtwB/Z60UumXxxUeW2LLiNuvUyzDV/eSfMv86zazsXOejLP98V90fyLrVvJvI7SfbtRVmPe5+H5jEEvzjrccf2/63IQiyg/smNpG9quTYMov7BwuZP+2C1laQbnYY7d3fJakWZW+pzBmM8YfCBP+3gxq/b92WF6POKMQSnNGey5Pl0vfX7k6H75i/6DyfTn++eS6Re3qqzM3WJ0DuvdNJOfFZvZkpmtfOGxpL8p6cOzaAsAAAAAAAAwr2Z15+D9kn57769NLUm/EkL4vRm1BQAAAAAAAJhLM7k4GEL4jKQvn8W2AQAAAAAAANw2y4IkAAAAAIBTwOcIKsocPDCvm+Z0lWfTLLf1R6vMtc3L6brZbpr5NVipttO/PEjmddbS7LPtW1W24c659Kuui9bTMI2bk0K0Py7Prwwu63BYPXeepcu28rSN8XPl7nl9PmE7rzLK1hZ2Rs6TpKwmIdJnlC23037q5VU24Go73c7ZdprltjFMj2XSBtepPgcx7rdFl/N2qXUzmT6XTX5poh+q9t8s0355tVhJpuOMvgsuC/Aj2w8l069speteXkyXj/l9vdy+sf94JUv7tHCZlfG0zxjsWZrbuBNabrp9x8eSNAij+7Bn6bgsXcJcnDM4cNmFbZed5/fvcu5Pprtvq0z76WUX55dbvGzaDz5jsGfpvj8fdfm6yydczNJ+a0djftP1tx8T8Tl6pUyzVT+xczmZvr6b5rJe30mXPw4zyRwEAAAAAAAAMHtcHAQAAAAAAADm1Kn5WXFomXbOt8cvOMLOGRu/0AjDxenXlaSaCtYTaW2PvmV8EvnO9OuX7Wb73rTqdpOr12WnWdvLGZ4dVjRre2g1GzPDpfHL1PE/2zjUusPxy9Su37TvGgy6otes35tsW5Ks2eYbbb/pcbvXWCll/fHLjdK6Nf2LZ8iaDaSy1+yFO/Saven1l6Z/8T23tDV+oRqtvFnbN7Z6jdZvIv6J2DRWl3fGLzTC52+dbbTt93/kDY3Wr/ll3USsO/2bVtZu8IZ3F5TD6d/zwqDhm07Z7P02X2o2Zgf59H3/wvW1RtueNTN7WNL/I+mSpFLS0yGEnzWzc5J+TdKjkj4n6W+HEF6bVTsBAKfHqbk4CAAAAADQUNKPhRA+ZGYrkj5oZn8o6QclvS+E8JNm9g5J75D0D2ufKUghRFfY4z8wBXfl3eUKWiv6Ktly+X5LaQbZ4Hw6vbtSbae1mW6nu55Ov/bGKAuwm/4RpyjSi9yhW627fT6d52LHDvxRNEQX24fuDodNlzk4jLY7LhtwsV09187QZZRl6UXulW7VyKW2y0lzWYb+j0Ibgyob8EJvM5nncwWLaH86Wf2Fep8/F/NZdD6bsYhu9VjLt5N5D7vMweVs8oy1gartXinSjLhndy8m090oM24npMu+sH2mdjsPLtzYf7xRpNmLPpvxYpRnmCs9rptlum4e/VXLZwHuuttjduXGYvRcdRmDkrRU85fqXXdc4/xCnyV5Jk/H08N5+rxtW65tx7SKULXjs8N0rPVcG9fLqv0PtNIxnbnMx+fc68ZWtO5Klp53bbedQc3dFG2N/qPVR3ceTKZfG6bj/Vo/vTunCM3+ADcNflYMAAAAAKdECOGlEMKH9h6vS/qYpAclvUXSu/YWe5ek755JAwEApw4XBwEAAADgFDKzRyW9WdL7Jd0fQnhJun0BUdJ9M2waAOAU4eIgAAAAAJwyZrYs6Tcl/f0Qwq1xy0frvc3MnjGzZ3bL7fErAADueVwcBAAAAIBTxMzaun1h8N0hhN/a++dXzOzy3vzLkl6907ohhKdDCE+GEJ7sZAt3WgQAMGcoSAIAAAAAp4SZmaR/JuljIYR/Gs16j6QfkPSTe///nSmeu5pwRUZ8QRJ1qhD/ZD1JoZUuW7bSe1J6N6rg/nwnDfHvn0nXHS5WxRt8le1imE63+1U7di6kbRosp4VOhmfSogXtpaowhFm6bMcVGYmnz3TTuy97rbTAxNawKoJhlhamWHMFVlY6VeGQNVdEpJWlyw7LtJ+63Wp/HlpIi1RvDNPtxsU1fFGRzJWAL6MCDLnrl8wmLxcfF+yQpIv55OtulWmRiPWyavPLxdl0XtFLph/pXt1//MmdS8m8qztpEYiVTlpoYzkqvOELs/gCK3Hxj9z1YVzswy/bs6FbNj3vtlwxk7oiJNMWIPFWs3TsPeqKx1xuHU0BEu/jg2p/Vtz494VoHo7OO38H3IuuwFBcgOT2c1fja1wBkni6684d7zPDc/uPXxmsJfM23Tm5OUz3Zxa4OAgAAAAAp8dTkr5f0l+Z2Z/v/ds/0u2Lgr9uZn9P0uclfe9smgcAOG24OAgAAAAAp0QI4d9JshGzv/E42wIAuDeQOQgAAAAAAADMKe4cBAAAAABIWXRDos8YdLmCKqMsQH/LSeb+wcXLda9H+X4hnbl13+ivqJnL6MtbaT7YcKVq8/DBNPtvbW0rmW63XN5ft8o3O9tNl21lbjtltX/nu5sj2ytJrSjDbNXlCJ5pp7l13WwQPU6z6Iox9/X0ssHIeXXr+szB0t2UGk/7PEK/rneuVfXN6ztpfZwL+ZJffKSB0u1cLarMuOd2z6dtcv22E+XLvdxfTeZtDdKct0eWX07nl9X8XOkYWMvTMdKzqv93lZ47HddP6bLpsdkMaZt8NmARHQ+fDejFOYM+uzBz2Xrn8439x693GYOPtY8nY/D/G2wk0+eirnmuSNv/cJ7mK2ZRvxwmY1BKcwaL4LJK3YubzyRMlnXHMh6bG0V9xmB/OPtLc9w5CAAAAAAAAMwpLg4CAAAAAAAAc4qLgwAAAAAAAMCcmv0PmwEAAAAAx88ki/IBQ7fKxTKXGxgGLtOuH2V+ddJML+XuHhQXV9jaHJ2P195Kc+2yQbXysssNfGD1VjL92tmF/ccPr9xI5r2ytZJML7TSNrx++dr+46VWmmd2Y7CQTA/LKtPsfDvNHByENO8snr/sctLqcgL984ybrlOGUcWt77DdMn3eOIOtkw/94umy7t6jOJfPZ8RJbsxEtso0E269TDP7Xi7O7D++Pkzz8B7ovJZMXxlWx/1aP805zF2W5GorzfDrl9XlkkWXU7eSp3mRsXHHpmdVPx48ruklmsJl3sV5hd5u8Fl71fns8yEvttJz5/WtKu/vodbxZAy+NEwzBttumD47rM67+11/r2Udt2zVp+tlOs9nDPrszHiMb7n+9xmDft3Yc8MzyXQ8NrcLlyVZ+OM8+Tl6VLhzEAAAAAAAAJhTXBwEAAAAAAAA5hQXBwEAAAAAAIA5xcVBAAAAAAAAYE6dmoIkZVvavDT9tczB0vhlRrHRmZMTaW2NX+akKrrjl6ldv9ds/bIcv8wooeGl73LyjN87r9/g7Mr6zQJJs/qs4LFchuuhFd3pT5rMJ1YfUsibnbCzzIK1BuNdkqzhcW+0ff7UlAjdUsPHdsYvOMLikg/tnlw7L8YvVGO5uzt+oRrdVrOBuNSafvurndHh4JN4KVtrtP6wmP6NY1g0O4l8sPVh9VqjA87HeXTteqNtL7Sn37YkvXD1TKP1h9cbfFjZafim0fAzphYaPMFis3O1tzb965QknVvZHL9QjcUG42ZYNjvfPtVo7RPGMqkXFSGJP3wPx7yf5NVrnrXT4hJl7opadNM+z9vVdH4rHUvt9XS7Ra96ffvKS59P5n3p0ovJ9CuD1f3HD3XS16Z/XX5JMj2s+bC/OUy/BPmiBBd7VSGFc61Nt2za/rW8mt9x8+oKSKyX6WtTUY4u4OH5oiKl+2KUR19wM/dld7sYvZ1c9R8W/b4/3n15//F9+WLturGB0ue56tr0wuBs1aYDBSPS6auDqiDJxiA9rl1XYKXrvkT1h62R8+oKg/iiIm33IT2PXvw3lR4rPya8paw6XzbLdH+2Sj9uq358sJUWavmidvo59UJ+PEVIbpbVZ7V198Vrx+37WrSvF/O0T18pXNGaUO17z/V3242J3I35eLv+XPfrxgbuC9CV4Woy3Y/GwcCdg8NDFBQ6LnydAwAAAAAAAOYUFwcBAAAAAACAOcXFQQAAAAAAAGBOnZrMQQAAAADAXZRnKlcW9idtu8rxMp85aOl9JdaLcuB6adZZaKd5Wv0zLgOvUy2/2K/PvxxeqLLdfMbgI50ryfRKXuWZHcie204z1fouJ7YT5c9d6KY5gg90byTTF1rr0TbT7LaVLM2/jbPp1suFZJ7Pl4uz6nxOYHmIXO5izD1ArSiLzmf29WuyDVtZfeagz198tHUz2k59pt1WWY299TIde68WaZbbK4MqI3jNBfzvhDQ8/cag6vOtQTrv0tKtZPpAfmGUTefnHZg+RIhsER3LQc0YkA7mFcbzd0LbLZv228VWtX8HMwYbFGVo4Nlhte9nsrTP1l0e7MXonNwJblnXT/H5spSleYQ+Y9Dn/5VR9qHPHKxzpUj7cL1Iz++d6FzyGadNs2+PwslrEQAAAAAAAIBjwcVBAAAAAAAAYE5xcRAAAAAAAACYU2QOAgAAAMAcCiaFTvWVMFuv8vLCMM06s1aamRXOrFSPOy7/azHNQtu8lN6TMlysptcWV5J5wUXrLa5V+X4+n+3KMM2iizMHP9u/L5n34rW1ZNpc3tlat8pke2TxejovT3MEF7P+/uM4U1A6mEVX1tyP4/Pl4iy6Ikx/H0/pOtFnrC1nVZszl5Xnsw3jDDafaeePx4VWmuF3IU/n1xmoeu6bLp/tepHmFcZ97DMer7oxsTlM8zBjnawYOc/z/VSncH3Yq1nXjwHPj684Z9DnUq7maf7io62N/ccX8vrMx6Py/HAjmV6JuuZ6kb5OLLp8xcWsGgc3Sz/20n2Px6Y/Vj4Pcscdnzhb8jDH2WeIrhe9ZDo+D+tyDk8K7hwEAAAAAAAA5hQXBwEAAAAAAIA5xcVBAAAAAAAAYE6ROQgAAAAAc8iCZMMoI293MHrhLL2vZHB+af9xaLs8rdxl3qWxgtp9vMqJu1WkuV2dNLZOS73d/cef2b6YzDvTTjPW3rz47P5jn/813E1z7FqdNMOslY3O1stcjmBu0+XP+Sw6L86Q89l/h+GzAL1uVmW7+WULl4XWyatlfU5arrRfXtd2WY1Zemxj/TBw09VzXSuXknk+W7LOVtlJpoc12Y1xPxwlP37qxoSf9pmVcUahH6eX8vTkeV3r+HMG/XH1+ZFrUc7jjstbXMvSdXtW5UVeDfV5lz0bfSzHnXeHsRW1ebNM8yz9ORu30edDNskUPSonr0UAAAAAAAAAjgUXBwEAAAAAAIA5xcVBAAAAAAAAYE5xcRAAAAAAAACYUxQkAQAAAIB5VJayzZ39ydDvj17WFSQpO1XY/nApLQ5ghSvY4eoBrK1WhUQ2l9LCIZ2brkBDObqYgC+QcRiWpW3MoiIjmSs44gtvxNO5L1aidN3dmqIEB4qBxIUqDlmwIF5+WNYXJIn1y/pLAr7oRWyttZ1MP9xy1WQ0uiDGwBWYWC9D9DgdE764zGJejVNfsGOrSAuS7BZVX/jj6qfrNCkQc+C5omPlj7MfP148ZlbytP/vz31Boa6O23q5m0y3Xbf1QzwvHQOujpGKUC2826CAhy+y4183DjMO4v7fKdsj50npeegL4zR57Toqp+bioJVSe2Pyg+Zlu+OXGb3xButKqnk9nchgpVkDdicv7nRA2Zq+zyXJat7Mj1rT8y00PDtCg77L+80a395otn7R8H1kuDR+mVHKbrMx17TwU5PzNRs0HXTNVs+Gs9t+0zFzr+m2h3r95atTr9/kA0PTDxudrNmblh3iA9ad1FUVHGdr2Bm/UI1u3qxq4YXlzanX7Q9n+5FsWM7uxyRnu1vjF6px/qHp+12Shg9M/iXa6zQcM03Pt3aD9bOmbzoNDRq+Yd/aHV0FdZyXN1fGLwQAwBzhZ8UAAAAAAADAnOLiIAAAAAAAADCnTs3PigEAAAAAd1GQbFjEkxOzcIil3aJxlIH/hXk2TBcOM8jm8jmCd0vh7s3x03EW3WEz7nzeWaybpREIcSTBuEiO2szBPI2FuJBPHtNQuEGxHuW3rRf1sQE9q7L1fGaf7wef8xjLGhznw2ZCTqpt6bHarTmuK9lOMn0hnz5u4W7ZCj6vM1VMmdmWN4itObDuDFI1fOzPScwc5M5BAAAAAAAAYE5xcRAAAAAAAACYU1wcBAAAAAAAAOYUmYMAAAAAAMmie0dCfR6blVFw15gMLxejluQI+ug2Pz0oRmeu+cy+OCuwLivvTrKancga5J3FigO5Yy6DsEGOXZzjOHDPs5wN/OLRemn/tl0OX7zvPs/v4c61ZHotG515t1XuJtN9N742Q3f/8Y1iMZnnMyB70f7shvSShm/jMOoLa3Acxx2bOEsvP8JQuzgnccnSPm1bfX7kcRjcxV3PbXSf+rzIuizDg+venUb6sebHSJwb6jMGi/Lk3ad38loEAAAAAAAA4FhwcRAAAAAAAACYU1wcBAAAAAAAAOYUmYMAAAAAACmLcrHGRfYVk+d2Ze65hsMoq8vfruKiw8oyyu2Sz+wbnTN22MzBaWWqz2YcRJl4pdtZn5Pm9+8w4vwzn2XYzdLQxzgbzecTLuRpPmHcxz2XXfhgftO1oqtJbZbBTVfrbpX1zxMf2023rB8TdWOkCX8sa5f1eYU1Y9PnK/ocuzg/z+dDzsogTHeu+fHvX1KKEO9rOs/nCNZlDs5KnDHqsz19/uhJwJ2DAAAAAAAAwJzi4iAAAAAAAAAwp7g4CAAAAAAAAMwpLg4CAAAAAAAAc4qCJAAAAACAeiEtAGDl5AVJ5AsNFFUYf8jTmcHS+1cGgyrIf+BC/eUmY21XhEMNCgDkY4qOxOoKlPjiEgcKVRyC74v4udquAkxmaR/3y9GXAeoKuSznO8n0BVe8pK4gycBVuNlx+74VqnV3ynYyzxdCiYs5xIVYJGkYfL/cncIPvliMP5ZHpa7wSfsQ4/KkiAuJ+PE/cH0cj5l0RBwsxjKIxkxp7li55/XFTJKiPGOGS3x++HOlbqw1KTZ0XLhzEAAAAAAAAJhTp+fOwVBb8Xus1s74ZUYZ9qZfV5LKzmyvEpetQ/xVzwk1f42bhP+D3WE1qc5edpptu+xM329Ss7b7v64e67YlZbvN1tfC9KuWvRn/BWx3+r+ZhNDwwA0avlY0XL3sTt/+pufbvWZY5rq6sTT1+lk2/XnQaTV4s5QU8mYDaVA2+7vj7nD6jyadVrM3neV2sxe/lXZ/6nW7ecM3zBlaazf4kCVpqTV9v0n1d7lMwt9Rc1q2LUkbxei7dMa5vrvYaNu3dhu82UvaHDZ749jYnX7919ab7TsAAPca7hwEAAAAAAAA5tTpuXMQAAAAAHAyRDe+mv/1hLsJ/cCviaJsrgPRbW66jO5IH5cfF+fA+VwxL5QnLwPsMDl2heuoONOs5e5qzt3PiqbNHFzLt5PpM1n95YQijP4lxFZI190sR98JXdemvssn9FmMoWbM5A3v3h7F5076zLvsED/zOvhc0Rg/ovY30XbdveP6v1fT5sItO4jGT9vqcwPjPj7Q3+55s6Y/05tQnKl4t7IvjxJ3DgIAAAAAAABziouDAAAAAAAAwJzi4iAAAAAAAAAwp8gcBABgj5nlkp6R9EII4TvN7JykX5P0qKTPSfrbIYTXZtdCAABOhgM5g3XconH+VsjTmT52rxxW/1CXlTdWVt/e0gclHgGfE+iz0Q7ThmHpnivq06U8DXn0GWvDUOXytV3+na+iPoiWfbB9PZm3nPVq29gPVTu2yjQ3cCekGYNx5qDPSGxbuj+DKK8wbp8kDd0Aiqf9vjXJnvPbjfn2H8ahcidPSI5dVjNufRvjY+CzJP34L6Pj07W0X/yxizMID+T7jemm+Dz0bfDb8efLpPz5ehKd/BYCAHB8flTSx6Lpd0h6XwjhcUnv25sGAAAAgHsGFwcBAJBkZg9J+g5Jvxj981skvWvv8bskffcxNwsAAAAAjhQXBwEAuO1nJP24pPj3AveHEF6SpL3/3zdqZTN7m5k9Y2bPDG9uHmlDAQAAAOBuIXMQADD3zOw7Jb0aQvigmX39NM8RQnha0tOStPj4A9OH2AAAcI/xMV1xjuC4PLBQTJ6rVpcDZ+MyB+9Sfls5Jlcw3eb09+r4/MKYz3Lz/RLnwHVcPqHvh/i5LuW33JY6kzRVkrTjuj/OGJSkjWJ0fmEvGyTTcS7fVpG2oV+klziKKOutnaf90srS6eMS5+P5fMLDZA6eFHmUB9g+xHo+v8+fK7tRrumypWO4Y7vJdBY9V905J0m5y55sED1ZK8lWPSH5kHW4OAgAgPSUpO8ys2+X1JO0ama/LOkVM7scQnjJzC5LenWmrQQAAACAu+z0XZYGAOAuCyG8M4TwUAjhUUlvlfRvQgjfJ+k9kn5gb7EfkPQ7M2oiAAAAABwJLg4CADDaT0r6ZjP7lKRv3psGAAAAgHsGPysGACASQvhjSX+89/iapG+cZXsAAAAA4ChxcRAAAAAAcDjl5Cn+viBJUgDA/ZbtQE2RQbXA0BVr8MUz4iIdmduoL0hymAIBdcU/DuNuFT057HP1y9Ff+33xEm8l39l/fC7bdXPrC5KUqo7Bjjt2OyEtXbFTVtO+SEemdDpe1xdb2S3T6WFUkKTriq94d6sYSO4qXOzWFMrxy+6E9Fi1LW1zXGxjcAJ/CJpbOi5L+elo2QP7nvbToOYlpu2Gv3+u2ja6ZQ+zblv+xWwyvh9OopM3mgAAAAAAAAAcCy4OAgAAAAAAAHPq1PysOGTScLHJE0y/atlpdgvogdvoD6m12aDxkro2ffuHC402rdBwhBWd6fe9rL/Lffy2e836Xa3p188GzcbcsNdodZXdZvterDQY9FnDfi+b9V3oTN/2ouGYy7ea/b2mrP+lxPj12+OXwWRCkAbF6J+QjJM3+NnRStf/5OdwzvU2G62/2Bo0Wt//ZOxQ65bNzqHz3Wb7fr49/fqLebPjdrPhG/atBm8cm8Nuo21f21xqtH6TMSM1+5lfK2v2IW+x1ey4r7T6U6+72mBdSdqt+ZniJDaHDd80AQDAXXNqLg4CAAAAAE4g/7ddd83d3ywR4j/m+ixAf70+WnbcH4GKaOUDuWJ52oiySJ8rfu67lT13lErXxnbUyZml+z4oR//B0i/rLUeZgyvZ4f6YUkTHoO/y5LbK0X9YWjyQbZiK1912f50fun2N96+dp/mKPm/xbmZC3i0+f7GMxulJbK8/c4qarL2227dNd2dRmTz2OZSj1W3zTuIx4tfNGtxlFucMnsRj5Z38Vz0AAAAAAAAAR4KLgwAAAAAAAMCc4uIgAAAAAAAAMKfIHAQAAAAAHJ84f2tMFJcN4szB+mJfZXTvS+byzFqtNF+u7zIHQxidD3YS8sLG5SDG++tz6ny/5VHGWu6y3Ap3/9D5fGP/8VpWX7xqEAo3XT33TkizATdd5mDc5m6WFjcbuCy6rajy5HaRVtTbKdJl4zy5pkWgJuXHXhnSNsZ5mG1LKwoW6o1c1ttpWv3zCLTNZWH68RXtjt83n6MZ5/8VPkP0wHar7QxclcWyQYHWJoqa15STaGZ3DprZt5rZJ8zs02b2jlm1AwAAAAAAAJhXM7k4aGa5pJ+T9G2S3iTp75jZm2bRFgAAAAAAAGBezerOwa+S9OkQwmdCCLuSflXSW2bUFgAAAAAAAGAuzepH6g9Kei6afl7SV/uFzOxtkt4mSe3ls8fTMgAAAACYR+XofLMDskNkaJU187J0mwei9crR2/H5eHEun8/Sy7L6fSuTfLPp76HJand2euMyy+JsvQPrumDHrCbHzjsXZQ62rT7z0WcO7oRqO5suc/BmsTjyeXqWZgP6vMKtYnTmYN9lDlrUL708zTJsm89IHL1/h+mzoxSPzWJcYOcJkLtxGZ8d486yOLMvzq+UpHaDHMHj6rc4QzGQOTjSnXrmwNkWQng6hPBkCOHJ1sLSMTQLAAAAAAAAmB+zujj4vKSHo+mHJL04o7YAAAAAAAAAc2lWFwc/IOlxM3vMzDqS3irpPTNqCwAAAAAAADCXZnJxMIQwlPR2Sb8v6WOSfj2E8JFZtAUAAAAA7hVm9q1m9gkz+7SZvWPW7QEAnHyzKkiiEMJ7Jb13VtsHAAAAgHuJmeWSfk7SN+t2lNMHzOw9IYSPTvQE4WiKaRxQtxmXTm/RssMD1UpSdQUl2nlafGLbzR+W1XOPK/5xGPkRFbLwRUbiAizjimdkNvoA9LK0aMf5fDOaSguDjNOPmrFZdpN5O2VaSMRvN3ket+xGUT3X1jBtU1GmY6TbGu4/Xm31k3ntLB0T/rliufmCGMWIJceLC2L48TFuvMTHeRD85ZxjOn9r5AfG5ej9yd1pVjcux2939HZ8AZL2iOUOvc0G7T2JZvWzYgAAAADA3fVVkj4dQvhMCGFX0q9KesuM2wQAOOG4OAgAAAAA94YHJT0XTT+/92/7zOxtZvaMmT2zW24da+MAACcTFwcBAAAA4N5wp9/DJr+3CyE8HUJ4MoTwZCdbPKZmAQBOMgvhaDIQ7jYzuyLp2ZpFLki6ekzNuZfQb9Oj76ZDv03vXu67R0IIF2fdiLtlgvesJu7lcXDU6Lvp0G/To++mc9L77cS+Z5nZ10r6iRDCt+xNv1OSQgj/64jlv/B+ddL7/KSgnyZDP02GfpoM/TSZO/XTxO9Xp+bi4Dhm9kwI4clZt+O0od+mR99Nh36bHn0HiXHQBH03HfptevTddOi36ZlZS9InJX2jpBckfUDSfx5C+MiY9ejzCdBPk6GfJkM/TYZ+mkzTfppZtWIAAAAAwN0TQhia2dsl/b6kXNIvjbswCAAAFwcBAAAA4B4RQnivpPfOuh0AgNPjXipI8vSsG3BK0W/To++mQ79Nj76DxDhogr6bDv02PfpuOvTb8aPPJ0M/TYZ+mgz9NBn6aTKN+umeyRwEAAAAAAAAcDj30p2DAAAAAAAAAA6Bi4MAAAAAAADAnDr1FwfN7FvN7BNm9mkze8es23OamNnnzOyvzOzPzeyZWbfnJDOzXzKzV83sw9G/nTOzPzSzT+39/+ws23gSjei3nzCzF/bG3Z+b2bfPso0nkZk9bGZ/ZGYfM7OPmNmP7v07Y24OmVluZn9mZr+7N804mNAd+o7Xnwnc6fMB4268Ef3GmJuAmZ0xs98ws4/vvfd9LWPu+PB96s74PDY5PquMx+vcZMzsv9k73z5sZv/SzHr00+GvR5jZO/de0z9hZt8yyTZO9cVBM8sl/Zykb5P0Jkl/x8zeNNtWnTp/I4TwRAjhyVk35IT755K+1f3bOyS9L4TwuKT37U0j9c91sN8k6af3xt0TexX1kBpK+rEQwpdI+hpJP7z32saYm08/Kulj0TTjYHK+7yRefyblPx8w7iZzp89VjLnxflbS74UQvljSl+v2ecuYOwZ8n6rF57HJ8VllPF7nxjCzByX9iKQnQwhfKimX9FbRT9IhrkfsvU69VdJf21vn/9x7ra91qi8OSvoqSZ8OIXwmhLAr6VclvWXGbcI9KITwJ5Kuu39+i6R37T1+l6TvPs42nQYj+g1jhBBeCiF8aO/xum5/eHhQjLm5Y2YPSfoOSb8Y/TPjYAIj+g7TY9zhSJjZqqSvk/TPJCmEsBtCuCHG3HHh+9QIfB6bDJ9VxuN17lBakhbMrCVpUdKLop8Oez3iLZJ+NYTQDyF8VtKndfu1vtZpvzj4oKTnounn9/4NkwmS/sDMPmhmb5t1Y06h+0MIL0m3PzxIum/G7TlN3m5mf7l3e/Tc3RZ+GGb2qKQ3S3q/GHPz6Gck/bikMvo3xsFkfkYH+07i9WcSd/p8wLgbb9TnKsZcvddLuiLp/977WeIvmtmSGHPHhe9TE+DzWK2fEZ9VxuF1bgIhhBck/WNJn5f0kqSbIYQ/EP00yqh+mep1/bRfHLQ7/Fs49lacXk+FEL5Ct39G8MNm9nWzbhDmws9LeoOkJ3T7Rf+fzLQ1J5iZLUv6TUl/P4Rwa9btwfEys++U9GoI4YOzbstpU9N3vP5Mhs8H07lTvzHmxmtJ+gpJPx9CeLOkTc3nT8Zmhe9TY/B5bDQ+q0yM17kJ7P0B7S2SHpP0gKQlM/u+2bbqVJrqdf20Xxx8XtLD0fRDun3bKSYQQnhx7/+vSvptTXCrKRKvmNllSdr7/6szbs+pEEJ4JYRQhBBKSb8gxt0dmVlbtz+IvjuE8Ft7/8yYmy9PSfouM/ucbv/M6xvM7JfFOJjEHfuO15/JjPh8wLgb4079xpibyPOSng8hvH9v+jd0+0s0Y+548H2qBp/HxuKzymR4nZvMN0n6bAjhSghhIOm3JP2nop9GGdUvU72un/aLgx+Q9LiZPWZmHd0OXXzPjNt0KpjZkpmtfOGxpL8p6cP1a8F5j6Qf2Hv8A5J+Z4ZtOTW+8AK253vEuDvAzEy3M0k+FkL4p9EsxtwcCSG8M4TwUAjhUd1+f/s3IYTvE+NgrFF9x+vPeDWfDxh3NUb1G2NuvBDCy5KeM7M37v3TN0r6qBhzx4XvUyPweWw8PqtMhte5iX1e0teY2eLe+feNup31ST/d2ah+eY+kt5pZ18wek/S4pD8d92StI2niMQkhDM3s7ZJ+X7cr2fxSCOEjM27WaXG/pN++fc6pJelXQgi/N9smnVxm9i8lfb2kC2b2vKT/QdJPSvp1M/t7uv1C9r2za+HJNKLfvt7MntDtW5s/J+mHZtW+E+wpSd8v6a/M7M/3/u0fiTGH2xgH0/spXn/GuuPnAzP7gBh3dUb1279gzE3kv5b07r2LU5+R9Hd1+yYGxtwR4/tULT6PTY8+OojXuTFCCO83s9+Q9CHdrhb+Z5KelrSsOe+nw1yPCCF8xMx+XbcvQA8l/XAIoRi7jRCIlAAAAAAAAADm0Wn/WTEAAAAAAACAKXFxEAAAAAAAAJhTXBwEAAAAAAAA5hQXBwEAAAAAAIA5xcVBAAAAAAAAYE5xcRAAAAAAAACYU1wcBI6ZmT1gZr9xyHV+0Mz+j6NqEwAAd8J7FgDgtOA9C5hea9YNAOZNCOFFSX9r1u0AAGAc3rMAAKcF71nA9LhzELhLzOwrzewvzaxnZktm9hEz+9I7LPeomX147/EPmtlvmdnvmdmnzOynouX+rpl90sz+raSnon+/aGa/aWYf2Pvvqb1//x0z+y/2Hv+Qmb37yHcaAHAq8Z4FADgteM8Cjh53DgJ3SQjhA2b2Hkn/s6QFSb8cQvjwBKs+IenNkvqSPmFm/7ukoaT/UdJfl3RT0h9J+rO95X9W0k+HEP6dmb1O0u9L+hJJb5P0783ss5J+TNLX3K19AwDcW3jPAgCcFrxnAUePi4PA3fU/SfqApB1JPzLhOu8LIdyUJDP7qKRHJF2Q9MchhCt7//5rkv6TveW/SdKbzOwL66+a2UoI4RUz++91+w3ue0II1+/GDgEA7lm8ZwEATgves4AjxMVB4O46J2lZUltST9LmBOv0o8eFqvMyjFg+k/S1IYTtO8z7MknXJD0wUWsBAPOM9ywAwGnBexZwhMgcBO6upyX9d5LeLel/a/A875f09WZ23szakr43mvcHkt7+hQkze2Lv/18l6dt0+9b5f2BmjzXYPgDg3sd7FgDgtOA9CzhCXBwE7pK9kNphCOFXJP2kpK80s2+Y5rlCCC9J+glJ/0HSv5b0oWj2j0h6ci+U96OS/isz60r6BUn/5V6Vrh+T9EsW3RMPAMAX8J4FADgteM8Cjp6FMOqOWgAAAAAAAAD3Mu4cBAAAAAAAAOYUBUmAI2JmXybpX7h/7ocQvnoW7QEAYBTeswAApwXvWcDdx8+KAQAAAAAAgDnFz4oBAAAAAACAOcXFQQAAAAAAAGBOcXEQAAAAAAAAmFNcHAQAAAAAAADm1P8P2IxC+BEj8a8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "EXAMPLE_I = 2\n", + "CHANNEL_I = 7\n", + "HISTORY_LENGTH = 6\n", + "sat_data = sat_batch[\"data\"].sel(example=EXAMPLE_I, channels_index=CHANNEL_I)\n", + "channel_name = sat_batch[\"channels\"].sel(example=EXAMPLE_I, channels_index=CHANNEL_I).values\n", + "opt_flow_data = opt_flow_batch[\"data\"].sel(example=EXAMPLE_I, channels_index=CHANNEL_I)\n", + "min_pixel_val = min(sat_data.min(), opt_flow_data.min())\n", + "max_pixel_val = min(sat_data.max(), opt_flow_data.max())\n", + "imshow_kwargs = dict(x='x_index', y='y_index', add_colorbar=False, vmin=min_pixel_val, vmax=max_pixel_val)\n", + "\n", + "fig, axes = plt.subplots(figsize=(18, 7), ncols=3)\n", + "\n", + "ax = axes[0]\n", + "sat_img = sat_data.isel(time_index=0).plot.imshow(ax=ax, **imshow_kwargs)\n", + "\n", + "ax = axes[1]\n", + "opt_flow_cropped_img = crop_center(\n", + " opt_flow_data.isel(time_index=0),\n", + " 24,\n", + " 24\n", + ").plot.imshow(ax=ax, **imshow_kwargs)\n", + "OPT_FLOW_TITLE = \"Optical flow precitions (cropped) \"\n", + "ax.set_title(OPT_FLOW_TITLE)\n", + "\n", + "ax = axes[2]\n", + "opt_flow_img = opt_flow_data.isel(time_index=0).plot.imshow(ax=ax, **imshow_kwargs)\n", + "ax.set_title(\"Optical flow precitions (zoomed out)\")\n", + "\n", + "\n", + "def format_date(dt: np.datetime64) -> str:\n", + " return pd.Timestamp(dt).strftime(\"%Y-%m-%d %H:%M\")\n", + "\n", + "plt.tight_layout()\n", + "\n", + "def init():\n", + " sat_img.set_data(sat_data.isel(time_index=0))\n", + " axes[1].set_title(OPT_FLOW_TITLE)\n", + " opt_flow_cropped_img.set_data(np.full(shape=sat_data.isel(time_index=0).shape, fill_value=np.NaN))\n", + " opt_flow_img.set_data(np.full(shape=opt_flow_data.isel(time_index=0).shape, fill_value=np.NaN))\n", + " return sat_img, opt_flow_cropped_img, opt_flow_img\n", + "\n", + "def update(i):\n", + " # SAT DATA\n", + " sat_img.set_data(sat_data.isel(time_index=i))\n", + " datetime = sat_batch[\"time\"].isel(example=EXAMPLE_I, time_index=i).values\n", + " axes[0].set_title(\"Real satellite data | \" + format_date(datetime) + \" | chan = \" + channel_name)\n", + " \n", + " # OPTICAL FLOW PREDICTIONS\n", + " if i > HISTORY_LENGTH:\n", + " opt_flow_datetime = opt_flow_batch[\"time\"].isel(example=EXAMPLE_I, time_index=i-HISTORY_LENGTH).values\n", + " axes[1].set_title(OPT_FLOW_TITLE + format_date(opt_flow_datetime))\n", + " new_opt_flow_data = opt_flow_data.isel(time_index=i-HISTORY_LENGTH).values.copy()\n", + " opt_flow_cropped_img.set_data(\n", + " crop_center(\n", + " new_opt_flow_data,\n", + " 24,\n", + " 24\n", + " )\n", + " )\n", + " new_opt_flow_data[[39, 63], 39:63] = 0\n", + " new_opt_flow_data[39:64, [39, 63]] = 0\n", + " opt_flow_img.set_data(new_opt_flow_data)\n", + " return sat_img, opt_flow_cropped_img, opt_flow_img\n", + "\n", + "anim = FuncAnimation(fig, func=update, frames=np.arange(30), init_func=init, interval=250, blit=True)\n", + "#anim.save('optical_flow.gif', writer='imagemagick')\n", + "html = anim.to_html5_video()\n", + "HTML(html)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b906a09-e147-44e7-93ed-c60e14b37031", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nowcasting_dataset", + "language": "python", + "name": "nowcasting_dataset" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 36c2c9521bfe789a5204ed92869c610abc19e1c5 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 13:30:28 +0000 Subject: [PATCH 176/197] fix linter error with local_temp_path_to_path_object_expanduser --- nowcasting_dataset/config/model.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index bc9e46ca..dadff776 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -429,8 +429,10 @@ class Process(BaseModel): @validator("local_temp_path") def local_temp_path_to_path_object_expanduser(cls, v): - """Convert the path in string format to a `pathlib.PosixPath` object - and call `expanduser` on the latter.""" + """Convert the path in string format to a `pathlib.PosixPath` object. + + Also calls `expanduser` on the latter. + """ return Path(v).expanduser() From 268ed89335fd56695d1ee46c586f879384cff18d Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 14:19:58 +0000 Subject: [PATCH 177/197] remove total_number_batches from DataSource.create_batches --- .../data_sources/data_source.py | 50 ++++++++----------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 8eb40588..218b8202 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -142,7 +142,6 @@ def create_batches( dst_path: Path, local_temp_path: Path, upload_every_n_batches: int, - total_number_batches: int = None, ) -> None: """Create multiple batches and save them to disk. @@ -150,34 +149,32 @@ def create_batches( Args: spatial_and_temporal_locations_of_each_example (pd.DataFrame): A DataFrame where each - row specifies the spatial and temporal location of an example. The number of rows - must be an exact multiple of `batch_size`. - Columns are: t0_datetime_UTC, x_center_OSGB, y_center_OSGB. + row specifies the spatial and temporal location of an example. The number of rows + must be an exact multiple of `batch_size`. + Columns are: t0_datetime_UTC, x_center_OSGB, y_center_OSGB. idx_of_first_batch (int): The batch number of the first batch to create. batch_size (int): The number of examples per batch. dst_path (Path): The final destination path for the batches. Must exist. local_temp_path (Path): The local temporary path. This is only required when dst_path - is a cloud storage bucket, so files must first be created on the VM's local disk in - temp_path and then uploaded to dst_path every `upload_every_n_batches`. Must exist. - Will be emptied. + is a cloud storage bucket, so files must first be created on the VM's local disk in + temp_path and then uploaded to dst_path every `upload_every_n_batches`. Must exist. + Will be emptied. upload_every_n_batches (int): Upload the contents of temp_path to dst_path after this - number of batches have been created. If 0 then will write directly to `dst_path`. - total_number_batches (int, optional): If specified it will be used to compute the batch - size (`batch_size` will not be used in that case). + number of batches have been created. If 0 then will write directly to `dst_path`. """ # Sanity checks: - assert idx_of_first_batch >= 0, ( - "The batch number of the first batch to create should be" " greater than 0" + assert ( + idx_of_first_batch >= 0 + ), "The batch number of the first batch to create should be greater than 0" + assert batch_size > 0, ( + "The batch size should be strictly greater than 0. Otherwise," + " you should specify 'total_number_batches' to compute the batch size from" + " 'spatial_and_temporal_locations_of_each_example'" + ) + assert len(spatial_and_temporal_locations_of_each_example) % batch_size == 0, ( + f"{len(spatial_and_temporal_locations_of_each_example)=} must be" + f" exactly divisible by {batch_size=}" ) - - if total_number_batches is None: - assert batch_size > 0, ( - "The batch size should be strictly greater than 0. Otherwise," - " you should specify 'total_number_batches' to compute the batch size from" - " 'spatial_and_temporal_locations_of_each_example'" - ) - assert len(spatial_and_temporal_locations_of_each_example) % batch_size == 0 - assert upload_every_n_batches >= 0, "'upload_every_n_batches' should be greater than 0" spatial_and_temporal_locations_of_each_example_columns = ( @@ -186,8 +183,8 @@ def create_batches( assert spatial_and_temporal_locations_of_each_example_columns == list( SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES ), ( - f"The provided data columns ({spatial_and_temporal_locations_of_each_example_columns})" - f"do not match {SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES}" + f"The provided data columns {spatial_and_temporal_locations_of_each_example_columns}" + f" do not match {SPATIAL_AND_TEMPORAL_LOCATIONS_COLUMN_NAMES=}" ) self.open() @@ -199,13 +196,10 @@ def create_batches( path_to_write_to = local_temp_path if save_batches_locally_and_upload else dst_path # Split locations per example into batches: - if total_number_batches is not None: - batch_size = len(spatial_and_temporal_locations_of_each_example) // total_number_batches - else: - total_number_batches = len(spatial_and_temporal_locations_of_each_example) // batch_size + n_batches = len(spatial_and_temporal_locations_of_each_example) // batch_size locations_for_batches = [] - for batch_idx in range(total_number_batches): + for batch_idx in range(n_batches): start_example_idx, end_example_idx = get_start_and_end_example_index( batch_idx=batch_idx, batch_size=batch_size ) From 906d6acf37e32572fdbb4f9095f90085c53bbf44 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 14:21:32 +0000 Subject: [PATCH 178/197] remove total_number_batches from assertion message --- nowcasting_dataset/data_sources/data_source.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 218b8202..76c34607 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -166,11 +166,7 @@ def create_batches( assert ( idx_of_first_batch >= 0 ), "The batch number of the first batch to create should be greater than 0" - assert batch_size > 0, ( - "The batch size should be strictly greater than 0. Otherwise," - " you should specify 'total_number_batches' to compute the batch size from" - " 'spatial_and_temporal_locations_of_each_example'" - ) + assert batch_size > 0, "The batch size should be strictly greater than 0." assert len(spatial_and_temporal_locations_of_each_example) % batch_size == 0, ( f"{len(spatial_and_temporal_locations_of_each_example)=} must be" f" exactly divisible by {batch_size=}" From cbdaf46b22569cf673ac05c47134b52b0f527558 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 14:27:05 +0000 Subject: [PATCH 179/197] improve comments --- nowcasting_dataset/data_sources/data_source.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index 76c34607..bc9f27a3 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -171,7 +171,7 @@ def create_batches( f"{len(spatial_and_temporal_locations_of_each_example)=} must be" f" exactly divisible by {batch_size=}" ) - assert upload_every_n_batches >= 0, "'upload_every_n_batches' should be greater than 0" + assert upload_every_n_batches >= 0, "`upload_every_n_batches` must be >= 0" spatial_and_temporal_locations_of_each_example_columns = ( spatial_and_temporal_locations_of_each_example.columns.to_list() @@ -193,7 +193,6 @@ def create_batches( # Split locations per example into batches: n_batches = len(spatial_and_temporal_locations_of_each_example) // batch_size - locations_for_batches = [] for batch_idx in range(n_batches): start_example_idx, end_example_idx = get_start_and_end_example_index( @@ -218,7 +217,7 @@ def create_batches( ) # Save batch to disk. - # TODO: Use DataSourceOutput.save_netcdf + # TODO: Issue #524: Use DataSourceOutput.save_netcdf in place of to_netcdf netcdf_filename = path_to_write_to / nd_utils.get_netcdf_filename(batch_idx) encoding = {name: {"compression": "lzf"} for name in batch.data_vars} batch.to_netcdf(netcdf_filename, engine="h5netcdf", encoding=encoding) From 31a3f3c7fab5356a24c2c6c1f70a924e69a177bb Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 14:30:30 +0000 Subject: [PATCH 180/197] improve comments --- nowcasting_dataset/data_sources/data_source.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/data_sources/data_source.py b/nowcasting_dataset/data_sources/data_source.py index bc9f27a3..234c4834 100644 --- a/nowcasting_dataset/data_sources/data_source.py +++ b/nowcasting_dataset/data_sources/data_source.py @@ -271,15 +271,18 @@ def get_batch( ) future_examples.append(future_example) - # Get the examples back. future_example.result() will raise an exception - # if the worker thread raised an exception. + # Get the examples back. Loop round each future so we can log a helpful error. + # If the worker thread raised an exception then the exception won't "bubble up" + # until we call future_example.result(). examples = [] for example_i, future_example in enumerate(future_examples): try: - examples.append(future_example.result()) + result = future_example.result() except Exception: logger.error(f"Exception when processing {example_i=}!") raise + else: + examples.append(result) # Get the DataSource class, this could be one of the data sources like Sun cls = self.get_data_model_for_batch() From 1b36c8d45dd8b0f47b4ca3e9ecffce515f24f3c2 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 14:35:52 +0000 Subject: [PATCH 181/197] more comments! --- .../data_sources/satellite/satellite_data_source.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py index e47a8b63..3ef50e5f 100644 --- a/nowcasting_dataset/data_sources/satellite/satellite_data_source.py +++ b/nowcasting_dataset/data_sources/satellite/satellite_data_source.py @@ -98,6 +98,8 @@ def get_spatial_region_of_interest( # Get the index into x and y nearest to x_center_osgb and y_center_osgb: x_index_at_center = np.searchsorted(data_array.x.values, x_center_osgb) - 1 y_index_at_center = np.searchsorted(data_array.y.values, y_center_osgb) - 1 + # Put x_index_at_center and y_index_at_center into a pd.Series so we can operate + # on them both in a single line of code. x_and_y_index_at_center = pd.Series({"x": x_index_at_center, "y": y_index_at_center}) half_image_size_pixels = self._square.size_pixels // 2 min_x_and_y_index = x_and_y_index_at_center - half_image_size_pixels @@ -112,6 +114,8 @@ def get_spatial_region_of_interest( ) * 2 ) + # If the requested region does step outside the available data then raise an exception + # with a helpful message: if suggested_reduction_of_image_size_pixels > 0: new_suggested_image_size_pixels = ( self._square.size_pixels - suggested_reduction_of_image_size_pixels From 1baa27c37efeb45d736177c508d4477abc046f0f Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 14:41:20 +0000 Subject: [PATCH 182/197] update nwp_size_test.yaml --- tests/config/nwp_size_test.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/config/nwp_size_test.yaml b/tests/config/nwp_size_test.yaml index dd3fff75..9bb3ea4e 100644 --- a/tests/config/nwp_size_test.yaml +++ b/tests/config/nwp_size_test.yaml @@ -23,8 +23,13 @@ input_data: topographic: topographic_filename: tests/data/europe_dem_2km_osgb.tif opticalflow: - number_previous_timesteps_to_use: 1 - opticalflow_image_size_pixels: 32 + history_minutes: 5 + forecast_minutes: 30 + opticalflow_zarr_path: tests/data/sat_data.zarr + opticalflow_input_image_size_pixels: 32 + opticalflow_output_image_size_pixels: 8 + opticalflow_channels: + - IR_016 output_data: filepath: not used by unittests! process: From 0de7c0044f412ff0b88df2a7b4625017f2983957 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 14:47:19 +0000 Subject: [PATCH 183/197] update gcp.yaml --- nowcasting_dataset/config/gcp.yaml | 37 ++++++++++++++++++++++ nowcasting_dataset/config/on_premises.yaml | 1 + 2 files changed, 38 insertions(+) diff --git a/nowcasting_dataset/config/gcp.yaml b/nowcasting_dataset/config/gcp.yaml index 12915f26..54e68069 100644 --- a/nowcasting_dataset/config/gcp.yaml +++ b/nowcasting_dataset/config/gcp.yaml @@ -4,10 +4,14 @@ general: input_data: default_forecast_minutes: 60 default_history_minutes: 30 + + #---------------------- GSP ------------------- gsp: forecast_minutes: 60 gsp_zarr_path: gs://solar-pv-nowcasting-data/PV/GSP/v3/pv_gsp.zarr history_minutes: 60 + + #---------------------- NWP ------------------- nwp: forecast_minutes: 60 history_minutes: 60 @@ -24,12 +28,16 @@ input_data: - hcc nwp_image_size_pixels: 64 nwp_zarr_path: gs://solar-pv-nowcasting-data/NWP/UK_Met_Office/UKV__2018-01_to_2019-12__chunks__variable10__init_time1__step1__x548__y704__.zarr + + #---------------------- PV ------------------- pv: forecast_minutes: 60 history_minutes: 30 pv_filename: gs://solar-pv-nowcasting-data/PV/Passive/ocf_formatted/v0/passiv.netcdf pv_metadata_filename: gs://solar-pv-nowcasting-data/PV/Passive/ocf_formatted/v0/system_metadata.csv get_center: false + + #---------------------- Satellite ------------- satellite: forecast_minutes: 60 history_minutes: 30 @@ -48,14 +56,43 @@ input_data: - WV_073 satellite_image_size_pixels: 64 satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr + + #---------------------- HRVSatellite ------------- + # The satellite Zarr data on GCP is the older Zarr, which contains + # HRV and the non-HRV channels in a single Zarr. + + # ------------------------- Sun ------------------------ sun: forecast_minutes: 60 history_minutes: 30 sun_zarr_path: gs://solar-pv-nowcasting-data/Sun/v0/sun.zarr + + # ------------------------- Topographic ---------------- topographic: forecast_minutes: 60 history_minutes: 30 topographic_filename: gs://solar-pv-nowcasting-data/Topographic/europe_dem_1km_osgb.tif + + # ------------------------- Optical Flow --------------- + opticalflow: + opticalflow_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr + opticalflow_history_minutes: 5 + opticalflow_forecast_minutes: 120 + opticalflow_input_image_size_pixels: 102 + opticalflow_output_image_size_pixels: 24 + opticalflow_source_data_source_class_name: SatelliteDataSource + opticalflow_channels: + - IR_016 + - IR_039 + - IR_087 + - IR_097 + - IR_108 + - IR_120 + - IR_134 + - VIS006 + - VIS008 + - WV_062 + - WV_073 output_data: filepath: gs://solar-pv-nowcasting-data/prepared_ML_training_data/v9/ process: diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index 3e72efed..7daee2bc 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -48,6 +48,7 @@ input_data: satellite_image_size_pixels: 24 satellite_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* + #---------------------- HRVSatellite ------------- hrvsatellite: hrvsatellite_channels: - HRV From ee6207942e227e7699cf5d4329b647afb3187e25 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 14:58:57 +0000 Subject: [PATCH 184/197] clip fake PV data after smoothing --- nowcasting_dataset/data_sources/fake.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index 543733d0..6e7e7282 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -348,14 +348,16 @@ def create_gsp_pv_dataset( seq_length, number_of_systems, ) - data = data.clip(min=0) - # smooth the data, the convolution method smooeths that data across systems first, + # smooth the data, the convolution method smooths that data across systems first, # and then a bit across time (depending what you set N) N = int(seq_length / 2) data = np.convolve(data.ravel(), np.ones(N) / N, mode="same").reshape( (seq_length, number_of_systems) ) + # Need to clip at 0 *after* smoothing, because the smoothing method might push + # non-zero data below zero. + data = data.clip(min=0) # make into a Data Array data_array = xr.DataArray( From 5d9efc0f789af5dc730267912dfcf5b0a1e221c8 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 15:08:22 +0000 Subject: [PATCH 185/197] I the PV test is fixed. Not entirely sure --- nowcasting_dataset/data_sources/fake.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index 6e7e7282..032e612b 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -344,10 +344,7 @@ def create_gsp_pv_dataset( coords = [(dim, ALL_COORDS[dim]) for dim in dims] # make pv yield - data = np.random.randn( - seq_length, - number_of_systems, - ) + data = np.random.random(size=(seq_length, number_of_systems)) # smooth the data, the convolution method smooths that data across systems first, # and then a bit across time (depending what you set N) @@ -355,9 +352,10 @@ def create_gsp_pv_dataset( data = np.convolve(data.ravel(), np.ones(N) / N, mode="same").reshape( (seq_length, number_of_systems) ) - # Need to clip at 0 *after* smoothing, because the smoothing method might push - # non-zero data below zero. - data = data.clip(min=0) + # Need to clip *after* smoothing, because the smoothing method might push + # non-zero data below zero. Clip at 0.1 instead of 0 so we don't get div-by-zero errors + # if capacity is zero (capacity is computed as the max of the random numbers). + data = data.clip(min=0.1) # make into a Data Array data_array = xr.DataArray( From b0ee333868da8b2495f4da9ca4be2d978c6f80f5 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 15:38:01 +0000 Subject: [PATCH 186/197] revert back to using np.random.randn --- nowcasting_dataset/data_sources/fake.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nowcasting_dataset/data_sources/fake.py b/nowcasting_dataset/data_sources/fake.py index 032e612b..150f1d01 100644 --- a/nowcasting_dataset/data_sources/fake.py +++ b/nowcasting_dataset/data_sources/fake.py @@ -343,8 +343,9 @@ def create_gsp_pv_dataset( } coords = [(dim, ALL_COORDS[dim]) for dim in dims] - # make pv yield - data = np.random.random(size=(seq_length, number_of_systems)) + # make pv yield. randn samples from a Normal distribution (and so can go negative). + # The values are clipped to be positive later. + data = np.random.randn(seq_length, number_of_systems) # smooth the data, the convolution method smooths that data across systems first, # and then a bit across time (depending what you set N) From 84055b767b739c3cd66fcbfb25754ff674f21105 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 15:57:00 +0000 Subject: [PATCH 187/197] no need for Manager to convert local_temp_path to Path --- nowcasting_dataset/manager.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index 56cf1c1c..c3f34531 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -34,7 +34,6 @@ class Manager: geospatial locations of each example. save_batches_locally_and_upload: bool: Set to True by `load_yaml_configuration()` if `config.process.upload_every_n_batches > 0`. - local_temp_path: Path: `config.process.local_temp_path` with `~` expanded. """ def __init__(self) -> None: # noqa: D107 @@ -48,8 +47,6 @@ def load_yaml_configuration(self, filename: str) -> None: self.config = config.load_yaml_configuration(filename) self.config = config.set_git_commit(self.config) self.save_batches_locally_and_upload = self.config.process.upload_every_n_batches > 0 - - self.local_temp_path = self.config.process.local_temp_path logger.debug(f"config={self.config}") def save_yaml_configuration(self): @@ -458,7 +455,7 @@ def create_batches(self, overwrite_batches: bool) -> None: # TODO: Issue 455: Guarantee that local temp path is unique and empty. local_temp_path = ( - self.local_temp_path + self.config.process.local_temp_path / split_name.value / data_source_name / f"worker_{worker_id}" From cade179b857490cf72ba8b7d11505bb93bd0fb1f Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 15:58:49 +0000 Subject: [PATCH 188/197] use Path as default for local_temp_path --- nowcasting_dataset/config/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index dadff776..be0c2c7f 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -425,7 +425,7 @@ class Process(BaseModel): ), ) - local_temp_path: str = Field("~/temp/") + local_temp_path: Path = Field(Path("~/temp/")) @validator("local_temp_path") def local_temp_path_to_path_object_expanduser(cls, v): From 1d9f02dbd012b2894bca86ebd92392c7e98e9f94 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 16:04:03 +0000 Subject: [PATCH 189/197] update description for local_temp_path --- nowcasting_dataset/config/model.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index be0c2c7f..dba81059 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -425,7 +425,14 @@ class Process(BaseModel): ), ) - local_temp_path: Path = Field(Path("~/temp/")) + local_temp_path: Path = Field( + Path("~/temp/").expanduser(), + description=( + "This is only necessary if using a VM on a public cloud and when the finished batches" + " will be uploaded to a cloud bucket. This is the local temporary path on the VM." + " This will be emptied." + ), + ) @validator("local_temp_path") def local_temp_path_to_path_object_expanduser(cls, v): From cf4b18e4ac5bcd7608e6502bda7fd6b18ffa4e53 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 16:20:50 +0000 Subject: [PATCH 190/197] avoid divide by zero --- .../data_sources/optical_flow/optical_flow_data_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 2e6b179d..2b06d4a2 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -235,7 +235,9 @@ def _convert_arrays_to_uint8(*arrays: tuple[np.ndarray]) -> tuple[np.ndarray]: # Rescale pixel values to be in the range [0, 1]: stacked -= stacked.min() - stacked /= stacked.max() + stacked_max = stacked.max() + if stacked_max > 0.0: + stacked /= stacked.max() # Convert to uint8 (uint8 can represent integers in the range [0, 255]): stacked *= 255 From 765ae1fdd7050514331b356a86a960f2848a1954 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 16:26:17 +0000 Subject: [PATCH 191/197] raise numpy errors for division --- .../data_sources/optical_flow/optical_flow_data_source.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index 2b06d4a2..d63280cf 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -237,7 +237,10 @@ def _convert_arrays_to_uint8(*arrays: tuple[np.ndarray]) -> tuple[np.ndarray]: stacked -= stacked.min() stacked_max = stacked.max() if stacked_max > 0.0: - stacked /= stacked.max() + # If there is still an invalid value then we want to know about it! + # Adapted from https://stackoverflow.com/a/33701974/732596 + with np.errstate(all="raise"): + stacked /= stacked.max() # Convert to uint8 (uint8 can represent integers in the range [0, 255]): stacked *= 255 From 36eee38df3ea64439bfe33eeb9423178b16c0ca5 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Fri, 3 Dec 2021 16:27:45 +0000 Subject: [PATCH 192/197] fix test_load_yaml_configuration --- tests/test_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_manager.py b/tests/test_manager.py index fc5edbee..0eee83ec 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -72,12 +72,11 @@ def test_load_yaml_configuration(): # noqa: D103 filename = local_path / "tests" / "config" / "test.yaml" manager.load_yaml_configuration(filename=filename) - local_temp_path = manager.local_temp_path manager.initialise_data_sources() assert len(manager.data_sources) == 8 assert isinstance(manager.data_source_which_defines_geospatial_locations, GSPDataSource) - assert isinstance(local_temp_path, Path) + assert isinstance(manager.config.process.local_temp_path, Path) def test_get_daylight_datetime_index(): From 7dd2e5ec5ae48ea932811a41b4310739377bcb2e Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 6 Dec 2021 11:26:40 +0000 Subject: [PATCH 193/197] add error log when file is not there --- nowcasting_dataset/dataset/batch.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nowcasting_dataset/dataset/batch.py b/nowcasting_dataset/dataset/batch.py index 96407bf3..20fd74ef 100644 --- a/nowcasting_dataset/dataset/batch.py +++ b/nowcasting_dataset/dataset/batch.py @@ -181,6 +181,11 @@ def load_netcdf( filename_or_obj=local_netcdf_filename, ) future_examples_per_source.append([data_source_name, future_examples]) + else: + _LOG.error( + f"{local_netcdf_filename} does not exists," + f"this is for {data_source_name} data source" + ) # Collect results from each thread. for data_source_name, future_examples in future_examples_per_source: From 3302ed6b08dc03c47ad6bdbafb3307e5122d3067 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 6 Dec 2021 13:12:00 +0000 Subject: [PATCH 194/197] add doc strings --- nowcasting_dataset/manager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nowcasting_dataset/manager.py b/nowcasting_dataset/manager.py index c3f34531..a2d20274 100644 --- a/nowcasting_dataset/manager.py +++ b/nowcasting_dataset/manager.py @@ -478,14 +478,18 @@ def create_batches(self, overwrite_batches: bool) -> None: # Logger messages for callbacks: def _callback(result): + """Create callback for 'pool.apply_async'""" logger.info( f"{data_source_name} has finished created batches for {split_name}!" ) def _error_callback(exception, data_source_name): - # Need to pass in data_source_name rather than rely on data_source_name - # in the outer scope, because otherwise the error message will contain - # the wrong data_source_name (due to stuff happening concurrently!) + """Create error callback for 'pool.apply_async' + + Need to pass in data_source_name rather than rely on data_source_name + in the outer scope, because otherwise the error message will contain + the wrong data_source_name (due to stuff happening concurrently!) + """ logger.exception( f"Exception raised by {data_source_name} whilst creating batches for" f" {split_name.value}\n{exception.__class__.__name__}: {exception}" From c7c06604edc57bb20b810d7da82c8c590cf684f8 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 6 Dec 2021 13:16:31 +0000 Subject: [PATCH 195/197] refactor functions - just move code to separate files --- .../optical_flow/format_images.py | 67 +++++++++++++ .../optical_flow/optical_flow_data_source.py | 93 +------------------ nowcasting_dataset/dataset/xr_utils.py | 25 +++++ 3 files changed, 95 insertions(+), 90 deletions(-) create mode 100644 nowcasting_dataset/data_sources/optical_flow/format_images.py diff --git a/nowcasting_dataset/data_sources/optical_flow/format_images.py b/nowcasting_dataset/data_sources/optical_flow/format_images.py new file mode 100644 index 00000000..df8e012a --- /dev/null +++ b/nowcasting_dataset/data_sources/optical_flow/format_images.py @@ -0,0 +1,67 @@ +""" Functions that format images """ +import cv2 +import numpy as np + + +def remap_image( + image: np.ndarray, + flow: np.ndarray, + border_mode: int = cv2.BORDER_REPLICATE, +) -> np.ndarray: + """ + Takes an image and warps it forwards in time according to the flow field. + + Args: + image: The grayscale image to warp. + flow: A 3D array. The first two dimensions must be the same size as the first two + dimensions of the image. The third dimension represented the x and y displacement. + border_mode: One of cv2's BorderTypes such as cv2.BORDER_CONSTANT or cv2.BORDER_REPLICATE. + If border_mode=cv2.BORDER_CONSTANT then the border will be set to -1. + For details of other border_mode settings, see the Open CV docs here: + docs.opencv.org/4.5.4/d2/de8/group__core__array.html#ga209f2f4869e304c82d07739337eae7c5 + + Returns: Warped image. + """ + # Adapted from https://github.com/opencv/opencv/issues/11068 + height, width = flow.shape[:2] + remap = -flow.copy() + remap[..., 0] += np.arange(width) # map_x + remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y + # remap docs: + # docs.opencv.org/4.5.4/da/d54/group__imgproc__transform.html#gab75ef31ce5cdfb5c44b6da5f3b908ea4 + # TODO: Maybe use integer remap: docs say that might be faster? + remapped_image = cv2.remap( + src=image, + map1=remap, + map2=None, + interpolation=cv2.INTER_LINEAR, + borderMode=border_mode, + borderValue=-1, + ) + return remapped_image + + +def crop_center(image: np.ndarray, output_image_size_pixels: int) -> np.ndarray: + """ + Crop center of a 2D numpy image. + + Args: + image: The input image to crop. + output_image_size_pixels: The requested size of the output image. + + Returns: + The cropped image, of size output_image_size_pixels x output_image_size_pixels + """ + input_size_y, input_size_x = image.shape + assert ( + input_size_x >= output_image_size_pixels + ), "output_image_size_pixels is larger than the input image!" + assert ( + input_size_y >= output_image_size_pixels + ), "output_image_size_pixels is larger than the input image!" + half_output_image_size_pixels = output_image_size_pixels // 2 + start_x = (input_size_x // 2) - half_output_image_size_pixels + start_y = (input_size_y // 2) - half_output_image_size_pixels + end_x = start_x + output_image_size_pixels + end_y = start_y + output_image_size_pixels + return image[start_y:end_y, start_x:end_x] diff --git a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py index d63280cf..71c1eb2b 100644 --- a/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py +++ b/nowcasting_dataset/data_sources/optical_flow/optical_flow_data_source.py @@ -12,7 +12,9 @@ import nowcasting_dataset.filesystem.utils as nd_fs_utils from nowcasting_dataset.data_sources import DataSource +from nowcasting_dataset.data_sources.optical_flow.format_images import crop_center, remap_image from nowcasting_dataset.data_sources.optical_flow.optical_flow_model import OpticalFlow +from nowcasting_dataset.dataset.xr_utils import convert_arrays_to_uint8 _LOG = logging.getLogger(__name__) @@ -225,31 +227,6 @@ def check_input_paths_exist(self) -> None: nd_fs_utils.check_path_exists(self.zarr_path) -def _convert_arrays_to_uint8(*arrays: tuple[np.ndarray]) -> tuple[np.ndarray]: - """Convert multiple arrays to uint8, using the same min and max to scale all arrays.""" - # First, stack into a single numpy array so we can work on all images at the same time: - stacked = np.stack(arrays) - - # Convert to float64 for normalisation: - stacked = stacked.astype(np.float64) - - # Rescale pixel values to be in the range [0, 1]: - stacked -= stacked.min() - stacked_max = stacked.max() - if stacked_max > 0.0: - # If there is still an invalid value then we want to know about it! - # Adapted from https://stackoverflow.com/a/33701974/732596 - with np.errstate(all="raise"): - stacked /= stacked.max() - - # Convert to uint8 (uint8 can represent integers in the range [0, 255]): - stacked *= 255 - stacked = stacked.round() - stacked = stacked.astype(np.uint8) - - return tuple(stacked) - - def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.ndarray: """ Compute the optical flow for a set of images @@ -267,7 +244,7 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n assert prev_image.dtype == next_image.dtype, "Images must be the same dtype!" # cv2.calcOpticalFlowFarneback expects images to be uint8: - prev_image, next_image = _convert_arrays_to_uint8(prev_image, next_image) + prev_image, next_image = convert_arrays_to_uint8(prev_image, next_image) # Docs for cv2.calcOpticalFlowFarneback: # https://docs.opencv.org/4.5.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af @@ -284,67 +261,3 @@ def compute_optical_flow(prev_image: np.ndarray, next_image: np.ndarray) -> np.n flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN, ) return flow - - -def remap_image( - image: np.ndarray, - flow: np.ndarray, - border_mode: int = cv2.BORDER_REPLICATE, -) -> np.ndarray: - """ - Takes an image and warps it forwards in time according to the flow field. - - Args: - image: The grayscale image to warp. - flow: A 3D array. The first two dimensions must be the same size as the first two - dimensions of the image. The third dimension represented the x and y displacement. - border_mode: One of cv2's BorderTypes such as cv2.BORDER_CONSTANT or cv2.BORDER_REPLICATE. - If border_mode=cv2.BORDER_CONSTANT then the border will be set to -1. - For details of other border_mode settings, see the Open CV docs here: - docs.opencv.org/4.5.4/d2/de8/group__core__array.html#ga209f2f4869e304c82d07739337eae7c5 - - Returns: Warped image. - """ - # Adapted from https://github.com/opencv/opencv/issues/11068 - height, width = flow.shape[:2] - remap = -flow.copy() - remap[..., 0] += np.arange(width) # map_x - remap[..., 1] += np.arange(height)[:, np.newaxis] # map_y - # remap docs: - # docs.opencv.org/4.5.4/da/d54/group__imgproc__transform.html#gab75ef31ce5cdfb5c44b6da5f3b908ea4 - # TODO: Maybe use integer remap: docs say that might be faster? - remapped_image = cv2.remap( - src=image, - map1=remap, - map2=None, - interpolation=cv2.INTER_LINEAR, - borderMode=border_mode, - borderValue=-1, - ) - return remapped_image - - -def crop_center(image: np.ndarray, output_image_size_pixels: int) -> np.ndarray: - """ - Crop center of a 2D numpy image. - - Args: - image: The input image to crop. - output_image_size_pixels: The requested size of the output image. - - Returns: - The cropped image, of size output_image_size_pixels x output_image_size_pixels - """ - input_size_y, input_size_x = image.shape - assert ( - input_size_x >= output_image_size_pixels - ), "output_image_size_pixels is larger than the input image!" - assert ( - input_size_y >= output_image_size_pixels - ), "output_image_size_pixels is larger than the input image!" - half_output_image_size_pixels = output_image_size_pixels // 2 - start_x = (input_size_x // 2) - half_output_image_size_pixels - start_y = (input_size_y // 2) - half_output_image_size_pixels - end_x = start_x + output_image_size_pixels - end_y = start_y + output_image_size_pixels - return image[start_y:end_y, start_x:end_x] diff --git a/nowcasting_dataset/dataset/xr_utils.py b/nowcasting_dataset/dataset/xr_utils.py index 79c7c75b..def4a11e 100644 --- a/nowcasting_dataset/dataset/xr_utils.py +++ b/nowcasting_dataset/dataset/xr_utils.py @@ -123,3 +123,28 @@ def validate_data_vars(cls, v: Any) -> Any: data_var in data_var_names ), f"{data_var} is not in all data_vars ({data_var_names}) in {cls.__name__}!" return v + + +def convert_arrays_to_uint8(*arrays: tuple[np.ndarray]) -> tuple[np.ndarray]: + """Convert multiple arrays to uint8, using the same min and max to scale all arrays.""" + # First, stack into a single numpy array so we can work on all images at the same time: + stacked = np.stack(arrays) + + # Convert to float64 for normalisation: + stacked = stacked.astype(np.float64) + + # Rescale pixel values to be in the range [0, 1]: + stacked -= stacked.min() + stacked_max = stacked.max() + if stacked_max > 0.0: + # If there is still an invalid value then we want to know about it! + # Adapted from https://stackoverflow.com/a/33701974/732596 + with np.errstate(all="raise"): + stacked /= stacked.max() + + # Convert to uint8 (uint8 can represent integers in the range [0, 255]): + stacked *= 255 + stacked = stacked.round() + stacked = stacked.astype(np.uint8) + + return tuple(stacked) From f5933ec01efe2e4954a57705c7c6837f39747b42 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 6 Dec 2021 13:25:13 +0000 Subject: [PATCH 196/197] reduce pydantic opticalflow model - DataSourceMixin --- nowcasting_dataset/config/gcp.yaml | 4 ++-- nowcasting_dataset/config/model.py | 22 ++++++++++------------ nowcasting_dataset/config/on_premises.yaml | 4 ++-- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/nowcasting_dataset/config/gcp.yaml b/nowcasting_dataset/config/gcp.yaml index 54e68069..8e74ba9d 100644 --- a/nowcasting_dataset/config/gcp.yaml +++ b/nowcasting_dataset/config/gcp.yaml @@ -76,8 +76,8 @@ input_data: # ------------------------- Optical Flow --------------- opticalflow: opticalflow_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr - opticalflow_history_minutes: 5 - opticalflow_forecast_minutes: 120 + history_minutes: 5 + forecast_minutes: 120 opticalflow_input_image_size_pixels: 102 opticalflow_output_image_size_pixels: 24 opticalflow_source_data_source_class_name: SatelliteDataSource diff --git a/nowcasting_dataset/config/model.py b/nowcasting_dataset/config/model.py index abf3f6fa..818c2e15 100644 --- a/nowcasting_dataset/config/model.py +++ b/nowcasting_dataset/config/model.py @@ -202,18 +202,16 @@ class OpticalFlow(DataSourceMixin): " satellite.satellite_zarr_path." ), ) - opticalflow_history_minutes: int = Field( - 5, - description=( - "Duration of historical data to use when computing the optical flow field." - " For example, set to 5 to use just two images: the t-1 and t0 images. Set to 10 to" - " compute the optical flow field separately for the image pairs (t-2, t-1), and" - " (t-1, t0) and to use the mean optical flow field." - ), - ) - opticalflow_forecast_minutes: int = Field( - 120, description="Duration of the optical flow predictions." - ) + + # history_minutes, set in DataSourceMixin. + # Duration of historical data to use when computing the optical flow field. + # For example, set to 5 to use just two images: the t-1 and t0 images. Set to 10 to + # compute the optical flow field separately for the image pairs (t-2, t-1), and + # (t-1, t0) and to use the mean optical flow field. + + # forecast_minutes, set in DataSourceMixin. + # Duration of the optical flow predictions. + opticalflow_meters_per_pixel: int = METERS_PER_PIXEL_FIELD opticalflow_input_image_size_pixels: int = Field( IMAGE_SIZE_PIXELS * 2, diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml index 7daee2bc..e239c25a 100644 --- a/nowcasting_dataset/config/on_premises.yaml +++ b/nowcasting_dataset/config/on_premises.yaml @@ -66,8 +66,8 @@ input_data: # ------------------------- Optical Flow --------------- opticalflow: opticalflow_zarr_path: /mnt/storage_ssd_8tb/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/zarr/v2/eumetsat_zarr_* - opticalflow_history_minutes: 5 - opticalflow_forecast_minutes: 120 + history_minutes: 5 + forecast_minutes: 120 opticalflow_input_image_size_pixels: 102 opticalflow_output_image_size_pixels: 24 opticalflow_source_data_source_class_name: SatelliteDataSource From bd250214334f7d4ac584b6679f8f46964107b999 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 6 Dec 2021 14:03:09 +0000 Subject: [PATCH 197/197] update --- nowcasting_dataset/filesystem/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nowcasting_dataset/filesystem/utils.py b/nowcasting_dataset/filesystem/utils.py index bbccccf4..e5b05f1f 100644 --- a/nowcasting_dataset/filesystem/utils.py +++ b/nowcasting_dataset/filesystem/utils.py @@ -97,7 +97,7 @@ def check_path_exists(path: Union[str, Path]): `path` can include wildcards. """ - if not bool(path): + if not path: raise FileNotFoundError("Not a valid path!") filesystem = get_filesystem(path) if not filesystem.exists(path):