diff --git a/datacube_wms/data.py b/datacube_wms/data.py index f1ed2b29..7b18d92e 100644 --- a/datacube_wms/data.py +++ b/datacube_wms/data.py @@ -18,7 +18,7 @@ from datacube_wms.cube_pool import cube -from datacube_wms.wms_layers import get_service_cfg +from datacube_wms.wms_layers import get_service_cfg, TIMERES_RAW, TIMERES_YR, TIMERES_MON from datacube_wms.wms_utils import img_coords_to_geopoint , GetMapParameters, \ GetFeatureInfoParameters, solar_correct_data from datacube_wms.ogc_utils import resp_headers, local_solar_date_range, local_date, dataset_center_time, \ @@ -34,22 +34,6 @@ tracer = get_opencensus_tracer() -# Read data for given datasets and measurements per the output_geobox -@log_call -@opencensus_trace_call(tracer=tracer) -def read_data(datasets, measurements, geobox, resampling=Resampling.nearest, **kwargs): - if not hasattr(datasets, "__iter__"): - datasets = [datasets] - - datasets = datacube.Datacube.group_datasets(datasets, 'solar_day') - data = datacube.Datacube.load_data( - datasets, - geobox, - measurements=measurements, - fuse_func=kwargs.get('fuse_func', None)) - # maintain compatibility with functions not expecting the data to have a time dimension - return data.squeeze(dim='time', drop=True) - class DataStacker(): @log_call @@ -64,7 +48,13 @@ def __init__(self, product, geobox, time, resampling=None, style=None, bands=Non self._needed_bands = [ self._product.band_idx.band(b) for b in bands ] else: self._needed_bands = self._product.band_idx.native_bands.index - self._time = local_solar_date_range(geobox, time) + + if self._product.time_resolution == TIMERES_MON: + self._time = time + elif self._product.time_resolution == TIMERES_YR: + self._time = str(time.year) + else: + self._time = local_solar_date_range(geobox, time) def needed_bands(self): return self._needed_bands @@ -128,7 +118,7 @@ def data(self, datasets, mask=False, manual_merge=False, skip_corrections=False, # process the data for the datasets individually to do solar correction. merged = None for ds in datasets: - d = read_data(ds, measurements, self._geobox, **kwargs) + d = self.read_data(ds, measurements, self._geobox, **kwargs) for band in self.needed_bands(): if band != self._product.pq_band: d[band] = solar_correct_data(d[band], ds) @@ -138,7 +128,7 @@ def data(self, datasets, mask=False, manual_merge=False, skip_corrections=False, merged = merged.combine_first(d) return merged else: - data = read_data(datasets, measurements, self._geobox, self._resampling, **kwargs) + data = self.read_data(datasets, measurements, self._geobox, self._resampling, **kwargs) return data @log_call @@ -152,7 +142,7 @@ def manual_data_stack(self, datasets, measurements, mask, skip_corrections, **kw else: bands = self.needed_bands() for ds in datasets: - d = read_data(ds, measurements, self._geobox, **kwargs) + d = self.read_data(ds, measurements, self._geobox, **kwargs) extent_mask = None for band in bands: for f in self._product.extent_mask_func: @@ -175,6 +165,26 @@ def manual_data_stack(self, datasets, measurements, mask, skip_corrections, **kw merged[band].attrs = d[band].attrs return merged + # Read data for given datasets and measurements per the output_geobox + @log_call + @opencensus_trace_call(tracer=tracer) + def read_data(self, datasets, measurements, geobox, resampling=Resampling.nearest, **kwargs): + if not hasattr(datasets, "__iter__"): + datasets = [datasets] + + if self._product.time_resolution == TIMERES_RAW: + datasets = datacube.Datacube.group_datasets(datasets, 'solar_day') + else: + datasets = datacube.Datacube.group_datasets(datasets, 'time') + data = datacube.Datacube.load_data( + datasets, + geobox, + measurements=measurements, + fuse_func=kwargs.get('fuse_func', None)) + # maintain compatibility with functions not expecting the data to have a time dimension + return data.squeeze(dim='time', drop=True) + + def bbox_to_geom(bbox, crs): return datacube.utils.geometry.box(bbox.left, bbox.bottom, bbox.right, bbox.top, crs) diff --git a/datacube_wms/ogc_utils.py b/datacube_wms/ogc_utils.py index ebbcc408..dd3b90e8 100644 --- a/datacube_wms/ogc_utils.py +++ b/datacube_wms/ogc_utils.py @@ -22,7 +22,7 @@ tf = TimezoneFinder(in_memory=True) # Use metadata time if possible as this is what WMS uses to calculate it's temporal extents -# datacube-core center time accessed through the dataset API is caluclated and may +# datacube-core center time accessed through the dataset API is calculated and may # not agree with the metadata document def dataset_center_time(dataset): center_time = dataset.center_time @@ -30,7 +30,11 @@ def dataset_center_time(dataset): metadata_time = dataset.metadata_doc['extent']['center_dt'] center_time = parse(metadata_time) except KeyError: - pass + try: + metadata_time = dataset.metadata_doc['properties']['dtr:start_datetime'] + center_time = parse(metadata_time) + except KeyError: + pass return center_time diff --git a/datacube_wms/wms_cfg_example.py b/datacube_wms/wms_cfg_example.py index f537d27c..0a5ca0d0 100644 --- a/datacube_wms/wms_cfg_example.py +++ b/datacube_wms/wms_cfg_example.py @@ -293,6 +293,19 @@ # Bands to always fetch from the Datacube, even if it is not used by the active style. # Useful for when a particular band is always needed for the extent_mask_func, "always_fetch_bands": [ ], + # Time resolution of the product. Controls the way ODC time query parameters are generated. + # + # Defaults to "raw". Supported values are: + # + # "raw": The default. For sub-day time resolution, e.g. raw (unsummarised) EO data. + # Also works fine for DEA packaged summary data where the "from" and "to" dates + # are both set to the start of the summary time period. + # + # "month": + # "year" : For data summarised to monthly or yearly resolution respectively. + # Needed for e.g. summary data with EO3-style metadata with the "from" and "to" + # dates set to the start and end of the summary time period. + "time_resolution": "raw", # Apply corrections for solar angle, for "Level 1" products. # (Defaults to false - should not be used for NBAR/NBAR-T or other Analysis Ready products "apply_solar_corrections": False, diff --git a/datacube_wms/wms_layers.py b/datacube_wms/wms_layers.py index efa306ca..1872b0e5 100644 --- a/datacube_wms/wms_layers.py +++ b/datacube_wms/wms_layers.py @@ -122,6 +122,13 @@ def __init__(self, cfg): self.format = cfg["format"] +TIMERES_RAW = "raw" +TIMERES_MON = "month" +TIMERES_YR = "year" + +TIMERES_VALS = [ TIMERES_RAW, TIMERES_MON, TIMERES_YR ] + + class ProductLayerDef(object): # pylint: disable=invalid-name, too-many-instance-attributes, bare-except, too-many-statements def __init__(self, product_cfg, platform_def, dc): @@ -170,6 +177,10 @@ def __init__(self, product_cfg, platform_def, dc): self.pq_name = self.pq_names[0] if self.pq_names is not None and len(self.pq_names) > 0 else None self.pq_band = product_cfg.get("pq_band") + self.time_resolution = product_cfg.get("time_resolution", TIMERES_RAW) + if self.time_resolution not in TIMERES_VALS: + raise ProductLayerException("Invalid time resolution value: %s" % self.time_resolution) + self.min_zoom = product_cfg.get("min_zoom_factor", 300.0) self.max_datasets_wms = product_cfg.get("max_datasets_wms", 0) self.zoom_fill = product_cfg.get("zoomed_out_fill_colour", [150, 180, 200]) diff --git a/update_ranges.py b/update_ranges.py index dc398eed..ee1f9d30 100755 --- a/update_ranges.py +++ b/update_ranges.py @@ -3,6 +3,7 @@ from datacube_wms.product_ranges import update_all_ranges, get_sqlconn, add_product_range, add_multiproduct_range, add_all, update_range from datacube import Datacube from psycopg2.sql import SQL, Identifier +import os import click @click.command() @@ -29,6 +30,11 @@ def main(product, multiproduct, merge_only, calculate_extent, schema, role): print("Sorry, cannot update schema without specifying a role") return 1 + if os.environ.get("PYDEV_DEBUG"): + import pydevd_pycharm + pydevd_pycharm.settrace('172.17.0.1', port=12321, stdoutToServer=True, stderrToServer=True) + + dc = Datacube(app="wms_update_ranges") if schema: print("Checking schema....")