From d7083b8ca48d5000cd1364a599a2c5ced2e89b35 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 16 Nov 2022 10:43:31 +1100 Subject: [PATCH 01/10] Do range searches with '[]' bounds. --- datacube/drivers/postgres/_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacube/drivers/postgres/_fields.py b/datacube/drivers/postgres/_fields.py index 26d880aff8..d16b4a9623 100755 --- a/datacube/drivers/postgres/_fields.py +++ b/datacube/drivers/postgres/_fields.py @@ -491,7 +491,7 @@ def __init__(self, field, low_value, high_value, _range_class): @property def alchemy_expression(self): return self.field.alchemy_expression.overlaps( - self._range_class(self.low_value, self.high_value) + self._range_class(self.low_value, self.high_value, bounds="[]") ) From 6d4b167d20a5b03474756ec8526921fd302a6327 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 16 Nov 2022 11:53:45 +1100 Subject: [PATCH 02/10] and for experimental driver. --- datacube/drivers/postgis/_fields.py | 2 +- integration_tests/index/test_config_docs.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datacube/drivers/postgis/_fields.py b/datacube/drivers/postgis/_fields.py index 7ac80845f6..140cd709cd 100755 --- a/datacube/drivers/postgis/_fields.py +++ b/datacube/drivers/postgis/_fields.py @@ -491,7 +491,7 @@ def __init__(self, field, low_value, high_value, _range_class): @property def alchemy_expression(self): return self.field.alchemy_expression.overlaps( - self._range_class(self.low_value, self.high_value) + self._range_class(self.low_value, self.high_value, bounds='[]') ) diff --git a/integration_tests/index/test_config_docs.py b/integration_tests/index/test_config_docs.py index 328aab35bf..79f9ec4655 100644 --- a/integration_tests/index/test_config_docs.py +++ b/integration_tests/index/test_config_docs.py @@ -159,6 +159,7 @@ def test_field_expression_unchanged_postgis( default_metadata_type: MetadataType, telemetry_metadata_type: MetadataType) -> None: # We're checking for accidental changes here in our field-to-SQL code + # Dubious test as it uses non-EO3 metadata types # If we started outputting a different expression they would quietly no longer match the expression # indexes that exist in our DBs. From 75670ee3b2568853c76fc5b19e0ad8f1436dfc0d Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 16 Nov 2022 11:54:08 +1100 Subject: [PATCH 03/10] Tests. --- integration_tests/data/eo3/ls8_dataset4.yaml | 2 -- integration_tests/index/test_search_eo3.py | 21 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/integration_tests/data/eo3/ls8_dataset4.yaml b/integration_tests/data/eo3/ls8_dataset4.yaml index 333745c7ab..7fb0dcf805 100644 --- a/integration_tests/data/eo3/ls8_dataset4.yaml +++ b/integration_tests/data/eo3/ls8_dataset4.yaml @@ -36,8 +36,6 @@ grids: properties: datetime: '2013-07-21T00:57:26.432563Z' dea:dataset_maturity: final - dtr:end_datetime: '2013-07-21T00:57:41.398421Z' - dtr:start_datetime: '2013-07-21T00:57:11.356786Z' eo:cloud_cover: 1.828773330949106e+01 eo:gsd: 1.5e+01 # Ground sample distance (m) eo:instrument: OLI_TIRS diff --git a/integration_tests/index/test_search_eo3.py b/integration_tests/index/test_search_eo3.py index 9568a7a520..12fd64679b 100644 --- a/integration_tests/index/test_search_eo3.py +++ b/integration_tests/index/test_search_eo3.py @@ -151,6 +151,27 @@ def test_search_dataset_ranges_eo3(index: Index, ls8_eo3_dataset: Dataset) -> No assert len(datasets) == 0 +def test_zero_width_range_search(index: Index, ls8_eo3_dataset4: Dataset) -> None: + # Test time search against zero-width time metadata + datasets = index.datasets.search_eager(time=Range( + begin=datetime.datetime(2013, 7, 21, 0, 57, 26, 432563, tzinfo=datetime.timezone.utc), + end=datetime.datetime(2013, 7, 21, 0, 57, 26, 432563, tzinfo=datetime.timezone.utc) + )) + assert len(datasets) == 1 + + datasets = index.datasets.search_eager(time=Range( + begin=datetime.datetime(2013, 7, 21, 0, 57, 26, 432563, tzinfo=datetime.timezone.utc), + end=datetime.datetime(2013, 7, 21, 0, 57, 27, 432563, tzinfo=datetime.timezone.utc) + )) + assert len(datasets) == 1 + + datasets = index.datasets.search_eager(time=Range( + begin=datetime.datetime(2013, 7, 21, 0, 57, 25, 432563, tzinfo=datetime.timezone.utc), + end=datetime.datetime(2013, 7, 21, 0, 57, 26, 432563, tzinfo=datetime.timezone.utc) + )) + assert len(datasets) == 1 + + def test_search_globally_eo3(index: Index, ls8_eo3_dataset: Dataset) -> None: # No expressions means get all. results = list(index.datasets.search()) From e538726aa380f1fef5f511777c4c120ac9f87ad6 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 16 Nov 2022 12:04:09 +1100 Subject: [PATCH 04/10] Update whats_new.rst. --- docs/about/whats_new.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index fa6596bee6..16aae585ab 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -20,6 +20,7 @@ v1.8.next - Add readthedoc preview github action (:pull:`1344`) - Update `nodata` in readthedoc for products page (:pull:`1347`) - Add `eo-datasets` to extensions & related software doc page (:pull:`1349`) +- Fix bug affecting searches against range types of zero width (:pull:`1352`) .. _PEP561: https://peps.python.org/pep-0561/ From 03f4864be39d5d58aeb0aa635007a253e895730d Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 17 Nov 2022 11:05:44 +1100 Subject: [PATCH 05/10] Add 1.8.9 release date and add missing PR to whats_new.rst --- docs/about/whats_new.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index 16aae585ab..6588f4321e 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -8,12 +8,16 @@ What's New v1.8.next ========= +v1.8.9 (17 November 2022) +========================= + - Performance improvements to CRS geometry class (:pull:`1322`) - Extend `patch_url` argument to `dc.load()` and `dc.load_data()` to Dask loading. (:pull:`1323`) - Add `sphinx.ext.autoselectionlabel` extension to readthedoc conf to support `:ref:` command (:pull:`1325`) - Add `pyspellcheck` for `.rst` documentation files and fix typos (:pull:`1327`) - Add `rst` documentation lint github action and apply best practices (:pull:`1328`) -- Follow PEP561_ to make type hints available to other packages (:issue:`1330`) +- Follow PEP561_ to make type hints available to other packages (:pull:`1331`) +- Updated GitHub actions config to remove deprecated `set-output` (:pull:`1333`) - Add what's new page link to menu and general doc fixes (:pull:`1335`) - Add `search_fields` to required for metadata type schema and update doc (:pull:`1339`) - Fix typo and update metadata documentation (:pull:`1340`) @@ -21,6 +25,7 @@ v1.8.next - Update `nodata` in readthedoc for products page (:pull:`1347`) - Add `eo-datasets` to extensions & related software doc page (:pull:`1349`) - Fix bug affecting searches against range types of zero width (:pull:`1352`) +- Add 1.8.9 release date and missing PR to `whats_news.rst` (:pull:`1353`) .. _PEP561: https://peps.python.org/pep-0561/ From a929a9bb19f3085c1b5724243f1064f59d1c26c4 Mon Sep 17 00:00:00 2001 From: "pin.jin@ga.gov.au" Date: Thu, 17 Nov 2022 04:29:13 +0000 Subject: [PATCH 06/10] add grid spec to list_product --- datacube/api/core.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/datacube/api/core.py b/datacube/api/core.py index 6fb2ff5a4b..aa18b2dad5 100644 --- a/datacube/api/core.py +++ b/datacube/api/core.py @@ -100,6 +100,7 @@ def list_products(self, with_pandas=True, dataset_count=False): 'license' 'default_crs' 'default_resolution' + 'grid_spec' 'dataset_count' (optional) :param bool with_pandas: @@ -120,6 +121,7 @@ def list_products(self, with_pandas=True, dataset_count=False): 'license', 'default_crs', 'default_resolution', + 'grid_spec', ] rows = [[getattr(pr, col, None) for col in cols] for pr in self.index.products.get_all()] @@ -870,7 +872,10 @@ def output_geobox(like=None, output_crs=None, resolution=None, align=None, resolution = grid_spec.resolution align = align or grid_spec.alignment else: - raise ValueError("Product has no default CRS. Must specify 'output_crs' and 'resolution'") + raise ValueError( + "Product has no default CRS nor grid_spec. \n" + "Must specify 'output_crs' and 'resolution' or 'grid_spec'" + ) # Try figuring out bounds # 1. Explicitly defined with geopolygon From ec7de308c95fe8599722368558076d44fb01a502 Mon Sep 17 00:00:00 2001 From: "pin.jin@ga.gov.au" Date: Thu, 17 Nov 2022 04:35:03 +0000 Subject: [PATCH 07/10] update whats new --- docs/about/whats_new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index 6588f4321e..0effe3acee 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -8,6 +8,8 @@ What's New v1.8.next ========= +- Add `grid_spec` to `list_products` (:pull:`1357`) + v1.8.9 (17 November 2022) ========================= From af9bf443ba111167f17bd7c450e8fd5fab2824c2 Mon Sep 17 00:00:00 2001 From: "pin.jin@ga.gov.au" Date: Thu, 17 Nov 2022 05:01:31 +0000 Subject: [PATCH 08/10] replace default with grid --- datacube/api/core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/datacube/api/core.py b/datacube/api/core.py index aa18b2dad5..f588501b7b 100644 --- a/datacube/api/core.py +++ b/datacube/api/core.py @@ -100,7 +100,6 @@ def list_products(self, with_pandas=True, dataset_count=False): 'license' 'default_crs' 'default_resolution' - 'grid_spec' 'dataset_count' (optional) :param bool with_pandas: @@ -121,10 +120,13 @@ def list_products(self, with_pandas=True, dataset_count=False): 'license', 'default_crs', 'default_resolution', - 'grid_spec', ] - rows = [[getattr(pr, col, None) for col in cols] - for pr in self.index.products.get_all()] + rows = [[ + getattr(pr, col, None) + if getattr(pr, col, None) and 'default' not in col + else getattr(pr.grid_spec, col.replace('default_', ''), None) + for col in cols] + for pr in self.index.products.get_all()] # Optionally compute dataset count for each product and add to row/cols # Product lists are sorted by product name to ensure 1:1 match From 7efec75274dd870e8066050a086727c2fb1f6090 Mon Sep 17 00:00:00 2001 From: "pin.jin@ga.gov.au" Date: Thu, 17 Nov 2022 05:01:59 +0000 Subject: [PATCH 09/10] revert valueerror message --- datacube/api/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datacube/api/core.py b/datacube/api/core.py index f588501b7b..612a7cc963 100644 --- a/datacube/api/core.py +++ b/datacube/api/core.py @@ -875,8 +875,8 @@ def output_geobox(like=None, output_crs=None, resolution=None, align=None, align = align or grid_spec.alignment else: raise ValueError( - "Product has no default CRS nor grid_spec. \n" - "Must specify 'output_crs' and 'resolution' or 'grid_spec'" + "Product has no default CRS. \n" + "Must specify 'output_crs' and 'resolution'" ) # Try figuring out bounds From 88e51b36e62289f817ee92d95f90c379f5382a7f Mon Sep 17 00:00:00 2001 From: "pin.jin@ga.gov.au" Date: Thu, 17 Nov 2022 05:44:59 +0000 Subject: [PATCH 10/10] update comment --- datacube/api/core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datacube/api/core.py b/datacube/api/core.py index 612a7cc963..9e358a6b87 100644 --- a/datacube/api/core.py +++ b/datacube/api/core.py @@ -98,8 +98,8 @@ def list_products(self, with_pandas=True, dataset_count=False): 'name' 'description' 'license' - 'default_crs' - 'default_resolution' + 'default_crs' or 'grid_spec.crs' + 'default_resolution' or 'grid_spec.crs' 'dataset_count' (optional) :param bool with_pandas: @@ -123,7 +123,11 @@ def list_products(self, with_pandas=True, dataset_count=False): ] rows = [[ getattr(pr, col, None) + # if 'default_crs' and 'default_resolution' are not None + # return 'default_crs' and 'default_resolution' if getattr(pr, col, None) and 'default' not in col + # else try 'grid_spec.crs' and 'grid_spec.resolution' + # as per output_geobox() handling logic else getattr(pr.grid_spec, col.replace('default_', ''), None) for col in cols] for pr in self.index.products.get_all()]