From 8a9938a567dc0c1e2853577e86daf435d7ce5176 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 1 Aug 2023 13:35:32 +1000 Subject: [PATCH 01/15] Formalised spatial index API. --- datacube/index/abstract.py | 31 +++++++++++++++++++++++++++++++ datacube/index/postgis/index.py | 2 ++ 2 files changed, 33 insertions(+) diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 0c4b56aaa..33ddd8c42 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -1862,6 +1862,8 @@ class AbstractIndex(ABC): supports_external_home = False # Supports ACID transactions supports_transactions = False + # Supports per-CRS spatial indexes + supports_spatial_indexes = False @property @abstractmethod @@ -1921,6 +1923,30 @@ def init_db(self, :return: true if the database was created, false if already exists """ + # Spatial Index API + + def create_spatial_index(self, crs: CRS) -> bool: + if not self.supports_spatial_indexes: + raise NotImplementedError("This index driver does not support the Spatial Index API") + else: + raise NotImplementedError() + + def spatial_indexes(self, refresh=False) -> Iterable[CRS]: + if not self.supports_spatial_indexes: + raise NotImplementedError("This index driver does not support the Spatial Index API") + else: + raise NotImplementedError() + + def update_spatial_index(self, + crses: Sequence[CRS] = [], + product_names: Sequence[str] = [], + dataset_ids: Sequence[DSID] = [] + ) -> int: + if not self.supports_spatial_indexes: + raise NotImplementedError("This index driver does not support the Spatial Index API") + else: + raise NotImplementedError() + def clone(self, origin_index: "AbstractIndex", batch_size: int = 1000, @@ -1957,6 +1983,11 @@ def clone(self, """ results = {} if not lineage_only: + if self.supports_spatial_indexes and origin_index.supports_spatial_indexes: + for crs in origin_index.spatial_indexes(refresh=True): + report_to_user(f"Creating spatial index for CRS {crs}") + self.create_spatial_index(crs) + self.update_spatial_index(crs) # Clone Metadata Types report_to_user("Cloning Metadata Types:") results["metadata_types"] = self.metadata_types.bulk_add(origin_index.metadata_types.get_all_docs(), diff --git a/datacube/index/postgis/index.py b/datacube/index/postgis/index.py index 1de718bd2..cfd9a352b 100644 --- a/datacube/index/postgis/index.py +++ b/datacube/index/postgis/index.py @@ -58,6 +58,8 @@ class Index(AbstractIndex): supports_external_home = True # Postgis driver supports ACID database transactions supports_transactions = True + # Postgis supports per-CRS spatial indexes + supports_spatial_indexes = True def __init__(self, db: PostGisDb) -> None: # POSTGIS driver is not stable with respect to database schema or internal APIs. From f3cf85970a952f2a3a4783015d2fab16d465630c Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 2 Aug 2023 15:33:03 +1000 Subject: [PATCH 02/15] API for dropping spatial indexes. --- datacube/drivers/postgis/_api.py | 2 +- datacube/drivers/postgis/_connections.py | 18 +++++++- datacube/drivers/postgis/_spatial.py | 29 +++++++++++- datacube/index/abstract.py | 58 ++++++++++++++++++++++++ datacube/index/postgis/index.py | 3 ++ datacube/scripts/system.py | 5 ++ 6 files changed, 111 insertions(+), 4 deletions(-) diff --git a/datacube/drivers/postgis/_api.py b/datacube/drivers/postgis/_api.py index 6b97fc923..659ad7542 100644 --- a/datacube/drivers/postgis/_api.py +++ b/datacube/drivers/postgis/_api.py @@ -219,7 +219,7 @@ def extract_dataset_fields(ds_metadata, fields): return result -class PostgisDbAPI(object): +class PostgisDbAPI: def __init__(self, parentdb, connection): self._db = parentdb self._connection = connection diff --git a/datacube/drivers/postgis/_connections.py b/datacube/drivers/postgis/_connections.py index 3d10407c3..b99a76a92 100755 --- a/datacube/drivers/postgis/_connections.py +++ b/datacube/drivers/postgis/_connections.py @@ -30,7 +30,7 @@ from . import _api from . import _core -from ._spatial import ensure_spindex, spindexes, spindex_for_crs +from ._spatial import ensure_spindex, spindexes, spindex_for_crs, drop_spindex from ._schema import SpatialIndex _LIB_ID = 'odc-' + str(datacube.__version__) @@ -223,7 +223,7 @@ def create_spatial_index(self, crs: CRS) -> Optional[Type[SpatialIndex]]: """ Create a spatial index across the database, for the named CRS. - :param crs_str: + :param crs: :return: """ spidx = self.spindexes.get(crs) @@ -236,6 +236,20 @@ def create_spatial_index(self, crs: CRS) -> Optional[Type[SpatialIndex]]: self.spindexes[crs] = spidx return spidx + def drop_spatial_index(self, crs: CRS) -> bool: + """ + Create a spatial index across the database, for the named CRS. + + :param crs: + :return: + """ + spidx = self.spindexes.get(crs) + if spidx is None: + return False + result = drop_spindex(self._engine, spidx) + self._refresh_spindexes() + return result + def spatial_index(self, crs: CRS) -> Optional[Type[SpatialIndex]]: return self.spindexes.get(crs) diff --git a/datacube/drivers/postgis/_spatial.py b/datacube/drivers/postgis/_spatial.py index 3a2c8e1d4..3272d07e5 100644 --- a/datacube/drivers/postgis/_spatial.py +++ b/datacube/drivers/postgis/_spatial.py @@ -10,7 +10,7 @@ from threading import Lock from typing import Mapping, Optional, Type, Union -from sqlalchemy import ForeignKey, select +from sqlalchemy import ForeignKey, select, delete from sqlalchemy.dialects import postgresql as postgres from geoalchemy2 import Geometry @@ -20,6 +20,8 @@ from odc.geo import CRS, Geometry as Geom from odc.geo.geom import multipolygon, polygon +from sqlalchemy.sql.ddl import DropTable + from ._core import METADATA from .sql import SCHEMA_NAME from ._schema import orm_registry, Dataset, SpatialIndex, SpatialIndexRecord @@ -131,6 +133,31 @@ def ensure_spindex(engine: Connectable, sp_idx: Type[SpatialIndex]) -> None: return +def drop_spindex(engine: Connectable, sp_idx: Type[SpatialIndex]): + with Session(engine) as session: + results = session.execute( + select(SpatialIndexRecord).where(SpatialIndexRecord.srid == sp_idx.__tablename__[8:]) + ) + spidx_record = None + for result in results: + spidx_record = result[0] + break + record_del_result = False + if spidx_record: + result = session.execute( + delete(SpatialIndexRecord).where(SpatialIndexRecord.srid == spidx_record.srid) + ) + record_del_result = (result.rowcount == 1) + + result = session.execute( + DropTable(sp_idx.__table__, if_exists=True) + ) + drop_table_result = (result.rowcount == 1) + print(f"spindex record deleted: {record_del_result} table dropped: {drop_table_result}") + + return True + + def spindexes(engine: Connectable) -> Mapping[CRS, Type[SpatialIndex]]: """ Return a CRS-to-Spatial Index ORM class mapping for indexes that exist in a particular database. diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 33ddd8c42..07de4d556 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -1926,12 +1926,31 @@ def init_db(self, # Spatial Index API def create_spatial_index(self, crs: CRS) -> bool: + """ + Create a spatial index for a CRS. + + Note that a newly created spatial index is empty. If there are already datatsets in the index whose + extents can be safely projected into the CRS, then it is necessary to also call update_spatial_index + otherwise they will not be found by queries against that CRS. + + Only implemented by index drivers with supports_spatial_indexes set to True. + + :param crs: The coordinate reference system to create a spatial index for. + :return: True if the spatial index was successfully created (or already exists) + """ if not self.supports_spatial_indexes: raise NotImplementedError("This index driver does not support the Spatial Index API") else: raise NotImplementedError() def spatial_indexes(self, refresh=False) -> Iterable[CRS]: + """ + Return the CRSs for which spatial indexes have been created. + + :param refresh: If true, query the backend for the list of current spatial indexes. If false (the default) + a cached list of spatial index CRSs may be returned. + :return: An iterable of CRSs for which spatial indexes exist in the index + """ if not self.supports_spatial_indexes: raise NotImplementedError("This index driver does not support the Spatial Index API") else: @@ -1942,6 +1961,45 @@ def update_spatial_index(self, product_names: Sequence[str] = [], dataset_ids: Sequence[DSID] = [] ) -> int: + """ + Populate a newly created spatial index (or indexes). + + Spatial indexes are automatically populated with new datasets as they are indexed, but if there were + datasets already in the index when a new spatial index is created, or if geometries have been added or + modified outside of the ODC in a populated index (e.g. with SQL) then the spatial indexies must be + updated manually with this method. + + This is a very slow operation. The product_names and dataset_ids lists can be used to break the + operation up into chunks or allow faster updating when the spatial index is only relevant to a + small portion of the entire index. + + :param crses: A list of CRSes whose spatial indexes are to be updated. + Default is to update all spatial indexes + :param product_names: A list of product names to update the spatial indexes. + Default is to update for all products + :param dataset_ids: A list of ids of specific datasets to update in the spatial index. + Default is to update for all datasets (or all datasts in the products + in the product_names list) + :return: The number of dataset extents processed - i.e. the number of datasets updated multiplied by the + number of spatial indexes updated. + """ + if not self.supports_spatial_indexes: + raise NotImplementedError("This index driver does not support the Spatial Index API") + else: + raise NotImplementedError() + + def drop_spatial_index(self, crs:CRS) -> bool: + """ + Remove a spatial index from the database. + + Note that creating spatial indexes on an existing index is a slow and expensive operation. Do not + delete spatial indexes unless you are absolutely certain it is no longer required by any users of + this ODC index. + + :param crs: The CRS whose spatial index is to be deleted. + :return: True if the spatial index was successfully dropped. + False if spatial index could not be dropped. + """ if not self.supports_spatial_indexes: raise NotImplementedError("This index driver does not support the Spatial Index API") else: diff --git a/datacube/index/postgis/index.py b/datacube/index/postgis/index.py index cfd9a352b..3429e42da 100644 --- a/datacube/index/postgis/index.py +++ b/datacube/index/postgis/index.py @@ -157,6 +157,9 @@ def update_spatial_index(self, def __repr__(self): return "Index".format(self._db) + def drop_spatial_index(self, crs:CRS) -> bool: + return self._db.drop_spatial_index(crs) + @contextmanager def _active_connection(self, transaction: bool = False) -> PostgisDbAPI: """ diff --git a/datacube/scripts/system.py b/datacube/scripts/system.py index d76aebf97..7c04f10ad 100644 --- a/datacube/scripts/system.py +++ b/datacube/scripts/system.py @@ -133,3 +133,8 @@ def clone(index: Index, batch_size: int, skip_lineage: bool, lineage_only: bool, handle_exception('Error Connecting to Source Database: %s', e) except IndexSetupError as e: handle_exception('Source database not initialised: %s', e) + + +@system.command('spindex', help='Create and/or update a spatial index for a particular CRS.') +def spindex(): + pass From 0c5715d8e22406977d6e44d7aeb0327261d6fca7 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Fri, 4 Aug 2023 12:31:17 +1000 Subject: [PATCH 03/15] CLI for spatial indexes. --- datacube/scripts/cli_app.py | 1 + datacube/scripts/spindex.py | 176 ++++++++++++++++++++++++++++++++++++ datacube/scripts/system.py | 5 - 3 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 datacube/scripts/spindex.py diff --git a/datacube/scripts/cli_app.py b/datacube/scripts/cli_app.py index 605059c07..d3ffc1170 100644 --- a/datacube/scripts/cli_app.py +++ b/datacube/scripts/cli_app.py @@ -13,6 +13,7 @@ import datacube.scripts.dataset # noqa: F401 import datacube.scripts.product # noqa: F401 import datacube.scripts.metadata # noqa: F401 +import datacube.scripts.spindex # noqa: F401 import datacube.scripts.system # noqa: F401 import datacube.scripts.user # noqa: F401 diff --git a/datacube/scripts/spindex.py b/datacube/scripts/spindex.py new file mode 100644 index 000000000..4c0157fce --- /dev/null +++ b/datacube/scripts/spindex.py @@ -0,0 +1,176 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2023 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 +import logging +import sys +from typing import Sequence + +import click +from click import echo, style, confirm +from odc.geo import CRS +from sqlalchemy.exc import OperationalError + +import datacube +from datacube.index import Index, index_connect +from datacube.drivers.postgres._connections import IndexSetupError +from datacube.ui import click as ui +from datacube.ui.click import cli, handle_exception +from datacube.config import LocalConfig + +_LOG = logging.getLogger('datacube-system') + + +@cli.group(name='spindex', help='System commands') +def system(): + pass + + +@system.command( + 'create', + help='Create unpopulated spatial index for particular CRSes ' + '(express CRSes as EPSG numbers - e.g. 3857, not "epsg:3857").') +@click.option( + '--update/--no-update', '-u', is_flag=True, default=False, + help="Populate the spatial index after creation (slow). " + "For finer grained updating, use the 'spindex update' command" +) +@click.argument('srids', type=int, nargs=-1) +@ui.pass_index() +def create(index: Index, update: bool, srids: Sequence[int]): + if not index.supports_spatial_indexes: + echo("The active index driver does not support spatial indexes") + exit(1) + if not srids: + echo("Must supply at least one CRS to create/update") + exit(1) + + crses = [CRS(f"epsg:{srid}") for srid in srids] + confirmed = [] + failed = [] + for crs in crses: + if crs in index.spatial_indexes(): + # A spatial index for crs already exists: skip silently + confirmed.append(crs) + elif index.create_spatial_index(crs): + # Creation attempted but failed + failed.append(crs) + else: + # Creation succeeded + confirmed.append(crs) + if failed: + echo(f"Could not create spatial indexes for: {','.join(crs.epsg for crs in failed)}") + if confirmed: + echo(f"Spatial indexes created for: {','.join(crs.epsg for crs in confirmed)}") + if update and failed: + echo("Skipping update") + elif update: + result = index.update_spatial_index(confirmed) + echo(f'{result} extents checked and updated in spatial indexes') + else: + echo("Newly created spatial indexes are unpopulated - run 'datacube spindex update' before use.") + exit(int(failed)) + + +@system.command( + "list", + help="List all CRSs for which spatial indexes exist in this index" +) +@ui.pass_index() +def list_spindex(index): + for crs in index.spatial_indexes(): + echo(f'epsg:{crs.epsg}') + + +@system.command( + 'update', + help='Update a spatial index for particular CRSs ' + '(express CRSs as EPSG numbers - e.g. 3857, not "epsg:3857").') +@click.option( + '--product', '-p', multiple=True, + help="The name of a product to update the spatial index for (can be used multiple times for multiple products" +) +@click.option( + '--dataset', '-d', multiple=True, + help="The id of a dataset to update the spatial index for (can be used multiple times for multiple datasets" +) +@click.argument('srids', type=int, nargs=-1) +@ui.pass_index() +def update(index: Index, product: Sequence[str], dataset: Sequence[str], srids: Sequence[int]): + if not index.supports_spatial_indexes: + echo("The active index driver does not support spatial indexes") + exit(1) + if not update and product: + echo("Cannot pass in product names when not updating") + exit(1) + if not update and dataset: + echo("Cannot pass in dataset ids when not updating") + exit(1) + if not srids: + echo("Must supply at least one CRS to create/update") + exit(1) + + crses = [CRS(f"epsg:{srid}") for srid in srids] + for_update = [] + for crs in crses: + if crs in index.spatial_indexes(): + for_update.append(crs) + else: + echo(f"No spatial index for crs {crs.epsg} exists: skipping") + if not for_update: + echo("Nothing to update!") + exit(0) + result = index.update_spatial_index(for_update, product_names=product, dataset_ids=dataset) + echo(f'{result} extents checked and updated in spatial indexes') + exit(0) + + +@system.command( + 'drop', + help='Drop existing spatial indexes for particular CRSs ' + '(express CRSs as EPSG numbers - e.g. 3857, not "epsg:3857").') +@click.option( + '--force/--no-force', '-f', is_flag=True, default=False, + help="If set, does not ask the user to confirm deletion" +) +@click.argument('srids', type=int, nargs=-1) +@ui.pass_index() +def drop(index: Index, force: bool, srids: Sequence[int]): + if not index.supports_spatial_indexes: + echo("The active index driver does not support spatial indexes") + exit(1) + if not srids: + echo("Must supply at least one CRS to drop") + exit(1) + crses = [CRS(f"epsg:{srid}") for srid in srids] + for_deletion = [] + for crs in crses: + if crs in index.spatial_indexes(): + for_deletion.append(crs) + else: + echo(f"No spatial index exists for CRS epsg:{crs.epsg} - skipping") + if for_deletion and not force: + echo("WARNING: Recreating spatial indexes may be slow and expensive for large databases.") + echo(f"You have requested to delete spatial indexes for the following CRSes: {','.join(crs.epsg for crs in for_deletion)}") + if sys.stdin.isatty(): + confirmed = confirm( + "Are you sure you want to delete these spatial indexes?", + default=False) + if not confirmed: + echo('OK aborting', err=True) + exit(1) + else: + echo("Use --force from non-interactive scripts. Aborting.") + exit(1) + errors = False + for crs in for_deletion: + click.echo(f"Deleting spatial index for CRS epsg:{crs.epsg}: ", nl=False) + if index.drop_spatial_index(crs): + click.echo("Done") + else: + click.echo("Failed") + errors = True + if errors: + exit(1) + else: + exit(0) diff --git a/datacube/scripts/system.py b/datacube/scripts/system.py index 7c04f10ad..d76aebf97 100644 --- a/datacube/scripts/system.py +++ b/datacube/scripts/system.py @@ -133,8 +133,3 @@ def clone(index: Index, batch_size: int, skip_lineage: bool, lineage_only: bool, handle_exception('Error Connecting to Source Database: %s', e) except IndexSetupError as e: handle_exception('Source database not initialised: %s', e) - - -@system.command('spindex', help='Create and/or update a spatial index for a particular CRS.') -def spindex(): - pass From 2adcd6ab95ee8a182994d46d72b1d40232edc326 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Fri, 4 Aug 2023 16:18:27 +1000 Subject: [PATCH 04/15] API tests and cleanup. --- datacube/drivers/postgis/_connections.py | 7 ++++--- datacube/drivers/postgis/_spatial.py | 6 +++++- integration_tests/index/test_postgis_index.py | 6 +++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/datacube/drivers/postgis/_connections.py b/datacube/drivers/postgis/_connections.py index b99a76a92..53ca3d9c4 100755 --- a/datacube/drivers/postgis/_connections.py +++ b/datacube/drivers/postgis/_connections.py @@ -30,7 +30,7 @@ from . import _api from . import _core -from ._spatial import ensure_spindex, spindexes, spindex_for_crs, drop_spindex +from ._spatial import ensure_spindex, spindexes, spindex_for_crs, drop_spindex, normalise_crs from ._schema import SpatialIndex _LIB_ID = 'odc-' + str(datacube.__version__) @@ -49,6 +49,7 @@ DEFAULT_IAM_TIMEOUT = 600 + class PostGisDb(object): """ A thin database access api. @@ -233,7 +234,7 @@ def create_spatial_index(self, crs: CRS) -> Optional[Type[SpatialIndex]]: _LOG.warning("Could not dynamically model an index for CRS %s", crs._str) return None ensure_spindex(self._engine, spidx) - self.spindexes[crs] = spidx + self._refresh_spindexes() return spidx def drop_spatial_index(self, crs: CRS) -> bool: @@ -251,7 +252,7 @@ def drop_spatial_index(self, crs: CRS) -> bool: return result def spatial_index(self, crs: CRS) -> Optional[Type[SpatialIndex]]: - return self.spindexes.get(crs) + return self.spindexes.get(normalise_crs(crs)) def spatial_indexes(self, refresh=False) -> Iterable[CRS]: if refresh: diff --git a/datacube/drivers/postgis/_spatial.py b/datacube/drivers/postgis/_spatial.py index 3272d07e5..c6438dd3a 100644 --- a/datacube/drivers/postgis/_spatial.py +++ b/datacube/drivers/postgis/_spatial.py @@ -110,6 +110,10 @@ def spindex_for_crs(crs: CRS) -> Type[SpatialIndex]: return spindex_for_epsg(crs.epsg) +def normalise_crs(crs_in: CRS) -> CRS: + return CRS(f'epsg:{crs_in.epsg}') + + def spindex_for_record(rec: SpatialIndexRecord) -> Type[SpatialIndex]: """Convert a Record of a SpatialIndex created in a particular database to an ORM class""" return spindex_for_crs(rec.crs) @@ -168,7 +172,7 @@ def spindexes(engine: Connectable) -> Mapping[CRS, Type[SpatialIndex]]: for result in results: epsg = int(result[0]) spindex = spindex_for_epsg(epsg) - crs = CRS(f'EPSG:{epsg}') + crs = CRS(f'epsg:{epsg}') out[crs] = spindex return out diff --git a/integration_tests/index/test_postgis_index.py b/integration_tests/index/test_postgis_index.py index 02ab14793..60c0bd925 100644 --- a/integration_tests/index/test_postgis_index.py +++ b/integration_tests/index/test_postgis_index.py @@ -13,7 +13,7 @@ @pytest.mark.parametrize('datacube_env_name', ('experimental',)) -def test_create_spatial_index(index: Index): +def test_create_drop_spatial_index(index: Index): # Default spatial index for 4326 assert list(index.spatial_indexes()) == [CRS("EPSG:4326")] # WKT CRS which cannot be mapped to an EPSG number. @@ -24,7 +24,11 @@ def test_create_spatial_index(index: Index): assert list(index.spatial_indexes()) == [CRS("EPSG:4326")] assert index.create_spatial_index(CRS("EPSG:3577")) assert index.create_spatial_index(CRS("WGS-84")) + assert set(index.spatial_indexes()) == {CRS("EPSG:3577"), CRS("EPSG:4326")} assert set(index.spatial_indexes(refresh=True)) == {CRS("EPSG:3577"), CRS("EPSG:4326")} + assert index.drop_spatial_index(CRS("EPSG:3577")) + assert index.spatial_indexes() == [CRS("EPSG:4326")] + assert index.spatial_indexes(refresh=True) == [CRS("EPSG:4326")] @pytest.mark.parametrize('datacube_env_name', ('experimental',)) From fdf4a607e6b73fa1784c287806b5927015b993ad Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Fri, 4 Aug 2023 17:06:24 +1000 Subject: [PATCH 05/15] Basic CLI tests. --- datacube/scripts/spindex.py | 14 ++++---- integration_tests/index/test_postgis_index.py | 16 +++++----- integration_tests/test_cli_spatial_indexes.py | 32 +++++++++++++++++++ 3 files changed, 47 insertions(+), 15 deletions(-) create mode 100644 integration_tests/test_cli_spatial_indexes.py diff --git a/datacube/scripts/spindex.py b/datacube/scripts/spindex.py index 4c0157fce..8c100bd8f 100644 --- a/datacube/scripts/spindex.py +++ b/datacube/scripts/spindex.py @@ -53,15 +53,15 @@ def create(index: Index, update: bool, srids: Sequence[int]): # A spatial index for crs already exists: skip silently confirmed.append(crs) elif index.create_spatial_index(crs): - # Creation attempted but failed - failed.append(crs) - else: # Creation succeeded confirmed.append(crs) + else: + # Creation attempted but failed + failed.append(crs) if failed: - echo(f"Could not create spatial indexes for: {','.join(crs.epsg for crs in failed)}") + echo(f"Could not create spatial indexes for: {','.join(str(crs.epsg) for crs in failed)}") if confirmed: - echo(f"Spatial indexes created for: {','.join(crs.epsg for crs in confirmed)}") + echo(f"Spatial indexes created for: {','.join(str(crs.epsg) for crs in confirmed)}") if update and failed: echo("Skipping update") elif update: @@ -69,7 +69,7 @@ def create(index: Index, update: bool, srids: Sequence[int]): echo(f'{result} extents checked and updated in spatial indexes') else: echo("Newly created spatial indexes are unpopulated - run 'datacube spindex update' before use.") - exit(int(failed)) + exit(len(failed)) @system.command( @@ -151,7 +151,7 @@ def drop(index: Index, force: bool, srids: Sequence[int]): echo(f"No spatial index exists for CRS epsg:{crs.epsg} - skipping") if for_deletion and not force: echo("WARNING: Recreating spatial indexes may be slow and expensive for large databases.") - echo(f"You have requested to delete spatial indexes for the following CRSes: {','.join(crs.epsg for crs in for_deletion)}") + echo(f"You have requested to delete spatial indexes for the following CRSes: {','.join(str(crs.epsg) for crs in for_deletion)}") if sys.stdin.isatty(): confirmed = confirm( "Are you sure you want to delete these spatial indexes?", diff --git a/integration_tests/index/test_postgis_index.py b/integration_tests/index/test_postgis_index.py index 60c0bd925..997865777 100644 --- a/integration_tests/index/test_postgis_index.py +++ b/integration_tests/index/test_postgis_index.py @@ -15,20 +15,20 @@ @pytest.mark.parametrize('datacube_env_name', ('experimental',)) def test_create_drop_spatial_index(index: Index): # Default spatial index for 4326 - assert list(index.spatial_indexes()) == [CRS("EPSG:4326")] + assert list(index.spatial_indexes()) == [CRS("epsg:4326")] # WKT CRS which cannot be mapped to an EPSG number. assert not index.create_spatial_index(CRS( 'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]]' ',PRIMEM["Weird",22.3],UNIT["Degree",0.017453292519943295]]' )) - assert list(index.spatial_indexes()) == [CRS("EPSG:4326")] - assert index.create_spatial_index(CRS("EPSG:3577")) + assert list(index.spatial_indexes()) == [CRS("epsg:4326")] + assert index.create_spatial_index(CRS("epsg:3577")) assert index.create_spatial_index(CRS("WGS-84")) - assert set(index.spatial_indexes()) == {CRS("EPSG:3577"), CRS("EPSG:4326")} - assert set(index.spatial_indexes(refresh=True)) == {CRS("EPSG:3577"), CRS("EPSG:4326")} - assert index.drop_spatial_index(CRS("EPSG:3577")) - assert index.spatial_indexes() == [CRS("EPSG:4326")] - assert index.spatial_indexes(refresh=True) == [CRS("EPSG:4326")] + assert set(index.spatial_indexes()) == {CRS("epsg:3577"), CRS("epsg:4326")} + assert set(index.spatial_indexes(refresh=True)) == {CRS("epsg:3577"), CRS("epsg:4326")} + assert index.drop_spatial_index(CRS("epsg:3577")) + assert index.spatial_indexes() == [CRS("epsg:4326")] + assert index.spatial_indexes(refresh=True) == [CRS("epsg:4326")] @pytest.mark.parametrize('datacube_env_name', ('experimental',)) diff --git a/integration_tests/test_cli_spatial_indexes.py b/integration_tests/test_cli_spatial_indexes.py new file mode 100644 index 000000000..c22429897 --- /dev/null +++ b/integration_tests/test_cli_spatial_indexes.py @@ -0,0 +1,32 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2023 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 + +import pytest + + +@pytest.mark.parametrize('datacube_env_name', ('experimental',)) +def test_cli_spatial_indexes(index, clirunner): + runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) + assert "epsg:4326" in runner.output + assert "epsg:3577" not in runner.output + assert runner.exit_code == 0 + + runner = clirunner(['spindex', 'create', '3577'], verbose_flag=False, expect_success=True) + assert runner.exit_code == 0 + + runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) + assert "epsg:4326" in runner.output + assert "epsg:3577" in runner.output + assert runner.exit_code == 0 + + runner = clirunner(['spindex', 'drop', '3577'], verbose_flag=False, expect_success=False) + assert runner.exit_code == 1 + runner = clirunner(['spindex', 'drop', '--force', '3577'], verbose_flag=False, expect_success=True) + assert runner.exit_code == 0 + + runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) + assert "epsg:4326" in runner.output + assert "epsg:3577" not in runner.output + assert runner.exit_code == 0 From 95990b7e6feee8368f6fd06d54efcc8ce5f74085 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Fri, 4 Aug 2023 17:20:57 +1000 Subject: [PATCH 06/15] Lintage. --- datacube/drivers/postgis/_connections.py | 1 - datacube/index/abstract.py | 45 +----------------------- datacube/index/postgis/index.py | 2 +- datacube/scripts/spindex.py | 13 +++---- 4 files changed, 7 insertions(+), 54 deletions(-) diff --git a/datacube/drivers/postgis/_connections.py b/datacube/drivers/postgis/_connections.py index 53ca3d9c4..6b3b6489f 100755 --- a/datacube/drivers/postgis/_connections.py +++ b/datacube/drivers/postgis/_connections.py @@ -49,7 +49,6 @@ DEFAULT_IAM_TIMEOUT = 600 - class PostGisDb(object): """ A thin database access api. diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 07de4d556..e776deb60 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -1988,7 +1988,7 @@ def update_spatial_index(self, else: raise NotImplementedError() - def drop_spatial_index(self, crs:CRS) -> bool: + def drop_spatial_index(self, crs: CRS) -> bool: """ Remove a spatial index from the database. @@ -2109,55 +2109,12 @@ def transaction(self) -> AbstractTransaction: :return: a Transaction context manager for this index. """ - @abstractmethod - def create_spatial_index(self, crs: CRS) -> bool: - """ - Create a spatial index using the nominated CRS. - - :param crs: The CRS to use in the spatial index. - :return: True is the index was successfully created or already exists. - None if spatial indexes are not supported. - """ - def thread_transaction(self) -> Optional["AbstractTransaction"]: """ :return: The existing Transaction object cached in thread-local storage for this index, if there is one. """ return thread_local_cache(f"txn-{self.index_id}", None) - def spatial_indexes(self, refresh=False) -> Iterable[CRS]: - """ - Return a list of CRSs for which spatiotemporal indexes exist in the database. - - :param refresh: If true, re-read from database record (e.g. to catch spatial - indexes recently created in another datacube session. - :return: - """ - _LOG.warning("Spatial index API is unstable and may change between releases.") - return [] - - def update_spatial_index(self, - crses: Sequence[CRS] = [], - product_names: Sequence[str] = [], - dataset_ids: Sequence[DSID] = [] - ) -> int: - """ - Update a spatial index - :param crs: CRSs for Spatial Indexes to update. Default=all indexes - :param product_names: Product names to update - :param dsids: Dataset IDs to update - - If neither product_names nor dataset ids are supplied, update for all datasets. - - If both are supplied, both the named products and identified datasets are updated. - - If spatial indexes are not supported by the index driver, always return zero. - - :return: Number of spatial index entries updated or verified as unindexed. - """ - _LOG.warning("Spatial index API is unstable and may change between releases.") - return 0 - def __enter__(self): return self diff --git a/datacube/index/postgis/index.py b/datacube/index/postgis/index.py index 3429e42da..d5b97977e 100644 --- a/datacube/index/postgis/index.py +++ b/datacube/index/postgis/index.py @@ -157,7 +157,7 @@ def update_spatial_index(self, def __repr__(self): return "Index".format(self._db) - def drop_spatial_index(self, crs:CRS) -> bool: + def drop_spatial_index(self, crs: CRS) -> bool: return self._db.drop_spatial_index(crs) @contextmanager diff --git a/datacube/scripts/spindex.py b/datacube/scripts/spindex.py index 8c100bd8f..0de3f70d2 100644 --- a/datacube/scripts/spindex.py +++ b/datacube/scripts/spindex.py @@ -7,16 +7,12 @@ from typing import Sequence import click -from click import echo, style, confirm +from click import echo, confirm from odc.geo import CRS -from sqlalchemy.exc import OperationalError -import datacube -from datacube.index import Index, index_connect -from datacube.drivers.postgres._connections import IndexSetupError +from datacube.index import Index from datacube.ui import click as ui -from datacube.ui.click import cli, handle_exception -from datacube.config import LocalConfig +from datacube.ui.click import cli _LOG = logging.getLogger('datacube-system') @@ -151,7 +147,8 @@ def drop(index: Index, force: bool, srids: Sequence[int]): echo(f"No spatial index exists for CRS epsg:{crs.epsg} - skipping") if for_deletion and not force: echo("WARNING: Recreating spatial indexes may be slow and expensive for large databases.") - echo(f"You have requested to delete spatial indexes for the following CRSes: {','.join(str(crs.epsg) for crs in for_deletion)}") + echo("You have requested to delete spatial indexes for the following " + f"CRSes: {','.join(str(crs.epsg) for crs in for_deletion)}") if sys.stdin.isatty(): confirmed = confirm( "Are you sure you want to delete these spatial indexes?", From 87293ddec5a7601de8617db39a6db85e7e861f0b Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Mon, 7 Aug 2023 11:21:49 +1000 Subject: [PATCH 07/15] Test coverage and cleanup. --- datacube/index/postgres/index.py | 3 --- integration_tests/index/test_search_legacy.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/datacube/index/postgres/index.py b/datacube/index/postgres/index.py index 428de7521..918083a29 100644 --- a/datacube/index/postgres/index.py +++ b/datacube/index/postgres/index.py @@ -117,9 +117,6 @@ def index_id(self) -> str: def transaction(self) -> AbstractTransaction: return PostgresTransaction(self._db, self.index_id) - def create_spatial_index(self, crs: CRS) -> None: - _LOG.warning("postgres driver does not support spatio-temporal indexes") - def __repr__(self): return "Index".format(self._db) diff --git a/integration_tests/index/test_search_legacy.py b/integration_tests/index/test_search_legacy.py index ba0db9c2e..57ebd1200 100644 --- a/integration_tests/index/test_search_legacy.py +++ b/integration_tests/index/test_search_legacy.py @@ -15,6 +15,7 @@ import pytest import yaml from dateutil import tz +from odc.geo import CRS from sqlalchemy.dialects.postgresql.ranges import Range as SQLARange from datacube.config import LocalConfig @@ -1061,3 +1062,19 @@ def test_query_dataset_multi_product(index: Index, ls5_dataset_w_children: Datas # Can we query multiple products in a tuple datasets = dc.find_datasets(product=('ls5_nbar_scene', 'ls5_level1_scene')) assert len(datasets) == 2 + + +@pytest.mark.parametrize('datacube_env_name', ('datacube',)) +def test_spatial_index_api_defaults(index: Index): + with pytest.raises(NotImplementedError) as e: + index.spatial_indexes() + assert "does not support the Spatial Index API" in str(e.value) + with pytest.raises(NotImplementedError) as e: + index.create_spatial_index(CRS("epsg:3577")) + assert "does not support the Spatial Index API" in str(e.value) + with pytest.raises(NotImplementedError) as e: + index.update_spatial_index([CRS("epsg:3577")]) + assert "does not support the Spatial Index API" in str(e.value) + with pytest.raises(NotImplementedError) as e: + index.drop_spatial_index(CRS("epsg:3577")) + assert "does not support the Spatial Index API" in str(e.value) From e7c0ee2559b743c7e2f4b22e489549a8822907db Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Mon, 7 Aug 2023 11:24:53 +1000 Subject: [PATCH 08/15] Lintage. --- datacube/index/postgres/index.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datacube/index/postgres/index.py b/datacube/index/postgres/index.py index 918083a29..757fdd58c 100644 --- a/datacube/index/postgres/index.py +++ b/datacube/index/postgres/index.py @@ -15,7 +15,6 @@ from datacube.index.abstract import AbstractIndex, AbstractIndexDriver, AbstractTransaction, \ default_metadata_type_docs from datacube.model import MetadataType -from odc.geo import CRS _LOG = logging.getLogger(__name__) From 460a20463e6bb6c843fb3495a19c0505d1f0df5f Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Mon, 7 Aug 2023 13:32:33 +1000 Subject: [PATCH 09/15] More test coverage. --- datacube/scripts/spindex.py | 58 ++++++++++++------- integration_tests/test_cli_spatial_indexes.py | 48 +++++++++++++++ 2 files changed, 86 insertions(+), 20 deletions(-) diff --git a/datacube/scripts/spindex.py b/datacube/scripts/spindex.py index 0de3f70d2..e96de45dc 100644 --- a/datacube/scripts/spindex.py +++ b/datacube/scripts/spindex.py @@ -7,6 +7,7 @@ from typing import Sequence import click +import pyproj from click import echo, confirm from odc.geo import CRS @@ -41,10 +42,14 @@ def create(index: Index, update: bool, srids: Sequence[int]): echo("Must supply at least one CRS to create/update") exit(1) - crses = [CRS(f"epsg:{srid}") for srid in srids] confirmed = [] failed = [] - for crs in crses: + for srid in srids: + try: + crs = CRS(f"epsg:{srid}") + except pyproj.exceptions.CRSError: + failed.append(srid) + continue if crs in index.spatial_indexes(): # A spatial index for crs already exists: skip silently confirmed.append(crs) @@ -53,9 +58,10 @@ def create(index: Index, update: bool, srids: Sequence[int]): confirmed.append(crs) else: # Creation attempted but failed - failed.append(crs) + failed.append(srid) if failed: - echo(f"Could not create spatial indexes for: {','.join(str(crs.epsg) for crs in failed)}") + str_failed = ','.join(f'epsg:{srid}' for srid in failed) + echo(f"Could not create spatial indexes for: {str_failed}") if confirmed: echo(f"Spatial indexes created for: {','.join(str(crs.epsg) for crs in confirmed)}") if update and failed: @@ -74,8 +80,12 @@ def create(index: Index, update: bool, srids: Sequence[int]): ) @ui.pass_index() def list_spindex(index): + if not index.supports_spatial_indexes: + echo("The active index driver does not support spatial indexes") + exit(1) for crs in index.spatial_indexes(): echo(f'epsg:{crs.epsg}') + exit(0) @system.command( @@ -96,29 +106,32 @@ def update(index: Index, product: Sequence[str], dataset: Sequence[str], srids: if not index.supports_spatial_indexes: echo("The active index driver does not support spatial indexes") exit(1) - if not update and product: - echo("Cannot pass in product names when not updating") - exit(1) - if not update and dataset: - echo("Cannot pass in dataset ids when not updating") - exit(1) if not srids: echo("Must supply at least one CRS to create/update") exit(1) - crses = [CRS(f"epsg:{srid}") for srid in srids] + # crses = [CRS(f"epsg:{srid}") for srid in srids] for_update = [] - for crs in crses: + cant_update = [] + for srid in srids: + try: + crs = CRS(f"epsg:{srid}") + except pyproj.exceptions.CRSError: + echo(f"epsg:{srid} is not a valid CRS: skipping") + cant_update.append(srid) + continue if crs in index.spatial_indexes(): for_update.append(crs) else: - echo(f"No spatial index for crs {crs.epsg} exists: skipping") + cant_update.append(srid) + echo(f"No spatial index for crs {srid} exists: skipping") + cant_update.append(srid) if not for_update: echo("Nothing to update!") - exit(0) + exit(len(cant_update)) result = index.update_spatial_index(for_update, product_names=product, dataset_ids=dataset) - echo(f'{result} extents checked and updated in spatial indexes') - exit(0) + echo(f'{result} extents checked and updated in {len(for_update)} spatial indexes') + exit(len(cant_update)) @system.command( @@ -138,13 +151,19 @@ def drop(index: Index, force: bool, srids: Sequence[int]): if not srids: echo("Must supply at least one CRS to drop") exit(1) - crses = [CRS(f"epsg:{srid}") for srid in srids] for_deletion = [] - for crs in crses: + errors = False + for srid in srids: + try: + crs = CRS(f"epsg:{srid}") + except pyproj.exceptions.CRSError: + echo(f"epsg:{srid} is not a valid CRS: skipping") + errors = True + continue if crs in index.spatial_indexes(): for_deletion.append(crs) else: - echo(f"No spatial index exists for CRS epsg:{crs.epsg} - skipping") + echo(f"No spatial index exists for CRS epsg:{srid} - skipping") if for_deletion and not force: echo("WARNING: Recreating spatial indexes may be slow and expensive for large databases.") echo("You have requested to delete spatial indexes for the following " @@ -159,7 +178,6 @@ def drop(index: Index, force: bool, srids: Sequence[int]): else: echo("Use --force from non-interactive scripts. Aborting.") exit(1) - errors = False for crs in for_deletion: click.echo(f"Deleting spatial index for CRS epsg:{crs.epsg}: ", nl=False) if index.drop_spatial_index(crs): diff --git a/integration_tests/test_cli_spatial_indexes.py b/integration_tests/test_cli_spatial_indexes.py index c22429897..43e451f68 100644 --- a/integration_tests/test_cli_spatial_indexes.py +++ b/integration_tests/test_cli_spatial_indexes.py @@ -16,6 +16,10 @@ def test_cli_spatial_indexes(index, clirunner): runner = clirunner(['spindex', 'create', '3577'], verbose_flag=False, expect_success=True) assert runner.exit_code == 0 + # Double creation succeeds silently + runner = clirunner(['spindex', 'create', '3577'], verbose_flag=False, expect_success=True) + assert runner.exit_code == 0 + runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) assert "epsg:4326" in runner.output assert "epsg:3577" in runner.output @@ -30,3 +34,47 @@ def test_cli_spatial_indexes(index, clirunner): assert "epsg:4326" in runner.output assert "epsg:3577" not in runner.output assert runner.exit_code == 0 + + +@pytest.mark.parametrize('datacube_env_name', ('datacube',)) +def test_cli_spatial_indexes_on_non_supporting_index(index, clirunner): + runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=False) + assert "does not support spatial indexes" in runner.output + assert runner.exit_code == 1 + + runner = clirunner(['spindex', 'create', '3577'], verbose_flag=False, expect_success=False) + assert "does not support spatial indexes" in runner.output + assert runner.exit_code == 1 + + runner = clirunner(['spindex', 'update', '3577'], verbose_flag=False, expect_success=False) + assert "does not support spatial indexes" in runner.output + assert runner.exit_code == 1 + + runner = clirunner(['spindex', 'drop', '3577'], verbose_flag=False, expect_success=False) + assert "does not support spatial indexes" in runner.output + assert runner.exit_code == 1 + + +@pytest.mark.parametrize('datacube_env_name', ('experimental',)) +def test_cli_spatial_indexes_no_srids(index, clirunner): + runner = clirunner(['spindex', 'create'], verbose_flag=False, expect_success=False) + assert "Must supply at least one CRS" in runner.output + assert runner.exit_code == 1 + + runner = clirunner(['spindex', 'update'], verbose_flag=False, expect_success=False) + assert "Must supply at least one CRS" in runner.output + assert runner.exit_code == 1 + + runner = clirunner(['spindex', 'drop'], verbose_flag=False, expect_success=False) + assert "Must supply at least one CRS" in runner.output + assert runner.exit_code == 1 + + +@pytest.mark.parametrize('datacube_env_name', ('experimental',)) +def test_cli_spatial_indexes_bad_srid(index, clirunner): + runner = clirunner(['spindex', 'create', '1'], verbose_flag=False, expect_success=False) + assert runner.exit_code == 1 + runner = clirunner(['spindex', 'update', '1'], verbose_flag=False, expect_success=False) + assert runner.exit_code == 1 + runner = clirunner(['spindex', 'drop', '1'], verbose_flag=False, expect_success=False) + assert runner.exit_code == 1 From e36fef0e811526a1fce17d92a5f4572debfe3dbc Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Mon, 7 Aug 2023 14:50:19 +1000 Subject: [PATCH 10/15] More test coverage. --- integration_tests/test_cli_spatial_indexes.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/integration_tests/test_cli_spatial_indexes.py b/integration_tests/test_cli_spatial_indexes.py index 43e451f68..6a4f250e3 100644 --- a/integration_tests/test_cli_spatial_indexes.py +++ b/integration_tests/test_cli_spatial_indexes.py @@ -36,6 +36,27 @@ def test_cli_spatial_indexes(index, clirunner): assert runner.exit_code == 0 +@pytest.mark.parametrize('datacube_env_name', ('experimental',)) +def test_cli_spatial_index_create_and_update(index, clirunner): + runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) + assert "epsg:4326" in runner.output + assert "epsg:3577" not in runner.output + assert runner.exit_code == 0 + + runner = clirunner(['spindex', 'create', '--update', '3577'], verbose_flag=False, expect_success=True) + assert runner.exit_code == 0 + + runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) + assert "epsg:4326" in runner.output + assert "epsg:3577" in runner.output + assert runner.exit_code == 0 + + runner = clirunner(['spindex', 'drop', '3577'], verbose_flag=False, expect_success=False) + assert runner.exit_code == 1 + runner = clirunner(['spindex', 'drop', '--force', '3577'], verbose_flag=False, expect_success=True) + assert runner.exit_code == 0 + + @pytest.mark.parametrize('datacube_env_name', ('datacube',)) def test_cli_spatial_indexes_on_non_supporting_index(index, clirunner): runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=False) @@ -74,6 +95,9 @@ def test_cli_spatial_indexes_no_srids(index, clirunner): def test_cli_spatial_indexes_bad_srid(index, clirunner): runner = clirunner(['spindex', 'create', '1'], verbose_flag=False, expect_success=False) assert runner.exit_code == 1 + runner = clirunner(['spindex', 'create', '--update', '1'], verbose_flag=False, expect_success=False) + assert "Skipping update" in runner.output + assert runner.exit_code == 1 runner = clirunner(['spindex', 'update', '1'], verbose_flag=False, expect_success=False) assert runner.exit_code == 1 runner = clirunner(['spindex', 'drop', '1'], verbose_flag=False, expect_success=False) From 4864a095cfc53b31a4be75952c336488106dc623 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Mon, 7 Aug 2023 15:12:05 +1000 Subject: [PATCH 11/15] More test coverage. --- datacube/scripts/spindex.py | 1 - integration_tests/test_cli_spatial_indexes.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datacube/scripts/spindex.py b/datacube/scripts/spindex.py index e96de45dc..4ed631888 100644 --- a/datacube/scripts/spindex.py +++ b/datacube/scripts/spindex.py @@ -125,7 +125,6 @@ def update(index: Index, product: Sequence[str], dataset: Sequence[str], srids: else: cant_update.append(srid) echo(f"No spatial index for crs {srid} exists: skipping") - cant_update.append(srid) if not for_update: echo("Nothing to update!") exit(len(cant_update)) diff --git a/integration_tests/test_cli_spatial_indexes.py b/integration_tests/test_cli_spatial_indexes.py index 6a4f250e3..7624bc4bf 100644 --- a/integration_tests/test_cli_spatial_indexes.py +++ b/integration_tests/test_cli_spatial_indexes.py @@ -20,6 +20,14 @@ def test_cli_spatial_indexes(index, clirunner): runner = clirunner(['spindex', 'create', '3577'], verbose_flag=False, expect_success=True) assert runner.exit_code == 0 + # Double creation succeeds silently + runner = clirunner(['spindex', 'update', '3577'], verbose_flag=False, expect_success=True) + assert runner.exit_code == 0 + + # Double creation succeeds silently + runner = clirunner(['spindex', 'update', '3857'], verbose_flag=False, expect_success=False) + assert runner.exit_code == 1 + runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) assert "epsg:4326" in runner.output assert "epsg:3577" in runner.output @@ -35,6 +43,10 @@ def test_cli_spatial_indexes(index, clirunner): assert "epsg:3577" not in runner.output assert runner.exit_code == 0 + # Drop non-existent spindex ignored. + runner = clirunner(['spindex', 'drop', '--force', '3577'], verbose_flag=False, expect_success=True) + assert runner.exit_code == 0 + @pytest.mark.parametrize('datacube_env_name', ('experimental',)) def test_cli_spatial_index_create_and_update(index, clirunner): From 89ed7f092cd2b807068f738acdb1a3e37dfc20cd Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 8 Aug 2023 10:28:33 +1000 Subject: [PATCH 12/15] Update whats_new.rst --- docs/about/whats_new.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index 4ce3ddbad..c1cb6baac 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -21,6 +21,7 @@ v1.9.next - Fix postgis queries for numeric custom search fields (:pull:`1475`) - Document best practice for pulling in changes from develop and update constraints.txt (:pull:`1478`) - Postgis index driver performance tuning (:pull:`1480`) +- Cleanup and formalise spatial index API and expose in CLI (:pull:`1481`) v1.8.next From 0b4fba56ac063806856a964de762b0544521dbb9 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Mon, 21 Aug 2023 12:59:07 +1000 Subject: [PATCH 13/15] Misc cleanup - thanks Ariana. --- datacube/drivers/postgis/_api.py | 12 ++++++------ datacube/drivers/postgis/_connections.py | 6 +++--- datacube/scripts/spindex.py | 3 +-- integration_tests/index/test_index_data.py | 15 ++++++--------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/datacube/drivers/postgis/_api.py b/datacube/drivers/postgis/_api.py index 659ad7542..606d954d6 100644 --- a/datacube/drivers/postgis/_api.py +++ b/datacube/drivers/postgis/_api.py @@ -372,7 +372,7 @@ def insert_dataset_spatial(self, dataset_id, crs, extent): values = generate_dataset_spatial_values(dataset_id, crs, extent) if values is None: return False - SpatialIndex = self._db.spatial_index(crs) # noqa: N806 + SpatialIndex = self._db.spatial_index_orm(crs) # noqa: N806 r = self._connection.execute( insert( SpatialIndex @@ -386,12 +386,12 @@ def insert_dataset_spatial(self, dataset_id, crs, extent): return r.rowcount > 0 def insert_dataset_spatial_bulk(self, crs, values): - SpatialIndex = self._db.spatial_index(crs) # noqa: N806 + SpatialIndex = self._db.spatial_index_orm(crs) # noqa: N806 r = self._connection.execute(insert(SpatialIndex).values(values)) return r.rowcount def spatial_extent(self, ids, crs): - SpatialIndex = self._db.spatial_index(crs) # noqa: N806 + SpatialIndex = self._db.spatial_index_orm(crs) # noqa: N806 if SpatialIndex is None: return None result = self._connection.execute( @@ -512,7 +512,7 @@ def delete_dataset(self, dataset_id): delete(table).where(table.dataset_ref == dataset_id) ) for crs in self._db.spatial_indexes(): - SpatialIndex = self._db.spatial_index(crs) # noqa: N806 + SpatialIndex = self._db.spatial_index_orm(crs) # noqa: N806 self._connection.execute( delete( SpatialIndex @@ -607,12 +607,12 @@ def search_datasets_query(self, # No? Convert to 4326 which we should always have a spatial index for by default if not geom.crs: raise ValueError("Search geometry must have a CRS") - SpatialIndex = self._db.spatial_index(geom.crs) # noqa: N806 + SpatialIndex = self._db.spatial_index_orm(geom.crs) # noqa: N806 if SpatialIndex is None: _LOG.info("No spatial index for crs %s - converting to 4326", geom.crs) default_crs = CRS("EPSG:4326") geom = geom.to_crs(default_crs) - SpatialIndex = self._db.spatial_index(default_crs) # noqa: N806 + SpatialIndex = self._db.spatial_index_orm(default_crs) # noqa: N806 geom_sql = geom_alchemy(geom) _LOG.info("query geometry = %s (%s)", geom.json, geom.crs) spatialquery = func.ST_Intersects(SpatialIndex.extent, geom_sql) diff --git a/datacube/drivers/postgis/_connections.py b/datacube/drivers/postgis/_connections.py index 6b3b6489f..2dc8f2d38 100755 --- a/datacube/drivers/postgis/_connections.py +++ b/datacube/drivers/postgis/_connections.py @@ -30,7 +30,7 @@ from . import _api from . import _core -from ._spatial import ensure_spindex, spindexes, spindex_for_crs, drop_spindex, normalise_crs +from ._spatial import ensure_spindex, spindexes, spindex_for_crs, drop_spindex from ._schema import SpatialIndex _LIB_ID = 'odc-' + str(datacube.__version__) @@ -250,8 +250,8 @@ def drop_spatial_index(self, crs: CRS) -> bool: self._refresh_spindexes() return result - def spatial_index(self, crs: CRS) -> Optional[Type[SpatialIndex]]: - return self.spindexes.get(normalise_crs(crs)) + def spatial_index_orm(self, crs: CRS) -> Optional[Type[SpatialIndex]]: + return self.spindexes.get(crs) def spatial_indexes(self, refresh=False) -> Iterable[CRS]: if refresh: diff --git a/datacube/scripts/spindex.py b/datacube/scripts/spindex.py index 4ed631888..5b8322dc1 100644 --- a/datacube/scripts/spindex.py +++ b/datacube/scripts/spindex.py @@ -107,10 +107,9 @@ def update(index: Index, product: Sequence[str], dataset: Sequence[str], srids: echo("The active index driver does not support spatial indexes") exit(1) if not srids: - echo("Must supply at least one CRS to create/update") + echo("Must supply at least one CRS to update") exit(1) - # crses = [CRS(f"epsg:{srid}") for srid in srids] for_update = [] cant_update = [] for srid in srids: diff --git a/integration_tests/index/test_index_data.py b/integration_tests/index/test_index_data.py index 6a53efa95..b7135c5a0 100755 --- a/integration_tests/index/test_index_data.py +++ b/integration_tests/index/test_index_data.py @@ -73,7 +73,7 @@ def test_archive_datasets(index, ls8_eo3_dataset): datasets = index.datasets.search_eager() assert len(datasets) == 1 - assert datasets[0].is_active + assert not datasets[0].is_archived index.datasets.archive([ls8_eo3_dataset.id]) datasets = index.datasets.search_eager() @@ -82,7 +82,6 @@ def test_archive_datasets(index, ls8_eo3_dataset): # The model should show it as archived now. indexed_dataset = index.datasets.get(ls8_eo3_dataset.id) assert indexed_dataset.is_archived - assert not indexed_dataset.is_active index.datasets.restore([ls8_eo3_dataset.id]) datasets = index.datasets.search_eager() @@ -90,17 +89,16 @@ def test_archive_datasets(index, ls8_eo3_dataset): # And now active indexed_dataset = index.datasets.get(ls8_eo3_dataset.id) - assert indexed_dataset.is_active assert not indexed_dataset.is_archived def test_archive_less_mature(index, final_dataset, nrt_dataset): # case 1: add nrt then final; nrt should get archived index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=True) - index.datasets.get(nrt_dataset.id).is_active + assert not index.datasets.get(nrt_dataset.id).is_archived index.datasets.add(final_dataset, with_lineage=False, archive_less_mature=True) assert index.datasets.get(nrt_dataset.id).is_archived - assert index.datasets.get(final_dataset.id).is_active + assert not index.datasets.get(final_dataset.id).is_archived # case 2: purge nrt; re-add with final already there index.datasets.purge([nrt_dataset.id]) @@ -113,17 +111,17 @@ def test_archive_less_mature(index, final_dataset, nrt_dataset): def test_archive_less_mature_approx_timestamp(index, ga_s2am_ard3_final, ga_s2am_ard3_interim): # test archive_less_mature where there's a slight difference in timestamps index.datasets.add(ga_s2am_ard3_interim, with_lineage=False) - index.datasets.get(ga_s2am_ard3_interim.id).is_active + assert not index.datasets.get(ga_s2am_ard3_interim.id).is_archived index.datasets.add(ga_s2am_ard3_final, with_lineage=False, archive_less_mature=True) assert index.datasets.get(ga_s2am_ard3_interim.id).is_archived - assert index.datasets.get(ga_s2am_ard3_final.id).is_active + assert not index.datasets.get(ga_s2am_ard3_final.id).is_archived def test_purge_datasets(index, ls8_eo3_dataset): assert index.datasets.has(ls8_eo3_dataset.id) datasets = index.datasets.search_eager() assert len(datasets) == 1 - assert datasets[0].is_active + assert not datasets[0].is_archived # Archive dataset index.datasets.archive([ls8_eo3_dataset.id]) @@ -133,7 +131,6 @@ def test_purge_datasets(index, ls8_eo3_dataset): # The model should show it as archived now. indexed_dataset = index.datasets.get(ls8_eo3_dataset.id) assert indexed_dataset.is_archived - assert not indexed_dataset.is_active # Purge dataset index.datasets.purge([ls8_eo3_dataset.id]) From 55b80b0644e17762bd37d1fa18a265e3118d0da0 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Mon, 21 Aug 2023 15:32:29 +1000 Subject: [PATCH 14/15] More cleanup. --- datacube/drivers/postgis/_api.py | 16 ++++++++-------- datacube/drivers/postgis/_connections.py | 16 ++++++++-------- datacube/drivers/postgis/_spatial.py | 11 +++-------- datacube/index/postgis/_datasets.py | 2 +- datacube/index/postgis/index.py | 2 +- 5 files changed, 21 insertions(+), 26 deletions(-) diff --git a/datacube/drivers/postgis/_api.py b/datacube/drivers/postgis/_api.py index 606d954d6..2e91f80a3 100644 --- a/datacube/drivers/postgis/_api.py +++ b/datacube/drivers/postgis/_api.py @@ -372,7 +372,7 @@ def insert_dataset_spatial(self, dataset_id, crs, extent): values = generate_dataset_spatial_values(dataset_id, crs, extent) if values is None: return False - SpatialIndex = self._db.spatial_index_orm(crs) # noqa: N806 + SpatialIndex = self._db.spatial_index(crs) # noqa: N806 r = self._connection.execute( insert( SpatialIndex @@ -386,12 +386,12 @@ def insert_dataset_spatial(self, dataset_id, crs, extent): return r.rowcount > 0 def insert_dataset_spatial_bulk(self, crs, values): - SpatialIndex = self._db.spatial_index_orm(crs) # noqa: N806 + SpatialIndex = self._db.spatial_index(crs) # noqa: N806 r = self._connection.execute(insert(SpatialIndex).values(values)) return r.rowcount def spatial_extent(self, ids, crs): - SpatialIndex = self._db.spatial_index_orm(crs) # noqa: N806 + SpatialIndex = self._db.spatial_index(crs) # noqa: N806 if SpatialIndex is None: return None result = self._connection.execute( @@ -511,8 +511,8 @@ def delete_dataset(self, dataset_id): self._connection.execute( delete(table).where(table.dataset_ref == dataset_id) ) - for crs in self._db.spatial_indexes(): - SpatialIndex = self._db.spatial_index_orm(crs) # noqa: N806 + for crs in self._db.spatially_indexed_crses(): + SpatialIndex = self._db.spatial_index(crs) # noqa: N806 self._connection.execute( delete( SpatialIndex @@ -607,12 +607,12 @@ def search_datasets_query(self, # No? Convert to 4326 which we should always have a spatial index for by default if not geom.crs: raise ValueError("Search geometry must have a CRS") - SpatialIndex = self._db.spatial_index_orm(geom.crs) # noqa: N806 + SpatialIndex = self._db.spatial_index(geom.crs) # noqa: N806 if SpatialIndex is None: _LOG.info("No spatial index for crs %s - converting to 4326", geom.crs) default_crs = CRS("EPSG:4326") geom = geom.to_crs(default_crs) - SpatialIndex = self._db.spatial_index_orm(default_crs) # noqa: N806 + SpatialIndex = self._db.spatial_index(default_crs) # noqa: N806 geom_sql = geom_alchemy(geom) _LOG.info("query geometry = %s (%s)", geom.json, geom.crs) spatialquery = func.ST_Intersects(SpatialIndex.extent, geom_sql) @@ -901,7 +901,7 @@ def update_spindex(self, crs_seq: Sequence[CRS] = [], if crs_seq: crses = [crs for crs in crs_seq] else: - crses = self._db.spatial_indexes() + crses = self._db.spatially_indexed_crses() # Update implementation. # Design will change, but this method should be fairly low level to be as efficient as possible diff --git a/datacube/drivers/postgis/_connections.py b/datacube/drivers/postgis/_connections.py index 2dc8f2d38..19ef392cb 100755 --- a/datacube/drivers/postgis/_connections.py +++ b/datacube/drivers/postgis/_connections.py @@ -70,7 +70,7 @@ def __init__(self, engine): # We don't recommend using this constructor directly as it may change. # Use static methods PostGisDb.create() or PostGisDb.from_config() self._engine = engine - self._spindexes: Optional[Mapping[CRS, Any]] = None + self._spindexes: Optional[Mapping[int, Any]] = None @classmethod def from_config(cls, config, application_name=None, validate_connection=True): @@ -214,7 +214,7 @@ def _refresh_spindexes(self): self._spindexes = spindexes(self._engine) @property - def spindexes(self) -> Mapping[CRS, Type[SpatialIndex]]: + def spindexes(self) -> Mapping[int, Type[SpatialIndex]]: if self._spindexes is None: self._refresh_spindexes() return self._spindexes @@ -226,7 +226,7 @@ def create_spatial_index(self, crs: CRS) -> Optional[Type[SpatialIndex]]: :param crs: :return: """ - spidx = self.spindexes.get(crs) + spidx = self.spindexes.get(crs.epsg) if spidx is None: spidx = spindex_for_crs(crs) if spidx is None: @@ -243,20 +243,20 @@ def drop_spatial_index(self, crs: CRS) -> bool: :param crs: :return: """ - spidx = self.spindexes.get(crs) + spidx = self.spindexes.get(crs.epsg) if spidx is None: return False result = drop_spindex(self._engine, spidx) self._refresh_spindexes() return result - def spatial_index_orm(self, crs: CRS) -> Optional[Type[SpatialIndex]]: - return self.spindexes.get(crs) + def spatial_index(self, crs: CRS) -> Optional[Type[SpatialIndex]]: + return self.spindexes.get(crs.epsg) - def spatial_indexes(self, refresh=False) -> Iterable[CRS]: + def spatially_indexed_crses(self, refresh=False) -> Iterable[CRS]: if refresh: self._refresh_spindexes() - return list(self.spindexes.keys()) + return list(CRS(epsg) for epsg in self.spindexes.keys()) @contextmanager def _connect(self): diff --git a/datacube/drivers/postgis/_spatial.py b/datacube/drivers/postgis/_spatial.py index c6438dd3a..bfb05b62b 100644 --- a/datacube/drivers/postgis/_spatial.py +++ b/datacube/drivers/postgis/_spatial.py @@ -110,10 +110,6 @@ def spindex_for_crs(crs: CRS) -> Type[SpatialIndex]: return spindex_for_epsg(crs.epsg) -def normalise_crs(crs_in: CRS) -> CRS: - return CRS(f'epsg:{crs_in.epsg}') - - def spindex_for_record(rec: SpatialIndexRecord) -> Type[SpatialIndex]: """Convert a Record of a SpatialIndex created in a particular database to an ORM class""" return spindex_for_crs(rec.crs) @@ -162,9 +158,9 @@ def drop_spindex(engine: Connectable, sp_idx: Type[SpatialIndex]): return True -def spindexes(engine: Connectable) -> Mapping[CRS, Type[SpatialIndex]]: +def spindexes(engine: Connectable) -> Mapping[int, Type[SpatialIndex]]: """ - Return a CRS-to-Spatial Index ORM class mapping for indexes that exist in a particular database. + Return a SRID-to-Spatial Index ORM class mapping for indexes that exist in a particular database. """ out = {} with Session(engine) as session: @@ -172,8 +168,7 @@ def spindexes(engine: Connectable) -> Mapping[CRS, Type[SpatialIndex]]: for result in results: epsg = int(result[0]) spindex = spindex_for_epsg(epsg) - crs = CRS(f'epsg:{epsg}') - out[crs] = spindex + out[epsg] = spindex return out diff --git a/datacube/index/postgis/_datasets.py b/datacube/index/postgis/_datasets.py index e0e1cd327..4ea756635 100755 --- a/datacube/index/postgis/_datasets.py +++ b/datacube/index/postgis/_datasets.py @@ -184,7 +184,7 @@ def _init_bulk_add_cache(self): def _add_batch(self, batch_ds: Iterable[DatasetTuple], cache: Mapping[str, Any]) -> BatchStatus: # Add a "batch" of datasets. b_started = monotonic() - crses = self._db.spatial_indexes() + crses = self._db.spatially_indexed_crses() batch = { "datasets": [], "uris": [], diff --git a/datacube/index/postgis/index.py b/datacube/index/postgis/index.py index d5b97977e..22ef5d88b 100644 --- a/datacube/index/postgis/index.py +++ b/datacube/index/postgis/index.py @@ -144,7 +144,7 @@ def create_spatial_index(self, crs: CRS) -> bool: return sp_idx is not None def spatial_indexes(self, refresh=False) -> Iterable[CRS]: - return self._db.spatial_indexes(refresh) + return self._db.spatially_indexed_crses(refresh) def update_spatial_index(self, crses: Sequence[CRS] = [], From 2f091fd0fa56302ec1d459b2f41c8b0e04f48ffa Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Mon, 21 Aug 2023 16:05:13 +1000 Subject: [PATCH 15/15] Make CLI less fussy about CRS input format. --- datacube/scripts/spindex.py | 55 +++++++++++-------- integration_tests/test_cli_spatial_indexes.py | 24 ++++---- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/datacube/scripts/spindex.py b/datacube/scripts/spindex.py index 5b8322dc1..f705c20cb 100644 --- a/datacube/scripts/spindex.py +++ b/datacube/scripts/spindex.py @@ -25,16 +25,15 @@ def system(): @system.command( 'create', - help='Create unpopulated spatial index for particular CRSes ' - '(express CRSes as EPSG numbers - e.g. 3857, not "epsg:3857").') + help='Create unpopulated spatial index for particular CRSes') @click.option( '--update/--no-update', '-u', is_flag=True, default=False, help="Populate the spatial index after creation (slow). " "For finer grained updating, use the 'spindex update' command" ) -@click.argument('srids', type=int, nargs=-1) +@click.argument('srids', nargs=-1) @ui.pass_index() -def create(index: Index, update: bool, srids: Sequence[int]): +def create(index: Index, update: bool, srids: Sequence[str]): if not index.supports_spatial_indexes: echo("The active index driver does not support spatial indexes") exit(1) @@ -46,9 +45,14 @@ def create(index: Index, update: bool, srids: Sequence[int]): failed = [] for srid in srids: try: - crs = CRS(f"epsg:{srid}") + crs = CRS(srid) except pyproj.exceptions.CRSError: failed.append(srid) + echo(f"{srid} cannot be converted to a valid CRS") + continue + if not crs.epsg: + failed.append(srid) + echo(f"{srid} cannot be converted to a CRS with an EPSG number") continue if crs in index.spatial_indexes(): # A spatial index for crs already exists: skip silently @@ -60,7 +64,7 @@ def create(index: Index, update: bool, srids: Sequence[int]): # Creation attempted but failed failed.append(srid) if failed: - str_failed = ','.join(f'epsg:{srid}' for srid in failed) + str_failed = ','.join(f'{srid}' for srid in failed) echo(f"Could not create spatial indexes for: {str_failed}") if confirmed: echo(f"Spatial indexes created for: {','.join(str(crs.epsg) for crs in confirmed)}") @@ -84,25 +88,24 @@ def list_spindex(index): echo("The active index driver does not support spatial indexes") exit(1) for crs in index.spatial_indexes(): - echo(f'epsg:{crs.epsg}') + echo(f'EPSG:{crs.epsg}') exit(0) @system.command( 'update', - help='Update a spatial index for particular CRSs ' - '(express CRSs as EPSG numbers - e.g. 3857, not "epsg:3857").') + help='Update a spatial index for particular CRSs.') @click.option( '--product', '-p', multiple=True, - help="The name of a product to update the spatial index for (can be used multiple times for multiple products" + help="The name of a product to update the spatial index for (can be used multiple times for multiple products)" ) @click.option( '--dataset', '-d', multiple=True, - help="The id of a dataset to update the spatial index for (can be used multiple times for multiple datasets" + help="The id of a dataset to update the spatial index for (can be used multiple times for multiple datasets)" ) -@click.argument('srids', type=int, nargs=-1) +@click.argument('srids', nargs=-1) @ui.pass_index() -def update(index: Index, product: Sequence[str], dataset: Sequence[str], srids: Sequence[int]): +def update(index: Index, product: Sequence[str], dataset: Sequence[str], srids: Sequence[str]): if not index.supports_spatial_indexes: echo("The active index driver does not support spatial indexes") exit(1) @@ -114,11 +117,16 @@ def update(index: Index, product: Sequence[str], dataset: Sequence[str], srids: cant_update = [] for srid in srids: try: - crs = CRS(f"epsg:{srid}") + crs = CRS(srid) except pyproj.exceptions.CRSError: - echo(f"epsg:{srid} is not a valid CRS: skipping") + echo(f"{srid} is not a valid CRS: skipping") + cant_update.append(srid) + continue + if not crs.epsg: + echo(f"{srid} cannot be converted to a CRS with an EPSG number") cant_update.append(srid) continue + if crs in index.spatial_indexes(): for_update.append(crs) else: @@ -134,15 +142,14 @@ def update(index: Index, product: Sequence[str], dataset: Sequence[str], srids: @system.command( 'drop', - help='Drop existing spatial indexes for particular CRSs ' - '(express CRSs as EPSG numbers - e.g. 3857, not "epsg:3857").') + help='Drop existing spatial indexes for particular CRSs') @click.option( '--force/--no-force', '-f', is_flag=True, default=False, help="If set, does not ask the user to confirm deletion" ) -@click.argument('srids', type=int, nargs=-1) +@click.argument('srids', nargs=-1) @ui.pass_index() -def drop(index: Index, force: bool, srids: Sequence[int]): +def drop(index: Index, force: bool, srids: Sequence[str]): if not index.supports_spatial_indexes: echo("The active index driver does not support spatial indexes") exit(1) @@ -153,15 +160,19 @@ def drop(index: Index, force: bool, srids: Sequence[int]): errors = False for srid in srids: try: - crs = CRS(f"epsg:{srid}") + crs = CRS(srid) except pyproj.exceptions.CRSError: - echo(f"epsg:{srid} is not a valid CRS: skipping") + echo(f"{srid} is not a valid CRS: skipping") + errors = True + continue + if not crs.epsg: + echo(f"{srid} cannot be converted to a CRS with an EPSG number") errors = True continue if crs in index.spatial_indexes(): for_deletion.append(crs) else: - echo(f"No spatial index exists for CRS epsg:{srid} - skipping") + echo(f"No spatial index exists for CRS {srid} - skipping") if for_deletion and not force: echo("WARNING: Recreating spatial indexes may be slow and expensive for large databases.") echo("You have requested to delete spatial indexes for the following " diff --git a/integration_tests/test_cli_spatial_indexes.py b/integration_tests/test_cli_spatial_indexes.py index 7624bc4bf..afae48dd9 100644 --- a/integration_tests/test_cli_spatial_indexes.py +++ b/integration_tests/test_cli_spatial_indexes.py @@ -9,11 +9,11 @@ @pytest.mark.parametrize('datacube_env_name', ('experimental',)) def test_cli_spatial_indexes(index, clirunner): runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) - assert "epsg:4326" in runner.output - assert "epsg:3577" not in runner.output + assert "EPSG:4326" in runner.output + assert "EPSG:3577" not in runner.output assert runner.exit_code == 0 - runner = clirunner(['spindex', 'create', '3577'], verbose_flag=False, expect_success=True) + runner = clirunner(['spindex', 'create', 'epsg:3577'], verbose_flag=False, expect_success=True) assert runner.exit_code == 0 # Double creation succeeds silently @@ -25,12 +25,12 @@ def test_cli_spatial_indexes(index, clirunner): assert runner.exit_code == 0 # Double creation succeeds silently - runner = clirunner(['spindex', 'update', '3857'], verbose_flag=False, expect_success=False) + runner = clirunner(['spindex', 'update', 'EPSG:3857'], verbose_flag=False, expect_success=False) assert runner.exit_code == 1 runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) - assert "epsg:4326" in runner.output - assert "epsg:3577" in runner.output + assert "EPSG:4326" in runner.output + assert "EPSG:3577" in runner.output assert runner.exit_code == 0 runner = clirunner(['spindex', 'drop', '3577'], verbose_flag=False, expect_success=False) @@ -39,8 +39,8 @@ def test_cli_spatial_indexes(index, clirunner): assert runner.exit_code == 0 runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) - assert "epsg:4326" in runner.output - assert "epsg:3577" not in runner.output + assert "EPSG:4326" in runner.output + assert "EPSG:3577" not in runner.output assert runner.exit_code == 0 # Drop non-existent spindex ignored. @@ -51,16 +51,16 @@ def test_cli_spatial_indexes(index, clirunner): @pytest.mark.parametrize('datacube_env_name', ('experimental',)) def test_cli_spatial_index_create_and_update(index, clirunner): runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) - assert "epsg:4326" in runner.output - assert "epsg:3577" not in runner.output + assert "EPSG:4326" in runner.output + assert "EPSG:3577" not in runner.output assert runner.exit_code == 0 runner = clirunner(['spindex', 'create', '--update', '3577'], verbose_flag=False, expect_success=True) assert runner.exit_code == 0 runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True) - assert "epsg:4326" in runner.output - assert "epsg:3577" in runner.output + assert "EPSG:4326" in runner.output + assert "EPSG:3577" in runner.output assert runner.exit_code == 0 runner = clirunner(['spindex', 'drop', '3577'], verbose_flag=False, expect_success=False)