diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml deleted file mode 100644 index 5a1dcc41f7..0000000000 --- a/.github/workflows/build-docs.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: build-docs - -on: - pull_request: - push: - branches: - - develop - - dra/docs-updates - -env: - DKR: opendatacube/datacube-tests:latest - -jobs: - main: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Pull Docker - run: | - docker pull ${DKR} - - - name: Build Docs - run: | - cat <> $GITHUB_OUTPUT + fi + - uses: dorny/paths-filter@v2 id: changes if: | @@ -112,10 +119,11 @@ jobs: ls -lh ./dist/ twine check ./dist/* EOF + - name: Publish to PyPi if: | github.event_name == 'push' - && github.ref == 'refs/heads/pypi/publish' + && steps.cfg.outputs.push_pypi == 'yes' run: | if [ -n "${TWINE_PASSWORD}" ]; then docker run --rm \ @@ -132,6 +140,7 @@ jobs: else echo "Skipping upload as 'PyPiToken' is not set" fi + env: TWINE_PASSWORD: ${{ secrets.PyPiToken }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e3a0412649..59143ce982 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/adrienverge/yamllint.git - rev: v1.30.0 + rev: v1.32.0 hooks: - id: yamllint - repo: https://github.com/pre-commit/pre-commit-hooks @@ -22,3 +22,18 @@ repos: rev: v3.0.0a5 # Use the sha / tag you want to point at hooks: - id: pylint + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.1 + hooks: + - id: forbid-crlf + - id: remove-crlf + - id: forbid-tabs + - id: remove-tabs + args: [--whitespaces-count, '2'] + - id: insert-license + files: ./(.*).py$ + args: + - --license-filepath + - license-template.txt + - --use-current-year + - --no-extra-eol diff --git a/README.rst b/README.rst index d64e696dc7..a730c11b4b 100644 --- a/README.rst +++ b/README.rst @@ -49,12 +49,12 @@ Developer setup - ``git clone https://github.com/opendatacube/datacube-core.git`` -2. Create a Python environment for using the ODC. We recommend `conda `__ as the +2. Create a Python environment for using the ODC. We recommend `Mambaforge `__ as the easiest way to handle Python dependencies. :: - conda create -f conda-environment.yml + mamba env create -f conda-environment.yml conda activate cubeenv 3. Install a develop version of datacube-core. @@ -72,14 +72,19 @@ Developer setup pre-commit install 5. Run unit tests + PyLint - ``./check-code.sh`` - - (this script approximates what is run by Travis. You can - alternatively run ``pytest`` yourself). Some test dependencies may need to be installed, attempt to install these using: +Install test dependencies using: + ``pip install --upgrade -e '.[test]'`` - If install for these fails please lodge them as issues. +If install for these fails, please lodge them as issues. + +Run unit tests with: + + ``./check-code.sh`` + + (this script approximates what is run by GitHub Actions. You can + alternatively run ``pytest`` yourself). 6. **(or)** Run all tests, including integration tests. @@ -87,11 +92,14 @@ Developer setup - Assumes a password-less Postgres database running on localhost called - ``agdcintegration`` + ``pgintegration`` - - Otherwise copy ``integration_tests/agdcintegration.conf`` to + - Otherwise copy ``integration_tests/integration.conf`` to ``~/.datacube_integration.conf`` and edit to customise. + - For instructions on setting up a password-less Postgres database, see + the `developer setup instructions `__. + Alternatively one can use the ``opendatacube/datacube-tests`` docker image to run tests. This docker includes database server pre-configured for running @@ -103,11 +111,13 @@ to ``./check-code.sh`` script. ./check-code.sh --with-docker integration_tests -To run individual test in docker container +To run individual tests in a docker container :: - docker run -ti -v /home/ubuntu/datacube-core:/code opendatacube/datacube-tests:latest pytest integration_tests/test_filename.py::test_function_name + docker build --tag=opendatacube/datacube-tests-local --no-cache --progress plain -f docker/Dockerfile . + + docker run -ti -v $(pwd):/code opendatacube/datacube-tests-local:latest pytest integration_tests/test_filename.py::test_function_name Developer setup on Ubuntu diff --git a/datacube/__init__.py b/datacube/__init__.py index 55b5d6924b..3429804308 100644 --- a/datacube/__init__.py +++ b/datacube/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Datacube diff --git a/datacube/__main__.py b/datacube/__main__.py index 8af90e515d..543af0ede3 100644 --- a/datacube/__main__.py +++ b/datacube/__main__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 if __name__ == "__main__": from .config import auto_config diff --git a/datacube/api/__init__.py b/datacube/api/__init__.py index 3154c299f5..b7676523e3 100644 --- a/datacube/api/__init__.py +++ b/datacube/api/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Modules for the Storage and Access Query API diff --git a/datacube/api/core.py b/datacube/api/core.py index 4ebdf04698..ed3c3e7783 100644 --- a/datacube/api/core.py +++ b/datacube/api/core.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2021 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import uuid @@ -236,12 +236,17 @@ def load(self, product=None, measurements=None, output_crs=None, resolution=None x=(1516200, 1541300), y=(-3867375, -3867350), crs='EPSG:3577' - The ``time`` dimension can be specified using a tuple of datetime objects or strings with - ``YYYY-MM-DD hh:mm:ss`` format. Data will be loaded inclusive of the start and finish times. E.g:: + The ``time`` dimension can be specified using a single or tuple of datetime objects or strings with + ``YYYY-MM-DD hh:mm:ss`` format. Data will be loaded inclusive of the start and finish times. + A ``None`` value in the range indicates an open range, with the provided date serving as either the + upper or lower bound. E.g:: time=('2000-01-01', '2001-12-31') time=('2000-01', '2001-12') time=('2000', '2001') + time=('2000') + time=('2000', None) # all data from 2000 onward + time=(None, '2000') # all data up to and including 2000 For 3D datasets, where the product definition contains an ``extra_dimension`` specification, these dimensions can be queried using that dimension's name. E.g.:: diff --git a/datacube/api/grid_workflow.py b/datacube/api/grid_workflow.py index e38d55766b..7c00304d6b 100644 --- a/datacube/api/grid_workflow.py +++ b/datacube/api/grid_workflow.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import xarray diff --git a/datacube/api/query.py b/datacube/api/query.py index 55a46ccb7d..78559b859b 100644 --- a/datacube/api/query.py +++ b/datacube/api/query.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Storage Query and Access API module @@ -136,8 +136,8 @@ def __init__(self, index=None, product=None, geopolygon=None, like=None, **searc if time_coord is not None: self.search['time'] = _time_to_search_dims( (pandas_to_datetime(time_coord.values[0]).to_pydatetime(), - pandas_to_datetime(time_coord.values[-1]).to_pydatetime() - + datetime.timedelta(milliseconds=1)) # TODO: inclusive time searches + pandas_to_datetime(time_coord.values[-1]).to_pydatetime() + + datetime.timedelta(milliseconds=1)) # TODO: inclusive time searches ) @property @@ -350,7 +350,11 @@ def _time_to_search_dims(time_range): if hasattr(tr_end, 'isoformat'): tr_end = tr_end.isoformat() + if tr_start is None: + tr_start = datetime.datetime.fromtimestamp(0) start = _to_datetime(tr_start) + if tr_end is None: + tr_end = datetime.datetime.now().strftime("%Y-%m-%d") end = _to_datetime(pandas.Period(tr_end) .end_time .to_pydatetime()) diff --git a/datacube/config.py b/datacube/config.py index 90911ec4fd..8caa1aa0b1 100755 --- a/datacube/config.py +++ b/datacube/config.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ User configuration. diff --git a/datacube/drivers/__init__.py b/datacube/drivers/__init__.py index 4544150027..0ab9b51789 100644 --- a/datacube/drivers/__init__.py +++ b/datacube/drivers/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ This module implements a simple plugin manager for storage and index drivers. diff --git a/datacube/drivers/_tools.py b/datacube/drivers/_tools.py index d43e3db515..39ec173060 100644 --- a/datacube/drivers/_tools.py +++ b/datacube/drivers/_tools.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from threading import Lock from typing import Any diff --git a/datacube/drivers/_types.py b/datacube/drivers/_types.py index 49bd41e631..15b8f7d541 100644 --- a/datacube/drivers/_types.py +++ b/datacube/drivers/_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Defines abstract types for IO drivers. """ diff --git a/datacube/drivers/datasource.py b/datacube/drivers/datasource.py index 7ad98e775a..91e64a0368 100644 --- a/datacube/drivers/datasource.py +++ b/datacube/drivers/datasource.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Defines abstract types for IO reader drivers. """ diff --git a/datacube/drivers/driver_cache.py b/datacube/drivers/driver_cache.py index 75c70b2bf5..6324170f41 100644 --- a/datacube/drivers/driver_cache.py +++ b/datacube/drivers/driver_cache.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from typing import Dict, Any, Tuple, Iterable diff --git a/datacube/drivers/indexes.py b/datacube/drivers/indexes.py index 5c231b22ce..1cda1d6e8a 100644 --- a/datacube/drivers/indexes.py +++ b/datacube/drivers/indexes.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import List, Optional diff --git a/datacube/drivers/netcdf/__init__.py b/datacube/drivers/netcdf/__init__.py index b0148d4ca0..9b0321f125 100644 --- a/datacube/drivers/netcdf/__init__.py +++ b/datacube/drivers/netcdf/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from ._write import write_dataset_to_netcdf, create_netcdf_storage_unit from . import writer as netcdf_writer diff --git a/datacube/drivers/netcdf/_write.py b/datacube/drivers/netcdf/_write.py index d759f20d4c..8cc15f208c 100644 --- a/datacube/drivers/netcdf/_write.py +++ b/datacube/drivers/netcdf/_write.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from pathlib import Path import logging diff --git a/datacube/drivers/netcdf/driver.py b/datacube/drivers/netcdf/driver.py index 56910c40e8..d68b04ab86 100644 --- a/datacube/drivers/netcdf/driver.py +++ b/datacube/drivers/netcdf/driver.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from urllib.parse import urlsplit diff --git a/datacube/drivers/netcdf/writer.py b/datacube/drivers/netcdf/writer.py index 397fa2e781..eb82bdf6e8 100644 --- a/datacube/drivers/netcdf/writer.py +++ b/datacube/drivers/netcdf/writer.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Create netCDF4 Storage Units and write data to them diff --git a/datacube/drivers/postgis/__init__.py b/datacube/drivers/postgis/__init__.py index 50348f828d..a43833f903 100644 --- a/datacube/drivers/postgis/__init__.py +++ b/datacube/drivers/postgis/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Lower-level database access. diff --git a/datacube/drivers/postgis/_api.py b/datacube/drivers/postgis/_api.py index f17109b17f..339e50368b 100644 --- a/datacube/drivers/postgis/_api.py +++ b/datacube/drivers/postgis/_api.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # We often have one-arg-per column, so these checks aren't so useful. diff --git a/datacube/drivers/postgis/_connections.py b/datacube/drivers/postgis/_connections.py index d4b486b9c1..3d10407c31 100755 --- a/datacube/drivers/postgis/_connections.py +++ b/datacube/drivers/postgis/_connections.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # We often have one-arg-per column, so these checks aren't so useful. diff --git a/datacube/drivers/postgis/_core.py b/datacube/drivers/postgis/_core.py index 7567805060..d7a8c81e88 100644 --- a/datacube/drivers/postgis/_core.py +++ b/datacube/drivers/postgis/_core.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Core SQL schema settings. diff --git a/datacube/drivers/postgis/_fields.py b/datacube/drivers/postgis/_fields.py index f17f702f45..95d0aeca5f 100755 --- a/datacube/drivers/postgis/_fields.py +++ b/datacube/drivers/postgis/_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Build and index fields within documents. diff --git a/datacube/drivers/postgis/_schema.py b/datacube/drivers/postgis/_schema.py index bcb6872fda..7887aada68 100644 --- a/datacube/drivers/postgis/_schema.py +++ b/datacube/drivers/postgis/_schema.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tables for indexing the datasets which were ingested into the AGDC. diff --git a/datacube/drivers/postgis/_spatial.py b/datacube/drivers/postgis/_spatial.py index 9200336448..3a2c8e1d4c 100644 --- a/datacube/drivers/postgis/_spatial.py +++ b/datacube/drivers/postgis/_spatial.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tracking spatial indexes diff --git a/datacube/drivers/postgis/sql.py b/datacube/drivers/postgis/sql.py index bdb4ff06de..02f9c741f4 100644 --- a/datacube/drivers/postgis/sql.py +++ b/datacube/drivers/postgis/sql.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Custom types for postgres & sqlalchemy diff --git a/datacube/drivers/postgres/__init__.py b/datacube/drivers/postgres/__init__.py index 6fe10be3c7..6f3e321349 100644 --- a/datacube/drivers/postgres/__init__.py +++ b/datacube/drivers/postgres/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Lower-level database access. diff --git a/datacube/drivers/postgres/_api.py b/datacube/drivers/postgres/_api.py index 36a1290df7..bd46ed2b20 100644 --- a/datacube/drivers/postgres/_api.py +++ b/datacube/drivers/postgres/_api.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # We often have one-arg-per column, so these checks aren't so useful. diff --git a/datacube/drivers/postgres/_connections.py b/datacube/drivers/postgres/_connections.py index 68f5d080a4..003617db3e 100755 --- a/datacube/drivers/postgres/_connections.py +++ b/datacube/drivers/postgres/_connections.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # We often have one-arg-per column, so these checks aren't so useful. diff --git a/datacube/drivers/postgres/_core.py b/datacube/drivers/postgres/_core.py index 94a5aa9c4a..7971019b60 100644 --- a/datacube/drivers/postgres/_core.py +++ b/datacube/drivers/postgres/_core.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Core SQL schema settings. diff --git a/datacube/drivers/postgres/_dynamic.py b/datacube/drivers/postgres/_dynamic.py index d7401038ba..8c0765bad8 100644 --- a/datacube/drivers/postgres/_dynamic.py +++ b/datacube/drivers/postgres/_dynamic.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Methods for managing dynamic dataset field indexes and views. diff --git a/datacube/drivers/postgres/_fields.py b/datacube/drivers/postgres/_fields.py index c4a464f4c9..fc3023b05e 100755 --- a/datacube/drivers/postgres/_fields.py +++ b/datacube/drivers/postgres/_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Build and index fields within documents. diff --git a/datacube/drivers/postgres/_schema.py b/datacube/drivers/postgres/_schema.py index 56bedf3ed6..d555bb6b4a 100644 --- a/datacube/drivers/postgres/_schema.py +++ b/datacube/drivers/postgres/_schema.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tables for indexing the datasets which were ingested into the AGDC. diff --git a/datacube/drivers/postgres/sql.py b/datacube/drivers/postgres/sql.py index 7c421b4d1d..8e949ff9ff 100644 --- a/datacube/drivers/postgres/sql.py +++ b/datacube/drivers/postgres/sql.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Custom types for postgres & sqlalchemy diff --git a/datacube/drivers/readers.py b/datacube/drivers/readers.py index 16d0f9e204..230e63b4c3 100644 --- a/datacube/drivers/readers.py +++ b/datacube/drivers/readers.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import List, Optional, Callable from .driver_cache import load_drivers diff --git a/datacube/drivers/rio/__init__.py b/datacube/drivers/rio/__init__.py index b74ea31e59..f41dba762d 100644 --- a/datacube/drivers/rio/__init__.py +++ b/datacube/drivers/rio/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ RasterIO based driver """ diff --git a/datacube/drivers/rio/_reader.py b/datacube/drivers/rio/_reader.py index f19ba7644c..ae53bbdc07 100644 --- a/datacube/drivers/rio/_reader.py +++ b/datacube/drivers/rio/_reader.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ reader """ diff --git a/datacube/drivers/writers.py b/datacube/drivers/writers.py index 970eede77a..cc652c409c 100644 --- a/datacube/drivers/writers.py +++ b/datacube/drivers/writers.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import List diff --git a/datacube/execution/__init__.py b/datacube/execution/__init__.py index c081ad5b41..45970a2a5b 100644 --- a/datacube/execution/__init__.py +++ b/datacube/execution/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/execution/worker.py b/datacube/execution/worker.py index 9cd20faf6c..cacc97e647 100644 --- a/datacube/execution/worker.py +++ b/datacube/execution/worker.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ This app launches workers for distributed work loads diff --git a/datacube/executor.py b/datacube/executor.py index 9a86b28977..b39b03f620 100644 --- a/datacube/executor.py +++ b/datacube/executor.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # # type: ignore diff --git a/datacube/helpers.py b/datacube/helpers.py index cf9c3da5b1..6f397fadf3 100644 --- a/datacube/helpers.py +++ b/datacube/helpers.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Useful functions for Datacube users diff --git a/datacube/index/__init__.py b/datacube/index/__init__.py index 1c4050cf53..c6a25f3011 100644 --- a/datacube/index/__init__.py +++ b/datacube/index/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Modules for interfacing with the index/database. diff --git a/datacube/index/_api.py b/datacube/index/_api.py index abd7e4c656..2ffb67919a 100644 --- a/datacube/index/_api.py +++ b/datacube/index/_api.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Access methods for indexing datasets & products. diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 44d6d60291..4377d25e56 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime import logging @@ -15,6 +15,7 @@ NamedTuple, Optional, Tuple, Union, Sequence) from uuid import UUID +from datetime import timedelta from datacube.config import LocalConfig from datacube.index.exceptions import TransactionException @@ -1067,7 +1068,8 @@ def bulk_has(self, ids_: Iterable[DSID]) -> Iterable[bool]: @abstractmethod def add(self, dataset: Dataset, - with_lineage: bool = True + with_lineage: bool = True, + archive_less_mature: bool = False, ) -> Dataset: """ Add ``dataset`` to the index. No-op if it is already present. @@ -1079,6 +1081,10 @@ def add(self, dataset: Dataset, - ``False`` record lineage relations, but do not attempt adding lineage datasets to the db + :param archive_less_mature: + - ``True`` search for less mature versions of the dataset + and archive them + :return: Persisted Dataset model """ @@ -1116,12 +1122,14 @@ def can_update(self, @abstractmethod def update(self, dataset: Dataset, - updates_allowed: Optional[Mapping[Offset, AllowPolicy]] = None + updates_allowed: Optional[Mapping[Offset, AllowPolicy]] = None, + archive_less_mature: bool = False, ) -> Dataset: """ Update dataset metadata and location :param Dataset dataset: Dataset model with unpersisted updates :param updates_allowed: Allowed updates + :param archive_less_mature: Find and archive less mature datasets :return: Persisted dataset model """ @@ -1133,6 +1141,52 @@ def archive(self, ids: Iterable[DSID]) -> None: :param Iterable[Union[str,UUID]] ids: list of dataset ids to archive """ + def archive_less_mature(self, ds: Dataset) -> None: + """ + Archive less mature versions of a dataset + + :param Dataset ds: dataset to search + """ + less_mature = self.find_less_mature(ds) + less_mature_ids = map(lambda x: x.id, less_mature) + + self.archive(less_mature_ids) + for lm_ds in less_mature_ids: + _LOG.info(f"Archived less mature dataset: {lm_ds}") + + def find_less_mature(self, ds: Dataset) -> Iterable[Dataset]: + """ + Find less mature versions of a dataset + + :param Dataset ds: Dataset to search + :return: Iterable of less mature datasets + """ + less_mature = [] + # 'expand' the date range by a millisecond to give a bit more leniency in datetime comparison + expanded_time_range = Range(ds.metadata.time.begin - timedelta(milliseconds=500), + ds.metadata.time.end + timedelta(milliseconds=500)) + dupes = self.search(product=ds.product.name, + region_code=ds.metadata.region_code, + time=expanded_time_range) + for dupe in dupes: + if dupe.id == ds.id: + continue + if dupe.metadata.dataset_maturity == ds.metadata.dataset_maturity: + # Duplicate has the same maturity, which one should be archived is unclear + raise ValueError( + f"A dataset with the same maturity as dataset {ds.id} already exists, " + f"with id: {dupe.id}" + ) + if dupe.metadata.dataset_maturity < ds.metadata.dataset_maturity: + # Duplicate is more mature than dataset + # Note that "final" < "nrt" + raise ValueError( + f"A more mature version of dataset {ds.id} already exists, with id: " + f"{dupe.id} and maturity: {dupe.metadata.dataset_maturity}" + ) + less_mature.append(dupe) + return less_mature + @abstractmethod def restore(self, ids: Iterable[DSID]) -> None: """ diff --git a/datacube/index/eo3.py b/datacube/index/eo3.py index 4a48cd40a1..6973156990 100644 --- a/datacube/index/eo3.py +++ b/datacube/index/eo3.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 # # type: ignore diff --git a/datacube/index/exceptions.py b/datacube/index/exceptions.py index e75a4135ae..c12344c491 100644 --- a/datacube/index/exceptions.py +++ b/datacube/index/exceptions.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/index/fields.py b/datacube/index/fields.py index 5c2c0fe7f8..85d3e59d85 100644 --- a/datacube/index/fields.py +++ b/datacube/index/fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Common datatypes for DB drivers. @@ -38,7 +38,7 @@ def evaluate(self, ctx): def as_expression(field: Field, value) -> Expression: """ - Convert a single field/value to expression, following the "simple" convensions. + Convert a single field/value to expression, following the "simple" conventions. """ if isinstance(value, Range): return field.between(value.begin, value.end) diff --git a/datacube/index/hl.py b/datacube/index/hl.py index 313367f8bb..b062b24677 100644 --- a/datacube/index/hl.py +++ b/datacube/index/hl.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ High level indexing operations/utilities diff --git a/datacube/index/memory/__init__.py b/datacube/index/memory/__init__.py index bea658a528..7bd83106eb 100644 --- a/datacube/index/memory/__init__.py +++ b/datacube/index/memory/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/datacube/index/memory/_datasets.py b/datacube/index/memory/_datasets.py index 2ee5808aeb..817b28223b 100755 --- a/datacube/index/memory/_datasets.py +++ b/datacube/index/memory/_datasets.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime import logging @@ -74,7 +74,8 @@ def bulk_has(self, ids_: Iterable[DSID]) -> Iterable[bool]: return (self.has(id_) for id_ in ids_) def add(self, dataset: Dataset, - with_lineage: bool = True) -> Dataset: + with_lineage: bool = True, + archive_less_mature: bool = False) -> Dataset: if with_lineage is None: with_lineage = True _LOG.info('indexing %s', dataset.id) @@ -102,6 +103,8 @@ def add(self, dataset: Dataset, self.by_product[dataset.product.name].append(dataset.id) else: self.by_product[dataset.product.name] = [dataset.id] + if archive_less_mature: + _LOG.warning("archive-less-mature functionality is not implemented for memory driver") return cast(Dataset, self.get(dataset.id)) def persist_source_relationship(self, ds: Dataset, src: Dataset, classifier: str) -> None: @@ -188,7 +191,8 @@ def can_update(self, def update(self, dataset: Dataset, - updates_allowed: Optional[Mapping[Offset, AllowPolicy]] = None + updates_allowed: Optional[Mapping[Offset, AllowPolicy]] = None, + archive_less_mature: bool = False ) -> Dataset: existing = self.get(dataset.id) if not existing: diff --git a/datacube/index/memory/_fields.py b/datacube/index/memory/_fields.py index 1ad6a7cd63..ba60fc7f33 100644 --- a/datacube/index/memory/_fields.py +++ b/datacube/index/memory/_fields.py @@ -1,3 +1,7 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2023 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 from typing import Any, Mapping, MutableMapping from datacube.model.fields import SimpleField, Field, get_dataset_fields as generic_get_dataset_fields from datacube.index.abstract import Offset diff --git a/datacube/index/memory/_metadata_types.py b/datacube/index/memory/_metadata_types.py index 83501d7cbd..8f22ecced1 100644 --- a/datacube/index/memory/_metadata_types.py +++ b/datacube/index/memory/_metadata_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from copy import deepcopy diff --git a/datacube/index/memory/_products.py b/datacube/index/memory/_products.py index d0cce50152..c5707687db 100644 --- a/datacube/index/memory/_products.py +++ b/datacube/index/memory/_products.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/memory/_users.py b/datacube/index/memory/_users.py index 6e597853f1..579829af55 100644 --- a/datacube/index/memory/_users.py +++ b/datacube/index/memory/_users.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource diff --git a/datacube/index/memory/index.py b/datacube/index/memory/index.py index bd259e3c2f..7d6b810ac4 100644 --- a/datacube/index/memory/index.py +++ b/datacube/index/memory/index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from threading import Lock diff --git a/datacube/index/null/__init__.py b/datacube/index/null/__init__.py index bea658a528..7bd83106eb 100644 --- a/datacube/index/null/__init__.py +++ b/datacube/index/null/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/datacube/index/null/_datasets.py b/datacube/index/null/_datasets.py index 42c9ec351e..6c998da14c 100755 --- a/datacube/index/null/_datasets.py +++ b/datacube/index/null/_datasets.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datacube.index.abstract import AbstractDatasetResource, DSID @@ -28,7 +28,8 @@ def bulk_has(self, ids_): return [False for id_ in ids_] def add(self, dataset: Dataset, - with_lineage: bool = True) -> Dataset: + with_lineage: bool = True, + archive_less_mature: bool = False) -> Dataset: raise NotImplementedError() def search_product_duplicates(self, product: Product, *args): @@ -37,7 +38,7 @@ def search_product_duplicates(self, product: Product, *args): def can_update(self, dataset, updates_allowed=None): raise NotImplementedError() - def update(self, dataset: Dataset, updates_allowed=None): + def update(self, dataset: Dataset, updates_allowed=None, archive_less_mature=False): raise NotImplementedError() def archive(self, ids): diff --git a/datacube/index/null/_metadata_types.py b/datacube/index/null/_metadata_types.py index 3346203f4f..808c18ee13 100644 --- a/datacube/index/null/_metadata_types.py +++ b/datacube/index/null/_metadata_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datacube.index.abstract import AbstractMetadataTypeResource diff --git a/datacube/index/null/_products.py b/datacube/index/null/_products.py index a2788a283b..7bef0ec24d 100644 --- a/datacube/index/null/_products.py +++ b/datacube/index/null/_products.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/null/_users.py b/datacube/index/null/_users.py index cc91c5dae0..e058c51582 100644 --- a/datacube/index/null/_users.py +++ b/datacube/index/null/_users.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource diff --git a/datacube/index/null/index.py b/datacube/index/null/index.py index 136095b82d..3c72aaa652 100644 --- a/datacube/index/null/index.py +++ b/datacube/index/null/index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/postgis/__init__.py b/datacube/index/postgis/__init__.py index e69de29bb2..45970a2a5b 100644 --- a/datacube/index/postgis/__init__.py +++ b/datacube/index/postgis/__init__.py @@ -0,0 +1,4 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2023 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/index/postgis/_datasets.py b/datacube/index/postgis/_datasets.py index 09e1595ecf..6d1413bd33 100755 --- a/datacube/index/postgis/_datasets.py +++ b/datacube/index/postgis/_datasets.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ API for dataset indexing, access and search. @@ -137,7 +137,7 @@ def bulk_has(self, ids_): map((lambda x: UUID(x) if isinstance(x, str) else x), ids_)] def add(self, dataset: Dataset, - with_lineage: bool = True) -> Dataset: + with_lineage: bool = True, archive_less_mature: bool = False) -> Dataset: """ Add ``dataset`` to the index. No-op if it is already present. @@ -148,6 +148,10 @@ def add(self, dataset: Dataset, - ``False`` record lineage relations, but do not attempt adding lineage datasets to the db + :param archive_less_mature: + - ``True`` search for less mature versions of the dataset + and archive them + :rtype: Dataset """ @@ -169,6 +173,8 @@ def add(self, dataset: Dataset, # 1c. Store locations if dataset.uris is not None: self._ensure_new_locations(dataset, transaction=transaction) + if archive_less_mature: + self.archive_less_mature(dataset) return dataset @@ -305,11 +311,12 @@ def can_update(self, dataset, updates_allowed=None): return not bad_changes, good_changes, bad_changes - def update(self, dataset: Dataset, updates_allowed=None): + def update(self, dataset: Dataset, updates_allowed=None, archive_less_mature=False): """ Update dataset metadata and location :param Dataset dataset: Dataset to update :param updates_allowed: Allowed updates + :param archive_less_mature: Find and archive less mature datasets :rtype: Dataset """ existing = self.get(dataset.id) @@ -342,6 +349,8 @@ def update(self, dataset: Dataset, updates_allowed=None): raise ValueError("Failed to update dataset %s..." % dataset.id) transaction.update_spindex(dsids=[dataset.id]) transaction.update_search_index(dsids=[dataset.id]) + if archive_less_mature: + self._archive_less_mature(dataset) self._ensure_new_locations(dataset, existing) diff --git a/datacube/index/postgis/_metadata_types.py b/datacube/index/postgis/_metadata_types.py index 9c593f9b9a..7e9593552b 100644 --- a/datacube/index/postgis/_metadata_types.py +++ b/datacube/index/postgis/_metadata_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from time import monotonic diff --git a/datacube/index/postgis/_products.py b/datacube/index/postgis/_products.py index 7e3cd86b70..7f50a218b7 100644 --- a/datacube/index/postgis/_products.py +++ b/datacube/index/postgis/_products.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/postgis/_transaction.py b/datacube/index/postgis/_transaction.py index 3fabcccb59..ac48a64f87 100644 --- a/datacube/index/postgis/_transaction.py +++ b/datacube/index/postgis/_transaction.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from contextlib import contextmanager diff --git a/datacube/index/postgis/_users.py b/datacube/index/postgis/_users.py index c604a676b4..ca2c6ec3a2 100644 --- a/datacube/index/postgis/_users.py +++ b/datacube/index/postgis/_users.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource diff --git a/datacube/index/postgis/index.py b/datacube/index/postgis/index.py index 40da0176e1..1de718bd20 100644 --- a/datacube/index/postgis/index.py +++ b/datacube/index/postgis/index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from contextlib import contextmanager diff --git a/datacube/index/postgres/__init__.py b/datacube/index/postgres/__init__.py index e69de29bb2..45970a2a5b 100644 --- a/datacube/index/postgres/__init__.py +++ b/datacube/index/postgres/__init__.py @@ -0,0 +1,4 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2023 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/index/postgres/_datasets.py b/datacube/index/postgres/_datasets.py index 522b17c701..77fbf17f79 100755 --- a/datacube/index/postgres/_datasets.py +++ b/datacube/index/postgres/_datasets.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ API for dataset indexing, access and search. @@ -133,7 +133,7 @@ def bulk_has(self, ids_): map((lambda x: UUID(x) if isinstance(x, str) else x), ids_)] def add(self, dataset: Dataset, - with_lineage: bool = True) -> Dataset: + with_lineage: bool = True, archive_less_mature: bool = False) -> Dataset: """ Add ``dataset`` to the index. No-op if it is already present. @@ -144,6 +144,10 @@ def add(self, dataset: Dataset, - ``False`` record lineage relations, but do not attempt adding lineage datasets to the db + :param archive_less_mature: + - ``True`` search for less mature versions of the dataset + and archive them + :rtype: Dataset """ @@ -188,6 +192,8 @@ def process_bunch(dss, main_ds, transaction): with self._db_connection(transaction=True) as transaction: process_bunch(dss, dataset, transaction) + if archive_less_mature: + self.archive_less_mature(dataset) return dataset @@ -279,11 +285,12 @@ def can_update(self, dataset, updates_allowed=None): return not bad_changes, good_changes, bad_changes - def update(self, dataset: Dataset, updates_allowed=None): + def update(self, dataset: Dataset, updates_allowed=None, archive_less_mature=False): """ Update dataset metadata and location :param Dataset dataset: Dataset to update :param updates_allowed: Allowed updates + :param archive_less_mature: Find and archive less mature datasets :rtype: Dataset """ existing = self.get(dataset.id) diff --git a/datacube/index/postgres/_lineage.py b/datacube/index/postgres/_lineage.py index 18c0714ddd..f27127b418 100644 --- a/datacube/index/postgres/_lineage.py +++ b/datacube/index/postgres/_lineage.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from time import monotonic from typing import Iterable diff --git a/datacube/index/postgres/_metadata_types.py b/datacube/index/postgres/_metadata_types.py index 4c1f4193f5..de77c23c12 100644 --- a/datacube/index/postgres/_metadata_types.py +++ b/datacube/index/postgres/_metadata_types.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/postgres/_products.py b/datacube/index/postgres/_products.py index a639f63363..e3b7fd773e 100644 --- a/datacube/index/postgres/_products.py +++ b/datacube/index/postgres/_products.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/index/postgres/_transaction.py b/datacube/index/postgres/_transaction.py index 5b02a5584c..6ff35d942a 100644 --- a/datacube/index/postgres/_transaction.py +++ b/datacube/index/postgres/_transaction.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from contextlib import contextmanager diff --git a/datacube/index/postgres/_users.py b/datacube/index/postgres/_users.py index 2dd4c81877..eeb76c4fc7 100644 --- a/datacube/index/postgres/_users.py +++ b/datacube/index/postgres/_users.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource diff --git a/datacube/index/postgres/index.py b/datacube/index/postgres/index.py index 4714adb619..428de75212 100644 --- a/datacube/index/postgres/index.py +++ b/datacube/index/postgres/index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from contextlib import contextmanager diff --git a/datacube/model/_base.py b/datacube/model/_base.py index 5223c18fa3..3a004f7633 100644 --- a/datacube/model/_base.py +++ b/datacube/model/_base.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from collections import namedtuple diff --git a/datacube/model/eo3.py b/datacube/model/eo3.py index 668f8c44cd..8897ea9339 100644 --- a/datacube/model/eo3.py +++ b/datacube/model/eo3.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datacube.utils.documents import InvalidDocException diff --git a/datacube/model/fields.py b/datacube/model/fields.py index 3f94ff2803..48826eb01a 100644 --- a/datacube/model/fields.py +++ b/datacube/model/fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """Non-db specific implementation of metadata search fields. diff --git a/datacube/model/utils.py b/datacube/model/utils.py index b89494c948..e001570ce8 100644 --- a/datacube/model/utils.py +++ b/datacube/model/utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime import os diff --git a/datacube/scripts/__init__.py b/datacube/scripts/__init__.py index c081ad5b41..45970a2a5b 100644 --- a/datacube/scripts/__init__.py +++ b/datacube/scripts/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/datacube/scripts/cli_app.py b/datacube/scripts/cli_app.py index 820d8179eb..6ab3901b25 100644 --- a/datacube/scripts/cli_app.py +++ b/datacube/scripts/cli_app.py @@ -2,7 +2,7 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Datacube command-line interface diff --git a/datacube/scripts/dataset.py b/datacube/scripts/dataset.py index 2a318a8545..c7a4e5ba6a 100644 --- a/datacube/scripts/dataset.py +++ b/datacube/scripts/dataset.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import csv import datetime @@ -158,6 +158,8 @@ def mk_dataset(ds, uri): @click.option('--confirm-ignore-lineage', help="Pretend that there is no lineage data in the datasets being indexed, without confirmation", is_flag=True, default=False) +@click.option('--archive-less-mature', help='Archive less mature versions of the dataset', + is_flag=True, default=False) @click.argument('dataset-paths', type=str, nargs=-1) @ui.pass_index() def index_cmd(index, product_names, @@ -167,9 +169,11 @@ def index_cmd(index, product_names, dry_run, ignore_lineage, confirm_ignore_lineage, + archive_less_mature, dataset_paths): if not dataset_paths: + click.echo('Error: no datasets provided\n') print_help_msg(index_cmd) sys.exit(1) @@ -203,7 +207,7 @@ def run_it(dataset_paths): index_datasets(dss, index, auto_add_lineage=auto_add_lineage and not confirm_ignore_lineage, - dry_run=dry_run) + dry_run=dry_run, archive_less_mature=archive_less_mature) # If outputting directly to terminal, show a progress bar. if sys.stdout.isatty(): @@ -213,12 +217,13 @@ def run_it(dataset_paths): run_it(dataset_paths) -def index_datasets(dss, index, auto_add_lineage, dry_run): +def index_datasets(dss, index, auto_add_lineage, dry_run, archive_less_mature): for dataset in dss: _LOG.info('Matched %s', dataset) if not dry_run: try: - index.datasets.add(dataset, with_lineage=auto_add_lineage) + index.datasets.add(dataset, with_lineage=auto_add_lineage, + archive_less_mature=archive_less_mature) except (ValueError, MissingRecordError) as e: _LOG.error('Failed to add dataset %s: %s', dataset.local_uri, e) @@ -245,10 +250,13 @@ def parse_update_rules(keys_that_can_change): - 'keep': keep as alternative location [default] - 'archive': mark as archived - 'forget': remove from the index''')) +@click.option('--archive-less-mature', help='Archive less mature versions of the dataset', + is_flag=True, default=False) @click.argument('dataset-paths', nargs=-1) @ui.pass_index() -def update_cmd(index, keys_that_can_change, dry_run, location_policy, dataset_paths): +def update_cmd(index, keys_that_can_change, dry_run, location_policy, dataset_paths, archive_less_mature): if not dataset_paths: + click.echo('Error: no datasets provided\n') print_help_msg(update_cmd) sys.exit(1) @@ -301,7 +309,8 @@ def loc_keep(new_ds, existing_ds): if not dry_run: try: - index.datasets.update(dataset, updates_allowed=updates_allowed) + index.datasets.update(dataset, updates_allowed=updates_allowed, + archive_less_mature=archive_less_mature) update_loc(dataset, existing_ds) success += 1 echo('Updated %s' % dataset.id) @@ -416,6 +425,7 @@ def info_cmd(index: Index, show_sources: bool, show_derived: bool, max_depth: int, ids: Iterable[str]) -> None: if not ids: + click.echo('Error: no datasets provided\n') print_help_msg(info_cmd) sys.exit(1) @@ -486,6 +496,7 @@ def uri_search_cmd(index: Index, paths: List[str], search_mode): PATHS may be either file paths or URIs """ if not paths: + click.echo('Error: no locations provided\n') print_help_msg(uri_search_cmd) sys.exit(1) @@ -511,6 +522,7 @@ def uri_search_cmd(index: Index, paths: List[str], search_mode): @ui.pass_index() def archive_cmd(index: Index, archive_derived: bool, dry_run: bool, all_ds: bool, ids: List[str]): if not ids and not all_ds: + click.echo('Error: no datasets provided\n') print_help_msg(archive_cmd) sys.exit(1) @@ -559,6 +571,7 @@ def archive_cmd(index: Index, archive_derived: bool, dry_run: bool, all_ds: bool def restore_cmd(index: Index, restore_derived: bool, derived_tolerance_seconds: int, dry_run: bool, all_ds: bool, ids: List[str]): if not ids and not all_ds: + click.echo('Error: no datasets provided\n') print_help_msg(restore_cmd) sys.exit(1) @@ -606,6 +619,7 @@ def within_tolerance(dataset): @ui.pass_index() def purge_cmd(index: Index, dry_run: bool, all_ds: bool, ids: List[str]): if not ids and not all_ds: + click.echo('Error: no datasets provided\n') print_help_msg(purge_cmd) sys.exit(1) diff --git a/datacube/scripts/ingest.py b/datacube/scripts/ingest.py index b86be8747c..4e74031e74 100644 --- a/datacube/scripts/ingest.py +++ b/datacube/scripts/ingest.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import time import logging diff --git a/datacube/scripts/metadata.py b/datacube/scripts/metadata.py index a4efd7f801..3b3f735c19 100644 --- a/datacube/scripts/metadata.py +++ b/datacube/scripts/metadata.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import json import logging diff --git a/datacube/scripts/product.py b/datacube/scripts/product.py index 294291493f..ef9095fb3f 100644 --- a/datacube/scripts/product.py +++ b/datacube/scripts/product.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import csv import json diff --git a/datacube/scripts/search_tool.py b/datacube/scripts/search_tool.py index aba233bb2c..25acc2204b 100755 --- a/datacube/scripts/search_tool.py +++ b/datacube/scripts/search_tool.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Query datasets. diff --git a/datacube/scripts/system.py b/datacube/scripts/system.py index eabd1e38bd..d76aebf976 100644 --- a/datacube/scripts/system.py +++ b/datacube/scripts/system.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging diff --git a/datacube/scripts/user.py b/datacube/scripts/user.py index 1c3b8ba5b3..6f4f084189 100644 --- a/datacube/scripts/user.py +++ b/datacube/scripts/user.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import click diff --git a/datacube/storage/__init__.py b/datacube/storage/__init__.py index 689d71de61..64388c59c6 100644 --- a/datacube/storage/__init__.py +++ b/datacube/storage/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Modules for creating and accessing Data Store Units diff --git a/datacube/storage/_base.py b/datacube/storage/_base.py index a7da9a1659..48508528b2 100644 --- a/datacube/storage/_base.py +++ b/datacube/storage/_base.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Optional, Dict, Any, Tuple, Callable from urllib.parse import urlparse diff --git a/datacube/storage/_hdf5.py b/datacube/storage/_hdf5.py index 4029c054e9..2b9a289839 100644 --- a/datacube/storage/_hdf5.py +++ b/datacube/storage/_hdf5.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from threading import RLock HDF5_LOCK = RLock() diff --git a/datacube/storage/_load.py b/datacube/storage/_load.py index a94e0d1a65..15edb6ac8c 100644 --- a/datacube/storage/_load.py +++ b/datacube/storage/_load.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Important functions are: diff --git a/datacube/storage/_read.py b/datacube/storage/_read.py index 08e88ce01b..2706223239 100644 --- a/datacube/storage/_read.py +++ b/datacube/storage/_read.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Dataset -> Raster """ @@ -133,13 +133,31 @@ def norm_read_args(roi, shape, extra_dim_index): pix = rdr.read(*norm_read_args(rr.roi_src, src_geobox.shape, extra_dim_index)) + # XSCALE and YSCALE are (currently) undocumented arguments that rasterio passed through to + # GDAL. Not using them results in very inaccurate warping in images with highly + # non-square (i.e. long and thin) aspect ratios. + # + # See https://github.com/OSGeo/gdal/issues/7750 as well as + # https://github.com/opendatacube/datacube-core/pull/1450 and + # https://github.com/opendatacube/datacube-core/issues/1456 + # + # In theory we might be able to get better results for queries with significantly + # different vertical and horizontal scales, but explicitly using XSCALE=1, YSCALE=1 + # appears to be most appropriate for most requests, and is demonstrably better + # than not setting them at all. + gdal_scale_params = { + "XSCALE": 1, + "YSCALE": 1, + } if rr.transform.linear is not None: A = (~src_geobox.transform)*dst_geobox.transform warp_affine(pix, dst, A, resampling, - src_nodata=rdr.nodata, dst_nodata=dst_nodata) + src_nodata=rdr.nodata, dst_nodata=dst_nodata, + **gdal_scale_params) else: rio_reproject(pix, dst, src_geobox, dst_geobox, resampling, - src_nodata=rdr.nodata, dst_nodata=dst_nodata) + src_nodata=rdr.nodata, dst_nodata=dst_nodata, + **gdal_scale_params) return rr.roi_dst diff --git a/datacube/storage/_rio.py b/datacube/storage/_rio.py index eedaf610b4..087a6fc21b 100644 --- a/datacube/storage/_rio.py +++ b/datacube/storage/_rio.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Driver implementation for Rasterio based reader. diff --git a/datacube/testutils/__init__.py b/datacube/testutils/__init__.py index 45792224ff..3db4915de2 100644 --- a/datacube/testutils/__init__.py +++ b/datacube/testutils/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Useful methods for tests (particularly: reading/writing and checking files) diff --git a/datacube/testutils/geom.py b/datacube/testutils/geom.py index f72e377d4b..b311b26e0d 100644 --- a/datacube/testutils/geom.py +++ b/datacube/testutils/geom.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np from affine import Affine diff --git a/datacube/testutils/io.py b/datacube/testutils/io.py index 8b2f8fc4d0..063b9b9d02 100644 --- a/datacube/testutils/io.py +++ b/datacube/testutils/io.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np import toolz diff --git a/datacube/testutils/iodriver.py b/datacube/testutils/iodriver.py index 4accaadb51..87b2046737 100644 --- a/datacube/testutils/iodriver.py +++ b/datacube/testutils/iodriver.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Reader driver construction for tests """ diff --git a/datacube/testutils/threads.py b/datacube/testutils/threads.py index 17ae13bba6..22bb22aeb8 100644 --- a/datacube/testutils/threads.py +++ b/datacube/testutils/threads.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ threads related stuff """ diff --git a/datacube/ui/__init__.py b/datacube/ui/__init__.py index 51ff28cf46..08fd367b1f 100644 --- a/datacube/ui/__init__.py +++ b/datacube/ui/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ User Interface Utilities diff --git a/datacube/ui/click.py b/datacube/ui/click.py index c339b8ae8e..75ab563cfb 100644 --- a/datacube/ui/click.py +++ b/datacube/ui/click.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Common functions for click-based cli scripts. @@ -340,16 +340,19 @@ def parsed_search_expressions(f): FIELD = VALUE FIELD in DATE-RANGE FIELD in [START, END] + TIME < DATE + TIME > DATE \b - DATE-RANGE is one of YYYY, YYYY-MM or YYYY-MM-DD START and END can be either numbers or dates + Dates follow YYYY, YYYY-MM, or YYYY-MM-DD format FIELD: x, y, lat, lon, time, product, ... \b eg. 'time in [1996-01-01, 1996-12-31]' 'time in 1996' + 'time > 2020-01' 'lon in [130, 140]' 'lat in [-40, -30]' product=ls5_nbar_albers diff --git a/datacube/ui/common.py b/datacube/ui/common.py index 33bc5933c5..7e8c67d280 100644 --- a/datacube/ui/common.py +++ b/datacube/ui/common.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Common methods for UI code. diff --git a/datacube/ui/expression.py b/datacube/ui/expression.py index c369631628..8ef1fdf73c 100644 --- a/datacube/ui/expression.py +++ b/datacube/ui/expression.py @@ -1,17 +1,19 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Search expression parsing for command line applications. -Three types of expressions are available: +Four types of expressions are available: FIELD = VALUE FIELD in DATE-RANGE FIELD in [START, END] + TIME > DATE + TIME < DATE -Where DATE-RANGE is one of YYYY, YYYY-MM or YYYY-MM-DD +Where DATE or DATE-RANGE is one of YYYY, YYYY-MM or YYYY-MM-DD and START, END are either numbers or dates. """ # flake8: noqa @@ -27,10 +29,14 @@ ?expression: equals_expr | time_in_expr | field_in_expr + | time_gt_expr + | time_lt_expr equals_expr: field "=" value time_in_expr: time "in" date_range field_in_expr: field "in" "[" orderable "," orderable "]" + time_gt_expr: time ">" date_gt + time_lt_expr: time "<" date_lt field: FIELD time: TIME @@ -48,6 +54,10 @@ ?date_range: date -> single_date | "[" date "," date "]" -> date_pair + date_gt: date -> range_lower_bound + + date_lt: date -> range_upper_bound + date: YEAR ["-" MONTH ["-" DAY ]] TIME: "time" @@ -86,6 +96,12 @@ def field_in_expr(self, field, lower, upper): def time_in_expr(self, time_field, date_range): return {str(time_field): date_range} + + def time_gt_expr(self, time_field, date_gt): + return {str(time_field): date_gt} + + def time_lt_expr(self, time_field, date_lt): + return {str(time_field): date_lt} # Convert the literals def string(self, val): @@ -101,6 +117,12 @@ def single_date(self, date): def date_pair(self, start, end): return _time_to_search_dims((start, end)) + + def range_lower_bound(self, date): + return _time_to_search_dims((date, None)) + + def range_upper_bound(self, date): + return _time_to_search_dims((None, date)) def date(self, y, m=None, d=None): return "-".join(x for x in [y, m, d] if x is not None) diff --git a/datacube/ui/task_app.py b/datacube/ui/task_app.py index 789ebddb4c..2077b1d3e9 100644 --- a/datacube/ui/task_app.py +++ b/datacube/ui/task_app.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import os diff --git a/datacube/utils/__init__.py b/datacube/utils/__init__.py index cf70b55f9a..09a17d156e 100644 --- a/datacube/utils/__init__.py +++ b/datacube/utils/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Utility functions diff --git a/datacube/utils/_misc.py b/datacube/utils/_misc.py index 65713fdbe8..25925fd019 100644 --- a/datacube/utils/_misc.py +++ b/datacube/utils/_misc.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Utility functions diff --git a/datacube/utils/aws/__init__.py b/datacube/utils/aws/__init__.py index 48e36676ac..a5061f7a7a 100644 --- a/datacube/utils/aws/__init__.py +++ b/datacube/utils/aws/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Helper methods for working with AWS diff --git a/datacube/utils/changes.py b/datacube/utils/changes.py index 74654de8bd..5e1cb03417 100644 --- a/datacube/utils/changes.py +++ b/datacube/utils/changes.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Validation of document/dictionary changes. diff --git a/datacube/utils/cog.py b/datacube/utils/cog.py index 6483e78e4a..4f2f39ea19 100644 --- a/datacube/utils/cog.py +++ b/datacube/utils/cog.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import warnings import toolz # type: ignore[import] diff --git a/datacube/utils/dask.py b/datacube/utils/dask.py index 9bb51a076b..e7affe0d14 100644 --- a/datacube/utils/dask.py +++ b/datacube/utils/dask.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Dask Distributed Tools diff --git a/datacube/utils/dates.py b/datacube/utils/dates.py index fb6671106c..293bbf38c8 100644 --- a/datacube/utils/dates.py +++ b/datacube/utils/dates.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Date and time utility functions diff --git a/datacube/utils/documents.py b/datacube/utils/documents.py index 13006d1fb7..8dc6f2e495 100644 --- a/datacube/utils/documents.py +++ b/datacube/utils/documents.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Functions for working with YAML documents and configurations diff --git a/datacube/utils/generic.py b/datacube/utils/generic.py index 327a10f9d0..0bef2fd977 100644 --- a/datacube/utils/generic.py +++ b/datacube/utils/generic.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import itertools import threading diff --git a/datacube/utils/geometry/__init__.py b/datacube/utils/geometry/__init__.py index 1a023611a0..1d5a6b2057 100644 --- a/datacube/utils/geometry/__init__.py +++ b/datacube/utils/geometry/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Geometric shapes and operations on them """ diff --git a/datacube/utils/geometry/_base.py b/datacube/utils/geometry/_base.py index 7647ca9661..a0cc2a4c9f 100644 --- a/datacube/utils/geometry/_base.py +++ b/datacube/utils/geometry/_base.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import functools import itertools diff --git a/datacube/utils/geometry/_warp.py b/datacube/utils/geometry/_warp.py index bbe2862a92..c2e2e6e674 100644 --- a/datacube/utils/geometry/_warp.py +++ b/datacube/utils/geometry/_warp.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Union, Optional import rasterio.warp # type: ignore[import] diff --git a/datacube/utils/geometry/gbox.py b/datacube/utils/geometry/gbox.py index 2cdeb80818..bd32f79f14 100644 --- a/datacube/utils/geometry/gbox.py +++ b/datacube/utils/geometry/gbox.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Geometric operations on GeoBox class """ diff --git a/datacube/utils/geometry/tools.py b/datacube/utils/geometry/tools.py index 15cca6006d..d698d20a3b 100644 --- a/datacube/utils/geometry/tools.py +++ b/datacube/utils/geometry/tools.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np import collections.abc diff --git a/datacube/utils/io.py b/datacube/utils/io.py index 85d50eb640..983ffeeb29 100644 --- a/datacube/utils/io.py +++ b/datacube/utils/io.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import os from pathlib import Path diff --git a/datacube/utils/masking.py b/datacube/utils/masking.py index 3c35b1c422..5a6c8467b5 100644 --- a/datacube/utils/masking.py +++ b/datacube/utils/masking.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tools for masking data based on a bit-mask variable with attached definition. diff --git a/datacube/utils/math.py b/datacube/utils/math.py index d97419d902..2026489448 100644 --- a/datacube/utils/math.py +++ b/datacube/utils/math.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Tuple, Union, Optional, Any from math import ceil diff --git a/datacube/utils/py.py b/datacube/utils/py.py index 456f24bf28..f869c8052b 100644 --- a/datacube/utils/py.py +++ b/datacube/utils/py.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import importlib import logging diff --git a/datacube/utils/rio/__init__.py b/datacube/utils/rio/__init__.py index 55db80b776..93128d0c26 100644 --- a/datacube/utils/rio/__init__.py +++ b/datacube/utils/rio/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ This will move into IO driver eventually. diff --git a/datacube/utils/rio/_rio.py b/datacube/utils/rio/_rio.py index 5d176bbdc3..04b2e9c7fd 100644 --- a/datacube/utils/rio/_rio.py +++ b/datacube/utils/rio/_rio.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ rasterio environment management tools """ diff --git a/datacube/utils/serialise.py b/datacube/utils/serialise.py index 8f180d7a2c..8306bc0386 100644 --- a/datacube/utils/serialise.py +++ b/datacube/utils/serialise.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Serialise function used in YAML output diff --git a/datacube/utils/uris.py b/datacube/utils/uris.py index 0abfb6cde8..7644e954a3 100644 --- a/datacube/utils/uris.py +++ b/datacube/utils/uris.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import os diff --git a/datacube/utils/xarray_geoextensions.py b/datacube/utils/xarray_geoextensions.py index 536abcad5a..fa0aa52275 100644 --- a/datacube/utils/xarray_geoextensions.py +++ b/datacube/utils/xarray_geoextensions.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Add geometric extensions to :class:`xarray.Dataset` and :class:`xarray.DataArray` for use diff --git a/datacube/virtual/__init__.py b/datacube/virtual/__init__.py index 16793105c3..b3de023514 100644 --- a/datacube/virtual/__init__.py +++ b/datacube/virtual/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Mapping, Any, cast import copy diff --git a/datacube/virtual/catalog.py b/datacube/virtual/catalog.py index aab9006043..e908367eef 100644 --- a/datacube/virtual/catalog.py +++ b/datacube/virtual/catalog.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Catalog of virtual products. diff --git a/datacube/virtual/expr.py b/datacube/virtual/expr.py index 23fb9f08a6..e19894ac70 100644 --- a/datacube/virtual/expr.py +++ b/datacube/virtual/expr.py @@ -1,3 +1,7 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2023 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 import lark import numpy diff --git a/datacube/virtual/impl.py b/datacube/virtual/impl.py index 624028c244..dc1c0524ab 100644 --- a/datacube/virtual/impl.py +++ b/datacube/virtual/impl.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Implementation of virtual products. Provides an interface for the products in the datacube diff --git a/datacube/virtual/transformations.py b/datacube/virtual/transformations.py index 9c929eacc7..2aeaf64b67 100644 --- a/datacube/virtual/transformations.py +++ b/datacube/virtual/transformations.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from typing import Optional, Collection import warnings diff --git a/datacube/virtual/utils.py b/datacube/virtual/utils.py index 27cb0acada..c26a7bde8b 100644 --- a/datacube/virtual/utils.py +++ b/datacube/virtual/utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Utilities to facilitate virtual product implementation. """ diff --git a/docker/assets/with_bootstrap b/docker/assets/with_bootstrap index 5da8866f65..02bdc77915 100755 --- a/docker/assets/with_bootstrap +++ b/docker/assets/with_bootstrap @@ -14,8 +14,8 @@ launch_db () { sudo -u postgres createuser --superuser "${dbuser}" sudo -u postgres createdb "${dbuser}" sudo -u postgres createdb datacube - sudo -u postgres createdb agdcintegration - sudo -u postgres createdb odcintegration + sudo -u postgres createdb pgintegration + sudo -u postgres createdb pgisintegration } # Become `odc` user with UID/GID compatible to datacube-core volume @@ -58,12 +58,12 @@ launch_db () { cat < $HOME/.datacube_integration.conf [datacube] db_hostname: -db_database: agdcintegration +db_database: pgintegration index_driver: default [experimental] db_hostname: -db_database: odcintegration +db_database: pgisintegration index_driver: postgis [no_such_driver_env] diff --git a/docs/about-core-concepts/metadata-types.rst b/docs/about-core-concepts/metadata-types.rst index 323f62e470..a101af4203 100644 --- a/docs/about-core-concepts/metadata-types.rst +++ b/docs/about-core-concepts/metadata-types.rst @@ -8,4 +8,7 @@ Metadata Types Metadata type yaml file must contain name, description and dataset keys. - Dataset key must contain id, sources, creation_dt, label and search_fields keys. + Dataset key must contain id, sources, creation_dt, label, and search_fields keys. + + For metadata types of spatial datasets, the dataset key must also contain grid_spatial, measurements, and format keys. + Support for non-spatial datasets is likely to be dropped in version 2.0. diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index 902c25e768..10f39f5e02 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -20,9 +20,31 @@ v1.9.next v1.8.next ========= -- Documentation fixes (:pull:`1417`, :pull:`1418`) +- Second attempt to address unexpected handling of image aspect ratios in rasterio and + GDAL. (:pull:`1457`) +- Fix broken pypi publishing Github action (:pull:`1454`) +- Documentation improvements (:pull:`1455`) +- Increase maturity leniency to +-500ms (:pull:`1458`) +v1.8.13 (6th June 2023) +======================= + +- Fix broken Github action workflows (:pull:`1425`, :pull:`1427`, :pull:`1433`) +- Setup Dependabot, and Dependabot-generated updates (:pull:`1416`, :pull:`1420`, :pull:`1423`, + :pull:`1428`, :pull:`1436`, :pull:`1447`) +- Documentation fixes (:pull:`1417`, :pull:`1418`, :pull:`1430`) +- ``datacube dataset`` cli commands print error message if missing argument (:pull:`1437`) +- Add pre-commit hook to verify license headers (:pull:`1438`) +- Support open-ended date ranges in `datacube dataset search`, `dc.load`, and `dc.find_datasets` (:pull:`1439`, :pull:`1443`) +- Pass Y and Y Scale factors through to rasterio.warp.reproject, to eliminate projection bug affecting + non-square Areas Of Interest (See `Issue #1448`_) (:pull:`1450`) +- Add `archive_less_mature` option to `datacube dataset add` and `datacube dataset update` (:pull:`1451`) +- Allow for +-1ms leniency in finding other maturity versions of a dataset (:pull:`1452`) +- Update whats_new.rst for release (:pull:`1453`) + +.. _`Issue #1448`: https://github.com/opendatacube/datacube-core/issues/1448 + v1.8.12 (7th March 2023) ======================== diff --git a/docs/click_utils.py b/docs/click_utils.py index 0b658d8eef..7558480895 100644 --- a/docs/click_utils.py +++ b/docs/click_utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pkg_resources from docutils.nodes import literal_block, section, title, make_id diff --git a/docs/conf.py b/docs/conf.py index bbcbec7b12..3b63a7cdb7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import os import sys diff --git a/docs/config_samples/ingester/ls5_nbar_albers.yaml b/docs/config_samples/ingester/ls5_nbar_albers.yaml index 42cfc44127..d37747848e 100644 --- a/docs/config_samples/ingester/ls5_nbar_albers.yaml +++ b/docs/config_samples/ingester/ls5_nbar_albers.yaml @@ -39,10 +39,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -57,11 +57,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-N_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: TM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -77,30 +77,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+ 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls5_nbart_albers.yaml b/docs/config_samples/ingester/ls5_nbart_albers.yaml index 89d942cb8d..a848a6af79 100644 --- a/docs/config_samples/ingester/ls5_nbart_albers.yaml +++ b/docs/config_samples/ingester/ls5_nbart_albers.yaml @@ -45,10 +45,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -63,11 +63,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-NT_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: TM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -83,30 +83,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+T 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls5_pq_albers.yaml b/docs/config_samples/ingester/ls5_pq_albers.yaml index 491f30b020..4eda5e55e6 100644 --- a/docs/config_samples/ingester/ls5_pq_albers.yaml +++ b/docs/config_samples/ingester/ls5_pq_albers.yaml @@ -19,11 +19,11 @@ global_attributes: or only sea pixels depending on their analytical requirements, leading to enhanced computationally efficient. PQ provides an assessment of the quality of observations at a pixel level and includes information about: - - Spectral Contiguity (lack of signal in any band) - - Saturation in any band - - Presence of cloud - - Presence of cloud shadow - - Land or sea + - Spectral Contiguity (lack of signal in any band) + - Saturation in any band + - Presence of cloud + - Presence of cloud shadow + - Land or sea As Landsat Imagery becomes more readily available, there has been a rapid increase in the amount of analyses undertaken by researchers around the globe. Most researchers use some form of quality masking schema in order to remove undesirable @@ -43,11 +43,11 @@ global_attributes: with OLI data, however it uses additional algorithms to detect cloud and cloud shadow, and is available for Landsat 5, 7 and 8. source: PQ_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: TM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,ETM+,TM,OLI,EARTH SCIENCE diff --git a/docs/config_samples/ingester/ls7_nbar_albers.yaml b/docs/config_samples/ingester/ls7_nbar_albers.yaml index 3ddba85437..fd3aa6b9e3 100644 --- a/docs/config_samples/ingester/ls7_nbar_albers.yaml +++ b/docs/config_samples/ingester/ls7_nbar_albers.yaml @@ -39,10 +39,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -57,11 +57,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-N_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: ETM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -77,30 +77,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+ 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls7_nbart_albers.yaml b/docs/config_samples/ingester/ls7_nbart_albers.yaml index 136927ad22..0f31c91004 100644 --- a/docs/config_samples/ingester/ls7_nbart_albers.yaml +++ b/docs/config_samples/ingester/ls7_nbart_albers.yaml @@ -45,10 +45,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -63,11 +63,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-NT_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: ETM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -83,30 +83,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+T 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls7_pq_albers.yaml b/docs/config_samples/ingester/ls7_pq_albers.yaml index 02b4a4bde1..ecd6c2deeb 100644 --- a/docs/config_samples/ingester/ls7_pq_albers.yaml +++ b/docs/config_samples/ingester/ls7_pq_albers.yaml @@ -19,11 +19,11 @@ global_attributes: or only sea pixels depending on their analytical requirements, leading to enhanced computationally efficient. PQ provides an assessment of the quality of observations at a pixel level and includes information about: - - Spectral Contiguity (lack of signal in any band) - - Saturation in any band - - Presence of cloud - - Presence of cloud shadow - - Land or sea + - Spectral Contiguity (lack of signal in any band) + - Saturation in any band + - Presence of cloud + - Presence of cloud shadow + - Land or sea As Landsat Imagery becomes more readily available, there has been a rapid increase in the amount of analyses undertaken by researchers around the globe. Most researchers use some form of quality masking schema in order to remove undesirable @@ -43,11 +43,11 @@ global_attributes: with OLI data, however it uses additional algorithms to detect cloud and cloud shadow, and is available for Landsat 5, 7 and 8. source: PQ_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: ETM keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,ETM+,TM,OLI,EARTH SCIENCE diff --git a/docs/config_samples/ingester/ls8_nbar_albers.yaml b/docs/config_samples/ingester/ls8_nbar_albers.yaml index 3139007794..0e7b6205f9 100644 --- a/docs/config_samples/ingester/ls8_nbar_albers.yaml +++ b/docs/config_samples/ingester/ls8_nbar_albers.yaml @@ -39,10 +39,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -57,11 +57,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-N_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: OLI keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -77,30 +77,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+ 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls8_nbart_albers.yaml b/docs/config_samples/ingester/ls8_nbart_albers.yaml index 42899fd4c9..27f22bf8cb 100644 --- a/docs/config_samples/ingester/ls8_nbart_albers.yaml +++ b/docs/config_samples/ingester/ls8_nbart_albers.yaml @@ -45,10 +45,10 @@ global_attributes: Surface Reflectance Correction Models Image radiance values recorded by passive EO sensors are a composite of - - surface reflectance; - - atmospheric condition; - - interaction between surface land cover, solar radiation and sensor view angle; - - land surface orientation relative to the imaging sensor. + - surface reflectance; + - atmospheric condition; + - interaction between surface land cover, solar radiation and sensor view angle; + - land surface orientation relative to the imaging sensor. It has been traditionally assumed that Landsat imagery display negligible variation in sun and sensor view angles, however these can vary significantly both within and between scenes, especially in different seasons and geographic regions (Li et al., 2012). The SR product delivers modeled surface reflectance from Landsat TM/ETM+/OLI @@ -63,11 +63,11 @@ global_attributes: Given the growing time series of EO imagery, this landmark facility will streamline the process of reliably monitoring long-term chnges in land and water resources. source: SR-NT_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: OLI keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE @@ -83,30 +83,30 @@ global_attributes: product_suite: Surface Reflectance NBAR+T 25m acknowledgment: Landsat data is provided by the United States Geological Survey (USGS) through direct reception of the data at Geoscience Australias satellite reception facility or download. references: | - - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. - - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. - - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. - - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. - - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. - - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. - - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. - - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. - - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. - - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. - - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. - - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. - - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. - - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. - - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. - - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. - - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf - - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. - - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. - - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. - - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. - - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. - - http://dx.doi.org/10.4225/25/5487CC0D4F40B - - http://dx.doi.org/10.1109/JSTARS.2010.2042281 + - Berk, A., Anderson, G.P., Acharya, P.K., Hoke, M.L., Chetwynd, J.H., Bernstein, L.S., Shettle, E.P., Matthew, M.W., and Adler-Golden, S.M. (2003) Modtran 4 Version 3 Revision 1 User s manual. Airforce Research Laboratory, Hanscom, MA, USA. + - Chander, G., Markham, B.L., and Helder, D.L. (2009) Summary of current radiometric calibration coefficients for Landsat MSS, TM, ETM+, and EO-1 ALI sensors. Remote Sensing of Environment 113, 893-903. + - Edberg, R., and Oliver, S. (2013) Projection-Independent Earth-Solar-Sensor Geometry for Surface Reflectance Correction. Submitted to IGARSS 2013, Melbourne. + - GA and CSIRO (2010) 1 second SRTM Derived Digital Elevation Models User Guide. Version 1.03. GA, Canberra. + - Forrest, R.B. (1981) Simulation of orbital image-sensor geometry, Photogrammetric Engineering and Remote Sensing 47, 1187-93. + - Irish, R. (2000) Landsat 7 Automatic Cloud Cover Assessment, sourced: http://landsathandbook.gsfc.nasa.gov/pdfs/ACCA_SPIE_paper.pdf, last accessed 12/11/2012. + - Irish, R.R., Barker, J.L., Goward, S.N., Arvidson, T. (2006) Characterization of the Landsat-7 ETM+ Automated Cloud -Cover Assessment (ACCA) Algorithm, Photogrammetric Engineering & Remote Sensing 72 (10), 1179-88. + - Irons, J.R., Dwyer, J.L., and Barsi, J.A. (2012) The next Landsat satellite: The Landsat Data Continuity Mission. Remote Sensing of Environment (2012), doi:10.1016/j.rse.2011.08.026. + - Kalnay, E. Kanamitsu, M., Kistler, R., Collins, W., Deaven, D., Gandin, L., Iredell, M., Saha, S., White, G., Woollen, J., Zhu, Y., Chelliah, M., Ebisuzaki, W., Higgins, W., Janowiak, J., Mo, K.C., Ropelewski, C., Wang, J., Leetmaa, A., Reynolds, R. Jenne, R., Joseph, D. (1996) The NCEP/NCAR 40-Year Reanalysis Project. Bulletin of the American Meteorological Society 77, 437-71. + - Li, F., Jupp, D.L.B., Reddy, S., Lymburner, L., Mueller, N., Tan, P., and Islam, A. (2010) An Evaluation of the Use of Atmospheric and BRDF Correction to Standardize Landsat Data. IEEE J. Selected Topics in Applied Earth Observations and Remote Sensing 3, 257-70. + - Li, F. (2010) ARG25 Algorithm Theoretical Basis Document. GA, Canberra. + - Li, F., Jupp, D.L.B., Thankappan, M., Lymburner, L., Mueller, N., Lewis, A., and Held, A. (2012) A physics-based atmopheric and BRDF correction for Landsat data over mountainous terrain. Remote Sensing of Environment 124, 756-70. + - Lubke, M. (2012) Landsat Geometry Calibration/Validation Update. Presentation at LTWG #21, 25 September 2012, Sioux Falls. USGS, USA. + - OGC (2006) OpenGIS Web Map Server Implementation Specification (Ed: Jeff de la Beaujardiere) Ref. OGC 06-042. + - OGC (2010) OGC WCS 2.0 Interface Standard - Core. (Ed: Peter Baumann) Ref. OGC 09-110r3. + - OGC (2013) CF-netCDF3 Data Model Extension Standard (Eds: Ben Domenico and Stefano Nativi) Ref. OGC 11-165r2. + - Strahler, A.H., and Muller, J.-P. (1999) MODIS BRDF/Albedo Product: Algorithm Theoretical Basis Document Version 5.0. http://modis.gsfc.nasa.gov/data/atbd/atbd_mod09.pdf + - TM World Borders vector file: http://thematicmapping.org/downloads/world_borders.php. + - USGS (2012a) Landsat Thematic Mapper (TM) Level 1 (L1) Data Format Control Book (DFCB). LS-DFCB-20 Version 4.0. USGS, USA. http://landsat.usgs.gov/documents/LS-DFCB-20.pdf. + - USGS (2012b) Landsat 7 ETM+ Level 1 Product Data Format Control Book (DFCB). LS-DFCB-04 Version 15.0. http://landsat.usgs.gov/documents/LS-DFCB-04.pdf. + - Vincenty, T. (1975) Direct and Inverse Solutions of Geodesies on the Ellipsoid with Application of Nested Equations. Survey Review 23, 88-93. + - Zhu, Z. and Woodcock, C. E. (2012) Object-based cloud and cloud shadow detection in Landsat imagery. Remote Sensing of Environment 118, 83-94. + - http://dx.doi.org/10.4225/25/5487CC0D4F40B + - http://dx.doi.org/10.1109/JSTARS.2010.2042281 storage: driver: NetCDF CF diff --git a/docs/config_samples/ingester/ls8_pq_albers.yaml b/docs/config_samples/ingester/ls8_pq_albers.yaml index 184cb81983..6caad8979f 100644 --- a/docs/config_samples/ingester/ls8_pq_albers.yaml +++ b/docs/config_samples/ingester/ls8_pq_albers.yaml @@ -19,11 +19,11 @@ global_attributes: or only sea pixels depending on their analytical requirements, leading to enhanced computationally efficient. PQ provides an assessment of the quality of observations at a pixel level and includes information about: - - Spectral Contiguity (lack of signal in any band) - - Saturation in any band - - Presence of cloud - - Presence of cloud shadow - - Land or sea + - Spectral Contiguity (lack of signal in any band) + - Saturation in any band + - Presence of cloud + - Presence of cloud shadow + - Land or sea As Landsat Imagery becomes more readily available, there has been a rapid increase in the amount of analyses undertaken by researchers around the globe. Most researchers use some form of quality masking schema in order to remove undesirable @@ -43,11 +43,11 @@ global_attributes: with OLI data, however it uses additional algorithms to detect cloud and cloud shadow, and is available for Landsat 5, 7 and 8. source: PQ_25_2 history: | - - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing - - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. - - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. - - DEM: 1 second SRTM DSM is used for Ortho-rectification. - - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. + - Ground Control Points (GCP): new GCP chips released by USGS in Dec 2015 are used for re-processing + - Geometric QA: each product undergoes geometric assessment and the assessment result will be recorded within v2 AGDC for filtering/masking purposes. + - Processing parameter settings: the minimum number of GCPs for Ortho-rectified product generation has been reduced from 30 to 10. + - DEM: 1 second SRTM DSM is used for Ortho-rectification. + - Updated Calibration Parameter File (CPF): the latest/current CPF is used for processing. institution: Commonwealth of Australia (Geoscience Australia) instrument: OLI keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,ETM+,TM,OLI,EARTH SCIENCE diff --git a/docs/config_samples/ingester/s2amsil1c_albers_10.yaml b/docs/config_samples/ingester/s2amsil1c_albers_10.yaml index c9701a6d2a..d7612e8b72 100644 --- a/docs/config_samples/ingester/s2amsil1c_albers_10.yaml +++ b/docs/config_samples/ingester/s2amsil1c_albers_10.yaml @@ -81,5 +81,5 @@ measurements: src_varname: '08' zlib: True attrs: - long_name: " Level 1 Top of Atmosphere Reflectance 842 nanometers (Near infrared)" + long_name: " Level 1 Top of Atmosphere Reflectance 842 nanometers (Near infrared)" alias: "Band8" diff --git a/docs/config_samples/metadata_types/bare_bone.yaml b/docs/config_samples/metadata_types/bare_bone.yaml index 6f6c3eda32..8ffc642482 100644 --- a/docs/config_samples/metadata_types/bare_bone.yaml +++ b/docs/config_samples/metadata_types/bare_bone.yaml @@ -2,10 +2,16 @@ name: barebone description: A minimalist metadata type file dataset: - id: [id] # No longer configurable in newer ODCs. - sources: [lineage, source_datasets] # No longer configurable in newer ODCs. + id: [id] # No longer configurable in newer ODCs. + sources: [lineage, source_datasets] # No longer configurable in newer ODCs. + creation_dt: [properties, 'odc:processing_datetime'] label: [label] + # The following keys are necessary if describing spatial datasets + # grid_spatial: [grid_spatial, projection] + # measurements: [measurements] + # format: [properties, 'odc:file_format'] + search_fields: platform: description: Platform code diff --git a/docs/installation/data-preparation-scripts.rst b/docs/installation/data-preparation-scripts.rst deleted file mode 100644 index b80dcde151..0000000000 --- a/docs/installation/data-preparation-scripts.rst +++ /dev/null @@ -1,147 +0,0 @@ -Data Preperation Scripts -======================== - -.. note:: - - Much of the below content is not updated and has not been tested recently. - - -Sometimes data to load into an Open Data Cube will come packaged with -compatible :ref:`dataset-metadata-doc` and be ready to :ref:`index ` -immediately. - -In other cases you will need to generate these :ref:`dataset-metadata-doc` yourself. -This is done using a ``Dataset Preparation Script``, which reads whatever format metadata -has been supplied with the data, and either writes out ODC compatible files, or adds -records directly to an ODC database. - -For common distribution formats there is likely to be a script already, but -other formats will require one to be modified. - -.. admonition:: Existing dataset-metadata-docs - :class: tip - - Examples of prepare scripts are found in the `datacube-dataset-config `_ repository - on Github. - - -Landsat Samples -=============== - -The two examples below show preparing USGS Landsat data for indexing into an Open Data Cube: - - -1. Preparing USGS Landsat Collection 1 - LEVEL1 -=============================================== - -Download the USGS Collection 1 landsat scenes from any of the links below: - -* `Earth-Explorer `_ -* `GloVis `_ -* `ESPA ordering `_ - -The prepare script for collection 1 - level 1 data is available in -`ls_usgs_prepare.py -`_. - -:: - - $ wget https://github.com/opendatacube/datacube-dataset-config/raw/master/old-prep-scripts/ls_usgs_prepare.py - $ python ls_usgs_prepare.py --help - Usage: ls_usgs_prepare.py [OPTIONS] [DATASETS]... - - Prepare USGS Landsat Collection 1 data for ingestion into the Data Cube. - This prepare script supports only for MTL.txt metadata file - To Set the Path for referring the datasets - - Download the Landsat scene data from Earth Explorer or GloVis into - 'some_space_available_folder' and unpack the file. - For example: yourscript.py --output [Yaml- which writes datasets into this file for indexing] - [Path for dataset as : /home/some_space_available_folder/] - - Options: - --output PATH Write datasets into this file - --help Show this message and exit. - - $ python ls_usgs_prepare.py --output ls8_usgs_lv1 ~/earth_explorer/Collection1/LANDSAT8 - -*ls8_usgs_lv1* is the output for required dataset for landsat 8 scene. - -To add the product definitions: - -For Landsat collection 1 level 1 product: - -:: - - $ datacube product add docs/config_samples/dataset_types/ls_usgs.yaml - Added "ls8_level1_usgs" - Added "ls7_level1_usgs" - Added "ls5_level1_usgs" - Added "ls8_l1_pc_usgs" - - -2. Preparing USGS Landsat Surface Reflectance - LEDAPS -====================================================== - -To prepare downloaded USGS LEDAPS Landsat scenes for use with the Data Cube, use -the script provided in -`usgs_ls_ard_prepare.py -`_ - -The following example generates the required Dataset Metadata files, named -`agdc-metadata.yaml` for three landsat scenes. - -:: - - $ wget https://github.com/opendatacube/datacube-dataset-config/raw/master/agdcv2-ingest/prepare_scripts/landsat_collection/usgs_ls_ard_prepare.py - $ python USGS_precollection_oldscripts/usgslsprepare.py --help - Usage: usgslsprepare.py [OPTIONS] [DATASETS]... - - Prepare USGS LS dataset for ingestion into the Data Cube. - - Options: - --help Show this message and exit. - - $ python usgslsprepare.py ~/USGS_LandsatLEDAPS/*/ - 2016-06-09 15:32:51,641 INFO Processing ~/USGS_LandsatLEDAPS/LC80960852015365-SC20160211222236 - 2016-06-09 15:32:52,096 INFO Writing ~/USGS_LandsatLEDAPS/LC80960852015365-SC20160211222236/agdc-metadata.yaml - 2016-06-09 15:32:52,119 INFO Processing ~/USGS_LandsatLEDAPS/LE70960852016024-SC20160211221824 - 2016-06-09 15:32:52,137 INFO Writing ~/USGS_LandsatLEDAPS/LE70960852016024-SC20160211221824/agdc-metadata.yaml - 2016-06-09 15:32:52,151 INFO Processing ~/USGS_LandsatLEDAPS/LT50960852011290-SC20160211221617 - 2016-06-09 15:32:52,157 INFO Writing ~/USGS_LandsatLEDAPS/LT50960852011290-SC20160211221617/agdc-metadata.yaml - - -The scenes are now ready to be :ref:`indexed ` and accessed using -the Data Cube. - -For Landsat Surface reflectance LEDAPS add: - -:: - - $ datacube product add docs/config_samples/dataset_types/* - ... - Added "ls5_ledaps_scene" - ... - Added "ls7_ledaps_scene" - ... - Added "ls8_ledaps_scene" - ... - -Then :ref:`index the data `. - -3. Indexing data on AWS, an example using Sentinel-2 -==================================================== - -To view an example of how to `index Sentinel-2 data from S3`_ check out the documentation -available in the datacube-dataset-config_ repository. - -.. _`index Sentinel-2 data from S3`: https://github.com/opendatacube/datacube-dataset-config/blob/master/sentinel-2-l2a-cogs.md -.. _datacube-dataset-config: https://github.com/opendatacube/datacube-dataset-config/ - -Custom Prepare Scripts -====================== - -We expect that many new Data Cube instances will require custom prepare scripts -to be written. It is generally a straightforward task of mapping metadata from -one form to another and writing out a YAML document. The code need not even be -written in Python, although starting with one of our examples is generally -the easiest way. diff --git a/docs/installation/database/setup.rst b/docs/installation/database/setup.rst index 74f8154cfd..4a758fa265 100644 --- a/docs/installation/database/setup.rst +++ b/docs/installation/database/setup.rst @@ -81,6 +81,18 @@ Alternately, you can configure the ODC connection to Postgres using environment DB_PASSWORD DB_DATABASE +To configure a database as a single connection url instead of individual environment variables:: + + export DATACUBE_DB_URL=postgresql://[username]:[password]@[hostname]:[port]/[database] + +Alternatively, for password-less access to a database on localhost:: + + export DATACUBE_DB_URL=postgresql:///[database] + +Further information on database configuration can be found `here `__. +Although the enhancement proposal details incoming changes in v1.9 and beyond, it should largely be compatible with the current behaviour, barring a few +obscure corner cases. + The desired environment can be specified: 1. in code, with the ``env`` argument to the ``datacube.Datacube`` constructor; diff --git a/docs/installation/index.rst b/docs/installation/index.rst index f2422569ce..7a811cbaf6 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -53,5 +53,4 @@ This section contains information on setting up and managing the Open Data Cube. .. toctree:: :caption: Legacy Approaches - data-preparation-scripts ingesting-data/index diff --git a/docs/installation/indexing-data/step-guide.rst b/docs/installation/indexing-data/step-guide.rst index 48010c57d3..d005d8265e 100644 --- a/docs/installation/indexing-data/step-guide.rst +++ b/docs/installation/indexing-data/step-guide.rst @@ -72,16 +72,9 @@ for searching, querying and accessing the data. The data from Geoscience Australia already comes with relevant files (named ``ga-metadata.yaml``), so no further steps are required for indexing them. -For third party datasets, see :ref:`prepare-scripts`. - - -.. admonition:: Note - - :class: info - - Some metadata requires cleanup before they are ready to be loaded. - -For more information see :ref:`dataset-metadata-doc`. +For third party datasets, see the examples detailed `here `__. +For common distribution formats, data can be indexed using one of the tools from `odc-apps-dc-tools `__. +In other cases, the metadata may need to be mapped to an ODC-compatible format. You can find examples of data preparation scripts `here `__. Step 3. Run the Indexing process diff --git a/docs/installation/ingesting-data/index.rst b/docs/installation/ingesting-data/index.rst index 021a2a3ce9..737be508d0 100644 --- a/docs/installation/ingesting-data/index.rst +++ b/docs/installation/ingesting-data/index.rst @@ -7,25 +7,9 @@ Ingesting Data .. note:: Ingestion is no longer recommended. While it was used as an optimised on-disk - storage mechanism, there are a range of reasons why this is no longer ideal. For example + storage mechanism, there are a range of reasons why this is no longer ideal. For example, the emergence of cloud optimised storage formats means that software such - as GDAL and Rasterio are optimised for reading many files over the network. Additionally - the limitation of NetCDF reading to a single thread means that reading from .TIF - files on disk could be faster in some situations. - - In addition to limited performance improvements, ingestion leads to duplication - of data and opinionated decisions, such as reprojection of data, which can lead - to a loss of data fidelity. - - The section below is being retained for completion, but should be considered optional. - - -.. note:: - - Ingestion is no longer recommended. While it was used as an optimised on-disk - storage mechanism, there are a range of reasons why this is no longer ideal. For example - the emergence of cloud optimised storage formats means that software such - as GDAL and Rasterio are optimised for reading many files over the network. Additionally + as GDAL and Rasterio are optimised for reading many files over the network. Additionally, the limitation of NetCDF reading to a single thread means that reading from .TIF files on disk could be faster in some situations. diff --git a/docs/installation/metadata-types.rst b/docs/installation/metadata-types.rst index 0f206016bf..9a4ac538a2 100644 --- a/docs/installation/metadata-types.rst +++ b/docs/installation/metadata-types.rst @@ -5,15 +5,17 @@ A Metadata Type defines which fields should be searchable in your product or dat Three metadata types are added by default called ``eo``, ``telemetry`` and ``eo3``. +You can see the default metadata types in the repository at `datacube/index/default-metadata-types.yaml `_. + You would create a new metadata type if you want custom fields to be searchable for your products, or if you want to structure your metadata documents differently. -You can see the default metadata type in the repository at `datacube/index/default-metadata-types.yaml `_. - To add or alter metadata types, you can use commands like: ``datacube metadata add `` and to update: ``datacube metadata update ``. Using ``--allow-unsafe`` will allow you to update metadata types where the changes may have unexpected consequences. +Note that the postgis driver only supports eo3-compatible metadata types, and from version 2.0 onward, support for non-eo3-compatible metadata types +will be fully deprecated. .. literalinclude:: ../config_samples/metadata_types/bare_bone.yaml :language: yaml @@ -22,4 +24,7 @@ you to update metadata types where the changes may have unexpected consequences. Metadata type yaml file must contain name, description and dataset keys. - Dataset key must contain id, sources, creation_dt, label and search_fields keys. + Dataset key must contain id, sources, creation_dt, label, and search_fields keys. + + For metadata types of spatial datasets, the dataset key must also contain grid_spatial, measurements, and format keys. + Support for non-spatial datasets is likely to be dropped in version 2.0. diff --git a/docs/installation/setup/common_install.rst b/docs/installation/setup/common_install.rst index 5e5966fd0d..49deaa20a3 100644 --- a/docs/installation/setup/common_install.rst +++ b/docs/installation/setup/common_install.rst @@ -7,36 +7,32 @@ Python and packages Python 3.8+ is required. -Anaconda Python ---------------- - -`Install Anaconda Python `_ - -Add conda-forge to package channels:: - - conda config --append channels conda-forge +Conda environment setup +----------------------- Conda environments are recommended for use in isolating your ODC development environment from your system installation and other Python environments. -Install required Python packages and create a conda environment named ``odc_env``. +We recommend you use Mambaforge to set up your conda virtual environment, as all the required packages are obtained from the conda-forge channel. +Download and install it from `here `_. -Python:: +Download the latest version of the Open Data Cube from the `repository `_:: - conda create --name odc_env python=3.8 datacube + git clone https://github.com/opendatacube/datacube-core + cd datacube-core -Activate the ``odc_env`` conda environment:: +Create a conda environment named ``cubeenv``:: - conda activate odc_env + mamba env create -f conda-environment.yml -Find out more about conda environments `here `_. +Activate the ``cubeenv`` conda environment:: -Install other packages:: + conda activate cubeenv - conda install jupyter matplotlib scipy pytest-cov hypothesis +Find out more about conda environments `here `_. -Postgres database configuration -=============================== +Postgres testing database configuration +======================================= This configuration supports local development using your login name. @@ -52,7 +48,7 @@ Set a password for the "postgres" database role using the command:: and set the password when prompted. The password text will be hidden from the console for security purposes. -Type **Control+D** or **\q** to exit the posgreSQL prompt. +Type **Control+D** or **\\q** to exit the posgreSQL prompt. By default in Ubuntu, Postgresql is configured to use ``ident sameuser`` authentication for any connections from the same machine which is useful for development. Check out the excellent Postgresql documentation for more information, but essentially this means that if your Ubuntu username is ``foo`` and you add ``foo`` as a Postgresql user then you can connect to a database without requiring a password for many functions. @@ -61,51 +57,19 @@ Since the only user who can connect to a fresh install is the postgres user, her sudo -u postgres createuser --superuser $USER sudo -u postgres psql - postgres=# \password $USER + postgres=# \password -Now we can create an ``agdcintegration`` database for testing:: +Now we can create databases for integration testing. You will need 2 databases - one for the Postgres driver and one for the PostGIS driver. +By default, these databases are called ``pgintegration`` and ``pgisintegration``, but you can name them however you want:: - createdb agdcintegration + postgres=# create database pgintegration; + postgres=# create database pgisintegration; + +Or, directly from the bash terminal:: -Connecting to your own database to try out some SQL should now be as easy as:: - - psql -d agdcintegration - - -Open Data Cube source and development configuration -=================================================== + createdb pgintegration + createdb pgisintegration -Download the latest version of the software from the `repository `_ :: - - git clone https://github.com/opendatacube/datacube-core - cd datacube-core - -We need to specify the database user and password for the ODC integration testing. To do this:: - - cp integration_tests/agdcintegration.conf ~/.datacube_integration.conf - -Then edit the ``~/.datacube_integration.conf`` with a text editor and add the following lines replacing ```` with your username and ```` with the database user password you set above (not the postgres one, your ```` one):: - - [datacube] - db_hostname: localhost - db_database: agdcintegration - db_username: - db_password: - -Note: For Ubuntu Setup the db_hostname should be set to "/var/run/postgresql". For more refer: https://github.com/opendatacube/datacube-core/issues/1329 - -Verify it all works -=================== - -Run the integration tests:: - - cd datacube-core - ./check-code.sh integration_tests - -Build the documentation:: - - cd datacube-core/docs - pip install -r requirements.txt - make html +Connecting to your own database to try out some SQL should now be as easy as:: -Then open :file:`_build/html/index.html` in your browser to view the Documentation. + psql -d pgintegration diff --git a/docs/installation/setup/macosx.rst b/docs/installation/setup/macosx.rst index e1dfe79ca9..09deab701c 100644 --- a/docs/installation/setup/macosx.rst +++ b/docs/installation/setup/macosx.rst @@ -23,3 +23,50 @@ Postgres: .. include:: common_install.rst + + +You can now specify the database user and password for ODC integration testing. To do this:: + + cp integration_tests/integration.conf ~/.datacube_integration.conf + +Then edit the ``~/.datacube_integration.conf`` with a text editor and add the following lines, replacing ```` with your username and ```` with the database user password you set above (not the postgres one, your ```` one):: + + [datacube] + db_hostname: localhost + db_database: pgintegration + index_driver: default + db_username: + db_password: + + [experimental] + db_hostname: localhost + db_database: pgisintegration + index_driver: postgis + db_username: + db_password: + + +Verify it all works +=================== + +Install additional test dependencies:: + + cd datacube-core + pip install --upgrade -e '.[test]' + +Run the integration tests:: + + ./check-code.sh integration_tests + +Note: if moto-based AWS-mock tests fail, you may need to unset all AWS environment variables. + +Build the documentation:: + + pip install --upgrade -e '.[doc]' + cd docs + pip install -r requirements.txt + sudo apt install make + sudo apt install pandoc + make html + +Then open :file:`_build/html/index.html` in your browser to view the Documentation. diff --git a/docs/installation/setup/ubuntu.rst b/docs/installation/setup/ubuntu.rst index 56c19ed4b7..2f999ae6e0 100644 --- a/docs/installation/setup/ubuntu.rst +++ b/docs/installation/setup/ubuntu.rst @@ -6,7 +6,7 @@ Base OS: Ubuntu 20.04 LTS This guide will setup an ODC core development environment and includes: - - Anaconda python using conda environments to isolate the odc development environment + - Mambaforge using conda environments to isolate the odc development environment - installation of required software and useful developer manuals for those libraries - Postgres database installation with a local user configuration - Integration tests to confirm both successful development setup and for ongoing testing @@ -20,13 +20,13 @@ GDAL, HDF5, and netCDF4:: sudo apt-get install libgdal-dev libhdf5-serial-dev libnetcdf-dev -Install the latest Postgres version `available ` for your +Install the latest Postgres version `available `_ for your Ubuntu distribution, eg:: - sudo apt-get install postgresql-12 + sudo apt-get install postgresql-14 - # Optionally, Postgis too - sudo apt-get install postgresql-12-postgis-3 + # Optionally, Postgis too (required for the postgis/experimental index driver) + sudo apt-get install postgresql-14-postgis-3 Ubuntu's official repositories usually ship older versions of Postgres. You can alternatively get the most recent version from `the official PostgreSQL repository `_. @@ -37,3 +37,58 @@ Optional packages (useful utilities, docs):: sudo apt-get install hdf5-tools netcdf-bin gdal-bin pgadmin3 .. include:: common_install.rst + + +If createdb or psql cannot connect to server, check which postgresql installation is being run:: + + which psql + +If it is running the mambaforge installation, you may need to run the global installation:: + + /usr/bin/psql -d pgintegration + + +You can now specify the database user and password for ODC integration testing. To do this:: + + cp integration_tests/integration.conf ~/.datacube_integration.conf + +Then edit the ``~/.datacube_integration.conf`` with a text editor and add the following lines, replacing ```` with your username and ```` with the database user password you set above (not the postgres one, your ```` one):: + + [datacube] + db_hostname: /var/run/postgresql + db_database: pgintegration + index_driver: default + db_username: + db_password: + + [experimental] + db_hostname: /var/run/postgresql + db_database: pgisintegration + index_driver: postgis + db_username: + db_password: + +Verify it all works +=================== + +Install additional test dependencies:: + + cd datacube-core + pip install --upgrade -e '.[test]' + +Run the integration tests:: + + ./check-code.sh integration_tests + +Note: if moto-based AWS-mock tests fail, you may need to unset all AWS environment variables. + +Build the documentation:: + + pip install --upgrade -e '.[doc]' + cd docs + pip install -r requirements.txt + sudo apt install make + sudo apt install pandoc + make html + +Then open :file:`_build/html/index.html` in your browser to view the Documentation. diff --git a/docs/installation/setup/windows.rst b/docs/installation/setup/windows.rst index c4a467f3dd..2a1f558d89 100644 --- a/docs/installation/setup/windows.rst +++ b/docs/installation/setup/windows.rst @@ -62,14 +62,22 @@ Download the latest version of the software from the `repository `` with your username and ```` with the database user password you set above (not the postgres one, your ```` one):: [datacube] db_hostname: localhost - db_database: agdcintegration + db_database: pgintegration + index_driver: default + db_username: + db_password: + + [experimental] + db_hostname: localhost + db_database: pgisintegration + index_driver: postgis db_username: db_password: diff --git a/docs/make.bat b/docs/make.bat index 4d607534cd..34f6c8c045 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -5,7 +5,7 @@ echo %cd% REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build + set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build @@ -17,15 +17,15 @@ if "%1" == "livehtml" goto livehtml %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% diff --git a/examples/io_plugin/dcio_example/__init__.py b/examples/io_plugin/dcio_example/__init__.py index c081ad5b41..45970a2a5b 100644 --- a/examples/io_plugin/dcio_example/__init__.py +++ b/examples/io_plugin/dcio_example/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/examples/io_plugin/dcio_example/pickles.py b/examples/io_plugin/dcio_example/pickles.py index 2f6eed8df8..f209b12e35 100644 --- a/examples/io_plugin/dcio_example/pickles.py +++ b/examples/io_plugin/dcio_example/pickles.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Example reader plugin """ diff --git a/examples/io_plugin/dcio_example/xarray_3d.py b/examples/io_plugin/dcio_example/xarray_3d.py index f57607a5a5..3a0e11dc7b 100644 --- a/examples/io_plugin/dcio_example/xarray_3d.py +++ b/examples/io_plugin/dcio_example/xarray_3d.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ xarray 3D driver plugin for 3D support testing """ diff --git a/examples/io_plugin/dcio_example/zeros.py b/examples/io_plugin/dcio_example/zeros.py index 1cd088e5c0..71b302c136 100644 --- a/examples/io_plugin/dcio_example/zeros.py +++ b/examples/io_plugin/dcio_example/zeros.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Sample plugin "reads" zeros each time every time diff --git a/examples/io_plugin/setup.py b/examples/io_plugin/setup.py index e5f41105ca..402dc1e062 100644 --- a/examples/io_plugin/setup.py +++ b/examples/io_plugin/setup.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from setuptools import setup, find_packages diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py index d0e5e97db3..7bd83106eb 100644 --- a/integration_tests/__init__.py +++ b/integration_tests/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index c87329f4e9..d7e749b163 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Common methods for index integration tests. @@ -48,7 +48,7 @@ PROJECT_ROOT = Path(__file__).parents[1] CONFIG_SAMPLES = PROJECT_ROOT / 'docs' / 'config_samples' -CONFIG_FILE_PATHS = [str(INTEGRATION_TESTS_DIR / 'agdcintegration.conf'), +CONFIG_FILE_PATHS = [str(INTEGRATION_TESTS_DIR / 'integration.conf'), os.path.expanduser('~/.datacube_integration.conf')] # Configure Hypothesis to allow slower tests, because we're testing datasets @@ -183,6 +183,11 @@ def extended_eo3_metadata_type_doc(): return get_eo3_test_data_doc("eo3_landsat_ard.odc-type.yaml") +@pytest.fixture +def eo3_sentinel_metadata_type_doc(): + return get_eo3_test_data_doc("eo3_sentinel_ard.odc-type.yaml") + + @pytest.fixture def extended_eo3_product_doc(): return get_eo3_test_data_doc("ard_ls8.odc-product.yaml") @@ -198,6 +203,47 @@ def africa_s2_product_doc(): return get_eo3_test_data_doc("s2_africa_product.yaml") +@pytest.fixture +def s2_ard_product_doc(): + return get_eo3_test_data_doc("ga_s2am_ard_3.odc-product.yaml") + + +@pytest.fixture +def final_dataset_doc(): + return ( + get_eo3_test_data_doc("final_dataset.yaml"), + 's3://dea-public-data/baseline/ga_ls8c_ard_3/090/086/2023/04/30' + 'ga_ls8c_ard_3-2-1_090086_2023-04-30_final.stac-item.json' + ) + + +@pytest.fixture +def nrt_dataset_doc(): + return ( + get_eo3_test_data_doc("nrt_dataset.yaml"), + 's3://dea-public-data/baseline/ga_ls8c_ard_3/090/086/2023/04/30_nrt' + 'ga_ls8c_ard_3-2-1_090086_2023-04-30_nrt.stac-item.json' + ) + + +@pytest.fixture +def ga_s2am_ard_3_interim_doc(): + return ( + get_eo3_test_data_doc("ga_s2am_ard_3_interim.yaml"), + 's3://dea-public-data/baseline/ga_s2am_ard_3/53/JNN/2021/07/24_interim' + '20230222T235626/ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim.odc-metadata.yaml' + ) + + +@pytest.fixture +def ga_s2am_ard_3_final_doc(): + return ( + get_eo3_test_data_doc("ga_s2am_ard_3_final.yaml"), + 's3://dea-public-data/baseline/ga_s2am_ard_3/53/JNN/2021/07/24' + '20210724T023436/ga_s2am_ard_3-2-1_53JNN_2021-07-24_final.odc-metadata.yaml' + ) + + def doc_to_ds(index, product_name, ds_doc, ds_path): from datacube.index.hl import Doc2Dataset resolver = Doc2Dataset(index, products=[product_name], verify_lineage=False) @@ -212,6 +258,14 @@ def doc_to_ds(index, product_name, ds_doc, ds_path): return index.datasets.get(ds.id) +def doc_to_ds_no_add(index, product_name, ds_doc, ds_path): + from datacube.index.hl import Doc2Dataset + resolver = Doc2Dataset(index, products=[product_name], verify_lineage=False) + ds, err = resolver(ds_doc, ds_path) + assert err is None and ds is not None + return ds + + @pytest.fixture def extended_eo3_metadata_type(index, extended_eo3_metadata_type_doc): return index.metadata_types.add( @@ -219,6 +273,13 @@ def extended_eo3_metadata_type(index, extended_eo3_metadata_type_doc): ) +@pytest.fixture +def eo3_sentinel_metadata_type(index, eo3_sentinel_metadata_type_doc): + return index.metadata_types.add( + index.metadata_types.from_doc(eo3_sentinel_metadata_type_doc) + ) + + @pytest.fixture def ls8_eo3_product(index, extended_eo3_metadata_type, extended_eo3_product_doc): return index.products.add_document(extended_eo3_product_doc) @@ -234,6 +295,11 @@ def africa_s2_eo3_product(index, africa_s2_product_doc): return index.products.add_document(africa_s2_product_doc) +@pytest.fixture +def ga_s2am_ard_3_product(index, eo3_sentinel_metadata_type, s2_ard_product_doc): + return index.products.add_document(s2_ard_product_doc) + + @pytest.fixture def eo3_products(index, extended_eo3_metadata_type, ls8_eo3_product, wo_eo3_product, @@ -287,6 +353,38 @@ def africa_eo3_dataset(index, africa_s2_eo3_product, eo3_africa_dataset_doc): *eo3_africa_dataset_doc) +@pytest.fixture +def nrt_dataset(index, extended_eo3_metadata_type, ls8_eo3_product, nrt_dataset_doc): + return doc_to_ds_no_add( + index, + ls8_eo3_product.name, + *nrt_dataset_doc) + + +@pytest.fixture +def final_dataset(index, extended_eo3_metadata_type, ls8_eo3_product, final_dataset_doc): + return doc_to_ds_no_add( + index, + ls8_eo3_product.name, + *final_dataset_doc) + + +@pytest.fixture +def ga_s2am_ard3_final(index, eo3_sentinel_metadata_type, ga_s2am_ard_3_product, ga_s2am_ard_3_final_doc): + return doc_to_ds_no_add( + index, + ga_s2am_ard_3_product.name, + *ga_s2am_ard_3_final_doc) + + +@pytest.fixture +def ga_s2am_ard3_interim(index, eo3_sentinel_metadata_type, ga_s2am_ard_3_product, ga_s2am_ard_3_interim_doc): + return doc_to_ds_no_add( + index, + ga_s2am_ard_3_product.name, + *ga_s2am_ard_3_interim_doc) + + @pytest.fixture def mem_index_fresh(in_memory_config): from datacube import Datacube diff --git a/integration_tests/data/eo3/eo3_sentinel_ard.odc-type.yaml b/integration_tests/data/eo3/eo3_sentinel_ard.odc-type.yaml new file mode 100644 index 0000000000..307eb9dd43 --- /dev/null +++ b/integration_tests/data/eo3/eo3_sentinel_ard.odc-type.yaml @@ -0,0 +1,361 @@ +--- +# Metadata Type +# url: https://explorer-aws.dea.ga.gov.au/metadata-types/eo3_sentinel_ard.odc-type.yaml +name: eo3_sentinel_ard +description: EO3 for Sentinel 2 ARD +dataset: + id: + - id + label: + - label + format: + - properties + - odc:file_format + sources: + - lineage + - source_datasets + creation_dt: + - properties + - odc:processing_datetime + grid_spatial: + - grid_spatial + - projection + measurements: + - measurements + search_fields: + lat: + type: double-range + max_offset: + - - extent + - lat + - end + min_offset: + - - extent + - lat + - begin + description: Latitude range + lon: + type: double-range + max_offset: + - - extent + - lon + - end + min_offset: + - - extent + - lon + - begin + description: Longitude range + time: + type: datetime-range + max_offset: + - - properties + - dtr:end_datetime + - - properties + - datetime + min_offset: + - - properties + - dtr:start_datetime + - - properties + - datetime + description: Acquisition time range + eo_gsd: + type: double + offset: + - properties + - eo:gsd + indexed: false + description: "Ground sampling distance of the sensor’s best resolution band\n\ + in metres; represents the size (or spatial resolution) of one pixel.\n" + crs_raw: + offset: + - crs + indexed: false + description: "The raw CRS string as it appears in metadata\n\n(e.g. ‘epsg:32654’)\n" + platform: + offset: + - properties + - eo:platform + indexed: false + description: Platform code + gqa_abs_x: + type: double + offset: + - properties + - gqa:abs_x + indexed: false + description: "Absolute value of the x-axis (east-to-west) GCP residuals, in\ + \ pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5\ + \ metres)\n" + gqa_abs_y: + type: double + offset: + - properties + - gqa:abs_y + indexed: false + description: "Absolute value of the y-axis (north-to-south) GCP residuals, in\ + \ pixel units based on a 25 metre resolution reference image (i.e. 0.2 = 5\ + \ metres)\n" + gqa_cep90: + type: double + offset: + - properties + - gqa:cep90 + indexed: false + description: "Circular error probable (90%) of the values of the GCP residuals,\ + \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ + \ = 5 metres)\n" + fmask_snow: + type: double + offset: + - properties + - fmask:snow + indexed: false + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains clear snow pixels according to the Fmask algorithm\n" + gqa_abs_xy: + type: double + offset: + - properties + - gqa:abs_xy + indexed: false + description: "Absolute value of the total GCP residuals, in pixel units based\ + \ on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" + gqa_mean_x: + type: double + offset: + - properties + - gqa:mean_x + indexed: false + description: "Mean of the values of the x-axis (east-to-west) GCP residuals,\ + \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ + \ = 5 metres)\n" + gqa_mean_y: + type: double + offset: + - properties + - gqa:mean_y + indexed: false + description: "Mean of the values of the y-axis (north-to-south) GCP residuals,\ + \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ + \ = 5 metres)\n" + instrument: + offset: + - properties + - eo:instrument + indexed: false + description: Instrument name + cloud_cover: + type: double + offset: + - properties + - eo:cloud_cover + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains cloud pixels.\n\nFor these ARD products, this value comes\ + \ from the Fmask algorithm.\n" + fmask_clear: + type: double + offset: + - properties + - fmask:clear + indexed: false + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains clear land pixels according to the Fmask algorithm\n" + fmask_water: + type: double + offset: + - properties + - fmask:water + indexed: false + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains clear water pixels according to the Fmask algorithm\n" + gqa_mean_xy: + type: double + offset: + - properties + - gqa:mean_xy + indexed: false + description: "Mean of the values of the GCP residuals, in pixel units based\ + \ on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" + region_code: + offset: + - properties + - odc:region_code + description: "Spatial reference code from the provider.\nFor Sentinel it is\ + \ MGRS code.\n" + gqa_stddev_x: + type: double + offset: + - properties + - gqa:stddev_x + indexed: false + description: "Standard Deviation of the values of the x-axis (east-to-west)\ + \ GCP residuals, in pixel units based on a 25 metre resolution reference image\ + \ (i.e. 0.2 = 5 metres)\n" + gqa_stddev_y: + type: double + offset: + - properties + - gqa:stddev_y + indexed: false + description: "Standard Deviation of the values of the y-axis (north-to-south)\ + \ GCP residuals, in pixel units based on a 25 metre resolution reference image\ + \ (i.e. 0.2 = 5 metres)\n" + gqa_stddev_xy: + type: double + offset: + - properties + - gqa:stddev_xy + indexed: false + description: "Standard Deviation of the values of the GCP residuals, in pixel\ + \ units based on a 25 metre resolution reference image (i.e. 0.2 = 5 metres)\n" + eo_sun_azimuth: + type: double + offset: + - properties + - eo:sun_azimuth + indexed: false + description: "The azimuth angle of the sun at the moment of acquisition, in\ + \ degree units measured clockwise from due north\n" + product_family: + offset: + - properties + - odc:product_family + indexed: false + description: Product family code + dataset_maturity: + offset: + - properties + - dea:dataset_maturity + indexed: false + description: One of - final|interim|nrt (near real time) + eo_sun_elevation: + type: double + offset: + - properties + - eo:sun_elevation + indexed: false + description: "The elevation angle of the sun at the moment of acquisition, in\ + \ degree units relative to the horizon.\n" + sentinel_tile_id: + offset: + - properties + - sentinel:sentinel_tile_id + indexed: false + description: "Granule ID according to the ESA naming convention\n\n(e.g. ‘S2A_OPER_MSI_L1C_TL_SGS__20161214T040601_A007721_T53KRB_N02.04’)\n" + s2cloudless_clear: + type: double + offset: + - properties + - s2cloudless:clear + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains clear land pixels according to s3cloudless\n" + s2cloudless_cloud: + type: double + offset: + - properties + - s2cloudless:cloud + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains cloud land pixels according to s3cloudless\n" + fmask_cloud_shadow: + type: double + offset: + - properties + - fmask:cloud_shadow + indexed: false + description: "The proportion (from 0 to 100) of the dataset's valid data area\ + \ that contains cloud shadow pixels according to the Fmask algorithm\n" + gqa_iterative_mean_x: + type: double + offset: + - properties + - gqa:iterative_mean_x + indexed: false + description: "Mean of the values of the x-axis (east-to-west) GCP residuals\ + \ after removal of outliers, in pixel units based on a 25 metre resolution\ + \ reference image (i.e. 0.2 = 5 metres)\n" + gqa_iterative_mean_y: + type: double + offset: + - properties + - gqa:iterative_mean_y + indexed: false + description: "Mean of the values of the y-axis (north-to-south) GCP residuals\ + \ after removal of outliers, in pixel units based on a 25 metre resolution\ + \ reference image (i.e. 0.2 = 5 metres)\n" + gqa_iterative_mean_xy: + type: double + offset: + - properties + - gqa:iterative_mean_xy + indexed: false + description: "Mean of the values of the GCP residuals after removal of outliers,\ + \ in pixel units based on a 25 metre resolution reference image (i.e. 0.2\ + \ = 5 metres)\n" + sentinel_datastrip_id: + offset: + - properties + - sentinel:datastrip_id + indexed: false + description: "Unique identifier for a datastrip relative to a given Datatake.\n\ + \n(e.g. ‘S2A_OPER_MSI_L1C_DS_SGS__20161214T040601_S20161214T005840_N02.04’)\n" + sentinel_product_name: + offset: + - properties + - sentinel:product_name + indexed: false + description: "ESA product URI, with the '.SAFE' ending removed.\n\n(e.g. 'S2A_MSIL1C_20220303T000731_N0400_R073_T56LNM_20220303T012845')\n" + gqa_iterative_stddev_x: + type: double + offset: + - properties + - gqa:iterative_stddev_x + indexed: false + description: "Standard Deviation of the values of the x-axis (east-to-west)\ + \ GCP residuals after removal of outliers, in pixel units based on a 25 metre\ + \ resolution reference image (i.e. 0.2 = 5 metres)\n" + gqa_iterative_stddev_y: + type: double + offset: + - properties + - gqa:iterative_stddev_y + indexed: false + description: "Standard Deviation of the values of the y-axis (north-to-south)\ + \ GCP residuals after removal of outliers, in pixel units based on a 25 metre\ + \ resolution reference image (i.e. 0.2 = 5 metres)\n" + gqa_iterative_stddev_xy: + type: double + offset: + - properties + - gqa:iterative_stddev_xy + indexed: false + description: "Standard Deviation of the values of the GCP residuals after removal\ + \ of outliers, in pixel units based on a 25 metre resolution reference image\ + \ (i.e. 0.2 = 5 metres)\n" + gqa_abs_iterative_mean_x: + type: double + offset: + - properties + - gqa:abs_iterative_mean_x + indexed: false + description: "Mean of the absolute values of the x-axis (east-to-west) GCP residuals\ + \ after removal of outliers, in pixel units based on a 25 metre resolution\ + \ reference image (i.e. 0.2 = 5 metres)\n" + gqa_abs_iterative_mean_y: + type: double + offset: + - properties + - gqa:abs_iterative_mean_y + indexed: false + description: "Mean of the absolute values of the y-axis (north-to-south) GCP\ + \ residuals after removal of outliers, in pixel units based on a 25 metre\ + \ resolution reference image (i.e. 0.2 = 5 metres)\n" + gqa_abs_iterative_mean_xy: + type: double + offset: + - properties + - gqa:abs_iterative_mean_xy + indexed: false + description: "Mean of the absolute values of the GCP residuals after removal\ + \ of outliers, in pixel units based on a 25 metre resolution reference image\ + \ (i.e. 0.2 = 5 metres)\n" +... diff --git a/integration_tests/data/eo3/final_dataset.yaml b/integration_tests/data/eo3/final_dataset.yaml new file mode 100644 index 0000000000..19c27ffca3 --- /dev/null +++ b/integration_tests/data/eo3/final_dataset.yaml @@ -0,0 +1,139 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: 6241395f-f341-41b0-96b7-60f486c577d0 + +label: ga_ls8c_ard_3-2-1_090086_2023-04-30_final +product: + name: ga_ls8c_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_ls8c_ard_3 + +crs: epsg:32655 +geometry: + type: Polygon + coordinates: [[[559816.0052888468, -4219857.620084257], [559721.8933982822, -4219833.106601718], + [559732.9800465275, -4219728.735567618], [560962.9813034026, -4214988.730722999], + [571612.9871046449, -4174023.7084477567], [604522.9880076076, -4048158.7049931744], + [608708.0157636222, -4032408.6003978983], [608918.1498917936, -4031628.1325757634], + [609157.5, -4030897.5], [609243.8473913191, -4030914.6899256567], [609255.0, + -4030905.0], [609449.0888722979, -4030955.549131081], [609461.1380343755, + -4030957.947862498], [611246.2821331235, -4031422.984647127], [614491.7279428138, + -4032268.87029345], [795172.5726569144, -4079325.971481828], [796033.5623962873, + -4079585.522912585], [796173.1066017178, -4079621.893398282], [747082.5, -4268617.5], + [746957.9666764413, -4268587.428373786], [746857.7239312489, -4268594.104275004], + [746137.4401493662, -4268414.031855924], [560077.4396544178, -4219964.0317270355], + [559816.0052888468, -4219857.620084257]]] +grids: + default: + shape: [7931, 7901] + transform: [30.0, 0.0, 559485.0, 0.0, -30.0, -4030785.0, 0.0, 0.0, 1.0] + panchromatic: + shape: [15861, 15801] + transform: [15.0, 0.0, 559492.5, 0.0, -15.0, -4030792.5, 0.0, 0.0, 1.0] + +properties: + datetime: 2023-04-30 23:50:34.384549Z + dea:dataset_maturity: final + dea:product_maturity: stable + dtr:end_datetime: 2023-04-30 23:50:48.771747Z + dtr:start_datetime: 2023-04-30 23:50:19.818416Z + eo:cloud_cover: 93.24450316290117 + eo:gsd: 15.0 # Ground sample distance (m) + eo:instrument: OLI_TIRS + eo:platform: landsat-8 + eo:sun_azimuth: 36.68926293 + eo:sun_elevation: 29.26893394 + fmask:clear: 6.177487638246016 + fmask:cloud: 93.24450316290117 + fmask:cloud_shadow: 0.14004371437490343 + fmask:snow: 0.006964025007482182 + fmask:water: 0.43100145947042295 + gqa:abs_iterative_mean_x: 0.19 + gqa:abs_iterative_mean_xy: 0.26 + gqa:abs_iterative_mean_y: 0.18 + gqa:abs_x: 0.65 + gqa:abs_xy: 0.73 + gqa:abs_y: 0.33 + gqa:cep90: 0.4 + gqa:iterative_mean_x: -0.15 + gqa:iterative_mean_xy: 0.18 + gqa:iterative_mean_y: 0.1 + gqa:iterative_stddev_x: 0.16 + gqa:iterative_stddev_xy: 0.24 + gqa:iterative_stddev_y: 0.18 + gqa:mean_x: 0.23 + gqa:mean_xy: 0.23 + gqa:mean_y: -0.02 + gqa:stddev_x: 1.42 + gqa:stddev_xy: 1.48 + gqa:stddev_y: 0.42 + landsat:collection_category: T1 + landsat:collection_number: 2 + landsat:landsat_product_id: LC08_L1TP_090086_20230430_20230509_02_T1 + landsat:landsat_scene_id: LC80900862023120LGN00 + landsat:wrs_path: 90 + landsat:wrs_row: 86 + odc:dataset_version: 3.2.1 + odc:file_format: GeoTIFF + odc:processing_datetime: 2023-05-10 10:57:11.431704Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: '090086' + +measurements: + nbart_blue: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band02.tif + nbart_coastal_aerosol: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band01.tif + nbart_green: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band03.tif + nbart_nir: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band05.tif + nbart_panchromatic: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band08.tif + grid: panchromatic + nbart_red: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band04.tif + nbart_swir_1: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band06.tif + nbart_swir_2: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_band07.tif + oa_azimuthal_exiting: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_exiting-angle.tif + oa_fmask: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_fmask.tif + oa_incident_angle: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_incident-angle.tif + oa_nbart_contiguity: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_nbart-contiguity.tif + oa_relative_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_relative-azimuth.tif + oa_relative_slope: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_relative-slope.tif + oa_satellite_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_satellite-azimuth.tif + oa_satellite_view: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_satellite-view.tif + oa_solar_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_solar-azimuth.tif + oa_solar_zenith: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_solar-zenith.tif + oa_time_delta: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_final_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_final_thumbnail.jpg + metadata:processor: + path: ga_ls8c_ard_3-2-1_090086_2023-04-30_final.proc-info.yaml + checksum:sha1: + path: ga_ls8c_ard_3-2-1_090086_2023-04-30_final.sha1 + +lineage: {} +... diff --git a/integration_tests/data/eo3/ga_s2am_ard_3.odc-product.yaml b/integration_tests/data/eo3/ga_s2am_ard_3.odc-product.yaml new file mode 100644 index 0000000000..edc86a770a --- /dev/null +++ b/integration_tests/data/eo3/ga_s2am_ard_3.odc-product.yaml @@ -0,0 +1,240 @@ +--- +name: ga_s2am_ard_3 +license: CC-BY-4.0 +metadata_type: eo3_sentinel_ard +description: Geoscience Australia Sentinel 2A MSI Analysis Ready Data Collection 3 +metadata: + product: + name: ga_s2am_ard_3 + properties: + eo:platform: sentinel-2a + odc:producer: ga.gov.au + eo:instrument: MSI + odc:product_family: ard + dea:product_maturity: stable +measurements: +- name: nbart_coastal_aerosol + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band01 + - coastal_aerosol +- name: nbart_blue + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band02 + - blue +- name: nbart_green + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band03 + - green +- name: nbart_red + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band04 + - red +- name: nbart_red_edge_1 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band05 + - red_edge_1 +- name: nbart_red_edge_2 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band06 + - red_edge_2 +- name: nbart_red_edge_3 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band07 + - red_edge_3 +- name: nbart_nir_1 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band08 + - nir_1 + - nbart_common_nir +- name: nbart_nir_2 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band8a + - nir_2 +- name: nbart_swir_2 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band11 + - swir_2 + - nbart_common_swir_1 + - swir2 +- name: nbart_swir_3 + dtype: int16 + units: '1' + nodata: -999 + aliases: + - nbart_band12 + - swir_3 + - nbart_common_swir_2 +- name: oa_fmask + dtype: uint8 + units: '1' + nodata: 0 + aliases: + - fmask + flags_definition: + fmask: + bits: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + values: + '0': nodata + '1': valid + '2': cloud + '3': shadow + '4': snow + '5': water + description: Fmask +- name: oa_nbart_contiguity + dtype: uint8 + units: '1' + nodata: 255 + aliases: + - nbart_contiguity + flags_definition: + contiguous: + bits: + - 0 + values: + '0': false + '1': true +- name: oa_azimuthal_exiting + dtype: float32 + units: '1' + nodata: NaN + aliases: + - azimuthal_exiting +- name: oa_azimuthal_incident + dtype: float32 + units: '1' + nodata: NaN + aliases: + - azimuthal_incident +- name: oa_combined_terrain_shadow + dtype: uint8 + units: '1' + nodata: 255 + aliases: + - combined_terrain_shadow +- name: oa_exiting_angle + dtype: float32 + units: '1' + nodata: NaN + aliases: + - exiting_angle +- name: oa_incident_angle + dtype: float32 + units: '1' + nodata: NaN + aliases: + - incident_angle +- name: oa_relative_azimuth + dtype: float32 + units: '1' + nodata: NaN + aliases: + - relative_azimuth +- name: oa_relative_slope + dtype: float32 + units: '1' + nodata: NaN + aliases: + - relative_slope +- name: oa_satellite_azimuth + dtype: float32 + units: '1' + nodata: NaN + aliases: + - satellite_azimuth +- name: oa_satellite_view + dtype: float32 + units: '1' + nodata: NaN + aliases: + - satellite_view +- name: oa_solar_azimuth + dtype: float32 + units: '1' + nodata: NaN + aliases: + - solar_azimuth +- name: oa_solar_zenith + dtype: float32 + units: '1' + nodata: NaN + aliases: + - solar_zenith +- name: oa_time_delta + dtype: float32 + units: '1' + nodata: NaN + aliases: + - time_delta +- name: oa_s2cloudless_mask + dtype: uint8 + units: '1' + nodata: 0 + aliases: + - s2cloudless_mask + flags_definition: + s2cloudless_mask: + bits: + - 0 + - 1 + - 2 + values: + '0': nodata + '1': valid + '2': cloud + description: s2cloudless mask +- name: oa_s2cloudless_prob + dtype: float64 + units: '1' + nodata: NaN + aliases: + - s2cloudless_prob +# Product +# url: https://explorer-aws.dea.ga.gov.au/products/ga_s2am_ard_3.odc-product.yaml +load: + crs: EPSG:3577 + align: + x: 0 + y: 0 + resolution: + x: 10 + y: -10 +... diff --git a/integration_tests/data/eo3/ga_s2am_ard_3_final.yaml b/integration_tests/data/eo3/ga_s2am_ard_3_final.yaml new file mode 100644 index 0000000000..0ee56c2ad9 --- /dev/null +++ b/integration_tests/data/eo3/ga_s2am_ard_3_final.yaml @@ -0,0 +1,150 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: 4123766b-5779-4f96-b71f-28e28a326434 + +label: ga_s2am_ard_3-2-1_53JNN_2021-07-24_final +product: + name: ga_s2am_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_s2am_ard_3 + +crs: epsg:32753 +geometry: + type: Polygon + coordinates: [[[499980.0, 7300000.0], [609780.0, 7300000.0], [609780.0, 7190200.0], + [499980.0, 7190200.0], [499980.0, 7300000.0]]] +grids: + default: + shape: [5490, 5490] + transform: [20.0, 0.0, 499980.0, 0.0, -20.0, 7300000.0, 0.0, 0.0, 1.0] + '10': + shape: [10980, 10980] + transform: [10.0, 0.0, 499980.0, 0.0, -10.0, 7300000.0, 0.0, 0.0, 1.0] + '60': + shape: [1830, 1830] + transform: [60.0, 0.0, 499980.0, 0.0, -60.0, 7300000.0, 0.0, 0.0, 1.0] + +properties: + datetime: 2021-07-24 01:14:10.195987Z + dea:dataset_maturity: final + dea:product_maturity: stable + eo:cloud_cover: 0.0 + eo:constellation: sentinel-2 + eo:gsd: 10.0 # Ground sample distance (m) + eo:instrument: MSI + eo:platform: sentinel-2a + eo:sun_azimuth: 33.4140301515032 + eo:sun_elevation: 52.1651784839186 + fmask:clear: 99.9999469145756 + fmask:cloud: 0.0 + fmask:cloud_shadow: 0.0 + fmask:snow: 0.0 + fmask:water: 5.30854244013789e-05 + gqa:abs_iterative_mean_x: 0.88 + gqa:abs_iterative_mean_xy: 2.17 + gqa:abs_iterative_mean_y: 1.98 + gqa:abs_x: 1.48 + gqa:abs_xy: 4.03 + gqa:abs_y: 3.74 + gqa:cep90: 4.78 + gqa:iterative_mean_x: 0.52 + gqa:iterative_mean_xy: 0.55 + gqa:iterative_mean_y: 0.18 + gqa:iterative_stddev_x: 1.41 + gqa:iterative_stddev_xy: 4.21 + gqa:iterative_stddev_y: 3.97 + gqa:mean_x: 0.69 + gqa:mean_xy: 0.74 + gqa:mean_y: -0.25 + gqa:stddev_x: 14.77 + gqa:stddev_xy: 39.88 + gqa:stddev_y: 37.04 + odc:dataset_version: 3.2.1 + odc:file_format: GeoTIFF + odc:processing_datetime: 2022-07-12 17:54:39.369087Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: 53JNN + s2cloudless:clear: 100.0 + s2cloudless:cloud: 0.0 + sat:orbit_state: descending + sat:relative_orbit: 45 + sentinel:datastrip_id: S2A_OPER_MSI_L1C_DS_VGS4_20210724T023436_S20210724T010731_N03.01 + sentinel:datatake_start_datetime: 2021-07-24 02:34:36Z + sentinel:product_name: S2A_MSIL1C_20210724T010731_N0301_R045_T53JNN_20210724T023436 + sentinel:sentinel_tile_id: S2A_OPER_MSI_L1C_TL_VGS4_20210724T023436_A031788_T53JNN_N03.01 + +measurements: + nbart_blue: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band02.tif + grid: '10' + nbart_coastal_aerosol: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band01.tif + grid: '60' + nbart_green: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band03.tif + grid: '10' + nbart_nir_1: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band08.tif + grid: '10' + nbart_nir_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band08a.tif + nbart_red: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band04.tif + grid: '10' + nbart_red_edge_1: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band05.tif + nbart_red_edge_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band06.tif + nbart_red_edge_3: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band07.tif + nbart_swir_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band11.tif + nbart_swir_3: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_band12.tif + oa_azimuthal_exiting: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_exiting-angle.tif + oa_fmask: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_fmask.tif + oa_incident_angle: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_incident-angle.tif + oa_nbart_contiguity: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_nbart-contiguity.tif + grid: '10' + oa_relative_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_relative-azimuth.tif + oa_relative_slope: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_relative-slope.tif + oa_s2cloudless_mask: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_s2cloudless-mask.tif + grid: '60' + oa_s2cloudless_prob: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_s2cloudless-prob.tif + grid: '60' + oa_satellite_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_satellite-azimuth.tif + oa_satellite_view: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_satellite-view.tif + oa_solar_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_solar-azimuth.tif + oa_solar_zenith: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_solar-zenith.tif + oa_time_delta: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_final_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_final_thumbnail.jpg + metadata:processor: + path: ga_s2am_ard_3-2-1_53JNN_2021-07-24_final.proc-info.yaml + checksum:sha1: + path: ga_s2am_ard_3-2-1_53JNN_2021-07-24_final.sha1 + +lineage: {} +... diff --git a/integration_tests/data/eo3/ga_s2am_ard_3_interim.yaml b/integration_tests/data/eo3/ga_s2am_ard_3_interim.yaml new file mode 100644 index 0000000000..41e828da2d --- /dev/null +++ b/integration_tests/data/eo3/ga_s2am_ard_3_interim.yaml @@ -0,0 +1,151 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: c0324cbb-f499-4e8d-8001-a38206ff6904 + +label: ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim +product: + name: ga_s2am_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_s2am_ard_3 + +crs: epsg:32753 +geometry: + type: Polygon + coordinates: [[[499980.0, 7300000.0], [609780.0, 7300000.0], [609780.0, 7190200.0], + [499980.0, 7190200.0], [499980.0, 7300000.0]]] +grids: + default: + shape: [5490, 5490] + transform: [20.0, 0.0, 499980.0, 0.0, -20.0, 7300000.0, 0.0, 0.0, 1.0] + '10': + shape: [10980, 10980] + transform: [10.0, 0.0, 499980.0, 0.0, -10.0, 7300000.0, 0.0, 0.0, 1.0] + '60': + shape: [1830, 1830] + transform: [60.0, 0.0, 499980.0, 0.0, -60.0, 7300000.0, 0.0, 0.0, 1.0] + +properties: + datetime: 2021-07-24 01:14:10.195809Z + dea:dataset_maturity: interim + dea:product_maturity: stable + eo:cloud_cover: 0.0 + eo:constellation: sentinel-2 + eo:gsd: 10.0 # Ground sample distance (m) + eo:instrument: MSI + eo:platform: sentinel-2a + eo:sun_azimuth: 33.4134402697606 + eo:sun_elevation: 52.1647723037447 + fmask:clear: 99.99995355025365 + fmask:cloud: 0.0 + fmask:cloud_shadow: 0.0 + fmask:snow: 0.0 + fmask:water: 4.6449746351206534e-05 + gqa:abs_iterative_mean_x: 0.48 + gqa:abs_iterative_mean_xy: 1.02 + gqa:abs_iterative_mean_y: 0.91 + gqa:abs_x: 0.81 + gqa:abs_xy: 2.04 + gqa:abs_y: 1.87 + gqa:cep90: 2.56 + gqa:iterative_mean_x: 0.33 + gqa:iterative_mean_xy: 0.41 + gqa:iterative_mean_y: 0.24 + gqa:iterative_stddev_x: 0.6 + gqa:iterative_stddev_xy: 1.7 + gqa:iterative_stddev_y: 1.6 + gqa:mean_x: 0.41 + gqa:mean_xy: 0.41 + gqa:mean_y: 0.03 + gqa:stddev_x: 3.58 + gqa:stddev_xy: 10.73 + gqa:stddev_y: 10.12 + odc:dataset_version: 3.2.1 + odc:file_format: GeoTIFF + odc:processing_datetime: 2023-05-31 21:12:25.979068Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: 53JNN + s2cloudless:clear: 100.0 + s2cloudless:cloud: 0.0 + sentinel:datastrip_id: S2A_OPER_MSI_L1C_DS_S2RP_20230222T235626_S20210724T010731_N05.00 + sentinel:datatake_start_datetime: 2023-02-22 23:56:26Z + sentinel:grid_square: NN + sentinel:latitude_band: J + sentinel:product_name: S2A_MSIL1C_20210724T010731_N0500_R045_T53JNN_20230222T235626.SAFE + sentinel:sentinel_tile_id: S2A_OPER_MSI_L1C_TL_S2RP_20230222T235626_A031788_T53JNN_N05.00 + sentinel:utm_zone: 53 + +measurements: + nbart_blue: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band02.tif + grid: '10' + nbart_coastal_aerosol: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band01.tif + grid: '60' + nbart_green: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band03.tif + grid: '10' + nbart_nir_1: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band08.tif + grid: '10' + nbart_nir_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band08a.tif + nbart_red: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band04.tif + grid: '10' + nbart_red_edge_1: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band05.tif + nbart_red_edge_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band06.tif + nbart_red_edge_3: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band07.tif + nbart_swir_2: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band11.tif + nbart_swir_3: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_band12.tif + oa_azimuthal_exiting: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_exiting-angle.tif + oa_fmask: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_fmask.tif + oa_incident_angle: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_incident-angle.tif + oa_nbart_contiguity: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_nbart-contiguity.tif + grid: '10' + oa_relative_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_relative-azimuth.tif + oa_relative_slope: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_relative-slope.tif + oa_s2cloudless_mask: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_s2cloudless-mask.tif + grid: '60' + oa_s2cloudless_prob: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_s2cloudless-prob.tif + grid: '60' + oa_satellite_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_satellite-azimuth.tif + oa_satellite_view: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_satellite-view.tif + oa_solar_azimuth: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_solar-azimuth.tif + oa_solar_zenith: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_solar-zenith.tif + oa_time_delta: + path: ga_s2am_oa_3-2-1_53JNN_2021-07-24_interim_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_s2am_nbart_3-2-1_53JNN_2021-07-24_interim_thumbnail.jpg + metadata:processor: + path: ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim.proc-info.yaml + checksum:sha1: + path: ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim.sha1 + +lineage: {} +... diff --git a/integration_tests/data/eo3/nrt_dataset.yaml b/integration_tests/data/eo3/nrt_dataset.yaml new file mode 100644 index 0000000000..2388e62fa0 --- /dev/null +++ b/integration_tests/data/eo3/nrt_dataset.yaml @@ -0,0 +1,136 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: 94db51ab-4279-4402-8bcf-ec2f4ec494e3 + +label: ga_ls8c_ard_3-2-1_090086_2023-04-30_nrt +product: + name: ga_ls8c_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_ls8c_ard_3 + +crs: epsg:32655 +geometry: + type: Polygon + coordinates: [[[559721.8933982822, -4219833.106601718], [559732.9800465275, -4219728.735567618], + [560962.9813034026, -4214988.730722999], [571612.9871046449, -4174023.7084477567], + [604522.9880076076, -4048158.7049931744], [608903.006475906, -4031688.6350602414], + [609157.5, -4030897.5], [609243.8827279789, -4030914.696960433], [609255.0, + -4030905.0], [609449.0888722979, -4030955.549131081], [609461.1380343755, + -4030957.947862498], [611246.2821331235, -4031422.984647127], [614491.7279428138, + -4032268.87029345], [795172.5726569144, -4079325.971481828], [796033.5623962873, + -4079585.522912585], [796173.1066017178, -4079621.893398282], [747082.5, -4268617.5], + [746957.9666764413, -4268587.428373786], [746857.7239312489, -4268594.104275004], + [746137.4401493662, -4268414.031855924], [560077.4396544178, -4219964.0317270355], + [559816.0052888468, -4219857.620084257], [559721.8933982822, -4219833.106601718]]] +grids: + default: + shape: [7931, 7901] + transform: [30.0, 0.0, 559485.0, 0.0, -30.0, -4030785.0, 0.0, 0.0, 1.0] + nbart:panchromatic: + shape: [15861, 15801] + transform: [15.0, 0.0, 559492.5, 0.0, -15.0, -4030792.5, 0.0, 0.0, 1.0] + +properties: + datetime: 2023-04-30 23:50:34.384549Z + dea:dataset_maturity: nrt + dea:product_maturity: stable + eo:cloud_cover: 93.23672818080765 + eo:gsd: 15.0 # Ground sample distance (m) + eo:instrument: OLI_TIRS + eo:platform: landsat-8 + eo:sun_azimuth: 36.68926293 + eo:sun_elevation: 29.26893394 + fmask:clear: 6.182802588011784 + fmask:cloud: 93.23672818080765 + fmask:cloud_shadow: 0.14184073083015938 + fmask:snow: 0.006917692167185185 + fmask:water: 0.4317108081832226 + gqa:abs_iterative_mean_x: 0.19 + gqa:abs_iterative_mean_xy: 0.26 + gqa:abs_iterative_mean_y: 0.18 + gqa:abs_x: 0.65 + gqa:abs_xy: 0.73 + gqa:abs_y: 0.33 + gqa:cep90: 0.4 + gqa:iterative_mean_x: -0.15 + gqa:iterative_mean_xy: 0.18 + gqa:iterative_mean_y: 0.1 + gqa:iterative_stddev_x: 0.16 + gqa:iterative_stddev_xy: 0.24 + gqa:iterative_stddev_y: 0.18 + gqa:mean_x: 0.23 + gqa:mean_xy: 0.23 + gqa:mean_y: -0.02 + gqa:stddev_x: 1.42 + gqa:stddev_xy: 1.48 + gqa:stddev_y: 0.42 + landsat:collection_category: RT + landsat:collection_number: 2 + landsat:landsat_product_id: LC08_L1TP_090086_20230430_20230501_02_RT + landsat:landsat_scene_id: LC80900862023120LGN00 + landsat:wrs_path: 90 + landsat:wrs_row: 86 + odc:dataset_version: 3.2.1 + odc:file_format: GeoTIFF + odc:processing_datetime: 2023-05-01 08:53:27.948360Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: '090086' + +measurements: + nbart_blue: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band02.tif + nbart_coastal_aerosol: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band01.tif + nbart_green: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band03.tif + nbart_nir: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band05.tif + nbart_panchromatic: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band08.tif + grid: nbart:panchromatic + nbart_red: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band04.tif + nbart_swir_1: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band06.tif + nbart_swir_2: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_band07.tif + oa_azimuthal_exiting: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_exiting-angle.tif + oa_fmask: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_fmask.tif + oa_incident_angle: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_incident-angle.tif + oa_nbart_contiguity: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_nbart-contiguity.tif + oa_relative_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_relative-azimuth.tif + oa_relative_slope: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_relative-slope.tif + oa_satellite_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_satellite-azimuth.tif + oa_satellite_view: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_satellite-view.tif + oa_solar_azimuth: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_solar-azimuth.tif + oa_solar_zenith: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_solar-zenith.tif + oa_time_delta: + path: ga_ls8c_oa_3-2-1_090086_2023-04-30_nrt_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_ls8c_nbart_3-2-1_090086_2023-04-30_nrt_thumbnail.jpg + metadata:processor: + path: ga_ls8c_ard_3-2-1_090086_2023-04-30_nrt.proc-info.yaml + checksum:sha1: + path: ga_ls8c_ard_3-2-1_090086_2023-04-30_nrt.sha1 + +lineage: {} +... diff --git a/integration_tests/data_utils.py b/integration_tests/data_utils.py index f2d9c0c8f1..cb9323ae75 100644 --- a/integration_tests/data_utils.py +++ b/integration_tests/data_utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ The start of some general purpose utilities for generating test data. diff --git a/integration_tests/index/__init__.py b/integration_tests/index/__init__.py index d0e5e97db3..7bd83106eb 100644 --- a/integration_tests/index/__init__.py +++ b/integration_tests/index/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/index/search_utils.py b/integration_tests/index/search_utils.py index 62595bb735..fc0e4bf775 100644 --- a/integration_tests/index/search_utils.py +++ b/integration_tests/index/search_utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import csv import io diff --git a/integration_tests/index/test_config_docs.py b/integration_tests/index/test_config_docs.py index 9ea1e0ea79..6653d8fadb 100644 --- a/integration_tests/index/test_config_docs.py +++ b/integration_tests/index/test_config_docs.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/index/test_index_data.py b/integration_tests/index/test_index_data.py index 08d54cf9c2..6a53efa95f 100755 --- a/integration_tests/index/test_index_data.py +++ b/integration_tests/index/test_index_data.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test database methods. @@ -94,6 +94,31 @@ def test_archive_datasets(index, ls8_eo3_dataset): assert not indexed_dataset.is_archived +def test_archive_less_mature(index, final_dataset, nrt_dataset): + # case 1: add nrt then final; nrt should get archived + index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=True) + index.datasets.get(nrt_dataset.id).is_active + index.datasets.add(final_dataset, with_lineage=False, archive_less_mature=True) + assert index.datasets.get(nrt_dataset.id).is_archived + assert index.datasets.get(final_dataset.id).is_active + + # case 2: purge nrt; re-add with final already there + index.datasets.purge([nrt_dataset.id]) + assert index.datasets.get(nrt_dataset.id) is None + with pytest.raises(ValueError): + # should error as more mature version of dataset already exists + index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=True) + + +def test_archive_less_mature_approx_timestamp(index, ga_s2am_ard3_final, ga_s2am_ard3_interim): + # test archive_less_mature where there's a slight difference in timestamps + index.datasets.add(ga_s2am_ard3_interim, with_lineage=False) + index.datasets.get(ga_s2am_ard3_interim.id).is_active + index.datasets.add(ga_s2am_ard3_final, with_lineage=False, archive_less_mature=True) + assert index.datasets.get(ga_s2am_ard3_interim.id).is_archived + assert index.datasets.get(ga_s2am_ard3_final.id).is_active + + def test_purge_datasets(index, ls8_eo3_dataset): assert index.datasets.has(ls8_eo3_dataset.id) datasets = index.datasets.search_eager() diff --git a/integration_tests/index/test_memory_index.py b/integration_tests/index/test_memory_index.py index 41758bdb80..f5ed510a5f 100644 --- a/integration_tests/index/test_memory_index.py +++ b/integration_tests/index/test_memory_index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime diff --git a/integration_tests/index/test_null_index.py b/integration_tests/index/test_null_index.py index 8c235f0058..2cf9ebd03c 100644 --- a/integration_tests/index/test_null_index.py +++ b/integration_tests/index/test_null_index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from unittest.mock import MagicMock diff --git a/integration_tests/index/test_pluggable_indexes.py b/integration_tests/index/test_pluggable_indexes.py index c2e8e1d112..7d2b711d8c 100644 --- a/integration_tests/index/test_pluggable_indexes.py +++ b/integration_tests/index/test_pluggable_indexes.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/index/test_postgis_index.py b/integration_tests/index/test_postgis_index.py index 52610b9bb3..02ab147937 100644 --- a/integration_tests/index/test_postgis_index.py +++ b/integration_tests/index/test_postgis_index.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from uuid import uuid4 as random_uuid diff --git a/integration_tests/index/test_search_eo3.py b/integration_tests/index/test_search_eo3.py index dfd40f14f0..d4cf1b32af 100644 --- a/integration_tests/index/test_search_eo3.py +++ b/integration_tests/index/test_search_eo3.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/index/test_search_legacy.py b/integration_tests/index/test_search_legacy.py index e88050a5db..ba0db9c2ea 100644 --- a/integration_tests/index/test_search_legacy.py +++ b/integration_tests/index/test_search_legacy.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/index/test_update_columns.py b/integration_tests/index/test_update_columns.py index 9f3662f4ae..106c23c275 100644 --- a/integration_tests/index/test_update_columns.py +++ b/integration_tests/index/test_update_columns.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test creation of added/updated columns during diff --git a/integration_tests/agdcintegration.conf b/integration_tests/integration.conf similarity index 79% rename from integration_tests/agdcintegration.conf rename to integration_tests/integration.conf index ac3df9a70d..10d0599d1f 100644 --- a/integration_tests/agdcintegration.conf +++ b/integration_tests/integration.conf @@ -1,11 +1,11 @@ [datacube] db_hostname: -db_database: agdcintegration +db_database: pgintegration index_driver: default [experimental] db_hostname: -db_database: odcintegration +db_database: pgisintegration index_driver: postgis [no_such_driver_env] diff --git a/integration_tests/test_3d.py b/integration_tests/test_3d.py index 921031edc2..b8ad691b59 100644 --- a/integration_tests/test_3d.py +++ b/integration_tests/test_3d.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging from copy import deepcopy diff --git a/integration_tests/test_cli_output.py b/integration_tests/test_cli_output.py index 44c6292478..e2050e04b2 100644 --- a/integration_tests/test_cli_output.py +++ b/integration_tests/test_cli_output.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/integration_tests/test_config_tool.py b/integration_tests/test_config_tool.py index 991c6fb62c..264254d247 100644 --- a/integration_tests/test_config_tool.py +++ b/integration_tests/test_config_tool.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/integration_tests/test_dataset_add.py b/integration_tests/test_dataset_add.py index 0de7fffc05..0969117be6 100644 --- a/integration_tests/test_dataset_add.py +++ b/integration_tests/test_dataset_add.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import math diff --git a/integration_tests/test_double_ingestion.py b/integration_tests/test_double_ingestion.py index 3534def553..39485c9405 100644 --- a/integration_tests/test_double_ingestion.py +++ b/integration_tests/test_double_ingestion.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import netCDF4 diff --git a/integration_tests/test_end_to_end.py b/integration_tests/test_end_to_end.py index 708e77381f..1d571a5ef4 100644 --- a/integration_tests/test_end_to_end.py +++ b/integration_tests/test_end_to_end.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import shutil from pathlib import Path diff --git a/integration_tests/test_environments.py b/integration_tests/test_environments.py index 2ce64f07ac..78bb9a589c 100644 --- a/integration_tests/test_environments.py +++ b/integration_tests/test_environments.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/test_full_ingestion.py b/integration_tests/test_full_ingestion.py index a72cd9d7b9..30b03d84a0 100644 --- a/integration_tests/test_full_ingestion.py +++ b/integration_tests/test_full_ingestion.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import hashlib import warnings diff --git a/integration_tests/test_index_datasets_search.py b/integration_tests/test_index_datasets_search.py index 0a8373faf1..50b6a213f5 100644 --- a/integration_tests/test_index_datasets_search.py +++ b/integration_tests/test_index_datasets_search.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from pathlib import PurePosixPath diff --git a/integration_tests/test_index_out_of_bound.py b/integration_tests/test_index_out_of_bound.py index 8cbd343255..7054830d08 100644 --- a/integration_tests/test_index_out_of_bound.py +++ b/integration_tests/test_index_out_of_bound.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/test_model.py b/integration_tests/test_model.py index 6fe0dd06f1..189150fdfc 100644 --- a/integration_tests/test_model.py +++ b/integration_tests/test_model.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/test_validate_ingestion.py b/integration_tests/test_validate_ingestion.py index e204cb299a..68358fced6 100644 --- a/integration_tests/test_validate_ingestion.py +++ b/integration_tests/test_validate_ingestion.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest diff --git a/integration_tests/utils.py b/integration_tests/utils.py index 4744c54efc..c29b126c01 100644 --- a/integration_tests/utils.py +++ b/integration_tests/utils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging import os diff --git a/license-headers.md b/license-headers.md index 92090fad06..e5cd086bbb 100644 --- a/license-headers.md +++ b/license-headers.md @@ -1,29 +1,23 @@ # Applying or updating license headers -To add or update license headers in this or other Open Data Cube -projects, you can do the following: +To add or update license headers in this or other Open Data Cube projects, you can do the following: -Download the tool from [https://github.com/johann-petrak/licenseheaders](https://github.com/johann-petrak/licenseheaders) and make it executable. +Ensure you have pre-commit hooks installed ```bash -wget https://raw.githubusercontent.com/johann-petrak/licenseheaders/master/licenseheaders.py -chmod +x licenseheaders.py +pre-commit install ``` -Change the sections on `python` files, to remove the `headerStartLine` and -`headerEndLine`, like: +Run the `insert-license` hook: -```python - "headerStartLine": "", - "headerEndLine": "", +```bash +pre-commit run insert-license --all-files ``` -Run the tool: +To remove the license headers, add the `--remove-header` arg to `.pre-commit-config.yaml` before running the hook. -```bash -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir datacube -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir integration_tests -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir tests -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir docs -python3 ./licenseheaders.py --tmpl license-template.txt --years 2015-2020 --ext py --dir examples -``` +To make updates to the license text, first remove the headers, then update `license-template.txt` before rerunning the hook as usual to add them back. + +Note that the date range is automatically updated to include the current year. + +See the full documentation here: [https://github.com/Lucas-C/pre-commit-hooks#insert-license](https://github.com/Lucas-C/pre-commit-hooks#insert-license) diff --git a/license-template.txt b/license-template.txt index 649c117012..f15b36fd20 100644 --- a/license-template.txt +++ b/license-template.txt @@ -1,4 +1,4 @@ This file is part of the Open Data Cube, see https://opendatacube.org for more information -Copyright (c) ${years} ODC Contributors +Copyright (c) 2015-2023 ODC Contributors SPDX-License-Identifier: Apache-2.0 diff --git a/tests/__init__.py b/tests/__init__.py index c081ad5b41..45970a2a5b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/api/__init__.py b/tests/api/__init__.py index c081ad5b41..45970a2a5b 100644 --- a/tests/api/__init__.py +++ b/tests/api/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/api/test_core.py b/tests/api/test_core.py index c96f58824b..2bada262cf 100644 --- a/tests/api/test_core.py +++ b/tests/api/test_core.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import xarray as xr import numpy as np diff --git a/tests/api/test_grid_workflow.py b/tests/api/test_grid_workflow.py index d44dd3e996..7aab2d964e 100644 --- a/tests/api/test_grid_workflow.py +++ b/tests/api/test_grid_workflow.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import numpy diff --git a/tests/api/test_masking.py b/tests/api/test_masking.py index 456680a00e..63f9299ad9 100644 --- a/tests/api/test_masking.py +++ b/tests/api/test_masking.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import yaml import pytest diff --git a/tests/api/test_query.py b/tests/api/test_query.py index a32506e5c8..aa59561db3 100644 --- a/tests/api/test_query.py +++ b/tests/api/test_query.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime import pandas @@ -140,7 +140,11 @@ def format_test(start_out, end_out): ((datetime.datetime(2008, 1, 1), datetime.datetime(2008, 1, 10, 23, 59, 40)), format_test('2008-01-01T00:00:00', '2008-01-10T23:59:40.999999')), ((datetime.date(2008, 1, 1)), - format_test('2008-01-01T00:00:00', '2008-01-01T23:59:59.999999')) + format_test('2008-01-01T00:00:00', '2008-01-01T23:59:59.999999')), + ((datetime.date(2008, 1, 1), None), + format_test('2008-01-01T00:00:00', datetime.datetime.now().strftime("%Y-%m-%dT23:59:59.999999"))), + ((None, '2008'), + format_test(datetime.datetime.fromtimestamp(0).strftime("%Y-%m-%dT%H:%M:%S"), '2008-12-31T23:59:59.999999')) ] diff --git a/tests/api/test_virtual.py b/tests/api/test_virtual.py index 45588583a8..b906e93291 100644 --- a/tests/api/test_virtual.py +++ b/tests/api/test_virtual.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from collections import OrderedDict from datetime import datetime diff --git a/tests/conftest.py b/tests/conftest.py index d11d3c0f79..7a2f9ad5b5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ py.test configuration fixtures diff --git a/tests/data/lbg/LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323/metadata.xml b/tests/data/lbg/LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323/metadata.xml index 6c1abf7382..880c39e0a4 100644 --- a/tests/data/lbg/LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323/metadata.xml +++ b/tests/data/lbg/LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323/metadata.xml @@ -1,233 +1,233 @@ - - - - 19920323T23:09:13 - ALSP - acquisition - - D - 19920323T23:17:56 - - 42876 - completed - instantaneous - - preProgrammed - - TM - Multi-spectral - - Landsat-5 - - - 1 - Color JPEG Image - LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323.jpg - JPG - 4 - 7 - 237.329101562 - - - - - - - - - - Ortho-rectified, processed by Geoscience Australia - - - - 20120917T01:25:07 - creation - - - - - - - - </SOURCECITATION> - <SOURCEREFERENCESYSTEM/> - <SOURCERESOURCEID/> - <SOURCESCALE/> - <SOURCESTEP/> - </LINEAGESOURCE> - <PROCESSINGSTEP> - <ALGORITHMCITATION> - <EDITION>Git version ed0cd604d3b171ded4d8d3767feedb76e1b8ce58</EDITION> - <TITLE>Pinkmatter Landsat Processor - - - NBAR version GANBARv3.00.00-dev + + + + 19920323T23:09:13 + ALSP + acquisition + + D + 19920323T23:17:56 + + 42876 + completed + instantaneous + + preProgrammed + + TM + Multi-spectral + + Landsat-5 + + + 1 + Color JPEG Image + LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323.jpg + JPG + 4 + 7 + 237.329101562 + + + + + + + + + + Ortho-rectified, processed by Geoscience Australia + + + + 20120917T01:25:07 + creation + + + + + + + + </SOURCECITATION> + <SOURCEREFERENCESYSTEM/> + <SOURCERESOURCEID/> + <SOURCESCALE/> + <SOURCESTEP/> + </LINEAGESOURCE> + <PROCESSINGSTEP> + <ALGORITHMCITATION> + <EDITION>Git version ed0cd604d3b171ded4d8d3767feedb76e1b8ce58</EDITION> + <TITLE>Pinkmatter Landsat Processor + + + NBAR version GANBARv3.00.00-dev BRDF (MODIS_BRDF) = BAND 1 [0.008570 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b09.500m_0459_0479nm_brdf_par_fgeo.hdf), 0.043934 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b07.500m_0459_0479nm_brdf_par_fiso.hdf), 0.016253 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b08.500m_0459_0479nm_brdf_par_fvol.hdf)] - BAND 2 [0.015374 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b12.500m_0545_0565nm_brdf_par_fgeo.hdf), 0.078902 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b10.500m_0545_0565nm_brdf_par_fiso.hdf), 0.037976 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b11.500m_0545_0565nm_brdf_par_fvol.hdf)] - BAND 3 [0.017559 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b03.500m_0620_0670nm_brdf_par_fgeo.hdf), 0.085431 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b01.500m_0620_0670nm_brdf_par_fiso.hdf), 0.034050 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b02.500m_0620_0670nm_brdf_par_fvol.hdf)] - BAND 4 [0.023475 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b06.500m_0841_0876nm_brdf_par_fgeo.hdf), 0.273303 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b04.500m_0841_0876nm_brdf_par_fiso.hdf), 0.201959 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b05.500m_0841_0876nm_brdf_par_fvol.hdf)] - BAND 5 [0.043315 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b18.500m_1628_1652nm_brdf_par_fgeo.hdf), 0.279249 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b16.500m_1628_1652nm_brdf_par_fiso.hdf), 0.106509 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b17.500m_1628_1652nm_brdf_par_fvol.hdf)] - BAND 7 [0.032661 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b21.500m_2105_2155nm_brdf_par_fgeo.hdf), 0.165904 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b19.500m_2105_2155nm_brdf_par_fiso.hdf), 0.045391 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b20.500m_2105_2155nm_brdf_par_fvol.hdf)] + BAND 2 [0.015374 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b12.500m_0545_0565nm_brdf_par_fgeo.hdf), 0.078902 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b10.500m_0545_0565nm_brdf_par_fiso.hdf), 0.037976 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b11.500m_0545_0565nm_brdf_par_fvol.hdf)] + BAND 3 [0.017559 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b03.500m_0620_0670nm_brdf_par_fgeo.hdf), 0.085431 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b01.500m_0620_0670nm_brdf_par_fiso.hdf), 0.034050 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b02.500m_0620_0670nm_brdf_par_fvol.hdf)] + BAND 4 [0.023475 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b06.500m_0841_0876nm_brdf_par_fgeo.hdf), 0.273303 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b04.500m_0841_0876nm_brdf_par_fiso.hdf), 0.201959 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b05.500m_0841_0876nm_brdf_par_fvol.hdf)] + BAND 5 [0.043315 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b18.500m_1628_1652nm_brdf_par_fgeo.hdf), 0.279249 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b16.500m_1628_1652nm_brdf_par_fiso.hdf), 0.106509 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b17.500m_1628_1652nm_brdf_par_fvol.hdf)] + BAND 7 [0.032661 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b21.500m_2105_2155nm_brdf_par_fgeo.hdf), 0.165904 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b19.500m_2105_2155nm_brdf_par_fiso.hdf), 0.045391 (/g/data1/v10/eoancillarydata/brdf-jl/data/073/MCD43A1.JLWKAV.073.aust.005.b20.500m_2105_2155nm_brdf_par_fvol.hdf)] OZONE (GA 2 deg pixel size) = 0.261 WATER VAPOUR (NOAA) = 2.27 DEM (GA 1 deg pixel size) = 0.409 AOD (/g/data1/v10/eoancillarydata/aerosol/AATSR/2.0/aot_mean_Mar_All_Aerosols.cmp) = 0.043009 - - dataset - - - - 4326 - 151.1469421 - - - -35.6039314 - 148.4684601 - -35.5414848 - 151.1469421 - -33.5805855 - -34.5911 - 149.775 - -35.6039314 - 19920323 23:14:13 - 19920323 23:14:38 - - -33.6386375 - 148.4342346 - -33.5805855 - 151.0505676 - - - - - - 148.4342346 - - - area - - sample - 25.0 - 9721 - - - line - 25.0 - 8721 - - - 754512.500000 - 6167987.500000 - 0 - - - GDA94 - GRS80 - 633012.5 - 6058987.5 - - 876012.5 - 6058987.5 - - 633012.5 - 6276987.5 - - 876012.5 - 6276987.5 - - UpperLeft - UTM - 55 - - 2 - 1 - - - - 1, 2, 3, 4, 5, 7 - - - 23.0 - 35.8386447 - 57.6238557 - - NBAR - 90 - 84 - SAM - 0 - 0 - - - - - - - - - - notPlanned - - - - - - - - Landsat-5 NBAR - usAscii - - Landsat-5 TM NBAR x090 y084 19920323 version 1 status completed - 2014-03-17 05:53:14 - creation - - - - Medhavy Thankappan - Geoscience Australia - Science and Strategy Project Leader - pointOfContact - - - Landsat-5 TM NBAR x090 y084 19920323 version 1 status completed - - - - - - - - 9 - - eng - NCI PBS 1.0.0 | Linux r3398 2.6.32-358.23.2.el6.x86_64 #1 SMP Wed Oct 16 18:37:12 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux - 161 - - GEOTIFF - 1 - - dataset - dataset - - - Medhavy Thankappan - Geoscience Australia - Science and Strategy Project Leader - pointOfContact - - LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323 - GA Metadata Profile: A Geoscience Australia Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata - 1.0 - - LS5_March_1992_GA/ULA_STND_v2.1.5 - 25.0 - completed - Processed Image - - - environment - + + dataset + + + + 4326 + 151.1469421 + + + -35.6039314 + 148.4684601 + -35.5414848 + 151.1469421 + -33.5805855 + -34.5911 + 149.775 + -35.6039314 + 19920323 23:14:13 + 19920323 23:14:38 + + -33.6386375 + 148.4342346 + -33.5805855 + 151.0505676 + + + + + + 148.4342346 + + + area + + sample + 25.0 + 9721 + + + line + 25.0 + 8721 + + + 754512.500000 + 6167987.500000 + 0 + + + GDA94 + GRS80 + 633012.5 + 6058987.5 + + 876012.5 + 6058987.5 + + 633012.5 + 6276987.5 + + 876012.5 + 6276987.5 + + UpperLeft + UTM + 55 + + 2 + 1 + + + + 1, 2, 3, 4, 5, 7 + + + 23.0 + 35.8386447 + 57.6238557 + + NBAR + 90 + 84 + SAM + 0 + 0 + + + + + + + + + + notPlanned + + + + + + + + Landsat-5 NBAR + usAscii + + Landsat-5 TM NBAR x090 y084 19920323 version 1 status completed + 2014-03-17 05:53:14 + creation + + + + Medhavy Thankappan + Geoscience Australia + Science and Strategy Project Leader + pointOfContact + + + Landsat-5 TM NBAR x090 y084 19920323 version 1 status completed + + + + + + + + 9 + + eng + NCI PBS 1.0.0 | Linux r3398 2.6.32-358.23.2.el6.x86_64 #1 SMP Wed Oct 16 18:37:12 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux + 161 + + GEOTIFF + 1 + + dataset + dataset + + + Medhavy Thankappan + Geoscience Australia + Science and Strategy Project Leader + pointOfContact + + LS5_TM_NBAR_P54_GANBAR01-002_090_084_19920323 + GA Metadata Profile: A Geoscience Australia Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata + 1.0 + + LS5_March_1992_GA/ULA_STND_v2.1.5 + 25.0 + completed + Processed Image + + + environment + diff --git a/tests/data/lbg/LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323/metadata.xml b/tests/data/lbg/LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323/metadata.xml index 383b8c68a4..9116dae376 100644 --- a/tests/data/lbg/LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323/metadata.xml +++ b/tests/data/lbg/LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323/metadata.xml @@ -1,70 +1,70 @@ - - - - 19920323T23:09:13 - ALSP - acquisition - - - 19920323T23:17:56 - - 42876 - - instantaneous - - automatic - - TM - Multi-spectral - - Landsat-5 - - - - - - - - - - - - - - - - - - - The pixel quality algorithm uses data from both the L1T (Systematic Terrain Correction) and ARG25 (Australian Reflectance Grid 25m) products. - - - - 20120917T01:25:07 - creation - - - - - - - - </SOURCECITATION> - <SOURCEREFERENCESYSTEM/> - <SOURCERESOURCEID/> - <SOURCESCALE/> - <SOURCESTEP/> - </LINEAGESOURCE> - <PROCESSINGSTEP> - <ALGORITHMCITATION> - <EDITION>Git version 3.2.0</EDITION> - <TITLE>Pixel Quality - - - + + + + 19920323T23:09:13 + ALSP + acquisition + + + 19920323T23:17:56 + + 42876 + + instantaneous + + automatic + + TM + Multi-spectral + + Landsat-5 + + + + + + + + + + + + + + + + + + + The pixel quality algorithm uses data from both the L1T (Systematic Terrain Correction) and ARG25 (Australian Reflectance Grid 25m) products. + + + + 20120917T01:25:07 + creation + + + + + + + + </SOURCECITATION> + <SOURCEREFERENCESYSTEM/> + <SOURCERESOURCEID/> + <SOURCESCALE/> + <SOURCESTEP/> + </LINEAGESOURCE> + <PROCESSINGSTEP> + <ALGORITHMCITATION> + <EDITION>Git version 3.2.0</EDITION> + <TITLE>Pixel Quality + + + The pixel quality algorithm assesses quality aspects such as saturation, band/spectral contiguity, land/sea, cloud and cloud shadow. Saturation Band1 (Bit 0): Run Saturation Band2 (Bit 1): Run @@ -83,158 +83,158 @@ Cloud Shadow (Fmask) (Bit 13): Run Empty Test (Bit 14): Not Run Empty Test (Bit 15): Not Run - - dataset - - - - 4326 - 151.1469421 - - - -35.6039314 - 148.4684601 - -35.5414848 - 151.1469421 - -33.5805855 - -34.5911 - 149.775 - -35.6039314 - 19920323 23:14:13 - 19920323 23:14:38 - - -33.6386375 - 148.4342346 - -33.5805855 - 151.0505676 - - - - - - 148.4342346 - - - area - - sample - 25.0 - 9721 - - - line - 25.0 - 8721 - - - 754512.500000 - 6167987.500000 - 0 - - - GDA94 - GRS80 - 633012.5 - 6058987.5 - - 876012.5 - 6058987.5 - - 633012.5 - 6276987.5 - - 876012.5 - 6276987.5 - - UpperLeft - UTM - 55 - - 2 - 1 - - - - PQ - - 23.0 - 35.8386447 - 57.6238557 - - Pixel Quality - 90 - 84 - SAM - 0 - 0 - - - - - - - - - - notPlanned - - - - - - - - Landsat-5 PQ - utf8 - - Landsat-5 TM PQ x090 y084 19920323 - 20140601T01:22:30 - creation - - - - NEMO Group - Geoscience Australia - NEMO Operation - pointOfContact - - - Landsat-5 TM PQ x090 y084 19920323 - - - - - - - - 1 - - eng - NCI PBS 1.0.0 | Linux r3398 2.6.32-431.11.2.el6.x86_64 #1 SMP Tue Mar 25 19:59:55 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux - 161 - - GEOTIFF - 1 - - dataset - Satellite Imagery - - - NEMO Group - Geoscience Australia - NEMO Operation - pointOfContact - - LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323 - ANZLIC Metadata Profile: An Australian/New Zealand Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata - 1.0 - - Pixel Quality - 25.0 - completed - Processed Image - grid - + + dataset + + + + 4326 + 151.1469421 + + + -35.6039314 + 148.4684601 + -35.5414848 + 151.1469421 + -33.5805855 + -34.5911 + 149.775 + -35.6039314 + 19920323 23:14:13 + 19920323 23:14:38 + + -33.6386375 + 148.4342346 + -33.5805855 + 151.0505676 + + + + + + 148.4342346 + + + area + + sample + 25.0 + 9721 + + + line + 25.0 + 8721 + + + 754512.500000 + 6167987.500000 + 0 + + + GDA94 + GRS80 + 633012.5 + 6058987.5 + + 876012.5 + 6058987.5 + + 633012.5 + 6276987.5 + + 876012.5 + 6276987.5 + + UpperLeft + UTM + 55 + + 2 + 1 + + + + PQ + + 23.0 + 35.8386447 + 57.6238557 + + Pixel Quality + 90 + 84 + SAM + 0 + 0 + + + + + + + + + + notPlanned + + + + + + + + Landsat-5 PQ + utf8 + + Landsat-5 TM PQ x090 y084 19920323 + 20140601T01:22:30 + creation + + + + NEMO Group + Geoscience Australia + NEMO Operation + pointOfContact + + + Landsat-5 TM PQ x090 y084 19920323 + + + + + + + + 1 + + eng + NCI PBS 1.0.0 | Linux r3398 2.6.32-431.11.2.el6.x86_64 #1 SMP Tue Mar 25 19:59:55 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux + 161 + + GEOTIFF + 1 + + dataset + Satellite Imagery + + + NEMO Group + Geoscience Australia + NEMO Operation + pointOfContact + + LS5_TM_PQ_P55_GAPQ01-002_090_084_19920323 + ANZLIC Metadata Profile: An Australian/New Zealand Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata + 1.0 + + Pixel Quality + 25.0 + completed + Processed Image + grid + The pixel quality algorithm uses data from both the L1T (Systematic Terrain Correction) and ARG25 (Australian Reflectance Grid 25m) products. ACCA cloud cover is reported as a percentage of the entire data grid, while Fmask is reported as a percentage of the valid image data only. @@ -245,6 +245,6 @@ Cloud shadow is reported as a percentage of the entire data grid for both analys CLOUD SHADOW PERCENTAGE ACCA 0.38 CLOUD SHADOW PERCENTAGE Fmask 0.38 - environment - + environment + diff --git a/tests/data/ls8-eods-nbar/data/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012/metadata.xml b/tests/data/ls8-eods-nbar/data/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012/metadata.xml index b0eb00c2ea..b976062191 100644 --- a/tests/data/ls8-eods-nbar/data/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012/metadata.xml +++ b/tests/data/ls8-eods-nbar/data/LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012/metadata.xml @@ -1,223 +1,223 @@ - - - - 20141012T03:23:36 - LGS - acquisition - - D - 20141012T03:29:10 - - 8846 - completed - instantaneous - - preProgrammed - - OLI-TIRS - Multi-spectral - - Landsat-8 - - - 2 - Color JPEG Image - LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.jpg - JPG - 5 - 7 - 223.657226562 - - - - - - - - - - Ortho-rectified, processed by Geoscience Australia - - - - - - - - - - - - - </SOURCECITATION> - <SOURCEREFERENCESYSTEM/> - <SOURCERESOURCEID/> - <SOURCESCALE/> - <SOURCESTEP/> - </LINEAGESOURCE> - <PROCESSINGSTEP> - <ALGORITHMCITATION> - <EDITION>Git version 3.2.1</EDITION> - <TITLE>NBAR - - - Resampling=CC,RadiometricCorrection=CPF,Orientation=NUP,LPGS_version=3.3.3104,Hemisphere=S,CPF_NAME=L8CPF20141001_20141231.01,BPF_NAME_TIRS=LT8BPF20141012002432_20141012011154.02,BPF_NAME_OLI=LO8BPF20141012002825_20141012011100.01 - - dataset - - - - 4326 - 136.24838 - - - -26.96528 - 133.96233 - -26.96338 - 136.26962 - -24.97000 - -25.967 - 135.125 - -26.96338 - 20141012 00:55:54 - 20141012 00:56:18 - - -24.97 - 133.97969 - -24.96826 - 136.24838 - - - - - - 133.96233 - - - area - - sample - 25.0 - 9161 - - - line - 25.0 - 8841 - - - 511512.500000 - 7127487.500000 - 0 - eoancillarydataGCP - GDA94 / MGA zone 53 - GDA94 - GRS80 - 397012.5 - 7016987.5 - - 626012.5 - 7016987.5 - - 397012.5 - 7237987.5 - - 626012.5 - 7237987.5 - - UpperLeft - UTM - 53 - - 2 - 1 - - - - 1, 2, 3, 4, 5, 6, 7 - - - 0.0 - 58.00268508 - 59.41814014 - - NBAR - 101 - 78 - PUSH-BROOM - 0 - 0 - - - - - - - - - - notPlanned - - - - - - - - Processed Image: Landsat Nadir BRDF-Adjusted Reflectance products generated by Geoscience Australia. Data Source: GA - usAscii - - Landsat-8 OLI_TIRS NBAR x101 y078 20141012 - 2014-11-06 00:32:06 - creation - - - - Medhavy Thankappan - Geoscience Australia - Science and Strategy Project Leader - pointOfContact - - - Landsat-8 OLI_TIRS NBAR x101 y078 20141012 - - - - - - - - 9 - - eng - NCI PBS 1.0.0 | Linux r1699 2.6.32-431.29.2.el6.x86_64 #1 SMP Tue Sep 9 21:36:05 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux - 154 - - GEOTIFF - 1 - - dataset - Satellite Imagery - - - Medhavy Thankappan - Geoscience Australia - Science and Strategy Project Leader - pointOfContact - - LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012 - GA Metadata Profile: A Geoscience Australia Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata - 1.0 - - - 25.0 - completed - Processed Image - - - environment - + + + + 20141012T03:23:36 + LGS + acquisition + + D + 20141012T03:29:10 + + 8846 + completed + instantaneous + + preProgrammed + + OLI-TIRS + Multi-spectral + + Landsat-8 + + + 2 + Color JPEG Image + LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012.jpg + JPG + 5 + 7 + 223.657226562 + + + + + + + + + + Ortho-rectified, processed by Geoscience Australia + + + + + + + + + + + + + </SOURCECITATION> + <SOURCEREFERENCESYSTEM/> + <SOURCERESOURCEID/> + <SOURCESCALE/> + <SOURCESTEP/> + </LINEAGESOURCE> + <PROCESSINGSTEP> + <ALGORITHMCITATION> + <EDITION>Git version 3.2.1</EDITION> + <TITLE>NBAR + + + Resampling=CC,RadiometricCorrection=CPF,Orientation=NUP,LPGS_version=3.3.3104,Hemisphere=S,CPF_NAME=L8CPF20141001_20141231.01,BPF_NAME_TIRS=LT8BPF20141012002432_20141012011154.02,BPF_NAME_OLI=LO8BPF20141012002825_20141012011100.01 + + dataset + + + + 4326 + 136.24838 + + + -26.96528 + 133.96233 + -26.96338 + 136.26962 + -24.97000 + -25.967 + 135.125 + -26.96338 + 20141012 00:55:54 + 20141012 00:56:18 + + -24.97 + 133.97969 + -24.96826 + 136.24838 + + + + + + 133.96233 + + + area + + sample + 25.0 + 9161 + + + line + 25.0 + 8841 + + + 511512.500000 + 7127487.500000 + 0 + eoancillarydataGCP + GDA94 / MGA zone 53 + GDA94 + GRS80 + 397012.5 + 7016987.5 + + 626012.5 + 7016987.5 + + 397012.5 + 7237987.5 + + 626012.5 + 7237987.5 + + UpperLeft + UTM + 53 + + 2 + 1 + + + + 1, 2, 3, 4, 5, 6, 7 + + + 0.0 + 58.00268508 + 59.41814014 + + NBAR + 101 + 78 + PUSH-BROOM + 0 + 0 + + + + + + + + + + notPlanned + + + + + + + + Processed Image: Landsat Nadir BRDF-Adjusted Reflectance products generated by Geoscience Australia. Data Source: GA + usAscii + + Landsat-8 OLI_TIRS NBAR x101 y078 20141012 + 2014-11-06 00:32:06 + creation + + + + Medhavy Thankappan + Geoscience Australia + Science and Strategy Project Leader + pointOfContact + + + Landsat-8 OLI_TIRS NBAR x101 y078 20141012 + + + + + + + + 9 + + eng + NCI PBS 1.0.0 | Linux r1699 2.6.32-431.29.2.el6.x86_64 #1 SMP Tue Sep 9 21:36:05 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux + 154 + + GEOTIFF + 1 + + dataset + Satellite Imagery + + + Medhavy Thankappan + Geoscience Australia + Science and Strategy Project Leader + pointOfContact + + LS8_OLI_TIRS_NBAR_P54_GANBAR01-015_101_078_20141012 + GA Metadata Profile: A Geoscience Australia Profile of AS/NZS ISO 19115:2005, Geographic information - Metadata + 1.0 + + + 25.0 + completed + Processed Image + + + environment + diff --git a/tests/drivers/fail_drivers/dc_tests_io/__init__.py b/tests/drivers/fail_drivers/dc_tests_io/__init__.py index c081ad5b41..45970a2a5b 100644 --- a/tests/drivers/fail_drivers/dc_tests_io/__init__.py +++ b/tests/drivers/fail_drivers/dc_tests_io/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/drivers/fail_drivers/dc_tests_io/dummy.py b/tests/drivers/fail_drivers/dc_tests_io/dummy.py index f4efb60242..2d3143fd4a 100644 --- a/tests/drivers/fail_drivers/dc_tests_io/dummy.py +++ b/tests/drivers/fail_drivers/dc_tests_io/dummy.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/drivers/fail_drivers/setup.py b/tests/drivers/fail_drivers/setup.py index 9ca7932ceb..ed71cb1276 100644 --- a/tests/drivers/fail_drivers/setup.py +++ b/tests/drivers/fail_drivers/setup.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from setuptools import setup, find_packages diff --git a/tests/drivers/test_rio_reader.py b/tests/drivers/test_rio_reader.py index 1402b5823d..8f2a7a57d1 100644 --- a/tests/drivers/test_rio_reader.py +++ b/tests/drivers/test_rio_reader.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tests for new RIO reader driver """ diff --git a/tests/index/__init__.py b/tests/index/__init__.py index d0e5e97db3..7bd83106eb 100644 --- a/tests/index/__init__.py +++ b/tests/index/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/index/test_api_index_dataset.py b/tests/index/test_api_index_dataset.py index 93a57a84ee..6c86f9addc 100644 --- a/tests/index/test_api_index_dataset.py +++ b/tests/index/test_api_index_dataset.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime from collections import namedtuple diff --git a/tests/index/test_fields.py b/tests/index/test_fields.py index af2cc12da0..3f26ac33a3 100644 --- a/tests/index/test_fields.py +++ b/tests/index/test_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/index/test_hl_index.py b/tests/index/test_hl_index.py index 4b4cd00f77..df36ebf775 100644 --- a/tests/index/test_hl_index.py +++ b/tests/index/test_hl_index.py @@ -1,3 +1,7 @@ +# This file is part of the Open Data Cube, see https://opendatacube.org for more information +# +# Copyright (c) 2015-2023 ODC Contributors +# SPDX-License-Identifier: Apache-2.0 import pytest from unittest.mock import MagicMock diff --git a/tests/index/test_postgis_fields.py b/tests/index/test_postgis_fields.py index d5725fbdb3..cfa606b919 100644 --- a/tests/index/test_postgis_fields.py +++ b/tests/index/test_postgis_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2022 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import datetime diff --git a/tests/index/test_query.py b/tests/index/test_query.py index 60f6967652..477d6ed2d9 100644 --- a/tests/index/test_query.py +++ b/tests/index/test_query.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/index/test_validate_dataset_type.py b/tests/index/test_validate_dataset_type.py index 058c2e4655..565ad6dce0 100644 --- a/tests/index/test_validate_dataset_type.py +++ b/tests/index/test_validate_dataset_type.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py index d0e5e97db3..7bd83106eb 100644 --- a/tests/scripts/__init__.py +++ b/tests/scripts/__init__.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/scripts/test_search_tool.py b/tests/scripts/test_search_tool.py index 8e2b44e2bc..57bf4ffeb9 100644 --- a/tests/scripts/test_search_tool.py +++ b/tests/scripts/test_search_tool.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index f14b05c26b..823c71b2ee 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from datacube.storage import BandInfo diff --git a/tests/storage/test_netcdfwriter.py b/tests/storage/test_netcdfwriter.py index 177039e548..5fa785c4d9 100644 --- a/tests/storage/test_netcdfwriter.py +++ b/tests/storage/test_netcdfwriter.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import netCDF4 import numpy diff --git a/tests/storage/test_storage.py b/tests/storage/test_storage.py index 761d296548..393b3a422d 100644 --- a/tests/storage/test_storage.py +++ b/tests/storage/test_storage.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from contextlib import contextmanager @@ -234,7 +234,8 @@ def assert_same_read_results(source, dst_shape, dst_dtype, dst_transform, dst_no dst_transform=dst_transform, dst_crs=str(dst_projection), dst_nodata=dst_nodata, - resampling=resampling) + resampling=resampling, + XSCALE=1, YSCALE=1) result = np.full(dst_shape, dst_nodata, dtype=dst_dtype) H, W = dst_shape diff --git a/tests/storage/test_storage_load.py b/tests/storage/test_storage_load.py index 3277079a33..5c90ca2ee7 100644 --- a/tests/storage/test_storage_load.py +++ b/tests/storage/test_storage_load.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test New IO driver loading """ diff --git a/tests/storage/test_storage_read.py b/tests/storage/test_storage_read.py index f0ba445eed..7754918725 100644 --- a/tests/storage/test_storage_read.py +++ b/tests/storage/test_storage_read.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np diff --git a/tests/test_3d.py b/tests/test_3d.py index 6b2d1a31be..06df9763dd 100644 --- a/tests/test_3d.py +++ b/tests/test_3d.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from collections import OrderedDict diff --git a/tests/test_concurrent_executor.py b/tests/test_concurrent_executor.py index c8dac01326..f11212da43 100644 --- a/tests/test_concurrent_executor.py +++ b/tests/test_concurrent_executor.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Tests for MultiprocessingExecutor diff --git a/tests/test_config.py b/tests/test_config.py index dc8c20dc0e..0d134e532d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/test_driver.py b/tests/test_driver.py index 19bf25d683..3c9331d684 100644 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import yaml diff --git a/tests/test_dynamic_db_passwd.py b/tests/test_dynamic_db_passwd.py index 62579b99b0..17b7b9640c 100644 --- a/tests/test_dynamic_db_passwd.py +++ b/tests/test_dynamic_db_passwd.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from sqlalchemy.exc import OperationalError diff --git a/tests/test_eo3.py b/tests/test_eo3.py index 36750ee496..44cb5e0da9 100644 --- a/tests/test_eo3.py +++ b/tests/test_eo3.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from affine import Affine import pytest diff --git a/tests/test_load_data.py b/tests/test_load_data.py index d1c1305b67..66a3857d53 100644 --- a/tests/test_load_data.py +++ b/tests/test_load_data.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datacube import Datacube from datacube.api.query import query_group_by diff --git a/tests/test_metadata_fields.py b/tests/test_metadata_fields.py index 00baa21e3b..20b9ad1740 100644 --- a/tests/test_metadata_fields.py +++ b/tests/test_metadata_fields.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import yaml import datetime diff --git a/tests/test_model.py b/tests/test_model.py index ac43f29059..967135e7b0 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import numpy diff --git a/tests/test_testutils.py b/tests/test_testutils.py index fd168fe2c4..2db2a1c6e6 100644 --- a/tests/test_testutils.py +++ b/tests/test_testutils.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from datacube.testutils.threads import FakeThreadPoolExecutor diff --git a/tests/test_utils_aws.py b/tests/test_utils_aws.py index 359acfbf4b..a86cdc296e 100644 --- a/tests/test_utils_aws.py +++ b/tests/test_utils_aws.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from unittest import mock diff --git a/tests/test_utils_changes.py b/tests/test_utils_changes.py index 5499a0e41c..9f5108a09e 100644 --- a/tests/test_utils_changes.py +++ b/tests/test_utils_changes.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from datacube.utils.changes import ( diff --git a/tests/test_utils_cog.py b/tests/test_utils_cog.py index 3e7b773ece..94960b6077 100644 --- a/tests/test_utils_cog.py +++ b/tests/test_utils_cog.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from pathlib import Path diff --git a/tests/test_utils_dask.py b/tests/test_utils_dask.py index 093e629878..c7b2d2d298 100644 --- a/tests/test_utils_dask.py +++ b/tests/test_utils_dask.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import moto diff --git a/tests/test_utils_dates.py b/tests/test_utils_dates.py index a6fdd0c474..b7547f8647 100644 --- a/tests/test_utils_dates.py +++ b/tests/test_utils_dates.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import numpy as np import pytest diff --git a/tests/test_utils_docs.py b/tests/test_utils_docs.py index 4ae0811ce8..e9e6447a41 100644 --- a/tests/test_utils_docs.py +++ b/tests/test_utils_docs.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test utility functions from :module:`datacube.utils` diff --git a/tests/test_utils_generic.py b/tests/test_utils_generic.py index d8c899538c..15392bd0b7 100644 --- a/tests/test_utils_generic.py +++ b/tests/test_utils_generic.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from queue import Queue from datacube.utils.generic import ( diff --git a/tests/test_utils_other.py b/tests/test_utils_other.py index c4bdb9d9d8..53a962003e 100644 --- a/tests/test_utils_other.py +++ b/tests/test_utils_other.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Test utility functions from :module:`datacube.utils` diff --git a/tests/test_utils_rio.py b/tests/test_utils_rio.py index 558b3523b3..e834ac2fec 100644 --- a/tests/test_utils_rio.py +++ b/tests/test_utils_rio.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest from unittest import mock diff --git a/tests/test_xarray_extension.py b/tests/test_xarray_extension.py index 12f856b9cd..3b3f35215a 100644 --- a/tests/test_xarray_extension.py +++ b/tests/test_xarray_extension.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest import xarray as xr diff --git a/tests/ui/__init__.py b/tests/ui/__init__.py index c081ad5b41..45970a2a5b 100644 --- a/tests/ui/__init__.py +++ b/tests/ui/__init__.py @@ -1,4 +1,4 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/ui/test_common.py b/tests/ui/test_common.py index 51c4d3865a..8c8352002f 100644 --- a/tests/ui/test_common.py +++ b/tests/ui/test_common.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/tests/ui/test_expression_parsing.py b/tests/ui/test_expression_parsing.py index 40526ecfe2..3f30748c2e 100644 --- a/tests/ui/test_expression_parsing.py +++ b/tests/ui/test_expression_parsing.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 from datetime import datetime from dateutil.tz import tzutc diff --git a/tests/ui/test_task_app.py b/tests/ui/test_task_app.py index eac53d2ea8..6d5a4a7549 100644 --- a/tests/ui/test_task_app.py +++ b/tests/ui/test_task_app.py @@ -1,6 +1,6 @@ # This file is part of the Open Data Cube, see https://opendatacube.org for more information # -# Copyright (c) 2015-2020 ODC Contributors +# Copyright (c) 2015-2023 ODC Contributors # SPDX-License-Identifier: Apache-2.0 """ Module diff --git a/wordlist.txt b/wordlist.txt index a3d3f93d2a..4cd2ffb996 100644 --- a/wordlist.txt +++ b/wordlist.txt @@ -258,6 +258,8 @@ LTS lv macosx MakeMask +Mambaforge +mambaforge ManagingODC mapbox matplotlib @@ -269,6 +271,7 @@ mk mkdir MODIS mongolia +moto MTL multiband multigeom @@ -312,6 +315,7 @@ nx ny ODC odc +odcintegration ODCv OGC ogr @@ -325,11 +329,14 @@ osgeo osr OSX ows +pandoc param params pc petewa pgadmin +pgintegration +pgisintegration pixelquality pkgs Pluggable @@ -346,14 +353,13 @@ pq pre precollection prefetch -Preperation -preperation PRIMEM prog Proj provence psql pts +pwd py pydata pyenv @@ -462,6 +468,7 @@ usgs USGS's usgslsprepare UsingODC +usr utils UUID uuid