Porting recent develop (1.8.x) changes forward into develop-1.9 branch.

.github/workflows/main.yml

@@ -27,6 +27,13 @@ jobs:
       with:
         fetch-depth: 0
 
+    - name: Config
+      id: cfg
+      run: |
+        if "${GITHUB_REF}" in "refs/tags/"*; then
+          echo "push_pypi=yes" >> $GITHUB_OUTPUT
+        fi
+
     - uses: dorny/paths-filter@v2
       id: changes
       if: |
@@ -112,10 +119,11 @@ jobs:
         ls -lh ./dist/
         twine check ./dist/*
         EOF
+
     - name: Publish to PyPi
       if: |
         github.event_name == 'push'
-        && github.ref == 'refs/heads/pypi/publish'
+        && steps.cfg.outputs.push_pypi == 'yes'
       run: |
         if [ -n "${TWINE_PASSWORD}" ]; then
           docker run --rm  \
@@ -132,6 +140,7 @@ jobs:
         else
            echo "Skipping upload as 'PyPiToken' is not set"
         fi
+
       env:
         TWINE_PASSWORD: ${{ secrets.PyPiToken }}
 

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
     - repo: https://github.com/adrienverge/yamllint.git
-      rev: v1.30.0
+      rev: v1.32.0
       hooks:
         - id: yamllint
     - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -22,3 +22,18 @@ repos:
         rev: v3.0.0a5  # Use the sha / tag you want to point at
         hooks:
         -   id: pylint
+    - repo: https://github.com/Lucas-C/pre-commit-hooks
+      rev: v1.5.1
+      hooks:
+        - id: forbid-crlf
+        - id: remove-crlf
+        - id: forbid-tabs
+        - id: remove-tabs
+          args: [--whitespaces-count, '2'] 
+        - id: insert-license
+          files: ./(.*).py$
+          args:
+            - --license-filepath
+            - license-template.txt
+            - --use-current-year
+            - --no-extra-eol

README.rst

@@ -49,12 +49,12 @@ Developer setup
 
    -  ``git clone https://github.com/opendatacube/datacube-core.git``
 
-2. Create a Python environment for using the ODC.  We recommend `conda <https://docs.conda.io/en/latest/miniconda.html>`__ as the
+2. Create a Python environment for using the ODC.  We recommend `Mambaforge <https://mamba.readthedocs.io/en/latest/user_guide/mamba.html>`__ as the
    easiest way to handle Python dependencies.
 
 ::
 
-   conda create -f conda-environment.yml
+   mamba env create -f conda-environment.yml
    conda activate cubeenv
 
 3. Install a develop version of datacube-core.
@@ -72,26 +72,34 @@ Developer setup
    pre-commit install
 
 5. Run unit tests + PyLint
-   ``./check-code.sh``
-
-   (this script approximates what is run by Travis. You can
-   alternatively run ``pytest`` yourself). Some test dependencies may need to be installed, attempt to install these using:
 
+Install test dependencies using:
+
    ``pip install --upgrade -e '.[test]'``
 
-   If install for these fails please lodge them as issues.
+If install for these fails, please lodge them as issues.
+
+Run unit tests with:
+
+   ``./check-code.sh``
+
+   (this script approximates what is run by GitHub Actions. You can
+   alternatively run ``pytest`` yourself). 
 
 6. **(or)** Run all tests, including integration tests.
 
    ``./check-code.sh integration_tests``
 
    -  Assumes a password-less Postgres database running on localhost called
 
-   ``agdcintegration``
+   ``pgintegration``
 
-   -  Otherwise copy ``integration_tests/agdcintegration.conf`` to
+   -  Otherwise copy ``integration_tests/integration.conf`` to
       ``~/.datacube_integration.conf`` and edit to customise.
 
+   - For instructions on setting up a password-less Postgres database, see
+      the `developer setup instructions <https://datacube-core.readthedocs.io/en/latest/installation/setup/ubuntu.html#postgres-database-configuration>`__.
+
 
 Alternatively one can use the ``opendatacube/datacube-tests`` docker image to run
 tests. This docker includes database server pre-configured for running
@@ -103,11 +111,13 @@ to ``./check-code.sh`` script.
    ./check-code.sh --with-docker integration_tests
 
 
-To run individual test in docker container
+To run individual tests in a docker container
 
 ::
 
-    docker run -ti -v /home/ubuntu/datacube-core:/code opendatacube/datacube-tests:latest pytest integration_tests/test_filename.py::test_function_name
+    docker build --tag=opendatacube/datacube-tests-local --no-cache --progress plain -f docker/Dockerfile .
+
+    docker run -ti -v $(pwd):/code opendatacube/datacube-tests-local:latest pytest integration_tests/test_filename.py::test_function_name
 
 
 Developer setup on Ubuntu

datacube/__init__.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """
 Datacube

datacube/__main__.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 if __name__ == "__main__":
     from .config import auto_config

datacube/api/__init__.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """
 Modules for the Storage and Access Query API

datacube/api/core.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2021 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import uuid
@@ -236,12 +236,17 @@ def load(self, product=None, measurements=None, output_crs=None, resolution=None
 
                 x=(1516200, 1541300), y=(-3867375, -3867350), crs='EPSG:3577'
 
-            The ``time`` dimension can be specified using a tuple of datetime objects or strings with
-            ``YYYY-MM-DD hh:mm:ss`` format. Data will be loaded inclusive of the start and finish times. E.g::
+            The ``time`` dimension can be specified using a single or tuple of datetime objects or strings with
+            ``YYYY-MM-DD hh:mm:ss`` format. Data will be loaded inclusive of the start and finish times.
+            A ``None`` value in the range indicates an open range, with the provided date serving as either the
+            upper or lower bound. E.g::
 
                 time=('2000-01-01', '2001-12-31')
                 time=('2000-01', '2001-12')
                 time=('2000', '2001')
+                time=('2000')
+                time=('2000', None)  # all data from 2000 onward
+                time=(None, '2000')  # all data up to and including 2000
 
             For 3D datasets, where the product definition contains an ``extra_dimension`` specification,
             these dimensions can be queried using that dimension's name. E.g.::

datacube/api/grid_workflow.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import xarray

datacube/api/query.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """
 Storage Query and Access API module
@@ -136,8 +136,8 @@ def __init__(self, index=None, product=None, geopolygon=None, like=None, **searc
                 if time_coord is not None:
                     self.search['time'] = _time_to_search_dims(
                         (pandas_to_datetime(time_coord.values[0]).to_pydatetime(),
-                         pandas_to_datetime(time_coord.values[-1]).to_pydatetime()
-                         + datetime.timedelta(milliseconds=1))  # TODO: inclusive time searches
+                            pandas_to_datetime(time_coord.values[-1]).to_pydatetime()
+                            + datetime.timedelta(milliseconds=1))  # TODO: inclusive time searches
                     )
 
     @property
@@ -350,7 +350,11 @@ def _time_to_search_dims(time_range):
         if hasattr(tr_end, 'isoformat'):
             tr_end = tr_end.isoformat()
 
+        if tr_start is None:
+            tr_start = datetime.datetime.fromtimestamp(0)
         start = _to_datetime(tr_start)
+        if tr_end is None:
+            tr_end = datetime.datetime.now().strftime("%Y-%m-%d")
         end = _to_datetime(pandas.Period(tr_end)
                            .end_time
                            .to_pydatetime())

datacube/config.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """
 User configuration.

datacube/drivers/__init__.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """
 This module implements a simple plugin manager for storage and index drivers.

datacube/drivers/_tools.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 from threading import Lock
 from typing import Any

datacube/drivers/_types.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """ Defines abstract types for IO drivers.
 """

datacube/drivers/datasource.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """ Defines abstract types for IO reader drivers.
 """

datacube/drivers/driver_cache.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 import logging
 from typing import Dict, Any, Tuple, Iterable

datacube/drivers/indexes.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 from typing import List, Optional
 

datacube/drivers/netcdf/__init__.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 from ._write import write_dataset_to_netcdf, create_netcdf_storage_unit
 from . import writer as netcdf_writer

datacube/drivers/netcdf/_write.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 from pathlib import Path
 import logging

datacube/drivers/netcdf/driver.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 from urllib.parse import urlsplit
 

datacube/drivers/netcdf/writer.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """
 Create netCDF4 Storage Units and write data to them

datacube/drivers/postgis/__init__.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """
 Lower-level database access.

datacube/drivers/postgis/_api.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 
 # We often have one-arg-per column, so these checks aren't so useful.

datacube/drivers/postgis/_connections.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 
 # We often have one-arg-per column, so these checks aren't so useful.

datacube/drivers/postgis/_core.py

@@ -1,6 +1,6 @@
 # This file is part of the Open Data Cube, see https://opendatacube.org for more information
 #
-# Copyright (c) 2015-2020 ODC Contributors
+# Copyright (c) 2015-2023 ODC Contributors
 # SPDX-License-Identifier: Apache-2.0
 """
 Core SQL schema settings.