Merge branch 'eodatasets3' of github.com:GeoscienceAustralia/eo-datas…

…ets into C3_version_fix

.gitignore

@@ -27,3 +27,5 @@ nosetests.xml
 MANIFEST
 venv
 build
+
+.vscode

eodatasets3/images.py

@@ -1,31 +1,29 @@
-import os
-import sys
-import tempfile
-from collections import defaultdict
-from pathlib import Path
-from typing import Tuple, Dict, List, Sequence, Optional, Iterable
-from typing import Union, Generator
-
 import attr
 import numpy
-import numpy as np
+import os
 import rasterio
 import rasterio.features
 import shapely
 import shapely.affinity
 import shapely.ops
+import sys
+import tempfile
 import xarray
 from affine import Affine
-from eodatasets3.model import GridDoc, MeasurementDoc, DatasetDoc
-from eodatasets3.properties import FileFormat
+from collections import defaultdict
+from pathlib import Path
 from rasterio import DatasetReader
 from rasterio.coords import BoundingBox
 from rasterio.crs import CRS
 from rasterio.enums import Resampling
 from rasterio.io import DatasetWriter
 from rasterio.shutil import copy as rio_copy
-from rasterio.warp import reproject, calculate_default_transform
-from shapely.geometry.base import BaseGeometry, CAP_STYLE, JOIN_STYLE
+from rasterio.warp import calculate_default_transform, reproject
+from shapely.geometry.base import CAP_STYLE, JOIN_STYLE, BaseGeometry
+from typing import Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union
+
+from eodatasets3.model import DatasetDoc, GridDoc, MeasurementDoc
+from eodatasets3.properties import FileFormat
 
 DEFAULT_OVERVIEWS = (8, 16, 32)
 
@@ -82,7 +80,6 @@ def from_odc_xarray(cls, dataset: xarray.Dataset) -> "GridSpec":
             transform=dataset.geobox.transform,
             crs=CRS.from_wkt(dataset.geobox.crs.crs_str),
         )
-        pass
 
     @property
     def bounds(self):
@@ -493,7 +490,7 @@ def write_from_ndarray(
 
         # convert any bools to uin8
         if dtype == "bool":
-            array = np.uint8(array)
+            array = numpy.uint8(array)
             dtype = "uint8"
 
         ndims = array.ndim
@@ -686,6 +683,66 @@ def create_thumbnail(
                                 index,
                             )
 
+    def create_thumbnail_singleband(
+        self,
+        in_file: Path,
+        out_file: Path,
+        bit: int = None,
+        lookup_table: Dict[int, Tuple[int, int, int]] = None,
+    ):
+        """
+        Write out a JPG thumbnail from a singleband image.
+        This takes in a path to a valid raster dataset and writes
+        out a file with only the values of the bit (integer) as white
+        """
+        if bit is not None and lookup_table is not None:
+            raise ValueError(
+                "Please set either bit or lookup_table, and not both of them"
+            )
+
+        with rasterio.open(in_file) as dataset:
+            data = dataset.read()
+
+            if bit is not None:
+                out_data = numpy.copy(data)
+                out_data[data != bit] = 0
+                stretch = [0, bit]
+            if lookup_table is not None:
+                out_data = [
+                    numpy.full_like(data, 0),
+                    numpy.full_like(data, 0),
+                    numpy.full_like(data, 0),
+                ]
+                stretch = [0, 255]
+
+                for value, rgb in lookup_table.items():
+                    for index in range(3):
+                        out_data[index][data == value] = rgb[index]
+
+        meta = dataset.meta
+        meta["driver"] = "GTiff"
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if bit:
+                # Only use one file, three times
+                temp_file = Path(temp_dir) / "temp.tif"
+
+                with rasterio.open(temp_file, "w", **meta) as tmpdataset:
+                    tmpdataset.write(out_data)
+                self.create_thumbnail(
+                    [temp_file, temp_file, temp_file],
+                    out_file,
+                    static_stretch=stretch,
+                )
+            else:
+                # Use three different files
+                temp_files = [Path(temp_dir) / f"temp_{i}.tif" for i in range(3)]
+
+                for i in range(3):
+                    with rasterio.open(temp_files[i], "w", **meta) as tmpdataset:
+                        tmpdataset.write(out_data[i])
+                self.create_thumbnail(temp_files, out_file, static_stretch=stretch)
+
 
 def _write_quicklook(
     rgb: Sequence[Path],
@@ -752,7 +809,7 @@ def _write_quicklook(
                     image_null_mask=~valid_data_mask,
                     in_range=(static_range or calculated_range),
                     out_range=(1, 255),
-                    out_dtype=np.uint8,
+                    out_dtype=numpy.uint8,
                 ),
                 reprojected_data,
                 src_crs=input_geobox.crs,
@@ -770,7 +827,7 @@ def _write_quicklook(
     return reproj_grid
 
 
-LazyImages = Iterable[Tuple[np.ndarray, int]]
+LazyImages = Iterable[Tuple[numpy.ndarray, int]]
 
 
 def _iter_images(rgb: Sequence[Path]) -> LazyImages:
@@ -805,7 +862,7 @@ def read_valid_mask_and_value_range(
             the_data = array[valid_data_mask]
             # Check if there's a non-empty array first
             if the_data.any():
-                low, high = np.percentile(
+                low, high = numpy.percentile(
                     the_data, calculate_percentiles, interpolation="nearest"
                 )
                 calculated_range = (
@@ -817,14 +874,14 @@ def read_valid_mask_and_value_range(
 
 
 def rescale_intensity(
-    image: np.ndarray,
+    image: numpy.ndarray,
     in_range: Tuple[int, int],
     out_range: Optional[Tuple[int, int]] = None,
     image_nodata: int = None,
-    image_null_mask: np.ndarray = None,
-    out_dtype=np.uint8,
+    image_null_mask: numpy.ndarray = None,
+    out_dtype=numpy.uint8,
     out_nodata=0,
-) -> np.ndarray:
+) -> numpy.ndarray:
     """
     Based on scikit-image's rescale_intensity, but does fewer copies/allocations of the array.
 
@@ -836,13 +893,13 @@ def rescale_intensity(
         image_null_mask = image == image_nodata
 
     imin, imax = in_range
-    omin, omax = out_range or (np.iinfo(out_dtype).min, np.iinfo(out_dtype).max)
+    omin, omax = out_range or (numpy.iinfo(out_dtype).min, numpy.iinfo(out_dtype).max)
 
     # The intermediate calculation will need floats.
     # We'll convert to it immediately to avoid modifying the input array
-    image = image.astype(np.float64)
+    image = image.astype(numpy.float64)
 
-    np.clip(image, imin, imax, out=image)
+    numpy.clip(image, imin, imax, out=image)
     image -= imin
     image /= float(imax - imin)
     image *= omax - omin

tests/integration/__init__.py

@@ -4,11 +4,43 @@
 """
 from __future__ import absolute_import
 
+import binascii
 import hashlib
+import rasterio
 import tempfile
 from pathlib import Path
-
-import binascii
+from rasterio import DatasetReader
+from typing import Dict, Tuple
+import numpy
+
+allow_anything = object()
+
+
+def assert_image(
+    image: Path,
+    overviews=allow_anything,
+    nodata=allow_anything,
+    unique_pixel_counts: Dict = allow_anything,
+    bands=1,
+    shape: Tuple[int, int] = None,
+):
+    __tracebackhide__ = True
+    with rasterio.open(image) as d:
+        d: DatasetReader
+        assert d.count == bands, f"Expected {bands} band{'s' if bands > 1 else ''}"
+
+        if overviews is not allow_anything:
+            assert d.overviews(1) == overviews
+        if nodata is not allow_anything:
+            assert d.nodata == nodata
+
+        if unique_pixel_counts is not allow_anything:
+            array = d.read(1)
+            value_counts = dict(zip(*numpy.unique(array, return_counts=True)))
+            assert value_counts == unique_pixel_counts
+
+        if shape:
+            assert shape == d.shape, f"Unexpected shape: {shape!r} != {d.shape!r}"
 
 
 def load_checksum_filenames(output_metadata_path):

tests/integration/common.py

@@ -1,12 +1,11 @@
-from functools import partial
-from pathlib import Path
-from pprint import pformat, pprint
-from typing import Dict
-
 import rapidjson
 from click.testing import CliRunner, Result
 from deepdiff import DeepDiff
+from functools import partial
+from pathlib import Path
+from pprint import pformat, pprint
 from ruamel import yaml
+from typing import Dict
 
 diff = partial(DeepDiff, significant_digits=6)
 

tests/integration/conftest.py

@@ -27,6 +27,8 @@
     / "LS8_OLITIRS_STD-MD_P00_LC80840720742017365LGN00_084_072-074_20180101T004644Z20180101T004824_1"
 )
 
+WOFS_PATH: Path = Path(__file__).parent / "data" / "wofs"
+
 
 def path_offset(base: Path, offset: str):
     return str(normalise_nci_symlinks(base.absolute().joinpath(offset)))
@@ -592,3 +594,13 @@ def l1_ls5_tarball_md_expected(
         },
         "lineage": {},
     }
+
+
+@pytest.fixture
+def input_uint8_tif() -> Path:
+    return Path(WOFS_PATH / "ga_ls_wofs_3_099081_2020-07-26_interim_water_clipped.tif")
+
+
+@pytest.fixture
+def input_uint8_tif_2() -> Path:
+    return Path(WOFS_PATH / "ga_ls_wofs_3_090081_1993_01_05_interim_water_clipped.tif")

tests/integration/test_packagewagl.py

@@ -1,20 +1,19 @@
-from binascii import crc32
-from datetime import datetime, timedelta, timezone
-from pathlib import Path
-from typing import Dict, Tuple
-
-import numpy as np
 import pytest
 import rasterio
+from binascii import crc32
 from click.testing import CliRunner
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
 from rasterio import DatasetReader
 from rasterio.enums import Compression
 from rio_cogeo import cogeo
-from tests.integration.common import assert_same_as_file
 
 import eodatasets3
 from eodatasets3.model import DatasetDoc
 from tests import assert_file_structure
+from tests.integration.common import assert_same_as_file
+
+from . import assert_image
 
 h5py = pytest.importorskip(
     "h5py",
@@ -121,10 +120,10 @@ def test_whole_wagl_package(
 
     # Verify the computed contiguity looks the same. (metadata fields will depend on it)
     [image] = expected_folder.rglob("*_oa_*nbar-contiguity.tif")
-    _assert_image(image, nodata=255, unique_pixel_counts={0: 1978, 1: 4184})
+    assert_image(image, nodata=255, unique_pixel_counts={0: 1978, 1: 4184})
 
     [image] = expected_folder.rglob("*_oa_*nbart-contiguity.tif")
-    _assert_image(image, nodata=255, unique_pixel_counts={0: 1979, 1: 4183})
+    assert_image(image, nodata=255, unique_pixel_counts={0: 1979, 1: 4183})
 
     assert_same_as_file(
         {
@@ -453,44 +452,14 @@ def test_whole_wagl_package(
     for image in oa_images:
         # fmask is the only OA that should have overviews according to spec (and Josh).
         if "fmask" in image.name:
-            _assert_image(image, overviews=[8, 16, 26])
+            assert_image(image, overviews=[8, 16, 26])
         else:
-            _assert_image(image, overviews=[])
+            assert_image(image, overviews=[])
 
     # Check we didn't get height/width mixed up again :)
     # (The small size of our test data makes this slightly silly, though...)
     [thumb_path] = expected_folder.rglob("*_nbar_*.jpg")
-    _assert_image(thumb_path, bands=3, shape=(7, 8))
-
-
-allow_anything = object()
-
-
-def _assert_image(
-    image: Path,
-    overviews=allow_anything,
-    nodata=allow_anything,
-    unique_pixel_counts: Dict = allow_anything,
-    bands=1,
-    shape: Tuple[int, int] = None,
-):
-    __tracebackhide__ = True
-    with rasterio.open(image) as d:
-        d: DatasetReader
-        assert d.count == bands, f"Expected {bands} band{'s' if bands > 1 else ''}"
-
-        if overviews is not allow_anything:
-            assert d.overviews(1) == overviews
-        if nodata is not allow_anything:
-            assert d.nodata == nodata
-
-        if unique_pixel_counts is not allow_anything:
-            array = d.read(1)
-            value_counts = dict(zip(*np.unique(array, return_counts=True)))
-            assert value_counts == unique_pixel_counts
-
-        if shape:
-            assert shape == d.shape, f"Unexpected shape: {shape!r} != {d.shape!r}"
+    assert_image(thumb_path, bands=3, shape=(7, 8))
 
 
 def test_maturity_calculation():