Merge pull request #27 from pkgw/lsst-demo

Work for LSST demo

README.md

@@ -68,8 +68,9 @@ and [PyPI](https://pypi.org/project/toasty/#history).
 
 [toasty] is a Python package so, yes, Python is required.
 
-- [astropy]
+- [astropy] if using FITS files or WCS coordinates
 - [cython]
+- [filelock]
 - [healpy] if using [HEALPix] maps
 - [numpy]
 - [pillow]
@@ -80,6 +81,7 @@ and [PyPI](https://pypi.org/project/toasty/#history).
 
 [astropy]: https://www.astropy.org/
 [cython]: https://cython.org/
+[filelock]: https://github.com/benediktschmitt/py-filelock
 [healpy]: https://healpy.readthedocs.io/
 [HEALPix]: https://healpix.jpl.nasa.gov/
 [numpy]: https://numpy.org/

ci/azure-build-and-test.yml

@@ -84,7 +84,7 @@ jobs:
       set -euo pipefail
       source activate-conda.sh
       set -x
-      \conda create -y -n build setuptools pip
+      \conda create -y -n build setuptools pip python=3.8
       conda activate build
       pip install $BASH_WORKSPACE/sdist/*.tar.gz
     displayName: Install from sdist
@@ -96,6 +96,7 @@ jobs:
       set -x
       \conda install -y \
         cython \
+        filelock \
         healpy \
         numpy \
         openexr-python \
@@ -123,7 +124,7 @@ jobs:
       set -euo pipefail
       source activate-conda.sh
       set -x
-      \conda create -y -n build setuptools pip
+      \conda create -y -n build setuptools pip python=3.8
       conda activate build
       pip install $BASH_WORKSPACE/sdist/*.tar.gz
     displayName: Install from sdist

ci/azure-sdist.yml

@@ -53,6 +53,7 @@ jobs:
       conda config --add channels conda-forge
       conda install -y \
         cython \
+        filelock \
         numpy \
         pillow \
         pip \

docs/api/toasty.image.Image.rst

@@ -28,6 +28,7 @@ Image
       ~Image.from_pil
       ~Image.make_thumbnail_bitmap
       ~Image.save_default
+      ~Image.update_into_maskable_buffer
 
    .. rubric:: Attributes Documentation
 
@@ -47,3 +48,4 @@ Image
    .. automethod:: from_pil
    .. automethod:: make_thumbnail_bitmap
    .. automethod:: save_default
+   .. automethod:: update_into_maskable_buffer

docs/api/toasty.pyramid.PyramidIO.rst

@@ -13,15 +13,17 @@ PyramidIO
       ~PyramidIO.get_path_scheme
       ~PyramidIO.open_metadata_for_read
       ~PyramidIO.open_metadata_for_write
-      ~PyramidIO.read_toasty_image
+      ~PyramidIO.read_image
       ~PyramidIO.tile_path
-      ~PyramidIO.write_toasty_image
+      ~PyramidIO.update_image
+      ~PyramidIO.write_image
 
    .. rubric:: Methods Documentation
 
    .. automethod:: get_path_scheme
    .. automethod:: open_metadata_for_read
    .. automethod:: open_metadata_for_write
-   .. automethod:: read_toasty_image
+   .. automethod:: read_image
    .. automethod:: tile_path
-   .. automethod:: write_toasty_image
+   .. automethod:: update_image
+   .. automethod:: write_image

docs/api/toasty.study.StudyTiling.rst

@@ -11,6 +11,7 @@ StudyTiling
    .. autosummary::
 
       ~StudyTiling.apply_to_imageset
+      ~StudyTiling.compute_for_subimage
       ~StudyTiling.count_populated_positions
       ~StudyTiling.generate_populated_positions
       ~StudyTiling.image_to_tile
@@ -20,6 +21,7 @@ StudyTiling
    .. rubric:: Methods Documentation
 
    .. automethod:: apply_to_imageset
+   .. automethod:: compute_for_subimage
    .. automethod:: count_populated_positions
    .. automethod:: generate_populated_positions
    .. automethod:: image_to_tile

setup.py

@@ -72,6 +72,7 @@ def get_long_desc():
     },
 
     install_requires = [
+        'filelock>=3',
         'numpy>=1.7',
         'pillow>=7.0',
         'PyYAML>=5.0',

toasty/image.py

@@ -547,6 +547,48 @@ def fill_into_maskable_buffer(self, buffer, iy_idx, ix_idx, by_idx, bx_idx):
         else:
             raise Exception('unhandled mode in fill_into_maskable_buffer')
 
+    def update_into_maskable_buffer(self, buffer, iy_idx, ix_idx, by_idx, bx_idx):
+        """
+        Update a maskable buffer with data from this image.
+
+        Parameters
+        ----------
+        buffer : :class:`Image`
+            The destination buffer image, created with :meth:`ImageMode.make_maskable_buffer`.
+        iy_idx : slice or other indexer
+            The indexer into the Y axis of the source image (self).
+        ix_idx : slice or other indexer
+            The indexer into the X axis of the source image (self).
+        by_idx : slice or other indexer
+            The indexer into the Y axis of the destination *buffer*.
+        bx_idx : slice or other indexer
+            The indexer into the X axis of the destination *buffer*.
+
+        Notes
+        -----
+        Unlike :meth:`fill_into_maskable_buffer`, this function does not clear
+        the entire buffer. It only overwrites the portion of the buffer covered
+        by non-NaN-like values of the input image.
+
+        """
+        i = self.asarray()
+        b = buffer.asarray()
+
+        sub_b = b[by_idx,bx_idx]
+        sub_i = i[iy_idx,ix_idx]
+
+        if self.mode == ImageMode.RGB:
+            sub_b[...,:3] = sub_i
+            sub_b[...,3] = 255
+        elif self.mode == ImageMode.RGBA:
+            valid = (sub_i[...,3] != 0)
+            np.putmask(sub_b, valid, sub_i)
+        elif self.mode == ImageMode.F32:
+            valid = ~np.isnan(sub_i)
+            np.putmask(sub_b, valid, sub_i)
+        else:
+            raise Exception('unhandled mode in update_into_maskable_buffer')
+
     def save_default(self, path_or_stream):
         """
         Save this image to a filesystem path or stream

toasty/merge.py

@@ -31,6 +31,7 @@
 import os
 import sys
 from tqdm import tqdm
+import warnings
 
 from . import pyramid
 from .image import Image
@@ -50,7 +51,12 @@ def averaging_merger(data):
 
     """
     s = (data.shape[0] // 2, 2, data.shape[1] // 2, 2) + data.shape[2:]
-    return np.nanmean(data.reshape(s), axis=(1, 3)).astype(data.dtype)
+
+    # nanmean will raise a RuntimeWarning if there are all-NaN quartets. This
+    # gets annoying, so we silence them.
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore')
+        return np.nanmean(data.reshape(s), axis=(1, 3)).astype(data.dtype)
 
 
 def cascade_images(pio, mode, start, merger, parallel=None, cli_progress=False):
@@ -108,10 +114,10 @@ def _cascade_images_serial(pio, mode, start, merger, cli_progress):
             # processed.
             children = pyramid.pos_children(pos)
 
-            img0 = pio.read_toasty_image(children[0], mode, default='none')
-            img1 = pio.read_toasty_image(children[1], mode, default='none')
-            img2 = pio.read_toasty_image(children[2], mode, default='none')
-            img3 = pio.read_toasty_image(children[3], mode, default='none')
+            img0 = pio.read_image(children[0], mode, default='none')
+            img1 = pio.read_image(children[1], mode, default='none')
+            img2 = pio.read_image(children[2], mode, default='none')
+            img3 = pio.read_image(children[3], mode, default='none')
 
             if img0 is None and img1 is None and img2 is None and img3 is None:
                 progress.update(1)
@@ -128,7 +134,7 @@ def _cascade_images_serial(pio, mode, start, merger, cli_progress):
                     buf.asarray()[slidx] = subimg.asarray()
 
             merged = Image.from_array(mode, merger(buf.asarray()))
-            pio.write_toasty_image(pos, merged)
+            pio.write_image(pos, merged)
             progress.update(1)
 
     if cli_progress:
@@ -153,10 +159,9 @@ def _cascade_images_parallel(pio, mode, start, merger, cli_progress, parallel):
 
     first_level_to_do = start - 1
     n_todo = pyramid.depth2tiles(first_level_to_do)
-    ready_queue = mp.Queue(maxsize = 2 * parallel)
+    ready_queue = mp.Queue()
     done_queue = mp.Queue(maxsize = 2 * parallel)
 
-
     dispatcher = mp.Process(
         target=_mp_cascade_dispatcher,
         args=(done_queue, ready_queue, n_todo, cli_progress)
@@ -267,10 +272,10 @@ def _mp_cascade_worker(done_queue, ready_queue, pio, merger, mode):
         # processed.
         children = pyramid.pos_children(pos)
 
-        img0 = pio.read_toasty_image(children[0], mode, default='none')
-        img1 = pio.read_toasty_image(children[1], mode, default='none')
-        img2 = pio.read_toasty_image(children[2], mode, default='none')
-        img3 = pio.read_toasty_image(children[3], mode, default='none')
+        img0 = pio.read_image(children[0], mode, default='none')
+        img1 = pio.read_image(children[1], mode, default='none')
+        img2 = pio.read_image(children[2], mode, default='none')
+        img3 = pio.read_image(children[3], mode, default='none')
 
         if img0 is None and img1 is None and img2 is None and img3 is None:
             pass  # No data here; ignore
@@ -286,6 +291,6 @@ def _mp_cascade_worker(done_queue, ready_queue, pio, merger, mode):
                     buf.asarray()[slidx] = subimg.asarray()
 
             merged = Image.from_array(mode, merger(buf.asarray()))
-            pio.write_toasty_image(pos, merged)
+            pio.write_image(pos, merged)
 
         done_queue.put(pos)