Merge branch 'maint-1.0'

CHANGES.txt

@@ -1,6 +1,17 @@
 Changes
 =======
 
+1.0.8 (2018-10-02)
+------------------
+
+Bug fixes:
+
+- Datasets contained in MemoryFile buffers are now opened in r+ or w+ mode
+  instead of r or w.
+- The namedtuple _asdict method is overridden in BoundingBox to work around a
+  bug in Python 3.4.3 (#1486, #1488). This unblocks creation of manylinux1
+  wheels for Python 3.4.
+
 1.0.7 (2018-09-26)
 ------------------
 

rasterio/__init__.py

@@ -43,7 +43,7 @@ def emit(self, record):
 
 
 __all__ = ['band', 'open', 'pad', 'Env']
-__version__ = "1.0.7"
+__version__ = "1.0.8"
 __gdal_version__ = gdal_version()
 
 # Rasterio attaches NullHandler to the 'rasterio' logger and its

rasterio/_io.pyx

@@ -397,7 +397,40 @@ cdef class DatasetReaderBase(DatasetBase):
                     indexes, out, Window(0, 0, window.width, window.height),
                     None, resampling=resampling)
 
-                if masked:
+                if masked and all_valid:
+                    blank_path = UnparsedPath('/vsimem/blank-{}.tif'.format(uuid.uuid4()))
+                    transform = Affine.translation(self.transform.xoff, self.transform.yoff) * (Affine.scale(self.width / 3, self.height / 3) * (Affine.translation(-self.transform.xoff, -self.transform.yoff) * self.transform))
+                    with DatasetWriterBase(
+                            blank_path, 'w',
+                            driver='GTiff', count=self.count, height=3, width=3,
+                            dtype='uint8', crs=self.crs, transform=transform) as blank_dataset:
+                        blank_dataset.write(
+                            np.ones((self.count, 3, 3), dtype='uint8'))
+
+                    with DatasetReaderBase(blank_path) as blank_dataset:
+                        mask_vrt_doc = _boundless_vrt_doc(
+                            blank_dataset, nodata=0,
+                            width=max(self.width, window.width) + 1,
+                            height=max(self.height, window.height) + 1,
+                            transform=self.window_transform(window)).decode('ascii')
+
+                        with DatasetReaderBase(UnparsedPath(mask_vrt_doc), **vrt_kwds) as mask_vrt:
+                            mask = np.zeros(out.shape, 'uint8')
+                            mask = ~mask_vrt._read(
+                                indexes, mask, Window(0, 0, window.width, window.height), None).astype('bool')
+
+                            kwds = {'mask': mask}
+                            # Set a fill value only if the read bands share a
+                            # single nodata value.
+                            if fill_value is not None:
+                                kwds['fill_value'] = fill_value
+                            elif len(set(nodatavals)) == 1:
+                                if nodatavals[0] is not None:
+                                    kwds['fill_value'] = nodatavals[0]
+
+                            out = np.ma.array(out, **kwds)
+
+                elif masked:
                     mask = np.zeros(out.shape, 'uint8')
                     mask = ~vrt._read(
                         indexes, mask, Window(0, 0, window.width, window.height), None, masks=True).astype('bool')
@@ -545,33 +578,61 @@ cdef class DatasetReaderBase(DatasetBase):
         # If this is a boundless read we will create an in-memory VRT
         # in order to use GDAL's windowing and compositing logic.
         else:
-            vrt_doc = _boundless_vrt_doc(
-                self, width=max(self.width, window.width) + 1,
-                height=max(self.height, window.height) + 1,
-                transform=self.window_transform(window)).decode('ascii')
+
+            enums = self.mask_flag_enums
+            all_valid = all([MaskFlags.all_valid in flags for flags in enums])
 
             if not gdal_version().startswith('1'):
                 vrt_kwds = {'driver': 'VRT'}
             else:
                 vrt_kwds = {}
-            with DatasetReaderBase(UnparsedPath(vrt_doc), **vrt_kwds) as vrt:
-
-                out = vrt._read(
-                    indexes, out, Window(0, 0, window.width, window.height),
-                    None, resampling=resampling, masks=True)
 
-                # TODO: we need to determine why `out` can contain data
-                # that looks more like the source's band 1 when doing
-                # this kind of boundless read. It looks like
-                # hmask = GDALGetMaskBand(band) may be returning the
-                # a pointer to the band instead of the mask band in 
-                # this case.
-                # 
-                # Temporary solution: convert all non-zero pixels to
-                # 255 and log that we have done so.
+            if all_valid:
+                blank_path = UnparsedPath('/vsimem/blank-{}.tif'.format(uuid.uuid4()))
+                transform = Affine.translation(self.transform.xoff, self.transform.yoff) * (Affine.scale(self.width / 3, self.height / 3) * (Affine.translation(-self.transform.xoff, -self.transform.yoff) * self.transform))
+                with DatasetWriterBase(
+                        blank_path, 'w',
+                        driver='GTiff', count=self.count, height=3, width=3,
+                        dtype='uint8', crs=self.crs, transform=transform) as blank_dataset:
+                    blank_dataset.write(
+                        np.full((self.count, 3, 3), 255, dtype='uint8'))
+
+                with DatasetReaderBase(blank_path) as blank_dataset:
+                    mask_vrt_doc = _boundless_vrt_doc(
+                        blank_dataset, nodata=0,
+                        width=max(self.width, window.width) + 1,
+                        height=max(self.height, window.height) + 1,
+                        transform=self.window_transform(window)).decode('ascii')
+
+                    with DatasetReaderBase(UnparsedPath(mask_vrt_doc), **vrt_kwds) as mask_vrt:
+                        out = np.zeros(out.shape, 'uint8')
+                        out = mask_vrt._read(
+                            indexes, out, Window(0, 0, window.width, window.height), None).astype('bool')
 
-                out = np.where(out != 0, 255, 0)
-                log.warn("Nonzero values in mask have been converted to 255, see note in rasterio/_io.pyx, read_masks()")
+            else:
+                vrt_doc = _boundless_vrt_doc(
+                    self, width=max(self.width, window.width) + 1,
+                    height=max(self.height, window.height) + 1,
+                    transform=self.window_transform(window)).decode('ascii')
+
+                with DatasetReaderBase(UnparsedPath(vrt_doc), **vrt_kwds) as vrt:
+
+                    out = vrt._read(
+                        indexes, out, Window(0, 0, window.width, window.height),
+                        None, resampling=resampling, masks=True)
+
+                    # TODO: we need to determine why `out` can contain data
+                    # that looks more like the source's band 1 when doing
+                    # this kind of boundless read. It looks like
+                    # hmask = GDALGetMaskBand(band) may be returning the
+                    # a pointer to the band instead of the mask band in 
+                    # this case.
+                    # 
+                    # Temporary solution: convert all non-zero pixels to
+                    # 255 and log that we have done so.
+
+                    out = np.where(out != 0, 255, 0)
+                    log.warn("Nonzero values in mask have been converted to 255, see note in rasterio/_io.pyx, read_masks()")
 
         if return2d:
             out.shape = out.shape[1:]
@@ -1863,7 +1924,7 @@ cdef class BufferedDatasetWriterBase(DatasetWriterBase):
         # Validate write mode arguments.
 
         log.debug("Path: %s, mode: %s, driver: %s", path, mode, driver)
-        if mode == 'w':
+        if mode in ('w', 'w+'):
             if not isinstance(driver, string_types):
                 raise TypeError("A driver name string is required.")
             try:
@@ -1912,7 +1973,7 @@ cdef class BufferedDatasetWriterBase(DatasetWriterBase):
 
         memdrv = GDALGetDriverByName("MEM")
 
-        if self.mode == 'w':
+        if self.mode in ('w', 'w+'):
             # Find the equivalent GDAL data type or raise an exception
             # We've mapped numpy scalar types to GDAL types so see
             # if we can crosswalk those.

rasterio/coords.py

@@ -1,6 +1,6 @@
 """Bounding box tuple, and disjoint operator."""
 
-from collections import namedtuple
+from collections import namedtuple, OrderedDict
 
 _BoundingBox = namedtuple('BoundingBox', ('left', 'bottom', 'right', 'top'))
 
@@ -24,7 +24,8 @@ class BoundingBox(_BoundingBox):
         Top coordinate
     """
 
-    pass
+    def _asdict(self):
+        return OrderedDict(zip(self._fields, self))
 
 
 def disjoint_bounds(bounds1, bounds2):

rasterio/io.py

@@ -127,10 +127,10 @@ def open(self, driver=None, width=None, height=None, count=None, crs=None,
             raise IOError("I/O operation on closed file.")
         if self.exists():
             log.debug("VSI path: {}".format(vsi_path.path))
-            return DatasetReader(vsi_path, driver=driver, **kwargs)
+            return DatasetReader(vsi_path, mode='r+', driver=driver, **kwargs)
         else:
             writer = get_writer_for_driver(driver)
-            return writer(vsi_path, 'w', driver=driver, width=width,
+            return writer(vsi_path, 'w+', driver=driver, width=width,
                           height=height, count=count, crs=crs,
                           transform=transform, dtype=dtype,
                           nodata=nodata, **kwargs)

rasterio/vrt.py

@@ -161,8 +161,8 @@ def _boundless_vrt_doc(
         dstrect = ET.SubElement(simplesource, 'DstRect')
         dstrect.attrib['xOff'] = str((src_dataset.transform.xoff - transform.xoff) / transform.a)
         dstrect.attrib['yOff'] = str((src_dataset.transform.yoff - transform.yoff) / transform.e)
-        dstrect.attrib['xSize'] = str(src_dataset.width)
-        dstrect.attrib['ySize'] = str(src_dataset.height)
+        dstrect.attrib['xSize'] = str(src_dataset.width * src_dataset.transform.a / transform.a)
+        dstrect.attrib['ySize'] = str(src_dataset.height * src_dataset.transform.e / transform.e)
 
         if src_dataset.nodata is not None:
             nodata_elem = ET.SubElement(simplesource, 'NODATA')

setup.py

@@ -332,7 +332,7 @@ def run(self):
 
 # Runtime requirements.
 inst_reqs = [
-    'affine', 'attrs', 'cligj>=0.5', 'numpy', 'snuggs>=1.4.1', 'click-plugins']
+    'affine', 'attrs', 'click>=4.0,<8', 'cligj>=0.5', 'numpy', 'snuggs>=1.4.1', 'click-plugins']
 
 if sys.version_info < (3, 4):
     inst_reqs.append('enum34')

tests/test_boundless_read.py

@@ -82,6 +82,26 @@ def test_boundless_fill_value():
     assert (filled[-1, :] == 5).all()
 
 
+def test_boundless_masked_special():
+    """Confirm resolution of special case in issue #1471"""
+    with rasterio.open("tests/data/green.tif") as src:
+        masked = src.read(2, boundless=True, masked=True, window=Window(-1, -1, 66, 66))
+    assert not masked[:, 0].any()
+    assert not masked[:, -1].any()
+    assert not masked[0, :].any()
+    assert not masked[-1, :].any()
+
+
+def test_boundless_mask_special():
+    """Confirm resolution of special case in issue #1471"""
+    with rasterio.open("tests/data/green.tif") as src:
+        mask = src.read_masks(2, boundless=True, window=Window(-1, -1, 66, 66))
+    assert not mask[:, 0].any()
+    assert not mask[:, -1].any()
+    assert not mask[0, :].any()
+    assert not mask[-1, :].any()
+
+
 def test_boundless_fill_value_overview_masks():
     """Confirm a more general resolution to issue #1471"""
     with rasterio.open("tests/data/cogeo.tif") as src:

tests/test_memoryfile.py

@@ -5,14 +5,14 @@
 import logging
 import os.path
 
+from affine import Affine
+import numpy
 import pytest
 
 import rasterio
 from rasterio.io import MemoryFile, ZipMemoryFile
 from rasterio.env import GDALVersion
 
-logging.basicConfig(level=logging.DEBUG)
-
 
 # Skip ENTIRE module if not GDAL >= 2.x.
 # pytestmark is a keyword that instructs pytest to skip this module.
@@ -233,3 +233,27 @@ def test_vrt_memfile():
             assert src.count == 3
             assert src.dtypes == ('uint8', 'uint8', 'uint8')
             assert src.read().shape == (3, 768, 1024)
+
+
+def test_write_plus_mode():
+    with MemoryFile() as memfile:
+        with memfile.open(driver='GTiff', dtype='uint8', count=3, height=32, width=32, crs='epsg:3226', transform=Affine.identity() * Affine.scale(0.5, -0.5)) as dst:
+            dst.write(numpy.full((32, 32), 255, dtype='uint8'), 1)
+            dst.write(numpy.full((32, 32), 204, dtype='uint8'), 2)
+            dst.write(numpy.full((32, 32), 153, dtype='uint8'), 3)
+            data = dst.read()
+            assert (data[0] == 255).all()
+            assert (data[1] == 204).all()
+            assert (data[2] == 153).all()
+
+
+def test_write_plus_model_jpeg():
+    with MemoryFile() as memfile:
+        with memfile.open(driver='JPEG', dtype='uint8', count=3, height=32, width=32, crs='epsg:3226', transform=Affine.identity() * Affine.scale(0.5, -0.5)) as dst:
+            dst.write(numpy.full((32, 32), 255, dtype='uint8'), 1)
+            dst.write(numpy.full((32, 32), 204, dtype='uint8'), 2)
+            dst.write(numpy.full((32, 32), 153, dtype='uint8'), 3)
+            data = dst.read()
+            assert (data[0] == 255).all()
+            assert (data[1] == 204).all()
+            assert (data[2] == 153).all()