Merge f45490b into ff8d344

rasterio/__init__.py

@@ -174,7 +174,7 @@ def open(fp, mode='r', driver=None, width=None, height=None, count=None,
         @contextmanager
         def fp_reader(fp):
             memfile = MemoryFile(fp.read())
-            dataset = memfile.open()
+            dataset = memfile.open(driver=driver, sharing=sharing)
             try:
                 yield dataset
             finally:
@@ -190,7 +190,7 @@ def fp_writer(fp):
             memfile = MemoryFile()
             dataset = memfile.open(driver=driver, width=width, height=height,
                                    count=count, crs=crs, transform=transform,
-                                   dtype=dtype, nodata=nodata, **kwargs)
+                                   dtype=dtype, nodata=nodata, sharing=sharing, **kwargs)
             try:
                 yield dataset
             finally:
@@ -213,15 +213,16 @@ def fp_writer(fp):
         # be taken over by the dataset's context manager if it is not
         # None.
         if mode == 'r':
-            s = DatasetReader(path, driver=driver, **kwargs)
+            s = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
         elif mode == 'r+':
-            s = get_writer_for_path(path)(path, mode, driver=driver, **kwargs)
+            s = get_writer_for_path(path)(path, mode, driver=driver, sharing=sharing, **kwargs)
         elif mode.startswith("w"):
             s = get_writer_for_driver(driver)(path, mode, driver=driver,
                                               width=width, height=height,
                                               count=count, crs=crs,
                                               transform=transform,
                                               dtype=dtype, nodata=nodata,
+                                              sharing=sharing,
                                               **kwargs)
         else:
             raise ValueError(

rasterio/_base.pyx

@@ -197,6 +197,8 @@ cdef class DatasetBase(object):
         cdef int flags = 0
         cdef int sharing_flag = (0x20 if sharing else 0x0)
 
+        log.debug("Sharing flag: %r", sharing_flag)
+
         self._hds = NULL
 
         if path is not None:

rasterio/_env.pyx

@@ -21,6 +21,8 @@ from rasterio._base cimport _safe_osr_release
 from rasterio._err import CPLE_BaseError
 from rasterio._err cimport exc_wrap_ogrerr, exc_wrap_int
 
+from libc.stdio cimport stderr
+
 
 level_map = {
     0: 0,
@@ -400,3 +402,6 @@ cdef class GDALEnv(ConfigEnv):
             result[key] = val
 
         return result
+
+    def _dump_open_datasets(self):
+        GDALDumpOpenDatasets(stderr)

rasterio/env.py

@@ -245,6 +245,13 @@ def drivers(self):
         """Return a mapping of registered drivers."""
         return local._env.drivers()
 
+    def _dump_open_datasets(self):
+        """Writes descriptions of open datasets to stderr
+
+        For debugging and testing purposes.
+        """
+        return local._env._dump_open_datasets()
+
     def __enter__(self):
         log.debug("Entering env context: %r", self)
         if local._env is None:

rasterio/gdal.pxi

@@ -280,6 +280,9 @@ cdef extern from "gdal.h" nogil:
     CPLErr GDALSetRasterScale(GDALRasterBandH hBand, double dfNewScale)
     CPLErr GDALSetRasterOffset(GDALRasterBandH hBand, double dfNewOffset)
 
+    int GDALDumpOpenDatasets(FILE *fp)
+
+
 cdef extern from "ogr_api.h" nogil:
 
     ctypedef void * OGRLayerH

rasterio/io.py

@@ -106,7 +106,7 @@ def __init__(self, file_or_bytes=None, filename=None, ext=''):
 
     @ensure_env
     def open(self, driver=None, width=None, height=None, count=None, crs=None,
-             transform=None, dtype=None, nodata=None, **kwargs):
+             transform=None, dtype=None, nodata=None, sharing=False, **kwargs):
         """Open the file and return a Rasterio dataset object.
 
         If data has already been written, the file is opened in 'r'
@@ -127,13 +127,13 @@ def open(self, driver=None, width=None, height=None, count=None, crs=None,
             raise IOError("I/O operation on closed file.")
         if self.exists():
             log.debug("VSI path: {}".format(vsi_path.path))
-            return DatasetReader(vsi_path, driver=driver, **kwargs)
+            return DatasetReader(vsi_path, driver=driver, sharing=sharing, **kwargs)
         else:
             writer = get_writer_for_driver(driver)
             return writer(vsi_path, 'w+', driver=driver, width=width,
                           height=height, count=count, crs=crs,
                           transform=transform, dtype=dtype,
-                          nodata=nodata, **kwargs)
+                          nodata=nodata, sharing=sharing, **kwargs)
 
     def __enter__(self):
         self._env = env_ctx_if_needed()
@@ -156,7 +156,7 @@ def __init__(self, file_or_bytes=None):
         super(ZipMemoryFile, self).__init__(file_or_bytes, ext='zip')
 
     @ensure_env
-    def open(self, path, driver=None, **kwargs):
+    def open(self, path, driver=None, sharing=False, **kwargs):
         """Open a dataset within the zipped stream.
 
         Parameters
@@ -165,6 +165,9 @@ def open(self, path, driver=None, **kwargs):
             Path to a dataset in the zip file, relative to the root of the
             archive.
 
+        Other parameters are optional and have the same semantics as the
+        parameters of `rasterio.open()`.
+
         Returns
         -------
         A Rasterio dataset object
@@ -173,7 +176,7 @@ def open(self, path, driver=None, **kwargs):
 
         if self.closed:
             raise IOError("I/O operation on closed file.")
-        return DatasetReader(vsi_path, driver=driver, **kwargs)
+        return DatasetReader(vsi_path, driver=driver, sharing=sharing, **kwargs)
 
 
 def get_writer_for_driver(driver):

tests/conftest.py

@@ -488,6 +488,11 @@ def path_rgb_msk_byte_tif(data_dir):
     return os.path.join(data_dir, 'RGB2.byte.tif')
 
 
+@pytest.fixture(scope='session')
+def path_cogeo_tif(data_dir):
+    return os.path.join(data_dir, 'cogeo.tif')
+
+
 @pytest.fixture(scope='function')
 def _path_multiband_no_colorinterp(tmpdir):
 

tests/test_open_overview_level.py

@@ -0,0 +1,21 @@
+"""Tests for fix of #1504"""
+
+import rasterio
+
+from .conftest import requires_gdal2
+
+
+@requires_gdal2
+def test_overview_levels(path_cogeo_tif):
+    """With sharing turned off, problem noted in #1504 vanishes"""
+    olevel = 0
+    with rasterio.open(path_cogeo_tif, overview_level=olevel) as src:
+        assert src.shape == (512, 512)
+
+        olevel = 1
+        with rasterio.open(path_cogeo_tif, sharing=False, overview_level=olevel) as src:
+            assert src.shape == (256, 256)
+
+            olevel = 2
+            with rasterio.open(path_cogeo_tif, sharing=False, overview_level=olevel) as src:
+                assert src.shape == (128, 128)

tests/test_open_sharing.py

@@ -0,0 +1,45 @@
+"""Tests of dataset connection sharing"""
+
+import rasterio
+
+from .conftest import requires_gdal2
+
+
+@requires_gdal2
+def test_sharing_on(capfd, path_rgb_byte_tif):
+    """Datasets are shared"""
+    with rasterio.Env() as env:
+
+        # Opens a new file.
+        with rasterio.open(path_rgb_byte_tif, sharing=False) as srcx:
+            env._dump_open_datasets()
+            captured = capfd.readouterr()
+            assert "1 N GTiff" in captured.err
+            assert "1 S GTiff" not in captured.err
+
+            # Does not open a new file.
+            with rasterio.open(path_rgb_byte_tif, sharing=True) as srcy:
+                env._dump_open_datasets()
+                captured = capfd.readouterr()
+                assert "1 N GTiff" in captured.err
+                assert "1 S GTiff" in captured.err
+
+
+@requires_gdal2
+def test_sharing_off(capfd, path_rgb_byte_tif):
+    """Datasets are not shared"""
+    with rasterio.Env() as env:
+
+        # Opens a new file.
+        with rasterio.open(path_rgb_byte_tif, sharing=False) as srcx:
+            env._dump_open_datasets()
+            captured = capfd.readouterr()
+            assert "1 N GTiff" in captured.err
+            assert "1 S GTiff" not in captured.err
+
+            # Does not open a new file.
+            with rasterio.open(path_rgb_byte_tif, sharing=False) as srcy:
+                env._dump_open_datasets()
+                captured = capfd.readouterr()
+                assert captured.err.count("1 N GTiff") == 2
+                assert "1 S GTiff" not in captured.err