From 2a3baf4365e1a20ce4d7df0956db37994b397be7 Mon Sep 17 00:00:00 2001 From: Chris Helm Date: Tue, 19 Dec 2017 15:52:50 -0700 Subject: [PATCH] making rasterio optional and only used for creating geotiffs --- appveyor.yml | 2 -- docs/conf.py | 5 +---- environment.yml | 2 -- gbdxtools/images/tms_image.py | 19 +++++++------------ gbdxtools/ipe/interface.py | 12 +++--------- gbdxtools/ipe/io.py | 8 +++++++- install_windows.md | 7 ------- meta.yaml | 2 -- requirements.txt | 1 - tests/unit/test_catalog_image.py | 1 - tests/unit/test_dem_image.py | 1 - tests/unit/test_geoeye_image.py | 1 - tests/unit/test_ikonos_image.py | 1 - tests/unit/test_ipe_image.py | 11 ----------- tests/unit/test_landsat_image.py | 1 - tests/unit/test_quickbird_image.py | 1 - tests/unit/test_s3_image.py | 1 - tests/unit/test_sentinel2_image.py | 1 - tests/unit/test_worldview.py | 1 - 19 files changed, 18 insertions(+), 60 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 289006f9..9c17b216 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -39,8 +39,6 @@ install: - "%PYTHON%\\python.exe -m pip install GDAL-2.1.3-cp27-cp27m-win32.whl" - "appveyor DownloadFile https://github.com/DigitalGlobe/gbdxtools-windows-binaries/raw/master/Shapely-1.5.17-cp27-cp27m-win32.whl" - "%PYTHON%\\python.exe -m pip install Shapely-1.5.17-cp27-cp27m-win32.whl" - - "appveyor DownloadFile https://github.com/DigitalGlobe/gbdxtools-windows-binaries/raw/master/rasterio-1.0a5-cp27-cp27m-win32.whl" - - "%PYTHON%\\python.exe -m pip install rasterio-1.0a5-cp27-cp27m-win32.whl" - "appveyor DownloadFile https://github.com/DigitalGlobe/gbdxtools-windows-binaries/raw/master/pyproj-1.9.5.1-cp27-cp27m-win32.whl" - "%PYTHON%\\python.exe -m pip install pyproj-1.9.5.1-cp27-cp27m-win32.whl" - "%PYTHON%\\python.exe -m pip install matplotlib pytest" diff --git a/docs/conf.py b/docs/conf.py index d7049070..06970503 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ class Mock(MagicMock): def __getattr__(cls, name): return Mock() -MOCK_MODULES = ['pycurl', 'pyproj', 'rasterio', 'rio_hist'] +MOCK_MODULES = ['pycurl', 'pyproj'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) # 'boto', @@ -49,9 +49,6 @@ def __getattr__(cls, name): # 'dask.array', # 'numpy', # 'numpy.linalg', -# 'rasterio', -# 'rasterio.io', -# 'rasterio.transform', # 'skimage', # 'skimage.transform', # 'skimage.transform._geometric'] diff --git a/environment.yml b/environment.yml index 18b8360d..927a4182 100644 --- a/environment.yml +++ b/environment.yml @@ -215,8 +215,6 @@ dependencies: 'pywavelets=0.5.2=py27_1' - !!python/unicode 'pyyaml=3.12=py27_1' -- !!python/unicode - 'rasterio=0.36.0=py27_1' - !!python/unicode 'readline=6.2=0' - !!python/unicode diff --git a/gbdxtools/images/tms_image.py b/gbdxtools/images/tms_image.py index 35409834..4fa3f565 100644 --- a/gbdxtools/images/tms_image.py +++ b/gbdxtools/images/tms_image.py @@ -16,12 +16,8 @@ from cachetools.func import lru_cache import numpy as np -import rasterio -try: - from rasterio import RasterioIOError -except ImportError: - from rasterio.errors import RasterioIOError -from rasterio.transform import from_bounds as transform_from_bounds +from affine import Affine +from scipy.misc import imread import mercantile @@ -57,11 +53,8 @@ def load_url(url, shape=(8, 256, 256)): try: if(code != 200): raise TypeError("Request for {} returned unexpected error code: {}".format(url, code)) - temp.file.flush() - temp.close() - with rasterio.open(temp.name) as dataset: - arr = dataset.read() - except (TypeError, RasterioIOError) as e: + arr = np.rollaxis(imread(temp), 2, 0) + except Exception as e: print(e) temp.seek(0) print(temp.read()) @@ -69,6 +62,7 @@ def load_url(url, shape=(8, 256, 256)): _curl.close() del _curl_pool[thread_id] finally: + temp.file.flush() temp.close() os.remove(temp.name) return arr @@ -149,7 +143,8 @@ def chunks(self): @property def __geo_transform__(self): - tfm = transform_from_bounds(*tuple(e for e in chain(self.bounds, self.shape[2:0:-1]))) + west, south, east, north = self.bounds + tfm = Affine.translation(west, north) * Affine.scale((east - west) / self.shape[2], (south - north) / self.shape[1]) return AffineTransform(tfm, "EPSG:3857") def _collect_urls(self, bounds): diff --git a/gbdxtools/ipe/interface.py b/gbdxtools/ipe/interface.py index 12c87531..9c795001 100644 --- a/gbdxtools/ipe/interface.py +++ b/gbdxtools/ipe/interface.py @@ -16,12 +16,7 @@ except ImportError: from cachetools.func import lru_cache -import rasterio -try: - from rasterio import RasterioIOError -except ImportError: - from rasterio.errors import RasterioIOError - +from skimage.io import imread import pycurl import numpy as np @@ -70,11 +65,10 @@ def load_url(url, token, shape=(8, 256, 256)): raise TypeError("Request for {} returned unexpected error code: {}".format(url, code)) temp.file.flush() temp.close() - with rasterio.open(temp.name) as dataset: - arr = dataset.read() + arr = np.rollaxis(imread(temp.name), 2, 0) success = True return arr - except (TypeError, RasterioIOError) as e: + except Exception as e: print(e) _curl.close() del _curl_pool[thread_id] diff --git a/gbdxtools/ipe/io.py b/gbdxtools/ipe/io.py index 608125d1..02f54c44 100644 --- a/gbdxtools/ipe/io.py +++ b/gbdxtools/ipe/io.py @@ -1,4 +1,9 @@ -import rasterio +try: + import rasterio + has_rasterio = True +except: + has_rasterio = False + import numpy as np from dask.array import store @@ -12,6 +17,7 @@ def __setitem__(self, location, chunk): self.dst.write(chunk, window=window) def to_geotiff(arr, path='./output.tif', proj=None, bands=None, **kwargs): + assert has_rasterio, "To create geotiff images please install rasterio" if bands is not None: arr = arr[bands,...] diff --git a/install_windows.md b/install_windows.md index 3be78266..496ca3d1 100644 --- a/install_windows.md +++ b/install_windows.md @@ -26,13 +26,6 @@ curl -outf Shapely-1.5.17-cp27-cp27m-win32.whl https://github.com/DigitalGlobe/g pip install .\Shapely-1.5.17-cp27-cp27m-win32.whl ``` -Install rasterio: - -``` -curl -outf rasterio-1.0a5-cp27-cp27m-win32.whl https://github.com/DigitalGlobe/gbdxtools-windows-binaries/blob/master/rasterio-1.0a5-cp27-cp27m-win32.whl?raw=true -pip install .\rasterio-1.0a5-cp27-cp27m-win32.whl -``` - Install pyproj: ``` diff --git a/meta.yaml b/meta.yaml index 7da06e83..374c9f72 100644 --- a/meta.yaml +++ b/meta.yaml @@ -26,7 +26,6 @@ requirements: - dask ==0.15.4 - numpy - pycurl - - rasterio >=0.36.0 - pyproj - requests-futures - configparser @@ -50,7 +49,6 @@ requirements: - dask ==0.15.4 - {{ pin_compatible('numpy', min_pin='1.9') }} - pycurl - - rasterio >=0.36.0 - pyproj - requests-futures - configparser diff --git a/requirements.txt b/requirements.txt index ceb9716a..a5522911 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,6 @@ cloudpickle dask==0.15.4 numpy pycurl -rasterio>=0.36 pyproj requests_futures configparser diff --git a/tests/unit/test_catalog_image.py b/tests/unit/test_catalog_image.py index 99a3b04c..245293ec 100644 --- a/tests/unit/test_catalog_image.py +++ b/tests/unit/test_catalog_image.py @@ -10,7 +10,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio # How to use the mock_gbdx_session and vcr to create unit tests: # 1. Add a new test that is dependent upon actually hitting GBDX APIs. diff --git a/tests/unit/test_dem_image.py b/tests/unit/test_dem_image.py index 9b1f933f..831808b1 100644 --- a/tests/unit/test_dem_image.py +++ b/tests/unit/test_dem_image.py @@ -12,7 +12,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio import dask.array as da try: diff --git a/tests/unit/test_geoeye_image.py b/tests/unit/test_geoeye_image.py index 010a924b..49882610 100644 --- a/tests/unit/test_geoeye_image.py +++ b/tests/unit/test_geoeye_image.py @@ -12,7 +12,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio import dask.array as da def force(r1, r2): diff --git a/tests/unit/test_ikonos_image.py b/tests/unit/test_ikonos_image.py index 2ef1d9f3..08b0107a 100644 --- a/tests/unit/test_ikonos_image.py +++ b/tests/unit/test_ikonos_image.py @@ -12,7 +12,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio import dask.array as da def force(r1, r2): diff --git a/tests/unit/test_ipe_image.py b/tests/unit/test_ipe_image.py index ba80a26b..60ab348f 100644 --- a/tests/unit/test_ipe_image.py +++ b/tests/unit/test_ipe_image.py @@ -152,14 +152,3 @@ def test_ipe_image_ortho(self): aoi.ortho = read_mock ortho = aoi.warp() assert isinstance(ortho, DaskImage) - - #@my_vcr.use_cassette('tests/unit/cassettes/test_ipe_image_geotiff.yaml', filter_headers=['authorization']) - #def test_ipe_image_geotiff(self): - # idahoid = '179269b9-fdb3-49d8-bb62-d15de54ad15d' - # img = self.gbdx.idaho_image(idahoid, bbox=[-110.85299491882326,32.167148499672855,-110.84870338439943,32.170236308395644]) - # tif = img.geotiff(path='/tmp/tmp.tif', dtype='uint16') - # with rasterio.open(tif) as src: - # assert [round(x, 3) for x in list(src.bounds)] == [-110.853, 32.167, -110.849, 32.17] - # assert src.meta['width'] == 239 - # assert src.meta['height'] == 172 - # assert src.meta['dtype'] == 'uint16' diff --git a/tests/unit/test_landsat_image.py b/tests/unit/test_landsat_image.py index f0f629d6..949c06de 100644 --- a/tests/unit/test_landsat_image.py +++ b/tests/unit/test_landsat_image.py @@ -12,7 +12,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio import dask.array as da def force(r1, r2): diff --git a/tests/unit/test_quickbird_image.py b/tests/unit/test_quickbird_image.py index 4e4abc8a..0c7905d0 100644 --- a/tests/unit/test_quickbird_image.py +++ b/tests/unit/test_quickbird_image.py @@ -12,7 +12,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio import dask.array as da def force(r1, r2): diff --git a/tests/unit/test_s3_image.py b/tests/unit/test_s3_image.py index c136f622..705248a6 100644 --- a/tests/unit/test_s3_image.py +++ b/tests/unit/test_s3_image.py @@ -12,7 +12,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio import dask.array as da def force(r1, r2): diff --git a/tests/unit/test_sentinel2_image.py b/tests/unit/test_sentinel2_image.py index a47e7349..7eea7dc7 100644 --- a/tests/unit/test_sentinel2_image.py +++ b/tests/unit/test_sentinel2_image.py @@ -12,7 +12,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio import dask.array as da def force(r1, r2): diff --git a/tests/unit/test_worldview.py b/tests/unit/test_worldview.py index 72b41dfe..39540d68 100644 --- a/tests/unit/test_worldview.py +++ b/tests/unit/test_worldview.py @@ -12,7 +12,6 @@ from os.path import join, isfile, dirname, realpath import tempfile import unittest -import rasterio try: from urlparse import urlparse