Merge pull request #152 from developmentseed/usgs-fallback

Usgs fallback

.travis.yml

@@ -32,7 +32,6 @@ deploy:
     repo: developmentseed/landsat-util
     branch:
       - master
-      - develop
 
 after_deploy:
   if [ "$TRAVIS_BRANCH" == "master" ]; then

CHANGES.txt

@@ -1,6 +1,10 @@
 Changes
 =======
 
+0.12.0 (2016-02-18)
+------------------
+- Add USGS download fallback closes #89
+
 0.11.0 (2016-01-12)
 ------------------
 - a hotfix for search command not showing outputs #137

Dockerfile

@@ -1,6 +1,6 @@
 FROM    ubuntu:14.04
 RUN     apt-get -y update
-RUN     apt-get install --yes python-pip python-skimage python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev libglib2.0-dev zlib1g-dev python-pycurl
+RUN     apt-get install --yes git-core python-pip python-skimage python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev libglib2.0-dev zlib1g-dev python-pycurl
 ADD     landsat /usr/local/lib/python2.7/dist-packages/landsat
 ADD     bin/landsat /usr/local/bin/
 ADD     . /landsat

landsat/__init__.py

@@ -1 +1 @@
-__version__ = '0.11.0'
+__version__ = '0.12.0'

landsat/downloader.py

@@ -1,9 +1,11 @@
 # Landsat Util
 # License: CC0 1.0 Universal
+from xml.etree import ElementTree
 from os.path import join, exists, getsize
 
-from homura import download as fetch
 import requests
+from usgs import api, USGSError
+from homura import download as fetch
 
 from utils import check_create_folder, url_builder
 from mixins import VerbosityMixin
@@ -20,13 +22,20 @@ class IncorrectSceneId(Exception):
     pass
 
 
+class USGSInventoryAccessMissing(Exception):
+    """ Exception for when User does not have Inventory Service access """
+    pass
+
+
 class Downloader(VerbosityMixin):
     """ The downloader class """
 
-    def __init__(self, verbose=False, download_dir=None):
+    def __init__(self, verbose=False, download_dir=None, usgs_user=None, usgs_pass=None):
         self.download_dir = download_dir if download_dir else settings.DOWNLOAD_DIR
         self.google = settings.GOOGLE_STORAGE
         self.s3 = settings.S3_LANDSAT
+        self.usgs_user = usgs_user
+        self.usgs_pass = usgs_pass
 
         # Make sure download directory exist
         check_create_folder(self.download_dir)
@@ -110,7 +119,22 @@ def google_storage(self, scene, path):
             return self.fetch(url, path, filename)
 
         else:
-            raise RemoteFileDoesntExist('%s is not available on Google Storage' % filename)
+            # download from usgs if login information is provided
+            if self.usgs_user and self.usgs_pass:
+                try:
+                    api_key = api.login(self.usgs_user, self.usgs_pass)
+                except USGSError as e:
+                    error_tree = ElementTree.fromstring(str(e.message))
+                    error_text = error_tree.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultstring", api.NAMESPACES).text
+                    raise USGSInventoryAccessMissing(error_text)
+
+                download_url = api.download('LANDSAT_8', 'EE', [scene], api_key=api_key)
+                if download_url:
+                    return self.fetch(download_url[0], path, filename)
+
+                raise RemoteFileDoesntExist('%s is not available on AWS S3, Google or USGS Earth Explorer' % filename)
+
+            raise RemoteFileDoesntExist('%s is not available on AWS S3 or Google Storage' % filename)
 
     def amazon_s3(self, scene, band, path):
         """

landsat/landsat.py

@@ -15,7 +15,7 @@
 import pycurl
 from boto.exception import NoAuthHandlerFound
 
-from downloader import Downloader, IncorrectSceneId
+from downloader import Downloader, IncorrectSceneId, RemoteFileDoesntExist, USGSInventoryAccessMissing
 from search import Search
 from uploader import Uploader
 from utils import reformat_date, convert_to_integer_list, timer, exit, get_file, convert_to_float_list
@@ -109,6 +109,12 @@
 
                 --force-unzip       Force unzip tar file
 
+                --username          USGS Eros account Username (only works if the account has special
+                                    inventory access). Username and password as a fallback if the image
+                                    is not found on AWS S3 or Google Storage
+
+                --password          USGS Eros account Password
+
         Process:
             landsat.py process path [-h] [-b --bands] [-p --pansharpen]
 
@@ -220,6 +226,10 @@ def args_options():
                                  '50.2682767372753')
     parser_download.add_argument('-u', '--upload', action='store_true',
                                  help='Upload to S3 after the image processing completed')
+    parser_download.add_argument('--username', help='USGS Eros account Username (only works if the account has' +
+                                 ' special inventory access). Username and password as a fallback if the image' +
+                                 'is not found on AWS S3 or Google Storage')
+    parser_download.add_argument('--password', help='USGS Eros username, used as a fallback')
     parser_download.add_argument('--key', help='Amazon S3 Access Key (You can also be set AWS_ACCESS_KEY_ID as '
                                  'Environment Variables)')
     parser_download.add_argument('--secret', help='Amazon S3 Secret Key (You can also be set AWS_SECRET_ACCESS_KEY '
@@ -368,7 +378,7 @@ def main(args):
                 return json.dumps(result)
 
         elif args.subs == 'download':
-            d = Downloader(download_dir=args.dest)
+            d = Downloader(download_dir=args.dest, usgs_user=args.username, usgs_pass=args.password)
             try:
                 bands = convert_to_integer_list(args.bands)
 
@@ -415,6 +425,8 @@ def main(args):
                     return ['Download Completed', 0]
             except IncorrectSceneId:
                 return ['The SceneID provided was incorrect', 1]
+            except (RemoteFileDoesntExist, USGSInventoryAccessMissing) as e:
+                return [e.message, 1]
 
 
 def process_image(path, bands=None, verbose=False, pansharpen=False, ndvi=False, force_unzip=None,

requirements/docker.txt

@@ -2,9 +2,10 @@ requests==2.7.0
 python-dateutil>=2.4.2
 termcolor>=1.1.0
 rasterio>=0.27.0
-six==1.9.0
+six==1.8.0
 homura>=0.1.2
 boto>=2.38.0
 polyline==1.1
 geocoder>=1.5.1
 jsonschema==2.5.1
+git+git://github.com/developmentseed/usgs@develop

setup.py

@@ -35,19 +35,23 @@ def readme():
     license='CCO',
     platforms='Posix; MacOS X; Windows',
     install_requires=[
+        'usgs==0.1.9',
         'requests==2.7.0',
         'python-dateutil>=2.4.2',
         'numpy>=1.9.3',
         'termcolor>=1.1.0',
         'rasterio>=0.26.0',
-        'six==1.9.0',
+        'six==1.8.0',
         'scipy>=0.16.0',
         'scikit-image>=0.11.3',
         'homura>=0.1.2',
         'boto>=2.38.0',
         'polyline==1.1',
         'geocoder>=1.5.1'
     ],
+    dependency_links=[
+        "git+https://github.com/developmentseed/usgs@develop#egg=usgs-0.1.9"
+    ],
     test_suite='nose.collector',
     tests_require=test_requirements
 )

tests/test_landsat.py

@@ -97,7 +97,7 @@ def test_download_correct(self, mock_downloader):
 
         args = ['download', 'LC80010092015051LGN00', '-b', '11,', '-d', self.mock_path]
         output = landsat.main(self.parser.parse_args(args))
-        mock_downloader.assert_called_with(download_dir=self.mock_path)
+        mock_downloader.assert_called_with(download_dir=self.mock_path, usgs_pass=None, usgs_user=None)
         mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], [11])
         self.assertEquals(output, ['Download Completed', 0])
 
@@ -108,7 +108,7 @@ def test_download_correct_zip(self, mock_downloader):
 
         args = ['download', 'LC80010092015051LGN00', '-d', self.mock_path]
         output = landsat.main(self.parser.parse_args(args))
-        mock_downloader.assert_called_with(download_dir=self.mock_path)
+        mock_downloader.assert_called_with(download_dir=self.mock_path, usgs_pass=None, usgs_user=None)
         mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], [])
         self.assertEquals(output, ['Download Completed', 0])
 

travis-dockerfile

@@ -1,4 +1,6 @@
 FROM    developmentseed/landsat-util:dev
 ADD     . /test
+RUN     apt-get -y update
+RUN     apt-get install --yes git-core
 RUN     cd /test && pip install -r requirements/docker.txt
 RUN     pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0