Merge pull request #125 from developmentseed/mileswwatkins-miles/feat…

…ure/address_search

Add street address search

.travis.yml

@@ -1,10 +1,24 @@
 sudo: required
 
 services:
-  - docker
+- docker
+
+env:
+  global:
+  - secure: QsF7ignSAbH/WCyO6v9bw1exmCWDQR0DqmHkwJ5swc9N44OOOzbWGsaMSYB5y9h+d70fz4arbxQDhsk2KvX4Zd1/2YIMOrIsbgDYeegpkhVPgyQNPKmVqiX+Tb47t1C/TgkC7A07tiPpuefYcLNMZ8gzz7oKhh1UKapYftqzZ+g=
+  - secure: HxjeKWSROBQYy9NuNkgQeaK1ubTF8vH5FcR8nUTSAYxxw/qOzKpqkiq4BcJSRcIwTbkvaBf4MshLGVOxPjMeyJFe06UD/6LvTUGS3bwdya+m0RFjHe5/3wzS8/MxLbTlvgzmuGLLKOsJjXCi9eQQchKfHv+QuhGxhYVLQpnbU9E=
+  - secure: Zq0Z2UA2A7/ieXX8XoMweClJTp8hiVBxoQ1ylJYNd7qsRSk0QvZhn62db5/x48L9S1kELk0sG64q5Pf96/RPLpdjkBUAdEkS7qF+QOvRvAv2woNEHutjlMUvP6jwYGbug+AORg76btZ57OwMOi3aTkagQMMKnokfo7KGbffy0Jo=
+
+
+before_script:
+- docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" .
 
 script:
-- docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests
+- docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis nosetests
+
+after_success:
+- docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}
+- docker push developmentseed/landsat-util:travis
 
 deploy:
   provider: pypi
@@ -18,8 +32,7 @@ deploy:
 after_deploy:
   if [ "$TRAVIS_BRANCH" == "master" ]; then
     echo "Start Docker Hub Push"
-    VER=$(python -c "import landsat; print landsat.__version__")
+    VER=$(docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis python landsat/landsat.py --version | sed s/[^0-9\.]//g)
     docker build . -t developmentseed/landsat-util:$VER
-    docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}
     docker push developmentseed/landsat-util:$VER
   fi

Dockerfile

@@ -5,4 +5,3 @@ ADD     landsat /usr/local/lib/python2.7/dist-packages/landsat
 ADD     bin/landsat /usr/local/bin/
 ADD     . /landsat
 RUN     cd /landsat && pip install -r requirements/docker.txt
-RUN     pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0

README.rst

@@ -21,7 +21,7 @@ Landsat-util is a command line utility that makes it easy to search, download, a
 Docs
 +++++
 
-For full documentation visit: http://landsat-util.readthedocs.org/
+For full documentation visit: https://pythonhosted.org/landsat-util/
 
 To run the documentation locally::
 
@@ -32,14 +32,21 @@ To run the documentation locally::
 Travis Tests
 ++++++++++++
 
-To speed up testing on travis, we use a docker image. If you add new packages, to ensure that tests will pass on Travis, make sure to rebuild the docker image and push it to DockerHub with `travis` tag.
+To speed up testing on travis, we use a docker image.
+
+To test with docker image locally run:
+
+.. code::
+
+    $ docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests
 
 Recently Added Features
 +++++++++++++++++++++++
 
-- Add longitude latitude search
-- Improve console output
-- Add more color options such as false color, true color, etc.
+- Improved pansharpening
+- Use BQA bands for cloud/snow coverage and use in color correction
+- Add support for different NDVI color maps (three included)
+- Add support for image clipping using the new `--clip` flag
 
 Change Log
 ++++++++++

docs/commands.rst

@@ -20,6 +20,8 @@ Commands
 
                     --lon               Longitude
 
+                    --address           Street address
+
                     -l LIMIT, --limit LIMIT
                                         Search return results limit default is 10
 

docs/todo.rst

@@ -1,9 +1,6 @@
 To Do List
 ++++++++++
 
-- Add Sphinx Documentation
-- Add capacity for NDVI output
 - Add alternative projections (currently only option is default web-mercator; EPSG: 3857)
-- Connect search to Google Address API
 - Include 16-bit image variant in output
 - Add support for color correct looping over multiple compressed inputs (currently just 1)

landsat/landsat.py

@@ -32,7 +32,7 @@
 
     Commands:
         Search:
-            landsat.py search [-p --pathrow] [--lat] [--lon] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h]
+            landsat.py search [-p --pathrow] [--lat] [--lon] [--address] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h]
 
             optional arguments:
                 -p, --pathrow       Paths and Rows in order separated by comma. Use quotes "001,003".
@@ -42,6 +42,8 @@
 
                 --lon               Longitude
 
+                --address           Street address
+
                 -l LIMIT, --limit LIMIT
                                     Search return results limit default is 10
 
@@ -184,6 +186,7 @@ def args_options():
                                'Example: path,row,path,row 001,001,190,204')
     parser_search.add_argument('--lat', type=float, help='The latitude')
     parser_search.add_argument('--lon', type=float, help='The longitude')
+    parser_search.add_argument('--address', type=str, help='The address')
     parser_search.add_argument('--json', action='store_true', help='Returns a bare JSON response')
 
     parser_download = subparsers.add_parser('download',
@@ -308,9 +311,14 @@ def main(args):
             except ValueError:
                 return ["The latitude and longitude values must be valid numbers", 1]
 
+            address = args.address
+            if address and (lat and lon):
+                return ["Cannot specify both address and latitude-longitude"]
+
             result = s.search(paths_rows=args.pathrow,
                               lat=lat,
                               lon=lon,
+                              address=address,
                               limit=args.limit,
                               start_date=args.start,
                               end_date=args.end,

landsat/search.py

@@ -6,7 +6,7 @@
 import requests
 
 import settings
-from utils import three_digit, create_paired_list
+from utils import three_digit, create_paired_list, geocode
 
 
 class Search(object):
@@ -15,7 +15,7 @@ class Search(object):
     def __init__(self):
         self.api_url = settings.API_URL
 
-    def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None,
+    def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None,
                cloud_max=None, limit=1):
         """
         The main method of Search class. It searches Development Seed's Landsat API.
@@ -32,6 +32,10 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=
             The The longitude
         :type lon:
             String, float, integer
+        :param address:
+            The address
+        :type address:
+            String
         :param start_date:
             Date string. format: YYYY-MM-DD
         :type start_date:
@@ -78,7 +82,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=
                 }
         """
 
-        search_string = self.query_builder(paths_rows, lat, lon, start_date, end_date, cloud_min, cloud_max)
+        search_string = self.query_builder(paths_rows, lat, lon, address, start_date, end_date, cloud_min, cloud_max)
 
         # Have to manually build the URI to bypass requests URI encoding
         # The api server doesn't accept encoded URIs
@@ -109,7 +113,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=
 
         return result
 
-    def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None,
+    def query_builder(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None,
                       cloud_min=None, cloud_max=None):
         """ Builds the proper search syntax (query) for Landsat API.
 
@@ -125,6 +129,10 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en
             The The longitude
         :type lon:
             String, float, integer
+        :param address:
+            The address
+        :type address:
+            String
         :param start_date:
             Date string. format: YYYY-MM-DD
         :type start_date:
@@ -171,7 +179,9 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en
         elif cloud_max:
             query.append(self.cloud_cover_prct_range_builder('-1', cloud_max))
 
-        if lat and lon:
+        if address:
+            query.append(self.address_builder(address))
+        elif lat and lon:
             query.append(self.lat_lon_builder(lat, lon))
 
         if query:
@@ -241,6 +251,20 @@ def cloud_cover_prct_range_builder(self, min=0, max=100):
         """
         return 'cloudCoverFull:[%s+TO+%s]' % (min, max)
 
+    def address_builder(self, address):
+        """ Builds lat and lon query from a geocoded address.
+
+        :param address:
+            The address
+        :type address:
+            String
+
+        :returns:
+            String
+        """
+        geocoded = geocode(address)
+        return self.lat_lon_builder(**geocoded)
+
     def lat_lon_builder(self, lat=0, lon=0):
         """ Builds lat and lon query.
 

landsat/utils.py

@@ -7,6 +7,7 @@
 import re
 from cStringIO import StringIO
 from datetime import datetime
+import geocoder
 
 from mixins import VerbosityMixin
 
@@ -280,6 +281,52 @@ def convert_to_integer_list(value):
         return s
 
 
+# Geocoding confidence scores, from https://github.com/DenisCarriere/geocoder/blob/master/docs/features/Confidence%20Score.md
+geocode_confidences = {
+    10: 0.25,
+    9: 0.5,
+    8: 1.,
+    7: 5.,
+    6: 7.5,
+    5: 10.,
+    4: 15.,
+    3: 20.,
+    2: 25.,
+    1: 99999.,
+    # 0: unable to locate at all
+}
+
+
+def geocode(address, required_precision_km=1.):
+    """ Identifies the coordinates of an address
+
+    :param address:
+        the address to be geocoded
+    :type value:
+        String
+    :param required_precision_km:
+        the maximum permissible geographic uncertainty for the geocoding
+    :type required_precision_km:
+        float
+
+    :returns:
+        dict
+
+    :example:
+        >>> geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')
+        {'lat': 38.89767579999999, 'lon': -77.0364827}
+
+    """
+    geocoded = geocoder.google(address)
+    precision_km = geocode_confidences[geocoded.confidence]
+
+    if precision_km <= required_precision_km:
+        (lon, lat) = geocoded.geometry['coordinates']
+        return {'lat': lat, 'lon': lon}
+    else:
+        raise ValueError("Address could not be precisely located")
+
+
 def convert_to_float_list(value):
     """ Converts a comma separate string to a list
 

requirements/docker.txt

@@ -6,3 +6,4 @@ six==1.9.0
 homura>=0.1.2
 boto>=2.38.0
 polyline==1.1
+geocoder>=1.5.1

tests/test_utils.py

@@ -105,6 +105,11 @@ def test_convert_to_integer_list(self):
         r = utils.convert_to_integer_list('1,11,10,QA')
         self.assertEqual([1, 11, 10, 'QA'], r)
 
+    def test_geocode(self):
+        self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364823}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500'))
+        self.assertRaises(ValueError, utils.geocode, 'Pennsylvania Ave NW, Washington, DC')
+        self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.))
+
     def test_convert_to_float_list(self):
         # correct input
         r = utils.convert_to_float_list('-1,2,-3')
@@ -166,5 +171,6 @@ def test_adjust_bounding_box(self):
 
         self.assertEqual(utils.adjust_bounding_box(origin, target), origin)
 
+
 if __name__ == '__main__':
     unittest.main()

travis-dockerfile

@@ -0,0 +1,4 @@
+FROM    developmentseed/landsat-util:dev
+ADD     . /test
+RUN     cd /test && pip install -r requirements/docker.txt
+RUN     pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0