diff --git a/.travis.yml b/.travis.yml index 686db65..0692e6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,24 @@ sudo: required services: - - docker +- docker + +env: + global: + - secure: QsF7ignSAbH/WCyO6v9bw1exmCWDQR0DqmHkwJ5swc9N44OOOzbWGsaMSYB5y9h+d70fz4arbxQDhsk2KvX4Zd1/2YIMOrIsbgDYeegpkhVPgyQNPKmVqiX+Tb47t1C/TgkC7A07tiPpuefYcLNMZ8gzz7oKhh1UKapYftqzZ+g= + - secure: HxjeKWSROBQYy9NuNkgQeaK1ubTF8vH5FcR8nUTSAYxxw/qOzKpqkiq4BcJSRcIwTbkvaBf4MshLGVOxPjMeyJFe06UD/6LvTUGS3bwdya+m0RFjHe5/3wzS8/MxLbTlvgzmuGLLKOsJjXCi9eQQchKfHv+QuhGxhYVLQpnbU9E= + - secure: Zq0Z2UA2A7/ieXX8XoMweClJTp8hiVBxoQ1ylJYNd7qsRSk0QvZhn62db5/x48L9S1kELk0sG64q5Pf96/RPLpdjkBUAdEkS7qF+QOvRvAv2woNEHutjlMUvP6jwYGbug+AORg76btZ57OwMOi3aTkagQMMKnokfo7KGbffy0Jo= + + +before_script: +- docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . script: -- docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests +- docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis nosetests + +after_success: +- docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} +- docker push developmentseed/landsat-util:travis deploy: provider: pypi @@ -18,8 +32,7 @@ deploy: after_deploy: if [ "$TRAVIS_BRANCH" == "master" ]; then echo "Start Docker Hub Push" - VER=$(python -c "import landsat; print landsat.__version__") + VER=$(docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis python landsat/landsat.py --version | sed s/[^0-9\.]//g) docker build . -t developmentseed/landsat-util:$VER - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} docker push developmentseed/landsat-util:$VER fi diff --git a/Dockerfile b/Dockerfile index be6f2ef..8bf1789 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,3 @@ ADD landsat /usr/local/lib/python2.7/dist-packages/landsat ADD bin/landsat /usr/local/bin/ ADD . /landsat RUN cd /landsat && pip install -r requirements/docker.txt -RUN pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0 diff --git a/README.rst b/README.rst index 9dbd1d8..150181c 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ Landsat-util is a command line utility that makes it easy to search, download, a Docs +++++ -For full documentation visit: http://landsat-util.readthedocs.org/ +For full documentation visit: https://pythonhosted.org/landsat-util/ To run the documentation locally:: @@ -32,14 +32,21 @@ To run the documentation locally:: Travis Tests ++++++++++++ -To speed up testing on travis, we use a docker image. If you add new packages, to ensure that tests will pass on Travis, make sure to rebuild the docker image and push it to DockerHub with `travis` tag. +To speed up testing on travis, we use a docker image. + +To test with docker image locally run: + +.. code:: + + $ docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests Recently Added Features +++++++++++++++++++++++ -- Add longitude latitude search -- Improve console output -- Add more color options such as false color, true color, etc. +- Improved pansharpening +- Use BQA bands for cloud/snow coverage and use in color correction +- Add support for different NDVI color maps (three included) +- Add support for image clipping using the new `--clip` flag Change Log ++++++++++ diff --git a/docs/commands.rst b/docs/commands.rst index acdd84d..c2de47c 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -20,6 +20,8 @@ Commands --lon Longitude + --address Street address + -l LIMIT, --limit LIMIT Search return results limit default is 10 diff --git a/docs/todo.rst b/docs/todo.rst index afb754d..704c1ab 100644 --- a/docs/todo.rst +++ b/docs/todo.rst @@ -1,9 +1,6 @@ To Do List ++++++++++ -- Add Sphinx Documentation -- Add capacity for NDVI output - Add alternative projections (currently only option is default web-mercator; EPSG: 3857) -- Connect search to Google Address API - Include 16-bit image variant in output - Add support for color correct looping over multiple compressed inputs (currently just 1) diff --git a/landsat/landsat.py b/landsat/landsat.py index 01472ac..376ac2f 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -32,7 +32,7 @@ Commands: Search: - landsat.py search [-p --pathrow] [--lat] [--lon] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h] + landsat.py search [-p --pathrow] [--lat] [--lon] [--address] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h] optional arguments: -p, --pathrow Paths and Rows in order separated by comma. Use quotes "001,003". @@ -42,6 +42,8 @@ --lon Longitude + --address Street address + -l LIMIT, --limit LIMIT Search return results limit default is 10 @@ -184,6 +186,7 @@ def args_options(): 'Example: path,row,path,row 001,001,190,204') parser_search.add_argument('--lat', type=float, help='The latitude') parser_search.add_argument('--lon', type=float, help='The longitude') + parser_search.add_argument('--address', type=str, help='The address') parser_search.add_argument('--json', action='store_true', help='Returns a bare JSON response') parser_download = subparsers.add_parser('download', @@ -308,9 +311,14 @@ def main(args): except ValueError: return ["The latitude and longitude values must be valid numbers", 1] + address = args.address + if address and (lat and lon): + return ["Cannot specify both address and latitude-longitude"] + result = s.search(paths_rows=args.pathrow, lat=lat, lon=lon, + address=address, limit=args.limit, start_date=args.start, end_date=args.end, diff --git a/landsat/search.py b/landsat/search.py index 65ebc46..5ee0be8 100644 --- a/landsat/search.py +++ b/landsat/search.py @@ -6,7 +6,7 @@ import requests import settings -from utils import three_digit, create_paired_list +from utils import three_digit, create_paired_list, geocode class Search(object): @@ -15,7 +15,7 @@ class Search(object): def __init__(self): self.api_url = settings.API_URL - def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None, + def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None, limit=1): """ The main method of Search class. It searches Development Seed's Landsat API. @@ -32,6 +32,10 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= The The longitude :type lon: String, float, integer + :param address: + The address + :type address: + String :param start_date: Date string. format: YYYY-MM-DD :type start_date: @@ -78,7 +82,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= } """ - search_string = self.query_builder(paths_rows, lat, lon, start_date, end_date, cloud_min, cloud_max) + search_string = self.query_builder(paths_rows, lat, lon, address, start_date, end_date, cloud_min, cloud_max) # Have to manually build the URI to bypass requests URI encoding # The api server doesn't accept encoded URIs @@ -109,7 +113,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= return result - def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, + def query_builder(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None): """ Builds the proper search syntax (query) for Landsat API. @@ -125,6 +129,10 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en The The longitude :type lon: String, float, integer + :param address: + The address + :type address: + String :param start_date: Date string. format: YYYY-MM-DD :type start_date: @@ -171,7 +179,9 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en elif cloud_max: query.append(self.cloud_cover_prct_range_builder('-1', cloud_max)) - if lat and lon: + if address: + query.append(self.address_builder(address)) + elif lat and lon: query.append(self.lat_lon_builder(lat, lon)) if query: @@ -241,6 +251,20 @@ def cloud_cover_prct_range_builder(self, min=0, max=100): """ return 'cloudCoverFull:[%s+TO+%s]' % (min, max) + def address_builder(self, address): + """ Builds lat and lon query from a geocoded address. + + :param address: + The address + :type address: + String + + :returns: + String + """ + geocoded = geocode(address) + return self.lat_lon_builder(**geocoded) + def lat_lon_builder(self, lat=0, lon=0): """ Builds lat and lon query. diff --git a/landsat/utils.py b/landsat/utils.py index 3107ffc..840ea91 100644 --- a/landsat/utils.py +++ b/landsat/utils.py @@ -7,6 +7,7 @@ import re from cStringIO import StringIO from datetime import datetime +import geocoder from mixins import VerbosityMixin @@ -280,6 +281,52 @@ def convert_to_integer_list(value): return s +# Geocoding confidence scores, from https://github.com/DenisCarriere/geocoder/blob/master/docs/features/Confidence%20Score.md +geocode_confidences = { + 10: 0.25, + 9: 0.5, + 8: 1., + 7: 5., + 6: 7.5, + 5: 10., + 4: 15., + 3: 20., + 2: 25., + 1: 99999., + # 0: unable to locate at all +} + + +def geocode(address, required_precision_km=1.): + """ Identifies the coordinates of an address + + :param address: + the address to be geocoded + :type value: + String + :param required_precision_km: + the maximum permissible geographic uncertainty for the geocoding + :type required_precision_km: + float + + :returns: + dict + + :example: + >>> geocode('1600 Pennsylvania Ave NW, Washington, DC 20500') + {'lat': 38.89767579999999, 'lon': -77.0364827} + + """ + geocoded = geocoder.google(address) + precision_km = geocode_confidences[geocoded.confidence] + + if precision_km <= required_precision_km: + (lon, lat) = geocoded.geometry['coordinates'] + return {'lat': lat, 'lon': lon} + else: + raise ValueError("Address could not be precisely located") + + def convert_to_float_list(value): """ Converts a comma separate string to a list diff --git a/requirements/docker.txt b/requirements/docker.txt index 1df380f..e4a8c18 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -6,3 +6,4 @@ six==1.9.0 homura>=0.1.2 boto>=2.38.0 polyline==1.1 +geocoder>=1.5.1 diff --git a/tests/test_utils.py b/tests/test_utils.py index 2b959ab..66c9fdb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -105,6 +105,11 @@ def test_convert_to_integer_list(self): r = utils.convert_to_integer_list('1,11,10,QA') self.assertEqual([1, 11, 10, 'QA'], r) + def test_geocode(self): + self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364823}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')) + self.assertRaises(ValueError, utils.geocode, 'Pennsylvania Ave NW, Washington, DC') + self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.)) + def test_convert_to_float_list(self): # correct input r = utils.convert_to_float_list('-1,2,-3') @@ -166,5 +171,6 @@ def test_adjust_bounding_box(self): self.assertEqual(utils.adjust_bounding_box(origin, target), origin) + if __name__ == '__main__': unittest.main() diff --git a/travis-dockerfile b/travis-dockerfile new file mode 100644 index 0000000..92d8fa1 --- /dev/null +++ b/travis-dockerfile @@ -0,0 +1,4 @@ +FROM developmentseed/landsat-util:dev +ADD . /test +RUN cd /test && pip install -r requirements/docker.txt +RUN pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0