From 97e3ad193bc275dc05c96ad47883396bc9a04f8b Mon Sep 17 00:00:00 2001 From: Miles Watkins Date: Tue, 18 Aug 2015 01:46:56 -0400 Subject: [PATCH 01/12] Add street address search --- landsat/landsat.py | 10 ++++++++- landsat/search.py | 34 ++++++++++++++++++++++++----- landsat/utils.py | 47 +++++++++++++++++++++++++++++++++++++++++ requirements/base.txt | 1 + requirements/travis.txt | 1 + tests/test_utils.py | 5 +++++ 6 files changed, 92 insertions(+), 6 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 3a6a152..1fb5c50 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -29,7 +29,7 @@ Commands: Search: - landsat.py search [-p --pathrow] [--lat] [--lon] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h] + landsat.py search [-p --pathrow] [--lat] [--lon] [--address] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h] optional arguments: -p, --pathrow Paths and Rows in order separated by comma. Use quotes "001,003". @@ -39,6 +39,8 @@ --lon Longitude + --address Street address + -l LIMIT, --limit LIMIT Search return results limit default is 10 @@ -165,6 +167,7 @@ def args_options(): 'Example: path,row,path,row 001,001,190,204') parser_search.add_argument('--lat', type=float, help='The latitude') parser_search.add_argument('--lon', type=float, help='The longitude') + parser_search.add_argument('--address', type=str, help='The address') parser_download = subparsers.add_parser('download', help='Download images from Google Storage') @@ -268,9 +271,14 @@ def main(args): except ValueError: return ["The latitude and longitude values must be valid numbers", 1] + address = args.address + if address and (lat and lon): + return ["Cannot specify both address and latitude-longitude"] + result = s.search(paths_rows=args.pathrow, lat=lat, lon=lon, + address=address, limit=args.limit, start_date=args.start, end_date=args.end, diff --git a/landsat/search.py b/landsat/search.py index 65ebc46..5ee0be8 100644 --- a/landsat/search.py +++ b/landsat/search.py @@ -6,7 +6,7 @@ import requests import settings -from utils import three_digit, create_paired_list +from utils import three_digit, create_paired_list, geocode class Search(object): @@ -15,7 +15,7 @@ class Search(object): def __init__(self): self.api_url = settings.API_URL - def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None, + def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None, limit=1): """ The main method of Search class. It searches Development Seed's Landsat API. @@ -32,6 +32,10 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= The The longitude :type lon: String, float, integer + :param address: + The address + :type address: + String :param start_date: Date string. format: YYYY-MM-DD :type start_date: @@ -78,7 +82,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= } """ - search_string = self.query_builder(paths_rows, lat, lon, start_date, end_date, cloud_min, cloud_max) + search_string = self.query_builder(paths_rows, lat, lon, address, start_date, end_date, cloud_min, cloud_max) # Have to manually build the URI to bypass requests URI encoding # The api server doesn't accept encoded URIs @@ -109,7 +113,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= return result - def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, + def query_builder(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None): """ Builds the proper search syntax (query) for Landsat API. @@ -125,6 +129,10 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en The The longitude :type lon: String, float, integer + :param address: + The address + :type address: + String :param start_date: Date string. format: YYYY-MM-DD :type start_date: @@ -171,7 +179,9 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en elif cloud_max: query.append(self.cloud_cover_prct_range_builder('-1', cloud_max)) - if lat and lon: + if address: + query.append(self.address_builder(address)) + elif lat and lon: query.append(self.lat_lon_builder(lat, lon)) if query: @@ -241,6 +251,20 @@ def cloud_cover_prct_range_builder(self, min=0, max=100): """ return 'cloudCoverFull:[%s+TO+%s]' % (min, max) + def address_builder(self, address): + """ Builds lat and lon query from a geocoded address. + + :param address: + The address + :type address: + String + + :returns: + String + """ + geocoded = geocode(address) + return self.lat_lon_builder(**geocoded) + def lat_lon_builder(self, lat=0, lon=0): """ Builds lat and lon query. diff --git a/landsat/utils.py b/landsat/utils.py index 3002cb8..912475d 100644 --- a/landsat/utils.py +++ b/landsat/utils.py @@ -7,6 +7,7 @@ import re from cStringIO import StringIO from datetime import datetime +import geocoder from mixins import VerbosityMixin @@ -278,3 +279,49 @@ def convert_to_integer_list(value): except ValueError: pass return s + + +# Geocoding confidence scores, from https://github.com/DenisCarriere/geocoder/blob/master/docs/features/Confidence%20Score.md +geocode_confidences = { + 10: 0.25, + 9: 0.5, + 8: 1., + 7: 5., + 6: 7.5, + 5: 10., + 4: 15., + 3: 20., + 2: 25., + 1: 99999., + # 0: unable to locate at all +} + + +def geocode(address, required_precision_km=1.): + """ Identifies the coordinates of an address + + :param address: + the address to be geocoded + :type value: + String + :param required_precision_km: + the maximum permissible geographic uncertainty for the geocoding + :type required_precision_km: + float + + :returns: + dict + + :example: + >>> geocode('1600 Pennsylvania Ave NW, Washington, DC 20500') + {'lat': 38.89767579999999, 'lon': -77.0364827} + + """ + geocoded = geocoder.google(address) + precision_km = geocode_confidences[geocoded.confidence] + + if precision_km <= required_precision_km: + (lon, lat) = geocoded.geometry['coordinates'] + return {'lat': lat, 'lon': lon} + else: + raise ValueError("Address could not be precisely located") diff --git a/requirements/base.txt b/requirements/base.txt index a17e798..789f50c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,3 +8,4 @@ scikit-image>=0.11.3 homura>=0.1.1 scipy>=0.15.1 boto>=2.38.0 +geocoder>=1.5.1 diff --git a/requirements/travis.txt b/requirements/travis.txt index 2509138..b03a045 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -4,3 +4,4 @@ termcolor>=1.1.0 python-dateutil>=2.4.2 mock>=1.0.1 boto>=2.38.0 +geocoder>=1.5.1 diff --git a/tests/test_utils.py b/tests/test_utils.py index b42c28b..81af806 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -105,6 +105,11 @@ def test_convert_to_integer_list(self): r = utils.convert_to_integer_list('1,11,10,QA') self.assertEqual([1, 11, 10, 'QA'], r) + def test_geocode(self): + self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364827}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')) + self.assertRaises(ValueError, utils.geocode, 'Pennsylvania Ave NW, Washington, DC') + self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.)) + if __name__ == '__main__': unittest.main() From c8397889a33946ee3b8a3db7c23c7d03f483ab12 Mon Sep 17 00:00:00 2001 From: Miles Watkins Date: Tue, 27 Oct 2015 13:41:33 -0400 Subject: [PATCH 02/12] Tweak precision on White House address test --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index e148f03..66c9fdb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -106,7 +106,7 @@ def test_convert_to_integer_list(self): self.assertEqual([1, 11, 10, 'QA'], r) def test_geocode(self): - self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364827}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')) + self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364823}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')) self.assertRaises(ValueError, utils.geocode, 'Pennsylvania Ave NW, Washington, DC') self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.)) From 6a120142cbba99cf934a57d1f09dd9b484f1daa3 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:43:54 -0400 Subject: [PATCH 03/12] add docker test command --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 9dbd1d8..145df35 100644 --- a/README.rst +++ b/README.rst @@ -34,6 +34,12 @@ Travis Tests To speed up testing on travis, we use a docker image. If you add new packages, to ensure that tests will pass on Travis, make sure to rebuild the docker image and push it to DockerHub with `travis` tag. +To test with docker image locally run: + +.. code:: + + $ docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests + Recently Added Features +++++++++++++++++++++++ From 6cf52c17d156ee10a0a156e7cca659b50fe9f802 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:49:14 -0400 Subject: [PATCH 04/12] update todo list --- docs/todo.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/todo.rst b/docs/todo.rst index afb754d..704c1ab 100644 --- a/docs/todo.rst +++ b/docs/todo.rst @@ -1,9 +1,6 @@ To Do List ++++++++++ -- Add Sphinx Documentation -- Add capacity for NDVI output - Add alternative projections (currently only option is default web-mercator; EPSG: 3857) -- Connect search to Google Address API - Include 16-bit image variant in output - Add support for color correct looping over multiple compressed inputs (currently just 1) From ae22ea3d3226a8bb318bd624a79366f5bd5c7a1f Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:49:21 -0400 Subject: [PATCH 05/12] update recently added features --- README.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 145df35..f54cac1 100644 --- a/README.rst +++ b/README.rst @@ -43,9 +43,10 @@ To test with docker image locally run: Recently Added Features +++++++++++++++++++++++ -- Add longitude latitude search -- Improve console output -- Add more color options such as false color, true color, etc. +- Improved pansharpening +- Use BQA bands for cloud/snow coverage and use in color correction +- Add support for different NDVI color maps (three included) +- Add support for image clipping using the new `--clip` flag Change Log ++++++++++ From 499ab1c24ccc9b645622c68f71fbb64a265594f7 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:53:58 -0400 Subject: [PATCH 06/12] update link to docs [ci skip] --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f54cac1..0f15ce9 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ Landsat-util is a command line utility that makes it easy to search, download, a Docs +++++ -For full documentation visit: http://landsat-util.readthedocs.org/ +For full documentation visit: https://pythonhosted.org/landsat-util/ To run the documentation locally:: From fad40e25132ca571007bdae942df1ec489b427a5 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:55:29 -0400 Subject: [PATCH 07/12] update commands [ci skip] --- docs/commands.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/commands.rst b/docs/commands.rst index acdd84d..c2de47c 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -20,6 +20,8 @@ Commands --lon Longitude + --address Street address + -l LIMIT, --limit LIMIT Search return results limit default is 10 From f4fa584d3fd052ba705aa0d7df6e5ee1118cc6ee Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:00:22 -0400 Subject: [PATCH 08/12] build docker image first --- .travis.yml | 3 +++ README.rst | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 686db65..ad7d4a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ sudo: required services: - docker +before_script: +- docker build -t 'developmentseed/landsat-util:travis' . + script: - docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests diff --git a/README.rst b/README.rst index 0f15ce9..150181c 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ To run the documentation locally:: Travis Tests ++++++++++++ -To speed up testing on travis, we use a docker image. If you add new packages, to ensure that tests will pass on Travis, make sure to rebuild the docker image and push it to DockerHub with `travis` tag. +To speed up testing on travis, we use a docker image. To test with docker image locally run: From 579a339bbef2bfbefe9f3577a10b2f00f107850b Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:25:02 -0400 Subject: [PATCH 09/12] use special dockerfile for travis --- .travis.yml | 3 ++- Dockerfile | 1 - travis-dockerfile | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 travis-dockerfile diff --git a/.travis.yml b/.travis.yml index ad7d4a9..8e9fd05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ services: - docker before_script: -- docker build -t 'developmentseed/landsat-util:travis' . +- docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . script: - docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests @@ -26,3 +26,4 @@ after_deploy: docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} docker push developmentseed/landsat-util:$VER fi + docker push developmentseed/landsat-util:travis diff --git a/Dockerfile b/Dockerfile index be6f2ef..8bf1789 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,3 @@ ADD landsat /usr/local/lib/python2.7/dist-packages/landsat ADD bin/landsat /usr/local/bin/ ADD . /landsat RUN cd /landsat && pip install -r requirements/docker.txt -RUN pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0 diff --git a/travis-dockerfile b/travis-dockerfile new file mode 100644 index 0000000..92d8fa1 --- /dev/null +++ b/travis-dockerfile @@ -0,0 +1,4 @@ +FROM developmentseed/landsat-util:dev +ADD . /test +RUN cd /test && pip install -r requirements/docker.txt +RUN pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0 From 0a936a63249acc1442fa78bd7c1c214d9ea25392 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:25:46 -0400 Subject: [PATCH 10/12] always login to docker --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8e9fd05..e10d4b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,11 @@ deploy: branch: master after_deploy: + docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} if [ "$TRAVIS_BRANCH" == "master" ]; then echo "Start Docker Hub Push" VER=$(python -c "import landsat; print landsat.__version__") docker build . -t developmentseed/landsat-util:$VER - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} docker push developmentseed/landsat-util:$VER fi docker push developmentseed/landsat-util:travis From 2bfb4aac5f34ed2541eb5cdeeb678947f60f7a6d Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:31:51 -0400 Subject: [PATCH 11/12] push to docker after success --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e10d4b3..9f3fa3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,10 @@ before_script: script: - docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests +after_success: +- docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} +- docker push developmentseed/landsat-util:travis + deploy: provider: pypi user: devseed @@ -19,11 +23,11 @@ deploy: branch: master after_deploy: - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} if [ "$TRAVIS_BRANCH" == "master" ]; then echo "Start Docker Hub Push" VER=$(python -c "import landsat; print landsat.__version__") + docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} docker build . -t developmentseed/landsat-util:$VER docker push developmentseed/landsat-util:$VER fi - docker push developmentseed/landsat-util:travis + From 19369f370b147cfd5fe0501c874e6b414e57bc94 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:50:33 -0400 Subject: [PATCH 12/12] add docker hub login info --- .travis.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f3fa3b..0692e6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,20 @@ sudo: required services: - - docker +- docker + +env: + global: + - secure: QsF7ignSAbH/WCyO6v9bw1exmCWDQR0DqmHkwJ5swc9N44OOOzbWGsaMSYB5y9h+d70fz4arbxQDhsk2KvX4Zd1/2YIMOrIsbgDYeegpkhVPgyQNPKmVqiX+Tb47t1C/TgkC7A07tiPpuefYcLNMZ8gzz7oKhh1UKapYftqzZ+g= + - secure: HxjeKWSROBQYy9NuNkgQeaK1ubTF8vH5FcR8nUTSAYxxw/qOzKpqkiq4BcJSRcIwTbkvaBf4MshLGVOxPjMeyJFe06UD/6LvTUGS3bwdya+m0RFjHe5/3wzS8/MxLbTlvgzmuGLLKOsJjXCi9eQQchKfHv+QuhGxhYVLQpnbU9E= + - secure: Zq0Z2UA2A7/ieXX8XoMweClJTp8hiVBxoQ1ylJYNd7qsRSk0QvZhn62db5/x48L9S1kELk0sG64q5Pf96/RPLpdjkBUAdEkS7qF+QOvRvAv2woNEHutjlMUvP6jwYGbug+AORg76btZ57OwMOi3aTkagQMMKnokfo7KGbffy0Jo= + before_script: - docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . script: -- docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests +- docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis nosetests after_success: - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} @@ -25,9 +32,7 @@ deploy: after_deploy: if [ "$TRAVIS_BRANCH" == "master" ]; then echo "Start Docker Hub Push" - VER=$(python -c "import landsat; print landsat.__version__") - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} + VER=$(docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis python landsat/landsat.py --version | sed s/[^0-9\.]//g) docker build . -t developmentseed/landsat-util:$VER docker push developmentseed/landsat-util:$VER fi -