diff --git a/.travis.yml b/.travis.yml index 7fc2963..ff65cec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,44 +1,27 @@ -language: python +sudo: required -sudo: false - -cache: - directories: - - ~/.cache/pip +services: +- docker env: global: - - PIP_WHEEL_DIR=$HOME/.cache/pip/wheels - - PIP_FIND_LINKS=file://$HOME/.cache/pip/wheels - -virtualenv: - system_site_packages: true - -addons: - apt: - packages: - - libgdal1h - - gdal-bin - - libgdal-dev - - libatlas-dev - - libatlas-base-dev - - gfortran - - python-numpy - - python-scipy - -python: -- '2.7' - -before_install: - - pip install -U pip - - pip install wheel + - secure: QsF7ignSAbH/WCyO6v9bw1exmCWDQR0DqmHkwJ5swc9N44OOOzbWGsaMSYB5y9h+d70fz4arbxQDhsk2KvX4Zd1/2YIMOrIsbgDYeegpkhVPgyQNPKmVqiX+Tb47t1C/TgkC7A07tiPpuefYcLNMZ8gzz7oKhh1UKapYftqzZ+g= + - secure: HxjeKWSROBQYy9NuNkgQeaK1ubTF8vH5FcR8nUTSAYxxw/qOzKpqkiq4BcJSRcIwTbkvaBf4MshLGVOxPjMeyJFe06UD/6LvTUGS3bwdya+m0RFjHe5/3wzS8/MxLbTlvgzmuGLLKOsJjXCi9eQQchKfHv+QuhGxhYVLQpnbU9E= + - secure: Zq0Z2UA2A7/ieXX8XoMweClJTp8hiVBxoQ1ylJYNd7qsRSk0QvZhn62db5/x48L9S1kELk0sG64q5Pf96/RPLpdjkBUAdEkS7qF+QOvRvAv2woNEHutjlMUvP6jwYGbug+AORg76btZ57OwMOi3aTkagQMMKnokfo7KGbffy0Jo= + install: - - "pip wheel -r requirements/dev.txt" - - "pip install -r requirements/dev.txt" +- pip install --user twine + +before_script: +- docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . script: -- nosetests +- docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis nosetests + +after_success: +- docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} +- docker push developmentseed/landsat-util:travis deploy: provider: pypi @@ -47,13 +30,14 @@ deploy: secure: WtawFW/999XYszmZfj1Qk82l00OSyP2JBVOOGCERrW1gVO7MYtYsgP31HKRSzNTCTHJNVDpdK4WZWY6zPQqC3l2UfWYYsvRn0hCoI8AJxE5VCUEg6Ccpe6fMJuhp1pq6Zy7yrfBSZcOB9aqSHLBJsunD2o3mNlTC8WV8vNK74ck= on: repo: developmentseed/landsat-util - branch: master + branch: + - master + - develop after_deploy: if [ "$TRAVIS_BRANCH" == "master" ]; then echo "Start Docker Hub Push" - VER=$(python -c "import landsat; print landsat.__version__") + VER=$(docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis python landsat/landsat.py --version | sed s/[^0-9\.]//g) docker build . -t developmentseed/landsat-util:$VER - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} docker push developmentseed/landsat-util:$VER fi diff --git a/AUTHORS.txt b/AUTHORS.txt index 9f2131d..20bd3c7 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -5,7 +5,5 @@ Scisco https://github.com/scisco Marc Farra https://github.com/kamicut Drew Bollinger https://github.com/drewbo Alex Kappel https://twitter.com/alex_kappel -Eric Miller https://github.com/millerEric -Yuriy Czoli https://github.com/YKCzoli See also https://github.com/developmentseed/landsat-util/graphs/contributors. diff --git a/CHANGES.txt b/CHANGES.txt index e5aadcc..3243704 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,19 @@ Changes ======= +0.10.0 (2016-01-05) +------------------ +- add support for bare json output +- faster travis tests +- add street search +- apply ndvigrey to process and download +- fixes #127 +- update commands help file +- apply pansharpen and ndvi only if -p is used +- download zip if bands are not specified +- better handle url joins to a fix error on Windows +- other small bug fixes + 0.9.1 (2015-10-26) ------------------ - Add missing package (polyline) to setup.py diff --git a/Dockerfile b/Dockerfile index 77d498c..8bf1789 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,3 @@ ADD landsat /usr/local/lib/python2.7/dist-packages/landsat ADD bin/landsat /usr/local/bin/ ADD . /landsat RUN cd /landsat && pip install -r requirements/docker.txt - diff --git a/MANIFEST.in b/MANIFEST.in index 5f54a81..23d7d34 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,4 +7,4 @@ recursive-exclude * __pycache__ recursive-exclude * *.py[co] recursive-include docs *.rst conf.py Makefile make.bat -recursive-include landsat/tests/samples *.tar.bz2 +recursive-include landsat/maps *.txt diff --git a/README.rst b/README.rst index 123a4d9..150181c 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ Landsat-util is a command line utility that makes it easy to search, download, a Docs +++++ -For full documentation visit: http://landsat-util.readthedocs.org/ +For full documentation visit: https://pythonhosted.org/landsat-util/ To run the documentation locally:: @@ -29,12 +29,24 @@ To run the documentation locally:: $ cd docs $ make html +Travis Tests +++++++++++++ + +To speed up testing on travis, we use a docker image. + +To test with docker image locally run: + +.. code:: + + $ docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests + Recently Added Features +++++++++++++++++++++++ -- Add longitude latitude search -- Improve console output -- Add more color options such as false color, true color, etc. +- Improved pansharpening +- Use BQA bands for cloud/snow coverage and use in color correction +- Add support for different NDVI color maps (three included) +- Add support for image clipping using the new `--clip` flag Change Log ++++++++++ diff --git a/docs/commands.rst b/docs/commands.rst index bd97fdd..c2daeb1 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -20,6 +20,8 @@ Commands --lon Longitude + --address Street address + -l LIMIT, --limit LIMIT Search return results limit default is 10 @@ -35,6 +37,8 @@ Commands -c CLOUD, --cloud CLOUD Maximum cloud percentage. Default: 20 perct + --json Returns a bare JSON response + -h, --help Show this help message and exit Download: @@ -60,6 +64,8 @@ Commands --ndvi Calculates NDVI and produce a RGB GTiff with seperate colorbar. + --ndvigrey Calculates NDVI and produce a greyscale GTiff. + --clip Clip the image with the bounding box provided. Values must be in WGS84 datum, and with longitude and latitude units of decimal degrees separated by comma. Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625,50.2682767372753 diff --git a/docs/overview.rst b/docs/overview.rst index f952151..43c7c4c 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -36,6 +36,9 @@ Search by latitude and longitude:: $: landsat search --lat 38.9004204 --lon -77.0237117 +Search by latitude and longitude with pure json output:: + + $: landsat search --lat 38.9004204 --lon -77.0237117 --json Download ++++++++ @@ -50,7 +53,7 @@ Download images by their custom sceneID, which you get from landsat search:: $: landsat download LC80090452014008LGN00 -Download only band 4, 3 and 2 for a particular sceneID:: +By default landsat-util downloads the full zip file from Google Storage unless you specify the bands or run an image processing right after download. For example to download only band 4, 3 and 2 for a particular sceneID run:: $: landsat download LC80090452014008LGN00 --bands 432 diff --git a/docs/todo.rst b/docs/todo.rst index afb754d..704c1ab 100644 --- a/docs/todo.rst +++ b/docs/todo.rst @@ -1,9 +1,6 @@ To Do List ++++++++++ -- Add Sphinx Documentation -- Add capacity for NDVI output - Add alternative projections (currently only option is default web-mercator; EPSG: 3857) -- Connect search to Google Address API - Include 16-bit image variant in output - Add support for color correct looping over multiple compressed inputs (currently just 1) diff --git a/landsat/__init__.py b/landsat/__init__.py index 8969d49..9d1bb72 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.9.1' +__version__ = '0.10.0' diff --git a/landsat/downloader.py b/landsat/downloader.py index e2627c5..7641298 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -5,7 +5,7 @@ from homura import download as fetch import requests -from utils import check_create_folder +from utils import check_create_folder, url_builder from mixins import VerbosityMixin import settings @@ -192,7 +192,7 @@ def google_storage_url(self, sat): (String) The URL to a google storage file """ filename = sat['scene'] + '.tar.bz' - return join(self.google, sat['sat'], sat['path'], sat['row'], filename) + return url_builder([self.google, sat['sat'], sat['path'], sat['row'], filename]) def amazon_s3_url(self, sat, filename): """ @@ -210,7 +210,7 @@ def amazon_s3_url(self, sat, filename): :returns: (String) The URL to a S3 file """ - return join(self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename) + return url_builder([self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename]) def remote_file_exists(self, url): """ Checks whether the remote file exists. @@ -284,4 +284,4 @@ def scene_interpreter(self, scene): d = Downloader() - # d.download(['LC81990242015046LGN00', 'LC80030172015001LGN00']) \ No newline at end of file + # d.download(['LC81990242015046LGN00', 'LC80030172015001LGN00']) diff --git a/landsat/landsat.py b/landsat/landsat.py index c033423..fb87f1e 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -3,6 +3,7 @@ # Landsat Util # License: CC0 1.0 Universal +import sys import argparse import textwrap import json @@ -31,7 +32,8 @@ Commands: Search: - landsat.py search [-p --pathrow] [--lat] [--lon] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h] + landsat.py search [-p --pathrow] [--lat] [--lon] [--address] [-l LIMIT] [-s START] [-e END] [-c CLOUD] + [-h] optional arguments: -p, --pathrow Paths and Rows in order separated by comma. Use quotes "001,003". @@ -41,6 +43,8 @@ --lon Longitude + --address Street address + -l LIMIT, --limit LIMIT Search return results limit default is 10 @@ -56,6 +60,8 @@ -c CLOUD, --cloud CLOUD Maximum cloud percentage. Default: 20 perct + --json Returns a bare JSON response + -h, --help Show this help message and exit Download: @@ -81,9 +87,12 @@ --ndvi Calculates NDVI and produce a RGB GTiff with seperate colorbar. + --ndvigrey Calculates NDVI and produce a greyscale GTiff. + --clip Clip the image with the bounding box provided. Values must be in WGS84 datum, and with longitude and latitude units of decimal degrees separated by comma. - Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625,50.2682767372753 + Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625, + 50.2682767372753 -u --upload Upload to S3 after the image processing completed @@ -119,7 +128,8 @@ --clip Clip the image with the bounding box provided. Values must be in WGS84 datum, and with longitude and latitude units of decimal degrees separated by comma. - Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625,50.2682767372753 + Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625, + 50.2682767372753 -v, --verbose Show verbose output @@ -181,6 +191,8 @@ def args_options(): 'Example: path,row,path,row 001,001,190,204') parser_search.add_argument('--lat', type=float, help='The latitude') parser_search.add_argument('--lon', type=float, help='The longitude') + parser_search.add_argument('--address', type=str, help='The address') + parser_search.add_argument('--json', action='store_true', help='Returns a bare JSON response') parser_download = subparsers.add_parser('download', help='Download images from Google Storage') @@ -190,7 +202,7 @@ def args_options(): help="Provide Full sceneID, e.g. LC81660392014196LGN00") parser_download.add_argument('-b', '--bands', help='If you specify bands, landsat-util will try to download ' - 'the band from S3. If the band does not exist, an error is returned', default='432') + 'the band from S3. If the band does not exist, an error is returned', default='') parser_download.add_argument('-d', '--dest', help='Destination path') parser_download.add_argument('-p', '--process', help='Process the image after download', action='store_true') parser_download.add_argument('--pansharpen', action='store_true', @@ -198,6 +210,7 @@ def args_options(): 'image. Pansharpening requires larger memory') parser_download.add_argument('--ndvi', action='store_true', help='Whether to run the NDVI process. If used, bands parameter is disregarded') + parser_download.add_argument('--ndvigrey', action='store_true', help='Create an NDVI map in grayscale (grey)') parser_download.add_argument('--clip', help='Clip the image with the bounding box provided. Values must be in ' + 'WGS84 datum, and with longitude and latitude units of decimal degrees ' + 'separated by comma.' + @@ -290,7 +303,7 @@ def main(args): if args.latest > 0: args.limit = 25 end = datetime.now() - start = end-relativedelta(days=+365) + start = end - relativedelta(days=+365) args.end = end.strftime("%Y-%m-%d") args.start = start.strftime("%Y-%m-%d") except (TypeError, ValueError): @@ -304,15 +317,23 @@ def main(args): except ValueError: return ["The latitude and longitude values must be valid numbers", 1] + address = args.address + if address and (lat and lon): + return ["Cannot specify both address and latitude-longitude"] + result = s.search(paths_rows=args.pathrow, lat=lat, lon=lon, + address=address, limit=args.limit, start_date=args.start, end_date=args.end, cloud_max=args.cloud) if result['status'] == 'SUCCESS': + if args.json: + return json.dumps(result) + if args.latest > 0: datelist = [] for i in range(0, result['total_returned']): @@ -341,15 +362,22 @@ def main(args): d = Downloader(download_dir=args.dest) try: bands = convert_to_integer_list(args.bands) - if args.pansharpen: - bands.append(8) - if args.ndvi: - bands = [4, 5] + if args.process: + if args.pansharpen: + bands.append(8) + + if args.ndvi or args.ndvigrey: + bands = [4, 5] + + if not args.bands: + bands = [4, 3, 2] downloaded = d.download(args.scenes, bands) if args.process: + if not args.bands: + args.bands = '432' force_unzip = True if args.force_unzip else False for scene, src in downloaded.iteritems(): if args.dest: @@ -362,7 +390,7 @@ def main(args): path = path + '.tar.bz' stored = process_image(path, args.bands, False, args.pansharpen, args.ndvi, force_unzip, - bounds=bounds) + args.ndvigrey, bounds=bounds) if args.upload: try: @@ -431,8 +459,13 @@ def __main__(): global parser parser = args_options() args = parser.parse_args() - with timer(): - exit(*main(args)) + if args.subs == 'search': + if args.json: + print main(args) + sys.exit(0) + else: + with timer(): + exit(*main(args)) if __name__ == "__main__": try: diff --git a/landsat/maps/colormap_ndvi_cfastie.txt b/landsat/maps/colormap_ndvi_cfastie.txt index 095f1d9..5814e53 100644 --- a/landsat/maps/colormap_ndvi_cfastie.txt +++ b/landsat/maps/colormap_ndvi_cfastie.txt @@ -165,99 +165,99 @@ mode = 255 26 233 47 17 240 31 8 247 15 -0 255 0 -7 255 0 -15 255 0 -23 255 0 -31 255 0 -39 255 0 -47 255 0 -55 255 0 -63 255 0 -71 255 0 -79 255 0 -87 255 0 -95 255 0 -103 255 0 -111 255 0 -119 255 0 -127 255 0 -135 255 0 -143 255 0 -151 255 0 -159 255 0 -167 255 0 -175 255 0 -183 255 0 -191 255 0 -199 255 0 -207 255 0 -215 255 0 -223 255 0 -231 255 0 -239 255 0 -247 255 0 -255 255 0 -255 249 0 -255 244 0 -255 239 0 -255 233 0 -255 228 0 -255 223 0 -255 217 0 -255 212 0 -255 207 0 -255 201 0 -255 196 0 -255 191 0 -255 185 0 -255 180 0 -255 175 0 -255 170 0 -255 164 0 -255 159 0 -255 154 0 -255 148 0 -255 143 0 -255 138 0 -255 132 0 -255 127 0 -255 122 0 -255 116 0 -255 111 0 -255 106 0 -255 100 0 -255 95 0 -255 90 0 -255 85 0 -255 79 0 -255 74 0 -255 69 0 -255 63 0 -255 58 0 -255 53 0 -255 47 0 -255 42 0 -255 37 0 -255 31 0 -255 26 0 -255 21 0 -255 15 0 -255 10 0 -255 5 0 -255 0 0 -255 0 15 -255 0 31 -255 0 47 -255 0 63 -255 0 79 -255 0 95 -255 0 111 -255 0 127 -255 0 143 -255 0 159 -255 0 175 -255 0 191 -255 0 207 -255 0 223 -255 0 239 +1 255 1 +7 255 1 +15 255 1 +23 255 1 +31 255 1 +39 255 1 +47 255 1 +55 255 1 +63 255 1 +71 255 1 +79 255 1 +87 255 1 +95 255 1 +103 255 1 +111 255 1 +119 255 1 +127 255 1 +135 255 1 +143 255 1 +151 255 1 +159 255 1 +167 255 1 +175 255 1 +183 255 1 +191 255 1 +199 255 1 +207 255 1 +215 255 1 +223 255 1 +231 255 1 +239 255 1 +247 255 1 +255 255 1 +255 249 1 +255 244 1 +255 239 1 +255 233 1 +255 228 1 +255 223 1 +255 217 1 +255 212 1 +255 207 1 +255 201 1 +255 196 1 +255 191 1 +255 185 1 +255 180 1 +255 175 1 +255 170 1 +255 164 1 +255 159 1 +255 154 1 +255 148 1 +255 143 1 +255 138 1 +255 132 1 +255 127 1 +255 122 1 +255 116 1 +255 111 1 +255 106 1 +255 100 1 +255 95 1 +255 90 1 +255 85 1 +255 79 1 +255 74 1 +255 69 1 +255 63 1 +255 58 1 +255 53 1 +255 47 1 +255 42 1 +255 37 1 +255 31 1 +255 26 1 +255 21 1 +255 15 1 +255 10 1 +255 5 1 +255 1 1 +255 1 15 +255 1 31 +255 1 47 +255 1 63 +255 1 79 +255 1 95 +255 1 111 +255 1 127 +255 1 143 +255 1 159 +255 1 175 +255 1 191 +255 1 207 +255 1 223 +255 1 239 diff --git a/landsat/ndvi.py b/landsat/ndvi.py index edda4ac..931192c 100644 --- a/landsat/ndvi.py +++ b/landsat/ndvi.py @@ -88,8 +88,6 @@ def write_band(self, output_band, output_file, image_data): output.write_band(1, output_band) - cmap = {k: v[:3] for k, v in self.cmap.iteritems()} - output.write_colormap(1, cmap) self.output("Writing to file", normal=True, color='green', indent=1) return output_file @@ -123,7 +121,7 @@ def write_band(self, output_band, output_file, image_data): crs=self.dst_crs) as output: for i in range(3): - output.write_band(i+1, rgb_bands[i]) + output.write_band(i + 1, rgb_bands[i]) self.output("Writing to file", normal=True, color='green', indent=1) return output_file diff --git a/landsat/search.py b/landsat/search.py index 65ebc46..5ee0be8 100644 --- a/landsat/search.py +++ b/landsat/search.py @@ -6,7 +6,7 @@ import requests import settings -from utils import three_digit, create_paired_list +from utils import three_digit, create_paired_list, geocode class Search(object): @@ -15,7 +15,7 @@ class Search(object): def __init__(self): self.api_url = settings.API_URL - def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None, + def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None, limit=1): """ The main method of Search class. It searches Development Seed's Landsat API. @@ -32,6 +32,10 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= The The longitude :type lon: String, float, integer + :param address: + The address + :type address: + String :param start_date: Date string. format: YYYY-MM-DD :type start_date: @@ -78,7 +82,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= } """ - search_string = self.query_builder(paths_rows, lat, lon, start_date, end_date, cloud_min, cloud_max) + search_string = self.query_builder(paths_rows, lat, lon, address, start_date, end_date, cloud_min, cloud_max) # Have to manually build the URI to bypass requests URI encoding # The api server doesn't accept encoded URIs @@ -109,7 +113,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= return result - def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, + def query_builder(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None): """ Builds the proper search syntax (query) for Landsat API. @@ -125,6 +129,10 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en The The longitude :type lon: String, float, integer + :param address: + The address + :type address: + String :param start_date: Date string. format: YYYY-MM-DD :type start_date: @@ -171,7 +179,9 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en elif cloud_max: query.append(self.cloud_cover_prct_range_builder('-1', cloud_max)) - if lat and lon: + if address: + query.append(self.address_builder(address)) + elif lat and lon: query.append(self.lat_lon_builder(lat, lon)) if query: @@ -241,6 +251,20 @@ def cloud_cover_prct_range_builder(self, min=0, max=100): """ return 'cloudCoverFull:[%s+TO+%s]' % (min, max) + def address_builder(self, address): + """ Builds lat and lon query from a geocoded address. + + :param address: + The address + :type address: + String + + :returns: + String + """ + geocoded = geocode(address) + return self.lat_lon_builder(**geocoded) + def lat_lon_builder(self, lat=0, lon=0): """ Builds lat and lon query. diff --git a/landsat/utils.py b/landsat/utils.py index 3107ffc..c6a3545 100644 --- a/landsat/utils.py +++ b/landsat/utils.py @@ -7,6 +7,7 @@ import re from cStringIO import StringIO from datetime import datetime +import geocoder from mixins import VerbosityMixin @@ -280,6 +281,52 @@ def convert_to_integer_list(value): return s +# Geocoding confidence scores, from https://github.com/DenisCarriere/geocoder/blob/master/docs/features/Confidence%20Score.md +geocode_confidences = { + 10: 0.25, + 9: 0.5, + 8: 1., + 7: 5., + 6: 7.5, + 5: 10., + 4: 15., + 3: 20., + 2: 25., + 1: 99999., + # 0: unable to locate at all +} + + +def geocode(address, required_precision_km=1.): + """ Identifies the coordinates of an address + + :param address: + the address to be geocoded + :type value: + String + :param required_precision_km: + the maximum permissible geographic uncertainty for the geocoding + :type required_precision_km: + float + + :returns: + dict + + :example: + >>> geocode('1600 Pennsylvania Ave NW, Washington, DC 20500') + {'lat': 38.89767579999999, 'lon': -77.0364827} + + """ + geocoded = geocoder.google(address) + precision_km = geocode_confidences[geocoded.confidence] + + if precision_km <= required_precision_km: + (lon, lat) = geocoded.geometry['coordinates'] + return {'lat': lat, 'lon': lon} + else: + raise ValueError("Address could not be precisely located") + + def convert_to_float_list(value): """ Converts a comma separate string to a list @@ -344,3 +391,17 @@ def adjust_bounding_box(bounds1, bounds2): new_bounds[3] = bounds1[3] return tuple(new_bounds) + + +def remove_slash(value): + + assert(isinstance(value, str)) + return re.sub('(^\/|\/$)', '', value) + + +def url_builder(segments): + + # Only accept list or tuple + assert((isinstance(segments, list) or isinstance(segments, tuple))) + return "/".join([remove_slash(s) for s in segments]) + diff --git a/requirements/docker.txt b/requirements/docker.txt index 1df380f..e4a8c18 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -6,3 +6,4 @@ six==1.9.0 homura>=0.1.2 boto>=2.38.0 polyline==1.1 +geocoder>=1.5.1 diff --git a/setup.py b/setup.py index e4afc11..27c50b8 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ def readme(): return f.read() test_requirements = [ - 'nose>=1.3.7', - 'mock>=1.3.0' + 'nose>=1.3.7', + 'mock>=1.3.0' ] setup( @@ -35,17 +35,18 @@ def readme(): license='CCO', platforms='Posix; MacOS X; Windows', install_requires=[ - 'requests==2.7.0', - 'python-dateutil>=2.4.2', - 'numpy>=1.9.3', - 'termcolor>=1.1.0', - 'rasterio>=0.26.0', - 'six==1.9.0', - 'scipy>=0.16.0', - 'scikit-image>=0.11.3', - 'homura>=0.1.2', - 'boto>=2.38.0', - 'polyline==1.1' + 'requests==2.7.0', + 'python-dateutil>=2.4.2', + 'numpy>=1.9.3', + 'termcolor>=1.1.0', + 'rasterio>=0.26.0', + 'six==1.9.0', + 'scipy>=0.16.0', + 'scikit-image>=0.11.3', + 'homura>=0.1.2', + 'boto>=2.38.0', + 'polyline==1.1', + 'geocoder>=1.5.1' ], test_suite='nose.collector', tests_require=test_requirements diff --git a/tests/test_landsat.py b/tests/test_landsat.py index 2c56d30..e7667e4 100644 --- a/tests/test_landsat.py +++ b/tests/test_landsat.py @@ -3,6 +3,7 @@ """Tests for landsat""" +import json import unittest import subprocess import errno @@ -67,6 +68,15 @@ def test_search_pr_wrong_input(self): self.assertEquals(landsat.main(self.parser.parse_args(args)), ['Check your request and try again', 1]) + def test_search_json_output(self): + """Test json output in search""" + args = ['search', '--latest', '10', '--json'] + + output = landsat.main(self.parser.parse_args(args)) + j = json.loads(output) + + self.assertEquals(type(j), dict) + @mock.patch('landsat.landsat.Downloader') def test_download_correct(self, mock_downloader): """Test download command with correct input""" @@ -78,6 +88,29 @@ def test_download_correct(self, mock_downloader): mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], [11]) self.assertEquals(output, ['Download Completed', 0]) + @mock.patch('landsat.landsat.Downloader') + def test_download_correct_zip(self, mock_downloader): + """Download command should download zip if no bands are given""" + mock_downloader.download.return_value = True + + args = ['download', 'LC80010092015051LGN00', '-d', self.mock_path] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(download_dir=self.mock_path) + mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], []) + self.assertEquals(output, ['Download Completed', 0]) + + @mock.patch('landsat.landsat.process_image') + @mock.patch('landsat.landsat.Downloader.download') + def test_download_no_bands_with_process(self, mock_downloader, mock_process): + """Download command should not download zip if no bands are given but process flag is used""" + mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'} + mock_process.return_value = 'image.TIF' + + args = ['download', 'LC80010092015051LGN00', '-p', '-d', self.mock_path] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) + self.assertEquals(output, ["The output is stored at image.TIF", 0]) + def test_download_incorrect(self): """Test download command with incorrect input""" args = ['download', 'LT813600'] @@ -97,7 +130,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', - False, False, False, False, bounds=None) + False, False, False, False, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with force unzip flag @@ -106,7 +139,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False, - True, bounds=None) + True, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with pansharpen @@ -115,7 +148,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2, 8]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False, - False, bounds=None) + False, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with pansharpen and clipping @@ -124,7 +157,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2, 8]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False, - False, bounds=[-180.0, -180.0, 0.0, 0.0]) + False, False, bounds=[-180.0, -180.0, 0.0, 0.0]) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with ndvi @@ -133,7 +166,16 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 5]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, True, - False, bounds=None) + False, False, bounds=None) + self.assertEquals(output, ["The output is stored at image.TIF", 0]) + + # Call with ndvigrey + args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + self.mock_path, '-p', '--ndvigrey'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 5]) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False, + False, True, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) @mock.patch('landsat.landsat.Uploader') @@ -150,7 +192,7 @@ def test_download_process_continuous_with_upload(self, mock_downloader, mock_pro output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False, - False, bounds=None) + False, False, bounds=None) mock_upload.assert_called_with('somekey', 'somesecret', 'this') mock_upload.return_value.run.assert_called_with('mybucket', 'image.TIF', 'image.TIF') self.assertEquals(output, ['The output is stored at image.TIF', 0]) @@ -167,7 +209,7 @@ def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False, - False, bounds=None) + False, False, bounds=None) self.assertEquals(output, ['Could not authenticate with AWS', 1]) @mock.patch('landsat.landsat.process_image') diff --git a/tests/test_utils.py b/tests/test_utils.py index 2b959ab..9bbd38d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -105,6 +105,15 @@ def test_convert_to_integer_list(self): r = utils.convert_to_integer_list('1,11,10,QA') self.assertEqual([1, 11, 10, 'QA'], r) + def test_geocode(self): + loc = utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500') + + self.assertEqual(round(loc['lat'], 3), 38.898) + self.assertEqual(round(loc['lon'], 3), -77.037) + self.assertRaises(ValueError, utils.geocode, 'Pennsylvania Ave NW, Washington, DC') + self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, + utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.)) + def test_convert_to_float_list(self): # correct input r = utils.convert_to_float_list('-1,2,-3') @@ -166,5 +175,16 @@ def test_adjust_bounding_box(self): self.assertEqual(utils.adjust_bounding_box(origin, target), origin) + def test_url_builder(self): + + self.assertEqual('http://example.com/segment1/segment2', + utils.url_builder(['/http://example.com', 'segment1/', '/segment2'])) + + self.assertEqual('http://example.com/segment1/segment2', + utils.url_builder(('/http://example.com', 'segment1/', '/segment2',))) + + with self.assertRaises(AssertionError): + utils.url_builder('example.com') + if __name__ == '__main__': unittest.main() diff --git a/travis-dockerfile b/travis-dockerfile new file mode 100644 index 0000000..92d8fa1 --- /dev/null +++ b/travis-dockerfile @@ -0,0 +1,4 @@ +FROM developmentseed/landsat-util:dev +ADD . /test +RUN cd /test && pip install -r requirements/docker.txt +RUN pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0