diff --git a/landsat/downloader.py b/landsat/downloader.py index 8a5080c..10bbc6e 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -45,10 +45,12 @@ def download(self, scenes, bands=None): List :returns: - Boolean + (List) includes downloaded scenes as key and source as value (aws or google) """ if isinstance(scenes, list): + output = {} + for scene in scenes: # If bands are provided the image is from 2015 or later use Amazon if (bands and int(scene[12]) > 4): @@ -61,14 +63,18 @@ def download(self, scenes, bands=None): bands_plus.append('MTL') for band in bands_plus: self.amazon_s3(scene, band, path) + output[scene] = 'aws' except RemoteFileDoesntExist: self.google_storage(scene, self.download_dir) + output[scene] = 'google' + else: raise Exception('Expected bands list') else: self.google_storage(scene, self.download_dir) + output[scene] = 'google' - return True + return output else: raise Exception('Expected sceneIDs list') diff --git a/landsat/image.py b/landsat/image.py index ccdb759..0a7d928 100644 --- a/landsat/image.py +++ b/landsat/image.py @@ -4,7 +4,7 @@ import warnings import sys -from os.path import join +from os.path import join, isdir import tarfile import glob import subprocess @@ -49,10 +49,14 @@ class Process(VerbosityMixin): Whether the output should be verbose. Default is False. :type verbose: boolean + :param force_unzip: + Whether to force unzip the tar file. Default is False + :type force_unzip: + boolean """ - def __init__(self, path, bands=None, dst_path=None, verbose=False): + def __init__(self, path, bands=None, dst_path=None, verbose=False, force_unzip=False): self.projection = {'init': 'epsg:3857'} self.dst_crs = {'init': u'epsg:3857'} @@ -71,7 +75,7 @@ def __init__(self, path, bands=None, dst_path=None, verbose=False): self.scene_path = join(self.src_path, self.scene) if self._check_if_zipped(path): - self._unzip(join(self.src_path, get_file(path)), join(self.src_path, self.scene), self.scene) + self._unzip(join(self.src_path, get_file(path)), join(self.src_path, self.scene), self.scene, force_unzip) self.bands_path = [] for band in self.bands: @@ -287,14 +291,19 @@ def _get_boundaries(self, src): def _percent_cut(self, color, low, high): return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (low, high)) - def _unzip(self, src, dst, scene): + def _unzip(self, src, dst, scene, force_unzip=False): """ Unzip tar files """ self.output("Unzipping %s - It might take some time" % scene, normal=True, arrow=True) try: - tar = tarfile.open(src, 'r') - tar.extractall(path=dst) - tar.close() + # check if file is already unzipped, skip + if isdir(dst) and not force_unzip: + self.output("%s is already unzipped." % scene, normal=True, arrow=True) + return + else: + tar = tarfile.open(src, 'r') + tar.extractall(path=dst) + tar.close() except tarfile.ReadError: check_create_folder(dst) subprocess.check_call(['tar', '-xf', src, '-C', dst]) diff --git a/landsat/landsat.py b/landsat/landsat.py index 328f01a..2ba073d 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -86,6 +86,8 @@ --region URL to S3 region e.g. s3-us-west-2.amazonaws.com + --force-unzip Force unzip tar file + Process: landsat.py process path [-h] [-b --bands] [-p --pansharpen] @@ -115,6 +117,8 @@ --bucket Bucket name (required if uploading to s3) --region URL to S3 region e.g. s3-us-west-2.amazonaws.com + + --force-unzip Force unzip tar file """ @@ -179,6 +183,7 @@ def args_options(): 'as Environment Variables)') parser_download.add_argument('--bucket', help='Bucket name (required if uploading to s3)') parser_download.add_argument('--region', help='URL to S3 region e.g. s3-us-west-2.amazonaws.com') + parser_download.add_argument('--force-unzip', help='Force unzip tar file', action='store_true') parser_process = subparsers.add_parser('process', help='Process Landsat imagery') parser_process.add_argument('path', @@ -198,6 +203,7 @@ def args_options(): 'as Environment Variables)') parser_process.add_argument('--bucket', help='Bucket name (required if uploading to s3)') parser_process.add_argument('--region', help='URL to S3 region e.g. s3-us-west-2.amazonaws.com') + parser_process.add_argument('--force-unzip', help='Force unzip tar file', action='store_true') return parser @@ -221,9 +227,11 @@ def main(args): v = VerbosityMixin() if args: + if args.subs == 'process': verbose = True if args.verbose else False - stored = process_image(args.path, args.bands, verbose, args.pansharpen) + force_unzip = True if args.force_unzip else False + stored = process_image(args.path, args.bands, verbose, args.pansharpen, force_unzip) if args.upload: u = Uploader(args.key, args.secret, args.region) @@ -269,18 +277,21 @@ def main(args): elif args.subs == 'download': d = Downloader(download_dir=args.dest) try: - if d.download(args.scenes, convert_to_integer_list(args.bands)): - if args.process: + downloaded = d.download(args.scenes, convert_to_integer_list(args.bands)) + + if args.process: + force_unzip = True if args.force_unzip else False + for scene, src in downloaded.iteritems(): if args.dest: - path = join(args.dest, args.scenes[0]) + path = join(args.dest, scene) else: - path = join(settings.DOWNLOAD_DIR, args.scenes[0]) + path = join(settings.DOWNLOAD_DIR, scene) # Keep using Google if the image is before 2015 - if (int(args.scenes[0][12]) < 5 or not args.bands): + if src == 'google': path = path + '.tar.bz' - stored = process_image(path, args.bands, False, args.pansharpen) + stored = process_image(path, args.bands, False, args.pansharpen, force_unzip) if args.upload: try: @@ -291,14 +302,16 @@ def main(args): return ["Connection timeout. Probably the region parameter is incorrect", 1] u.run(args.bucket, get_file(stored), stored) - return ["The output is stored at %s" % stored] - else: - return ['Download Completed', 0] + v.output("The output is stored at %s" % stored, normal=True, arrow=True) + + return ['Image Processing Completed', 0] + else: + return ['Download Completed', 0] except IncorrectSceneId: return ['The SceneID provided was incorrect', 1] -def process_image(path, bands=None, verbose=False, pansharpen=False): +def process_image(path, bands=None, verbose=False, pansharpen=False, force_unzip=None): """ Handles constructing and image process. :param path: @@ -323,7 +336,7 @@ def process_image(path, bands=None, verbose=False, pansharpen=False): """ try: bands = convert_to_integer_list(bands) - p = Process(path, bands=bands, verbose=verbose) + p = Process(path, bands=bands, verbose=verbose, force_unzip=force_unzip) except IOError: exit("Zip file corrupted", 1) except FileDoesNotExist as e: diff --git a/landsat/tests/test_download.py b/landsat/tests/test_download.py index 6166497..4d7d06d 100644 --- a/landsat/tests/test_download.py +++ b/landsat/tests/test_download.py @@ -28,7 +28,9 @@ def setUpClass(cls): cls.d = Downloader() cls.temp_folder = mkdtemp() cls.scene = 'LT81360082013127LGN01' + cls.scene_2 = 'LC82050312014229LGN00' cls.scene_s3 = 'LC80010092015051LGN00' + cls.scene_s3_2 = 'LC82050312015136LGN00' cls.scene_size = 59204484 @classmethod @@ -49,21 +51,19 @@ def assertSize(self, url, path): def test_download(self, mock_fetch): mock_fetch.return_value = True - # download one list + # download one scene self.d.download([self.scene]) - self.assertTrue(self.d.download([self.scene])) + self.assertEqual({self.scene: 'google'}, self.d.download([self.scene])) + + # download multiple scenes + self.assertEqual({self.scene: 'google', self.scene_2: 'google'}, self.d.download([self.scene, self.scene_2])) # Test if error is raised when passing scene as string instead of list self.assertRaises(Exception, self.d.download, self.scene) - # Test if download works when passing scenes as list - self.d.download([self.scene, self.scene]) - self.assertTrue(self.d.download([self.scene])) - # Test when passing band list along with sceneID - self.d.download([self.scene_s3], bands=[11]) - - self.assertTrue(self.d.download([self.scene])) + self.assertEqual({self.scene_s3: 'aws', self.scene_s3_2: 'aws'}, + self.d.download([self.scene_s3, self.scene_s3_2], bands=[11])) # Test whether passing band as string raises an exception self.assertRaises(Exception, self.d.download, self.scene, 4) diff --git a/landsat/tests/test_landsat.py b/landsat/tests/test_landsat.py index d6a74c1..0a35641 100644 --- a/landsat/tests/test_landsat.py +++ b/landsat/tests/test_landsat.py @@ -94,21 +94,30 @@ def test_download_incorrect(self): @mock.patch('landsat.landsat.Downloader.download') def test_download_process_continuous(self, mock_downloader, mock_process): """Test download and process commands together""" - mock_downloader.return_value = True + mock_downloader.return_value = {'LC80010092015051LGN00': 'aws', + 'LC80010092014051LGN00': 'aws'} mock_process.return_value = 'image.TIF' - args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p'] + args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', self.mock_path, '-p'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) - self.assertEquals(output, ["The output is stored at image.TIF"]) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2']) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False) + self.assertEquals(output, ["Image Processing Completed", 0]) + + # Call with force unzip flag + args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + self.mock_path, '-p', '--force-unzip'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2']) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, True) + self.assertEquals(output, ["Image Processing Completed", 0]) @mock.patch('landsat.landsat.Uploader') @mock.patch('landsat.landsat.process_image') @mock.patch('landsat.landsat.Downloader.download') def test_download_process_continuous_with_upload(self, mock_downloader, mock_process, mock_upload): """Test download and process commands together""" - mock_downloader.return_value = True + mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'} mock_process.return_value = 'image.TIF' mock_upload.run.return_value = True @@ -116,23 +125,23 @@ def test_download_process_continuous_with_upload(self, mock_downloader, mock_pro '-u', '--key', 'somekey', '--secret', 'somesecret', '--bucket', 'mybucket', '--region', 'this'] output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False) mock_upload.assert_called_with('somekey', 'somesecret', 'this') mock_upload.return_value.run.assert_called_with('mybucket', 'image.TIF', 'image.TIF') - self.assertEquals(output, ["The output is stored at image.TIF"]) + self.assertEquals(output, ["Image Processing Completed", 0]) @mock.patch('landsat.landsat.process_image') @mock.patch('landsat.landsat.Downloader.download') def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock_process): """Test download and process commands together""" - mock_downloader.return_value = True + mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'} mock_process.return_value = 'image.TIF' args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p', '-u', '--region', 'whatever'] output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False) self.assertEquals(output, ['Could not authenticate with AWS', 1]) @mock.patch('landsat.landsat.process_image') @@ -143,7 +152,7 @@ def test_process_correct(self, mock_process): args = ['process', 'path/to/folder/LC80010092015051LGN00'] output = landsat.main(self.parser.parse_args(args)) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False, False) self.assertEquals(output, ["The output is stored at image.TIF"]) @mock.patch('landsat.landsat.process_image') @@ -154,7 +163,7 @@ def test_process_correct_pansharpen(self, mock_process): args = ['process', '--pansharpen', 'path/to/folder/LC80010092015051LGN00'] output = landsat.main(self.parser.parse_args(args)) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True, False) self.assertEquals(output, ["The output is stored at image.TIF"]) def test_process_incorrect(self):