From e9606155e36cde38f89790ab33e89a638d4b0372 Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Tue, 9 Apr 2024 09:35:17 -0700 Subject: [PATCH 1/9] use temporal search directly in asf --- tools/bin/ariaDownload.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index 8f1b541a..cec16be2 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -108,6 +108,7 @@ def createParser(): '--log-level', default='warning', help='Logger log level') return parser + def make_bbox(inp_bbox): """Make a WKT from SNWE or a shapefile""" if inp_bbox is None: @@ -197,8 +198,10 @@ def __init__(self, args): else: LOGGER.setLevel('INFO') + def __call__(self): scenes = self.query_asf() + urls, ifgs = get_url_ifg(scenes) # subset everything by version @@ -226,10 +229,9 @@ def __call__(self): # optionally match other conditions (st/end date, elapsed time) else: - sten_chk = sti >= self.args.start and eni <= self.args.end elap = (eni - sti).days elap_chk = elap >= self.args.daysgt and elap <= self.args.dayslt - if sten_chk and elap_chk: + if elap_chk: idx.append(i) else: continue @@ -284,6 +286,7 @@ def __call__(self): return + def query_asf(self): """Get the scenes from ASF""" bbox = make_bbox(self.args.bbox) @@ -306,11 +309,15 @@ def query_asf(self): processingLevel=asf_search.constants.GUNW_STD, relativeOrbit=tracks, flightDirection=flight_direction, - intersectsWith=bbox) + intersectsWith=bbox, + start=self.args.start, + end=self.args.end) + scenes = asf_search.geo_search(**dct_kw) return scenes + def main(): parser = createParser() args = parser.parse_args() @@ -328,5 +335,6 @@ def main(): raise Exception('Must specify either a bbox or track') Downloader(args)() + if __name__ == '__main__': main() From 29da26ebfa7339d92a2d426e799ce60c99a71fbb Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Tue, 9 Apr 2024 10:56:14 -0700 Subject: [PATCH 2/9] placeholder for NISAR asf_search support --- tools/bin/ariaDownload.py | 57 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index cec16be2..2169ae1e 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -15,6 +15,7 @@ import logging import shapely import warnings +from getpass import getpass import asf_search import ARIAtools.util.log @@ -31,16 +32,18 @@ def createParser(): see: https://github.com/asfadmin/Discovery-asf_search """ parser = argparse.ArgumentParser( - description='Command line interface to download GUNW products from ' - 'the ASF DAAC. GUNW products are hosted at the NASA ASF ' - 'DAAC.\nDownloading them requires a NASA Earthdata URS ' - 'user login and requires users to add "GRFN Door (PROD)" ' - 'and "ASF Datapool Products" to their URS approved ' - 'applications.', + description='Command line interface to download Sentinel-1/NISAR GUNW ' + 'products from the ASF DAAC. \nDownloading them requires a ' + 'NASA Earthdata URS user login and requires users to add ' + '"GRFN Door (PROD)" and "ASF Datapool Products" to their ' + 'URS approved applications. Access to NISAR products ' + 'requires an Earthdata Bearer token from: ' + 'https://urs.earthdata.nasa.gov/documentation/for_users/user_token', epilog='Examples of use:\n' '\t ariaDownload.py --track 004 --output count\n' '\t ariaDownload.py --bbox "36.75 37.225 -76.655 -75.928"\n' - '\t ariaDownload.py -t 004,077 --start 20190101 -o count', + '\t ariaDownload.py -t 004,077 --start 20190101 -o count' + '\t ariaDownload.py --mission NISAR -o count', formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( @@ -76,6 +79,11 @@ def createParser(): help='NASA Earthdata URS user password. Users must add "GRFN Door ' '(PROD)" and "ASF Datapool Products" to their URS approved ' 'applications.') + + parser.add_argument( + '--mission', default='S1', type=str.upper, choices=('S1', 'NISAR'), + help='Sentinel-1 (S1) or NISAR. Default is S1') + parser.add_argument( '-l', '--daysless', dest='dayslt', default=math.inf, type=int, help='Take pairs with a temporal baseline -- days less than this value.') @@ -305,15 +313,34 @@ def query_asf(self): else: tracks = self.args.track - dct_kw = dict(platform=asf_search.constants.SENTINEL1, - processingLevel=asf_search.constants.GUNW_STD, - relativeOrbit=tracks, - flightDirection=flight_direction, - intersectsWith=bbox, - start=self.args.start, - end=self.args.end) - scenes = asf_search.geo_search(**dct_kw) + if self.args.mission.upper() == 'S1': + dct_kw = dict(dataset='ARIA S1 GUNW', + processingLevel=asf_search.constants.GUNW_STD, + relativeOrbit=tracks, + flightDirection=flight_direction, + intersectsWith=bbox, + start=self.args.start, + end=self.args.end) + scenes = asf_search.geo_search(**dct_kw) + + elif self.args.mission.upper() == 'NISAR': + session = asf_search.ASFSession() + session.auth_with_token(getpass('EDL Token:')) + LOGGER.info ('Token accepted.') + ## populate once available on GUNWs are available + search_opts = asf_search.ASFSearchOptions( + dataset=asf_search.DATASET.NISAR, + session=session + # processingLevel=asf_search.constants.GUNW_STD, + # relativeOrbit=tracks, + # flightDirection=flight_direction, + # intersectsWith=bbox, + # start=self.args.start, + # end=self.args.end) + ) + scenes = asf_search.search(opts=search_opts, maxResults=250) + raise Exception('NISAR GUNWs not futher supported') return scenes From d626d954e15c0ce38bce538b07bd0579b22b7e37 Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Tue, 9 Apr 2024 12:33:55 -0700 Subject: [PATCH 3/9] keep fine subsetting --- tools/bin/ariaDownload.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index 2169ae1e..f331fef5 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -15,6 +15,7 @@ import logging import shapely import warnings +from dateutil.relativedelta import relativedelta from getpass import getpass import asf_search @@ -237,9 +238,10 @@ def __call__(self): # optionally match other conditions (st/end date, elapsed time) else: + sten_chk = sti >= self.args.start and eni <= self.args.end elap = (eni - sti).days elap_chk = elap >= self.args.daysgt and elap <= self.args.dayslt - if elap_chk: + if sten_chk and elap_chk: idx.append(i) else: continue @@ -314,14 +316,18 @@ def query_asf(self): tracks = self.args.track + ## buffer a bit for loose subsetting and speedup + st0 = self.args.start + relativedelta(months=-3) + en0 = self.args.end + relativedelta(months=-3) + if self.args.mission.upper() == 'S1': dct_kw = dict(dataset='ARIA S1 GUNW', processingLevel=asf_search.constants.GUNW_STD, relativeOrbit=tracks, flightDirection=flight_direction, intersectsWith=bbox, - start=self.args.start, - end=self.args.end) + start=st0, + end=en0) scenes = asf_search.geo_search(**dct_kw) elif self.args.mission.upper() == 'NISAR': From 6d548ae66d734eab1d769b9b2bb869a541f941c4 Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Tue, 9 Apr 2024 15:12:38 -0700 Subject: [PATCH 4/9] placeholder for NISAR GUNWs --- tools/bin/ariaDownload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index f331fef5..fef5724d 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -336,7 +336,7 @@ def query_asf(self): LOGGER.info ('Token accepted.') ## populate once available on GUNWs are available search_opts = asf_search.ASFSearchOptions( - dataset=asf_search.DATASET.NISAR, + shortName='NISAR_L2_GUNW_BETA_V1', session=session # processingLevel=asf_search.constants.GUNW_STD, # relativeOrbit=tracks, @@ -346,6 +346,7 @@ def query_asf(self): # end=self.args.end) ) scenes = asf_search.search(opts=search_opts, maxResults=250) + print (scenes[0]) raise Exception('NISAR GUNWs not futher supported') return scenes From 0c01e79049a3621ed4513bf71dc7b38e003fac90 Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Wed, 10 Apr 2024 09:26:56 -0700 Subject: [PATCH 5/9] styling; add months to end --- tools/bin/ariaDownload.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index fef5724d..aa2e592b 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -317,8 +317,8 @@ def query_asf(self): ## buffer a bit for loose subsetting and speedup - st0 = self.args.start + relativedelta(months=-3) - en0 = self.args.end + relativedelta(months=-3) + start = self.args.start + relativedelta(months=-3) + end = self.args.end + relativedelta(months=3) if self.args.mission.upper() == 'S1': dct_kw = dict(dataset='ARIA S1 GUNW', @@ -326,15 +326,16 @@ def query_asf(self): relativeOrbit=tracks, flightDirection=flight_direction, intersectsWith=bbox, - start=st0, - end=en0) + start=start, + end=end) scenes = asf_search.geo_search(**dct_kw) elif self.args.mission.upper() == 'NISAR': session = asf_search.ASFSession() session.auth_with_token(getpass('EDL Token:')) - LOGGER.info ('Token accepted.') + LOGGER.info('Token accepted.') ## populate once available on GUNWs are available + # TODO: search_opts = asf_search.ASFSearchOptions( shortName='NISAR_L2_GUNW_BETA_V1', session=session @@ -346,8 +347,11 @@ def query_asf(self): # end=self.args.end) ) scenes = asf_search.search(opts=search_opts, maxResults=250) - print (scenes[0]) - raise Exception('NISAR GUNWs not futher supported') + if len(scenes)>0: + LOGGER.info('Found NISAR GUNW Betas.') + else: + LOGGER.warning('No NISAR GUNW Betas found.') + raise Exception('Exiting, NISAR GUNWs not futher supported.') return scenes From e1355efa2cc19d2ebdbc4a6ce47d16f95c15b325 Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Wed, 10 Apr 2024 15:10:55 -0700 Subject: [PATCH 6/9] replace relativedelta with datetime --- tools/bin/ariaDownload.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index aa2e592b..9415dd51 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -15,7 +15,6 @@ import logging import shapely import warnings -from dateutil.relativedelta import relativedelta from getpass import getpass import asf_search @@ -317,17 +316,17 @@ def query_asf(self): ## buffer a bit for loose subsetting and speedup - start = self.args.start + relativedelta(months=-3) - end = self.args.end + relativedelta(months=3) + start = self.args.start + datetime.timedelta(days=-90) + end = self.args.end + relativedelta(days=90) if self.args.mission.upper() == 'S1': dct_kw = dict(dataset='ARIA S1 GUNW', - processingLevel=asf_search.constants.GUNW_STD, - relativeOrbit=tracks, - flightDirection=flight_direction, - intersectsWith=bbox, - start=start, - end=end) + processingLevel=asf_search.constants.GUNW_STD, + relativeOrbit=tracks, + flightDirection=flight_direction, + intersectsWith=bbox, + start=start, + end=end) scenes = asf_search.geo_search(**dct_kw) elif self.args.mission.upper() == 'NISAR': @@ -339,12 +338,12 @@ def query_asf(self): search_opts = asf_search.ASFSearchOptions( shortName='NISAR_L2_GUNW_BETA_V1', session=session - # processingLevel=asf_search.constants.GUNW_STD, - # relativeOrbit=tracks, - # flightDirection=flight_direction, - # intersectsWith=bbox, - # start=self.args.start, - # end=self.args.end) + # processingLevel=asf_search.constants.GUNW_STD, + # relativeOrbit=tracks, + # flightDirection=flight_direction, + # intersectsWith=bbox, + # start=self.args.start, + # end=self.args.end) ) scenes = asf_search.search(opts=search_opts, maxResults=250) if len(scenes)>0: From d9d155eec024b1ed82dd4060f7a343322cf95a4a Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Thu, 11 Apr 2024 11:15:04 -0700 Subject: [PATCH 7/9] bugfix --- tools/bin/ariaDownload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index 9415dd51..1bc23836 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -317,7 +317,7 @@ def query_asf(self): ## buffer a bit for loose subsetting and speedup start = self.args.start + datetime.timedelta(days=-90) - end = self.args.end + relativedelta(days=90) + end = self.args.end + datetime.timedelta(days=90) if self.args.mission.upper() == 'S1': dct_kw = dict(dataset='ARIA S1 GUNW', From 0d4967539a2eca500c0ec576f8d884958ca5c144 Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Thu, 11 Apr 2024 13:23:12 -0700 Subject: [PATCH 8/9] fix bug introduced in refactor --- tools/bin/ariaDownload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index 1bc23836..790618d0 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -164,7 +164,7 @@ def get_url_ifg(scenes): return urls, ifgs -def fmt_dst(inps): +def fmt_dst(args): """Format the save name""" ext = '.kmz' if args.output == 'Kml' else '.txt' @@ -258,7 +258,7 @@ def __call__(self): # 'Revert to an older version of ARIAtools') elif self.args.output == 'Url': - dst = fmt_dst(inps) + dst = fmt_dst(self.args) with open(dst, 'w') as fh: for url in urls: print(url, sep='\n', file=fh) From 8cfa1d02523accd76502cf9344e77b2cab958a0c Mon Sep 17 00:00:00 2001 From: Brett Buzzanga Date: Thu, 11 Apr 2024 19:07:59 -0700 Subject: [PATCH 9/9] another bug from prior refactoring; use full collection --- tools/bin/ariaDownload.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/bin/ariaDownload.py b/tools/bin/ariaDownload.py index 790618d0..111acaeb 100755 --- a/tools/bin/ariaDownload.py +++ b/tools/bin/ariaDownload.py @@ -287,7 +287,7 @@ def __call__(self): LOGGER.info( f'Download complete. Wrote -- {len(scenes)} -- products to: ' - '{self.args.wd}') + f'{self.args.wd}') if self.args.verbose: for scene in scenes: @@ -320,13 +320,15 @@ def query_asf(self): end = self.args.end + datetime.timedelta(days=90) if self.args.mission.upper() == 'S1': - dct_kw = dict(dataset='ARIA S1 GUNW', + dct_kw = dict(dataset=asf_search.DATASET.ARIA_S1_GUNW, processingLevel=asf_search.constants.GUNW_STD, relativeOrbit=tracks, flightDirection=flight_direction, intersectsWith=bbox, start=start, - end=end) + end=end, + ) + # collectionAlias=False) scenes = asf_search.geo_search(**dct_kw) elif self.args.mission.upper() == 'NISAR':