From b0801de6578a6c3a643b8b4b233a0c8bc2905a60 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 14 Oct 2024 10:23:36 -0700 Subject: [PATCH 1/5] feat: add AUDIT log level for upload --- .../upload_arbitrary_files_as_granules.py | 1 + .../stage_in_out/upload_granules_abstract.py | 15 + .../upload_granules_by_complete_catalog_s3.py | 2 +- tests/integration_tests/.env.tpl | 6 + tests/integration_tests/__init__.py | 0 tests/integration_tests/test_docker_entry.py | 375 ++++ .../integration_tests/test_docker_stage_in.py | 1928 +++++++++++++++++ .../test_docker_stage_out.py | 1334 ++++++++++++ 8 files changed, 3660 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/.env.tpl create mode 100644 tests/integration_tests/__init__.py create mode 100644 tests/integration_tests/test_docker_entry.py create mode 100644 tests/integration_tests/test_docker_stage_in.py create mode 100644 tests/integration_tests/test_docker_stage_out.py diff --git a/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py b/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py index 9a53a75..a545ce4 100644 --- a/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py +++ b/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py @@ -77,6 +77,7 @@ def execute_job(self, job_obj, lock) -> bool: sample_stac_item = self.generate_sample_stac(job_obj) updating_assets = {} try: + LOGGER.audit(f'uploading auxiliary file: {job_obj}') s3_url = self.__s3.upload(job_obj, self.__staging_bucket, f'{self.__collection_id}/{self.__collection_id}:{sample_stac_item.id}', self.__delete_files) updating_assets[os.path.basename(s3_url)] = s3_url uploading_current_granule_stac = f'{s3_url}.stac.json' diff --git a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py index 66543e5..5d803d0 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py @@ -1,3 +1,18 @@ +import logging + +# Define the new log level AUDIT +AUDIT_LEVEL = 60 +logging.addLevelName(AUDIT_LEVEL, "AUDIT") + + +# Define a custom method for logging at AUDIT level +def audit(self, message, *args, **kwargs): + if self.isEnabledFor(AUDIT_LEVEL): + self._log(AUDIT_LEVEL, message, args, **kwargs) + + +logging.Logger.audit = audit + import os from abc import ABC, abstractmethod diff --git a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py index 409a41b..2f7d8e7 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py @@ -64,7 +64,7 @@ def execute_job(self, each_child, lock) -> bool: uploading_current_granule_stac = None for asset_type, asset_hrefs in current_assets.items(): for each_asset_href in asset_hrefs: - LOGGER.debug(f'uploading {asset_type}, {each_asset_href}') + LOGGER.audit(f'uploading {asset_type}: {each_asset_href}') s3_url = self.__s3.upload(each_asset_href, self.__staging_bucket, f'{self.__collection_id}/{self.__collection_id}:{current_granule_id}', self.__delete_files) diff --git a/tests/integration_tests/.env.tpl b/tests/integration_tests/.env.tpl new file mode 100644 index 0000000..5bb2ddf --- /dev/null +++ b/tests/integration_tests/.env.tpl @@ -0,0 +1,6 @@ +USERNAME=base64-encoded-str +PASSWORD=base64-encoded-str +CLIENT_ID=7a1fglm2d54eoggj13lccivp25 +COGNITO_URL=https://cognito-idp.us-west-2.amazonaws.com +UNITY_URL=https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev +UNITY_STAGE=sbx-uds-dapa diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration_tests/test_docker_entry.py b/tests/integration_tests/test_docker_entry.py new file mode 100644 index 0000000..df328b3 --- /dev/null +++ b/tests/integration_tests/test_docker_entry.py @@ -0,0 +1,375 @@ +import logging + +logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +import math +from unittest.mock import patch, MagicMock +import json +import os +import tempfile +from glob import glob +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from cumulus_lambda_functions.docker_entrypoint.__main__ import choose_process +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerEntry(TestCase): + def test_01_search_part_01(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030' + os.environ['LIMITS'] = '4000' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + search_result_str = choose_process() + search_result = json.loads(search_result_str) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 4000, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 4000, f'wrong length. not unique') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + self.assertEqual(sorted(json.dumps(FileUtils.read_json(os.environ['OUTPUT_FILE']))), sorted(search_result_str), f'not identical result') + return + + def test_01_search_part_02(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'SNDR_SNPP_ATMS_L1A___1' + os.environ['LIMITS'] = '100' + os.environ['DATE_FROM'] = '2016-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2016-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + print(search_result) + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 20, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result),20, f'wrong length. not unique') + return + + def test_01_search_part_03(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'L0_SNPP_ATMS_SCIENCE___1' + os.environ['LIMITS'] = '-1' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 4381, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 4381, f'wrong length. not unique') + return + + def test_01_search_part_04(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'L0_SNPP_ATMS_SCIENCE___1' + os.environ['LIMITS'] = '347' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 347, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 347, f'wrong length. not unique') + return + + def test_01_search_part_05(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['API_PREFIX'] = 'sbx-uds-2-dapa' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['COLLECTION_ID'] = 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030' + os.environ['LIMITS'] = '37' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2024-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 37, f'wrong length') + return + + def test_01_1_search_cmr_part_01(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'C1996881146-POCLOUD' # 'C1666605425-PODAAC' # C1996881146-POCLOUD + os.environ['LIMITS'] = '2120' + os.environ['DATE_FROM'] = '2002-06-01T12:06:00.000Z' + os.environ['DATE_TO'] = '2011-10-04T06:51:45.000Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'CMR' + os.environ['CMR_BASE_URL'] = 'https://cmr.earthdata.nasa.gov' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 2120, f'wrong length') + return + + def test_01_1_search_cmr_part_02(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'C1996881146-POCLOUD' # 'C1666605425-PODAAC' # C1996881146-POCLOUD + os.environ['LIMITS'] = '23' + os.environ['DATE_FROM'] = '2002-06-01T12:06:00.000Z' + os.environ['DATE_TO'] = '2011-10-04T06:51:45.000Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'CMR' + os.environ['CMR_BASE_URL'] = 'https://cmr.earthdata.nasa.gov' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 23, f'wrong length') + return + + def test_04_catalog(self): + upload_result = [{'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9', 'assets': {'metadata': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas'}, 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc'}}}] + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['UPLOADED_FILES_JSON'] = json.dumps(upload_result) + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertEqual('registered', catalog_result, 'wrong status') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_04_catalog_from_file(self): + upload_result = [{'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9', 'assets': {'metadata': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas'}, 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc'}}}] + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['DELAY_SECOND'] = '35' + os.environ['REPEAT_TIMES'] = '3' + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], 'registered', f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + + status_result = catalog_result['status_result'] + + self.assertTrue('cataloged' in status_result, f'missing cataloged') + self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + # Example result: {'cataloging_request_status': 'registered', 'status_result': {'cataloged': False, 'missing_granules': ['NEW_COLLECTION_EXAMPLE_L1B___9:test_file01'], 'registered_granules': []}} + return + + def test_04_catalog_from_file_item_collection(self): + upload_result = {'type': 'FeatureCollection', 'features': [ + {'type': 'Feature', 'stac_version': '1.0.0', 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', 'end_datetime': '2016-01-31T19:59:59.991043Z', + 'created': '2016-02-01T02:45:59.639000Z', 'updated': '2022-03-23T15:48:21.578000Z', + 'datetime': '1970-01-01T00:00:00Z'}, 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, + 'links': [], 'assets': { + 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, 'bbox': [0.0, 0.0, 0.0, 0.0], 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + # os.environ['DELAY_SECOND'] = '5' + # os.environ['REPEAT_TIMES'] = '3' + os.environ['DELAY_SECOND'] = '35' + os.environ['REPEAT_TIMES'] = '3' + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], 'registered', f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + # TODO update this after it is deployed to MCP Test + status_result = catalog_result['status_result'] + + self.assertTrue('cataloged' in status_result, f'missing cataloged') + self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + return + + def test_04_catalog_from_file_item_collection_large(self): + upload_result = FileUtils.read_json('./stage-out-results.json') + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ[Constants.CLIENT_ID] = '71g0c73jl77gsqhtlfg2ht388c' # MCP Dev + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['CHUNK_SIZE'] = '250' + # os.environ['DELAY_SECOND'] = '5' + # os.environ['REPEAT_TIMES'] = '3' + + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue(isinstance(catalog_result, list), f'catalog_result is not list. {catalog_result}') + self.assertEqual(len(catalog_result), math.ceil(len(upload_result['features']) / 250), f'mismatched catalog_result count') + + catalog_result = catalog_result[0] + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], {'message': 'processing'}, f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + + status_result = catalog_result['status_result'] + # TODO disabling this as we are not waiting for them to be registered. + # self.assertTrue('cataloged' in status_result, f'missing cataloged') + # self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + # self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + # self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + return diff --git a/tests/integration_tests/test_docker_stage_in.py b/tests/integration_tests/test_docker_stage_in.py new file mode 100644 index 0000000..6336640 --- /dev/null +++ b/tests/integration_tests/test_docker_stage_in.py @@ -0,0 +1,1928 @@ +import os + +from mdps_ds_lib.stage_in_out.download_granules_factory import DownloadGranulesFactory + +os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES' +os.environ['no_proxy'] = '*' +import logging + +import requests + +logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +import math +from unittest.mock import patch, MagicMock +import json +import tempfile +from glob import glob +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerStageIn(TestCase): + def test_02_download(self): + granule_json = '{"numberMatched": 20, "numberReturned": 20, "stac_version": "1.0.0", "type": "FeatureCollection", "links": [{"rel": "self", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}, {"rel": "root", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com"}, {"rel": "next", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100"}, {"rel": "prev", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}], "features": [{"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", "properties": {"start_datetime": "2016-01-14T09:54:00Z", "end_datetime": "2016-01-14T10:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:39.830000Z", "datetime": "2022-08-15T06:26:37.029000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", "properties": {"start_datetime": "2016-01-14T10:36:00Z", "end_datetime": "2016-01-14T10:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.078000Z", "datetime": "2022-08-15T06:26:19.333000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", "properties": {"start_datetime": "2016-01-14T10:24:00Z", "end_datetime": "2016-01-14T10:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.068000Z", "datetime": "2022-08-15T06:26:18.641000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", "properties": {"start_datetime": "2016-01-14T11:36:00Z", "end_datetime": "2016-01-14T11:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.060000Z", "datetime": "2022-08-15T06:26:19.698000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", "properties": {"start_datetime": "2016-01-14T10:12:00Z", "end_datetime": "2016-01-14T10:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.050000Z", "datetime": "2022-08-15T06:26:19.491000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.16", "properties": {"start_datetime": "2016-01-14T11:24:00Z", "end_datetime": "2016-01-14T11:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.917000Z", "datetime": "2022-08-15T06:26:19.027000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.17", "properties": {"start_datetime": "2016-01-14T11:30:00Z", "end_datetime": "2016-01-14T11:36:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.907000Z", "datetime": "2022-08-15T06:26:19.042000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.10", "properties": {"start_datetime": "2016-01-14T10:48:00Z", "end_datetime": "2016-01-14T10:54:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.446000Z", "datetime": "2022-08-15T06:26:18.730000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.14", "properties": {"start_datetime": "2016-01-14T11:12:00Z", "end_datetime": "2016-01-14T11:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.354000Z", "datetime": "2022-08-15T06:26:17.758000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.12", "properties": {"start_datetime": "2016-01-14T11:00:00Z", "end_datetime": "2016-01-14T11:06:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.344000Z", "datetime": "2022-08-15T06:26:17.938000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.09", "properties": {"start_datetime": "2016-01-14T10:42:00Z", "end_datetime": "2016-01-14T10:48:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:24.910000Z", "datetime": "2022-08-15T06:26:20.688000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.20", "properties": {"start_datetime": "2016-01-14T11:48:00Z", "end_datetime": "2016-01-14T11:54:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.929000Z", "datetime": "2022-08-15T06:26:19.091000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.15", "properties": {"start_datetime": "2016-01-14T11:18:00Z", "end_datetime": "2016-01-14T11:24:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.732000Z", "datetime": "2022-08-15T06:26:19.282000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.07", "properties": {"start_datetime": "2016-01-14T10:30:00Z", "end_datetime": "2016-01-14T10:36:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.371000Z", "datetime": "2022-08-15T06:26:19.047000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.19", "properties": {"start_datetime": "2016-01-14T11:42:00Z", "end_datetime": "2016-01-14T11:48:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.268000Z", "datetime": "2022-08-15T06:26:18.576000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.03", "properties": {"start_datetime": "2016-01-14T10:06:00Z", "end_datetime": "2016-01-14T10:12:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.930000Z", "datetime": "2022-08-15T06:26:17.714000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.11", "properties": {"start_datetime": "2016-01-14T10:54:00Z", "end_datetime": "2016-01-14T11:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.863000Z", "datetime": "2022-08-15T06:26:17.648000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.05", "properties": {"start_datetime": "2016-01-14T10:18:00Z", "end_datetime": "2016-01-14T10:24:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.649000Z", "datetime": "2022-08-15T06:26:18.060000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.13", "properties": {"start_datetime": "2016-01-14T11:06:00Z", "end_datetime": "2016-01-14T11:12:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.277000Z", "datetime": "2022-08-15T06:26:18.090000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.02", "properties": {"start_datetime": "2016-01-14T10:00:00Z", "end_datetime": "2016-01-14T10:06:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.169000Z", "datetime": "2022-08-15T06:26:17.466000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}]}' + granule_json = json.loads(granule_json) + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac(self): + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-01T21:00:00.000Z", "start_datetime": "2002-06-01T21:00:00.000Z", "end_datetime": "2002-06-02T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106835-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-02T21:00:00.000Z", "start_datetime": "2002-06-02T21:00:00.000Z", "end_datetime": "2002-06-03T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106890-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-03T21:00:00.000Z", "start_datetime": "2002-06-03T21:00:00.000Z", "end_datetime": "2002-06-04T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106962-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-04T21:00:00.000Z", "start_datetime": "2002-06-04T21:00:00.000Z", "end_datetime": "2002-06-05T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106862-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.umm_json"}]}]}' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'metadata' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['metadata']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['metadata']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_from_url(self): + os.environ['STAC_JSON'] = 'https://cmr.earthdata.nasa.gov/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&page_num=1&page_size=1' + granule_json = requests.get(os.environ['STAC_JSON']) + granule_json = json.loads(granule_json.text) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + # os.environ['STAC_JSON'] = json.dumps(granule_json) + + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'data,metadata' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + print(download_result) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * len(os.environ['DOWNLOADING_KEYS'].split(',')) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match: {glob(os.path.join(tmp_dir_name, "*"))} v. {download_result["features"]}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + print(download_result) + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac__from_file(self): + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-01T21:00:00.000Z", "start_datetime": "2002-06-01T21:00:00.000Z", "end_datetime": "2002-06-02T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106835-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-02T21:00:00.000Z", "start_datetime": "2002-06-02T21:00:00.000Z", "end_datetime": "2002-06-03T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106890-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-03T21:00:00.000Z", "start_datetime": "2002-06-03T21:00:00.000Z", "end_datetime": "2002-06-04T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106962-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-04T21:00:00.000Z", "start_datetime": "2002-06-04T21:00:00.000Z", "end_datetime": "2002-06-05T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106862-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.umm_json"}]}]}' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'metadata' + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['metadata']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['metadata']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_502__from_file(self): + granule_json = '''{ + "type": "FeatureCollection", + "stac_version": "1.0.0", + "numberMatched": 242, + "numberReturned": 10, + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&page_size=10&temporal%5B%5D=2021-02-01T00%3A00%3A00Z%2C2021-02-02T00%3A00%3A00Z&page_num=1" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "next", + "body": { + "collection_concept_id": "C2011289787-GES_DISC", + "page_num": "2", + "page_size": "10", + "temporal": [ + "2021-02-01T00:00:00Z,2021-02-02T00:00:00Z" + ], + "temporal[]": "2021-02-01T00:00:00Z,2021-02-02T00:00:00Z" + }, + "method": "POST", + "merge": true, + "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac" + } + ], + "context": { + "returned": 10, + "limit": 1000000, + "matched": 242 + }, + "features": [ + { + "properties": { + "datetime": "2021-01-31T23:54:00.000Z", + "start_datetime": "2021-01-31T23:54:00.000Z", + "end_datetime": "2021-02-01T00:00:00.000Z" + }, + "bbox": [ + -164.22, + -59.19, + -127.42, + -34.4 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/031/SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/031/SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -127.42, + -53.49 + ], + [ + -140.59, + -34.4 + ], + [ + -164.22, + -38.4 + ], + [ + -161.84, + -59.19 + ], + [ + -127.42, + -53.49 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327589-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:00:00.000Z", + "start_datetime": "2021-02-01T00:00:00.000Z", + "end_datetime": "2021-02-01T00:06:00.000Z" + }, + "bbox": [ + -167.61, + -38.28, + -140.62, + -14.06 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -140.62, + -34.28 + ], + [ + -147.98, + -14.06 + ], + [ + -167.61, + -17.44 + ], + [ + -164.31, + -38.28 + ], + [ + -140.62, + -34.28 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327600-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:06:00.000Z", + "start_datetime": "2021-02-01T00:06:00.000Z", + "end_datetime": "2021-02-01T00:12:00.000Z" + }, + "bbox": [ + -171.93, + -17.31, + -148.0, + 6.74 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -148.0, + -13.93 + ], + [ + -153.06, + 6.74 + ], + [ + -171.93, + 3.48 + ], + [ + -167.7, + -17.31 + ], + [ + -148.0, + -13.93 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327624-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:12:00.000Z", + "start_datetime": "2021-02-01T00:12:00.000Z", + "end_datetime": "2021-02-01T00:18:00.000Z" + }, + "bbox": [ + -177.83, + 3.6, + -153.07, + 27.71 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -153.07, + 6.87 + ], + [ + -156.97, + 27.71 + ], + [ + -177.83, + 24.15 + ], + [ + -172.02, + 3.6 + ], + [ + -153.07, + 6.87 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327611-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:18:00.000Z", + "start_datetime": "2021-02-01T00:18:00.000Z", + "end_datetime": "2021-02-01T00:24:00.000Z" + }, + "bbox": [ + 172.75, + 24.27, + -156.97, + 48.71 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -156.97, + 27.84 + ], + [ + -160.07, + 48.71 + ], + [ + 172.75, + 44.19 + ], + [ + -177.94, + 24.27 + ], + [ + -156.97, + 27.84 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327811-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:24:00.000Z", + "start_datetime": "2021-02-01T00:24:00.000Z", + "end_datetime": "2021-02-01T00:30:00.000Z" + }, + "bbox": [ + 152.97, + 44.29, + -160.05, + 69.67 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -160.05, + 48.84 + ], + [ + -161.92, + 69.67 + ], + [ + 152.97, + 62.28 + ], + [ + 172.6, + 44.29 + ], + [ + -160.05, + 48.84 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327763-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:30:00.000Z", + "start_datetime": "2021-02-01T00:30:00.000Z", + "end_datetime": "2021-02-01T00:36:00.000Z" + }, + "bbox": [ + -180.0, + 62.35, + 180.0, + 90.0 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -161.83, + 69.79 + ], + [ + -45.22, + 88.83 + ], + [ + 101.33, + 71.69 + ], + [ + 152.69, + 62.35 + ], + [ + -161.83, + 69.79 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327782-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:36:00.000Z", + "start_datetime": "2021-02-01T00:36:00.000Z", + "end_datetime": "2021-02-01T00:42:00.000Z" + }, + "bbox": [ + -41.63, + 61.83, + 100.91, + 89.26695841076085 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -41.63, + 88.73 + ], + [ + 6.42, + 68.41 + ], + [ + 50.76, + 61.83 + ], + [ + 100.91, + 71.65 + ], + [ + -41.63, + 88.73 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327806-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:42:00.000Z", + "start_datetime": "2021-02-01T00:42:00.000Z", + "end_datetime": "2021-02-01T00:48:00.000Z" + }, + "bbox": [ + 4.45, + 43.65, + 50.62, + 68.30202151196218 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 6.29, + 68.29 + ], + [ + 4.45, + 47.46 + ], + [ + 31.46, + 43.65 + ], + [ + 50.62, + 61.71 + ], + [ + 6.29, + 68.29 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327773-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:48:00.000Z", + "start_datetime": "2021-02-01T00:48:00.000Z", + "end_datetime": "2021-02-01T00:54:00.000Z" + }, + "bbox": [ + 1.29, + 23.6, + 31.41, + 47.33 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.35, + 47.33 + ], + [ + 1.29, + 26.46 + ], + [ + 22.18, + 23.6 + ], + [ + 31.41, + 43.52 + ], + [ + 4.35, + 47.33 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327751-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.umm_json" + } + ] + } + ] + }''' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'data' + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_error(self): # TODO update this later + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601161248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00414.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601172624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00415.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601190536-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00416.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601204344-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00417.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601222152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00418.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602000000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00419.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602013912-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00420.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602031720-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00421.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602045528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00422.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602063440-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00423.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602081248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00424.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602095056-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00425.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602112904-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00426.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602130816-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00427.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602144624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00428.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602162432-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00429.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602180240-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00430.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602194152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00431.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602212000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00432.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602225808-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00433.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603003616-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00434.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603021528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00435.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603035336-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00436.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}]}' + # granule_json = [{"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601161248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00414.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601172624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00415.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601190536-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00416.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601204344-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00417.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601222152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00418.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602000000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00419.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602013912-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00420.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602031720-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00421.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602045528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00422.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602063440-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00423.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602081248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00424.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602095056-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00425.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602112904-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00426.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602130816-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00427.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602144624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00428.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602162432-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00429.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602180240-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00430.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602194152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00431.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602212000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00432.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602225808-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00433.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603003616-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00434.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603021528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00435.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603035336-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00436.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}] + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + # TODO this is downloading a login page HTML + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + print(glob(os.path.join(tmp_dir_name, '*'))) + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_dapa_url(self): + dapa_url = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000/items?limit=37&offset=0&datetime=1990-01-14T08:00:00Z/2024-01-14T11:59:59Z' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['API_PREFIX'] = 'sbx-uds-2-dapa' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['STAC_AUTH_TYPE'] = 'UNITY' + + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['STAC_JSON'] = dapa_url + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + print(download_result) + expected_downloaded_feature = [{'type': 'Feature', 'stac_version': '1.0.0', + 'id': 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000:test_file05', + 'properties': {'tag': '#sample', 'c_data1': [1, 10, 100, 1000], + 'c_data2': [False, True, True, False, True], + 'c_data3': ['Bellman Ford'], + 'datetime': '2024-01-31T22:34:06.556000Z', + 'start_datetime': '2016-01-31T18:00:00.009000Z', + 'end_datetime': '2016-01-31T19:59:59.991000Z', + 'created': '1970-01-01T00:00:00Z', + 'updated': '2024-01-31T22:34:49.583000Z', + 'status': 'completed', 'provider': 'unity'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, + 'links': [{'rel': 'collection', 'href': '.'}], 'assets': { + 'test_file05.cmr.xml': {'href': './test_file05.cmr.xml', 'title': 'test_file05.cmr.xml', + 'description': 'size=1768;checksumType=md5;checksum=29d1b69df5587d7ee0a945250adfd16f;', + 'roles': ['metadata']}, + 'test_file05.nc.stac.json': {'href': './test_file05.nc.stac.json', + 'title': 'test_file05.nc.stac.json', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['metadata']}, + 'test_file05.nc.cas': {'href': './test_file05.nc.cas', 'title': 'test_file05.nc.cas', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['metadata']}, + 'test_file05.data.stac.json': {'href': './test_file05.data.stac.json', + 'title': 'test_file05.data.stac.json', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['data']}}, 'bbox': [-180.0, -90.0, 180.0, 90.0], + 'stac_extensions': [], + 'collection': 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000'}] + + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['test_file05.data.stac.json']['href'] for k in download_result]) + for each_granule in zip(expected_downloaded_feature, download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['test_file05.data.stac.json']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file(self): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + print(downloaded_file_hrefs) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + print() + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file_large_github_data_file(self): + granule_json = ''' + { + "numberMatched": 20, + "numberReturned": 20, + "stac_version": "1.0.0", + "type": "FeatureCollection", + "links": [ + { + "rel": "self", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + }, + { + "rel": "root", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com" + }, + { + "rel": "next", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100" + }, + { + "rel": "prev", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + } + ], + "features": [ + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", + "properties": { + "start_datetime": "2016-01-14T09:54:00Z", + "end_datetime": "2016-01-14T10:00:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:39.830000Z", + "datetime": "2022-08-15T06:26:37.029000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "data": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", + "title": "SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", + "description": "SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc" + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", + "properties": { + "start_datetime": "2016-01-14T10:36:00Z", + "end_datetime": "2016-01-14T10:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.078000Z", + "datetime": "2022-08-15T06:26:19.333000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "data": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", + "title": "SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", + "description": "SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc" + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + } + ] + } + ''' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + with tempfile.TemporaryDirectory() as tmp_dir_name: + # tmp_dir_name = '/tmp/unity-ds' + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 40 * 2**20, f'empty file: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + @patch('requests.get') + def test_02_download__from_file_with_http(self, mock_requests): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + # granule_json = json.loads(granule_json) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.return_value.content.decode.return_value = granule_json + + # mock_response.content = granule_json.encode() + # mock_response.content.return_value = granule_json.encode() + # mock_response.json.return_value = json.loads(granule_json) + # specify the return value of the get() method + mock_requests.return_value.get.return_value = mock_response + mock_requests.return_value.content.decode.return_value = granule_json + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + os.environ['STAC_JSON'] = 'https://example.com/get_feature_collection' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + granule_json = json.loads(granule_json) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + @patch('requests.get') + def test_02_download__from_file_with_http(self, mock_requests): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + # granule_json = json.loads(granule_json) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.return_value.content.decode.return_value = granule_json + + # mock_response.content = granule_json.encode() + # mock_response.content.return_value = granule_json.encode() + # mock_response.json.return_value = json.loads(granule_json) + # specify the return value of the get() method + mock_requests.return_value.get.return_value = mock_response + mock_requests.return_value.content.decode.return_value = granule_json + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + os.environ['STAC_JSON'] = 'https://example.com/get_feature_collection' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + granule_json = json.loads(granule_json) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file_large(self): + granule_json = FileUtils.read_json('./stage-in.json') + # granule_json['features'] = granule_json['features'][0:5] + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + # os.environ['PARALLEL_COUNT'] = '5' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + print(tmp_dir_name) + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + print(len(download_result['features'])) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_http(self): + granule_json = '{"numberMatched": 20, "numberReturned": 20, "stac_version": "1.0.0", "type": "FeatureCollection", "links": [{"rel": "self", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}, {"rel": "root", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com"}, {"rel": "next", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100"}, {"rel": "prev", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}], "features": [{"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", "properties": {"start_datetime": "2016-01-14T09:54:00Z", "end_datetime": "2016-01-14T10:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:39.830000Z", "datetime": "2022-08-15T06:26:37.029000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/README.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", "properties": {"start_datetime": "2016-01-14T10:36:00Z", "end_datetime": "2016-01-14T10:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.078000Z", "datetime": "2022-08-15T06:26:19.333000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CHANGELOG.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", "properties": {"start_datetime": "2016-01-14T10:24:00Z", "end_datetime": "2016-01-14T10:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.068000Z", "datetime": "2022-08-15T06:26:18.641000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CODE_OF_CONDUCT.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", "properties": {"start_datetime": "2016-01-14T11:36:00Z", "end_datetime": "2016-01-14T11:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.060000Z", "datetime": "2022-08-15T06:26:19.698000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CONTRIBUTING.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", "properties": {"start_datetime": "2016-01-14T10:12:00Z", "end_datetime": "2016-01-14T10:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.050000Z", "datetime": "2022-08-15T06:26:19.491000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/LICENSE", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}]}' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + os.environ['PARALLEL_COUNT'] = '3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + os.environ['DOWNLOADING_KEYS'] = 'data' + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]} v. {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_http_with_role(self): + granule_json = '''{ + "numberMatched": 20, + "numberReturned": 20, + "stac_version": "1.0.0", + "type": "FeatureCollection", + "links": [ + { + "rel": "self", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + }, + { + "rel": "root", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com" + }, + { + "rel": "next", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100" + }, + { + "rel": "prev", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + } + ], + "features": [ + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", + "properties": { + "start_datetime": "2016-01-14T09:54:00Z", + "end_datetime": "2016-01-14T10:00:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:39.830000Z", + "datetime": "2022-08-15T06:26:37.029000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.01.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/README.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", + "properties": { + "start_datetime": "2016-01-14T10:36:00Z", + "end_datetime": "2016-01-14T10:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.078000Z", + "datetime": "2022-08-15T06:26:19.333000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.08.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CHANGELOG.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", + "properties": { + "start_datetime": "2016-01-14T10:24:00Z", + "end_datetime": "2016-01-14T10:30:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.068000Z", + "datetime": "2022-08-15T06:26:18.641000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.06.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CODE_OF_CONDUCT.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", + "properties": { + "start_datetime": "2016-01-14T11:36:00Z", + "end_datetime": "2016-01-14T11:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.060000Z", + "datetime": "2022-08-15T06:26:19.698000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.18.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CONTRIBUTING.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", + "properties": { + "start_datetime": "2016-01-14T10:12:00Z", + "end_datetime": "2016-01-14T10:18:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.050000Z", + "datetime": "2022-08-15T06:26:19.491000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.04.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/LICENSE", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + } + ] + }''' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + os.environ['PARALLEL_COUNT'] = '3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + os.environ['DOWNLOADING_ROLES'] = 'data' + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]} v. {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([list(k['assets'].values())[0]['href'] for k in download_result]) + # print(granule_json['features'], download_result) + print(FileUtils.read_json(f'{downloading_dir}/downloaded_feature_collection.json')) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = [k['href'] for k in each_granule[0]['assets'].values() if 'data' in k['roles']] + remote_filename = os.path.basename(remote_filename[0]) + # self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, f'mismatched: {remote_filename}') + self.assertTrue(remote_filename in downloaded_file_hrefs, f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + # self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py new file mode 100644 index 0000000..a1b92b5 --- /dev/null +++ b/tests/integration_tests/test_docker_stage_out.py @@ -0,0 +1,1334 @@ +import logging +logging.basicConfig(level=20, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +from datetime import datetime + +from mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3 import \ + UploadGranulesByCompleteCatalogS3 + +from mdps_ds_lib.lib.aws.aws_s3 import AwsS3 +from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory + + +import json +import os +import tempfile +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerStageOut(TestCase): + def setUp(self) -> None: + super().setUp() + self.tenant = 'UDS_MY_LOCAL_ARCHIVE_TEST' # 'uds_local_test' # 'uds_sandbox' + self.tenant_venue = 'DEV' # 'DEV1' # 'dev' + self.collection_name = 'UDS_UNIT_COLLECTION' # 'uds_collection' # 'sbx_collection' + self.collection_version = '24.08.29.09.00'.replace('.', '') # '2402011200' + + def not_in_used_test_03_upload(self): + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-test-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + os.environ['GRANULES_UPLOAD_TYPE'] = 'S3' + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = tmp_dir_name + with open(os.path.join(tmp_dir_name, 'test_file01.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + print(upload_result) + self.assertEqual(1, len(upload_result), 'wrong length of upload_result features') + upload_result = upload_result[0] + self.assertTrue('assets' in upload_result, 'missing assets') + self.assertTrue('metadata' in upload_result['assets'], 'missing assets#metadata') + self.assertTrue('href' in upload_result['assets']['metadata'], 'missing assets#metadata#href') + self.assertTrue( + upload_result['assets']['metadata']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue('data' in upload_result['assets'], 'missing assets#data') + self.assertTrue('href' in upload_result['assets']['data'], 'missing assets#data#href') + self.assertTrue(upload_result['assets']['data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def not_in_used_test_03_upload_catalog(self): + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-test-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + os.environ['GRANULES_UPLOAD_TYPE'] = 'CATALOG_S3' + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + with open(os.path.join(tmp_dir_name, 'test_file01.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id='NA', + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + collection='NA', + assets={ + 'data': Asset(os.path.join(tmp_dir_name, 'test_file01.nc'), title='main data'), + 'metadata__cas': Asset(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), title='metadata cas'), + 'metadata__stac': Asset(os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), title='metadata stac'), + }) + with open(os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + catalog.add_link(Link('child', os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), 'application/json')) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + print(upload_result) + self.assertEqual(1, len(upload_result), 'wrong length of upload_result features') + upload_result = upload_result[0] + self.assertTrue('assets' in upload_result, 'missing assets') + self.assertTrue('metadata__cas' in upload_result['assets'], 'missing assets#metadata__cas') + self.assertTrue('href' in upload_result['assets']['metadata__cas'], 'missing assets#metadata__cas#href') + self.assertTrue( + upload_result['assets']['metadata__cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue('data' in upload_result['assets'], 'missing assets#data') + self.assertTrue('href' in upload_result['assets']['data'], 'missing assets#data#href') + self.assertTrue(upload_result['assets']['data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_03_upload_complete_catalog(self): + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['RESULT_PATH_PREFIX'] = 'integration_test/stage_out' + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_02_upload_complete_catalog(self): + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{UploadGranulesByCompleteCatalogS3.DEFAULT_RESULT_PATH_PREFIX}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_03_upload_auxiliary_files(self): + temp_collection_id = f'URN:NASA:UNITY:{self.tenant}:{self.tenant_venue}:{self.collection_name}___{self.collection_version}' + os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_AUXILIARY_FILE_AS_GRANULE' + os.environ['COLLECTION_ID'] = temp_collection_id + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['RESULT_PATH_PREFIX'] = 'stage_out' + os.environ['PARALLEL_COUNT'] = '1' + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + real_base_dir = os.path.join(tmp_dir_name, 'auxiliary_base') + FileUtils.mk_dir_p(real_base_dir) + os.environ['BASE_DIRECTORY'] = real_base_dir + + with open(os.path.join(tmp_dir_name, 'excluding_file.json'), 'w') as ff: + ff.write('{"message": "excluding file"}') + with open(os.path.join(real_base_dir, 'test_file_0.json'), 'w') as ff: + ff.write('{"message": "some file at root"}') + sub_folders = [ + os.path.join(real_base_dir, 'son'), + os.path.join(real_base_dir, 'daughter'), + os.path.join(real_base_dir, os.path.join('son', 'grandson')), + os.path.join(real_base_dir, os.path.join('son', 'granddaughter')), + os.path.join(real_base_dir, os.path.join('daughter', 'grandson')), + os.path.join(real_base_dir, os.path.join('daughter', 'granddaughter')), + ] + for i, each_sub_folder in enumerate(sub_folders): + FileUtils.mk_dir_p(each_sub_folder) + with open(os.path.join(each_sub_folder, f'test_file_{i}.json'), 'w') as ff: + ff.write(json.dumps({"message": f"some file at {each_sub_folder}"})) + FileUtils.mk_dir_p(os.path.join(real_base_dir, 'nephew')) # should not throw error for empty folders + FileUtils.mk_dir_p(os.path.join(real_base_dir, os.path.join('nephew', 'grandson'))) # should not throw error for empty folders + total_files = len(sub_folders) + 1 + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + self.assertTrue(f'{result_key}.stac.json' in upload_result['assets'], f'missing assets#metadata asset: test_file_0.json') + self.assertTrue('href' in upload_result['assets'][f'{result_key}.stac.json'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key}.stac.json']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/'), f"wrong HREF (no S3?): upload_result['assets'][f'{result_key}.stac.json']['href']") + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_upload_complete_catalog_invalid_bucket(self): + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'invalid_bucket' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), 0, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), total_files, f'wrong length: {failed_feature_collection}') + + upload_result = failed_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('properties' in upload_result, 'missing properties') + self.assertTrue('upload_error' in upload_result['properties'], 'missing upload_error') + self.assertTrue('An error occurred (NoSuchBucket)'in upload_result['properties']['upload_error'], f"wrong upload_error: {upload_result['properties']['upload_error']}") + + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + return + + def test_03_upload_complete_catalog_missing_data(self): + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'invalid_bucket' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data1']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), 0, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), total_files, f'wrong length: {failed_feature_collection}') + + upload_result = failed_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('properties' in upload_result, 'missing properties') + self.assertTrue('upload_error' in upload_result['properties'], 'missing upload_error') + self.assertTrue('missing "data" in assets'in upload_result['properties']['upload_error'], f"wrong upload_error: {upload_result['properties']['upload_error']}") + + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + return From 00d6bded6edaeb42a0d554103e00302296781da2 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 14 Oct 2024 10:31:23 -0700 Subject: [PATCH 2/5] chore: update outdated tests --- tests/integration_tests/test_docker_entry.py | 26 +++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/integration_tests/test_docker_entry.py b/tests/integration_tests/test_docker_entry.py index df328b3..c5b396d 100644 --- a/tests/integration_tests/test_docker_entry.py +++ b/tests/integration_tests/test_docker_entry.py @@ -1,5 +1,8 @@ import logging +from mdps_ds_lib.stage_in_out.catalog_granules_factory import CatalogGranulesFactory +from mdps_ds_lib.stage_in_out.search_granules_factory import SearchGranulesFactory + logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") import math @@ -42,7 +45,7 @@ def test_01_search_part_01(self): argv.append('SEARCH') with tempfile.TemporaryDirectory() as tmp_dir_name: os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - search_result_str = choose_process() + search_result_str = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result_str) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -73,7 +76,7 @@ def test_01_search_part_02(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() print(search_result) search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') @@ -103,7 +106,7 @@ def test_01_search_part_03(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -132,7 +135,7 @@ def test_01_search_part_04(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -161,7 +164,7 @@ def test_01_search_part_05(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -188,7 +191,7 @@ def test_01_1_search_cmr_part_01(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -215,7 +218,8 @@ def test_01_1_search_cmr_part_02(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() + search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -239,7 +243,7 @@ def test_04_catalog(self): argv.append('CATALOG') with tempfile.TemporaryDirectory() as tmp_dir_name: os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertEqual('registered', catalog_result, 'wrong status') self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') @@ -266,7 +270,7 @@ def test_04_catalog_from_file(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') self.assertTrue('status_result' in catalog_result, f'missing status_result') @@ -317,7 +321,7 @@ def test_04_catalog_from_file_item_collection(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') self.assertTrue('status_result' in catalog_result, f'missing status_result') @@ -355,7 +359,7 @@ def test_04_catalog_from_file_item_collection_large(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue(isinstance(catalog_result, list), f'catalog_result is not list. {catalog_result}') self.assertEqual(len(catalog_result), math.ceil(len(upload_result['features']) / 250), f'mismatched catalog_result count') From a51314b0f34e70a66368f45d04bcc582c76fda57 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 21 Oct 2024 12:12:14 -0700 Subject: [PATCH 3/5] fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val --- mdps_ds_lib/stage_in_out/upload_granules_abstract.py | 2 ++ tests/integration_tests/test_docker_stage_out.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py index 5d803d0..ceedbee 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py @@ -47,6 +47,8 @@ def _set_props_from_env(self): self._collection_id = os.environ.get(self.COLLECTION_ID_KEY) self._staging_bucket = os.environ.get(self.STAGING_BUCKET_KEY) self._result_path_prefix = os.environ.get(self.RESULT_PATH_PREFIX, self.DEFAULT_RESULT_PATH_PREFIX) + if self._result_path_prefix is None or self._result_path_prefix.strip() == '': + self._result_path_prefix = self.DEFAULT_RESULT_PATH_PREFIX self._result_path_prefix = self._result_path_prefix[:-1] if self._result_path_prefix.endswith('/') else self._result_path_prefix self._result_path_prefix = self._result_path_prefix[1:] if self._result_path_prefix.startswith('/') else self._result_path_prefix diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py index a1b92b5..7ee0298 100644 --- a/tests/integration_tests/test_docker_stage_out.py +++ b/tests/integration_tests/test_docker_stage_out.py @@ -1,4 +1,5 @@ import logging + logging.basicConfig(level=20, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") from datetime import datetime @@ -30,7 +31,7 @@ def setUp(self) -> None: self.tenant = 'UDS_MY_LOCAL_ARCHIVE_TEST' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' self.collection_name = 'UDS_UNIT_COLLECTION' # 'uds_collection' # 'sbx_collection' - self.collection_version = '24.08.29.09.00'.replace('.', '') # '2402011200' + self.collection_version = '24.10.21.12.00'.replace('.', '') # '2402011200' def not_in_used_test_03_upload(self): os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' @@ -791,7 +792,7 @@ def test_03_03_upload_auxiliary_files(self): os.environ['COLLECTION_ID'] = temp_collection_id os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' os.environ['VERIFY_SSL'] = 'FALSE' - os.environ['RESULT_PATH_PREFIX'] = 'stage_out' + os.environ['RESULT_PATH_PREFIX'] = '' os.environ['PARALLEL_COUNT'] = '1' if len(argv) > 1: @@ -882,7 +883,7 @@ def test_03_03_upload_auxiliary_files(self): """ s3 = AwsS3() s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], - f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + f"stage_out/successful_features_{starting_time}", )] s3_keys = sorted(s3_keys) print(f's3_keys: {s3_keys}') From 494505d64900c476e46e5f76a88379a74127a4a9 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 7 Apr 2025 12:19:11 -0500 Subject: [PATCH 4/5] fix: allowing optional original stac item --- .../lib/cumulus_stac/item_transformer.py | 37 +++++++++++-------- .../lib/cumulus_stac/test_item_transformer.py | 3 ++ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/mdps_ds_lib/lib/cumulus_stac/item_transformer.py b/mdps_ds_lib/lib/cumulus_stac/item_transformer.py index e12d6cb..5e6d9a3 100644 --- a/mdps_ds_lib/lib/cumulus_stac/item_transformer.py +++ b/mdps_ds_lib/lib/cumulus_stac/item_transformer.py @@ -360,7 +360,7 @@ def __get_asset_obj(self, input_dict): ) return asset - def to_stac(self, source: dict) -> dict: + def to_stac(self, source: dict, source_stac: Item=None) -> dict: """ Sample: Cumulus granule { @@ -478,25 +478,32 @@ def to_stac(self, source: dict) -> dict: } stac_item = Item( id=source['granuleId'], - stac_extensions=["https://stac-extensions.github.io/file/v2.1.0/schema.json"], - bbox=[-180.0, -90.0, 180.0, 90.0], - properties={ - **custom_metadata, - **cumulus_properties, - # "datetime": f"{TimeUtils.decode_datetime(source['createdAt'], False)}Z", - # "start_datetime": datetime_to_str(self.get_time_obj(source['beginningDateTime'])), - # "end_datetime": datetime_to_str(self.get_time_obj(source['endingDateTime'])), - # "created": datetime_to_str(self.get_time_obj(source['productionDateTime'])), - # "updated": datetime_to_str(TimeUtils().parse_from_unix(source['updatedAt'], True).get_datetime_obj()), - }, - collection=source['collectionId'], - assets={self.__get_asset_name(k): self.__get_asset_obj(k) for k in validated_files}, geometry={ "type": "Point", "coordinates": [0.0, 0.0] }, + bbox=[-180.0, -90.0, 180.0, 90.0], datetime=TimeUtils().parse_from_unix(source['createdAt'], True).get_datetime_obj(), - ) + properties={} + )if source_stac is None else source_stac + + stac_item.collection_id = source['collectionId'] + stac_item.id = source['granuleId'] + stac_item.datetime = TimeUtils().parse_from_unix(source['createdAt'], True).get_datetime_obj() + existing_stac_extensions = stac_item.stac_extensions + stac_item.stac_extensions = list(set(existing_stac_extensions + ["https://stac-extensions.github.io/file/v2.1.0/schema.json"])) + stac_item.assets = {self.__get_asset_name(k): self.__get_asset_obj(k) for k in validated_files} + stac_item.properties = { + **stac_item.properties, + **custom_metadata, + **cumulus_properties, + + # "datetime": f"{TimeUtils.decode_datetime(source['createdAt'], False)}Z", + # "start_datetime": datetime_to_str(self.get_time_obj(source['beginningDateTime'])), + # "end_datetime": datetime_to_str(self.get_time_obj(source['endingDateTime'])), + # "created": datetime_to_str(self.get_time_obj(source['productionDateTime'])), + # "updated": datetime_to_str(TimeUtils().parse_from_unix(source['updatedAt'], True).get_datetime_obj()), + } stac_item.links = [ Link(rel='collection', target='.') ] diff --git a/tests/mdps_ds_lib/lib/cumulus_stac/test_item_transformer.py b/tests/mdps_ds_lib/lib/cumulus_stac/test_item_transformer.py index 3427b4f..6e269df 100644 --- a/tests/mdps_ds_lib/lib/cumulus_stac/test_item_transformer.py +++ b/tests/mdps_ds_lib/lib/cumulus_stac/test_item_transformer.py @@ -1,8 +1,11 @@ import json from unittest import TestCase +from pystac import Item + from mdps_ds_lib.lib.cumulus_stac.item_transformer import ItemTransformer, STAC_ITEM_SCHEMA from mdps_ds_lib.lib.utils.json_validator import JsonValidator +from mdps_ds_lib.lib.utils.time_utils import TimeUtils class TestItemTransformer(TestCase): From 30d0272ec583f3c7bb54ca03c47695fe526421e7 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 19 May 2025 09:52:28 -0700 Subject: [PATCH 5/5] feat: sns accept external role option --- mdps_ds_lib/ds_client/ds_client_user.py | 18 +++ mdps_ds_lib/lib/aws/aws_sns.py | 26 ++++- .../ds_client/test_ds_client_unity.py | 105 ++++++++---------- 3 files changed, 86 insertions(+), 63 deletions(-) diff --git a/mdps_ds_lib/ds_client/ds_client_user.py b/mdps_ds_lib/ds_client/ds_client_user.py index 3a98e10..9060f88 100644 --- a/mdps_ds_lib/ds_client/ds_client_user.py +++ b/mdps_ds_lib/ds_client/ds_client_user.py @@ -256,3 +256,21 @@ def delete_single_granule(self): response.raise_for_status() response = json.loads(response.text) return response + + def archive_granule(self): + if self.tenant is None or self.tenant_venue is None or self.collection is None or self.granule is None: + raise ValueError(f'require to set tenant & tenant_venue & collection & granule') + collection_id_for_granules = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()]) + granule_id_complete = ':'.join([collection_id_for_granules, self.granule]) + request_url = f'{self._uds_url}collections/' + request_url = f'{request_url}{collection_id_for_granules}/archive/{granule_id_complete}/' + print(request_url) + s = requests.session() + s.trust_env = self._trust_env + response = s.put(url=request_url, headers={ + 'Authorization': f'Bearer {self._token_retriever.get_token()}', + }, verify=self._trust_env) + response.raise_for_status() + response = json.loads(response.text) + return response + diff --git a/mdps_ds_lib/lib/aws/aws_sns.py b/mdps_ds_lib/lib/aws/aws_sns.py index 291734c..58d46fd 100644 --- a/mdps_ds_lib/lib/aws/aws_sns.py +++ b/mdps_ds_lib/lib/aws/aws_sns.py @@ -5,16 +5,38 @@ class AwsSns(AwsCred): def __init__(self): super().__init__() self.__sns_client = self.get_client('sns') + self.__special_sns_client = None self.__topic_arn = '' def set_topic_arn(self, topic_arn): self.__topic_arn = topic_arn return self - def publish_message(self, msg_str: str): + def set_external_role(self, external_role_arn: str, external_role_session_name: str, external_role_duration: int =900): + sts_client = self.get_client('sts') + assumed_role = sts_client.assume_role( + RoleArn=external_role_arn, + RoleSessionName=external_role_session_name, + DurationSeconds=external_role_duration # 12 hours max + ) + + credentials = assumed_role['Credentials'] + + self.__special_sns_client = self.get_session().client( + "sns", + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + ) + return self + + def publish_message(self, msg_str: str, is_with_daac_role: bool=False): if self.__topic_arn == '': raise ValueError('missing topic arn to publish message') - response = self.__sns_client.publish( + if is_with_daac_role and self.__special_sns_client is None: + raise ValueError('sns client with external role NOT set') + my_sns = self.__special_sns_client if is_with_daac_role else self.__sns_client + response = my_sns.publish( TopicArn=self.__topic_arn, # TargetArn='string', # not needed coz of we are using topic arn # PhoneNumber='string', # not needed coz of we are using topic arn diff --git a/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py b/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py index 35247c5..dd25835 100644 --- a/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py +++ b/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py @@ -2,6 +2,8 @@ import os from unittest import TestCase +from dotenv import load_dotenv + from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract from mdps_ds_lib.ds_client.auth_token.token_factory import TokenFactory from mdps_ds_lib.ds_client.ds_client_admin import DsClientAdmin @@ -9,6 +11,9 @@ class TestDsClientAdmin(TestCase): + def setUp(self) -> None: + super().setUp() + load_dotenv() def test_01_admin(self): os.environ['TRUST_ENV'] = 'TRUE' os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' @@ -33,12 +38,6 @@ def test_01_admin(self): def test_query_granules_across_collections(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') @@ -55,12 +54,6 @@ def test_query_granules_across_collections(self): def test_query_collections(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') @@ -70,12 +63,6 @@ def test_query_collections(self): def test_query_single_collection(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') @@ -92,12 +79,6 @@ def test_query_single_collection(self): def test_query_granules(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') @@ -117,12 +98,6 @@ def test_query_granules(self): def test_query_granules02(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa') @@ -151,12 +126,6 @@ def test_query_granules02(self): def test_query_custom_properties(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa') @@ -178,12 +147,6 @@ def test_query_custom_properties(self): def test_query_single_granule(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa') @@ -202,12 +165,6 @@ def test_query_single_granule(self): def test_update_admin(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientAdmin(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') @@ -225,12 +182,6 @@ def test_update_admin(self): def test_delete_single_granule(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') @@ -243,21 +194,53 @@ def test_delete_single_granule(self): client.tenant_venue = 'DEV' client.collection = 'DDD-01' client.collection_venue = '001' - client.granule = 'test_file10' # urn:nasa:unity:uds_local_test:DEV1:CHRP_16_DAY_REBIN___10:SNDR.SS1330.CHIRP.20230101T0000.m06.g001.L1_J1.std.v02_48.G.200101070318_REBIN print(client.delete_single_granule()) return def test_query_catalog(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') # data-sbx' print(json.dumps(client.query_catalog(), indent=4)) return + + def test_query_granules01(self): + os.environ['TRUST_ENV'] = 'TRUE' + # https://api.test.mdps.mcp.nasa.gov/am-uds-dapa/collections/URN:NASA:UNITY:unity:test:TRPSDL2ALLCRS1MGLOS___2/items + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://api.test.mdps.mcp.nasa.gov', 'am-uds-dapa') + # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data') + + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'unity' + client.tenant_venue = 'test' + client.collection = 'TRPSDL2ALLCRS1MGLOS' + client.collection_venue = '2' + result = client.query_granules(sort_keys='+properties.datetime,-id') # bbox='-114,32.5,-113,33.5' + print(json.dumps(result, indent=4)) + return + + def test_archive_one(self): + os.environ['TRUST_ENV'] = 'TRUE' + # https://api.test.mdps.mcp.nasa.gov/am-uds-dapa/collections/URN:NASA:UNITY:unity:test:TRPSDL2ALLCRS1MGLOS___2/items + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://api.test.mdps.mcp.nasa.gov', 'am-uds-dapa') + # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data') + + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'unity' + client.tenant_venue = 'test' + client.collection = 'TRPSDL2ALLCRS1MGLOS' + client.collection_venue = '2' + client.granule = 'TROPESS_CrIS-JPSS1_L2_Standard_NH3_20250108_MUSES_R1p23_megacity_los_angeles_MGLOS_F2p5_J0' + + print(client.archive_granule()) + return \ No newline at end of file