diff --git a/docs/cli/cli-data.md b/docs/cli/cli-data.md index a00034221..5b60a062d 100644 --- a/docs/cli/cli-data.md +++ b/docs/cli/cli-data.md @@ -326,6 +326,25 @@ that are of standard (aka not test) quality. Therefore, these filters can be eas planet data filter --permission --std-quality --asset ortho_analytic_8b_sr | planet data search PSScene --filter - ``` +## `data asset` command basics + +To activate an asset for download three commands must be queried, in sequence: +1. `asset-activate` - activate an asset +2. `asset-wait` - wait for an asset to be activated +3. `asset-download` - download an activated asset + +For example, if we want to download a `basic_udm2` asset from item ID +`20221003_002705_38_2461`, a `PSScene` item type: + +``` +planet data asset-activate PSScene 20221003_002705_38_2461 basic_udm2 && \ +planet data asset-wait PSScene 20221003_002705_38_2461 basic_udm2 && \ +planet data asset-download PSScene 20221003_002705_38_2461 basic_udm2 --directory /path/to/data/ +00:00 - order my asset - state: active +{'_links': {'_self': 'https://api.planet.com/data/v1/assets/eyJpIjogIjIwMjIxMDAzXzAwMjcwNV8zOF8yNDYxIiwgImMiOiAiUFNTY2VuZSIsICJ0IjogImJhc2ljX3VkbTIiLCAiY3QiOiAiaXRlbS10eXBlIn0', 'activate': 'https://api.planet.com/data/v1/assets/eyJpIjogIjIwMjIxMDAzXzAwMjcwNV8zOF8yNDYxIiwgImMiOiAiUFNTY2VuZSIsICJ0IjogImJhc2ljX3VkbTIiLCAiY3QiOiAiaXRlbS10eXBlIn0/activate', 'type': 'https://api.planet.com/data/v1/asset-types/basic_udm2'}, '_permissions': ['download'], 'expires_at': '2023-03-02T19:30:48.942718', 'location': 'https://api.planet.com/data/v1/download?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJQYWtGNHZuUEs3WXRmSFNGUklHY2I3YTNXT3piaTlaam4zWUpZMmxnd0x5cVlFMVBRSHU5QXNCcjR5Q3FxSjBNbl9yN3VwVEFQYUI1ZzhYNUJmcDhmUT09IiwiZXhwIjoxNjc3Nzg1NDQ4LCJ0b2tlbl90eXBlIjoidHlwZWQtaXRlbSIsIml0ZW1fdHlwZV9pZCI6IlBTU2NlbmUiLCJpdGVtX2lkIjoiMjAyMjEwMDNfMDAyNzA1XzM4XzI0NjEiLCJhc3NldF90eXBlIjoiYmFzaWNfdWRtMiJ9.Dd0opDjW3bBS6qLLZoNiJkfBsO2n5Xz9pM5apEUz_K6viDPFexhJiy6bMbaySbby8W0YvuATdb1uYXS2FkweDg', 'md5_digest': '3a9f7dd1ce500f699d0a96afdd0e3aa2', 'status': 'active', 'type': 'basic_udm2'} +/path/to/data/20221003_002705_38_2461_1A_udm2.tif: 100%|██████████████████████████████████| 3.16k/3.16k [00:00<00:00, 32.0MB/s] +``` + ## Stats TODO diff --git a/planet/cli/data.py b/planet/cli/data.py index a232edec1..12d73e940 100644 --- a/planet/cli/data.py +++ b/planet/cli/data.py @@ -14,17 +14,20 @@ """The Planet Data CLI.""" from typing import List, Optional from contextlib import asynccontextmanager +from pathlib import Path import click +from planet.reporting import StateBar from planet import data_filter, DataClient, exceptions -from planet.clients.data import (LIST_SEARCH_TYPE, +from planet.clients.data import (SEARCH_SORT, + LIST_SEARCH_TYPE, LIST_SEARCH_TYPE_DEFAULT, LIST_SORT_ORDER, LIST_SORT_DEFAULT, - SEARCH_SORT, SEARCH_SORT_DEFAULT, STATS_INTERVAL) + from planet.specs import (get_item_types, validate_item_type, SpecificationException) @@ -68,8 +71,8 @@ def assets_to_filter(ctx, param, assets: List[str]) -> Optional[dict]: def check_item_types(ctx, param, item_types) -> Optional[List[dict]]: - '''Validates the item type by comparing the inputted item type to all - supported item types.''' + '''Validates each item types provided by comparing them to all supported + item types.''' try: for item_type in item_types: validate_item_type(item_type) @@ -78,6 +81,17 @@ def check_item_types(ctx, param, item_types) -> Optional[List[dict]]: raise click.BadParameter(str(e)) +def check_item_type(ctx, param, item_type) -> Optional[List[dict]]: + '''Validates the item type provided by comparing it to all supported + item types.''' + try: + validate_item_type(item_type) + except SpecificationException as e: + raise click.BadParameter(str(e)) + + return item_type + + def check_search_id(ctx, param, search_id) -> str: '''Ensure search id is a valix hex string''' try: @@ -480,8 +494,128 @@ async def search_update(ctx, echo_json(items, pretty) +@data.command() +@click.pass_context +@translate_exceptions +@coro +@click.argument("item_type", type=str, callback=check_item_type) +@click.argument("item_id") +@click.argument("asset_type") +@click.option('--directory', + default='.', + help=('Base directory for file download.'), + type=click.Path(exists=True, + resolve_path=True, + writable=True, + file_okay=False)) +@click.option('--filename', + default=None, + help=('Custom name to assign to downloaded file.'), + type=str) +@click.option('--overwrite', + is_flag=True, + default=False, + help=('Overwrite files if they already exist.')) +@click.option('--checksum', + is_flag=True, + default=None, + help=('Verify that checksums match.')) +async def asset_download(ctx, + item_type, + item_id, + asset_type, + directory, + filename, + overwrite, + checksum): + """Download an activated asset. + + This function will fail if the asset state is not activated. Consider + calling `asset-wait` before this command to ensure the asset is activated. + + If --checksum is provided, the associated checksums given in the manifest + are compared against the downloaded files to verify that they match. + + If --checksum is provided, files are already downloaded, and --overwrite is + not specified, this will simply validate the checksums of the files against + the manifest. + + Output: + The full path of the downloaded file. If the quiet flag is not set, this + also provides ANSI download status reporting. + """ + quiet = ctx.obj['QUIET'] + async with data_client(ctx) as cl: + asset = await cl.get_asset(item_type, item_id, asset_type) + path = await cl.download_asset(asset=asset, + filename=filename, + directory=Path(directory), + overwrite=overwrite, + progress_bar=not quiet) + if checksum: + cl.validate_checksum(asset, path) + + +@data.command() +@click.pass_context +@translate_exceptions +@coro +@click.argument("item_type", type=str, callback=check_item_type) +@click.argument("item_id") +@click.argument("asset_type") +async def asset_activate(ctx, item_type, item_id, asset_type): + '''Activate an asset.''' + async with data_client(ctx) as cl: + asset = await cl.get_asset(item_type, item_id, asset_type) + await cl.activate_asset(asset) + + +@data.command() +@click.pass_context +@translate_exceptions +@coro +@click.argument("item_type", type=str, callback=check_item_type) +@click.argument("item_id") +@click.argument("asset_type") +@click.option('--delay', + type=int, + default=5, + help='Time (in seconds) between polls.') +@click.option('--max-attempts', + type=int, + default=200, + show_default=True, + help='Maximum number of polls. Set to zero for no limit.') +async def asset_wait(ctx, item_type, item_id, asset_type, delay, max_attempts): + '''Wait for an asset to be activated. + + Returns when the asset state has reached "activated" and the asset is + available. + ''' + quiet = ctx.obj['QUIET'] + async with data_client(ctx) as cl: + asset = await cl.get_asset(item_type, item_id, asset_type) + with StateBar(order_id="my asset", disable=quiet) as bar: + state = await cl.wait_asset(asset, + delay, + max_attempts, + callback=bar.update_state) + click.echo(state) + + +# @data.command() +# @click.pass_context +# @translate_exceptions +# @coro +# @click.argument("item_type") +# @click.argument("item_id") +# @click.argument("asset_type_id") +# @pretty +# async def asset_get(ctx, item_type, item_id, asset_type_id, pretty): +# '''Get an item asset.''' +# async with data_client(ctx) as cl: +# asset = await cl.get_asset(item_type, item_id, asset_type_id) +# echo_json(asset, pretty) + # TODO: search_run()". # TODO: item_get()". -# TODO: asset_activate()". -# TODO: asset_wait()". -# TODO: asset_download()". diff --git a/tests/integration/test_data_cli.py b/tests/integration/test_data_cli.py index c8f37b824..5172d3ced 100644 --- a/tests/integration/test_data_cli.py +++ b/tests/integration/test_data_cli.py @@ -15,6 +15,8 @@ from http import HTTPStatus import json import logging +from pathlib import Path +import math import httpx import respx @@ -49,6 +51,69 @@ def _invoke(extra_args, runner=None, **kwargs): return _invoke +@pytest.fixture +def item_type(): + return 'PSScene' + + +@pytest.fixture +def item_id(): + return '20221003_002705_38_2461xx' + + +@pytest.fixture +def asset_type(): + return 'basic_udm2' + + +@pytest.fixture +def dl_url(): + return f'{TEST_URL}/1?token=IAmAToken' + + +@pytest.fixture +def mock_asset_get_response(item_type, item_id, asset_type, dl_url): + + def _func(): + basic_udm2_asset = { + "_links": { + "_self": "SELFURL", + "activate": "ACTIVATEURL", + "type": "https://api.planet.com/data/v1/asset-types/basic_udm2" + }, + "_permissions": ["download"], + "md5_digest": None, + "status": 'active', + "location": dl_url, + "type": asset_type + } + + page_response = { + "basic_analytic_4b": { + "_links": { + "_self": + "SELFURL", + "activate": + "ACTIVATEURL", + "type": ('https://api.planet.com/data/v1/asset-types/' + 'basic_analytic_4b') + }, + "_permissions": ["download"], + "md5_digest": None, + "status": "inactive", + "type": "basic_analytic_4b" + }, + "basic_udm2": basic_udm2_asset + } + + # Mock the response for get_asset + mock_resp_get = httpx.Response(HTTPStatus.OK, json=page_response) + asset_url = f'{TEST_URL}/item-types/{item_type}/items/{item_id}/assets' + respx.get(asset_url).return_value = mock_resp_get + + return _func + + @pytest.fixture def item_descriptions(get_test_file_json): item_ids = [ @@ -71,6 +136,9 @@ def test_data_command_registered(invoke): assert "search-get" in result.output assert "search-delete" in result.output assert "search-update" in result.output + assert "asset-download" in result.output + assert "asset-activate" in result.output + assert "asset-wait" in result.output # Add other sub-commands here. @@ -871,12 +939,171 @@ def test_search_update_fail(invoke, search_id, search_filter): assert result.exception -# TODO: basic test for "planet data search-create". -# TODO: basic test for "planet data search-get". +@respx.mock +@pytest.mark.asyncio +@pytest.mark.parametrize("exists, overwrite", + [(False, False), (True, False), (True, True), + (False, True)]) +def test_asset_download_default(invoke, + open_test_img, + exists, + overwrite, + mock_asset_get_response, + item_type, + item_id, + asset_type, + dl_url): + + mock_asset_get_response() + + img_headers = { + 'Content-Type': 'image/tiff', + 'Content-Length': '527', + 'Content-Disposition': 'attachment; filename="img.tif"' + } + + async def _stream_img(): + data = open_test_img.read() + v = memoryview(data) + + chunksize = 100 + for i in range(math.ceil(len(v) / (chunksize))): + yield v[i * chunksize:min((i + 1) * chunksize, len(v))] + + # Mock the response for download_asset + mock_resp_download = httpx.Response(HTTPStatus.OK, + stream=_stream_img(), + headers=img_headers, + request='donotcloneme') + respx.get(dl_url).return_value = mock_resp_download + + runner = CliRunner() + with runner.isolated_filesystem() as folder: + if exists: + Path(folder, 'img.tif').write_bytes(b'01010') + + asset_download_command = [ + 'asset-download', + item_type, + item_id, + asset_type, + f'--directory={Path(folder)}', + '--filename', + 'img.tif' + ] + if overwrite: + asset_download_command.append('--overwrite') + + result = invoke(asset_download_command, runner=runner) + assert result.exit_code == 0 + + path = Path(folder, 'img.tif') + + assert path.name == 'img.tif' + assert path.is_file() + + if exists and not overwrite: + assert len(path.read_bytes()) == 5 + assert len(result.output) == 0 + else: + assert len(path.read_bytes()) == 527 + assert path.name in result.output + + +@respx.mock +def test_asset_activate(invoke, + mock_asset_get_response, + item_type, + item_id, + asset_type, + dl_url): + + mock_asset_get_response() + + # Mock the response for activate_asset + mock_resp_activate = httpx.Response(HTTPStatus.OK) + respx.get(dl_url).return_value = mock_resp_activate + + runner = CliRunner() + result = invoke(['asset-activate', item_type, item_id, asset_type], + runner=runner) + + assert not result.exception + + +@respx.mock +def test_asset_wait(invoke, + mock_asset_get_response, + item_type, + item_id, + asset_type, + dl_url): + + mock_asset_get_response() + + # Mock the response for wait_asset + mock_resp_wait = httpx.Response(HTTPStatus.OK) + respx.get(dl_url).return_value = mock_resp_wait + + runner = CliRunner() + result = invoke( + ['asset-wait', item_type, item_id, asset_type, '--delay', '0'], + runner=runner) + + assert not result.exception + assert "state: active" in result.output + + +# @respx.mock +# def test_asset_get(invoke): +# item_type = 'PSScene' +# item_id = '20221003_002705_38_2461xx' +# asset_type_id = 'basic_udm2' +# dl_url = f'{TEST_URL}/1?token=IAmAToken' + +# basic_udm2_asset = { +# "_links": { +# "_self": "SELFURL", +# "activate": "ACTIVATEURL", +# "type": "https://api.planet.com/data/v1/asset-types/basic_udm2" +# }, +# "_permissions": ["download"], +# "md5_digest": None, +# "status": 'active', +# "location": dl_url, +# "type": "basic_udm2" +# } + +# page_response = { +# "basic_analytic_4b": { +# "_links": { +# "_self": +# "SELFURL", +# "activate": +# "ACTIVATEURL", +# "type": +# "https://api.planet.com/data/v1/asset-types/basic_analytic_4b" +# }, +# "_permissions": ["download"], +# "md5_digest": None, +# "status": "inactive", +# "type": "basic_analytic_4b" +# }, +# "basic_udm2": basic_udm2_asset +# } + +# mock_resp = httpx.Response(HTTPStatus.OK, json=page_response) +# assets_url = f'{TEST_URL}/item-types/{item_type}/items/{item_id}/assets' +# respx.get(assets_url).return_value = mock_resp + +# runner = CliRunner() +# result = invoke(['asset-get', item_type, item_id, asset_type_id], +# runner=runner) + +# assert not result.exception +# assert json.dumps(basic_udm2_asset) in result.output + # TODO: basic test for "planet data search-list". # TODO: basic test for "planet data search-run". # TODO: basic test for "planet data item-get". -# TODO: basic test for "planet data asset-activate". -# TODO: basic test for "planet data asset-wait". -# TODO: basic test for "planet data asset-download". # TODO: basic test for "planet data stats". diff --git a/tests/unit/test_data_callbacks.py b/tests/unit/test_data_callbacks.py index 41d7b78d5..6a6099ebf 100644 --- a/tests/unit/test_data_callbacks.py +++ b/tests/unit/test_data_callbacks.py @@ -14,7 +14,7 @@ import logging import pytest import click -from planet.cli.data import check_item_types +from planet.cli.data import check_item_types, check_item_type LOGGER = logging.getLogger(__name__) @@ -43,13 +43,32 @@ def __init__(self): 'PSScene4Band', 'REScene' ]) -def test_item_type_success(item_types): +def test_item_types_success(item_types): ctx = MockContext() result = check_item_types(ctx, 'item_types', [item_types]) assert result == [item_types] +def test_item_types_fail(): + ctx = MockContext() + with pytest.raises(click.BadParameter): + check_item_types(ctx, 'item_types', "bad_item_type") + + +def test_item_type_success(): + ctx = MockContext() + item_type = "PSScene" + result = check_item_type(ctx, 'item_type', item_type) + assert result == item_type + + def test_item_type_fail(): ctx = MockContext() with pytest.raises(click.BadParameter): - check_item_types(ctx, 'item_type', "bad_item_type") + check_item_type(ctx, 'item_type', "bad_item_type") + + +def test_item_type_too_many_item_types(): + ctx = MockContext() + with pytest.raises(click.BadParameter): + check_item_types(ctx, 'item_type', "PSScene,SkySatScene")