From b6eb8355c21d9e92ce171ee47bbd97bb2d3606e0 Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Wed, 3 Nov 2021 14:13:30 +1300 Subject: [PATCH 1/6] docs: Document how to install Geostore CLI --- USAGE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/USAGE.md b/USAGE.md index 6fa15829c..d8a84745b 100644 --- a/USAGE.md +++ b/USAGE.md @@ -119,6 +119,8 @@ Example of AWS service account authentication and authorization in to Geostore u ## Command line +To install the Geostore CLI, run `pip3 install boto3 geostore typer`. + The general synopsis is `geostore NOUN VERB [PARAMETER…]`. `NOUN` is the type of thing the command is operating on, for example `version` when dealing with dataset versions. `VERB` is the action it is telling the system to take, for example `list` to show a listing of the relevant objects, or From 5cb9bcf4c363af82204698dfd02d75f3ee42ad14 Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Wed, 3 Nov 2021 13:07:28 +1300 Subject: [PATCH 2/6] feat: List datasets using CLI --- USAGE.md | 8 +++--- geostore/cli.py | 69 +++++++++++++++++++++++++++++++++++------------ tests/test_cli.py | 57 ++++++++++++++++++++------------------- 3 files changed, 84 insertions(+), 50 deletions(-) diff --git a/USAGE.md b/USAGE.md index d8a84745b..df93accdb 100644 --- a/USAGE.md +++ b/USAGE.md @@ -160,17 +160,15 @@ Examples: ```console $ geostore dataset list - Dataset ID | Title | Description - Auckland_2020-01F9ZFRK12V0WFXJ94S0DHCP65 | Auckland_2020 | Aerial imagery from April 2020 - Wellington_2020-01FJJDQJ2X0MPTPYPMM246DSH1 | Wellington_2020 | Aerial imagery from March 2020 + Auckland_2020-01F9ZFRK12V0WFXJ94S0DHCP65 + Wellington_2020-01FJJDQJ2X0MPTPYPMM246DSH1 ``` - Filter to a single dataset: ```console $ geostore dataset list --id=Auckland_2020-01F9ZFRK12V0WFXJ94S0DHCP65 - Dataset ID | Title | Description - Auckland_2020-01F9ZFRK12V0WFXJ94S0DHCP65 | Auckland_2020 | Aerial imagery from April 2020 + Auckland_2020-01F9ZFRK12V0WFXJ94S0DHCP65 ``` #### Delete diff --git a/geostore/cli.py b/geostore/cli.py index a75129f1a..c15749db1 100644 --- a/geostore/cli.py +++ b/geostore/cli.py @@ -2,6 +2,7 @@ from enum import IntEnum from http import HTTPStatus from json import dumps, load +from typing import Callable, Union import boto3 from botocore.exceptions import NoCredentialsError, NoRegionError @@ -10,8 +11,10 @@ from .api_keys import MESSAGE_KEY from .aws_keys import BODY_KEY, HTTP_METHOD_KEY, STATUS_CODE_KEY +from .dataset_keys import DATASET_KEY_SEPARATOR from .resources import ResourceName from .step_function_keys import DATASET_ID_SHORT_KEY, DESCRIPTION_KEY, TITLE_KEY +from .types import JsonList, JsonObject app = Typer() dataset_app = Typer() @@ -29,6 +32,53 @@ class ExitCode(IntEnum): @dataset_app.command(name="create") def dataset_create(title: str = Option(...), description: str = Option(...)) -> None: + request_object = { + HTTP_METHOD_KEY: "POST", + BODY_KEY: {TITLE_KEY: title, DESCRIPTION_KEY: description}, + } + + def get_output(response_body: JsonObject) -> str: + dataset_id: str = response_body[DATASET_ID_SHORT_KEY] + return dataset_id + + handle_api_request(request_object, get_output) + + +@dataset_app.command(name="list") +def dataset_list() -> None: + request_object = {HTTP_METHOD_KEY: "GET", BODY_KEY: {}} + + def get_output(response_body: JsonList) -> str: + lines = [] + for entry in response_body: + lines.append(f"{entry[TITLE_KEY]}{DATASET_KEY_SEPARATOR}{entry[DATASET_ID_SHORT_KEY]}") + return "\n".join(lines) + + handle_api_request(request_object, get_output) + + +def handle_api_request( + request_object: JsonObject, + get_output: Union[Callable[[JsonList], str], Callable[[JsonObject], str]], +) -> None: + response_payload = invoke_lambda(request_object) + status_code = response_payload[STATUS_CODE_KEY] + response_body = response_payload[BODY_KEY] + + if status_code in [HTTPStatus.OK, HTTPStatus.CREATED]: + output = get_output(response_body) + secho(output, fg=GREEN) + sys.exit(ExitCode.SUCCESS) + + if status_code == HTTPStatus.CONFLICT: + secho(response_body[MESSAGE_KEY], err=True, fg=YELLOW) + sys.exit(ExitCode.CONFLICT) + + secho(dumps(response_body), err=True, fg=RED) + sys.exit(ExitCode.UNKNOWN) + + +def invoke_lambda(request_object: JsonObject) -> JsonObject: try: client = boto3.client("lambda") except NoRegionError: @@ -39,10 +89,6 @@ def dataset_create(title: str = Option(...), description: str = Option(...)) -> ) sys.exit(ExitCode.NO_REGION_SETTING) - request_object = { - HTTP_METHOD_KEY: "POST", - BODY_KEY: {TITLE_KEY: title, DESCRIPTION_KEY: description}, - } request_payload = dumps(request_object).encode() try: @@ -55,19 +101,8 @@ def dataset_create(title: str = Option(...), description: str = Option(...)) -> ) sys.exit(ExitCode.NO_CREDENTIALS) - response_payload = load(response["Payload"]) - exit_code = {HTTPStatus.CREATED: ExitCode.SUCCESS, HTTPStatus.CONFLICT: ExitCode.CONFLICT}.get( - response_payload[STATUS_CODE_KEY], ExitCode.UNKNOWN - ) - color = {ExitCode.SUCCESS: GREEN, ExitCode.UNKNOWN: RED}.get(exit_code, YELLOW) - response_body = response_payload[BODY_KEY] - output = response_body.get( - DATASET_ID_SHORT_KEY, response_body.get(MESSAGE_KEY, dumps(response_body)) - ) - - secho(output, err=exit_code != ExitCode.SUCCESS, fg=color) - - sys.exit(exit_code) + response_payload: JsonObject = load(response["Payload"]) + return response_payload if __name__ == "__main__": diff --git a/tests/test_cli.py b/tests/test_cli.py index 18e06c8ea..3bff9a078 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -18,7 +18,7 @@ from geostore.step_function_keys import DATASET_ID_SHORT_KEY from geostore.types import JsonObject -from .aws_utils import LAMBDA_EXECUTED_VERSION, delete_s3_key, wait_for_s3_key +from .aws_utils import LAMBDA_EXECUTED_VERSION, Dataset, delete_s3_key, wait_for_s3_key from .general_generators import any_dictionary_key, any_name from .stac_generators import any_dataset_description, any_dataset_id, any_dataset_title @@ -146,15 +146,7 @@ def should_print_error_message_when_authentication_missing( boto3_client_mock.return_value.invoke.side_effect = NoCredentialsError() # When - result = CLI_RUNNER.invoke( - app, - [ - "dataset", - "create", - f"--title={any_dataset_title()}", - f"--description={any_dataset_description()}", - ], - ) + result = CLI_RUNNER.invoke(app, ["dataset", "list"]) # Then with subtests.test(msg="should print nothing to standard output"): @@ -175,15 +167,7 @@ def should_print_error_message_when_region_missing( boto3_client_mock.side_effect = NoRegionError() # When - result = CLI_RUNNER.invoke( - app, - [ - "dataset", - "create", - f"--title={any_dataset_title()}", - f"--description={any_dataset_description()}", - ], - ) + result = CLI_RUNNER.invoke(app, ["dataset", "list"]) # Then with subtests.test(msg="should print nothing to standard output"): @@ -215,15 +199,7 @@ def should_report_arbitrary_dataset_creation_failure( ) # When - result = CLI_RUNNER.invoke( - app, - [ - "dataset", - "create", - f"--title={any_dataset_title()}", - f"--description={any_dataset_description()}", - ], - ) + result = CLI_RUNNER.invoke(app, ["dataset", "list"]) # Then with subtests.test(msg="should print nothing to standard output"): @@ -236,5 +212,30 @@ def should_report_arbitrary_dataset_creation_failure( assert result.exit_code == 1 +@mark.infrastructure +def should_list_datasets(subtests: SubTests) -> None: + # Given two datasets + with Dataset() as first_dataset, Dataset() as second_dataset: + # When + result = CLI_RUNNER.invoke(app, ["dataset", "list"]) + + # Then + with subtests.test(msg="should print datasets to standard output"): + assert ( + f"{first_dataset.title}{DATASET_KEY_SEPARATOR}{first_dataset.dataset_id}\n" + in result.stdout + ) + assert ( + f"{second_dataset.title}{DATASET_KEY_SEPARATOR}{second_dataset.dataset_id}\n" + in result.stdout + ) + + with subtests.test(msg="should print nothing to standard error"): + assert result.stderr == "" + + with subtests.test(msg="should indicate success via exit code"): + assert result.exit_code == 0 + + def get_response_object(status_code: int, body: JsonObject) -> JsonObject: return {STATUS_CODE_KEY: status_code, BODY_KEY: body} From 7cf3c60be7929abba2dcbe8626881fefc5b3fb48 Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Wed, 3 Nov 2021 15:22:31 +1300 Subject: [PATCH 3/6] feat: Enable filtering dataset listing --- geostore/cli.py | 40 +++++++++++++++++++++++++++------------- tests/test_cli.py | 21 +++++++++++++++++++++ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/geostore/cli.py b/geostore/cli.py index c15749db1..f6095bed7 100644 --- a/geostore/cli.py +++ b/geostore/cli.py @@ -2,7 +2,7 @@ from enum import IntEnum from http import HTTPStatus from json import dumps, load -from typing import Callable, Union +from typing import Callable, Optional, Union import boto3 from botocore.exceptions import NoCredentialsError, NoRegionError @@ -20,6 +20,8 @@ dataset_app = Typer() app.add_typer(dataset_app, name="dataset") +GetOutputFunctionType = Union[Callable[[JsonList], str], Callable[[JsonObject], str]] + class ExitCode(IntEnum): SUCCESS = 0 @@ -45,22 +47,34 @@ def get_output(response_body: JsonObject) -> str: @dataset_app.command(name="list") -def dataset_list() -> None: - request_object = {HTTP_METHOD_KEY: "GET", BODY_KEY: {}} +def dataset_list(id_: Optional[str] = Option(None, "--id")) -> None: + body = {} + get_output: GetOutputFunctionType - def get_output(response_body: JsonList) -> str: - lines = [] - for entry in response_body: - lines.append(f"{entry[TITLE_KEY]}{DATASET_KEY_SEPARATOR}{entry[DATASET_ID_SHORT_KEY]}") - return "\n".join(lines) + if id_ is None: - handle_api_request(request_object, get_output) + def get_list_output(response_body: JsonList) -> str: + lines = [] + for entry in response_body: + lines.append( + f"{entry[TITLE_KEY]}{DATASET_KEY_SEPARATOR}{entry[DATASET_ID_SHORT_KEY]}" + ) + return "\n".join(lines) + + get_output = get_list_output + + else: + + def get_single_output(response_body: JsonObject) -> str: + return f"{response_body['title']}{DATASET_KEY_SEPARATOR}{response_body['id']}" + + body[DATASET_ID_SHORT_KEY] = id_ + get_output = get_single_output + + handle_api_request({HTTP_METHOD_KEY: "GET", BODY_KEY: body}, get_output) -def handle_api_request( - request_object: JsonObject, - get_output: Union[Callable[[JsonList], str], Callable[[JsonObject], str]], -) -> None: +def handle_api_request(request_object: JsonObject, get_output: GetOutputFunctionType) -> None: response_payload = invoke_lambda(request_object) status_code = response_payload[STATUS_CODE_KEY] response_body = response_payload[BODY_KEY] diff --git a/tests/test_cli.py b/tests/test_cli.py index 3bff9a078..fc8845f3d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -237,5 +237,26 @@ def should_list_datasets(subtests: SubTests) -> None: assert result.exit_code == 0 +@mark.infrastructure +def should_filter_datasets_listing(subtests: SubTests) -> None: + # Given two datasets + with Dataset() as first_dataset, Dataset(): + # When + result = CLI_RUNNER.invoke(app, ["dataset", "list", f"--id={first_dataset.dataset_id}"]) + + # Then + with subtests.test(msg="should print dataset to standard output"): + assert ( + result.stdout + == f"{first_dataset.title}{DATASET_KEY_SEPARATOR}{first_dataset.dataset_id}\n" + ) + + with subtests.test(msg="should print nothing to standard error"): + assert result.stderr == "" + + with subtests.test(msg="should indicate success via exit code"): + assert result.exit_code == 0 + + def get_response_object(status_code: int, body: JsonObject) -> JsonObject: return {STATUS_CODE_KEY: status_code, BODY_KEY: body} From 25a64a73220132aafcdf8831c2ab21586963d8ce Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Thu, 4 Nov 2021 10:44:30 +1300 Subject: [PATCH 4/6] feat: Delete datasets using CLI --- geostore/cli.py | 16 ++++++++++++---- tests/test_cli.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/geostore/cli.py b/geostore/cli.py index f6095bed7..4321e5ef7 100644 --- a/geostore/cli.py +++ b/geostore/cli.py @@ -74,14 +74,22 @@ def get_single_output(response_body: JsonObject) -> str: handle_api_request({HTTP_METHOD_KEY: "GET", BODY_KEY: body}, get_output) -def handle_api_request(request_object: JsonObject, get_output: GetOutputFunctionType) -> None: +@dataset_app.command(name="delete") +def dataset_delete(id_: str = Option(..., "--id")) -> None: + handle_api_request({HTTP_METHOD_KEY: "DELETE", BODY_KEY: {DATASET_ID_SHORT_KEY: id_}}, None) + + +def handle_api_request( + request_object: JsonObject, get_output: Optional[GetOutputFunctionType] +) -> None: response_payload = invoke_lambda(request_object) status_code = response_payload[STATUS_CODE_KEY] response_body = response_payload[BODY_KEY] - if status_code in [HTTPStatus.OK, HTTPStatus.CREATED]: - output = get_output(response_body) - secho(output, fg=GREEN) + if status_code in [HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.NO_CONTENT]: + if get_output is not None: + output = get_output(response_body) + secho(output, fg=GREEN) sys.exit(ExitCode.SUCCESS) if status_code == HTTPStatus.CONFLICT: diff --git a/tests/test_cli.py b/tests/test_cli.py index fc8845f3d..700a0b501 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -258,5 +258,23 @@ def should_filter_datasets_listing(subtests: SubTests) -> None: assert result.exit_code == 0 +@mark.infrastructure +def should_delete_dataset(subtests: SubTests) -> None: + # Given + with Dataset() as dataset: + # When + result = CLI_RUNNER.invoke(app, ["dataset", "delete", f"--id={dataset.dataset_id}"]) + + # Then + with subtests.test(msg="should print nothing to standard output"): + assert result.stdout == "" + + with subtests.test(msg="should print nothing to standard error"): + assert result.stderr == "" + + with subtests.test(msg="should indicate success via exit code"): + assert result.exit_code == 0 + + def get_response_object(status_code: int, body: JsonObject) -> JsonObject: return {STATUS_CODE_KEY: status_code, BODY_KEY: body} From 6e3b8647aa54dad9958b4dec295724542634ce3f Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Thu, 4 Nov 2021 15:26:01 +1300 Subject: [PATCH 5/6] feat: Create dataset version via CLI --- geostore/cli.py | 63 ++++++++++++++++++++++++++++++++++++++--------- tests/test_cli.py | 48 ++++++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/geostore/cli.py b/geostore/cli.py index 4321e5ef7..25966075f 100644 --- a/geostore/cli.py +++ b/geostore/cli.py @@ -13,12 +13,24 @@ from .aws_keys import BODY_KEY, HTTP_METHOD_KEY, STATUS_CODE_KEY from .dataset_keys import DATASET_KEY_SEPARATOR from .resources import ResourceName -from .step_function_keys import DATASET_ID_SHORT_KEY, DESCRIPTION_KEY, TITLE_KEY +from .step_function_keys import ( + DATASET_ID_SHORT_KEY, + DESCRIPTION_KEY, + EXECUTION_ARN_KEY, + METADATA_URL_KEY, + S3_ROLE_ARN_KEY, + TITLE_KEY, + VERSION_ID_KEY, +) from .types import JsonList, JsonObject +HTTP_METHOD_CREATE = "POST" + app = Typer() dataset_app = Typer() +dataset_version_app = Typer() app.add_typer(dataset_app, name="dataset") +app.add_typer(dataset_version_app, name="version") GetOutputFunctionType = Union[Callable[[JsonList], str], Callable[[JsonObject], str]] @@ -35,7 +47,7 @@ class ExitCode(IntEnum): @dataset_app.command(name="create") def dataset_create(title: str = Option(...), description: str = Option(...)) -> None: request_object = { - HTTP_METHOD_KEY: "POST", + HTTP_METHOD_KEY: HTTP_METHOD_CREATE, BODY_KEY: {TITLE_KEY: title, DESCRIPTION_KEY: description}, } @@ -43,7 +55,9 @@ def get_output(response_body: JsonObject) -> str: dataset_id: str = response_body[DATASET_ID_SHORT_KEY] return dataset_id - handle_api_request(request_object, get_output) + handle_api_request( + ResourceName.DATASETS_ENDPOINT_FUNCTION_NAME.value, request_object, get_output + ) @dataset_app.command(name="list") @@ -71,18 +85,47 @@ def get_single_output(response_body: JsonObject) -> str: body[DATASET_ID_SHORT_KEY] = id_ get_output = get_single_output - handle_api_request({HTTP_METHOD_KEY: "GET", BODY_KEY: body}, get_output) + handle_api_request( + ResourceName.DATASETS_ENDPOINT_FUNCTION_NAME.value, + {HTTP_METHOD_KEY: "GET", BODY_KEY: body}, + get_output, + ) @dataset_app.command(name="delete") def dataset_delete(id_: str = Option(..., "--id")) -> None: - handle_api_request({HTTP_METHOD_KEY: "DELETE", BODY_KEY: {DATASET_ID_SHORT_KEY: id_}}, None) + handle_api_request( + ResourceName.DATASETS_ENDPOINT_FUNCTION_NAME.value, + {HTTP_METHOD_KEY: "DELETE", BODY_KEY: {DATASET_ID_SHORT_KEY: id_}}, + None, + ) + + +@dataset_version_app.command(name="create") +def dataset_version_create( + dataset_id: str = Option(...), metadata_url: str = Option(...), s3_role_arn: str = Option(...) +) -> None: + def get_output(response_body: JsonObject) -> str: + return f"{response_body[VERSION_ID_KEY]}\t{response_body[EXECUTION_ARN_KEY]}" + + handle_api_request( + ResourceName.DATASET_VERSIONS_ENDPOINT_FUNCTION_NAME.value, + { + HTTP_METHOD_KEY: HTTP_METHOD_CREATE, + BODY_KEY: { + DATASET_ID_SHORT_KEY: dataset_id, + METADATA_URL_KEY: metadata_url, + S3_ROLE_ARN_KEY: s3_role_arn, + }, + }, + get_output, + ) def handle_api_request( - request_object: JsonObject, get_output: Optional[GetOutputFunctionType] + function_name: str, request_object: JsonObject, get_output: Optional[GetOutputFunctionType] ) -> None: - response_payload = invoke_lambda(request_object) + response_payload = invoke_lambda(function_name, request_object) status_code = response_payload[STATUS_CODE_KEY] response_body = response_payload[BODY_KEY] @@ -100,7 +143,7 @@ def handle_api_request( sys.exit(ExitCode.UNKNOWN) -def invoke_lambda(request_object: JsonObject) -> JsonObject: +def invoke_lambda(function_name: str, request_object: JsonObject) -> JsonObject: try: client = boto3.client("lambda") except NoRegionError: @@ -114,9 +157,7 @@ def invoke_lambda(request_object: JsonObject) -> JsonObject: request_payload = dumps(request_object).encode() try: - response = client.invoke( - FunctionName=ResourceName.DATASETS_ENDPOINT_FUNCTION_NAME.value, Payload=request_payload - ) + response = client.invoke(FunctionName=function_name, Payload=request_payload) except NoCredentialsError: secho( "Unable to locate credentials. Make sure to log in to AWS first.", err=True, fg=YELLOW diff --git a/tests/test_cli.py b/tests/test_cli.py index 700a0b501..498b1f03e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,8 @@ from http import HTTPStatus from io import BytesIO from json import dumps, loads +from os import environ +from re import MULTILINE, match from unittest.mock import MagicMock, patch from botocore.exceptions import NoCredentialsError, NoRegionError @@ -10,7 +12,7 @@ from pytest_subtests import SubTests from typer.testing import CliRunner -from geostore.aws_keys import BODY_KEY, STATUS_CODE_KEY +from geostore.aws_keys import AWS_DEFAULT_REGION_KEY, BODY_KEY, STATUS_CODE_KEY from geostore.cli import app from geostore.dataset_keys import DATASET_KEY_SEPARATOR from geostore.populate_catalog.task import CATALOG_FILENAME @@ -18,10 +20,22 @@ from geostore.step_function_keys import DATASET_ID_SHORT_KEY from geostore.types import JsonObject -from .aws_utils import LAMBDA_EXECUTED_VERSION, Dataset, delete_s3_key, wait_for_s3_key +from .aws_utils import ( + LAMBDA_EXECUTED_VERSION, + Dataset, + any_role_arn, + any_s3_url, + delete_s3_key, + wait_for_s3_key, +) from .general_generators import any_dictionary_key, any_name from .stac_generators import any_dataset_description, any_dataset_id, any_dataset_title +DATASET_VERSION_ID_REGEX = ( + r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z_[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{16}" +) +AWS_REGION = environ[AWS_DEFAULT_REGION_KEY] + CLI_RUNNER = CliRunner(mix_stderr=False) @@ -276,5 +290,35 @@ def should_delete_dataset(subtests: SubTests) -> None: assert result.exit_code == 0 +@mark.infrastructure +def should_create_dataset_version(subtests: SubTests) -> None: + # Given + with Dataset() as dataset: + result = CLI_RUNNER.invoke( + app, + [ + "version", + "create", + f"--dataset-id={dataset.dataset_id}", + f"--metadata-url={any_s3_url()}", + f"--s3-role-arn={any_role_arn()}", + ], + ) + + # Then + with subtests.test(msg="should print dataset version ID and execution ARN to standard output"): + assert match( + f"^({DATASET_VERSION_ID_REGEX})\tarn:aws:states:{AWS_REGION}:\\d+:execution:.*:\\1\n$", + result.stdout, + flags=MULTILINE, + ) + + with subtests.test(msg="should print nothing to standard error"): + assert result.stderr == "" + + with subtests.test(msg="should indicate success via exit code"): + assert result.exit_code == 0, result + + def get_response_object(status_code: int, body: JsonObject) -> JsonObject: return {STATUS_CODE_KEY: status_code, BODY_KEY: body} From b909b38263d415ec22ce380c07743a241f7d5ba3 Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Thu, 4 Nov 2021 15:42:51 +1300 Subject: [PATCH 6/6] feat: Install boto3 and typer everywhere This is a workaround for the lack of Poetry subproject support , discussed here . Unfortunately this means boto3 and typer will be installed whenever we run `poetry install`, including into Lambdas. --- .github/workflows/build.yml | 2 +- USAGE.md | 2 +- poetry.lock | 35 +++++++++++++++++------------------ pyproject.toml | 19 ++----------------- 4 files changed, 21 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e0bff61e8..8064d02bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install poetry - python -m poetry install --extras=cli --no-dev --no-root + python -m poetry install --no-dev --no-root - name: Build run: poetry build diff --git a/USAGE.md b/USAGE.md index df93accdb..eebd4df15 100644 --- a/USAGE.md +++ b/USAGE.md @@ -119,7 +119,7 @@ Example of AWS service account authentication and authorization in to Geostore u ## Command line -To install the Geostore CLI, run `pip3 install boto3 geostore typer`. +To install the Geostore CLI, run `pip3 install geostore`. The general synopsis is `geostore NOUN VERB [PARAMETER…]`. `NOUN` is the type of thing the command is operating on, for example `version` when dealing with dataset versions. `VERB` is the action it diff --git a/poetry.lock b/poetry.lock index c1ff26126..6383bad1d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1265,7 +1265,7 @@ name = "boto3" version = "1.17.53" description = "The AWS SDK for Python" category = "main" -optional = true +optional = false python-versions = ">= 2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] @@ -1590,7 +1590,7 @@ name = "botocore" version = "1.20.53" description = "Low-level, data-driven core of boto 3." category = "main" -optional = true +optional = false python-versions = ">= 2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] @@ -1903,7 +1903,7 @@ name = "jmespath" version = "0.10.0" description = "JSON Matching Expressions" category = "main" -optional = true +optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] @@ -2567,7 +2567,7 @@ name = "s3transfer" version = "0.3.4" description = "An Amazon S3 Transfer Manager" category = "main" -optional = true +optional = false python-versions = "*" [package.dependencies] @@ -2697,7 +2697,7 @@ name = "typer" version = "0.4.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." category = "main" -optional = true +optional = false python-versions = ">=3.6" [package.dependencies] @@ -2845,26 +2845,25 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt [extras] cdk = ["aws-cdk.aws-dynamodb", "aws-cdk.aws-ec2", "aws-cdk.aws-ecr", "aws-cdk.aws-ecr_assets", "aws-cdk.aws-ecs", "aws-cdk.aws-events", "aws-cdk.aws-events-targets", "aws-cdk.aws-iam", "aws-cdk.aws-lambda", "aws-cdk.aws-lambda-event-sources", "aws-cdk.aws-lambda-python", "aws-cdk.aws-s3", "aws-cdk.aws-sns", "aws-cdk.aws-stepfunctions", "aws-cdk.aws-stepfunctions_tasks", "awscli", "cattrs"] -check_files_checksums = ["boto3", "linz-logger", "multihash", "pynamodb"] -check_stac_metadata = ["boto3", "jsonschema", "linz-logger", "pynamodb", "strict-rfc3339"] -cli = ["boto3", "typer"] +check_files_checksums = ["linz-logger", "multihash", "pynamodb"] +check_stac_metadata = ["jsonschema", "linz-logger", "pynamodb", "strict-rfc3339"] content_iterator = ["jsonschema", "linz-logger", "pynamodb"] dataset_versions = ["jsonschema", "linz-logger", "pynamodb", "ulid-py"] -datasets = ["boto3", "jsonschema", "linz-logger", "pynamodb", "pystac", "ulid-py"] -import_asset_file = ["boto3", "linz-logger", "smart-open"] -import_dataset = ["boto3", "jsonschema", "linz-logger", "pynamodb", "smart-open", "ulid-py"] -import_metadata_file = ["boto3", "linz-logger"] -import_status = ["boto3", "jsonschema", "linz-logger", "pynamodb"] -notify_status_update = ["boto3", "jsonschema", "linz-logger", "pynamodb", "slack-sdk"] -populate_catalog = ["boto3", "jsonschema", "linz-logger", "pystac"] -update_dataset_catalog = ["boto3", "jsonschema", "linz-logger", "pynamodb", "ulid-py"] -upload_status = ["boto3", "jsonschema", "linz-logger", "pynamodb"] +datasets = ["jsonschema", "linz-logger", "pynamodb", "pystac", "ulid-py"] +import_asset_file = ["linz-logger", "smart-open"] +import_dataset = ["jsonschema", "linz-logger", "pynamodb", "smart-open", "ulid-py"] +import_metadata_file = ["linz-logger"] +import_status = ["jsonschema", "linz-logger", "pynamodb"] +notify_status_update = ["jsonschema", "linz-logger", "pynamodb", "slack-sdk"] +populate_catalog = ["jsonschema", "linz-logger", "pystac"] +update_dataset_catalog = ["jsonschema", "linz-logger", "pynamodb", "ulid-py"] +upload_status = ["jsonschema", "linz-logger", "pynamodb"] validation_summary = ["jsonschema", "linz-logger", "pynamodb"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "6fec6ad8ae2976a96a75a933b5b03c103b353a30ef4ed8dedb6f0f1cbd6121c2" +content-hash = "6c2052a0479b8e58ecce60168c8aafa4c47686a485b09d206144e586b5f45b55" [metadata.files] appdirs = [ diff --git a/pyproject.toml b/pyproject.toml index 73368a304..804777aa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,7 +98,7 @@ python = "^3.8" "aws-cdk.aws-stepfunctions" = {version = "*", optional = true} "aws-cdk.aws-stepfunctions_tasks" = {version = "*", optional = true} awscli = {version = "*", optional = true} -boto3 = {version = "*", optional = true} +boto3 = "*" cattrs = {version = "*", optional = true} jsonschema = {version = "*", extras = ["format"], optional = true} multihash = {version = "*", optional = true} @@ -107,7 +107,7 @@ pystac = {version = "*", optional = true} slack-sdk = {version = "*", extras = ["models", "webhook"], optional = true} smart-open = {version = "*", extras = ["s3"], optional = true} strict-rfc3339 = {optional = true, version = "*"} -typer = {version = "*", optional = true} +typer = "*" ulid-py = {version = "*", optional = true} linz-logger = {version = "*", optional = true} @@ -158,29 +158,22 @@ cdk = [ "cattrs", ] check_files_checksums = [ - "boto3", "linz-logger", "multihash", "pynamodb", ] check_stac_metadata = [ - "boto3", "jsonschema", "linz-logger", "pynamodb", "strict-rfc3339", ] -cli = [ - "boto3", - "typer", -] content_iterator = [ "jsonschema", "linz-logger", "pynamodb", ] datasets = [ - "boto3", "jsonschema", "linz-logger", "pynamodb", @@ -194,12 +187,10 @@ dataset_versions = [ "ulid-py", ] import_asset_file = [ - "boto3", "linz-logger", "smart-open", ] import_dataset = [ - "boto3", "jsonschema", "linz-logger", "pynamodb", @@ -207,37 +198,31 @@ import_dataset = [ "ulid-py", ] import_metadata_file = [ - "boto3", "linz-logger", ] import_status = [ - "boto3", "jsonschema", "linz-logger", "pynamodb", ] notify_status_update = [ - "boto3", "jsonschema", "linz-logger", "pynamodb", "slack-sdk" ] populate_catalog = [ - "boto3", "jsonschema", "linz-logger", "pystac", ] update_dataset_catalog = [ - "boto3", "jsonschema", "linz-logger", "pynamodb", "ulid-py" ] upload_status = [ - "boto3", "jsonschema", "linz-logger", "pynamodb",