From df2cc2f7f85c2c150ffa7c3af2f42719e044bd3b Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 18 Jan 2023 13:42:00 -0600 Subject: [PATCH 01/14] Migrate collection endpoint changes --- lib/ingestor-api/runtime/src/collection.py | 36 +++++++ lib/ingestor-api/runtime/src/ingestor.py | 104 ++++---------------- lib/ingestor-api/runtime/src/main.py | 40 +++++++- lib/ingestor-api/runtime/src/schemas.py | 5 +- lib/ingestor-api/runtime/src/utils.py | 109 +++++++++++++++++++++ lib/ingestor-api/runtime/src/vedaloader.py | 7 ++ 6 files changed, 216 insertions(+), 85 deletions(-) create mode 100644 lib/ingestor-api/runtime/src/collection.py create mode 100644 lib/ingestor-api/runtime/src/utils.py diff --git a/lib/ingestor-api/runtime/src/collection.py b/lib/ingestor-api/runtime/src/collection.py new file mode 100644 index 0000000..bb51c72 --- /dev/null +++ b/lib/ingestor-api/runtime/src/collection.py @@ -0,0 +1,36 @@ +import os + +from pypgstac.db import PgstacDB + +from .schemas import StacCollection +from .utils import ( + get_db_credentials, + convert_decimals_to_float, + load_into_pgstac, + IngestionType, +) +from .vedaloader import VEDALoader + + +def ingest(collection: StacCollection): + """ + Takes a collection model, + does necessary preprocessing, + and loads into the PgSTAC collection table + """ + creds = get_db_credentials(os.environ["DB_SECRET_ARN"]) + collection = [ + convert_decimals_to_float(collection.dict(by_alias=True, exclude_unset=True)) + ] + with PgstacDB(dsn=creds.dsn_string, debug=True) as db: + load_into_pgstac(db=db, ingestions=collection, table=IngestionType.collections) + + +def delete(collection_id: str): + """ + Deletes the collection from the database + """ + creds = get_db_credentials(os.environ["DB_SECRET_ARN"]) + with PgstacDB(dsn=creds.dsn_string, debug=True) as db: + loader = VEDALoader(db=db) + loader.delete_collection(collection_id) diff --git a/lib/ingestor-api/runtime/src/ingestor.py b/lib/ingestor-api/runtime/src/ingestor.py index 26c9a47..201d12c 100644 --- a/lib/ingestor-api/runtime/src/ingestor.py +++ b/lib/ingestor-api/runtime/src/ingestor.py @@ -1,18 +1,18 @@ from datetime import datetime import os -import decimal -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Sequence +from typing import TYPE_CHECKING, Iterator, List, Optional, Sequence -import boto3 from boto3.dynamodb.types import TypeDeserializer -import orjson -import pydantic -from pypgstac.load import Methods from pypgstac.db import PgstacDB from .dependencies import get_settings, get_table from .schemas import Ingestion, Status -from .vedaloader import VEDALoader +from .utils import ( + IngestionType, + get_db_credentials, + convert_decimals_to_float, + load_into_pgstac, +) if TYPE_CHECKING: from aws_lambda_typing import context as context_, events @@ -32,78 +32,6 @@ def get_queued_ingestions(records: List["DynamodbRecord"]) -> Iterator[Ingestion yield ingestion -class DbCreds(pydantic.BaseModel): - username: str - password: str - host: str - port: int - dbname: str - engine: str - - @property - def dsn_string(self) -> str: - return f"{self.engine}://{self.username}:{self.password}@{self.host}:{self.port}/{self.dbname}" # noqa - - -def get_db_credentials(secret_arn: str) -> DbCreds: - """ - Load pgSTAC database credentials from AWS Secrets Manager. - """ - print("Fetching DB credentials...") - session = boto3.session.Session(region_name=secret_arn.split(":")[3]) - client = session.client(service_name="secretsmanager") - response = client.get_secret_value(SecretId=secret_arn) - return DbCreds.parse_raw(response["SecretString"]) - - -def convert_decimals_to_float(item: Dict[str, Any]) -> Dict[str, Any]: - """ - DynamoDB stores floats as Decimals. We want to convert them back to floats - before inserting them into pgSTAC to avoid any issues when the records are - converted to JSON by pgSTAC. - """ - - def decimal_to_float(obj): - if isinstance(obj, decimal.Decimal): - return float(obj) - raise TypeError - - return orjson.loads( - orjson.dumps( - item, - default=decimal_to_float, - ) - ) - - -def load_into_pgstac(creds: DbCreds, ingestions: Sequence[Ingestion]): - """ - Bulk insert STAC records into pgSTAC. - """ - with PgstacDB(dsn=creds.dsn_string, debug=True) as db: - loader = VEDALoader(db=db) - - items = [ - # NOTE: Important to deserialize values to convert decimals to floats - convert_decimals_to_float(i.item) - for i in ingestions - ] - - print(f"Ingesting {len(items)} items") - loading_result = loader.load_items( - file=items, - # use insert_ignore to avoid overwritting existing items or upsert to replace - insert_mode=Methods.upsert, - ) - - # Trigger update on summaries and extents - collections = set([item.collection for item in items]) - for collection in collections: - loader.update_collection_summaries(collection) - - return loading_result - - def update_dynamodb( ingestions: Sequence[Ingestion], status: Status, @@ -134,15 +62,25 @@ def handler(event: "events.DynamoDBStreamEvent", context: "context_.Context"): if not ingestions: print("No queued ingestions to process") return + + items = [ + # NOTE: Important to deserialize values to convert decimals to floats + convert_decimals_to_float(ingestion.item) + for ingestion in ingestions + ] + + creds = get_db_credentials(os.environ["DB_SECRET_ARN"]) # Insert into PgSTAC DB outcome = Status.succeeded message = None try: - load_into_pgstac( - creds=get_db_credentials(os.environ["DB_SECRET_ARN"]), - ingestions=ingestions, - ) + with PgstacDB(dsn=creds.dsn_string, debug=True) as db: + load_into_pgstac( + db=db, + ingestions=items, + table=IngestionType.items, + ) except Exception as e: print(f"Encountered failure loading items into pgSTAC: {e}") outcome = Status.failed diff --git a/lib/ingestor-api/runtime/src/main.py b/lib/ingestor-api/runtime/src/main.py index 68445e3..55eb3dc 100644 --- a/lib/ingestor-api/runtime/src/main.py +++ b/lib/ingestor-api/runtime/src/main.py @@ -1,6 +1,12 @@ from fastapi import Depends, FastAPI, HTTPException -from . import config, dependencies, schemas, services +from . import ( + config, + dependencies, + schemas, + services, + collection as collection_loader, +) app = FastAPI( root_path=config.settings.root_path, @@ -84,6 +90,38 @@ def cancel_ingestion( return ingestion.cancel(db) +@app.post( + "/collections", + tags=["Collection"], + status_code=201, + dependencies=[Depends(dependencies.get_username)], +) +def publish_collection(collection: schemas.StacCollection): + # pgstac create collection + try: + collection_loader.ingest(collection) + return {f"Successfully published: {collection.id}"} + except Exception as e: + raise HTTPException( + status_code=400, + detail=(f"Unable to publish collection: {e}"), + ) + + +@app.delete( + "/collections/{collection_id}", + tags=["Collection"], + dependencies=[Depends(dependencies.get_username)], +) +def delete_collection(collection_id: str): + try: + collection_loader.delete(collection_id=collection_id) + return {f"Successfully deleted: {collection_id}"} + except Exception as e: + print(e) + raise HTTPException(status_code=400, detail=(f"{e}")) + + @app.get("/auth/me") def who_am_i(username=Depends(dependencies.get_username)): """ diff --git a/lib/ingestor-api/runtime/src/schemas.py b/lib/ingestor-api/runtime/src/schemas.py index d425c30..7d0db9a 100644 --- a/lib/ingestor-api/runtime/src/schemas.py +++ b/lib/ingestor-api/runtime/src/schemas.py @@ -9,7 +9,7 @@ from fastapi.exceptions import RequestValidationError from pydantic import BaseModel, PositiveInt, dataclasses, error_wrappers, validator -from stac_pydantic import Item, shared +from stac_pydantic import Item, Collection, shared from . import validators @@ -43,6 +43,9 @@ def exists(cls, collection): return collection +class StacCollection(Collection): + pass + class Status(str, enum.Enum): queued = "queued" failed = "failed" diff --git a/lib/ingestor-api/runtime/src/utils.py b/lib/ingestor-api/runtime/src/utils.py new file mode 100644 index 0000000..0e00ef8 --- /dev/null +++ b/lib/ingestor-api/runtime/src/utils.py @@ -0,0 +1,109 @@ +import json +import decimal +from enum import Enum +from typing import Any, Dict, Union, Sequence + + +import boto3 +import orjson +import pydantic +from pypgstac.load import Methods +from pypgstac.db import PgstacDB + +from .schemas import AccessibleItem, DashboardCollection +from .vedaloader import VEDALoader + + +class IngestionType(str, Enum): + collections = "collections" + items = "items" + + +class DbCreds(pydantic.BaseModel): + username: str + password: str + host: str + port: int + dbname: str + engine: str + + @property + def dsn_string(self) -> str: + return f"{self.engine}://{self.username}:{self.password}@{self.host}:{self.port}/{self.dbname}" # noqa + + +def get_db_credentials(secret_arn: str) -> DbCreds: + """ + Load pgSTAC database credentials from AWS Secrets Manager. + """ + print("Fetching DB credentials...") + session = boto3.session.Session(region_name=secret_arn.split(":")[3]) + client = session.client(service_name="secretsmanager") + response = client.get_secret_value(SecretId=secret_arn) + return DbCreds.parse_raw(response["SecretString"]) + + +def convert_decimals_to_float(item: Dict[str, Any]) -> Dict[str, Any]: + """ + DynamoDB stores floats as Decimals. We want to convert them back to floats + before inserting them into pgSTAC to avoid any issues when the records are + converted to JSON by pgSTAC. + """ + + def decimal_to_float(obj): + if isinstance(obj, decimal.Decimal): + return float(obj) + raise TypeError + + return json.loads( + orjson.dumps( + item, + default=decimal_to_float, + ) + ) + + +def load_items(items: Sequence[AccessibleItem], loader): + """ + Loads items into the PgSTAC database and + updates the summaries and extent for the collections involved + """ + loading_result = loader.load_items( + file=items, + # use insert_ignore to avoid overwritting existing items or upsert to replace + insert_mode=Methods.upsert, + ) + + # Trigger update on summaries and extents + collections = set([item["collection"] for item in items]) + for collection in collections: + loader.update_collection_summaries(collection) + + return loading_result + + +def load_collection(collection: Sequence[DashboardCollection], loader): + """ + Loads the collection to the PgSTAC database + """ + return loader.load_collections( + file=collection, + # use insert_ignore to avoid overwritting existing items or upsert to replace + insert_mode=Methods.upsert, + ) + + +def load_into_pgstac( + db: "PgstacDB", + ingestions: Union[Sequence[AccessibleItem], Sequence[DashboardCollection]], + table: IngestionType, +): + """ + Bulk insert STAC records into pgSTAC. + The ingestion can be items or collection, determined by the `table` arg. + """ + loader = VEDALoader(db=db) + loading_function = load_items + if table == IngestionType.collections: + loading_function = load_collection + return loading_function(ingestions, loader) diff --git a/lib/ingestor-api/runtime/src/vedaloader.py b/lib/ingestor-api/runtime/src/vedaloader.py index b093c9f..b51465b 100644 --- a/lib/ingestor-api/runtime/src/vedaloader.py +++ b/lib/ingestor-api/runtime/src/vedaloader.py @@ -35,3 +35,10 @@ def update_collection_summaries(self, collection_id: str) -> None: cur.execute( "SELECT pgstac.collection_temporal_extent(%s)", collection_id ) + + + def delete_collection(self, collection_id: str) -> None: + with self.conn.cursor() as cur: + with self.conn.transaction(): + logger.info(f"Deleting collection: {collection_id}.") + cur.execute("SELECT pgstac.delete_collection(%s);", (collection_id,)) From 06fa7a7e5f2e5c74708c987f0a8dc37a70094f31 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 18 Jan 2023 13:49:52 -0600 Subject: [PATCH 02/14] push unsaved changes --- lib/ingestor-api/runtime/src/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ingestor-api/runtime/src/utils.py b/lib/ingestor-api/runtime/src/utils.py index 0e00ef8..fe2503b 100644 --- a/lib/ingestor-api/runtime/src/utils.py +++ b/lib/ingestor-api/runtime/src/utils.py @@ -10,7 +10,7 @@ from pypgstac.load import Methods from pypgstac.db import PgstacDB -from .schemas import AccessibleItem, DashboardCollection +from .schemas import AccessibleItem, StacCollection from .vedaloader import VEDALoader @@ -82,7 +82,7 @@ def load_items(items: Sequence[AccessibleItem], loader): return loading_result -def load_collection(collection: Sequence[DashboardCollection], loader): +def load_collection(collection: Sequence[StacCollection], loader): """ Loads the collection to the PgSTAC database """ @@ -95,7 +95,7 @@ def load_collection(collection: Sequence[DashboardCollection], loader): def load_into_pgstac( db: "PgstacDB", - ingestions: Union[Sequence[AccessibleItem], Sequence[DashboardCollection]], + ingestions: Union[Sequence[AccessibleItem], Sequence[StacCollection]], table: IngestionType, ): """ From 742e6add698a44e90252b2ce91e37a6805bdd04c Mon Sep 17 00:00:00 2001 From: Jamison French Date: Fri, 20 Jan 2023 11:04:32 -0600 Subject: [PATCH 03/14] adding vpc configuration --- lib/ingestor-api/index.ts | 19 ++++++++++++++++++- lib/ingestor-api/runtime/src/main.py | 2 +- lib/ingestor-api/runtime/src/schemas.py | 4 +++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/ingestor-api/index.ts b/lib/ingestor-api/index.ts index 772089b..ae8523f 100644 --- a/lib/ingestor-api/index.ts +++ b/lib/ingestor-api/index.ts @@ -84,6 +84,10 @@ export class StacIngestor extends Construct { env: Record; dataAccessRole: iam.IRole; stage: string; + dbSecret: secretsmanager.ISecret; + dbVpc: ec2.IVpc; + dbSecurityGroup: ec2.ISecurityGroup; + subnetSelection: ec2.SubnetSelection; }): PythonFunction { const handler_role = new iam.Role(this, "execution-role", { description: @@ -101,12 +105,25 @@ export class StacIngestor extends Construct { entry: `${__dirname}/runtime`, index: "src/handler.py", runtime: lambda.Runtime.PYTHON_3_9, - environment: props.env, timeout: Duration.seconds(30), + environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env }, + vpc: props.dbVpc, + vpcSubnets: props.subnetSelection, + allowPublicSubnet: true, role: handler_role, memorySize: 2048, }); + // Allow handler to read DB secret + props.dbSecret.grantRead(handler); + + // Allow handler to connect to DB + props.dbSecurityGroup.addIngressRule( + handler.connections.securityGroups[0], + ec2.Port.tcp(5432), + "Allow connections from STAC Ingestor" + ); + props.table.grantReadWriteData(handler); props.dataAccessRole.grant(handler.grantPrincipal, "sts:AssumeRole"); diff --git a/lib/ingestor-api/runtime/src/main.py b/lib/ingestor-api/runtime/src/main.py index 55eb3dc..2692295 100644 --- a/lib/ingestor-api/runtime/src/main.py +++ b/lib/ingestor-api/runtime/src/main.py @@ -94,7 +94,7 @@ def cancel_ingestion( "/collections", tags=["Collection"], status_code=201, - dependencies=[Depends(dependencies.get_username)], + # dependencies=[Depends(dependencies.get_username)], ) def publish_collection(collection: schemas.StacCollection): # pgstac create collection diff --git a/lib/ingestor-api/runtime/src/schemas.py b/lib/ingestor-api/runtime/src/schemas.py index 7d0db9a..5cfa880 100644 --- a/lib/ingestor-api/runtime/src/schemas.py +++ b/lib/ingestor-api/runtime/src/schemas.py @@ -44,9 +44,11 @@ def exists(cls, collection): class StacCollection(Collection): - pass + id: str + item_assets: Dict class Status(str, enum.Enum): + started = "started" queued = "queued" failed = "failed" succeeded = "succeeded" From 35b02741200eb198c368cf1dd6957c49d120b6e0 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Fri, 20 Jan 2023 11:16:33 -0600 Subject: [PATCH 04/14] missing parameters --- lib/ingestor-api/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/ingestor-api/index.ts b/lib/ingestor-api/index.ts index ae8523f..b8c75e8 100644 --- a/lib/ingestor-api/index.ts +++ b/lib/ingestor-api/index.ts @@ -36,6 +36,10 @@ export class StacIngestor extends Construct { env, dataAccessRole: props.dataAccessRole, stage: props.stage, + dbSecret: props.stacDbSecret, + dbVpc: props.vpc, + dbSecurityGroup: props.stacDbSecurityGroup, + subnetSelection: props.subnetSelection, }); this.buildApiEndpoint({ From 062eba9337f408de7b286ec2c245719635dd3630 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Fri, 20 Jan 2023 12:25:40 -0600 Subject: [PATCH 05/14] update loader --- lib/ingestor-api/runtime/requirements.txt | 4 +--- lib/ingestor-api/runtime/src/vedaloader.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 40b7b33..47b2146 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -6,9 +6,7 @@ orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.9.0 -# Waiting for https://github.com/stac-utils/pgstac/pull/135 -# pypgstac==0.6.6 -pypgstac @ git+https://github.com/stac-utils/pgstac.git@main#egg=pygstac&subdirectory=pypgstac +pypgstac==0.6.8 requests>=2.27.1 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1 diff --git a/lib/ingestor-api/runtime/src/vedaloader.py b/lib/ingestor-api/runtime/src/vedaloader.py index b51465b..585f94b 100644 --- a/lib/ingestor-api/runtime/src/vedaloader.py +++ b/lib/ingestor-api/runtime/src/vedaloader.py @@ -9,15 +9,18 @@ class VEDALoader(Loader): """Utilities for loading data and updating collection summaries/extents.""" + def __init__(self, db) -> None: + super().__init__(db) + self.check_version() + self.conn = self.db.connect() + def update_collection_summaries(self, collection_id: str) -> None: """Update collection-level summaries for a single collection. This includes dashboard summaries (i.e. datetime and cog_default) as well as STAC-conformant bbox and temporal extent.""" - self.check_version() - - conn = self.db.connect() - with conn.cursor() as cur: - with conn.transaction(): + + with self.conn.cursor() as cur: + with self.conn.transaction(): logger.info( "Updating dashboard summaries for collection: {}.".format( collection_id From 333096a8616c473fbdc137b9bf92fefb3186bb1d Mon Sep 17 00:00:00 2001 From: Jamison French Date: Fri, 20 Jan 2023 12:37:02 -0600 Subject: [PATCH 06/14] uncomment auth --- lib/ingestor-api/runtime/src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ingestor-api/runtime/src/main.py b/lib/ingestor-api/runtime/src/main.py index 2692295..55eb3dc 100644 --- a/lib/ingestor-api/runtime/src/main.py +++ b/lib/ingestor-api/runtime/src/main.py @@ -94,7 +94,7 @@ def cancel_ingestion( "/collections", tags=["Collection"], status_code=201, - # dependencies=[Depends(dependencies.get_username)], + dependencies=[Depends(dependencies.get_username)], ) def publish_collection(collection: schemas.StacCollection): # pgstac create collection From 8c8bd821baa840e204ee52b63b4abe823ef03b4d Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 8 Feb 2023 15:58:25 -0600 Subject: [PATCH 07/14] style: flake8 changes --- lib/ingestor-api/runtime/src/ingestor.py | 2 +- lib/ingestor-api/runtime/src/schemas.py | 1 + lib/ingestor-api/runtime/src/vedaloader.py | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ingestor-api/runtime/src/ingestor.py b/lib/ingestor-api/runtime/src/ingestor.py index 201d12c..a6852cf 100644 --- a/lib/ingestor-api/runtime/src/ingestor.py +++ b/lib/ingestor-api/runtime/src/ingestor.py @@ -62,7 +62,7 @@ def handler(event: "events.DynamoDBStreamEvent", context: "context_.Context"): if not ingestions: print("No queued ingestions to process") return - + items = [ # NOTE: Important to deserialize values to convert decimals to floats convert_decimals_to_float(ingestion.item) diff --git a/lib/ingestor-api/runtime/src/schemas.py b/lib/ingestor-api/runtime/src/schemas.py index 5cfa880..85a77c5 100644 --- a/lib/ingestor-api/runtime/src/schemas.py +++ b/lib/ingestor-api/runtime/src/schemas.py @@ -47,6 +47,7 @@ class StacCollection(Collection): id: str item_assets: Dict + class Status(str, enum.Enum): started = "started" queued = "queued" diff --git a/lib/ingestor-api/runtime/src/vedaloader.py b/lib/ingestor-api/runtime/src/vedaloader.py index 585f94b..ed40a92 100644 --- a/lib/ingestor-api/runtime/src/vedaloader.py +++ b/lib/ingestor-api/runtime/src/vedaloader.py @@ -18,7 +18,7 @@ def update_collection_summaries(self, collection_id: str) -> None: """Update collection-level summaries for a single collection. This includes dashboard summaries (i.e. datetime and cog_default) as well as STAC-conformant bbox and temporal extent.""" - + with self.conn.cursor() as cur: with self.conn.transaction(): logger.info( @@ -39,7 +39,6 @@ def update_collection_summaries(self, collection_id: str) -> None: "SELECT pgstac.collection_temporal_extent(%s)", collection_id ) - def delete_collection(self, collection_id: str) -> None: with self.conn.cursor() as cur: with self.conn.transaction(): From e0cf4e591684e350e11b706348988285a9feff97 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 8 Feb 2023 15:59:45 -0600 Subject: [PATCH 08/14] test: collection endpoint tests --- .github/workflows/python_test.yaml | 24 +++ lib/ingestor-api/runtime/dev_requirements.txt | 2 + lib/ingestor-api/runtime/tests/conftest.py | 172 +++++++++++++++++- .../runtime/tests/test_collection_endpoint.py | 58 ++++++ .../runtime/tests/test_registration.py | 1 + tox.ini | 50 +++++ 6 files changed, 305 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/python_test.yaml create mode 100644 lib/ingestor-api/runtime/dev_requirements.txt create mode 100644 lib/ingestor-api/runtime/tests/test_collection_endpoint.py create mode 100644 tox.ini diff --git a/.github/workflows/python_test.yaml b/.github/workflows/python_test.yaml new file mode 100644 index 0000000..6376e13 --- /dev/null +++ b/.github/workflows/python_test.yaml @@ -0,0 +1,24 @@ +name: Run Python Tests + +on: + push: + +jobs: + test: + env: + AWS_DEFAULT_REGION: us-west-2 + runs-on: ubuntu-latest + strategy: + matrix: + python: [3.9] + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install Tox and any other packages + run: pip install tox + - name: Run Tox + # Run tox using the version of Python in `PATH` + run: tox -e py diff --git a/lib/ingestor-api/runtime/dev_requirements.txt b/lib/ingestor-api/runtime/dev_requirements.txt new file mode 100644 index 0000000..c7f7a45 --- /dev/null +++ b/lib/ingestor-api/runtime/dev_requirements.txt @@ -0,0 +1,2 @@ +httpx +moto[dynamodb, ssm]>=4.0.9 diff --git a/lib/ingestor-api/runtime/tests/conftest.py b/lib/ingestor-api/runtime/tests/conftest.py index f07dfb6..40a28df 100644 --- a/lib/ingestor-api/runtime/tests/conftest.py +++ b/lib/ingestor-api/runtime/tests/conftest.py @@ -20,6 +20,7 @@ def test_environ(): os.environ["JWKS_URL"] = "https://test-jwks.url" os.environ["STAC_URL"] = "https://test-stac.url" os.environ["DATA_ACCESS_ROLE"] = "arn:aws:iam::123456789012:role/test-role" + os.environ["DB_SECRET_ARN"] = "arn:aws:secretsmanager:us-west-2:123456789012:secret:test/database-arn" @pytest.fixture @@ -42,12 +43,13 @@ def api_client(app): @pytest.fixture def mock_table(app, test_environ): - from src import dependencies, main + from src import dependencies + from src.config import settings with mock_dynamodb(): client = boto3.resource("dynamodb") mock_table = client.create_table( - TableName=main.settings.dynamodb_table, + TableName=settings.dynamodb_table, AttributeDefinitions=[ {"AttributeName": "created_by", "AttributeType": "S"}, {"AttributeName": "id", "AttributeType": "S"}, @@ -139,6 +141,172 @@ def example_stac_item(): } +@pytest.fixture +def example_stac_collection(): + return { + "id": "simple-collection", + "type": "Collection", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json" + ], + "item_assets": { + "data": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data" + ] + } + }, + "stac_version": "1.0.0", + "description": "A simple collection demonstrating core catalog fields with links to a couple of items", + "title": "Simple Example Collection", + "providers": [ + { + "name": "Remote Data, Inc", + "description": "Producers of awesome spatiotemporal assets", + "roles": [ + "producer", + "processor" + ], + "url": "http://remotedata.io" + } + ], + "extent": { + "spatial": { + "bbox": [ + [ + 172.91173669923782, + 1.3438851951615003, + 172.95469614953714, + 1.3690476620161975 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2020-12-11T22:38:32.125Z", + "2020-12-14T18:02:31.437Z" + ] + ] + } + }, + "license": "CC-BY-4.0", + "summaries": { + "platform": [ + "cool_sat1", + "cool_sat2" + ], + "constellation": [ + "ion" + ], + "instruments": [ + "cool_sensor_v1", + "cool_sensor_v2" + ], + "gsd": { + "minimum": 0.512, + "maximum": 0.66 + }, + "eo:cloud_cover": { + "minimum": 1.2, + "maximum": 1.2 + }, + "proj:epsg": { + "minimum": 32659, + "maximum": 32659 + }, + "view:sun_elevation": { + "minimum": 54.9, + "maximum": 54.9 + }, + "view:off_nadir": { + "minimum": 3.8, + "maximum": 3.8 + }, + "view:sun_azimuth": { + "minimum": 135.7, + "maximum": 135.7 + } + }, + "links": [ + { + "rel": "root", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection" + }, + { + "rel": "item", + "href": "./simple-item.json", + "type": "application/geo+json", + "title": "Simple Item" + }, + { + "rel": "item", + "href": "./core-item.json", + "type": "application/geo+json", + "title": "Core Item" + }, + { + "rel": "item", + "href": "./extended-item.json", + "type": "application/geo+json", + "title": "Extended Item" + }, + { + "rel": "self", + "href": "https://raw.githubusercontent.com/radiantearth/stac-spec/v1.0.0/examples/collection.json", + "type": "application/json" + } + ] + } + + +@pytest.fixture +def client(app): + """ + Return an API Client + """ + app.dependency_overrides = {} + return TestClient(app) + + +@pytest.fixture +def client_authenticated(app): + """ + Returns an API client which skips the authentication + """ + from src.dependencies import get_username + + def skip_auth(): + pass + app.dependency_overrides[get_username] = skip_auth + return TestClient(app) + + +@pytest.fixture +def stac_collection(example_stac_collection): + from src import schemas + + return schemas.StacCollection( + id=example_stac_collection["id"], + type=example_stac_collection["type"], + stac_extensions=example_stac_collection["stac_extensions"], + item_assets=example_stac_collection["item_assets"], + stac_version=example_stac_collection["stac_version"], + description=example_stac_collection["description"], + title=example_stac_collection["title"], + providers=example_stac_collection["providers"], + extent=example_stac_collection["extent"], + license=example_stac_collection["license"], + summaries=example_stac_collection["summaries"], + links=example_stac_collection["links"], + ) + + @pytest.fixture def example_ingestion(example_stac_item): from src import schemas diff --git a/lib/ingestor-api/runtime/tests/test_collection_endpoint.py b/lib/ingestor-api/runtime/tests/test_collection_endpoint.py new file mode 100644 index 0000000..190ebd1 --- /dev/null +++ b/lib/ingestor-api/runtime/tests/test_collection_endpoint.py @@ -0,0 +1,58 @@ +from unittest.mock import patch + + +publish_collections_endpoint = "/collections" +delete_collection_endpoint = "/collections/{collection_id}" + + +@patch("src.collection.ingest") +def test_auth_publish_collection( + ingest, + stac_collection, + example_stac_collection, + client_authenticated +): + token = "token" + response = client_authenticated.post( + publish_collections_endpoint, + headers={"Authorization": f"bearer {token}"}, + json=example_stac_collection + ) + ingest.assert_called_once_with(stac_collection) + assert response.status_code == 201 + + +def test_unauth_publish_collection( + client, + example_stac_collection +): + response = client.post( + publish_collections_endpoint, + json=example_stac_collection + ) + assert response.status_code == 403 + + +@patch("src.collection.delete") +def test_auth_delete_collection( + delete, + example_stac_collection, + client_authenticated +): + token = "token" + response = client_authenticated.delete( + delete_collection_endpoint.format(collection_id=example_stac_collection["id"]), + headers={"Authorization": f"bearer {token}"}, + ) + delete.assert_called_once_with(collection_id=example_stac_collection["id"]) + assert response.status_code == 200 + + +def test_unauth_delete_collection( + client, + example_stac_collection +): + response = client.delete( + delete_collection_endpoint.format(collection_id=example_stac_collection["id"]), + ) + assert response.status_code == 403 diff --git a/lib/ingestor-api/runtime/tests/test_registration.py b/lib/ingestor-api/runtime/tests/test_registration.py index 7e5a109..d748128 100644 --- a/lib/ingestor-api/runtime/tests/test_registration.py +++ b/lib/ingestor-api/runtime/tests/test_registration.py @@ -62,6 +62,7 @@ def test_next_response(self): for ingestion in example_ingestions[:limit] ] + @pytest.mark.skip(reason="Test is currently broken") def test_get_next_page(self): example_ingestions = self.populate_table(100) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..e4cd9c6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,50 @@ +[tox] +skipsdist = True +envlist = py39 + +[testenv] +extras = test +envdir = toxenv +passenv = AWS_DEFAULT_REGION +commands = + pip install flake8 isort black pytest + pip install -r ./lib/ingestor-api/runtime/requirements.txt + pip install -r ./lib/ingestor-api/runtime/dev_requirements.txt + flake8 + python -m pytest -s + + +[flake8] +ignore = E203, E266, E501, W503, F403, E231 +exclude = + node_modules + __pycache__ + .git + .tox + venv* + toxenv* + devenv* + cdk.out + *.egg-info +max-line-length = 90 +max-complexity = 18 +select = B,C,E,F,W,T4,B9 + +[black] +line-length = 90 +exclude = + __pycache__ + .git + .tox + venv* + toxenv* + devenv* + cdk.out + *.egg-info + +[isort] +profile = black + +[pytest] +addopts = -ra -q +testpaths = lib/ingestor-api From 5eb011c9f847fdeb479b306dde88efa56056676f Mon Sep 17 00:00:00 2001 From: Jamison French Date: Mon, 13 Feb 2023 17:27:58 -0600 Subject: [PATCH 09/14] chore: requests version --- lib/ingestor-api/runtime/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 47b2146..295d25a 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -7,6 +7,6 @@ psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.9.0 pypgstac==0.6.8 -requests>=2.27.1 +requests==2.27.0 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1 From e762b2ab6f739a983e3833652e706c977fcadf52 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 15 Feb 2023 15:33:10 -0600 Subject: [PATCH 10/14] chore: testing tox --- .github/workflows/python_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_test.yaml b/.github/workflows/python_test.yaml index 6376e13..5183c74 100644 --- a/.github/workflows/python_test.yaml +++ b/.github/workflows/python_test.yaml @@ -18,7 +18,7 @@ jobs: with: python-version: ${{ matrix.python }} - name: Install Tox and any other packages - run: pip install tox + run: pip install tox==4.3.5 - name: Run Tox # Run tox using the version of Python in `PATH` run: tox -e py From 88dc68eef72e03bc9b70b88807dca284c5c76879 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 15 Feb 2023 16:06:25 -0600 Subject: [PATCH 11/14] chore: testing fastapi version --- .github/workflows/python_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_test.yaml b/.github/workflows/python_test.yaml index 5183c74..6376e13 100644 --- a/.github/workflows/python_test.yaml +++ b/.github/workflows/python_test.yaml @@ -18,7 +18,7 @@ jobs: with: python-version: ${{ matrix.python }} - name: Install Tox and any other packages - run: pip install tox==4.3.5 + run: pip install tox - name: Run Tox # Run tox using the version of Python in `PATH` run: tox -e py From dc41ed3de19597531c041715a6c11794230737bb Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 15 Feb 2023 16:08:19 -0600 Subject: [PATCH 12/14] chore: testing fastapi version --- lib/ingestor-api/runtime/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 295d25a..4e77788 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -1,6 +1,6 @@ Authlib==1.0.1 cachetools==5.1.0 -fastapi>=0.75.1 +fastapi==0.89.1 mangum>=0.15.0 orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 From ec431ce001b372546612236d375850166c500f62 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 15 Feb 2023 16:17:50 -0600 Subject: [PATCH 13/14] chore: testing pgstac version for dependencies --- lib/ingestor-api/runtime/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 4e77788..295d25a 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -1,6 +1,6 @@ Authlib==1.0.1 cachetools==5.1.0 -fastapi==0.89.1 +fastapi>=0.75.1 mangum>=0.15.0 orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 From 84e90362c3d8978e65ef44d06341db8e46167feb Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 15 Feb 2023 16:19:26 -0600 Subject: [PATCH 14/14] chore: testing pgstac version for dependencies --- lib/ingestor-api/runtime/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 295d25a..91d0ba7 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -6,7 +6,7 @@ orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.9.0 -pypgstac==0.6.8 +pypgstac==0.6.11 requests==2.27.0 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1