Fix issues discovered in initial deployment.

lib/ingestor-api/index.ts

@@ -107,8 +107,14 @@ export class StacIngestor extends Construct {
       memorySize: 2048,
     });
 
-    props.table.grantReadWriteData(handler);
     props.dataAccessRole.grant(handler.grantPrincipal, "sts:AssumeRole");
+    handler.addToRolePolicy(
+      new iam.PolicyStatement({
+        actions: ["s3:Get*", "s3:List*"],
+        resources: ["arn:aws:s3:::*"],
+      })
+    );
+    props.table.grantReadWriteData(handler);
 
     return handler;
   }

lib/ingestor-api/runtime/requirements.txt

@@ -6,9 +6,7 @@ orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
 pydantic_ssm_settings>=0.2.0
 pydantic>=1.9.0
-# Waiting for https://github.com/stac-utils/pgstac/pull/135
-# pypgstac==0.6.6
-pypgstac @ git+https://github.com/stac-utils/pgstac.git@main#egg=pygstac&subdirectory=pypgstac
-requests>=2.27.1
+pypgstac==0.6.8
+requests==2.27.0
 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116
 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1

lib/ingestor-api/runtime/src/ingestor.py

@@ -10,7 +10,8 @@
 from pypgstac.load import Methods
 from pypgstac.db import PgstacDB
 
-from .dependencies import get_settings, get_table
+from .dependencies import get_table
+from .config import settings
 from .schemas import Ingestion, Status
 from .vedaloader import VEDALoader
 
@@ -80,6 +81,7 @@ def load_into_pgstac(creds: DbCreds, ingestions: Sequence[Ingestion]):
     """
     Bulk insert STAC records into pgSTAC.
     """
+    print("Connecting to pgstac")
     with PgstacDB(dsn=creds.dsn_string, debug=True) as db:
         loader = VEDALoader(db=db)
 
@@ -88,18 +90,16 @@ def load_into_pgstac(creds: DbCreds, ingestions: Sequence[Ingestion]):
             convert_decimals_to_float(i.item)
             for i in ingestions
         ]
-
-        print(f"Ingesting {len(items)} items")
         loading_result = loader.load_items(
             file=items,
             # use insert_ignore to avoid overwritting existing items or upsert to replace
             insert_mode=Methods.upsert,
         )
-
+        
         # Trigger update on summaries and extents
-        collections = set([item.collection for item in items])
-        for collection in collections:
-            loader.update_collection_summaries(collection)
+        #  collections = set([item["collection"] for item in items])
+        #  for collection in collections:
+            #  loader.update_collection_summaries(collection)
 
         return loading_result
 
@@ -114,7 +114,7 @@ def update_dynamodb(
     """
     # Update records in DynamoDB
     print(f"Updating ingested items status in DynamoDB, marking as {status}...")
-    table = get_table(get_settings())
+    table = get_table(settings)
     with table.batch_writer(overwrite_by_pkeys=["created_by", "id"]) as batch:
         for ingestion in ingestions:
             batch.put_item(

lib/ingestor-api/runtime/src/validators.py

@@ -5,11 +5,11 @@
 
 @functools.cache
 def get_s3_credentials():
-    from .main import settings
+    from .config import settings
 
     print("Fetching S3 Credentials...")
-
-    response = boto3.client("sts").assume_role(
+    client = boto3.client("sts")
+    response = client.assume_role(
         RoleArn=settings.data_access_role,
         RoleSessionName="stac-ingestor-data-validation",
     )
@@ -24,14 +24,15 @@ def s3_object_is_accessible(bucket: str, key: str):
     """
     Ensure we can send HEAD requests to S3 objects.
     """
-    from .main import settings
+    from .config import settings
 
-    client = boto3.client("s3", **get_s3_credentials())
+    #  client = boto3.client("s3", **get_s3_credentials())
+    client = boto3.client("s3")
     try:
         client.head_object(
             Bucket=bucket,
             Key=key,
-            **{"RequestPayer": "requester"} if settings.requester_pays else {},
+            **{"RequestPayer": "requester"},
         )
     except client.exceptions.ClientError as e:
         raise ValueError(
@@ -56,7 +57,7 @@ def collection_exists(collection_id: str) -> bool:
     """
     Ensure collection exists in STAC
     """
-    from .main import settings
+    from .config import settings
 
     url = "/".join(
         f'{url.strip("/")}' for url in [settings.stac_url, "collections", collection_id]

package.json

@@ -41,6 +41,8 @@
     "@semantic-release/changelog": "^6.0.1",
     "@semantic-release/git": "^10.0.1",
     "@types/node": "^18.7.23",
+    "aws-cdk-lib": "2.47.0",
+    "constructs": "10.1.113",
     "jsii": "^1.68.0",
     "jsii-docgen": "^7.0.119",
     "jsii-pacmak": "^1.68.0",
@@ -52,11 +54,11 @@
   "dependencies": {
     "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.47.0-alpha.0",
     "@aws-cdk/aws-lambda-python-alpha": "^2.47.0-alpha.0",
-    "aws-cdk-lib": "^2.46.0",
+    "aws-cdk-lib": "^2.47.0",
     "constructs": "^10.1.113"
   },
   "peerDependencies": {
-    "aws-cdk-lib": "^2.46.0",
+    "aws-cdk-lib": "^2.47.0",
     "constructs": "^10.1.113"
   },
   "release": {