diff --git a/lib/ingestor-api/index.ts b/lib/ingestor-api/index.ts index 772089b..9d37802 100644 --- a/lib/ingestor-api/index.ts +++ b/lib/ingestor-api/index.ts @@ -107,8 +107,14 @@ export class StacIngestor extends Construct { memorySize: 2048, }); - props.table.grantReadWriteData(handler); props.dataAccessRole.grant(handler.grantPrincipal, "sts:AssumeRole"); + handler.addToRolePolicy( + new iam.PolicyStatement({ + actions: ["s3:Get*", "s3:List*"], + resources: ["arn:aws:s3:::*"], + }) + ); + props.table.grantReadWriteData(handler); return handler; } diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 40b7b33..295d25a 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -6,9 +6,7 @@ orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.9.0 -# Waiting for https://github.com/stac-utils/pgstac/pull/135 -# pypgstac==0.6.6 -pypgstac @ git+https://github.com/stac-utils/pgstac.git@main#egg=pygstac&subdirectory=pypgstac -requests>=2.27.1 +pypgstac==0.6.8 +requests==2.27.0 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1 diff --git a/lib/ingestor-api/runtime/src/ingestor.py b/lib/ingestor-api/runtime/src/ingestor.py index 26c9a47..c9d2598 100644 --- a/lib/ingestor-api/runtime/src/ingestor.py +++ b/lib/ingestor-api/runtime/src/ingestor.py @@ -10,7 +10,8 @@ from pypgstac.load import Methods from pypgstac.db import PgstacDB -from .dependencies import get_settings, get_table +from .dependencies import get_table +from .config import settings from .schemas import Ingestion, Status from .vedaloader import VEDALoader @@ -80,6 +81,7 @@ def load_into_pgstac(creds: DbCreds, ingestions: Sequence[Ingestion]): """ Bulk insert STAC records into pgSTAC. """ + print("Connecting to pgstac") with PgstacDB(dsn=creds.dsn_string, debug=True) as db: loader = VEDALoader(db=db) @@ -88,18 +90,16 @@ def load_into_pgstac(creds: DbCreds, ingestions: Sequence[Ingestion]): convert_decimals_to_float(i.item) for i in ingestions ] - - print(f"Ingesting {len(items)} items") loading_result = loader.load_items( file=items, # use insert_ignore to avoid overwritting existing items or upsert to replace insert_mode=Methods.upsert, ) - + # Trigger update on summaries and extents - collections = set([item.collection for item in items]) - for collection in collections: - loader.update_collection_summaries(collection) + # collections = set([item["collection"] for item in items]) + # for collection in collections: + # loader.update_collection_summaries(collection) return loading_result @@ -114,7 +114,7 @@ def update_dynamodb( """ # Update records in DynamoDB print(f"Updating ingested items status in DynamoDB, marking as {status}...") - table = get_table(get_settings()) + table = get_table(settings) with table.batch_writer(overwrite_by_pkeys=["created_by", "id"]) as batch: for ingestion in ingestions: batch.put_item( diff --git a/lib/ingestor-api/runtime/src/validators.py b/lib/ingestor-api/runtime/src/validators.py index 2c26e06..6375720 100644 --- a/lib/ingestor-api/runtime/src/validators.py +++ b/lib/ingestor-api/runtime/src/validators.py @@ -5,11 +5,11 @@ @functools.cache def get_s3_credentials(): - from .main import settings + from .config import settings print("Fetching S3 Credentials...") - - response = boto3.client("sts").assume_role( + client = boto3.client("sts") + response = client.assume_role( RoleArn=settings.data_access_role, RoleSessionName="stac-ingestor-data-validation", ) @@ -24,14 +24,15 @@ def s3_object_is_accessible(bucket: str, key: str): """ Ensure we can send HEAD requests to S3 objects. """ - from .main import settings + from .config import settings - client = boto3.client("s3", **get_s3_credentials()) + # client = boto3.client("s3", **get_s3_credentials()) + client = boto3.client("s3") try: client.head_object( Bucket=bucket, Key=key, - **{"RequestPayer": "requester"} if settings.requester_pays else {}, + **{"RequestPayer": "requester"}, ) except client.exceptions.ClientError as e: raise ValueError( @@ -56,7 +57,7 @@ def collection_exists(collection_id: str) -> bool: """ Ensure collection exists in STAC """ - from .main import settings + from .config import settings url = "/".join( f'{url.strip("/")}' for url in [settings.stac_url, "collections", collection_id] diff --git a/package-lock.json b/package-lock.json index ed42530..6e8379b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,13 +5,12 @@ "requires": true, "packages": { "": { - "name": "cdk-pgstac", "version": "2.5.0", "license": "ISC", "dependencies": { "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.47.0-alpha.0", "@aws-cdk/aws-lambda-python-alpha": "^2.47.0-alpha.0", - "aws-cdk-lib": "^2.46.0", + "aws-cdk-lib": "^2.47.0", "constructs": "^10.1.113" }, "devDependencies": { @@ -19,6 +18,8 @@ "@semantic-release/changelog": "^6.0.1", "@semantic-release/git": "^10.0.1", "@types/node": "^18.7.23", + "aws-cdk-lib": "2.47.0", + "constructs": "10.1.113", "jsii": "^1.68.0", "jsii-docgen": "^7.0.119", "jsii-pacmak": "^1.68.0", @@ -28,7 +29,7 @@ "semantic-release": "^19.0.5" }, "peerDependencies": { - "aws-cdk-lib": "^2.46.0", + "aws-cdk-lib": "^2.47.0", "constructs": "^10.1.113" } }, @@ -1035,7 +1036,16 @@ "minimatch", "punycode", "semver", - "yaml" + "yaml", + "at-least-node", + "balanced-match", + "brace-expansion", + "concat-map", + "graceful-fs", + "jsonfile", + "lru-cache", + "universalify", + "yallist" ], "dependencies": { "@balena/dockerignore": "^1.0.2", @@ -4173,7 +4183,128 @@ "treeverse", "validate-npm-package-name", "which", - "write-file-atomic" + "write-file-atomic", + "@colors/colors", + "@gar/promisify", + "@npmcli/disparity-colors", + "@npmcli/git", + "@npmcli/installed-package-contents", + "@npmcli/metavuln-calculator", + "@npmcli/move-file", + "@npmcli/name-from-folder", + "@npmcli/node-gyp", + "@npmcli/query", + "@tootallnate/once", + "agent-base", + "agentkeepalive", + "aggregate-error", + "ansi-regex", + "ansi-styles", + "aproba", + "are-we-there-yet", + "asap", + "balanced-match", + "bin-links", + "binary-extensions", + "brace-expansion", + "builtins", + "cidr-regex", + "clean-stack", + "clone", + "cmd-shim", + "color-convert", + "color-name", + "color-support", + "common-ancestor-path", + "concat-map", + "console-control-strings", + "cssesc", + "debug", + "debuglog", + "defaults", + "delegates", + "depd", + "dezalgo", + "diff", + "emoji-regex", + "encoding", + "env-paths", + "err-code", + "fs.realpath", + "function-bind", + "gauge", + "has", + "has-flag", + "has-unicode", + "http-cache-semantics", + "http-proxy-agent", + "https-proxy-agent", + "humanize-ms", + "iconv-lite", + "ignore-walk", + "imurmurhash", + "indent-string", + "infer-owner", + "inflight", + "inherits", + "ip", + "ip-regex", + "is-core-module", + "is-fullwidth-code-point", + "is-lambda", + "isexe", + "json-stringify-nice", + "jsonparse", + "just-diff", + "just-diff-apply", + "lru-cache", + "minipass-collect", + "minipass-fetch", + "minipass-flush", + "minipass-json-stream", + "minipass-sized", + "minizlib", + "mute-stream", + "negotiator", + "normalize-package-data", + "npm-bundled", + "npm-normalize-package-bin", + "npm-packlist", + "once", + "path-is-absolute", + "postcss-selector-parser", + "promise-all-reject-late", + "promise-call-limit", + "promise-inflight", + "promise-retry", + "promzard", + "read-cmd-shim", + "readable-stream", + "retry", + "safe-buffer", + "safer-buffer", + "set-blocking", + "signal-exit", + "smart-buffer", + "socks", + "socks-proxy-agent", + "spdx-correct", + "spdx-exceptions", + "spdx-expression-parse", + "spdx-license-ids", + "string_decoder", + "string-width", + "strip-ansi", + "supports-color", + "unique-filename", + "unique-slug", + "util-deprecate", + "validate-npm-package-license", + "walk-up-path", + "wcwidth", + "wide-align", + "wrappy", + "yallist" ], "dev": true, "dependencies": { diff --git a/package.json b/package.json index 6301437..3afc243 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,8 @@ "@semantic-release/changelog": "^6.0.1", "@semantic-release/git": "^10.0.1", "@types/node": "^18.7.23", + "aws-cdk-lib": "2.47.0", + "constructs": "10.1.113", "jsii": "^1.68.0", "jsii-docgen": "^7.0.119", "jsii-pacmak": "^1.68.0", @@ -52,11 +54,11 @@ "dependencies": { "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.47.0-alpha.0", "@aws-cdk/aws-lambda-python-alpha": "^2.47.0-alpha.0", - "aws-cdk-lib": "^2.46.0", + "aws-cdk-lib": "^2.47.0", "constructs": "^10.1.113" }, "peerDependencies": { - "aws-cdk-lib": "^2.46.0", + "aws-cdk-lib": "^2.47.0", "constructs": "^10.1.113" }, "release": {