From 6df56818fb6e3366570f776f6b1a20a16ac88d9d Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 12 Mar 2024 18:59:01 +0100 Subject: [PATCH 1/3] (fix): fix database bootstrap --- lib/database/bootstrapper_runtime/Dockerfile | 2 +- lib/database/bootstrapper_runtime/handler.py | 119 ++++++++++--------- lib/database/index.ts | 2 +- lib/ingestor-api/runtime/requirements.txt | 2 +- 4 files changed, 63 insertions(+), 62 deletions(-) diff --git a/lib/database/bootstrapper_runtime/Dockerfile b/lib/database/bootstrapper_runtime/Dockerfile index 09e0f9a..b466608 100644 --- a/lib/database/bootstrapper_runtime/Dockerfile +++ b/lib/database/bootstrapper_runtime/Dockerfile @@ -17,4 +17,4 @@ RUN rm -rf /asset/asyncio* # A command must be present avoid the following error on CDK deploy: # Error response from daemon: No command specified -CMD [ "echo", "ready to go!" ] \ No newline at end of file +CMD [ "echo", "ready to go!" ] diff --git a/lib/database/bootstrapper_runtime/handler.py b/lib/database/bootstrapper_runtime/handler.py index 3e995d7..e263acb 100644 --- a/lib/database/bootstrapper_runtime/handler.py +++ b/lib/database/bootstrapper_runtime/handler.py @@ -119,8 +119,8 @@ def create_user(cursor, username: str, password: str) -> None: ) -def create_permissions(cursor, db_name: str, username: str) -> None: - """Add permissions.""" +def update_user_permissions(cursor, db_name: str, username: str) -> None: + """Update eoAPI user permissions.""" cursor.execute( sql.SQL( "GRANT CONNECT ON DATABASE {db_name} TO {username};" @@ -181,94 +181,95 @@ def handler(event, context): try: params = event["ResourceProperties"] - connection_params = get_secret(params["conn_secret_arn"]) - user_params = get_secret(params["new_user_secret_arn"]) - - print("Connecting to admin DB...") - admin_db_conninfo = make_conninfo( - dbname=connection_params.get("dbname", "postgres"), - user=connection_params["username"], - password=connection_params["password"], - host=connection_params["host"], - port=connection_params["port"], + + # Admin (AWS RDS) user/password/dbname parameters + admin_params = get_secret(params["conn_secret_arn"]) + + # Custom eoAPI user/password/dbname parameters + eoapi_params = get_secret(params["new_user_secret_arn"]) + + print("Connecting to RDS...") + rds_conninfo = make_conninfo( + dbname=admin_params.get("dbname", "postgres"), + user=admin_params["username"], + password=admin_params["password"], + host=admin_params["host"], + port=admin_params["port"], ) - with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: + with psycopg.connect(rds_conninfo, autocommit=True) as conn: with conn.cursor() as cur: - print("Creating database...") + print(f"Creating eoAPI *{eoapi_params['dbname']}* database...") create_db( cursor=cur, - db_name=user_params["dbname"], + db_name=eoapi_params["dbname"], ) - print("Creating user...") + print(f"Creating eoAPI *{eoapi_params['username']}* user...") create_user( cursor=cur, - username=user_params["username"], - password=user_params["password"], + username=eoapi_params["username"], + password=eoapi_params["password"], ) - # Install extensions on the user DB with - # superuser permissions, since they will - # otherwise fail to install when run as - # the non-superuser within the pgstac - # migrations. - print("Connecting to STAC DB...") - stac_db_conninfo = make_conninfo( - dbname=user_params["dbname"], - user=connection_params["username"], - password=connection_params["password"], - host=connection_params["host"], - port=connection_params["port"], + # Install postgis and pgstac on the eoapi database with + # superuser permissions + print(f"Connecting to eoAPI *{eoapi_params['dbname']}* database...") + eoapi_db_admin_conninfo = make_conninfo( + dbname=eoapi_params["dbname"], + user=admin_params["username"], + password=admin_params["password"], + host=admin_params["host"], + port=admin_params["port"], ) - with psycopg.connect(stac_db_conninfo, autocommit=True) as conn: + with psycopg.connect(eoapi_db_admin_conninfo, autocommit=True) as conn: with conn.cursor() as cur: - print("Registering PostGIS ...") + print( + f"Registering Extension in *{eoapi_params['dbname']}* database..." + ) register_extensions(cursor=cur) - stac_db_admin_dsn = ( - "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( - dbname=user_params.get("dbname", "postgres"), - user=connection_params["username"], - password=connection_params["password"], - host=connection_params["host"], - port=connection_params["port"], - ) - ) - - with PgstacDB(dsn=stac_db_admin_dsn, debug=True) as pgdb: - print(f"Current {pgdb.version=}") + print("Starting PgSTAC Migration ") + with PgstacDB(connection=conn, debug=True) as pgdb: + print(f"Current PgSTAC Version: {pgdb.version}") - # As admin, run migrations - print("Running migrations...") - Migrate(pgdb).run_migration(params["pgstac_version"]) + print(f"Running migrations to PgSTAC {params['pgstac_version']}...") + Migrate(pgdb).run_migration(params["pgstac_version"]) - # Assign appropriate permissions to user (requires pgSTAC migrations to have run) - with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: + # Update permissions to eoAPI user to assume pgstac_* roles with conn.cursor() as cur: - print("Setting permissions...") - create_permissions( + print(f"Update *{eoapi_params['username']}* permissions...") + update_user_permissions( cursor=cur, - db_name=user_params["dbname"], - username=user_params["username"], + db_name=eoapi_params["dbname"], + username=eoapi_params["username"], ) - print("Customize PgSTAC database...") + stac_db_admin_dsn = ( + "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( + dbname=eoapi_params["dbname"], + user=admin_params["username"], + password=admin_params["password"], + host=admin_params["host"], + port=admin_params["port"], + ) + ) with psycopg.connect( stac_db_admin_dsn, autocommit=True, options="-c search_path=pgstac,public -c application_name=pgstac", ) as conn: + print("Customize PgSTAC database...") with conn.cursor() as cur: customization(cursor=cur, params=params) # Make sure the user can access the database stac_db_user_dsn = ( "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( - dbname=user_params.get("dbname", "postgres"), - user=user_params["username"], - password=user_params["password"], - host=connection_params["host"], - port=connection_params["port"], + dbname=eoapi_params["dbname"], + user=eoapi_params["username"], + password=eoapi_params["password"], + host=admin_params["host"], + port=admin_params["port"], ) ) with PgstacDB(dsn=stac_db_user_dsn, debug=True) as pgdb: diff --git a/lib/database/index.ts b/lib/database/index.ts index e1823d5..334006c 100644 --- a/lib/database/index.ts +++ b/lib/database/index.ts @@ -14,7 +14,7 @@ import { Construct } from "constructs"; import { CustomLambdaFunctionProps } from "../utils"; const instanceSizes: Record = require("./instance-memory.json"); -const DEFAULT_PGSTAC_VERSION = "0.7.1"; +const DEFAULT_PGSTAC_VERSION = "0.8.4"; let defaultPgSTACCustomOptions :{ [key: string]: any } = { "context": "FALSE", diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index eb2332a..6adc65d 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -5,7 +5,7 @@ orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.9.0 -pypgstac==0.7.1 +pypgstac==0.8.4 requests>=2.27.1 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1 From 495bc644beb50d4a58e0585ff119a500395bf49e Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 12 Mar 2024 20:58:29 +0100 Subject: [PATCH 2/3] fix handler --- lib/database/bootstrapper_runtime/handler.py | 59 ++++++++------------ 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/lib/database/bootstrapper_runtime/handler.py b/lib/database/bootstrapper_runtime/handler.py index e263acb..7c937d0 100644 --- a/lib/database/bootstrapper_runtime/handler.py +++ b/lib/database/bootstrapper_runtime/handler.py @@ -39,10 +39,6 @@ def send( It isn't available for source code that's stored in Amazon S3 buckets. For code in buckets, you must write your own functions to send responses. """ - responseUrl = event["ResponseURL"] - - print(responseUrl) - responseBody = {} responseBody["Status"] = responseStatus responseBody["Reason"] = ( @@ -56,20 +52,18 @@ def send( responseBody["Data"] = responseData json_responseBody = json.dumps(responseBody) - - print("Response body:\n" + json_responseBody) - - headers = {"content-type": "", "content-length": str(len(json_responseBody))} + print("Response body:\n " + json_responseBody) try: response = httpx.put( - responseUrl, + event["ResponseURL"], data=json_responseBody, - headers=headers, + headers={"content-type": "", "content-length": str(len(json_responseBody))}, timeout=30, ) - print("Status code: " + response.status_code) + print("Status code: ", response.status_code) logger.debug(f"OK - Status code: {response.status_code}") + except Exception as e: print("send(..) failed executing httpx.put(..): " + str(e)) logger.debug(f"NOK - failed executing PUT requests: {e}") @@ -89,9 +83,9 @@ def create_db(cursor, db_name: str) -> None: sql.SQL("SELECT 1 FROM pg_catalog.pg_database " "WHERE datname = %s"), [db_name] ) if cursor.fetchone(): - print(f"database {db_name} exists, not creating DB") + print(f" database {db_name} exists, not creating DB") else: - print(f"database {db_name} not found, creating...") + print(f" database {db_name} not found, creating...") cursor.execute( sql.SQL("CREATE DATABASE {db_name}").format(db_name=sql.Identifier(db_name)) ) @@ -198,13 +192,13 @@ def handler(event, context): ) with psycopg.connect(rds_conninfo, autocommit=True) as conn: with conn.cursor() as cur: - print(f"Creating eoAPI *{eoapi_params['dbname']}* database...") + print(f"Creating eoAPI '{eoapi_params['dbname']}' database...") create_db( cursor=cur, db_name=eoapi_params["dbname"], ) - print(f"Creating eoAPI *{eoapi_params['username']}* user...") + print(f"Creating eoAPI '{eoapi_params['username']}' user...") create_user( cursor=cur, username=eoapi_params["username"], @@ -213,7 +207,7 @@ def handler(event, context): # Install postgis and pgstac on the eoapi database with # superuser permissions - print(f"Connecting to eoAPI *{eoapi_params['dbname']}* database...") + print(f"Connecting to eoAPI '{eoapi_params['dbname']}' database...") eoapi_db_admin_conninfo = make_conninfo( dbname=eoapi_params["dbname"], user=admin_params["username"], @@ -224,7 +218,7 @@ def handler(event, context): with psycopg.connect(eoapi_db_admin_conninfo, autocommit=True) as conn: with conn.cursor() as cur: print( - f"Registering Extension in *{eoapi_params['dbname']}* database..." + f"Registering Extension in '{eoapi_params['dbname']}' database..." ) register_extensions(cursor=cur) @@ -235,35 +229,25 @@ def handler(event, context): print(f"Running migrations to PgSTAC {params['pgstac_version']}...") Migrate(pgdb).run_migration(params["pgstac_version"]) + with psycopg.connect( + eoapi_db_admin_conninfo, + autocommit=True, + options="-c search_path=pgstac,public -c application_name=pgstac", + ) as conn: + print("Customize PgSTAC database...") # Update permissions to eoAPI user to assume pgstac_* roles with conn.cursor() as cur: - print(f"Update *{eoapi_params['username']}* permissions...") + print(f"Update '{eoapi_params['username']}' permissions...") update_user_permissions( cursor=cur, db_name=eoapi_params["dbname"], username=eoapi_params["username"], ) - stac_db_admin_dsn = ( - "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( - dbname=eoapi_params["dbname"], - user=admin_params["username"], - password=admin_params["password"], - host=admin_params["host"], - port=admin_params["port"], - ) - ) - with psycopg.connect( - stac_db_admin_dsn, - autocommit=True, - options="-c search_path=pgstac,public -c application_name=pgstac", - ) as conn: - print("Customize PgSTAC database...") - with conn.cursor() as cur: customization(cursor=cur, params=params) # Make sure the user can access the database - stac_db_user_dsn = ( + eoapi_user_dsn = ( "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( dbname=eoapi_params["dbname"], user=eoapi_params["username"], @@ -272,8 +256,9 @@ def handler(event, context): port=admin_params["port"], ) ) - with PgstacDB(dsn=stac_db_user_dsn, debug=True) as pgdb: - print(f"User can access pgstac: {pgdb.version}") + print("Checking eoAPI user access to the PgSTAC database...") + with PgstacDB(dsn=eoapi_user_dsn, debug=True) as pgdb: + print(f" OK - User has access to pgstac db, pgstac schema version: {pgdb.version}") except Exception as e: print(f"Unable to bootstrap database with exception={e}") From cc02d0c7d75fe1717d6ffa96e46fed4c4d70eaf2 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Wed, 13 Mar 2024 15:32:00 +0100 Subject: [PATCH 3/3] update to pgstac 0.8.5 --- lib/database/index.ts | 2 +- lib/ingestor-api/runtime/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/database/index.ts b/lib/database/index.ts index 334006c..94f7267 100644 --- a/lib/database/index.ts +++ b/lib/database/index.ts @@ -14,7 +14,7 @@ import { Construct } from "constructs"; import { CustomLambdaFunctionProps } from "../utils"; const instanceSizes: Record = require("./instance-memory.json"); -const DEFAULT_PGSTAC_VERSION = "0.8.4"; +const DEFAULT_PGSTAC_VERSION = "0.8.5"; let defaultPgSTACCustomOptions :{ [key: string]: any } = { "context": "FALSE", diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 6adc65d..187cd77 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -5,7 +5,7 @@ orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.9.0 -pypgstac==0.8.4 +pypgstac==0.8.5 requests>=2.27.1 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1