-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(titiler-pgstac-api): add titiler-pgstac endpoint (#42)
* added a new feature, deploying a titiler-pgstac service with access to the pgstac database, and exposing its API.
- Loading branch information
Showing
8 changed files
with
191 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,114 @@ | ||
import { | ||
Stack, | ||
aws_iam as iam, | ||
aws_ec2 as ec2, | ||
aws_rds as rds, | ||
aws_lambda as lambda, | ||
aws_secretsmanager as secretsmanager, | ||
CfnOutput, | ||
Duration, | ||
aws_logs, | ||
} from "aws-cdk-lib"; | ||
import { HttpApi } from "@aws-cdk/aws-apigatewayv2-alpha"; | ||
import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; | ||
import { Construct } from "constructs"; | ||
|
||
export class TitilerPgstacApiLambda extends Construct { | ||
readonly url: string; | ||
public titilerPgstacLambdaFunction: lambda.Function; | ||
|
||
constructor(scope: Construct, id: string, props: TitilerPgStacApiLambdaProps) { | ||
super(scope, id); | ||
|
||
const titilerPgstacEnv = { | ||
"CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff", | ||
"GDAL_CACHEMAX": "200", | ||
"GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", | ||
"GDAL_INGESTED_BYTES_AT_OPEN": "32768", | ||
"GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES", | ||
"GDAL_HTTP_MULTIPLEX": "YES", | ||
"GDAL_HTTP_VERSION": "2", | ||
"PYTHONWARNINGS": "ignore", | ||
"VSI_CACHE": "TRUE", | ||
"VSI_CACHE_SIZE": "5000000", | ||
"DB_MIN_CONN_SIZE": "1", | ||
"DB_MAX_CONN_SIZE": "1", | ||
"PGSTAC_SECRET_ARN": props.dbSecret.secretArn, | ||
} | ||
|
||
|
||
this.titilerPgstacLambdaFunction = new lambda.Function(this, "lambda", { | ||
handler: "handler.handler", | ||
runtime: lambda.Runtime.PYTHON_3_8, | ||
code: lambda.Code.fromDockerBuild(__dirname, { | ||
file: "runtime/Dockerfile", | ||
buildArgs: { PYTHON_VERSION: '3.10' }, | ||
}), | ||
timeout: Duration.seconds(30), | ||
vpc: props.vpc, | ||
vpcSubnets: props.subnetSelection, | ||
allowPublicSubnet: true, | ||
memorySize: 3008, | ||
logRetention: aws_logs.RetentionDays.ONE_WEEK, | ||
environment: titilerPgstacEnv, | ||
}); | ||
|
||
// grant access to buckets using addToRolePolicy | ||
if (props.buckets) { | ||
props.buckets.forEach(bucket => { | ||
this.titilerPgstacLambdaFunction.addToRolePolicy(new iam.PolicyStatement({ | ||
actions: ["s3:GetObject"], | ||
resources: [`arn:aws:s3:::${bucket}/*`], | ||
})); | ||
}); | ||
} | ||
|
||
props.dbSecret.grantRead(this.titilerPgstacLambdaFunction); | ||
this.titilerPgstacLambdaFunction.connections.allowTo(props.db, ec2.Port.tcp(5432), "allow connections from titiler"); | ||
|
||
const stacApi = new HttpApi(this, `${Stack.of(this).stackName}-titiler-pgstac-api`, { | ||
defaultIntegration: new HttpLambdaIntegration("integration", this.titilerPgstacLambdaFunction), | ||
}); | ||
|
||
this.url = stacApi.url!; | ||
|
||
new CfnOutput(this, "titiler-pgstac-api-output", { | ||
exportName: `${Stack.of(this).stackName}-titiler-pgstac-url`, | ||
value: this.url, | ||
}); | ||
} | ||
} | ||
|
||
export interface TitilerPgStacApiLambdaProps { | ||
|
||
/** | ||
* VPC into which the lambda should be deployed. | ||
*/ | ||
readonly vpc: ec2.IVpc; | ||
|
||
/** | ||
* RDS Instance with installed pgSTAC. | ||
*/ | ||
readonly db: rds.IDatabaseInstance; | ||
|
||
/** | ||
* Subnet into which the lambda should be deployed. | ||
*/ | ||
readonly subnetSelection: ec2.SubnetSelection; | ||
|
||
/** | ||
* Secret containing connection information for pgSTAC database. | ||
*/ | ||
readonly dbSecret: secretsmanager.ISecret; | ||
|
||
/** | ||
* Customized environment variables to send to titiler-pgstac runtime. | ||
*/ | ||
readonly apiEnv?: Record<string, string>; | ||
|
||
/** | ||
* list of buckets the lambda will be granted access to. | ||
*/ | ||
readonly buckets?: string[]; | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
ARG PYTHON_VERSION | ||
|
||
FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} | ||
|
||
WORKDIR /tmp | ||
RUN python -m pip install pip -U | ||
|
||
COPY runtime/requirements.txt requirements.txt | ||
RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset | ||
|
||
# Reduce package size and remove useless files | ||
RUN find /asset -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; | ||
RUN find /asset -type d -name '__pycache__' -print0 | xargs -0 rm -rf | ||
RUN find /asset -type f -name '*.py' -print0 | xargs -0 rm -f | ||
RUN find /asset -type d -name 'tests' -print0 | xargs -0 rm -rf | ||
RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc | ||
|
||
COPY runtime/src/*.py /asset | ||
|
||
CMD ["echo", "hello world"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
uvicorn |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
titiler.pgstac==0.3.3 | ||
boto3>=1.26.139 | ||
psycopg[binary, pool] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
"""eoapi.raster.""" | ||
|
||
__version__ = "0.1.0" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
""" | ||
Handler for AWS Lambda. | ||
""" | ||
|
||
import os | ||
from mangum import Mangum | ||
from utils import get_secret_dict | ||
from titiler.pgstac.main import app | ||
|
||
pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"] | ||
|
||
secret = get_secret_dict(pgstac_secret_arn) | ||
os.environ.update( | ||
{ | ||
"postgres_host": secret["host"], | ||
"postgres_dbname": secret["dbname"], | ||
"postgres_user": secret["username"], | ||
"postgres_pass": secret["password"], | ||
"postgres_port": str(secret["port"]), | ||
} | ||
) | ||
|
||
handler = Mangum(app) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
import base64 | ||
import json | ||
import boto3 | ||
|
||
|
||
def get_secret_dict(secret_name: str): | ||
"""Retrieve secrets from AWS Secrets Manager | ||
Args: | ||
secret_name (str): name of aws secrets manager secret containing database connection secrets | ||
profile_name (str, optional): optional name of aws profile for use in debugger only | ||
Returns: | ||
secrets (dict): decrypted secrets in dict | ||
""" | ||
|
||
# Create a Secrets Manager client | ||
session = boto3.session.Session() | ||
client = session.client(service_name="secretsmanager") | ||
|
||
get_secret_value_response = client.get_secret_value(SecretId=secret_name) | ||
|
||
if "SecretString" in get_secret_value_response: | ||
return json.loads(get_secret_value_response["SecretString"]) | ||
else: | ||
return json.loads(base64.b64decode(get_secret_value_response["SecretBinary"])) |