From 2fef537ab6840d7ad7bc413745a8999a5afc43d0 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 24 Apr 2021 12:41:40 -0700 Subject: [PATCH 01/11] add fastapi to lambda --- Makefile | 1 - cdk-deployment/app.py | 4 +- cdk-deployment/build-libraries/.dockerignore | 28 ++++++ .../build-libraries/Dockerfile-libraries | 2 +- cdk-deployment/lambda/lambda.py | 90 +++++++++++-------- .../validator_cdk/validator_cdk_stack.py | 14 +-- 6 files changed, 89 insertions(+), 50 deletions(-) create mode 100644 cdk-deployment/build-libraries/.dockerignore diff --git a/Makefile b/Makefile index 48aec365..571cdd33 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,6 @@ build-libraries: # Build the libraries for layers. Used internally docker stop lambdalayer docker rm lambdalayer docker rmi lambdalayer - cp -r stac_validator cdk-deployment/lambda build-cdk: ## Build the libraries in preperation for CDK deployment make build-libraries diff --git a/cdk-deployment/app.py b/cdk-deployment/app.py index c55513fa..cab11e71 100644 --- a/cdk-deployment/app.py +++ b/cdk-deployment/app.py @@ -5,9 +5,9 @@ # with examples from the CDK Developer's Guide, which are in the process of # being updated to use `cdk`. You may delete this import if you don't need it. from aws_cdk import core -from validator_cdk.validator_cdk_stack import ValidatorCdkStack +from validator_cdk.validator_cdk_stack import FastAPICdkStack app = core.App() -ValidatorCdkStack(app, "ValidatorCdkStack") +FastAPICdkStack(app, "FastAPICdkStack") app.synth() diff --git a/cdk-deployment/build-libraries/.dockerignore b/cdk-deployment/build-libraries/.dockerignore new file mode 100644 index 00000000..05b10522 --- /dev/null +++ b/cdk-deployment/build-libraries/.dockerignore @@ -0,0 +1,28 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +env +pip-log.txt +pip-delete-this-directory.txt +.tox +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +*.log +.git +*.swp +package-lock.json +__pycache__ +.pytest_cache +.env +.venv +*.egg-info + +# CDK asset staging directory +.cdk.staging +cdk.out \ No newline at end of file diff --git a/cdk-deployment/build-libraries/Dockerfile-libraries b/cdk-deployment/build-libraries/Dockerfile-libraries index 91498cbd..cef66ac3 100644 --- a/cdk-deployment/build-libraries/Dockerfile-libraries +++ b/cdk-deployment/build-libraries/Dockerfile-libraries @@ -4,5 +4,5 @@ COPY . /code/ RUN apt-get -y update && \ apt-get -y install zip unzip && \ python -m pip install --upgrade pip && \ - pip install . requests>=2.19.1 pytest==6.2.2 pystac==0.5.6 jsonschema==3.2.0 click==7.1.2 -t ./python && \ + pip install . mangum fastapi[all] uvicorn requests>=2.19.1 pytest==6.2.2 pystac==0.5.6 jsonschema==3.2.0 click==7.1.2 -t ./python && \ zip -r libraries.zip ./python/ \ No newline at end of file diff --git a/cdk-deployment/lambda/lambda.py b/cdk-deployment/lambda/lambda.py index 1040b628..70c605ed 100644 --- a/cdk-deployment/lambda/lambda.py +++ b/cdk-deployment/lambda/lambda.py @@ -1,39 +1,51 @@ -import json -import tempfile - -from stac_validator import stac_validator - - -def handler(event, context): - body = json.loads(event["body"]) - if body.get("stac_url"): - body = body["stac_url"] - else: - # TODO: Look at alains devops template. store in mem - temp_stac_file = tempfile.NamedTemporaryFile( - delete=False, - mode="w+", - ) - json.dump(body, temp_stac_file) - temp_stac_file.flush() - temp_stac_file.close() - body = temp_stac_file.name - stac = stac_validator.StacValidate(body) - stac.run() - - output = stac.message[0] - - if "validation_method" in output: - output.pop("validation_method") - - return { - "statusCode": 200, - "isBase64Encoded": False, - "headers": { - "Content-Type": "application/json", - "Access-Control-Allow-Headers": "Content-Type", - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "OPTIONS,POST,GET", - }, - "body": json.dumps(output), - } +# import json +# import tempfile + +import Mangum +from fastapi import FastAPI + +# from stac_validator import stac_validator + +app = FastAPI() + + +@app.get("/hello") +async def root(): + return {"message": "Hello World"} + + +# def handler(event, context): +# body = json.loads(event["body"]) +# if body.get("stac_url"): +# body = body["stac_url"] +# else: +# # TODO: Look at alains devops template. store in mem +# temp_stac_file = tempfile.NamedTemporaryFile( +# delete=False, +# mode="w+", +# ) +# json.dump(body, temp_stac_file) +# temp_stac_file.flush() +# temp_stac_file.close() +# body = temp_stac_file.name +# stac = stac_validator.StacValidate(body) +# stac.run() + +# output = stac.message[0] + +# if "validation_method" in output: +# output.pop("validation_method") + +# return { +# "statusCode": 200, +# "isBase64Encoded": False, +# "headers": { +# "Content-Type": "application/json", +# "Access-Control-Allow-Headers": "Content-Type", +# "Access-Control-Allow-Origin": "*", +# "Access-Control-Allow-Methods": "OPTIONS,POST,GET", +# }, +# "body": json.dumps(output), +# } + +handler = Mangum(app) diff --git a/cdk-deployment/validator_cdk/validator_cdk_stack.py b/cdk-deployment/validator_cdk/validator_cdk_stack.py index dc7b7747..ba0b8f0c 100644 --- a/cdk-deployment/validator_cdk/validator_cdk_stack.py +++ b/cdk-deployment/validator_cdk/validator_cdk_stack.py @@ -7,26 +7,26 @@ from aws_cdk import core as cdk -class ValidatorCdkStack(cdk.Stack): +class FastAPICdkStack(cdk.Stack): def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) - stac_lib = _lambda.LayerVersion( + all_lib = _lambda.LayerVersion( self, - "stac-lib-layer", + "all-lib-layer", code=_lambda.AssetCode("lambda/libraries.zip"), compatible_runtimes=[_lambda.Runtime.PYTHON_3_8], ) # Defines an AWS Lambda resource - validator_lambda = _lambda.Function( + fast_api_lambda = _lambda.Function( self, - "STACHandler", + "STACFastAPI", runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset("lambda"), handler="lambda.handler", timeout=cdk.Duration.seconds(30), - layers=[stac_lib], + layers=[all_lib], ) cors = apigw.CorsOptions(allow_origins=["*"]) @@ -34,6 +34,6 @@ def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: apigw.LambdaRestApi( self, "Endpoint", - handler=validator_lambda, + handler=fast_api_lambda, default_cors_preflight_options=cors, ) From ec2fb67122ae0d118be550bcebbd9324da4aa222 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 24 Apr 2021 14:03:26 -0700 Subject: [PATCH 02/11] fastapi works in lambda --- cdk-deployment/build-libraries/Dockerfile-libraries | 2 +- cdk-deployment/lambda/lambda.py | 2 +- cdk-deployment/validator_cdk/validator_cdk_stack.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdk-deployment/build-libraries/Dockerfile-libraries b/cdk-deployment/build-libraries/Dockerfile-libraries index cef66ac3..0b1bd3dd 100644 --- a/cdk-deployment/build-libraries/Dockerfile-libraries +++ b/cdk-deployment/build-libraries/Dockerfile-libraries @@ -4,5 +4,5 @@ COPY . /code/ RUN apt-get -y update && \ apt-get -y install zip unzip && \ python -m pip install --upgrade pip && \ - pip install . mangum fastapi[all] uvicorn requests>=2.19.1 pytest==6.2.2 pystac==0.5.6 jsonschema==3.2.0 click==7.1.2 -t ./python && \ + pip install . mangum uvicorn fastapi[all] requests>=2.19.1 pystac==0.5.6 jsonschema==3.2.0 click==7.1.2 -t ./python && \ zip -r libraries.zip ./python/ \ No newline at end of file diff --git a/cdk-deployment/lambda/lambda.py b/cdk-deployment/lambda/lambda.py index 70c605ed..de892bd9 100644 --- a/cdk-deployment/lambda/lambda.py +++ b/cdk-deployment/lambda/lambda.py @@ -1,8 +1,8 @@ # import json # import tempfile -import Mangum from fastapi import FastAPI +from mangum import Mangum # from stac_validator import stac_validator diff --git a/cdk-deployment/validator_cdk/validator_cdk_stack.py b/cdk-deployment/validator_cdk/validator_cdk_stack.py index ba0b8f0c..bd6a7dd5 100644 --- a/cdk-deployment/validator_cdk/validator_cdk_stack.py +++ b/cdk-deployment/validator_cdk/validator_cdk_stack.py @@ -14,7 +14,7 @@ def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: all_lib = _lambda.LayerVersion( self, "all-lib-layer", - code=_lambda.AssetCode("lambda/libraries.zip"), + code=_lambda.AssetCode("build-libraries/libraries.zip"), compatible_runtimes=[_lambda.Runtime.PYTHON_3_8], ) From dcc6aed33cc03b87879b83b9e8f38525fd03c89f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 24 Apr 2021 14:04:04 -0700 Subject: [PATCH 03/11] fastapi works in lambda --- .gitignore | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index fae014a4..fca284e4 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,6 @@ venv.bak/ .vscode cdk-deployment/lambda/stac_validator/ -cdk-deployment/lambda/libraries.zip +cdk-deployment/build-libraries/libraries.zip Pipfile* \ No newline at end of file diff --git a/Makefile b/Makefile index 571cdd33..43e27f29 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ build: ## Build a Docker container build-libraries: # Build the libraries for layers. Used internally docker build -f "cdk-deployment/build-libraries/Dockerfile-libraries" -t lambdalayer:latest . docker run -d -it --name lambdalayer lambdalayer:latest - docker cp lambdalayer:code/libraries.zip ./cdk-deployment/lambda + docker cp lambdalayer:code/libraries.zip ./cdk-deployment/build-libraries docker stop lambdalayer docker rm lambdalayer docker rmi lambdalayer From bb284ea93b9fb5ab5460a3e309607093caf4bcd9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 24 Apr 2021 15:07:51 -0700 Subject: [PATCH 04/11] add GET route for stac url --- cdk-deployment/lambda/lambda.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/cdk-deployment/lambda/lambda.py b/cdk-deployment/lambda/lambda.py index de892bd9..4142be88 100644 --- a/cdk-deployment/lambda/lambda.py +++ b/cdk-deployment/lambda/lambda.py @@ -1,12 +1,25 @@ # import json -# import tempfile from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware from mangum import Mangum -# from stac_validator import stac_validator +from stac_validator import stac_validator + +# import tempfile + -app = FastAPI() +# app = FastAPI() + +app = FastAPI(title="STAC Validator", version=0.1, root_path="/prod/") + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) @app.get("/hello") @@ -14,6 +27,14 @@ async def root(): return {"message": "Hello World"} +@app.get("/validate") +async def validate(stac_url): + stac = stac_validator.StacValidate(str(stac_url)) + stac.run() + output = stac.message[0] + return {"body": output} + + # def handler(event, context): # body = json.loads(event["body"]) # if body.get("stac_url"): From 72e79ca0f4e602c0b5171b178b9a981b72f8547f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 24 Apr 2021 16:03:46 -0700 Subject: [PATCH 05/11] POST and GET requests working --- cdk-deployment/lambda/lambda.py | 59 +++++++++++---------------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/cdk-deployment/lambda/lambda.py b/cdk-deployment/lambda/lambda.py index 4142be88..503a5b9b 100644 --- a/cdk-deployment/lambda/lambda.py +++ b/cdk-deployment/lambda/lambda.py @@ -1,16 +1,12 @@ -# import json +import json +import tempfile -from fastapi import FastAPI +from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from mangum import Mangum from stac_validator import stac_validator -# import tempfile - - -# app = FastAPI() - app = FastAPI(title="STAC Validator", version=0.1, root_path="/prod/") app.add_middleware( @@ -28,45 +24,28 @@ async def root(): @app.get("/validate") -async def validate(stac_url): +async def validate_url(stac_url): stac = stac_validator.StacValidate(str(stac_url)) stac.run() output = stac.message[0] return {"body": output} -# def handler(event, context): -# body = json.loads(event["body"]) -# if body.get("stac_url"): -# body = body["stac_url"] -# else: -# # TODO: Look at alains devops template. store in mem -# temp_stac_file = tempfile.NamedTemporaryFile( -# delete=False, -# mode="w+", -# ) -# json.dump(body, temp_stac_file) -# temp_stac_file.flush() -# temp_stac_file.close() -# body = temp_stac_file.name -# stac = stac_validator.StacValidate(body) -# stac.run() - -# output = stac.message[0] - -# if "validation_method" in output: -# output.pop("validation_method") +@app.post("/json") +async def validate_json(request: Request): + stac_file = await request.json() + temp_stac_file = tempfile.NamedTemporaryFile( + delete=False, + mode="w+", + ) + json.dump(stac_file, temp_stac_file) + temp_stac_file.flush() + temp_stac_file.close() + body = temp_stac_file.name + stac = stac_validator.StacValidate(body) + stac.run() + output = stac.message[0] + return {"body": output} -# return { -# "statusCode": 200, -# "isBase64Encoded": False, -# "headers": { -# "Content-Type": "application/json", -# "Access-Control-Allow-Headers": "Content-Type", -# "Access-Control-Allow-Origin": "*", -# "Access-Control-Allow-Methods": "OPTIONS,POST,GET", -# }, -# "body": json.dumps(output), -# } handler = Mangum(app) From d83d9b13cf70d060a46e6566e9666a49b9299ef4 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 24 Apr 2021 16:05:26 -0700 Subject: [PATCH 06/11] remove hello world path --- cdk-deployment/lambda/lambda.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cdk-deployment/lambda/lambda.py b/cdk-deployment/lambda/lambda.py index 503a5b9b..2fbe77e9 100644 --- a/cdk-deployment/lambda/lambda.py +++ b/cdk-deployment/lambda/lambda.py @@ -18,11 +18,6 @@ ) -@app.get("/hello") -async def root(): - return {"message": "Hello World"} - - @app.get("/validate") async def validate_url(stac_url): stac = stac_validator.StacValidate(str(stac_url)) From bc46634e944754a8cb252b4b826b38a0657238da Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 24 Apr 2021 16:22:07 -0700 Subject: [PATCH 07/11] change get endpoint name to url --- cdk-deployment/lambda/lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdk-deployment/lambda/lambda.py b/cdk-deployment/lambda/lambda.py index 2fbe77e9..f615cf32 100644 --- a/cdk-deployment/lambda/lambda.py +++ b/cdk-deployment/lambda/lambda.py @@ -18,7 +18,7 @@ ) -@app.get("/validate") +@app.get("/url") async def validate_url(stac_url): stac = stac_validator.StacValidate(str(stac_url)) stac.run() From 0d45a337e40877d212c67b0abe951c407272da94 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 25 Apr 2021 15:29:21 -0700 Subject: [PATCH 08/11] add url POST route --- cdk-deployment/lambda/lambda.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cdk-deployment/lambda/lambda.py b/cdk-deployment/lambda/lambda.py index f615cf32..158ee48b 100644 --- a/cdk-deployment/lambda/lambda.py +++ b/cdk-deployment/lambda/lambda.py @@ -7,7 +7,7 @@ from stac_validator import stac_validator -app = FastAPI(title="STAC Validator", version=0.1, root_path="/prod/") +app = FastAPI(title="STAC Validator", version=2.0, root_path="/prod/") app.add_middleware( CORSMiddleware, @@ -19,7 +19,15 @@ @app.get("/url") -async def validate_url(stac_url): +async def validate_url_get_request(stac_url): + stac = stac_validator.StacValidate(str(stac_url)) + stac.run() + output = stac.message[0] + return {"body": output} + + +@app.post("/url") +async def validate_url_post_request(stac_url): stac = stac_validator.StacValidate(str(stac_url)) stac.run() output = stac.message[0] From cf84624e81930e1d27bec42f875c9f97d3c16efa Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 26 Apr 2021 09:44:17 -0700 Subject: [PATCH 09/11] streamline lambda --- cdk-deployment/lambda/lambda.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cdk-deployment/lambda/lambda.py b/cdk-deployment/lambda/lambda.py index 158ee48b..f1974ea8 100644 --- a/cdk-deployment/lambda/lambda.py +++ b/cdk-deployment/lambda/lambda.py @@ -18,19 +18,22 @@ ) -@app.get("/url") -async def validate_url_get_request(stac_url): - stac = stac_validator.StacValidate(str(stac_url)) +def validate(stac_object): + stac = stac_validator.StacValidate(stac_object) stac.run() output = stac.message[0] + return output + + +@app.get("/url") +async def validate_url_get_request(stac_url): + output = validate(str(stac_url)) return {"body": output} @app.post("/url") async def validate_url_post_request(stac_url): - stac = stac_validator.StacValidate(str(stac_url)) - stac.run() - output = stac.message[0] + output = validate(str(stac_url)) return {"body": output} @@ -45,9 +48,7 @@ async def validate_json(request: Request): temp_stac_file.flush() temp_stac_file.close() body = temp_stac_file.name - stac = stac_validator.StacValidate(body) - stac.run() - output = stac.message[0] + output = validate(body) return {"body": output} From 1c192bf944dda4c87e425b50d1c91a619ecfecb1 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 26 Apr 2021 13:30:25 -0700 Subject: [PATCH 10/11] clean up setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2cd1c8a3..38a7dab3 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name="stac_validator", version=__version__, - author="James Banting, Darren Wiens, Jonathan Healy", + author="James Banting, Jonathan Healy", author_email="jhealy@sparkgeo.com", description="A package to validate STAC files", license="MIT", From fe6ea99a39e6ff87cb01dbfba0ba330416336f96 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 26 Apr 2021 18:00:28 -0700 Subject: [PATCH 11/11] update for 2.0.0 pypi release --- .../validator_cdk/validator_cdk_stack.py | 14 +++++++------- pyproject.toml | 8 ++++++++ requirements-dev.txt | 3 +++ setup.cfg | 2 -- setup.py | 9 ++------- 5 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt delete mode 100644 setup.cfg diff --git a/cdk-deployment/validator_cdk/validator_cdk_stack.py b/cdk-deployment/validator_cdk/validator_cdk_stack.py index bd6a7dd5..e4fa9380 100644 --- a/cdk-deployment/validator_cdk/validator_cdk_stack.py +++ b/cdk-deployment/validator_cdk/validator_cdk_stack.py @@ -7,26 +7,26 @@ from aws_cdk import core as cdk -class FastAPICdkStack(cdk.Stack): +class ValidatorCdkStack(cdk.Stack): def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) - all_lib = _lambda.LayerVersion( + val_lib = _lambda.LayerVersion( self, - "all-lib-layer", + "validator-library-layer", code=_lambda.AssetCode("build-libraries/libraries.zip"), compatible_runtimes=[_lambda.Runtime.PYTHON_3_8], ) # Defines an AWS Lambda resource - fast_api_lambda = _lambda.Function( + validator_lambda = _lambda.Function( self, - "STACFastAPI", + "STACValidator", runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.asset("lambda"), handler="lambda.handler", timeout=cdk.Duration.seconds(30), - layers=[all_lib], + layers=[val_lib], ) cors = apigw.CorsOptions(allow_origins=["*"]) @@ -34,6 +34,6 @@ def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: apigw.LambdaRestApi( self, "Endpoint", - handler=fast_api_lambda, + handler=validator_lambda, default_cors_preflight_options=cors, ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..2f77e63b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + "requests", + "jsonschema", + "pystac", + "click", +] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..a9662a59 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +"pytest" +"pytest-mypy" +"pre-commit" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 224a7795..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py index 38a7dab3..de3fabd2 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -try: - from setuptools import setup -except ImportError: - from distutils.core import setup +from setuptools import setup __version__ = "2.0.0" @@ -38,13 +35,11 @@ "jsonschema==3.2.0", "pystac==0.5.6", "click==7.1.2", - "pytest", - "pytest-mypy", - "pre-commit", ], packages=["stac_validator"], entry_points={ "console_scripts": ["stac_validator = stac_validator.stac_validator:main"] }, + python_requires=">=3.6", tests_require=["pytest"], )