From ac765e0ba209f21808ee8d91ef9aa9edfded07b4 Mon Sep 17 00:00:00 2001 From: Patrick Huck Date: Tue, 5 Apr 2022 15:41:25 -0700 Subject: [PATCH] start mpcontribs-serverless --- mpcontribs-sam-download/README.md | 7 -- .../make_download/__init__.py | 0 .../make_download/requirements.txt | 3 - mpcontribs-sam-download/template.yaml | 56 ------------- mpcontribs-serverless/Makefile | 59 +++++++++++++ mpcontribs-serverless/README.md | 80 ++++++++++++++++++ mpcontribs-serverless/dependencies/Dockerfile | 7 ++ .../dependencies/requirements.txt | 3 + .../events/make_download.json | 0 .../make_download}/__init__.py | 0 .../make_download/app.py | 0 .../samconfig.toml | 5 +- mpcontribs-serverless/template.yaml | 83 +++++++++++++++++++ 13 files changed, 235 insertions(+), 68 deletions(-) delete mode 100644 mpcontribs-sam-download/README.md delete mode 100644 mpcontribs-sam-download/make_download/__init__.py delete mode 100644 mpcontribs-sam-download/make_download/requirements.txt delete mode 100644 mpcontribs-sam-download/template.yaml create mode 100644 mpcontribs-serverless/Makefile create mode 100644 mpcontribs-serverless/README.md create mode 100644 mpcontribs-serverless/dependencies/Dockerfile create mode 100644 mpcontribs-serverless/dependencies/requirements.txt rename mpcontribs-sam-download/events/event.json => mpcontribs-serverless/events/make_download.json (100%) rename {mpcontribs-sam-download => mpcontribs-serverless/make_download}/__init__.py (100%) rename {mpcontribs-sam-download => mpcontribs-serverless}/make_download/app.py (100%) rename {mpcontribs-sam-download => mpcontribs-serverless}/samconfig.toml (51%) create mode 100644 mpcontribs-serverless/template.yaml diff --git a/mpcontribs-sam-download/README.md b/mpcontribs-sam-download/README.md deleted file mode 100644 index d3a7e3d83..000000000 --- a/mpcontribs-sam-download/README.md +++ /dev/null @@ -1,7 +0,0 @@ -```bash -sam build -u -b ~/.aws-sam --docker-network docker_internet --parallel --region us-east-1 --debug --cached --cache-dir ~/.aws-sam/cache -sam local invoke -t ~/.aws-sam/template.yaml --docker-network docker_internet -e events/event.json -sam deploy # --guided -sam logs -n MPContribsMakeDownloadFunction --stack-name mpcontribs-sam-download --tail -# aws cloudformation delete-stack --stack-name mpcontribs-sam-download -``` diff --git a/mpcontribs-sam-download/make_download/__init__.py b/mpcontribs-sam-download/make_download/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/mpcontribs-sam-download/make_download/requirements.txt b/mpcontribs-sam-download/make_download/requirements.txt deleted file mode 100644 index 6fb9c93bf..000000000 --- a/mpcontribs-sam-download/make_download/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -mpcontribs-client -ddtrace -redis diff --git a/mpcontribs-sam-download/template.yaml b/mpcontribs-sam-download/template.yaml deleted file mode 100644 index fc113041a..000000000 --- a/mpcontribs-sam-download/template.yaml +++ /dev/null @@ -1,56 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 - -Parameters: - LogLevel: - Type: String - AllowedValues: - - DEBUG - - INFO - - WARNING - Default: WARNING - RedisAddress: - Type: String - Default: redis/13 - MemorySize: - Type: Number - Default: 1024 - Timeout: - Type: Number - Default: 900 - -Globals: - Function: - Handler: app.lambda_handler - Runtime: python3.9 - MemorySize: !Ref MemorySize - Timeout: !Ref Timeout - EventInvokeConfig: - MaximumRetryAttempts: 0 - VpcConfig: - SecurityGroupIds: - - sg-0c15eb3bae6647d91 - SubnetIds: - - subnet-064fe19cf24399bf4 - - subnet-037dc2c4a90f16ddb - Architectures: - - x86_64 - -Resources: - MPContribsMakeDownloadFunction: - Type: AWS::Serverless::Function - Properties: - CodeUri: make_download/ - Environment: - Variables: - MPCONTRIBS_CLIENT_LOG_LEVEL: !Ref LogLevel - REDIS_ADDRESS: !Ref RedisAddress - LAMBDA_TIMEOUT: !Ref Timeout - -Outputs: - MPContribsMakeDownloadFunction: - Description: "MPContribsMakeDownloadFunction Lambda Function ARN" - Value: !GetAtt MPContribsMakeDownloadFunction.Arn - MPContribsMakeDownloadFunctionIamRole: - Description: "Implicit IAM Role created for MPContribsMakeDownloadFunction" - Value: !GetAtt MPContribsMakeDownloadFunctionRole.Arn diff --git a/mpcontribs-serverless/Makefile b/mpcontribs-serverless/Makefile new file mode 100644 index 000000000..130a7e552 --- /dev/null +++ b/mpcontribs-serverless/Makefile @@ -0,0 +1,59 @@ +BASE := $(shell /bin/pwd) + +############# +# SAM vars # +############# + +# Name of Docker Network to connect to +# Helpful when you're running Amazon DynamoDB local etc. +NETWORK = docker_internet +BUILDDIR = ${HOME}/.aws-sam +REGION = us-east-1 + +target: + $(info ${HELP_MESSAGE}) + @exit 0 + +clean: ##=> Deletes current build environment and latest build + $(info [*] Who needs all that anyway? Destroying environment....) + rm -rf ${BUILDDIR} + +all: clean build + +build: ##=> Same as package except that we don't create a ZIP + sam build --use-container -b ${BUILDDIR} --docker-network ${NETWORK} --parallel --region ${REGION} --cached --cache-dir ${BUILDDIR}/cache + +deploy.guided: ##=> Guided deploy that is typically run for the first time only + sam deploy --guided --template-file ${BUILDDIR}/template.yaml + +deploy: ##=> Deploy app using previously saved SAM CLI configuration + sam deploy --template-file ${BUILDDIR}/template.yaml --config-file ${PWD}/samconfig.toml + +invoke: ##=> Run SAM Local function with a given event payload + sam local invoke --event events/make_download.json -t ${BUILDDIR}/template.yaml --docker-network ${NETWORK} + +############# +# Helpers # +############# + +define HELP_MESSAGE + Environment variables to be aware of or to hardcode depending on your use case: + + NETWORK + Default: "" + Info: Docker Network to connect to when running Lambda function locally + + Common usage: + + ...::: Builds Lambda function dependencies:::... + $ make build + + ...::: Deploy for the first time :::... + $ make deploy.guided + + ...::: Deploy subsequent changes :::... + $ make deploy + + ...::: Cleans up the environment - Deletes Virtualenv, ZIP builds and Dev env :::... + $ make clean +endef diff --git a/mpcontribs-serverless/README.md b/mpcontribs-serverless/README.md new file mode 100644 index 000000000..949abf886 --- /dev/null +++ b/mpcontribs-serverless/README.md @@ -0,0 +1,80 @@ +- **`make_download`** - Code for the application's Lambda function. +- **`events`** - Invocation events that you can use to invoke the function. +- **`tests`** - Unit tests for the application code. +- **`template.yaml`** - A template that defines the application's AWS resources. +- **`Makefile`** - Makefile for your convenience to install deps, build, invoke, and deploy your application. + +## Deploy the sample application + +> **Already know this sample? Run: `make hurry` - This command will install app deps, build, and deploy your Serverless application using SAM.** + +Build and deploy your application for the first time by running the following commands in your shell: + +```bash +mpcontribs-serverless$ make build +mpcontribs-serverless$ make deploy.guided +``` + +The first command will **build** the source of your application within a Docker container. The second command will **package and deploy** your application to AWS. Guided deploy means SAM CLI will ask you about the name of your deployment/stack, AWS Region, and whether you want to save your choices, so that you can use `make deploy` next time. + +## Use the SAM CLI to build and test locally + +Whenever you change your application code, you'll have to run build command: + +```bash +mpcontribs-serverless$ make build +``` + +The SAM CLI installs dependencies defined in `make_download/requirements.txt`, creates a deployment package, and saves it in the `.aws-sam/build` folder. + +Test a single function by invoking it directly with a test event: + +```bash +mpcontribs-serverless$ make invoke +``` + +> An event is a JSON document that represents the input that the function receives from the event source. Test events are included in the `events` folder in this project. + +## Fetch, tail, and filter Lambda function logs + +To simplify troubleshooting, SAM CLI has a command called `sam logs`. `sam logs` lets you fetch logs generated by your deployed Lambda function from the command line. In addition to printing the logs on the terminal, this command has several nifty features to help you quickly find the bug. + +```bash +mpcontribs-serverless$ sam logs -n MPContribsMakeDownloadFunction --stack-name --tail +``` + +## Unit tests + +Tests are defined in the `tests` folder in this project, and we use Pytest as the test runner for this sample project. + +Make sure you install dev dependencies before you run tests with `make dev`: + +```bash +mpcontribs-serverless$ make dev +mpcontribs-serverless$ make test +``` + +## Cleanup + +```bash +mpcontribs-serverless$ aws cloudformation delete-stack --stack-name mpcontribs-serverless +``` + +# Appendix + +## Makefile + +We included a `Makefile` for your convenience - You can find all commands you can use by running `make`. Under the hood, we're using SAM CLI commands to run these common tasks: + +* **`make build`**: `sam build --use-container` +* **`make deploy.guided`**: `sam deploy --guided` +* **`make invoke`**: `sam local invoke MPContribsMakeDownloadFunction --event events/make_download.json` +* **`make run`**: `sam local start-api` + +## Sync project with function dependencies + +Pipenv takes care of isolating dev dependencies and app dependencies. As SAM CLI requires a `requirements.txt` file, you'd need to generate one if new app dependencies have been added: + +```bash +mpcontribs-serverless$ pipenv lock -r > make_download/requirements.txt +``` diff --git a/mpcontribs-serverless/dependencies/Dockerfile b/mpcontribs-serverless/dependencies/Dockerfile new file mode 100644 index 000000000..cf0887d19 --- /dev/null +++ b/mpcontribs-serverless/dependencies/Dockerfile @@ -0,0 +1,7 @@ +FROM public.ecr.aws/lambda/python:3.9 + +ENV BUILDDIR /dependencies/python +RUN yum install gcc -y && mkdir $BUILDDIR +COPY requirements.txt . +ENTRYPOINT /bin/bash +CMD -c "pip install -r requirements.txt -t $BUILDDIR" diff --git a/mpcontribs-serverless/dependencies/requirements.txt b/mpcontribs-serverless/dependencies/requirements.txt new file mode 100644 index 000000000..5a03947f7 --- /dev/null +++ b/mpcontribs-serverless/dependencies/requirements.txt @@ -0,0 +1,3 @@ +mpcontribs-client==4.2.10 +ddtrace==0.60.1 +redis==4.2.1 diff --git a/mpcontribs-sam-download/events/event.json b/mpcontribs-serverless/events/make_download.json similarity index 100% rename from mpcontribs-sam-download/events/event.json rename to mpcontribs-serverless/events/make_download.json diff --git a/mpcontribs-sam-download/__init__.py b/mpcontribs-serverless/make_download/__init__.py similarity index 100% rename from mpcontribs-sam-download/__init__.py rename to mpcontribs-serverless/make_download/__init__.py diff --git a/mpcontribs-sam-download/make_download/app.py b/mpcontribs-serverless/make_download/app.py similarity index 100% rename from mpcontribs-sam-download/make_download/app.py rename to mpcontribs-serverless/make_download/app.py diff --git a/mpcontribs-sam-download/samconfig.toml b/mpcontribs-serverless/samconfig.toml similarity index 51% rename from mpcontribs-sam-download/samconfig.toml rename to mpcontribs-serverless/samconfig.toml index c20c3acef..c288debbc 100644 --- a/mpcontribs-sam-download/samconfig.toml +++ b/mpcontribs-serverless/samconfig.toml @@ -2,10 +2,11 @@ version = 0.1 [default] [default.deploy] [default.deploy.parameters] -stack_name = "mpcontribs-sam-download" +stack_name = "mpcontribs-serverless" s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-1s64twonjniyf" -s3_prefix = "mpcontribs-sam-download" +s3_prefix = "mpcontribs-serverless" region = "us-east-1" confirm_changeset = true capabilities = "CAPABILITY_IAM" +parameter_overrides = "LogLevel=\"WARNING\" RedisAddress=\"mat-el-1k23v7u8usk2p.xljae8.0001.use1.cache.amazonaws.com/13\" MemorySize=\"1024\" Timeout=\"900\"" image_repositories = [] diff --git a/mpcontribs-serverless/template.yaml b/mpcontribs-serverless/template.yaml new file mode 100644 index 000000000..cf36b15c0 --- /dev/null +++ b/mpcontribs-serverless/template.yaml @@ -0,0 +1,83 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: mpcontribs-serverless + +Parameters: + LogLevel: + Type: String + AllowedValues: + - DEBUG + - INFO + - WARNING + Default: WARNING + RedisAddress: + Type: String + Default: redis/13 + MemorySize: + Type: Number + Default: 1024 + Timeout: + Type: Number + Default: 900 + +Globals: + Function: + Handler: app.lambda_handler + MemorySize: !Ref MemorySize + Timeout: !Ref Timeout + Runtime: python3.9 + EventInvokeConfig: + MaximumRetryAttempts: 0 + VpcConfig: + SecurityGroupIds: + - sg-0c15eb3bae6647d91 + SubnetIds: + - subnet-064fe19cf24399bf4 + - subnet-037dc2c4a90f16ddb + Architectures: + - x86_64 + #FileSystemConfigs: + # - Arn: arn:aws:elasticfilesystem:us-east-1:416700756612:access-point/fsap-0717fb111fead3950 + # LocalMountPath: /mnt/efs + +Resources: + MPContribsMakeDownloadFunction: + Type: AWS::Serverless::Function + Properties: + FunctionName: mpcontribs-make-download + CodeUri: make_download/ + Description: make downloads for MPContribs projects + Policies: + - Version: "2012-10-17" + Statement: + - Sid: AWSLambdaVPCAccessExecutionRole + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ec2:CreateNetworkInterface + - ec2:DescribeNetworkInterfaces + - ec2:DeleteNetworkInterface + Resource: "*" + - Sid: AmazonElasticFileSystemClientFullAccess + Effect: Allow + Action: + - elasticfilesystem:ClientMount + - elasticfilesystem:ClientRootAccess + - elasticfilesystem:ClientWrite + - elasticfilesystem:DescribeMountTargets + Resource: "*" + Environment: + Variables: + MPCONTRIBS_CLIENT_LOG_LEVEL: !Ref LogLevel + REDIS_ADDRESS: !Ref RedisAddress + LAMBDA_TIMEOUT: !Ref Timeout + +Outputs: + MPContribsMakeDownloadFunction: + Description: "MPContribsMakeDownloadFunction Lambda Function ARN" + Value: !GetAtt MPContribsMakeDownloadFunction.Arn + MPContribsMakeDownloadFunctionIamRole: + Description: "Implicit IAM Role created for MPContribsMakeDownloadFunction" + Value: !GetAtt MPContribsMakeDownloadFunctionRole.Arn