diff --git a/kinesis-lambda-terraform/README.md b/kinesis-lambda-terraform/README.md new file mode 100644 index 0000000000..ef9105e2c6 --- /dev/null +++ b/kinesis-lambda-terraform/README.md @@ -0,0 +1,86 @@ +# AWS Kinesis Data Streams to AWS Lambda + +This pattern creates an AWS Kinesis Data Stream, a stream consumer, and an AWS Lambda function. When data is added to the stream, the Lambda function is invoked. + +Learn more about this pattern at Serverless Land Patterns: [https://serverlessland.com/patterns/kinesis-to-lambda-terraform](https://serverlessland.com/patterns/kinesis-to-lambda-terraform) + +Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example. + +## Requirements + +* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources. +* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured +* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) +* [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started) installed + +## Deployment Instructions + +1. Clone the project to your local working directory + + ```sh + git clone https://github.com/aws-samples/serverless-patterns/ + ``` + +2. Change the working directory to this pattern's directory + + ```sh + cd serverless-patterns/kinesis-lambda-terraform + ``` + +1. From the command line, initialize terraform to to downloads and installs the providers defined in the configuration: + ``` + terraform init + ``` +1. From the command line, apply the configuration in the main.tf file: + ``` + terraform apply + ``` +1. During the prompts: + + - Enter yes + +## Testing + +Use the Amazon Kinesis Data Generator for testing. The easiest way to use this tool is to use the [hosted generator](https://awslabs.github.io/amazon-kinesis-data-generator/web/producer.html) and follow the [setup instructions](https://awslabs.github.io/amazon-kinesis-data-generator/web/help.html). + +After you have the generator configured, you should have a custom URL to generate data for your Kinesis data stream. In your configuration steps, you created a username and password. Log in to the generator using those credentials. + +When you are logged in, you can generate data for your stream test. + +1. Choose the region you deployed the application to +2. For Stream/delivery stream, select your stream. +3. For Records per second, keep the default value of 100. +4. On the Template 1 tab, name the template Sensor1. +5. Use the following template: + ```JSON + { + "sensorId": {{random.number(50)}}, + "currentTemperature": {{random.number( + { + "min":10, + "max":150 + } + )}}, + "status": "{{random.arrayElement( + ["OK","FAIL","WARN"] + )}}" + } + ``` +6. Choose Send Data. +7. After several seconds, choose Stop Sending Data. + +## Cleanup + +1. Delete all created resources by terraform + ```bash + terraform destroy + ``` +2. During the prompts: + * Enter yes +3. Confirm all created resources has been deleted + ```bash + terraform show + ``` + +---- +Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. \ No newline at end of file diff --git a/kinesis-lambda-terraform/example-pattern.json b/kinesis-lambda-terraform/example-pattern.json new file mode 100644 index 0000000000..25156dbbd9 --- /dev/null +++ b/kinesis-lambda-terraform/example-pattern.json @@ -0,0 +1,58 @@ +{ + "title": "Kinesis Data Streams With Lambda Consumer", + "description": "When data is loaded into Kinesis data stream, the configured lambda function is invoked.", + "language": "", + "level": "200", + "framework": "Terraform", + "introBox": { + "headline": "How it works", + "text": [ + "This pattern creates a Kinesis Data Stream and lambda function (consumer) with event source mapping. ", + "When the data is loaded into the stream, the lambda function is invoked. " + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/kinesis-lambda-terraform", + "templateURL": "serverless-patterns/kinesis-lambda-terraform", + "projectFolder": "kinesis-lambda-terraform", + "templateFile": "main.tf" + } + }, + "resources": { + "bullets": [ + { + "text": "Kinesis Data Streams with Lambda consumers", + "link": "https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html" + }, + { + "text": "Lambda event source mapping", + "link": "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping.html" + } + ] + }, + "deploy": { + "text": [ + "terraform apply" + ] + }, + "testing": { + "text": [ + "See the GitHub repo for detailed testing instructions." + ] + }, + "cleanup": { + "text": [ + "Delete the stack: terraform destroy." + ] + }, + "authors": [ + { + "name": "Rahul Sringeri", + "image": "", + "bio": "Technical Account Manager at AWS for strategic accounts.", + "linkedin": "", + "twitter": "" + } + ] + } \ No newline at end of file diff --git a/kinesis-lambda-terraform/main.tf b/kinesis-lambda-terraform/main.tf new file mode 100644 index 0000000000..d9554945e3 --- /dev/null +++ b/kinesis-lambda-terraform/main.tf @@ -0,0 +1,57 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.22" + } + } + + required_version = ">= 0.14.9" +} + +provider "aws" { + region = "us-east-1" # Change to your desired region +} +resource "aws_kinesis_stream" "sample_stream" { + name = "sample-stream" + shard_count = 1 + retention_period = 24 +} +resource "aws_lambda_function" "sample_lambda" { + filename = "sample_lambda.zip" # Path to your Lambda code ZIP file + function_name = "sample-lambda" + role = aws_iam_role.lambda_role.arn + handler = "index.handler" + runtime = "nodejs14.x" # Change to your preferred runtime +} +resource "aws_iam_role" "lambda_role" { + name = "lambda-role" + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) +} + +resource "aws_lambda_event_source_mapping" "sample_mapping" { + event_source_arn = aws_kinesis_stream.sample_stream.arn + function_name = aws_lambda_function.sample_lambda.arn + starting_position = "LATEST" +} + +output "kinesis_data_stream" { + value = aws_kinesis_stream.sample_stream.arn + description = "Kinesis data stream with shards" +} + +output "consumer_function" { + value = aws_lambda_function.sample_function.arn + description = "Consumer Function function name" +} \ No newline at end of file diff --git a/kinesis-lambda-terraform/src/app.js b/kinesis-lambda-terraform/src/app.js new file mode 100644 index 0000000000..0aeebf5d5e --- /dev/null +++ b/kinesis-lambda-terraform/src/app.js @@ -0,0 +1,15 @@ +//Sample lambda code + +exports.handler = async (event) => { + console.log('Received Kinesis records:', JSON.stringify(event, null, 2)); + // Process each Kinesis record + event.Records.forEach((record) => { + const payload = Buffer.from(record.kinesis.data, 'base64').toString('utf-8'); + console.log('Record payload:', payload); + // Add your processing logic here + }); + return { + statusCode: 200, + body: 'Processed Kinesis records successfully', + }; + }; \ No newline at end of file