diff --git a/eventbridge-pipes-sqs-to-timestream/README.md b/eventbridge-pipes-sqs-to-timestream/README.md new file mode 100644 index 000000000..233acbfb7 --- /dev/null +++ b/eventbridge-pipes-sqs-to-timestream/README.md @@ -0,0 +1,66 @@ +# SQS Queue to Amazon Timestream with AWS EventBridge Pipes + +This pattern demostrates the process of ingesting data into Amazon Timestream from SQS Queue via EventBridge Pipes. + +Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/eventbridge-pipes-sqs-to-timestream + +Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example. + +## Requirements + +* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources. +* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured +* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) +* [AWS Serverless Application Model](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) (AWS SAM) installed + +## Deployment Instructions + +1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository: + ``` + git clone https://github.com/aws-samples/serverless-patterns + ``` +1. Change directory to the pattern directory: + ``` + cd eventbridge-pipes-sqs-to-timestream + ``` +1. From the command line, use AWS SAM to deploy the AWS resources for the pattern as specified in the template.yml file: + ``` + sam deploy --guided + ``` +1. During the prompts: + * Enter a stack name + * Enter the desired AWS Region + * Allow SAM CLI to create IAM roles with the required permissions. + + Once you have run `sam deploy --guided` mode once and saved arguments to a configuration file (samconfig.toml), you can use `sam deploy` in future to use these defaults. + +1. Note the outputs from the SAM deployment process. These contain the resource names and/or ARNs which are used for testing. + +## How it works + +The template will create Timestream table, SQS queue, Cloudwatch LogGroup and IAM Role, and connect it together with a new EventBridge Pipe. + +When records are sent to the source queue, the Pipe will convert into a valid Timestream record and ingest it to the Timestream table. + +Please follow [this documentation](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes.html) for more information. + +## Testing + +``` +sh command.sh +``` + +## Cleanup + +1. Delete the stack + ```bash + aws cloudformation delete-stack --stack-name STACK_NAME + ``` +1. Confirm the stack has been deleted + ```bash + aws cloudformation list-stacks --query "StackSummaries[?contains(StackName,'STACK_NAME')].StackStatus" + ``` +---- +Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +SPDX-License-Identifier: MIT-0 \ No newline at end of file diff --git a/eventbridge-pipes-sqs-to-timestream/command.sh b/eventbridge-pipes-sqs-to-timestream/command.sh new file mode 100644 index 000000000..4d6c723a5 --- /dev/null +++ b/eventbridge-pipes-sqs-to-timestream/command.sh @@ -0,0 +1,4 @@ + +DATE_TIME=$(date -u '+%Y-%m-%d %H:%M:%S.000') +DATA="{\"dimension_1\": \"dimension1\", \"measure_1\": \"1.0\", \"time\": \"${DATE_TIME}\", \"version\": \"1\" }" +aws sqs send-message --queue-url "$1" --message-body "$DATA" --region "$2" \ No newline at end of file diff --git a/eventbridge-pipes-sqs-to-timestream/eventbridge-pipes-sqs-to-timestream.json b/eventbridge-pipes-sqs-to-timestream/eventbridge-pipes-sqs-to-timestream.json new file mode 100644 index 000000000..fabf13db2 --- /dev/null +++ b/eventbridge-pipes-sqs-to-timestream/eventbridge-pipes-sqs-to-timestream.json @@ -0,0 +1,89 @@ +{ + "title": "SQS to Timestream using EventBridge Pipes", + "description": "This pattern demostrates the process of ingesting data into Amazon Timestream from SQS Queue via EventBridge Pipes.", + "language": "YAML", + "level": "200", + "framework": "CDK", + "introBox": { + "headline": "How it works", + "text": [ + "This project demostrates how to send data to Amazon Timestream directly from SQS. This pattern uses EventBridge Pipes to connect SQS and Timestream together.", + "The pattern creates an Amazon SQS queue, a Timestream table, Cloudwatch LogGroup, IAM Policy and an EventBridge Pipe." + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/eventbridge-pipes-sqs-to-timestream", + "templateURL": "serverless-patterns/eventbridge-pipes-sqs-to-timestream", + "projectFolder": "eventbridge-pipes-sqs-to-timestream", + "templateFile": "template.yaml" + } + }, + "resources": { + "bullets": [ + { + "text": "Send SQS data to Timestream", + "link": "https://docs.aws.amazon.com/timestream/latest/developerguide/SQS.html" + }, + { + "text": "Amazon Timestream", + "link": "https://aws.amazon.com/timestream/" + }, + { + "text": "AWS EventBridge Pipes Targets", + "link": "https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes-event-target.html" + } + ] + }, + "deploy": { + "text": [ + "sam deploy" + ] + }, + "testing": { + "text": [ + "See the GitHub repo for detailed testing instructions." + ] + }, + "cleanup": { + "text": [ + "Delete the stack: aws cloudformation delete-stack --stack-name STACK_NAME." + ] + }, + "authors": [ + { + "name": "Aayush Wadhwa", + "image": "https://avatars.githubusercontent.com/u/20082863?s=400&u=aaf983bea1df62c7f9c75869aa0a51a8877edfa8&v=4", + "bio": "Software Development Engineer at AWS", + "linkedin": "aayushwadhwa" + } + ], + "patternArch": { + "icon1": { + "x": 20, + "y": 50, + "service": "sqs", + "label": "Amazon SQS" + }, + "icon2": { + "x": 50, + "y": 50, + "service": "eventbridge-pipes", + "label": "Amazon EventBridge Pipes" + }, + "icon3": { + "x": 80, + "y": 50, + "service": "timestream", + "label": "Amazon Timestream" + }, + "line1": { + "from": "icon1", + "to": "icon2" + }, + "line2": { + "from": "icon2", + "to": "icon3" + } + } +} diff --git a/eventbridge-pipes-sqs-to-timestream/example-pattern.json b/eventbridge-pipes-sqs-to-timestream/example-pattern.json new file mode 100644 index 000000000..7da9c9f2d --- /dev/null +++ b/eventbridge-pipes-sqs-to-timestream/example-pattern.json @@ -0,0 +1,61 @@ +{ + "title": "SQS to Timestream using EventBridge Pipes", + "description": "This pattern demostrates the process of ingesting data into Amazon Timestream from SQS Queue via EventBridge Pipes.", + "language": "YAML", + "level": "200", + "framework": "CDK", + "introBox": { + "headline": "How it works", + "text": [ + "This project demostrates how to send data to Amazon Timestream directly from SQS. This pattern uses EventBridge Pipes to connect SQS and Timestream together.", + "The pattern creates an Amazon SQS queue, a Timestream table, Cloudwatch LogGroup, IAM Policy and an EventBridge Pipe." + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/eventbridge-pipes-sqs-to-timestream", + "templateURL": "serverless-patterns/eventbridge-pipes-sqs-to-timestream", + "projectFolder": "eventbridge-pipes-sqs-to-timestream", + "templateFile": "template.yaml" + } + }, + "resources": { + "bullets": [ + { + "text": "Send SQS data to Timestream", + "link": "https://docs.aws.amazon.com/timestream/latest/developerguide/SQS.html" + }, + { + "text": "Amazon Timestream", + "link": "https://aws.amazon.com/timestream/" + }, + { + "text": "AWS EventBridge Pipes Targets", + "link": "https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes-event-target.html" + } + ] + }, + "deploy": { + "text": [ + "sam deploy" + ] + }, + "testing": { + "text": [ + "See the GitHub repo for detailed testing instructions." + ] + }, + "cleanup": { + "text": [ + "Delete the stack: aws cloudformation delete-stack --stack-name STACK_NAME." + ] + }, + "authors": [ + { + "name": "Aayush Wadhwa", + "image": "https://avatars.githubusercontent.com/u/20082863?s=400&u=aaf983bea1df62c7f9c75869aa0a51a8877edfa8&v=4", + "bio": "Software Development Engineer at AWS", + "linkedin": "https://www.linkedin.com/in/aayushwadhwa/" + } + ] +} diff --git a/eventbridge-pipes-sqs-to-timestream/template.yaml b/eventbridge-pipes-sqs-to-timestream/template.yaml new file mode 100644 index 000000000..7304e811e --- /dev/null +++ b/eventbridge-pipes-sqs-to-timestream/template.yaml @@ -0,0 +1,108 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: Template to connect SQS Queue to Timestream + +Resources: + # Input SQS Queue + Source: + Type: AWS::SQS::Queue + Properties: + QueueName: !Sub ${AWS::StackName}-source + # Target + TimestreamTargetDatabase: + Type: AWS::Timestream::Database + Properties: + DatabaseName: !Sub ${AWS::StackName}-db + TimestreamTargetTable: + Type: AWS::Timestream::Table + Properties: + DatabaseName: !Ref TimestreamTargetDatabase + TableName: !Sub ${AWS::StackName}-tbl + RetentionProperties: + MemoryStoreRetentionPeriodInHours: "168" + MagneticStoreRetentionPeriodInDays: "7" + # Cloudwatch log group for debugging + CloudwatchLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub ${AWS::StackName}-pipe-logs + # IAM Role for Pipe + PipeRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - pipes.amazonaws.com + Action: + - sts:AssumeRole + Policies: + - PolicyName: SourcePolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - sqs:ReceiveMessage + - sqs:DeleteMessage + - sqs:GetQueueAttributes + Resource: !GetAtt Source.Arn + - PolicyName: TargetPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - timestream:WriteRecords + Resource: !GetAtt TimestreamTargetTable.Arn + - PolicyName: DescribeEndpointsPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - timestream:DescribeEndpoints + Resource: '*' + - PolicyName: CloudwatchLogPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - logs:PutLogEvents + Resource: !GetAtt CloudwatchLogGroup.Arn + # Pipe connecting SQS to Timestream + TimestreamPipe: + Type: AWS::Pipes::Pipe + Properties: + Name: !Sub ${AWS::StackName}-pipe + RoleArn: !GetAtt PipeRole.Arn + Source: !GetAtt Source.Arn + SourceParameters: + SqsQueueParameters: + BatchSize: 10 + MaximumBatchingWindowInSeconds: 5 + Target: !GetAtt TimestreamTargetTable.Arn + TargetParameters: + TimestreamParameters: + DimensionMappings: + - DimensionName: dimension_1 + DimensionValue: $.body.dimension_1 + DimensionValueType: VARCHAR + SingleMeasureMappings: + - MeasureName: measure_1 + MeasureValue: $.body.measure_1 + MeasureValueType: DOUBLE + TimeFieldType: TIMESTAMP_FORMAT + TimeValue: $.body.time + TimestampFormat: yyyy-MM-dd HH:mm:ss.SSS + VersionValue: $.body.version + LogConfiguration: + CloudwatchLogsLogDestination: + LogGroupArn: !GetAtt CloudwatchLogGroup.Arn + Level: TRACE + IncludeExecutionData: + - ALL \ No newline at end of file