Merge pull request #9 from brienze1/develop

.prettierrc.js

@@ -1,9 +1,8 @@
 module.exports = {
-    trailingComma: 'es5',
-    tabWidth: 4,
-    semi: true,
-    singleQuote: true,
-    bracketSpacing: true,
-    printWidth: 130
-
+  trailingComma: 'es5',
+  tabWidth: 4,
+  semi: true,
+  singleQuote: true,
+  bracketSpacing: true,
+  printWidth: 130,
 };

Dockerfile

@@ -17,7 +17,7 @@ RUN npm update -g
 # Enter the src directory, install dependencies
 RUN cd src && npm install && npm run build:localstack && npm prune --production
 
-# zip the src directory in the container
-RUN zip -r crypto-robot-data-digest.zip dist node_modules
+## zip the src directory in the container
+#RUN zip -r crypto-robot-data-digest.zip dist node_modules
 
 ENTRYPOINT []

build/cloudformation/crypto-robot-data-digest.yaml

@@ -0,0 +1,203 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: Crypto Robot Data Digest
+
+Parameters:
+  # DB config
+  ReadCapacityUnits:
+    Type: Number
+    Default: 5
+  WriteCapacityUnits:
+    Type: Number
+    Default: 5
+
+  # Tags
+  System:
+    Type: String
+    Default: 'crypto-robot'
+  Parent:
+    Type: String
+    Default: 'crypto-robot-data-digest'
+
+Resources:
+  CryptoDataDigestQueueDLQ:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: 'cryptoDataDigestQueueDLQ'
+    Tags:
+      - Key: type
+        Value: sqs-dlq
+      - Key: system
+        Value: !Ref System
+      - Key: parent
+        Value: !Ref Parent
+
+  CryptoDataDigestQueue:
+    Type: AWS::SQS::Queue
+#    DependsOn: CryptoDataDigestQueueDLQ
+    Properties:
+      QueueName: 'cryptoDataDigestQueue'
+      VisibilityTimeout: 30
+      RedrivePolicy:
+        deadLetterTargetArn: !Sub ${CryptoDataDigestQueueDLQ.Arn}
+        maxReceiveCount: 3
+    Tags:
+      - Key: type
+        Value: sqs
+      - Key: system
+        Value: !Ref System
+      - Key: parent
+        Value: !Ref Parent
+
+  CryptoAnalysisTopicToCryptoDataDigestQueue:
+    Type: AWS::SNS::Subscription
+#    DependsOn: CryptoDataDigestQueue
+    Properties:
+      TopicArn: 'arn:aws:sns:sa-east-1:000000000000:cryptoAnalysisTopic'
+      Endpoint: !Sub ${CryptoDataDigestQueue.Arn}
+      Protocol: sqs
+      RawMessageDelivery: true
+    Tags:
+      - Key: type
+        Value: sns-subscription
+      - Key: system
+        Value: !Ref System
+      - Key: parent
+        Value: !Ref Parent
+
+  CryptoAnalysisSummaryTopic:
+    Type: AWS::SNS::Topic
+    Properties:
+      TopicName: 'cryptoAnalysisSummaryTopic'
+    Tags:
+      - Key: type
+        Value: sns
+      - Key: system
+        Value: !Ref System
+      - Key: parent
+        Value: !Ref Parent
+
+  CryptoRobotAnalyzedDataDynamoDBTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      TableName: 'crypto_robot.analyzed_data'
+      AttributeDefinitions:
+        - AttributeName: 'interval'
+          AttributeType: 'S'
+      KeySchema:
+        - AttributeName: 'interval'
+          KeyType: 'HASH'
+      ProvisionedThroughput:
+        ReadCapacityUnits: !Ref ReadCapacityUnits
+        WriteCapacityUnits: !Ref WriteCapacityUnits
+    Tags:
+      - Key: type
+        Value: table
+      - Key: system
+        Value: !Ref System
+      - Key: parent
+        Value: !Ref Parent
+
+  CryptoDataDigestLambdaRole:
+    Type: AWS::IAM::Role
+#    DependsOn:
+#      - CryptoDataDigestQueue
+#      - CryptoAnalysisSummaryTopic
+#      - CryptoRobotAnalyzedDataDynamoDBTable
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action:
+              - sts:AssumeRole
+      Path: '/'
+      Policies:
+        - PolicyName: logs
+          PolicyDocument:
+            Statement:
+              - Effect: Allow
+                Action:
+                  - logs:CreateLogGroup
+                  - logs:CreateLogStream
+                  - logs:PutLogEvents
+                Resource: arn:aws:logs:*:*:*
+        - PolicyName: sqs
+          PolicyDocument:
+            Statement:
+              - Effect: Allow
+                Action:
+                  - sqs:ReceiveMessage
+                Resource: !Sub ${CryptoDataDigestQueue.Arn}
+        - PolicyName: sns
+          PolicyDocument:
+            Statement:
+              - Effect: Allow
+                Action:
+                  - sns:Publish
+                Resource: !Sub ${CryptoAnalysisSummaryTopic.Arn}
+        - PolicyName: dynamodb
+          PolicyDocument:
+            Statement:
+              - Effect: Allow
+                Action:
+                  - dynamodb:BatchGet*
+                  - dynamodb:DescribeTable
+                  - dynamodb:Get*
+                  - dynamodb:Query
+                  - dynamodb:Scan
+                  - dynamodb:Delete*
+                  - dynamodb:Update*
+                  - dynamodb:PutItem
+                Resource: !Sub ${CryptoRobotAnalyzedDataDynamoDBTable.Arn}
+    Tags:
+      - Key: type
+        Value: role
+      - Key: system
+        Value: !Ref System
+      - Key: parent
+        Value: !Ref Parent
+
+  CryptoDataDigestLambda:
+    Type: AWS::Lambda::Function
+#    DependsOn: CryptoDataDigestLambdaRole
+    Properties:
+      Runtime: nodejs16.x
+      Role: !Sub ${CryptoDataDigestLambdaRole.Arn}
+      Handler: ./dist/main/delivery/handler/Handler.execute
+      FunctionName: 'dataDigestLambda'
+      Code:
+        S3Bucket: lambda-functions
+        S3Key: crypto-robot-data-digest.zip
+      MemorySize: 128
+      Timeout: 60
+      Description: 'SQS Lambda handler for analysis data digestion.'
+    Tags:
+      - Key: type
+        Value: lambda
+      - Key: system
+        Value: !Ref System
+      - Key: parent
+        Value: !Ref Parent
+  #      Environment:
+  #        Variables:
+  #          TASK_QUEUE_URL: !Ref TaskQueue
+
+  CryptoDataDigestLambdaEventSourceMapping:
+    Type: AWS::Lambda::EventSourceMapping
+#    DependsOn:
+#      - CryptoDataDigestQueue
+#      - CryptoDataDigestLambda
+    Properties:
+      BatchSize: 1
+      Enabled: true
+      EventSourceArn: !Sub ${CryptoDataDigestQueue.Arn}
+      FunctionName: !Sub ${CryptoDataDigestLambda.Arn}
+    Tags:
+      - Key: type
+        Value: event-source-mapping
+      - Key: system
+        Value: !Ref System
+      - Key: parent
+        Value: !Ref Parent

build/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3"
+version: '3'
 
 services:
   localstack:
@@ -20,7 +20,9 @@ services:
       dockerfile: Dockerfile
     volumes:
       - ./lambda-files:/lambda-files
-    entrypoint: sh -c "cp crypto-robot-data-digest.zip /lambda-files"
+    entrypoint: >
+      sh -c "zip -r crypto-robot-data-digest.zip dist node_modules && 
+      cp crypto-robot-data-digest.zip /lambda-files"
   awscli:
     container_name: awscli
     depends_on:
@@ -30,6 +32,7 @@ services:
       - ./lambda-files:/lambda-files
       - ./scripts/localstack:/init-scripts/localstack
       - ./scripts/lambda:/init-scripts/crypto-data-digest-lambda
+      - ./cloudformation:/cloudformation
     entrypoint: >
       sh -c "sh /init-scripts/localstack/01-profile-creation.sh && 
       sh /init-scripts/crypto-data-digest-lambda/01-wait-for-localstack.sh && 

build/scripts/lambda/02-resources-creation.sh

@@ -3,64 +3,13 @@
 
 echo "-----------------Script-02----------------- [data-digest]"
 
-echo "########### Creating DLQ SQS ###########"
-aws sqs create-queue --queue-name cryptoAnalysisQueueDLQ --endpoint-url http://localstack:4566
-
-echo "########### Creating SQS ###########"
-aws sqs create-queue \
---queue-name cryptoAnalysisQueue \
---attributes '{
-    "RedrivePolicy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:sa-east-1:000000000000:cryptoAnalysisQueueDLQ\",\"maxReceiveCount\":\"3\"}",
-    "MessageRetentionPeriod": "259200",
-    "VisibilityTimeout": "30"
-}' \
---endpoint-url http://localstack:4566
-
-echo "########### Listing SQS ###########"
-aws sqs list-queues --endpoint-url http://localstack:4566
-
-echo "########### Subscribing SQS to SNS ###########"
-aws sns subscribe \
---topic-arn arn:aws:sns:sa-east-1:000000000000:cryptoAnalysisTopic \
---protocol sqs \
---notification-endpoint "http://localhost:4566/000000000000/cryptoAnalysisQueue" \
---endpoint-url http://localstack:4566
-
-echo "########### Listing SNS Subscriptions ###########"
-aws sns list-subscriptions --endpoint-url http://localstack:4566
-
-echo "########### Creating SNS ###########"
-aws sns create-topic --name cryptoAnalysisSummaryTopic --endpoint-url http://localstack:4566
-
-echo "########### Listing SNS ###########"
-aws sns list-topics --endpoint-url http://localstack:4566
-
 echo "########### Copy the lambda function to the S3 bucket ###########"
-aws s3 cp /lambda-files/crypto-robot-data-digest.zip s3://lambda-functions --endpoint-url http://localstack:4566
-
-echo "########### Create the lambda dataDigestLambda ###########"
-aws lambda create-function \
-  --endpoint-url http://localstack:4566 \
-  --function-name dataDigestLambda \
-  --role arn:aws:iam::000000000000:role/admin-role \
-  --code S3Bucket=lambda-functions,S3Key=crypto-robot-data-digest.zip \
-  --handler ./dist/main/delivery/handler/Handler.execute \
-  --runtime nodejs16.x \
-  --description "SQS Lambda handler for test sqs." \
-  --timeout 60 \
-  --memory-size 128
-
-echo "########### Map the cryptoAnalysisQueue to the dataDigestLambda lambda function ###########"
-aws lambda create-event-source-mapping \
-  --function-name dataDigestLambda \
-  --batch-size 1 \
-  --event-source-arn "arn:aws:sqs:sa-east-1:000000000000:cryptoAnalysisQueue" \
+aws s3 cp /lambda-files/crypto-robot-data-digest.zip s3://lambda-functions \
   --endpoint-url http://localstack:4566
 
-echo "########### Creating DynamoDB 'crypto_robot.analyzed_data' table ###########"
-aws dynamodb create-table \
---table-name crypto_robot.analyzed_data  \
---attribute-definitions AttributeName=interval,AttributeType=S \
---key-schema AttributeName=interval,KeyType=HASH \
---provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \
---endpoint-url=http://localstack:4566
+echo "########### Cloudformation Start ###########"
+aws cloudformation deploy \
+ --stack-name crypto-robot-data-digest \
+ --template-file "/cloudformation/crypto-robot-data-digest.yaml" \
+ --endpoint-url http://localstack:4566
+

build/scripts/localstack/02-resources-creation.sh

@@ -13,3 +13,4 @@ aws sns create-topic --name cryptoAnalysisTopic --endpoint-url http://localhost:
 
 echo "########### Listing SNS ###########"
 aws sns list-topics --endpoint-url http://localhost:4566
+

src/main/delivery/handler/Handler.ts

@@ -7,7 +7,7 @@ export const execute = async (event: SQSEvent, context: Context) => {
     global.correlationId = context.awsRequestId;
     logger.info('new event received', event, context);
 
-    const analysisIndicatorDto = JSON.parse(JSON.parse(event?.Records[0]?.body)?.Message || '{}');
+    const analysisIndicatorDto = JSON.parse(event?.Records[0]?.body);
 
     const analysisIndicator = new AnalysisIndicator(analysisIndicatorDto);
 

src/test/integrated/step-definitions/DataDigest.steps.ts

@@ -1,7 +1,7 @@
 import { after, before, binding, given, then, when } from 'cucumber-tsflow';
 import { defineParameterType, setDefaultTimeout } from '@cucumber/cucumber';
 import * as fs from 'fs';
-import { Context, SNSMessage, SQSEvent } from 'aws-lambda';
+import { Context, SQSEvent } from 'aws-lambda';
 import { AnalyzedDataDto } from '@/integration/dto/AnalyzedDataDto';
 import * as Handler from '@/delivery/handler/Handler';
 import { assert } from 'chai';
@@ -75,9 +75,9 @@ export class DataDigestSteps {
 
     @when('The handler function gets called')
     async theHandlerFunctionGetsCalled() {
-        const snsMessage = DataDigestSteps.generateSNSMessage(eventMessage);
+        // const snsMessage = DataDigestSteps.generateSNSMessage(eventMessage);
 
-        const sqsEvent = DataDigestSteps.generateSQSEvent(snsMessage);
+        const sqsEvent = DataDigestSteps.generateSQSEvent(eventMessage);
 
         const context = DataDigestSteps.generateContext();
 
@@ -140,23 +140,6 @@ export class DataDigestSteps {
         return JSON.parse(fs.readFileSync(`${__dirname}/../resources/${jsonFile}`, 'utf-8'));
     }
 
-    private static generateSNSMessage(message: any): SNSMessage {
-        return {
-            MessageAttributes: {},
-            Subject: '',
-            Type: 'Notification',
-            MessageId: '41cf51ea-1a79-4864-9132-b15c8dd040cd',
-            TopicArn: 'arn:aws:sns:sa-east-1:000000000000:cryptoAnalysisTopic',
-            Message: typeof message === 'string' ? message : JSON.stringify(message),
-            Timestamp: '2022-07-21T14:58:58.971',
-            SignatureVersion: '1',
-            Signature: 'EXAMPLEpH+..',
-            SigningCertUrl: 'https://sns.us-east-1.amazonaws.com/SimpleNotificationService-0000000000000000000000.pem',
-            UnsubscribeUrl:
-                'http://localhost:4566/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:sa-east-1:000000000000:cryptoAnalysisTopic:d141d0ea-1ae4-4a94-88cf-e9394155c697',
-        };
-    }
-
     private static generateSQSEvent(message: any): SQSEvent {
         return {
             Records: [