CI/CD for events router

.ci/detect-changed-services.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -e
+
+detect_changed_services() {
+ echo "----------------------------------------------"
+ echo "detecting changed folders for this commit"
+
+ # get a list of all the changed folders only
+ changed_folders=`git diff --name-only $TRAVIS_COMMIT_RANGE | grep / | awk 'BEGIN {FS="/"} {print $1}' | uniq`
+ echo "changed folders "$changed_folders
+
+ changed_services=()
+ for folder in $changed_folders
+ do
+  echo "Adding $folder to list of services to build"
+  if [[ $folder != ".ci" && $folder != ".git" && $folder != ".templates" ]]; then
+    changed_services+=("$folder")
+  fi
+ done
+
+#  # Iterate on each service and run the packaging script
+#  for service in $changed_services
+#  do
+#    echo "-------------------Running packaging for $service---------------------"
+#    # copy the common code to the service so that it can be packaged in the docker image
+#    cp -r ./_global $service
+#    pushd "$service"
+#    # move the build script to the root of the service
+#    mv ./_global/package-service.sh ./.
+#    ./package-service.sh "$service"
+#    popd
+#  done
+}
+
+# detect_changed_services

.ci/prod-deploy.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_PROD
+export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_PROD
+source .ci/detect-changed-services.sh && detect_changed_services && echo $changed_services
+echo "changed_services: $changed_services"
+
+
+auto_deployable_services="events-router,"
+if [[ $TRAVIS_PULL_REQUEST == "false" ]]; then
+  for service in $changed_services
+  do
+    echo "-------------------Running packaging for $service---------------------"
+    if [[ $auto_deployable_services = *"$service"* ]]; then
+      cd $service && yarn install && yarn deploy:prod; cd ..
+    fi
+  done
+fi

.ci/staging-deploy.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_DEV
+export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_DEV
+source .ci/detect-changed-services.sh && detect_changed_services && echo $changed_services
+echo "changed_services: $changed_services"
+
+auto_deployable_services="events-router,"
+if [[ $TRAVIS_PULL_REQUEST == "false" ]]; then
+  for service in $changed_services
+  do
+    echo "-------------------Running packaging for $service---------------------"
+    if [[ $auto_deployable_services = *"$service"* ]]; then
+      cd $service && yarn install && yarn deploy; cd ..
+    fi
+  done
+fi

.ci/test.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+source .ci/detect-changed-services.sh && detect_changed_services && echo $changed_services
+echo "changed_services: $changed_services"
+for service in $changed_services
+do
+  echo "-------------------Running tests for $service---------------------"
+  cd $service
+  yarn install && yarn test
+  if [[ $service == "lists-microservice" ]]; then
+    wget ${ES_DOWNLOAD_URL}
+    tar -xzf elasticsearch-${ES_VERSION}.tar.gz
+    ./elasticsearch-${ES_VERSION}/bin/elasticsearch &
+    wget -q --waitretry=3 --retry-connrefused -T 10 -O - http://127.0.0.1:9200
+    yarn run test:integration
+    #if [[ $TRAVIS_PULL_REQUEST == "false" ]]; then
+      # yarn run test:system
+    #fi
+  fi
+  cd ..
+done

.gitignore

@@ -61,7 +61,7 @@ typings/
 .next
 
 # macos stuff
-.DS_store
+**/.DS_store
 
 # Serverless directories
 .serverless
@@ -73,7 +73,12 @@ playground/
 
 /.vscode/settings.json
 
-**/config.*.json
-lists-microservice/public
+lists-microservice/config.*.json
+campaigns-microservice/config.*.json
+emails-microservice/config.*.json
+template-marketplace/config.*.json
+support/screenshots-service/config.*.json
 
-*.pem
+*.pem
+
+admin.env

.travis.kkk.yml

@@ -0,0 +1,64 @@
+language: node_js
+sudo: required
+node_js:
+- '8.10'
+before_install: # if "install" is overridden
+  # Repo for Yarn
+  - sudo apt-key adv --fetch-keys http://dl.yarnpkg.com/debian/pubkey.gpg
+  - echo "deb http://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+  - sudo apt-get update -qq
+  - sudo apt-get install -y -qq yarn
+  - yarn install
+env:
+  global:
+    - ES_VERSION=6.0.0
+    - ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
+  matrix:
+    - TEST_DIR=events
+    - TEST_DIR=api
+    - TEST_DIR=private
+    - TEST_DIR=extensions
+    - TEST_DIR=template-marketplace
+    - TEST_DIR=lists-microservice
+    - TEST_DIR=emails-microservice
+    - TEST_DIR=campaigns-microservice
+    - TEST_DIR=webhooks-microservice
+    - TEST_DIR=support/events-router
+
+script:
+- cd $TEST_DIR && yarn install && yarn test
+- |
+  if [[ $TEST_DIR == "lists-microservice" ]]; then
+    wget ${ES_DOWNLOAD_URL}
+    tar -xzf elasticsearch-${ES_VERSION}.tar.gz
+    ./elasticsearch-${ES_VERSION}/bin/elasticsearch &
+    wget -q --waitretry=3 --retry-connrefused -T 10 -O - http://127.0.0.1:9200
+    yarn run test:integration
+    #if [[ $TRAVIS_PULL_REQUEST == "false" ]]; then
+      # yarn run test:system
+    #fi
+  fi
+  if [[ $TEST_DIR == "support/events-router" ]]; then
+    if [[ $TRAVIS_PULL_REQUEST == "false" ]]; then
+      if [[ $TRAVIS_BRANCH == "master" ]]; then
+        yarn deploy
+      elif [[ $TRAVIS_BRANCH == "release" ]]; then
+        yarn deploy:prod
+      fi
+    fi
+  fi
+
+cache:
+  yarn: true
+  directories:
+  - "$(npm config get prefix)/lib/node_modules"
+  - node_modules
+  - events/node_modules
+  - api/node_modules
+  - private/node_modules
+  - extensions/node_modules
+  - template-marketplace/node_modules
+  - lists-microservice/node_modules
+  - emails-microservice/node_modules
+  - campaigns-microservice/node_modules
+  - support/events-router/node_modules

.travis.yml

@@ -9,36 +9,30 @@ before_install: # if "install" is overridden
   - sudo apt-get update -qq
   - sudo apt-get install -y -qq yarn
   - yarn install
+  - source .ci/detect-changed-services.sh && detect_changed_services && echo $changed_services
 env:
   global:
-    - ES_VERSION=6.0.0 
+    - ES_VERSION=6.0.0
     - ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
-  matrix:
-    - TEST_DIR=events
-    - TEST_DIR=api
-    - TEST_DIR=private
-    - TEST_DIR=extensions
-    - TEST_DIR=template-marketplace
-    - TEST_DIR=lists-microservice
-    - TEST_DIR=emails-microservice
-    - TEST_DIR=campaigns-microservice
-    - TEST_DIR=webhooks-microservice
-    - TEST_DIR=support/events-router
-
-script:
-- cd $TEST_DIR && yarn install && yarn test
-- |
-  if [[ $TEST_DIR == "lists-microservice" ]]; then
-    wget ${ES_DOWNLOAD_URL}
-    tar -xzf elasticsearch-${ES_VERSION}.tar.gz
-    ./elasticsearch-${ES_VERSION}/bin/elasticsearch &
-    wget -q --waitretry=3 --retry-connrefused -T 10 -O - http://127.0.0.1:9200
-    yarn run test:integration
-    if [[ $TRAVIS_PULL_REQUEST == "false" ]]; then
-      yarn run test:system
-    fi
-  fi
+
+stages:
+  - test
+  - name: staging-deploy
+    if: branch = master AND type = push
+  - name: prod-deploy
+    if: branch = release AND type = push
 
+jobs:
+  include:
+    - stage: test
+      script: ./.ci/test.sh     
+
+    - stage: staging-deploy
+      script: ./.ci/staging-deploy.sh
+
+    - stage: prod-deploy
+      script: ./.ci/prod-deploy.sh 
+
 cache:
   yarn: true
   directories:
@@ -52,4 +46,4 @@ cache:
   - lists-microservice/node_modules
   - emails-microservice/node_modules
   - campaigns-microservice/node_modules
-  - support/events-router/node_modules
+  - events-router/node_modules

events-router/config.dev.json

@@ -0,0 +1,3 @@
+{
+  "KINESIS_SHARDS": 1
+}

support/events-router/handler.js → events-router/handler.js

@@ -5,3 +5,14 @@ export function routeEvents(event, context, callback) {
     .then(res => callback(null, res))
     .catch(err => callback(err));
 }
+
+
+
+
+
+
+
+
+
+
+

support/events-router/package.json → events-router/package.json

@@ -5,7 +5,9 @@
   "main": "handler.js",
   "scripts": {
     "test": "NODE_ENV=test mocha --require choma --require babel-register $(find ./ \\( -name '*.test.js' -o -name '*.spec.js' \\) -not -path './node_modules/*' -not -path './**/node_modules/*')",
-    "watch": "NODE_ENV=test mocha --watch --require choma  --require babel-register $(find ./ \\( -name '*.test.js' -o -name '*.spec.js' \\) -not -path './node_modules/*' -not -path './**/node_modules/*')"
+    "watch": "NODE_ENV=test mocha --watch --require choma  --require babel-register $(find ./ \\( -name '*.test.js' -o -name '*.spec.js' \\) -not -path './node_modules/*' -not -path './**/node_modules/*')",
+    "deploy": "yarn sls deploy -s dev -r us-east-1",
+    "deploy:prod": "yarn sls deploy -s prod -r eu-west-1"
   },
   "author": "David García <david.garcia@microapps.com>",
   "license": "ISC",

support/events-router/serverless.yml → events-router/serverless.yml

@@ -10,11 +10,12 @@ custom:
   recipientsBucket: next-recipients.moonmail.${self:custom.currentStage}.${self:provider.region}
   config: ${file(./config.${self:custom.currentStage}.json)}
   ssmPath: /moonmail/events-router/${self:custom.currentStage}
+  defaultShardCount: 1
 
 provider:
   name: aws
   runtime: nodejs6.10
-  profile: ${opt:profile}
+  profile: ${opt:profile, ""}
   region: ${opt:region}
   timeout: 300
   environment:
@@ -52,7 +53,7 @@ resources:
       Properties:
         Name: ${self:custom.resourcesPrefix}-EventsIngestionStream
         RetentionPeriodHours: 168
-        ShardCount: ${self:custom.config.KINESIS_SHARDS}
+        ShardCount: ${self:custom.config.KINESIS_SHARDS, self:custom.defaultShardCount}
     EventsDeadLetterQueue:
       Type: "AWS::SQS::Queue"
       Properties: