From 7e67e9e8ef46348da0b80c1c83e789fda10767ca Mon Sep 17 00:00:00 2001 From: shudarshon Date: Wed, 29 Jan 2020 15:12:53 +0800 Subject: [PATCH 01/34] edit: disabled docker build and k8s deploy --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index feb1f07fa3..6c783675dd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,8 +87,8 @@ jobs: - git_checkout_from_cache - npm_install - build - - docker - - k8s_deploy + # - docker + #- k8s_deploy workflows: release: From 67b13316e39ad540f1564c7db4fb5631e1eb1b99 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Wed, 29 Jan 2020 17:04:15 +0800 Subject: [PATCH 02/34] edit: testing artifact storing configuration --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c783675dd..7414886321 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,9 +51,11 @@ commands: build: description: "Build" steps: - - run: - name: "Grunt deploy" - command: npm run deploy-ci + #- run: + # name: "Grunt deploy" + # command: npm run deploy-ci + - store_artifacts: + path: dist/ docker: description: "Build and Push image to docker hub" From a1382b8f63d19ae07e093e350769662e11c2f61c Mon Sep 17 00:00:00 2001 From: shudarshon Date: Wed, 29 Jan 2020 17:09:10 +0800 Subject: [PATCH 03/34] edit: edited project build command --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7414886321..e312e754bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,9 +51,9 @@ commands: build: description: "Build" steps: - #- run: - # name: "Grunt deploy" - # command: npm run deploy-ci + - run: + name: "Compile project" + command: npm run build - store_artifacts: path: dist/ From b55291b07390e08c8e6bc580afe6698375e1b1f3 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Wed, 29 Jan 2020 17:46:01 +0800 Subject: [PATCH 04/34] edit: edited artifact path --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e312e754bc..59273f37ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,7 +55,7 @@ commands: name: "Compile project" command: npm run build - store_artifacts: - path: dist/ + path: dist/compressed/* docker: description: "Build and Push image to docker hub" From 57c200e8c1509162a5d13e6abf8abb60bf7a1af1 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 11:13:18 +0800 Subject: [PATCH 05/34] using artifact path as compressed static content --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59273f37ea..2faa25c8c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,7 +55,7 @@ commands: name: "Compile project" command: npm run build - store_artifacts: - path: dist/compressed/* + path: dist/compressed/ docker: description: "Build and Push image to docker hub" From 083d00bb2728ad3ce9085f2bd4cf265e1d9b1b4c Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 12:28:21 +0800 Subject: [PATCH 06/34] edit: added artifact compression step --- .circleci/config.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2faa25c8c4..c07545e0af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,8 +54,17 @@ commands: - run: name: "Compile project" command: npm run build + + compress: + description: "Compress" + steps: + - run: + name: "Compress Artifact" + command: | + cd dist/compressed/ + tar -cvf myartifact.tar * - store_artifacts: - path: dist/compressed/ + path: myartifact.tar docker: description: "Build and Push image to docker hub" @@ -89,6 +98,7 @@ jobs: - git_checkout_from_cache - npm_install - build + - compress # - docker #- k8s_deploy From e67bc1f3eda74bfee6d87a8fb5f1cd684557165e Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 12:41:57 +0800 Subject: [PATCH 07/34] edit: added artifact moving step --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c07545e0af..cccdd94262 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,15 +56,17 @@ commands: command: npm run build compress: - description: "Compress" + description: "Compress and move" steps: - run: - name: "Compress Artifact" + name: "Compress and Move Artifact" command: | - cd dist/compressed/ + pushd dist/compressed/ tar -cvf myartifact.tar * + mv myartifact.tar ${OLDPWD}/ - store_artifacts: path: myartifact.tar + destination: chaks.tar docker: description: "Build and Push image to docker hub" From eddee1cd53370914e4e70a3fe6712bffb89b668d Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 13:25:00 +0800 Subject: [PATCH 08/34] edit: edited artifact name and desitnation configuration --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cccdd94262..56df9da38d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,11 +62,11 @@ commands: name: "Compress and Move Artifact" command: | pushd dist/compressed/ - tar -cvf myartifact.tar * - mv myartifact.tar ${OLDPWD}/ + tar -cvf artifact.tar * + mv artifact.tar ${OLDPWD}/ - store_artifacts: - path: myartifact.tar - destination: chaks.tar + path: artifact.tar + destination: content.tar docker: description: "Build and Push image to docker hub" From 2bd2fd05ac71e9d8fcf5d7a70b71392f53786a66 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 14:14:18 +0800 Subject: [PATCH 09/34] add: added artifact upload to s3 step --- .circleci/config.yml | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 56df9da38d..4862333381 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,7 @@ version: 2.1 orbs: k8s: circleci/kubernetes@0.7.0 + s3: circleci/aws-s3@1.0.13 commands: git_checkout_from_cache: description: "Git checkout and save cache" @@ -56,18 +57,33 @@ commands: command: npm run build compress: - description: "Compress and move" + description: "Compress" steps: - run: - name: "Compress and Move Artifact" + name: "Compress" command: | pushd dist/compressed/ tar -cvf artifact.tar * mv artifact.tar ${OLDPWD}/ - - store_artifacts: - path: artifact.tar - destination: content.tar - + #- store_artifacts: + # path: artifact.tar + # destination: content.tar + + upload_artifact: + description: "upload artifact to s3" + steps: + - s3/copy: + from: artifact.tar + to: "s3://${ARTIFACT_S3_BUCKET}/latest/" + aws-access-key-id: ${ARTIFACT_S3_AWS_ACCESS_KEY_ID} + aws-secret-access-key: ${ARTIFACT_S3_AWS_SECRET_ACCESS_KEY} + aws-region: ${ARTIFACT_S3_AWS_REGION} + arguments: | + --acl public-read \ + --cache-control "max-age=86400" + overwrite: true + + docker: description: "Build and Push image to docker hub" steps: @@ -101,6 +117,7 @@ jobs: - npm_install - build - compress + - upload_artifact # - docker #- k8s_deploy From 7fe1283e1c737fa5660fb2b93aaacb9eb5af8136 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 14:18:25 +0800 Subject: [PATCH 10/34] edit: added circle ci project name in s3 upload url --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4862333381..4c860bf294 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,9 +74,9 @@ commands: steps: - s3/copy: from: artifact.tar - to: "s3://${ARTIFACT_S3_BUCKET}/latest/" - aws-access-key-id: ${ARTIFACT_S3_AWS_ACCESS_KEY_ID} - aws-secret-access-key: ${ARTIFACT_S3_AWS_SECRET_ACCESS_KEY} + to: "s3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/" + aws-access-key-id: "${ARTIFACT_S3_AWS_ACCESS_KEY_ID}" + aws-secret-access-key: "${ARTIFACT_S3_AWS_SECRET_ACCESS_KEY}" aws-region: ${ARTIFACT_S3_AWS_REGION} arguments: | --acl public-read \ From c5df5715d6def2c07725949326bb0d9a5d80cf0c Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 14:27:39 +0800 Subject: [PATCH 11/34] test: testing environment variable integration --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c860bf294..5076df41a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,9 +75,9 @@ commands: - s3/copy: from: artifact.tar to: "s3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/" - aws-access-key-id: "${ARTIFACT_S3_AWS_ACCESS_KEY_ID}" - aws-secret-access-key: "${ARTIFACT_S3_AWS_SECRET_ACCESS_KEY}" - aws-region: ${ARTIFACT_S3_AWS_REGION} + aws-access-key-id: "$ARTIFACT_S3_AWS_ACCESS_KEY_ID" + aws-secret-access-key: "$ARTIFACT_S3_AWS_SECRET_ACCESS_KEY" + aws-region: $ARTIFACT_S3_AWS_REGION arguments: | --acl public-read \ --cache-control "max-age=86400" From 63ac8cb0da50efaf1cae042b06ddf7f6feed67c2 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 14:56:04 +0800 Subject: [PATCH 12/34] testing circle ci variable --- .circleci/config.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5076df41a5..dca34a77d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: k8s: circleci/kubernetes@0.7.0 - s3: circleci/aws-s3@1.0.13 + aws-s3: circleci/aws-s3@1.0.13 commands: git_checkout_from_cache: description: "Git checkout and save cache" @@ -72,18 +72,13 @@ commands: upload_artifact: description: "upload artifact to s3" steps: - - s3/copy: + - aws-s3/copy: from: artifact.tar - to: "s3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/" - aws-access-key-id: "$ARTIFACT_S3_AWS_ACCESS_KEY_ID" - aws-secret-access-key: "$ARTIFACT_S3_AWS_SECRET_ACCESS_KEY" - aws-region: $ARTIFACT_S3_AWS_REGION - arguments: | - --acl public-read \ - --cache-control "max-age=86400" - overwrite: true - - + to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' + aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID + aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY + aws-region: env_ARTIFACT_S3_AWS_REGION + docker: description: "Build and Push image to docker hub" steps: From ed15ef463869bd1e8d8524e5b8c5e6e6ff915829 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 15:21:07 +0800 Subject: [PATCH 13/34] edit: edited orb alias and removed unwanted artifact configuration --- .circleci/config.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dca34a77d3..302caf733a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: k8s: circleci/kubernetes@0.7.0 - aws-s3: circleci/aws-s3@1.0.13 + s3: circleci/aws-s3@1.0.13 commands: git_checkout_from_cache: description: "Git checkout and save cache" @@ -65,20 +65,17 @@ commands: pushd dist/compressed/ tar -cvf artifact.tar * mv artifact.tar ${OLDPWD}/ - #- store_artifacts: - # path: artifact.tar - # destination: content.tar - + upload_artifact: description: "upload artifact to s3" steps: - - aws-s3/copy: + - s3/copy: from: artifact.tar to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_ARTIFACT_S3_AWS_REGION - + docker: description: "Build and Push image to docker hub" steps: From f7e514cd9329376e37c9e9c1d2682cfa29b025c4 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 15:53:52 +0800 Subject: [PATCH 14/34] add: added commit-id/checksum as an added information regarding artficat --- .circleci/config.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 302caf733a..70d569a5a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,6 +76,21 @@ commands: aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_ARTIFACT_S3_AWS_REGION + upload_commit_id: + description: "upload commit id to s3" + steps: + - run: + name: + command: | + echo ${CIRCLE_SHA1} > artifact-info.txt + # sha256sum artifact.tar | cut -d ' ' -f 1 > artifact-info.txt + - s3/copy: + from: artifact-info.txt + to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' + aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID + aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY + aws-region: env_ARTIFACT_S3_AWS_REGION + docker: description: "Build and Push image to docker hub" steps: @@ -110,6 +125,7 @@ jobs: - build - compress - upload_artifact + - upload_commit_id # - docker #- k8s_deploy From dab9041f849a8bb513f4f211470fced7b5cc1ca3 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 16:01:11 +0800 Subject: [PATCH 15/34] edit: testing circle ci config with singel step artifact upload --- .circleci/config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 70d569a5a9..87da59705a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,8 +69,13 @@ commands: upload_artifact: description: "upload artifact to s3" steps: + - run: + name: + command: | + echo ${CIRCLE_SHA1} > artifact-info.txt + sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - s3/copy: - from: artifact.tar + from: artifact*.{tar,txt} to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY @@ -125,7 +130,7 @@ jobs: - build - compress - upload_artifact - - upload_commit_id + #- upload_commit_id # - docker #- k8s_deploy From 5cddd6bfb07fd77660f58375ed4e70549f7caaf6 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 16:11:32 +0800 Subject: [PATCH 16/34] fix: fix s3 copy error in pipeline script --- .circleci/config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 87da59705a..9b43a21d84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,13 @@ commands: echo ${CIRCLE_SHA1} > artifact-info.txt sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - s3/copy: - from: artifact*.{tar,txt} + from: artifact.tar + to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' + aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID + aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY + aws-region: env_ARTIFACT_S3_AWS_REGION + - s3/copy: + from: artifact-info.txt to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY From e396beaeee264c1a0e097b96a349bd9f509baea2 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 16:20:25 +0800 Subject: [PATCH 17/34] disabled checksum calculation and enabled commit id as artifact id --- .circleci/config.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b43a21d84..ef1a98885c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,32 +69,21 @@ commands: upload_artifact: description: "upload artifact to s3" steps: - - run: - name: - command: | - echo ${CIRCLE_SHA1} > artifact-info.txt - sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - s3/copy: from: artifact.tar to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_ARTIFACT_S3_AWS_REGION - - s3/copy: - from: artifact-info.txt - to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' - aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID - aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY - aws-region: env_ARTIFACT_S3_AWS_REGION upload_commit_id: - description: "upload commit id to s3" + description: "upload commit id and checksum to s3" steps: - run: name: command: | echo ${CIRCLE_SHA1} > artifact-info.txt - # sha256sum artifact.tar | cut -d ' ' -f 1 > artifact-info.txt + # sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - s3/copy: from: artifact-info.txt to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' From a301d60248b7face0231efc4786751c5558d13ed Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 16:24:48 +0800 Subject: [PATCH 18/34] edit: enabled commit id upload as a jobs step --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef1a98885c..6265f4fc04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -125,7 +125,7 @@ jobs: - build - compress - upload_artifact - #- upload_commit_id + - upload_commit_id # - docker #- k8s_deploy From 3baa4fad848106e633a21168435d58b6f21dbc9b Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 16:48:54 +0800 Subject: [PATCH 19/34] edit: uploading checksum instead of commit id --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6265f4fc04..971d21f7a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,14 +76,14 @@ commands: aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_ARTIFACT_S3_AWS_REGION - upload_commit_id: - description: "upload commit id and checksum to s3" + upload_checksum: + description: "upload artifact checksum to s3" steps: - run: name: command: | - echo ${CIRCLE_SHA1} > artifact-info.txt - # sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt + sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt + # echo ${CIRCLE_SHA1} > artifact-info.txt - s3/copy: from: artifact-info.txt to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' @@ -125,7 +125,7 @@ jobs: - build - compress - upload_artifact - - upload_commit_id + - upload_checksum # - docker #- k8s_deploy From 29e49d7a8b4277b287b81aff81583ab92ab9e977 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 17:36:42 +0800 Subject: [PATCH 20/34] add: add cron script for deployment --- deploy/cron.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 deploy/cron.sh diff --git a/deploy/cron.sh b/deploy/cron.sh new file mode 100755 index 0000000000..fff93775b2 --- /dev/null +++ b/deploy/cron.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -ex + +echo "downloading artifact and artifact-info.txt in filesystem from s3 bucket" + +aws s3 cp s3://binary-frontend-backup/webtrader/latest/artifact.tar /tmp/ +aws s3 cp s3://binary-frontend-backup/webtrader/latest/artifact-info.txt /tmp/ + +echo "calculating checksum of artifact" + +new_chksum=$(sha256sum /tmp/artifact.tar | cut -d ' ' -f 1) +orig_chksum=$(cat /tmp/artifact-info.txt) +app_dir=/home/chaks/www/app + +if [ "${new_chksum}" == "${orig_chksum}" ] +then + echo "checksum matched!! now extracting static contents in specific directory" + tar -xvf artifact.tar --directory ${app_dir}/ +else + echo "checksim did not match" + exit 1 +fi + +echo "removing artifacts and artifacts information from filesystem" +rm -f /tmp/artifact*.{tar,txt} From f7f98237f7701f8ca8687f0a1dde893351595282 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Thu, 30 Jan 2020 17:45:23 +0800 Subject: [PATCH 21/34] changed access settings in aws --- deploy/cron.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/cron.sh b/deploy/cron.sh index fff93775b2..e408163198 100755 --- a/deploy/cron.sh +++ b/deploy/cron.sh @@ -4,8 +4,8 @@ set -ex echo "downloading artifact and artifact-info.txt in filesystem from s3 bucket" -aws s3 cp s3://binary-frontend-backup/webtrader/latest/artifact.tar /tmp/ -aws s3 cp s3://binary-frontend-backup/webtrader/latest/artifact-info.txt /tmp/ +aws s3 cp s3://xx/yy/latest/artifact.tar /tmp/ +aws s3 cp s3://xx/yy/latest/artifact-info.txt /tmp/ echo "calculating checksum of artifact" From d3d4bf29849f340ff042bf2c86f3af025d558d04 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Tue, 4 Feb 2020 17:24:33 +0800 Subject: [PATCH 22/34] edit: enabled all job steps to execute --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 971d21f7a2..946e92b795 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,7 +83,6 @@ commands: name: command: | sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - # echo ${CIRCLE_SHA1} > artifact-info.txt - s3/copy: from: artifact-info.txt to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' @@ -126,8 +125,8 @@ jobs: - compress - upload_artifact - upload_checksum - # - docker - #- k8s_deploy + - docker + - k8s_deploy workflows: release: From 308d64032faf6073847f4ce420864ae08875d248 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Wed, 5 Feb 2020 13:55:20 +0800 Subject: [PATCH 23/34] remove: removed unnecesary deployment script --- deploy/cron.sh | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100755 deploy/cron.sh diff --git a/deploy/cron.sh b/deploy/cron.sh deleted file mode 100755 index e408163198..0000000000 --- a/deploy/cron.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -echo "downloading artifact and artifact-info.txt in filesystem from s3 bucket" - -aws s3 cp s3://xx/yy/latest/artifact.tar /tmp/ -aws s3 cp s3://xx/yy/latest/artifact-info.txt /tmp/ - -echo "calculating checksum of artifact" - -new_chksum=$(sha256sum /tmp/artifact.tar | cut -d ' ' -f 1) -orig_chksum=$(cat /tmp/artifact-info.txt) -app_dir=/home/chaks/www/app - -if [ "${new_chksum}" == "${orig_chksum}" ] -then - echo "checksum matched!! now extracting static contents in specific directory" - tar -xvf artifact.tar --directory ${app_dir}/ -else - echo "checksim did not match" - exit 1 -fi - -echo "removing artifacts and artifacts information from filesystem" -rm -f /tmp/artifact*.{tar,txt} From b308cc83f2f48fb63c60394d80fe2c56a26ad321 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Wed, 5 Feb 2020 14:10:28 +0800 Subject: [PATCH 24/34] edit: using circle ci context variable instead of project based envvariables --- .circleci/config.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 946e92b795..e5940f7e97 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,14 +67,14 @@ commands: mv artifact.tar ${OLDPWD}/ upload_artifact: - description: "upload artifact to s3" + description: "upload build artifact to s3 bucket" steps: - s3/copy: from: artifact.tar - to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' - aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID - aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY - aws-region: env_ARTIFACT_S3_AWS_REGION + to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' + aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID + aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY + aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION upload_checksum: description: "upload artifact checksum to s3" @@ -85,10 +85,10 @@ commands: sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - s3/copy: from: artifact-info.txt - to: 's3://${ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/latest/' - aws-access-key-id: env_ARTIFACT_S3_AWS_ACCESS_KEY_ID - aws-secret-access-key: env_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY - aws-region: env_ARTIFACT_S3_AWS_REGION + to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' + aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID + aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY + aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION docker: description: "Build and Push image to docker hub" @@ -123,8 +123,8 @@ jobs: - npm_install - build - compress - - upload_artifact - - upload_checksum + - upload_artifact # uploading the built code to s3 to create a backup of key services separate from Kubernetes deployment + - upload_checksum # uploading compressed artifact checksum to cross match artifact fingerprint before actual deployment - docker - k8s_deploy @@ -135,4 +135,5 @@ workflows: filters: branches: only: /^master$/ + context: binary-frontend-artifact-upload # using context to retrieve shared secret from circle ci From 4a2eced42f35863173582f9be07d626dd1dcfb55 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 17:02:23 +0800 Subject: [PATCH 25/34] test: testing s3 upload with metadata integration --- .circleci/config.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e5940f7e97..c0eb9b8520 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,8 +12,8 @@ commands: - source-v1-{{ .Branch }}-{{ .Revision }} - source-v1-{{ .Branch }}- - source-v1- - - run: - name: Fetch git tags + - run: + name: Fetch git tags command: | mkdir -p ~/.ssh echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== ' >> ~/.ssh/known_hosts @@ -24,7 +24,7 @@ commands: fi - checkout - run: - name: Compress git objects + name: Compress git objects command: git gc - save_cache: name: Git save cache @@ -65,40 +65,41 @@ commands: pushd dist/compressed/ tar -cvf artifact.tar * mv artifact.tar ${OLDPWD}/ - + upload_artifact: description: "upload build artifact to s3 bucket" steps: - - s3/copy: + - s3/copy: from: artifact.tar to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' - aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID + aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY - aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION + aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION + arguments: '--metadata "{"x-amz-artifact-id": "chaks" }"' upload_checksum: description: "upload artifact checksum to s3" steps: - - run: - name: + - run: + name: command: | sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - - s3/copy: + - s3/copy: from: artifact-info.txt to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' - aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID + aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY - aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION + aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION docker: description: "Build and Push image to docker hub" steps: - setup_remote_docker - - run: + - run: name: Building docker image command: | docker build -t ${DOCKHUB_ORGANISATION}/binary-static-webtrader:${CIRCLE_SHA1} . - - run: + - run: name: Pushing Image to docker hub command: | echo $DOCKERHUB_PASSWORD | docker login -u $DOCKERHUB_USERNAME --password-stdin @@ -108,7 +109,7 @@ commands: description: "Deploy to k8s cluster" steps: - k8s/install-kubectl - - run: + - run: name: Deploying to k8s cluster for service binary-webtrader command: | echo $CA_CRT | base64 --decode > ca.crt @@ -127,7 +128,7 @@ jobs: - upload_checksum # uploading compressed artifact checksum to cross match artifact fingerprint before actual deployment - docker - k8s_deploy - + workflows: release: jobs: @@ -135,5 +136,4 @@ workflows: filters: branches: only: /^master$/ - context: binary-frontend-artifact-upload # using context to retrieve shared secret from circle ci - + context: binary-frontend-deploy # using context to retrieve shared secret from circle ci From 6b84fc0f3cb9d2357694f46a3b33356cc05ac516 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 17:06:46 +0800 Subject: [PATCH 26/34] test: testing s3 orb metadata --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c0eb9b8520..df36d5a694 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ commands: aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION - arguments: '--metadata "{"x-amz-artifact-id": "chaks" }"' + arguments: '--metadata \"{\"x-amz-artifact-id\": \"chaks\" }\"' upload_checksum: description: "upload artifact checksum to s3" From 2a89ddcdd4a378d645499c723f5595278286e04e Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 17:18:54 +0800 Subject: [PATCH 27/34] test: testing circleci orb metadata integration --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df36d5a694..1187544764 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ commands: aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION - arguments: '--metadata \"{\"x-amz-artifact-id\": \"chaks\" }\"' + arguments: '--metadata "{\"x-amz-artifact-id\"= \"chaks\" }"' upload_checksum: description: "upload artifact checksum to s3" From eb81cf7ca13f41396d4bb78c6aefa0d81b18fd1f Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 17:21:02 +0800 Subject: [PATCH 28/34] test: testing circleci orb metadata integration --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1187544764..81060e2157 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ commands: aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION - arguments: '--metadata "{\"x-amz-artifact-id\"= \"chaks\" }"' + arguments: '--metadata "{\"x-amz-artifact-id\": \"chaks\" }"' upload_checksum: description: "upload artifact checksum to s3" From 7c68a9125e2d790858a46f82252c77345c5c0704 Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 17:27:59 +0800 Subject: [PATCH 29/34] test: testing bash variable integration with s3 orb metadata --- .circleci/config.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 81060e2157..2d2c8834c1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,21 +69,26 @@ commands: upload_artifact: description: "upload build artifact to s3 bucket" steps: + - run: + name: + command: | + sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt + export ARTIFACT_HASH=$(cat artifact-info.txt) - s3/copy: from: artifact.tar to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION - arguments: '--metadata "{\"x-amz-artifact-id\": \"chaks\" }"' + arguments: '--metadata "{\"x-amz-artifact-id\": \"${ARTIFACT_HASH}\" }"' upload_checksum: description: "upload artifact checksum to s3" steps: - - run: - name: - command: | - sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt + #- run: + # name: + # command: | + # sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - s3/copy: from: artifact-info.txt to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' From 80a413db65ce085a4189312a430799dbbd53296b Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 17:35:40 +0800 Subject: [PATCH 30/34] test: using BASH_ENV to retrieve bash environment value in pipeline --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d2c8834c1..be0a3a6bcb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,7 +73,7 @@ commands: name: command: | sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - export ARTIFACT_HASH=$(cat artifact-info.txt) + export ARTIFACT_HASH=$(cat artifact-info.txt) >> $BASH_ENV - s3/copy: from: artifact.tar to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' From b38b8e57cb219c75f5ae155b590feca07a81f37b Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 17:45:23 +0800 Subject: [PATCH 31/34] test: using to retrieve bash environment value in pipeline --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index be0a3a6bcb..03fb9b8bfa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,6 +74,7 @@ commands: command: | sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt export ARTIFACT_HASH=$(cat artifact-info.txt) >> $BASH_ENV + source $BASH_ENV - s3/copy: from: artifact.tar to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' From b3c722a51b6f868c2f636e6a4960783fbb558c3d Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 17:49:39 +0800 Subject: [PATCH 32/34] test: testing with commit id as artifact id --- .circleci/config.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03fb9b8bfa..f9615b00f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,16 +72,17 @@ commands: - run: name: command: | - sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - export ARTIFACT_HASH=$(cat artifact-info.txt) >> $BASH_ENV - source $BASH_ENV + #sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt + echo "${CIRCLE_SHA1}" > artifact-info.txt + #export ARTIFACT_HASH=$(cat artifact-info.txt) >> $BASH_ENV + #source $BASH_ENV - s3/copy: from: artifact.tar to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' aws-access-key-id: env_CONTEXT_ARTIFACT_S3_AWS_ACCESS_KEY_ID aws-secret-access-key: env_CONTEXT_ARTIFACT_S3_AWS_SECRET_ACCESS_KEY aws-region: env_CONTEXT_ARTIFACT_S3_AWS_REGION - arguments: '--metadata "{\"x-amz-artifact-id\": \"${ARTIFACT_HASH}\" }"' + arguments: '--metadata "{\"x-amz-artifact-id\": \"${CIRCLE_SHA1}\" }"' upload_checksum: description: "upload artifact checksum to s3" From 474404d38c7af490bc4961d8f0b9db63b0b0c85d Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 18:03:13 +0800 Subject: [PATCH 33/34] add: tagging commit id as artifact identifier --- .circleci/config.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f9615b00f6..1a0e0745c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,17 +65,13 @@ commands: pushd dist/compressed/ tar -cvf artifact.tar * mv artifact.tar ${OLDPWD}/ + - run: + name: "Tag commit id as artifact identifer" + command: echo "${CIRCLE_SHA1}" > artifact-info.txt upload_artifact: description: "upload build artifact to s3 bucket" steps: - - run: - name: - command: | - #sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - echo "${CIRCLE_SHA1}" > artifact-info.txt - #export ARTIFACT_HASH=$(cat artifact-info.txt) >> $BASH_ENV - #source $BASH_ENV - s3/copy: from: artifact.tar to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' @@ -87,10 +83,6 @@ commands: upload_checksum: description: "upload artifact checksum to s3" steps: - #- run: - # name: - # command: | - # sha256sum artifact.tar | cut -d ' ' -f 1 >> artifact-info.txt - s3/copy: from: artifact-info.txt to: 's3://${CONTEXT_ARTIFACT_S3_BUCKET}/${CIRCLE_PROJECT_REPONAME}/' From 3305c0a4ff3e4fd01a85826d1b2f887c881f413e Mon Sep 17 00:00:00 2001 From: shudarshon Date: Fri, 7 Feb 2020 18:20:16 +0800 Subject: [PATCH 34/34] edit: edited context name --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a0e0745c5..bbc82ceabc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -135,4 +135,4 @@ workflows: filters: branches: only: /^master$/ - context: binary-frontend-deploy # using context to retrieve shared secret from circle ci + context: binary-frontend-artifact-upload # using context to retrieve shared secret from circle ci