From 0cdf16d91759097d06dfffab980324e3ab0bba6c Mon Sep 17 00:00:00 2001 From: Khai Do Date: Mon, 29 Oct 2018 09:41:14 -0700 Subject: [PATCH] setup CD to AWS Beanstalk AWS Beanstalk is now setup to run Agora & Mongo[1]. Now we can auto deploy a new version of the Agora app on every code change. This setup also allows promoting code changes from Agora develop -> staging and then eventually prod environments. [1] http://agora-develop.ampadscicomp.org --- .ebextensions/eb-options.config | 5 +++ .ebextensions/firewall.config | 3 ++ .ebextensions/npm.config | 44 ++++++++++++++++++++++++++ .ebextensions/packages.config | 6 ++++ .ebextensions/proxy.config | 55 +++++++++++++++++++++++++++++++++ .ebextensions/setup_swap.sh | 14 +++++++++ .elasticbeanstalk/config.yml | 20 ++++++++++++ .travis.yml | 17 ++++++++++ 8 files changed, 164 insertions(+) create mode 100644 .ebextensions/eb-options.config create mode 100644 .ebextensions/firewall.config create mode 100644 .ebextensions/npm.config create mode 100644 .ebextensions/packages.config create mode 100644 .ebextensions/proxy.config create mode 100644 .ebextensions/setup_swap.sh create mode 100644 .elasticbeanstalk/config.yml diff --git a/.ebextensions/eb-options.config b/.ebextensions/eb-options.config new file mode 100644 index 000000000..72d40bf87 --- /dev/null +++ b/.ebextensions/eb-options.config @@ -0,0 +1,5 @@ +option_settings: + aws:elasticbeanstalk:container:nodejs: + ProxyServer: "nginx" + NodeCommand: "node --max_old_space_size=2000 ./dist/server.js" + NodeVersion: "8.11.4" diff --git a/.ebextensions/firewall.config b/.ebextensions/firewall.config new file mode 100644 index 000000000..26d6062da --- /dev/null +++ b/.ebextensions/firewall.config @@ -0,0 +1,3 @@ +container_commands: + "10_FirewallOff": + command: "/sbin/service iptables save && /sbin/service iptables stop" diff --git a/.ebextensions/npm.config b/.ebextensions/npm.config new file mode 100644 index 000000000..7a2d5261f --- /dev/null +++ b/.ebextensions/npm.config @@ -0,0 +1,44 @@ +files: + "/opt/elasticbeanstalk/hooks/appdeploy/pre/50npm.sh": + mode: "000755" + owner: root + group: users + content: | + #!/usr/bin/env bash + # + # Prevent installing or rebuilding like Elastic Beanstalk tries to do by + # default. + # + # Note that this *overwrites* Elastic Beanstalk's default 50npm.sh script + # (https://gist.github.com/wearhere/de51bb799f5099cec0ed28b9d0eb3663). + + "/opt/elasticbeanstalk/hooks/configdeploy/pre/50npm.sh": + mode: "000755" + owner: root + group: users + content: | + #!/usr/bin/env bash + # + # Prevent installing or rebuilding like Elastic Beanstalk tries to do by + # default. + # + # Note that this *overwrites* Elastic Beanstalk's default 50npm.sh script. + # But their default script actually doesn't work at all, since the app + # staging dir, where they try to run `npm install`, doesn't exist during + # config deploys, so ebnode.py just aborts: + # https://gist.github.com/wearhere/de51bb799f5099cec0ed28b9d0eb3663#file-ebnode-py-L140 + + +container_commands: + "11_SetupSwap": + command: "bash .ebextensions/setup_swap.sh" + "12_CleanNpm": + command: "rm -rf node_modules/* && $NODE_HOME/bin/node $NODE_HOME/bin/npm cache clean --force" + "13_NpmInstallGlobals": + command: "export PATH=$PATH:/usr/bin:$NODE_HOME/bin && $NODE_HOME/bin/node $NODE_HOME/bin/npm install --unsafe-perm=true --allow-root --scripts-prepend-node-path=true -g webpack webpack-dev-server karma-cli protractor rimraf typescript cross-env artillery" + "15_NpmInstall": + command: "export PATH=$PATH:/usr/bin:$NODE_HOME/bin && $NODE_HOME/bin/node $NODE_HOME/bin/npm install --unsafe-perm=true --allow-root --scripts-prepend-node-path=true" + "16_NpmBuildAgoraClient": + command: "export PATH=$PATH:/usr/bin:$NODE_HOME/bin && $NODE_HOME/bin/node $NODE_HOME/bin/npm run --scripts-prepend-node-path=true build:aot" + "17_NpmBuildAgoraServer": + command: "export PATH=$PATH:/usr/bin:$NODE_HOME/bin && $NODE_HOME/bin/node $NODE_HOME/bin/npm run --scripts-prepend-node-path=true build:server:prod" diff --git a/.ebextensions/packages.config b/.ebextensions/packages.config new file mode 100644 index 000000000..17b00a2a7 --- /dev/null +++ b/.ebextensions/packages.config @@ -0,0 +1,6 @@ +packages: + yum: + git: [] +commands: + 01_InstallGit: + command: "yum -y install git" diff --git a/.ebextensions/proxy.config b/.ebextensions/proxy.config new file mode 100644 index 000000000..89dcc6ace --- /dev/null +++ b/.ebextensions/proxy.config @@ -0,0 +1,55 @@ +files: + /etc/nginx/conf.d/proxy.conf: + mode: "000644" + owner: root + group: root + content: | + upstream nodejs { + server 127.0.0.1:8080; + keepalive 256; + } + + server { + listen 80; + + if ($time_iso8601 ~ "^(\d{4})-(\d{2})-(\d{2})T(\d{2})") { + set $year $1; + set $month $2; + set $day $3; + set $hour $4; + } + access_log /var/log/nginx/healthd/application.log.$year-$month-$day-$hour healthd; + access_log /var/log/nginx/access.log main; + + location / { + proxy_pass http://nodejs; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + gzip on; + gzip_comp_level 4; + gzip_types text/html text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + location /static { + alias /var/app/current/static; + } + + } + + /opt/elasticbeanstalk/hooks/configdeploy/post/99_kill_default_nginx.sh: + mode: "000755" + owner: root + group: root + content: | + #!/bin/bash -xe + rm -f /etc/nginx/conf.d/00_elastic_beanstalk_proxy.conf + service nginx stop + service nginx start + +container_commands: + removeconfig: + command: "rm -f /tmp/deployment/config/#etc#nginx#conf.d#00_elastic_beanstalk_proxy.conf /etc/nginx/conf.d/00_elastic_beanstalk_proxy.conf" diff --git a/.ebextensions/setup_swap.sh b/.ebextensions/setup_swap.sh new file mode 100644 index 000000000..b248e2422 --- /dev/null +++ b/.ebextensions/setup_swap.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +SWAPFILE=/var/swapfile +SWAP_MEGABYTES=1048 + +if [ -f $SWAPFILE ]; then + echo "Swapfile $SWAPFILE found, assuming already setup" + exit; +fi + +/bin/dd if=/dev/zero of=$SWAPFILE bs=1M count=$SWAP_MEGABYTES +/bin/chmod 600 $SWAPFILE +/sbin/mkswap $SWAPFILE +/sbin/swapon $SWAPFILE diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml new file mode 100644 index 000000000..81cc2354b --- /dev/null +++ b/.elasticbeanstalk/config.yml @@ -0,0 +1,20 @@ +# TEMPLATE FOR elastic beanstalk configuration used by 'eb deploy' +branch-defaults: + TRAVIS_BRANCH: + environment: agora-TRAVIS_BRANCH +environment-defaults: + agora-TRAVIS_BRANCH: + branch: null + repository: null +global: + application_name: agora-ampad + default_ec2_keyname: toptal + default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Node.js running on + 64bit Amazon Linux/4.5.4 + default_region: us-east-1 + instance_profile: null + platform_name: null + platform_version: null + profile: eb-cli + sc: git + workspace_type: Application diff --git a/.travis.yml b/.travis.yml index a11fee225..0b9ccc3eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,12 @@ addons: matrix: fast_finish: true +branches: + only: + - develop + - staging + - prod + before_install: - npm install npm@5 -g - npm cache verify @@ -23,10 +29,17 @@ install: - sudo apt-get install -y build-essential libssl-dev python-pip python2.7-dev - sudo pip install -U setuptools wheel docker-cloud synapseclient[pandas,pysftp] - docker login --username="$DOCKER_USER" --password="$DOCKER_PASSWORD" + # install aws cli + - wget https://github.com/Sage-Bionetworks/infra-utils/archive/master.zip -O /tmp/infra-utils.zip + - unzip -j -n /tmp/infra-utils.zip -x "infra-utils-master/.gitignore" "infra-utils-master/LICENSE" "infra-utils-master/*.md" "infra-utils-master/aws/*" + - ./setup_aws_cli.sh || travis_terminate 1 stages: - name: deploy-containers + if: branch = develop - name: test + if: branch = develop + - name: eb-deploy jobs: include: @@ -44,3 +57,7 @@ jobs: - stage: test script: npm run ci:travis node_js: 8 + - stage: eb-deploy + script: + - sed -i -e "s/TRAVIS_BRANCH/$TRAVIS_BRANCH/g" ./.elasticbeanstalk/config.yml + - eb deploy