BLUEBUTTON-314: Build ENV Agnostic AMI's (#1354)

* Adding env_config playbook and swithcing user_data to use * Saving * Switching env_config playbook to use test branch bluebutton-314 * Trimming env_vars to only build env and cert copy and not checkout code * Only copy env_config playbook. env_var playbook imported nested. * Test * Stripping build playbook of env specific tasks,. * Removing env declaration from packer job. * Removing ENV declaration and private key. * Syntax error, extra } not needed. * Jenkins file typo. Forgot stages. * More jenkins files typos. * Refactor env_var and fix env_config * Cleanup * Setting aws_region var in all_var. * moving app_pyapps_user definition to common vars. Same across all env. * removing app_pyapps_user password set. * Moving cf_app_log_dir to common vars as app_log_dir. * moving cf_app_pyapp_home to commons var. * Setting project_repo for all environments. * moving css settings to common vars * Setting project_parent in common vars * Moving cf_app_py_virtual_env to common vars * Typo * Moving python_bin_dir def to common vars. * Moving python version to common vars * Moving app_py_virtual_env to commons * Moving venv defintion to commons * Renaming venv full path var * Moving NGINX install to deployment phase of pipeline. * Switching to include_role. * Duplicate NGINX role. Removing one. * Instead of including specific task switch to include_role
CMSgov · Mar 20, 2019 · 0bd0be3 · 0bd0be3
1 parent dfa1bc8
commit 0bd0be3
Show file tree

Hide file tree

Showing 26 changed files with 315 additions and 284 deletions.
diff --git a/.terraform-version b/.terraform-version
@@ -1 +1 @@
-0.11.10
+0.11.11
diff --git a/Jenkinsfiles/Jenkinsfile.build_app_ami b/Jenkinsfiles/Jenkinsfile.build_app_ami
@@ -39,39 +39,9 @@ pipeline {
       description: 'The class/size of the ec2 instance to launch.',
       name: 'INSTANCE_CLASS'
     )
-    choice(
-      choices: ['Do Not Deploy', 'dev', 'test', 'impl', 'prod'],
-      description: 'The environment to deploy to. Required.',
-      name: 'ENV'
-    )
   }
 
   stages {
-    stage('Ensure BB20_APP_VERSION, ENV and SUBNET_ID') {
-      steps {
-        sh """
-        if [ -z "${params.BB20_APP_VERSION}" ] || [ -z "${params.ENV}" ] || [ -z "${params.SUBNET_ID}" ]
-        then
-          exit 1
-        fi
-        """
-      }
-    }
-
-    stage('Stop master deployments to IMPL and PROD') {
-      steps {
-        sh """
-        if [ "${params.BB20_APP_VERSION}" == "master" ]
-        then
-          if [ "${params.ENV}" == "impl" ] || [ "${params.ENV}" == "prod" ]
-          then
-            exit 1
-          fi
-        fi
-        """
-      }
-    }
-
     stage('Set release_version var') {
       steps {
         script {
@@ -89,14 +59,6 @@ pipeline {
 # THIS IS WHERE WE WILL ADD SLACK NOTIFICATIONS ONCE SLACK IS ONLINE
 */
 
-    stage('Set private key file') {
-      steps {
-        script {
-          private_key = "${params.ENV}-key"
-        }
-      }
-    }
-
     stage('Checkout') {
       steps {
         checkout([
@@ -156,7 +118,6 @@ pipeline {
                     -var 'vault_password_file=${vp}' \
                     -var 'git_branch=${params.BB20_APP_VERSION}' \
                     -var 'subnet_id=${params.SUBNET_ID}' \
-                    -var 'env=${params.ENV}' \
                     -var 'source_ami=${BB20_PLATINUM_AMI}' \
                     -var 'release_version=${release_version}' \
                     packer/build_app_ami.json
@@ -172,5 +133,4 @@ pipeline {
 /*
 # THIS IS WHERE WE WILL ADD SLACK NOTIFICATIONS ONCE SLACK IS ONLINE
 */
-
 }
diff --git a/packer/build_app_ami.json b/packer/build_app_ami.json
@@ -4,8 +4,7 @@
     "source_ami": "",
     "vault_password_file": "",
     "git_branch": "master",
-    "subnet_id": "",
-    "env": ""
+    "subnet_id": ""
   },
   "builders": [{
     "type": "amazon-ebs",
@@ -33,8 +32,7 @@
               "ANSIBLE_VAULT_PASSWORD_FILE={{user `vault_password_file`}}"
             ],
             "extra_arguments": [
-              "-e git_branch={{user `git_branch`}}",
-              "-e env={{user `env`}}"
+              "-e git_branch={{user `git_branch`}}"
             ]
         }
     ]

diff --git a/playbook/appherd/910_add_cron_exports.yml b/playbook/appherd/910_add_cron_exports.yml
@@ -82,7 +82,7 @@
     become: yes
     run_once: yes
     shell: |
-      source {{ cf_app_py_virtual_env }}/bin/activate
+      source {{ app_py_virtual_env }}/bin/activate
       cd {{ install_root }}/{{ project_name }}/
-      echo "$(date --iso-8601=seconds) Active_BBAPI_Applications:$({{ cf_app_py_virtual_env }}/bin/python3 manage.py dumpdata dot_ext.application --indent 2 |grep \"active\": | wc -l ) " >>{{ cf_app_log_dir }}/info.log
-      chown pyapps:logreader {{ cf_app_log_dir }}/info.log
+      echo "$(date --iso-8601=seconds) Active_BBAPI_Applications:$({{ app_py_virtual_env }}/bin/python3 manage.py dumpdata dot_ext.application --indent 2 |grep \"active\": | wc -l ) " >>{{ app_log_dir }}/info.log
+      chown pyapps:logreader {{ app_log_dir }}/info.log
diff --git a/playbook/appherd/roles/create_certstore/tasks/main.yml b/playbook/appherd/roles/create_certstore/tasks/main.yml
@@ -9,9 +9,9 @@
   become_user: "{{ remote_admin_account }}"
   become: yes
   file:
-    dest: "{{ cf_app_pyapp_home }}/certstore"
+    dest: "{{ app_pyapp_home }}/certstore"
     mode: 0755
-    owner: "{{ cf_app_pyapps_user }}"
+    owner: "{{ app_pyapps_user }}"
     group: "{{ app_group }}"
     recurse: yes
     state: directory
diff --git a/playbook/appherd/roles/django_export_model/tasks/main.yml b/playbook/appherd/roles/django_export_model/tasks/main.yml
@@ -13,8 +13,8 @@
     path: "{{ item }}"
     state: absent
   with_items:
-    - "{{ cf_app_log_dir }}/bb_{{ env }}_{{ export_app_model }}_list.csv"
-    - "{{ cf_app_log_dir }}/bb_{{ env }}_{{ export_dev_model }}_list.csv"
+    - "{{ app_log_dir }}/bb_{{ env }}_{{ export_app_model }}_list.csv"
+    - "{{ app_log_dir }}/bb_{{ env }}_{{ export_dev_model }}_list.csv"
 
 
 - name: "Django export {{ export_app_model }} CSV"
@@ -23,11 +23,11 @@
   # We only want to run the command on one server in the fleet
   run_once: yes
   shell: |
-    cd {{ cf_app_pyapp_home }}/{{ common_project_name }}
-    source {{ cf_app_py_virtual_env }}/bin/activate
-    export PATH={{ cf_app_py_virtual_env }}/bin:$PATH
-    ./manage.py model2csv --application {{ export_application }} --model {{ export_app_model }} --add_table_name True --filter_fields id,client_id,user,name,agree,created,updated,client_uri,active | sed '/WARNING:/d' | sed 's/{{ export_application }}.{{ export_app_model }}/{{ export_application }}.{{ export_app_model }}_info/' >{{ cf_app_log_dir }}/bb_{{ env }}_{{ export_app_model }}_list.csv
-    chown pyapps:logreader {{ cf_app_log_dir }}/bb_{{ env }}_{{ export_app_model }}_list.csv
+    cd {{ app_pyapp_home }}/{{ common_project_name }}
+    source {{ app_py_virtual_env }}/bin/activate
+    export PATH={{ app_py_virtual_env }}/bin:$PATH
+    ./manage.py model2csv --application {{ export_application }} --model {{ export_app_model }} --add_table_name True --filter_fields id,client_id,user,name,agree,created,updated,client_uri,active | sed '/WARNING:/d' | sed 's/{{ export_application }}.{{ export_app_model }}/{{ export_application }}.{{ export_app_model }}_info/' >{{ app_log_dir }}/bb_{{ env }}_{{ export_app_model }}_list.csv
+    chown pyapps:logreader {{ app_log_dir }}/bb_{{ env }}_{{ export_app_model }}_list.csv
   when: ( export_app_model is defined )
 
 
@@ -37,9 +37,9 @@
   # We only want to run the command on one server in the fleet
   run_once: yes
   shell: |
-    cd {{ cf_app_pyapp_home }}/{{ common_project_name }}
-    source {{ cf_app_py_virtual_env }}/bin/activate
-    export PATH={{ cf_app_py_virtual_env }}/bin:$PATH
-    ./manage.py model2csv --application {{ export_devuser_application }} --model {{ export_dev_model }} --add_table_name True --filter_fields id,user,organization_name,user_type | sed '/WARNING:/d' | sed 's/{{ export_devuser_application }}.{{ export_dev_model }}/{{ export_devuser_application }}.{{ export_dev_model }}_info/' >{{ cf_app_log_dir }}/bb_{{ env }}_{{ export_dev_model }}_list.csv
-    chown pyapps:logreader {{ cf_app_log_dir }}/bb_{{ env }}_{{ export_dev_model }}_list.csv
+    cd {{ app_pyapp_home }}/{{ common_project_name }}
+    source {{ app_py_virtual_env }}/bin/activate
+    export PATH={{ app_py_virtual_env }}/bin:$PATH
+    ./manage.py model2csv --application {{ export_devuser_application }} --model {{ export_dev_model }} --add_table_name True --filter_fields id,user,organization_name,user_type | sed '/WARNING:/d' | sed 's/{{ export_devuser_application }}.{{ export_dev_model }}/{{ export_devuser_application }}.{{ export_dev_model }}_info/' >{{ app_log_dir }}/bb_{{ env }}_{{ export_dev_model }}_list.csv
+    chown pyapps:logreader {{ app_log_dir }}/bb_{{ env }}_{{ export_dev_model }}_list.csv
   when: ( export_dev_model is defined )
diff --git a/playbook/build_app_ami/main.yml b/playbook/build_app_ami/main.yml
@@ -5,7 +5,6 @@
   gather_facts: no
   vars:
     ansible_ssh_pipelining: no
-    env: "dev"
     azone: "az1"
     sub_zone: "app"
     sg_zone: "appserver"
@@ -18,16 +17,11 @@
     - "./../../vars/all_var.yml"
 
   roles:
-    - ../../roles/nessus_update_key
-    - ../../roles/splunk
     - ../../roles/aws
     - ../../roles/app_user
     - ../../roles/app_prep
     - ../../roles/app_logs
     - ../../roles/app_install
-    - ../../roles/env_vars
-    - ../../roles/new_relic
-    - ../../roles/nginx
-
+    - ../../roles/env_config
     - ../../roles/semanage_apps
     - ../../roles/semanage_virtualenv
diff --git a/roles/app_env/tasks/main.yml b/roles/app_env/tasks/main.yml
@@ -24,4 +24,4 @@
 - name: "Make new files executable"
   become_user: "{{ remote_admin_account }}"
   become: yes
-  shell: "chmod +x {{ cf_app_pyapp_home }}"
+  shell: "chmod +x {{ app_pyapp_home }}"
diff --git a/roles/app_install/tasks/main.yml b/roles/app_install/tasks/main.yml
@@ -15,18 +15,18 @@
   become_user: "{{ remote_admin_account }}"
   become: yes
   file:
-    dest: "{{ cf_app_pyapp_home }}"
+    dest: "{{ app_pyapp_home }}"
     state: directory
     recurse: yes
-    owner: "{{ cf_app_pyapps_user }}"
+    owner: "{{ app_pyapps_user }}"
     group: "{{ app_group }}"
     mode: 0755
 
-- name: "set ownership to pyapps:www-data ({{ cf_app_pyapps_user }}:{{ app_group }})"
+- name: "set ownership to pyapps:www-data ({{ app_pyapps_user }}:{{ app_group }})"
   become_user: "{{ remote_admin_account }}"
   become: yes
   file:
-    owner: "{{ cf_app_pyapps_user }}"
+    owner: "{{ app_pyapps_user }}"
     group: "{{ app_group }}"
     dest: "/var/virtualenv"
     state: directory
@@ -41,7 +41,7 @@
     force: yes
     clone: yes
 
-- name: "update from git: {{ project_repo }} branch:{{ git_branch }} to {{ env }}"
+- name: "update from git: {{ project_repo }} branch:{{ git_branch }}"
   become_user: "{{ remote_admin_account }}"
   become: yes
   git:
@@ -59,24 +59,24 @@
     force: yes
     clone: yes
 
-- name: "set permissions to application code ( {{ cf_app_pyapps_user }}:{{ app_group }} )"
+- name: "set permissions to application code ( {{ app_pyapps_user }}:{{ app_group }} )"
   become_user: "{{ remote_admin_account }}"
   become: yes
   file:
-    dest: "{{ cf_app_pyapp_home }}"
+    dest: "{{ app_pyapp_home }}"
     state: directory
     recurse: yes
-    owner: "{{ cf_app_pyapps_user }}"
+    owner: "{{ app_pyapps_user }}"
     group: "{{ app_group }}"
     mode: 0755
 
 - name: "install requirements"
   become_user: "{{ remote_admin_account }}"
   become: yes
   shell: |
-    source {{ cf_app_py_virtual_env }}/bin/activate && make reqs-install
+    source {{ app_py_virtual_env }}/bin/activate && make reqs-install
   args:
-    chdir: "{{ cf_app_pyapp_home }}/{{ common_project_name }}"
+    chdir: "{{ app_pyapp_home }}/{{ common_project_name }}"
 
 - name: "install boto3 for collectstatic"
   become_user: "{{ remote_admin_account }}"
@@ -86,7 +86,7 @@
     version: 1.9.42
     state: present
     extra_args: " --upgrade {{ pip_extra_args }} "
-    virtualenv: "{{ venv }}"
+    virtualenv: "{{ venv_full }}"
 
 - import_tasks: ./../appherd/roles/create_certstore/tasks/main.yml
 

diff --git a/roles/app_logs/tasks/main.yml b/roles/app_logs/tasks/main.yml
@@ -3,8 +3,8 @@
   become_user: "{{ remote_admin_account }}"
   become: yes
   file:
-    dest: "{{ cf_app_log_dir }}"
-    owner: "{{ cf_app_pyapps_user }}"
+    dest: "{{ app_log_dir }}"
+    owner: "{{ app_pyapps_user }}"
     group: logreader
     state: directory
     mode: 0775
@@ -14,8 +14,8 @@
   become_user: "{{ remote_admin_account }}"
   become: yes
   file:
-    dest: "{{ cf_app_log_dir }}/{{ item }}"
-    owner: "{{ cf_app_pyapps_user }}"
+    dest: "{{ app_log_dir }}/{{ item }}"
+    owner: "{{ app_pyapps_user }}"
     group: logreader
     state: touch
     mode: 0776
@@ -31,7 +31,7 @@
   become_user: "{{ remote_admin_account }}"
   become: yes
   sefcontext:
-    target: "{{ cf_app_log_dir }}(/.*)?"
+    target: "{{ app_log_dir }}(/.*)?"
     setype: httpd_sys_content_t
     state: present
   ignore_errors: True
@@ -42,15 +42,15 @@
   copy:
     dest: "/etc/logrotate.d/pyapps"
     content: |
-      {{ cf_app_log_dir }}/*.log {
+      {{ app_log_dir }}/*.log {
       missingok
       notifempty
       compress
       size 10M
       rotate 10
       copytruncate
       daily
-      create 0776 {{ cf_app_pyapps_user }} logreader
+      create 0776 {{ app_pyapps_user }} logreader
       }
     owner: root
     group: logreader

diff --git a/roles/app_prep/tasks/main.yml b/roles/app_prep/tasks/main.yml
@@ -4,8 +4,8 @@
   become: yes
   file:
     dest: "/var/virtualenv"
-    owner: "{{ cf_app_pyapps_user }}"
-    group: "{{ cf_app_pyapps_user }}"
+    owner: "{{ app_pyapps_user }}"
+    group: "{{ app_pyapps_user }}"
     mode: 0755
     state: directory
 
@@ -14,7 +14,7 @@
   become: yes
   file:
     dest: "/var/pyapps"
-    owner: "{{ cf_app_pyapps_user }}"
-    group: "{{ cf_app_pyapps_user }}"
+    owner: "{{ app_pyapps_user }}"
+    group: "{{ app_pyapps_user }}"
     mode: 0755
     state: directory
diff --git a/roles/app_user/tasks/main.yml b/roles/app_user/tasks/main.yml
@@ -3,5 +3,5 @@
   become_user: "{{ remote_admin_account }}"
   become: yes
   user:
-    name: "{{ cf_app_pyapps_user }}"
-    password: "{{ cf_app_pyapps_pwd | password_hash('sha512') }}"
+    name: "{{ app_pyapps_user }}"
+#    password: "{{ cf_app_pyapps_pwd | password_hash('sha512') }}"