Refactored caching in prep for EMCECS#121, EMCECS#119, EMCECS#81

bootstrap.sh

@@ -93,21 +93,7 @@ cd $root
 echo "INSTALL_ROOT=${root}" >$HOME/.ecsinstallrc
 export INSTALL_ROOT="${root}"
 #
-# DEPRECATED: shell-style environment flags are deprecated and will be replaced
-# DEPRECATED: by new context system, but some scripts may still depend on these.
-DRONE=${DRONE:-false}
-BUILD=${BUILD:-false}
-MONITOR=${DRONE:-false}
-#
-# Imports and import configs
-source ${lib}/release.conf
-if [ -f "${root}/release.conf" ]; then
-    source "${root}/release.conf"
-fi
-source ${lib}/console.sh
-log_file="${root}/install.log"
-source ${lib}/osutils.sh
-source ${lib}/dockerutils.sh
+source ${lib}/includes.sh
 #
 ##############################################################################
 
@@ -258,6 +244,7 @@ ping_sudo || die "Unable to escalate using sudo."
 
 
 ### Import vars for this specific OS
+p Detecting OS
 detect_os
 route_os
 o "Environment is $os"
@@ -278,6 +265,7 @@ o "    $log_file"
 o ""
 o "It is perfectly fine to remove this log file at any time."
 o ""
+p Collecting system info
 collect_environment_info
 
 
@@ -287,31 +275,31 @@ o "minutes to a few hours depending on many factors, the most"
 o "important being the speed of your Internet connection."
 o "For reference, the ECS software docker image is around 1.5GiB."
 o ""
-p
 
 
 ### Set utility symlinks in current user's path for various callables
 v "Creating symlinks in ${HOME}/bin for easy access to commands"
+p Installing commands
 symlink_scripts
-p
 
 
 ### Do we need to do a MitM cert?
 if $mitm_flag; then
     v "Adding ${mitm_val} to the local trust store and installer queue"
+p Installing proxy cert
     set_mitm_cert "$mitm_val"
     if ! [ -d "${docker_host_root}/ssl" ]; then
         sudo mkdir -p "${docker_host_root}/ssl"
     fi
     sudo cp "$mitm_val" "${docker_host_root}/ssl/sslfw.pem"
 fi
-p
 
 
 ### If we got proxies then set them up in the local shell
 ### and the system environment settings
 if $proxy_flag; then
     v "Checking connectivity through proxy ${proxy_val}"
+p Checking proxy connection
     proxy_http_ping "${proxy_val}" "${proxy_test_val}" 2>&1 >/dev/null
     if [ $? -gt 0 ]; then
         v " failed!"
@@ -320,14 +308,14 @@ if $proxy_flag; then
         v "Connectivity OK!"
     fi
     v "Configuring system for proxy ${proxy_val}"
+p Setting system proxy
     export http_proxy="http://${proxy_val}"
     export https_proxy="https://${proxy_val}"
     export ftp_proxy="ftp://${proxy_val}"
     set_os_proxy || die "Couldn't write to /etc/environment"
     # set package manager proxy in package manager config
     # Set docker proxy after installing it
 fi
-p
 
 
 ### Refresh sudo timestamp
@@ -337,100 +325,103 @@ ping_sudo
 ### Configure system package manager repos for proxies
 if $proxy_flag; then
     v "Configuring system package manager for proxies"
+p Setting package mgr proxy
     set_repo_proxy_conf
     set_repo_proxy_idempotent
 fi
-p
 
 
 ### Configure system package manager to keep its cache
 ### This is so we can reuse it later for nodes
 v "Configuring system package manager to keep its cache so it can be used for other nodes"
+p Setting package mgr keepcache
 set_repo_keepcache_conf
-p
 
 
 ### Update repo databases and all system packages
 v "Updating system package manager databases pass (1/2)"
+p Updating package mgr DB
 up_repo_db 2>&1 | log
 ping_sudo
-p
 
 v "Updating all system packages pass (1/2)"
+p Updating installed packages
 up_repo_pkg_all 2>&1 | log
 ping_sudo
-p
 
 
 ### Do system package installs
 v "Installing bootstrap packages pass (1/3)"
+p Installing new packages
 in_prefix_packages 2>&1 | log
 if $proxy_flag; then
     set_repo_proxy_idempotent
 fi
 ping_sudo
-p
 
+
+p Installing packages
 v "Installing bootstrap packages pass (2/3)"
+p Installing new packages
 in_general_packages 2>&1 | log
 if $proxy_flag; then
     set_repo_proxy_idempotent
 fi
 ping_sudo
-p
 
+
+p Installing packages
 v "Installing bootstrap packages pass (3/3)"
+p Installing new packages
 in_suffix_packages 2>&1 | log
 if $proxy_flag; then
     set_repo_proxy_idempotent
 fi
 ping_sudo
-p
 
 
 ### Do we need VM guest additions?
 if $vm_flag; then
     v "Installing virtual machine guest additions"
+p Installing VM guest additions
     in_vm_packages 2>&1 | log
     if $proxy_flag; then
         set_repo_proxy_idempotent
     fi
     ping_sudo
 fi
-p
 
 
 ### Update repo databases and all system packages (again)
 ### This will pick up any updates pulled in from alternate repos.
-if ! $minimal_flag; then
-    v "Updating system package manager databases pass (2/2)"
-    up_repo_db 2>&1 | log
-    ping_sudo
-p
-    v "Updating all system packages pass (2/2)"
-    up_repo_pkg_all 2>&1 | log
-    ping_sudo
-fi
-p
+v "Updating system package manager databases pass (2/2)"
+p Updating package mgr DB
+up_repo_db 2>&1 | log
+ping_sudo
+
+v "Updating all system packages pass (2/2)"
+p Updating installed packages
+up_repo_pkg_all 2>&1 | log
+ping_sudo
 
 
 ### If Docker needs proxy configs, do that now.
 if $proxy_flag; then
     v "Configuring Docker proxy settings"
+p Setting Docker proxy
     set_docker_proxy 2>&1 | log
     ping_sudo
 fi
-p
 
 
 ### If Docker needs to use a custom registry, set that up now.
 docker_registry
 ping_sudo
-p
 
 
 ### Test Docker install
 v "Testing docker installation"
+p Testing Docker
 if docker_test; then
     v "Docker was installed correctly"
 else
@@ -440,28 +431,28 @@ else
       die "\$ sudo docker run --rm ${hello-image}"
 fi
 ping_sudo
-p
 
 
 ### Run post-install
 v "Running post-install scripts for ${os}"
+p Post-install scripts
 do_post_install 2>&1 | log
 ping_sudo
-p
 
 
 ### Create host paths
 v "Creating host paths"
+p Creating host paths
 for directory in "${docker_host_root}/ssl" "${docker_host_root}/ssh" "${docker_host_logs}"; do
 if ! [ -d "${directory}" ]; then
     sudo mkdir -p "${directory}"
 fi
 done
-p
 
 
 ### ECS-Install Docker image
 if $build_image_flag; then
+p Building ecs-install image
     if ! ui/build_image.sh 2>&1 | log; then
         error "We couldn't build the ecs-install image for some reason. Please check the logs."
         error "If it's something simple, such as a missing base image (we use python:2.7-alpine),"
@@ -472,6 +463,7 @@ if $build_image_flag; then
         die "If you still need more help after trying the above, you can reach us on github."
     fi
 else
+    p Pulling ecs-install image
     if ! ui/pull_image.sh 2>&1 | log; then
         error "We couldn't pull the ecs-install image for some reason. Please check the logs."
         error "If it's something simple, such as the ecs-install:latest image missing from"
@@ -481,11 +473,11 @@ else
     fi
 fi
 ping_sudo
-p
 
 
 ### ECS Docker Image
 v "Pulling ${release_artifact}:${release_tag} Docker image"
+p Pulling ECS image
 if $registry_flag; then
     if ! sudo docker pull ${registry_val}/${release_artifact} 2>&1 | log; then
         error "We couldn't pull the software image for some reason. Since you're using a custom"
@@ -506,12 +498,12 @@ else
         die "If you still need more help after trying the above, you can reach us on github."
     fi
 fi
-p
 
 
 ### Next steps
+p ''
 o ''
-o 'All done bootstrapping your install node.'
+q 'All done bootstrapping your install node.'
 o ''
 o 'To continue (after reboot if needed):'
 o "    $ cd ${root}"
@@ -520,39 +512,37 @@ o '    $ sudo cp deploy.yml /opt/emc/ecs-install/'
 o 'If not, check out the examples directory and create one.'
 o 'Once you have a deploy.yml, you can start the deployment'
 o 'by running:'
-o '    $ cli init access cache'
+o '    $ ecsdeploy init access cache'
 o ''
 
-if ! $minimal_flag; then
-    ### Needs rebooting?
-    if get_os_needs_restarting; then
-
-        ping_sudo
-        q "The system has indicated it wants to reboot."
-        o "Please reboot BEFORE continuing to ensure this node is"
-        o "operating with the latest kernel and system libraries."
-        o ''
-
-        if $override_flag; then
-            if $override_val; then
-                o "Automatically rebooting by user request (-y argument)"
-                log "REBOOT-REBOOTING-ARGUMENT"
-                do_reboot
-            else
-                o "Skipping reboot by user request (-n argument)"
-                log "REBOOT-SKIPPED-ARGUMENT"
-            fi
+### Needs rebooting?
+if get_os_needs_restarting; then
+
+    ping_sudo
+    q "The system has indicated it wants to reboot."
+    o "Please reboot BEFORE continuing to ensure this node is"
+    o "operating with the latest kernel and system libraries."
+    o ''
+
+    if $override_flag; then
+        if $override_val; then
+            q "Automatically rebooting by user request (-y argument)"
+            log "REBOOT-REBOOTING-ARGUMENT"
+            do_reboot
+        else
+            q "Skipping reboot by user request (-n argument)"
+            log "REBOOT-SKIPPED-ARGUMENT"
+        fi
+    else
+        ask "Would you like to reboot now?"
+        if [ $? -eq 0 ]; then
+            log "REBOOT-REBOOTING-INTERACTIVE"
+            do_reboot
         else
-            ask "Would you like to reboot now?"
-            if [ $? -eq 0 ]; then
-                log "REBOOT-REBOOTING-INTERACTIVE"
-                do_reboot
-            else
-                log "REBOOT-SKIPPED-INTERACTIVE"
-                o "Skipping reboot by user request"
-            fi
-            log "END-ASK-REBOOT"
+            log "REBOOT-SKIPPED-INTERACTIVE"
+            q "Skipping reboot by user request"
         fi
+        log "END-ASK-REBOOT"
     fi
 fi
 

examples/local-lab/README.md

@@ -23,7 +23,7 @@ _That's a lot of arguments!_  Let's look at what each of them does:
 | `-t` | Additionally, I test connectivity through my proxycache by connecting to `www.emc.com:443` to test that I can make outbound HTTPS connections. |
 
 ## Deployment Config File: deploy.yml
-Deployment configuration for ECS Community Editition can be boiled down to a single YAML file, `deploy.yml`.  Deployment details can be tuned by making changes to this file, copying it to `/opt/emc/ecs-install/deploy.yml`, and running `cli init` to build a complete deployment profile.
+Deployment configuration for ECS Community Editition can be boiled down to a single YAML file, `deploy.yml`.  Deployment details can be tuned by making changes to this file, copying it to `/opt/emc/ecs-install/deploy.yml`, and running `ecsdeploy init` to build a complete deployment profile.
 
 The deployment system uses [Ansible](https://github.com/ansible/ansible) to get most things done, and you'll notice a few Ansible-specific variables directly referenced in `deploy.yml`.   The `ansible_user` variable needs to be set to a user on the node with `sudo` access.  In this case I'm using a user generically-named "admin" on my lab VMs.  The `ansible_*_pass` variables need to be set to admin's password.  Ansible needs to know the password so it can login and update `$HOME/.ssh/authorized_keys` to enable public key authentication for the remainder of the deployment.
 

examples/local-lab/bootstrap-local.sh

@@ -2,4 +2,4 @@
 
 cd $HOME/ECS-CommunityEdition
 
-./bootstrap.sh -y -b -v -g -k contrib/sslproxycert/emc_ssl.pem -r cache.local:5000 -d examples/local-lab/registry.crt -p cache.local:3128 -t www.emc.com:443
+./bootstrap.sh -y -b -g -k contrib/sslproxycert/emc_ssl.pem -r cache.local:5000 -d examples/local-lab/registry.crt -p cache.local:3128 -t www.emc.com:443

ui/ansible/clicmd_access.yml

@@ -1,6 +1,5 @@
----
 - name: Common | Ping data nodes before doing anything else
-  hosts: data_node:!install_node
+  hosts: data_node
   gather_facts: false
   vars:
     ansible_ssh_pipelining: false

ui/ansible/clicmd_access_host.yml

@@ -0,0 +1,12 @@
+- name: Installer | Setup access between ecs-install and install node
+  hosts: install_node
+  vars:
+    ansible_ssh_pipelining: false
+    ansible_ssh_common_args: -o PreferredAuthentications=password -o PubkeyAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s
+  tasks:
+     - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }}_configure_access
+
+- name: CentOS 7 | Setup SSH on install node
+  hosts: os_CentOS_7_configure_access
+  roles:
+    - CentOS_7_configure_ssh

ui/ansible/clicmd_cache.yml

@@ -1,4 +1,4 @@
-- name: Build the package cache
+- name: Installer | Build the package cache
   hosts: ecs_install
   roles:
     - installer_build_cache