New dock init

init.sh

@@ -2,65 +2,40 @@
 
 # Entry-point script for dock initialization. Simply includes the `lib/dock.sh`
 # library and calls the master initialization function.
-#
-# NOTE This script will automatically update the `lib/` directory before
-#      the dock is initialized. This means that this script itself will not be
-#      automatically updated. To do so a new AMI must be baked.
-#
 # @author Ryan Sandor Richards
 
 export DOCK_INIT_BASE=/opt/runnable/dock-init
-export CONSUL_HOSTNAME
 export HOST_IP=$(hostname -i)
-export environment=""
 
 if [ -z "${CONSUL_PORT+x}" ]; then
   export CONSUL_PORT=8500
 else
   export CONSUL_PORT
 fi
 
+if [ -z "${CONSUL_HOSTNAME+x}" ]; then
+  export CONSUL_HOSTNAME=10.4.5.144
+else
+  export CONSUL_HOSTNAME
+fi
+
 source "${DOCK_INIT_BASE}/lib/consul.sh"
+source "${DOCK_INIT_BASE}/lib/aws.sh"
+source "${DOCK_INIT_BASE}/lib/dock.sh"
+source "${DOCK_INIT_BASE}/lib/container.sh"
 source "${DOCK_INIT_BASE}/lib/util/log.sh"
-source "${DOCK_INIT_BASE}/lib/util/rollbar.sh"
-source "${DOCK_INIT_BASE}/lib/util/backoff.sh"
-
-# Executes a command using an ssh agent with the id_rsa_runnabledock key
-# @param $1 action Comand to execute
-ssh_execute() {
-  local action="$1"
-  ssh-agent bash -c "ssh-add key/id_rsa_runnabledock; $action"
-}
-
-# Automatically updates dock-init to the version given in consul, if needed.
-# After consul has been updated this executes the main script.
-auto_update() {
-  log::info "Updating dock-init"
-  consul::connect
-
-  log::trace 'Fetching dock-init version from consul...'
-  local version=$(consul::get '/dock-init/version')
-  log::info "dock-init version found: $version"
-
-  log::trace "moving to dock init base directory ($DOCK_INIT_BASE)"
-  cd "$DOCK_INIT_BASE"
-
-  log::trace "fetching all from repository"
-  if [[ "$FETCH_ORIGIN_ALL" != "" ]]; then
-    ssh_execute "git fetch origin $version"
-  else
-    ssh_execute "git fetch origin"
-  fi
-
-  log::info "Checking out dock-init version: $version"
-  ssh_execute "git checkout $version"
-}
 
 # Initializes the dock
 main() {
-  source "${DOCK_INIT_BASE}/lib/dock.sh"
-  dock::init
+  consul::connect
+  consul::get_environment
+  consul::configure_consul_template
+  dock::generate_certs
+  aws::get_org_id
+  dock::set_hostname
+  dock::set_config_org
+  container::start
+  log::info "Init Done!"
 }
 
-# Attempt to auto-update then initialize the dock
-backoff auto_update && main
+main

lib/container.sh

@@ -7,12 +7,13 @@ source "${DOCK_INIT_BASE}/lib/consul.sh"
 source "${DOCK_INIT_BASE}/lib/util/backoff.sh"
 source "${DOCK_INIT_BASE}/lib/util/log.sh"
 source "${DOCK_INIT_BASE}/lib/util/rollbar.sh"
+source "${DOCK_INIT_BASE}/lib/upstart.sh"
 source "${DOCK_INIT_BASE}/lib/vault.sh"
 
 # Starts the docker swarm container
 container::_start_swarm_container() {
   local name="swarm"
-  local version="$(consul::get ${name}/version)"
+  local version="1.2.5"
 
   log::info "Starting swarm:${version} container"
   local docker_logs
@@ -37,7 +38,7 @@ container::_start_swarm_container() {
 # Starts the docker registry container
 container::_start_registry_container() {
   local name="registry"
-  local version="$(consul::get ${name}/version)"
+  local version="2.3.1"
   log::info "Starting ${name}:${version} container"
 
   local region="$(consul::get s3/region)"
@@ -131,12 +132,17 @@ container::_start_node_exporter_container() {
 # Starts all container services needed for the dock
 container::start() {
   log::info "Starting container services"
+  upstart::start_docker
   backoff container::_start_registry_container
   backoff container::_start_cadvisor_container
   backoff container::_start_node_exporter_container
 
   # swarm should be started last so we know everything is up
   backoff container::_start_swarm_container
+  # currently @henrymollman does not understand why restarting swarm works
+  # but without this line docker-listener will time out getting events
+  # and the stream will close. this is an intermittent error however
+  docker restart swarm
 }
 
 # Stops all dock container services

lib/dock.sh

@@ -9,38 +9,11 @@
 # @author Bryan Kendall
 # @module dock
 
-source "${DOCK_INIT_BASE}/lib/aws.sh"
 source "${DOCK_INIT_BASE}/lib/cert.sh"
-source "${DOCK_INIT_BASE}/lib/consul.sh"
-source "${DOCK_INIT_BASE}/lib/container.sh"
-source "${DOCK_INIT_BASE}/lib/upstart.sh"
-
 source "${DOCK_INIT_BASE}/lib/util/backoff.sh"
 source "${DOCK_INIT_BASE}/lib/util/log.sh"
 source "${DOCK_INIT_BASE}/lib/util/rollbar.sh"
 
-# An "on exit" trap to clean up sensitive keys and files on the dock itself.
-# Note that this will have no effect if the `DONT_DELETE_KEYS` environment has
-# been set (useful for testing)
-dock::cleanup::exit_trap() {
-  # Delete the keys unless the `DO_NOT_DELETE` flag is set
-  if [[ "${DONT_DELETE_KEYS}" == "" ]]; then
-    log::info '[CLEANUP TRAP] Removing Keys'
-    rm -f "${CERT_PATH}"/ca-key.pem \
-          "${CERT_PATH}"/pass \
-          "${DOCK_INIT_BASE}"/consul-resources/template-config.hcl \
-          "${DOCK_INIT_BASE}"/consul-resources/vault/**/auth-token \
-          "${DOCK_INIT_BASE}"/consul-resources/vault/**/token-* \
-          "${DOCK_INIT_BASE}"/key/rollbar.token
-  fi
-}
-
-# Sets the cleanup trap for the entire script
-dock::cleanup::set_exit_trap() {
-  log::info "Setting key cleanup trap"
-  trap 'dock::cleanup::exit_trap' EXIT
-}
-
 # Sets the value of `$ORG_ID` as the org label in the docker configuration
 dock::set_config_org() {
   log::info "Setting organization id in docker configuration"
@@ -68,60 +41,3 @@ dock::generate_certs() {
   backoff dock::generate_certs_backoff
 }
 
-# Generates the correct /etc/hosts file for the dock
-dock::generate_etc_hosts() {
-  log::info "Generating /etc/hosts"
-
-  rollbar::fatal_trap \
-    "Dock-Init: Failed to Add Host Registry Entry" \
-    "Consul-Template was unable to realize the registry template."
-
-  local template=''
-  template+="$DOCK_INIT_BASE/consul-resources/templates/hosts-registry.ctmpl"
-  template+=":$DOCK_INIT_BASE/hosts-registry.txt"
-  consul-template \
-    -config="${DOCK_INIT_BASE}"/consul-resources/template-config.hcl \
-    -once \
-    -template="${template}"
-
-  rollbar::clear_trap
-}
-
-# Sets the correct registry.runnable.com host
-dock::set_registry_host() {
-  local registry_host=$(cat "$DOCK_INIT_BASE/hosts-registry.txt")
-  log::info "Set registry host: $registry_host"
-  echo "$registry_host" >> /etc/hosts
-}
-
-# Remove docker key file so it generates a unique id
-dock::remove_docker_key_file() {
-  log::info "Removing docker key.json"
-  rm -f /etc/docker/key.json
-}
-
-# Master function for performing all tasks and initializing the dock
-dock::init() {
-  # Setup the exit trap and rollbar
-  dock::cleanup::set_exit_trap
-  rollbar::init
-
-  # Connect to and configure consul then collect various information we need
-  consul::connect
-  consul::get_environment
-  consul::configure_consul_template
-  aws::get_org_id
-
-  # Now that we have everything we need and consul is ready, initialize the dock
-  dock::set_hostname
-  dock::set_config_org
-  dock::generate_certs
-  dock::generate_etc_hosts
-  dock::set_registry_host
-  dock::remove_docker_key_file
-  upstart::start
-  container::start
-
-  # Give the all clear message!
-  log::info "Init Done!"
-}

lib/upstart.sh

@@ -5,95 +5,9 @@
 # @author Ryan Sandor Richards
 # @author Bryan Kendall
 
-source "${DOCK_INIT_BASE}/lib/consul.sh"
-source "${DOCK_INIT_BASE}/lib/util/backoff.sh"
 source "${DOCK_INIT_BASE}/lib/util/log.sh"
 source "${DOCK_INIT_BASE}/lib/util/rollbar.sh"
 
-# Generates upstart scripts for the dock
-upstart::generate_scripts() {
-  log::info "Generating Upstart Scripts"
-  rollbar::fatal_trap \
-    "Dock-Init: Failed to Generate Upstart Script" \
-    "Failed to generate the upstart scripts."
-  upstart::generate_scripts
-  rollbar::clear_trap
-}
-
-# Configures the template for a given service
-# @param $1 name Name of the service
-# @param $2 path Path to the servic
-upstart::configure_service() {
-  local name="${1}"
-  log::trace "Configuring $name"
-  rollbar::fatal_trap \
-    "Consul-Template: Failed to Render $name Config" \
-    "Consule-Template was unable to realize the given template."
-
-  local template_path="$DOCK_INIT_BASE/consul-resources/templates/services"
-  template_path+="/$name.conf.ctmpl"
-  template_path+=":/etc/init/$name.conf"
-
-  consul-template \
-    -config="$DOCK_INIT_BASE/consul-resources/template-config.hcl" \
-    -once \
-    -template="$template_path"
-  echo "manual" > /etc/init/"$name".override
-
-  rollbar::clear_trap
-}
-
-# Generates upstart scripts for thoses services that require environment info
-# from consul
-upstart::generate_scripts() {
-  log::info "Configuring Upstart Scripts"
-  upstart::configure_service "charon"
-  log::trace "Done Generating Upstart Scripts"
-}
-
-# Updates a service to the consul version, installs packages, then restarts it.
-# @param $1 Name of the service
-upstart::upstart_named_service() {
-  local name="${1}"
-  local attempt="${2}"
-  local data='{"attempt":'"${attempt}"'}'
-  local version="$(consul::get ${name}/version)"
-  local key_path="$DOCK_INIT_BASE/key/id_rsa_runnabledock"
-
-  rollbar::warning_trap \
-    "$name: Cannot Upstart Services" \
-    "Attempting to upstart the services and failing." \
-    "${data}"
-
-  log::info "Updating and restarting $name @ $version" &&
-  cd "/opt/runnable/$name" &&
-  ssh-agent bash -c "ssh-add $key_path; git fetch origin" &&
-  git checkout "$version" &&
-  ssh-agent bash -c "ssh-add $key_path; USERPROFILE=/home/ubuntu npm install --production" &&
-  service "$name" restart
-
-  rollbar::clear_trap
-}
-
-# Starts a service installed on the machine.
-# @param $1 Name of the service
-# @param $2 Attempt number
-upstart::upstart_service() {
-  local name="${1}"
-  local attempt="${2}"
-  local data='{"attempt":'"${attempt}"'}'
-
-  rollbar::warning_trap \
-    "$name: Cannot Upstart Service" \
-    "Attempting to upstart the service and failing." \
-    "${data}"
-
-  log::info "Starting $name"
-  service "$name" restart
-
-  rollbar::clear_trap
-}
-
 # Start dockers (due to manual override now set in /etc/init)
 upstart::start_docker() {
   log::info "Starting Docker"
@@ -105,7 +19,7 @@ upstart::start_docker() {
 
   log::info "Waiting for Docker"
   local attempt=1
-  local timeout=1
+  local timeout=.5
   while [ ! -e /var/run/docker.sock ]
   do
     log::info "Docker Sock N/A ($attempt)"
@@ -115,52 +29,5 @@ upstart::start_docker() {
     rollbar::report_warning "${title}" "${message}" "$data"
     sleep $timeout
     attempt=$(( attempt + 1 ))
-    timeout=$(( timeout * 2 ))
   done
 }
-
-# Upstarts services that are supposed to be running on the dock.
-# @param $1 attempt Attempt number.
-upstart::upstart_services_with_backoff_params() {
-  local attempt="${1}"
-  upstart::upstart_named_service "krain" $attempt
-  upstart::upstart_named_service "charon" $attempt
-  upstart::upstart_service "datadog-agent" $attempt
-}
-
-# Pulls the latest docker image for the runnable image builder
-# @param $1 attempt The current attempt for pulling image builder
-upstart::pull_image_builder() {
-  local attempt="${1}"
-  local name="image-builder"
-  local version="$(consul::get $name/version)"
-
-  log::info "Pulling image-builder:$version (${attempt})"
-  docker pull "registry.runnable.com/runnable/image-builder:$version"
-
-  if [[ "$?" -gt "0" ]]; then
-    local data='{"attempt":'"${attempt}"'}'
-    rollbar::report_warning \
-      "Dock-Init: Cannot Upstart Services" \
-      "Attempting to upstart the services and failing." \
-      "${data}"
-    return 1
-  fi
-}
-
-# Starts all services needed for the dock
-upstart::start() {
-  log::info "Upstarting dock"
-  upstart::generate_scripts
-  upstart::start_docker
-  backoff upstart::pull_image_builder
-  backoff upstart::upstart_services_with_backoff_params
-}
-
-# Stops all dock services
-upstart::stop() {
-  log::info "Stopping all dock upstart services"
-  service krain stop
-  service charon stop
-  service docker stop
-}