diff --git a/init.sh b/init.sh index 6dc5c45..fc874c8 100644 --- a/init.sh +++ b/init.sh @@ -2,17 +2,10 @@ # Entry-point script for dock initialization. Simply includes the `lib/dock.sh` # library and calls the master initialization function. -# -# NOTE This script will automatically update the `lib/` directory before -# the dock is initialized. This means that this script itself will not be -# automatically updated. To do so a new AMI must be baked. -# # @author Ryan Sandor Richards export DOCK_INIT_BASE=/opt/runnable/dock-init -export CONSUL_HOSTNAME export HOST_IP=$(hostname -i) -export environment="" if [ -z "${CONSUL_PORT+x}" ]; then export CONSUL_PORT=8500 @@ -20,47 +13,29 @@ else export CONSUL_PORT fi +if [ -z "${CONSUL_HOSTNAME+x}" ]; then + export CONSUL_HOSTNAME=10.4.5.144 +else + export CONSUL_HOSTNAME +fi + source "${DOCK_INIT_BASE}/lib/consul.sh" +source "${DOCK_INIT_BASE}/lib/aws.sh" +source "${DOCK_INIT_BASE}/lib/dock.sh" +source "${DOCK_INIT_BASE}/lib/container.sh" source "${DOCK_INIT_BASE}/lib/util/log.sh" -source "${DOCK_INIT_BASE}/lib/util/rollbar.sh" -source "${DOCK_INIT_BASE}/lib/util/backoff.sh" - -# Executes a command using an ssh agent with the id_rsa_runnabledock key -# @param $1 action Comand to execute -ssh_execute() { - local action="$1" - ssh-agent bash -c "ssh-add key/id_rsa_runnabledock; $action" -} - -# Automatically updates dock-init to the version given in consul, if needed. -# After consul has been updated this executes the main script. -auto_update() { - log::info "Updating dock-init" - consul::connect - - log::trace 'Fetching dock-init version from consul...' - local version=$(consul::get '/dock-init/version') - log::info "dock-init version found: $version" - - log::trace "moving to dock init base directory ($DOCK_INIT_BASE)" - cd "$DOCK_INIT_BASE" - - log::trace "fetching all from repository" - if [[ "$FETCH_ORIGIN_ALL" != "" ]]; then - ssh_execute "git fetch origin $version" - else - ssh_execute "git fetch origin" - fi - - log::info "Checking out dock-init version: $version" - ssh_execute "git checkout $version" -} # Initializes the dock main() { - source "${DOCK_INIT_BASE}/lib/dock.sh" - dock::init + consul::connect + consul::get_environment + consul::configure_consul_template + dock::generate_certs + aws::get_org_id + dock::set_hostname + dock::set_config_org + container::start + log::info "Init Done!" } -# Attempt to auto-update then initialize the dock -backoff auto_update && main +main diff --git a/lib/container.sh b/lib/container.sh index 093a336..0544ed7 100644 --- a/lib/container.sh +++ b/lib/container.sh @@ -7,12 +7,13 @@ source "${DOCK_INIT_BASE}/lib/consul.sh" source "${DOCK_INIT_BASE}/lib/util/backoff.sh" source "${DOCK_INIT_BASE}/lib/util/log.sh" source "${DOCK_INIT_BASE}/lib/util/rollbar.sh" +source "${DOCK_INIT_BASE}/lib/upstart.sh" source "${DOCK_INIT_BASE}/lib/vault.sh" # Starts the docker swarm container container::_start_swarm_container() { local name="swarm" - local version="$(consul::get ${name}/version)" + local version="1.2.5" log::info "Starting swarm:${version} container" local docker_logs @@ -37,7 +38,7 @@ container::_start_swarm_container() { # Starts the docker registry container container::_start_registry_container() { local name="registry" - local version="$(consul::get ${name}/version)" + local version="2.3.1" log::info "Starting ${name}:${version} container" local region="$(consul::get s3/region)" @@ -131,12 +132,17 @@ container::_start_node_exporter_container() { # Starts all container services needed for the dock container::start() { log::info "Starting container services" + upstart::start_docker backoff container::_start_registry_container backoff container::_start_cadvisor_container backoff container::_start_node_exporter_container # swarm should be started last so we know everything is up backoff container::_start_swarm_container + # currently @henrymollman does not understand why restarting swarm works + # but without this line docker-listener will time out getting events + # and the stream will close. this is an intermittent error however + docker restart swarm } # Stops all dock container services diff --git a/lib/dock.sh b/lib/dock.sh index fdb1b6c..99074b8 100644 --- a/lib/dock.sh +++ b/lib/dock.sh @@ -9,38 +9,11 @@ # @author Bryan Kendall # @module dock -source "${DOCK_INIT_BASE}/lib/aws.sh" source "${DOCK_INIT_BASE}/lib/cert.sh" -source "${DOCK_INIT_BASE}/lib/consul.sh" -source "${DOCK_INIT_BASE}/lib/container.sh" -source "${DOCK_INIT_BASE}/lib/upstart.sh" - source "${DOCK_INIT_BASE}/lib/util/backoff.sh" source "${DOCK_INIT_BASE}/lib/util/log.sh" source "${DOCK_INIT_BASE}/lib/util/rollbar.sh" -# An "on exit" trap to clean up sensitive keys and files on the dock itself. -# Note that this will have no effect if the `DONT_DELETE_KEYS` environment has -# been set (useful for testing) -dock::cleanup::exit_trap() { - # Delete the keys unless the `DO_NOT_DELETE` flag is set - if [[ "${DONT_DELETE_KEYS}" == "" ]]; then - log::info '[CLEANUP TRAP] Removing Keys' - rm -f "${CERT_PATH}"/ca-key.pem \ - "${CERT_PATH}"/pass \ - "${DOCK_INIT_BASE}"/consul-resources/template-config.hcl \ - "${DOCK_INIT_BASE}"/consul-resources/vault/**/auth-token \ - "${DOCK_INIT_BASE}"/consul-resources/vault/**/token-* \ - "${DOCK_INIT_BASE}"/key/rollbar.token - fi -} - -# Sets the cleanup trap for the entire script -dock::cleanup::set_exit_trap() { - log::info "Setting key cleanup trap" - trap 'dock::cleanup::exit_trap' EXIT -} - # Sets the value of `$ORG_ID` as the org label in the docker configuration dock::set_config_org() { log::info "Setting organization id in docker configuration" @@ -68,60 +41,3 @@ dock::generate_certs() { backoff dock::generate_certs_backoff } -# Generates the correct /etc/hosts file for the dock -dock::generate_etc_hosts() { - log::info "Generating /etc/hosts" - - rollbar::fatal_trap \ - "Dock-Init: Failed to Add Host Registry Entry" \ - "Consul-Template was unable to realize the registry template." - - local template='' - template+="$DOCK_INIT_BASE/consul-resources/templates/hosts-registry.ctmpl" - template+=":$DOCK_INIT_BASE/hosts-registry.txt" - consul-template \ - -config="${DOCK_INIT_BASE}"/consul-resources/template-config.hcl \ - -once \ - -template="${template}" - - rollbar::clear_trap -} - -# Sets the correct registry.runnable.com host -dock::set_registry_host() { - local registry_host=$(cat "$DOCK_INIT_BASE/hosts-registry.txt") - log::info "Set registry host: $registry_host" - echo "$registry_host" >> /etc/hosts -} - -# Remove docker key file so it generates a unique id -dock::remove_docker_key_file() { - log::info "Removing docker key.json" - rm -f /etc/docker/key.json -} - -# Master function for performing all tasks and initializing the dock -dock::init() { - # Setup the exit trap and rollbar - dock::cleanup::set_exit_trap - rollbar::init - - # Connect to and configure consul then collect various information we need - consul::connect - consul::get_environment - consul::configure_consul_template - aws::get_org_id - - # Now that we have everything we need and consul is ready, initialize the dock - dock::set_hostname - dock::set_config_org - dock::generate_certs - dock::generate_etc_hosts - dock::set_registry_host - dock::remove_docker_key_file - upstart::start - container::start - - # Give the all clear message! - log::info "Init Done!" -} diff --git a/lib/upstart.sh b/lib/upstart.sh index 03f97a3..48c8483 100644 --- a/lib/upstart.sh +++ b/lib/upstart.sh @@ -5,95 +5,9 @@ # @author Ryan Sandor Richards # @author Bryan Kendall -source "${DOCK_INIT_BASE}/lib/consul.sh" -source "${DOCK_INIT_BASE}/lib/util/backoff.sh" source "${DOCK_INIT_BASE}/lib/util/log.sh" source "${DOCK_INIT_BASE}/lib/util/rollbar.sh" -# Generates upstart scripts for the dock -upstart::generate_scripts() { - log::info "Generating Upstart Scripts" - rollbar::fatal_trap \ - "Dock-Init: Failed to Generate Upstart Script" \ - "Failed to generate the upstart scripts." - upstart::generate_scripts - rollbar::clear_trap -} - -# Configures the template for a given service -# @param $1 name Name of the service -# @param $2 path Path to the servic -upstart::configure_service() { - local name="${1}" - log::trace "Configuring $name" - rollbar::fatal_trap \ - "Consul-Template: Failed to Render $name Config" \ - "Consule-Template was unable to realize the given template." - - local template_path="$DOCK_INIT_BASE/consul-resources/templates/services" - template_path+="/$name.conf.ctmpl" - template_path+=":/etc/init/$name.conf" - - consul-template \ - -config="$DOCK_INIT_BASE/consul-resources/template-config.hcl" \ - -once \ - -template="$template_path" - echo "manual" > /etc/init/"$name".override - - rollbar::clear_trap -} - -# Generates upstart scripts for thoses services that require environment info -# from consul -upstart::generate_scripts() { - log::info "Configuring Upstart Scripts" - upstart::configure_service "charon" - log::trace "Done Generating Upstart Scripts" -} - -# Updates a service to the consul version, installs packages, then restarts it. -# @param $1 Name of the service -upstart::upstart_named_service() { - local name="${1}" - local attempt="${2}" - local data='{"attempt":'"${attempt}"'}' - local version="$(consul::get ${name}/version)" - local key_path="$DOCK_INIT_BASE/key/id_rsa_runnabledock" - - rollbar::warning_trap \ - "$name: Cannot Upstart Services" \ - "Attempting to upstart the services and failing." \ - "${data}" - - log::info "Updating and restarting $name @ $version" && - cd "/opt/runnable/$name" && - ssh-agent bash -c "ssh-add $key_path; git fetch origin" && - git checkout "$version" && - ssh-agent bash -c "ssh-add $key_path; USERPROFILE=/home/ubuntu npm install --production" && - service "$name" restart - - rollbar::clear_trap -} - -# Starts a service installed on the machine. -# @param $1 Name of the service -# @param $2 Attempt number -upstart::upstart_service() { - local name="${1}" - local attempt="${2}" - local data='{"attempt":'"${attempt}"'}' - - rollbar::warning_trap \ - "$name: Cannot Upstart Service" \ - "Attempting to upstart the service and failing." \ - "${data}" - - log::info "Starting $name" - service "$name" restart - - rollbar::clear_trap -} - # Start dockers (due to manual override now set in /etc/init) upstart::start_docker() { log::info "Starting Docker" @@ -105,7 +19,7 @@ upstart::start_docker() { log::info "Waiting for Docker" local attempt=1 - local timeout=1 + local timeout=.5 while [ ! -e /var/run/docker.sock ] do log::info "Docker Sock N/A ($attempt)" @@ -115,52 +29,5 @@ upstart::start_docker() { rollbar::report_warning "${title}" "${message}" "$data" sleep $timeout attempt=$(( attempt + 1 )) - timeout=$(( timeout * 2 )) done } - -# Upstarts services that are supposed to be running on the dock. -# @param $1 attempt Attempt number. -upstart::upstart_services_with_backoff_params() { - local attempt="${1}" - upstart::upstart_named_service "krain" $attempt - upstart::upstart_named_service "charon" $attempt - upstart::upstart_service "datadog-agent" $attempt -} - -# Pulls the latest docker image for the runnable image builder -# @param $1 attempt The current attempt for pulling image builder -upstart::pull_image_builder() { - local attempt="${1}" - local name="image-builder" - local version="$(consul::get $name/version)" - - log::info "Pulling image-builder:$version (${attempt})" - docker pull "registry.runnable.com/runnable/image-builder:$version" - - if [[ "$?" -gt "0" ]]; then - local data='{"attempt":'"${attempt}"'}' - rollbar::report_warning \ - "Dock-Init: Cannot Upstart Services" \ - "Attempting to upstart the services and failing." \ - "${data}" - return 1 - fi -} - -# Starts all services needed for the dock -upstart::start() { - log::info "Upstarting dock" - upstart::generate_scripts - upstart::start_docker - backoff upstart::pull_image_builder - backoff upstart::upstart_services_with_backoff_params -} - -# Stops all dock services -upstart::stop() { - log::info "Stopping all dock upstart services" - service krain stop - service charon stop - service docker stop -} diff --git a/test/container.sh b/test/container.sh index 597b393..11e469a 100644 --- a/test/container.sh +++ b/test/container.sh @@ -140,6 +140,8 @@ describe 'container.sh' stub container::_start_cadvisor_container stub container::_start_node_exporter_container stub container::_start_swarm_container + stub upstart::start_docker + stub docker it 'should start all required containers' container::start @@ -147,10 +149,12 @@ describe 'container.sh' container::_start_cadvisor_container::called container::_start_node_exporter_container::called container::_start_swarm_container::called + upstart::start_docker::called container::_start_registry_container::restore container::_start_cadvisor_container::restore container::_start_node_exporter_container::restore container::_start_swarm_container::restore + upstart::start_docker::restore end # end container::start end # container.sh diff --git a/test/upstart.sh b/test/upstart.sh deleted file mode 100644 index 18f8843..0000000 --- a/test/upstart.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# Unit tests for the `lib/upstart.sh` module. -# @author Anandkumar Patel - -source "$DOCK_INIT_BASE/lib/container.sh" -source "$DOCK_INIT_BASE/lib/upstart.sh" -source "$DOCK_INIT_BASE/test/fixtures/shtub.sh" - -describe 'upstart.sh' - stub log::info - - describe 'upstart::upstart_service' - stub rollbar::warning_trap - stub rollbar::clear_trap - - it 'should start the given service' - stub service - local service_name='foobar' - upstart::upstart_service "$service_name" - service::called_with "$service_name restart" - service::restore - end - - rollbar::warning_trap::restore - rollbar::clear_trap::restore - end - - describe 'upstart::upstart_services_with_backoff_params' - it 'should start all our services' - local storage="" - serviceStub() { storage+="$@ "; } - stub::exec upstart::upstart_named_service serviceStub - stub::exec upstart::upstart_service serviceStub - - local attempt=8 - upstart::upstart_services_with_backoff_params $attempt - - local expected="krain 8 charon 8 datadog-agent 8 " - - assert equal "$expected" "$storage" - - upstart::upstart_named_service::restore - upstart::upstart_service::restore - end - end - - describe 'upstart::pull_image_builder' - local image_builder_version='v1.2.3' - stub rollbar::report_warning - stub docker - stub::returns consul::get "$image_builder_version" - - it 'should attempt to pull image builder' - local registry="registry.runnable.com/runnable/image-builder" - upstart::pull_image_builder 1 - consul::get::called_with "image-builder/version" - docker::called_with "pull $registry:$image_builder_version" - end - - it 'should return 1 on pull failure' - docker::errors - upstart::pull_image_builder 1 - assert equal "$?" "1" - end - - it 'should report a warning on pull failure' - docker::errors - upstart::pull_image_builder 222 - rollbar::report_warning::called_with \ - "Dock-Init: Cannot Upstart Services" \ - "Attempting to upstart the services and failing." \ - '{"attempt":222}' - end - - # docker::restore - rollbar::report_warning::restore - consul::get::restore - end - - log::info::restore -end # upstart.sh