Add ASGRollout feature. (#128)

.rubocop.yml

@@ -2,14 +2,17 @@
 AllCops:
   TargetRubyVersion: 2.2
   Exclude:
-    - '*.gemspec'
+    - '.gemspec'
     - 'vendor/**/*'
     - 'sample/bin/aws-codedeploy-samples/**/*'
+    - 'sample/gems/**/*'
 Metrics/AbcSize:
-  Max: 20
+  Max: 30
 Metrics/MethodLength:
   Max: 30
 Metrics/LineLength:
   Max: 100
 Style/ClassAndModuleChildren:
   Enabled: false
+Metrics/ClassLength:
+  Max: 130

docs/plugins/asg_rollout.md

@@ -0,0 +1,93 @@
+# Auto-Scaling Group LaunchConfig Rollout Tool
+
+## Overview
+
+This plugin adds support for rolling out changes to Auto Scaling
+Groups that have happened after a stack update. It supports various
+pre- and post-actions on the instances (using Moonshot's native SSH
+support).
+
+## Example
+
+The ASGRollout class is intended to be used within a method in your
+project's derived class of the Moonshot::CLI class.
+
+```ruby
+#!/usr/bin/env ruby
+
+require 'moonshot'
+
+class MyService < Moonshot::CLI
+  # .. normal configuration ..
+
+  desc :asg_rollout, 'Update instances in the Auto Scaling Group'
+  def asg_rollout
+    ar = Moonshot::Tools::ASGRollout.new do |config|
+      config.controller = controller
+      config.logical_id = 'APIAutoScalingGroup'
+
+      # Trigger the worker process to initiate a clean shutdown.
+      config.predetach = ->(h) { 0 == h.exec('systemctl stop my-worker-1').exitstatus }
+
+      # Wait for that clean shutdown to happen.
+      config.terminate_when = ->(h) { 0 == h.exec('pgrep -f my-worker-1').exitstatus }
+    end
+    ar.run!
+  end
+end
+
+MyService.start
+```
+
+## Configuration
+
+The ASGRollout object accepts two required keyword arguments:
+  - **controller**: A Moonshot::Controller. If you are using
+    Moonshot::CLI the `controller` method is most often when you want
+    to use.
+  - **logical_id**: The Logical resource ID from the CloudFormation
+    stack to operate on.
+
+By default, the ASGRollout class will build a list of instances with
+non-conforming LaunchConfiguration, then one at a time perform the
+following actions:
+  1. Increase the *Max* and *Desired* sizes of the Auto Scaling Group by one.
+  2. Wait for the new instance to be *InService*.
+  3. Detach a non-conforming instance.
+  4. Wait for the instance to be *OutOfService* or removed from the
+     Auto Scaling Group.
+  5. If the Auto Scaling Group has an associated Elastic Load
+     Balancer, wait for the instance to be *Todo?* there as well.
+  6. Wait for a new instance to replace it.
+  7. Wait for that new instance to be *InService* in the Auto Scaling Group.
+  8. If the Auto Scaling Group has an associated Elastic Load
+     Balancer, wait for the instance to be *InService* there as well.
+  9. Terminate the detached non-conforming instance.
+  10. If there are other non-conforming instances, go to
+     step 3.
+  11. Restore the *Max* and *Desired* sizes of the Auto Scaling Group
+      to their original values.
+
+A config object is yielded by the constructor as illustrated in the
+example above, which accepts the following options:
+  - **pre_detach** *(Callable)*: This will be run before step 3
+    above. If it returns `false`, the process will be aborted.
+  - **terminate** *(Callable)*: This will be run to execute step 9. If
+    not specified, the default is to use EC2's Terminate API.
+  - **terminate_when** *(Callable)*: This will be run every 5 seconds
+    prior to step 9. Step 9 will continue as soon as this returns
+    `true`.
+  - **terminate_when_timeout** *(Integer)*: (Default 300s) Number of seconds to
+    wait for the terminate_when lambda to return true. After this
+    timeout, the process will be aborted.
+
+For the callables above, an instance of `HookExecEnvironment` is
+passed in, providing the following methods:
+  - **exec** *(Moonshot::SSHForkExecutor::Result)*: Run a command on
+    the instance and return the output and exit code.
+  - **ec2** *(Aws::EC2::Client)*: A configured Aws::EC2::Client for the region we're
+    operating in.
+  - **instance_id** *(String)*: The EC2 instance ID.
+  - **debug(msg)**: Log a debug message, shown when Moonshot is in
+    verbose mode.
+  - **info(msg)**: Log a message, shown always.

lib/moonshot.rb

@@ -39,7 +39,10 @@ module Plugins # rubocop:disable Documentation
   'build_mechanism/github_release',
   'build_mechanism/travis_deploy',
   'build_mechanism/version_proxy',
-  'deployment_mechanism/code_deploy'
+  'deployment_mechanism/code_deploy',
+
+  # Core Tools
+  'tools/asg_rollout'
 ].each { |f| require_relative "moonshot/#{f}" }
 
 # Bundled plugins

lib/moonshot/build_mechanism/script.rb

@@ -47,7 +47,7 @@ def post_build_hook(_version)
 
   private
 
-  def run_script(step, env: {}) # rubocop:disable AbcSize
+  def run_script(step, env: {})
     popen2e(env, @script) do |_, out, wait|
       output = []
 

lib/moonshot/deployment_mechanism/code_deploy.rb

@@ -278,7 +278,7 @@ def wait_for_deployment(id, step)
     end
   end
 
-  def handle_deployment_failure(deployment_id) # rubocop:disable AbcSize
+  def handle_deployment_failure(deployment_id)
     instances = cd_client.list_deployment_instances(deployment_id: deployment_id)
                          .instances_list.map do |instance_id|
       cd_client.get_deployment_instance(deployment_id: deployment_id,

lib/moonshot/shell.rb

@@ -2,7 +2,7 @@
 module Moonshot::Shell
   # Run a command, returning stdout. Stderr is suppressed unless the command
   # returns non-zero.
-  def sh_out(cmd, fail: true, stdin: '') # rubocop:disable AbcSize
+  def sh_out(cmd, fail: true, stdin: '')
     r_in, w_in = IO.pipe
     r_out, w_out = IO.pipe
     r_err, w_err = IO.pipe

lib/moonshot/ssh_command_builder.rb

@@ -15,7 +15,7 @@ def build(command = nil)
       cmd << "-i #{@config.ssh_identity_file}" if @config.ssh_identity_file
       cmd << "-l #{@config.ssh_user}" if @config.ssh_user
       cmd << instance_ip
-      cmd << command if command
+      cmd << Shellwords.escape(command) if command
       Result.new(cmd.join(' '), instance_ip)
     end
 

lib/moonshot/ssh_fork_executor.rb

@@ -0,0 +1,20 @@
+require 'open3'
+
+module Moonshot
+  # Run an SSH command via fork/exec.
+  class SSHForkExecutor
+    Result = Struct.new(:output, :exitstatus)
+
+    def run(cmd)
+      output = StringIO.new
+
+      exit_status = nil
+      Open3.popen3(cmd) do |_, stdout, _, wt|
+        output << stdout.read until stdout.eof?
+        exit_status = wt.value.exitstatus
+      end
+
+      Result.new(output.string.chomp, exit_status)
+    end
+  end
+end

lib/moonshot/ssh_target_selector.rb

@@ -6,7 +6,7 @@ def initialize(stack, asg_name: nil)
       @stack = stack
     end
 
-    def choose! # rubocop:disable AbcSize
+    def choose!
       groups = @stack.resources_of_type('AWS::AutoScaling::AutoScalingGroup')
 
       asg = if groups.count == 1

lib/moonshot/stack_asg_printer.rb

@@ -5,7 +5,7 @@
 module Moonshot
   # Display information about the AutoScaling Groups, associated ELBs, and
   # managed instances to the user.
-  class StackASGPrinter # rubocop:disable ClassLength
+  class StackASGPrinter
     include CredsHelper
 
     def initialize(stack, table)
@@ -79,7 +79,7 @@ def get_addl_info(instance_ids)
       data
     end
 
-    def add_asg_info(table, asg_info) # rubocop:disable AbcSize
+    def add_asg_info(table, asg_info)
       name = asg_info.auto_scaling_group_name.blue
       table.add_line "Name: #{name}"
 

lib/moonshot/tools/asg_rollout.rb

@@ -0,0 +1,170 @@
+require_relative 'asg_rollout_config'
+require_relative 'asg_rollout/asg'
+require_relative 'asg_rollout/hook_exec_environment'
+
+module Moonshot
+  module Tools
+    class ASGRollout # rubocop:disable Documentation
+      attr_accessor :config
+
+      def initialize(controller:, logical_id:)
+        @config = ASGRolloutConfig.new
+        @controller = controller
+        @logical_id = logical_id
+        yield @config if block_given?
+      end
+
+      def run!
+        increase_max_and_desired
+        new_instance = wait_for_new_instance
+        wait_for_in_service(new_instance)
+
+        targets = asg.non_conforming_instances
+        last_instance = targets.last
+
+        targets.each do |instance|
+          run_pre_detach(instance) if @config.pre_detach
+          detach(instance, decrement: instance == last_instance)
+          wait_for_out_of_service(instance)
+
+          unless instance == last_instance
+            new_instance = wait_for_new_instance
+            wait_for_in_service(new_instance)
+          end
+
+          wait_for_terminate_when_hook(instance) if @config.terminate_when
+          terminate(instance)
+        end
+      ensure
+        log.start_threaded 'Restoring MaxSize/DesiredCapacity values to normal...' do |s|
+          asg.set_max_and_desired(@max, @desired)
+
+          s.success 'Restored MaxSize/DesiredCapacity values to normal!'
+        end
+      end
+
+      private
+
+      def increase_max_and_desired
+        log.start_threaded 'Increasing MaxSize/DesiredCapacity by 1.' do |s|
+          @max, @desired = asg.current_max_and_desired
+
+          asg.set_max_and_desired(@max + 1, @desired + 1)
+          s.success 'Increased MaxSize/DesiredCapacity by 1.'
+        end
+      end
+
+      def wait_for_new_instance
+        new_instance = nil
+        log.start_threaded 'Waiting for a new instance to join Auto Scaling Group...' do |s|
+          new_instance = asg.wait_for_new_instance
+          s.success "A wild #{new_instance.blue} appears!"
+        end
+        new_instance
+      end
+
+      def wait_for_in_service(new_instance)
+        log.start_threaded "Waiting for #{new_instance.blue} to be InService..." do |s|
+          instance_health = nil
+
+          loop do
+            instance_health = asg.instance_health(new_instance)
+            break if instance_health.in_service?
+
+            s.continue "Instance #{new_instance.blue} is #{instance_health}..."
+
+            sleep @config.instance_health_delay
+          end
+
+          s.success "Instance #{new_instance.blue} is #{instance_health}!"
+        end
+      end
+
+      def run_pre_detach(instance)
+        if @config.pre_detach
+          log.start_threaded "Running PreDetach hook on #{instance.blue}..." do |s|
+            he = HookExecEnvironment.new(@controller.config, instance)
+            if false == @config.pre_detach.call(he)
+              s.failure "PreDetach hook failed for #{instance.blue}!"
+              raise "PreDetach hook failed for #{instance.blue}!"
+            end
+
+            s.success "PreDetach hook complete for #{instance.blue}!"
+          end
+        end
+      end
+
+      def detach(instance, decrement:)
+        log.start_threaded "Detaching instance #{instance.blue}..." do |s|
+          asg.detach_instance(instance, decrement: decrement)
+
+          if decrement
+            s.success "Detached instance #{instance.blue}, and decremented DesiredCapacity."
+          else
+            s.success "Detached instance #{instance.blue}."
+          end
+        end
+      end
+
+      def wait_for_out_of_service(instance)
+        log.start_threaded "Waiting for #{instance.blue} to be OutOfService..." do |s|
+          instance_health = nil
+
+          loop do
+            instance_health = asg.instance_health(instance)
+            break if instance_health.out_of_service?
+
+            s.continue "Instance #{instance.blue} is #{instance_health}..."
+
+            sleep @config.instance_health_delay
+          end
+
+          s.success "Instance #{instance.blue} is #{instance_health}!"
+        end
+      end
+
+      def wait_for_terminate_when_hook(instance)
+        log.start_threaded "Waiting for TerminateWhen hook for #{instance.blue}..." do |s|
+          start = Time.now.to_f
+          he = HookExecEnvironment.new(@controller.config, instance)
+          timeout = @config.terminate_when_timeout
+
+          loop do
+            break if @config.terminate_when.call(he)
+            sleep @config.terminate_when_delay
+
+            if Time.now.to_f - start > timeout
+              s.failure "TerminateWhen for #{instance.blue} did not complete in #{timeout} seconds!"
+              raise "TerminateWhen for #{instance.blue} did not complete in #{timeout} seconds!"
+            end
+          end
+
+          s.success "Completed TerminateWhen check for #{instance.blue}!"
+        end
+      end
+
+      def terminate(instance)
+        log.start_threaded "Terminating #{instance.blue}..." do |s|
+          he = HookExecEnvironment.new(@controller.config, instance)
+          @config.terminate.call(he)
+          s.success "Terminated #{instance.blue}!"
+        end
+      end
+
+      def asg
+        return @asg if @asg
+
+        asg_name = @controller.stack.physical_id_for(@logical_id)
+        unless asg_name
+          raise "Could not find Auto Scaling Group #{@logical_id}!"
+        end
+
+        @asg ||= ASGRollout::ASG.new(asg_name)
+      end
+
+      def log
+        @controller.config.interactive_logger
+      end
+    end
+  end
+end