Merge Released And Current Approved Changes

.gitignore

@@ -8,3 +8,5 @@ RemoteSystemsTempFiles/
 codedeploy-local.log*
 codedeploy-local.*.log
 deployment/
+.idea/
+.DS_STORE

buildspec-agent-rake.yml

@@ -0,0 +1,25 @@
+version: 0.2
+
+phases:
+  install:
+    commands:
+      - echo Installing bundler
+      - gem install bundler
+      - echo Using bundler to install remaining gems
+      - bundle install
+  pre_build:
+    commands:
+      - echo Build started on `date`
+      - echo Running rake  - unit tests
+      - rake
+  build:
+    commands:
+      - echo Build completed on `date`
+      - echo Running rake test-integration - integ tests
+      - rake test-integration
+  post_build:
+    commands:
+      - echo $CODEBUILD_SOURCE_VERSION > codebuild_source_commit_id
+artifacts:
+  files:
+    - codebuild_source_commit_id

codedeploy_agent-1.1.0.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency('aws-sdk-core', '~> 2.9')
   spec.add_dependency('simple_pid', '~> 0.2.1')
   spec.add_dependency('docopt', '~> 0.5.0')
+  spec.add_dependency('concurrent-ruby', '~> 1.0.5')
 
   spec.add_development_dependency('rake', '~> 10.0')
   spec.add_development_dependency('rspec', '~> 3.2.0')

features/resources/sample_app_bundle_windows/appspec.yml

@@ -1,6 +1,10 @@
 version: 0.0
 os: windows
 hooks:
+    BeforeBlockTraffic:
+        - location: scripts/before_block_traffic.bat
+    AfterBlockTraffic:
+        - location: scripts/after_block_traffic.bat
     ApplicationStop:
         - location: scripts/application_stop.bat
     BeforeInstall:
@@ -11,3 +15,7 @@ hooks:
         - location: scripts/application_start.bat
     ValidateService:
         - location: scripts/validate_service.bat
+    BeforeAllowTraffic:
+        - location: scripts/before_allow_traffic.bat
+    AfterAllowTraffic:
+        - location: scripts/after_allow_traffic.bat

features/step_definitions/agent_steps.rb

@@ -46,9 +46,6 @@ def eventually(options = {}, &block)
   @working_directory = Dir.mktmpdir
   configure_local_agent(@working_directory)
 
-  #Reset aws credentials to default location
-  Aws.config[:credentials] = Aws::SharedCredentials.new.credentials
-
   #instantiate these clients first so they use user's aws creds instead of assumed role creds
   @codedeploy_client = Aws::CodeDeploy::Client.new
   @iam_client = Aws::IAM::Client.new
@@ -60,8 +57,6 @@ def configure_local_agent(working_directory)
   InstanceAgent::Log.init(File.join(working_directory, 'codedeploy-agent.log'))
   InstanceAgent::Config.init
   InstanceAgent::Platform.util = StepConstants::IS_WINDOWS ? InstanceAgent::WindowsUtil : InstanceAgent::LinuxUtil
-
-  if StepConstants::IS_WINDOWS then configure_windows_certificate end
   InstanceAgent::Config.config[:on_premises_config_file] = "#{working_directory}/codedeploy.onpremises.yml"
 
   configuration_contents = <<-CONFIG
@@ -84,13 +79,6 @@ def configure_local_agent(working_directory)
   InstanceAgent::Config.load_config
 end
 
-def configure_windows_certificate
-  cert_dir = File.expand_path(File.join(File.dirname(__FILE__), '..\certs'))
-  Aws.config[:ssl_ca_bundle] = File.join(cert_dir, 'windows-ca-bundle.crt')
-  ENV['AWS_SSL_CA_DIRECTORY'] = File.join(cert_dir, 'windows-ca-bundle.crt')
-  ENV['SSL_CERT_FILE'] = File.join(cert_dir, 'windows-ca-bundle.crt')
-end
-
 After("@codedeploy-agent") do
   @thread.kill unless @thread.nil?
   @codedeploy_client.delete_application({:application_name => @application_name}) unless @application_name.nil?
@@ -234,12 +222,20 @@ def create_instance_role
   rescue Aws::IAM::Errors::EntityAlreadyExists
     #Using the existing role
   end
-  eventually do
+
+  instance_role_arn = eventually do
     instance_role = @iam_client.get_role({:role_name => INSTANCE_ROLE_NAME}).role
     expect(instance_role).not_to be_nil
     expect(instance_role.assume_role_policy_document).not_to be_nil
     instance_role.arn
   end
+
+  @iam_client.update_assume_role_policy({
+    policy_document: instance_role_policy,
+    role_name: INSTANCE_ROLE_NAME,
+  })
+
+  instance_role_arn
 end
 
 def instance_role_policy

features/step_definitions/codedeploy_local_steps.rb

@@ -14,10 +14,6 @@
 
 Before("@codedeploy-local") do
   @test_directory = Dir.mktmpdir
-
-  #Reset aws credentials to default location
-  Aws.config[:credentials] = Aws::SharedCredentials.new.credentials
-
   configure_local_agent(@test_directory)
 end
 
@@ -60,7 +56,7 @@ def tar_app_bundle(temp_directory_to_create_bundle)
   #Unfortunately Minitar will keep pack all the file paths as given, so unless you change directories into the location where you want to pack the files the bundle won't have the correct files and folders
   Dir.chdir @bundle_original_directory_location
 
-  File.open(tar_file_name, 'wb') { |tar| Minitar.pack(directories_and_files_inside(@bundle_original_directory_location), tar) }
+  File.open(tar_file_name, 'wb') { |tar| Minitar.pack(InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside(@bundle_original_directory_location), tar) }
 
   Dir.chdir old_direcory
   tar_file_name
@@ -102,7 +98,7 @@ def tgz_app_bundle(temp_directory_to_create_bundle)
 
   File.open(tgz_file_name, 'wb') do |file|
     Zlib::GzipWriter.wrap(file) do |gz|
-      Minitar.pack(directories_and_files_inside(@bundle_original_directory_location), gz)
+      Minitar.pack(InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside(@bundle_original_directory_location), gz)
     end
   end
 
@@ -134,7 +130,10 @@ def create_local_deployment(custom_events = nil, file_exists_behavior = nil)
     codeedeploy_command_suffix = " --file-exists-behavior #{file_exists_behavior}"
   end
 
-  system "bin/codedeploy-local --bundle-location #{@bundle_location} --type #{@bundle_type} --deployment-group #{LOCAL_DEPLOYMENT_GROUP_ID} --agent-configuration-file #{InstanceAgent::Config.config[:config_file]}#{codeedeploy_command_suffix}"
+  # Windows doesn't respect shebang lines so ruby needs to be specified
+  ruby_prefix_for_windows = StepConstants::IS_WINDOWS ? "ruby " : ""
+
+  system "#{ruby_prefix_for_windows}bin/codedeploy-local --bundle-location #{@bundle_location} --type #{@bundle_type} --deployment-group #{LOCAL_DEPLOYMENT_GROUP_ID} --agent-configuration-file #{InstanceAgent::Config.config[:config_file]}#{codeedeploy_command_suffix}"
 end
 
 Then(/^the local deployment command should succeed$/) do
@@ -146,7 +145,7 @@ def create_local_deployment(custom_events = nil, file_exists_behavior = nil)
 end
 
 Then(/^the expected files should have have been locally deployed to my host(| twice)$/) do |maybe_twice|
-  deployment_ids = directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{LOCAL_DEPLOYMENT_GROUP_ID}")
+  deployment_ids = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{LOCAL_DEPLOYMENT_GROUP_ID}")
   step "the expected files in directory #{bundle_original_directory_location}/scripts should have have been deployed#{maybe_twice} to my host during deployment with deployment group id #{LOCAL_DEPLOYMENT_GROUP_ID} and deployment ids #{deployment_ids.join(' ')}"
 end
 

features/step_definitions/common_steps.rb

@@ -3,24 +3,36 @@
 $:.unshift File.join(File.dirname(File.expand_path('../..', __FILE__)), 'features')
 require 'step_definitions/step_constants'
 
+@bucket_creation_count = 0;
 Given(/^I have a sample bundle uploaded to s3$/) do
-  s3 = Aws::S3::Client.new
-
-  begin
-    s3.create_bucket({
-      bucket: StepConstants::APP_BUNDLE_BUCKET, # required
-      create_bucket_configuration: {
-        location_constraint: Aws.config[:region],
-      }
-    })
-  rescue Aws::S3::Errors::BucketAlreadyOwnedByYou
-    #Already created the bucket
-  end
+=begin
+This fails if the s3 upload is attempted after assume_role is called in the first integration test. 
+This is because once you call assume role the next time it instantiates a client it is using different permissions. In my opinion thats a bug because it doesn't match the documentation for the AWS SDK.
+https://docs.aws.amazon.com/sdkforruby/api/index.html
+
+Their documentation says an assumed role is the LAST permission it will try to rely on but it looks like its always the first. But the s3 upload is the only place that this mattered so I simply forced this code so it doesn't do it again since the bundle is identical for both tests.
+=end
+  if @bucket_creation_count == 0
+    s3 = Aws::S3::Client.new
+
+    begin
+      s3.create_bucket({
+        bucket: StepConstants::APP_BUNDLE_BUCKET, # required
+        create_bucket_configuration: {
+          location_constraint: Aws.config[:region],
+        }
+      })
+    rescue Aws::S3::Errors::BucketAlreadyOwnedByYou
+      #Already created the bucket
+    end
 
-  Dir.mktmpdir do |temp_directory_to_create_zip_file|
-    File.open(zip_app_bundle(temp_directory_to_create_zip_file), 'rb') do |file|
-      s3.put_object(bucket: StepConstants::APP_BUNDLE_BUCKET, key: StepConstants::APP_BUNDLE_KEY, body: file)
+    Dir.mktmpdir do |temp_directory_to_create_zip_file|
+      File.open(zip_app_bundle(temp_directory_to_create_zip_file), 'rb') do |file|
+        s3.put_object(bucket: StepConstants::APP_BUNDLE_BUCKET, key: StepConstants::APP_BUNDLE_KEY, body: file)
+      end
     end
+
+    @bucket_creation_count += 1
   end
 
   @bundle_type = 'zip'
@@ -34,7 +46,7 @@ def zip_app_bundle(temp_directory_to_create_bundle)
 end
 
 def zip_directory(input_dir, output_file)
-  entries = directories_and_files_inside(input_dir)
+  entries = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside(input_dir)
   zip_io = Zip::File.open(output_file, Zip::File::CREATE)
 
   write_zip_entries(entries, '', input_dir, zip_io)
@@ -47,44 +59,41 @@ def write_zip_entries(entries, path, input_dir, zip_io)
     diskFilePath = File.join(input_dir, zipFilePath)
     if File.directory?(diskFilePath)
       zip_io.mkdir(zipFilePath)
-      folder_entries = directories_and_files_inside(diskFilePath)
+      folder_entries = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside(diskFilePath)
       write_zip_entries(folder_entries, zipFilePath, input_dir, zip_io)
     else
       zip_io.get_output_stream(zipFilePath){ |f| f.write(File.open(diskFilePath, "rb").read())}
     end
   end
 end
 
-def directories_and_files_inside(directory)
-  Dir.entries(directory) - %w(.. .)
-end
 
 Then(/^the expected files in directory (\S+) should have have been deployed(| twice) to my host during deployment with deployment group id (\S+) and deployment ids (.+)$/) do |expected_scripts_directory, maybe_twice, deployment_group_id, deployment_ids_space_separated|
   deployment_ids = deployment_ids_space_separated.split(' ')
-  directories_in_deployment_root_folder = directories_and_files_inside(InstanceAgent::Config.config[:root_dir])
-  expect(directories_in_deployment_root_folder.size).to eq(3)
+  directories_in_deployment_root_folder = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside(InstanceAgent::Config.config[:root_dir])
+  expect(directories_in_deployment_root_folder.size).to be >= 3
 
   #ordering of the directories depends on the deployment group id, so using include instead of eq
   expect(directories_in_deployment_root_folder).to include(*%W(deployment-instructions deployment-logs #{deployment_group_id}))
 
-  files_in_deployment_logs_folder = directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/deployment-logs")
+  files_in_deployment_logs_folder = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/deployment-logs")
   expect(files_in_deployment_logs_folder.size).to eq(1)
   expect(files_in_deployment_logs_folder).to eq(%w(codedeploy-agent-deployments.log))
 
-  directories_in_deployment_group_id_folder = directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{deployment_group_id}")
+  directories_in_deployment_group_id_folder = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{deployment_group_id}")
   expect(directories_in_deployment_group_id_folder.size).to eq(maybe_twice.empty? ? 1 : 2)
   expect(directories_in_deployment_group_id_folder).to eq(deployment_ids)
 
   deployment_id = deployment_ids.first
-  files_and_directories_in_deployment_id_folder = directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{deployment_group_id}/#{deployment_id}")
+  files_and_directories_in_deployment_id_folder = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{deployment_group_id}/#{deployment_id}")
   expect(files_and_directories_in_deployment_id_folder).to include(*%w(logs deployment-archive))
 
-  files_and_directories_in_deployment_archive_folder = directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{deployment_group_id}/#{deployment_id}/deployment-archive")
+  files_and_directories_in_deployment_archive_folder = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{deployment_group_id}/#{deployment_id}/deployment-archive")
   expect(files_and_directories_in_deployment_archive_folder.size).to eq(2)
   expect(files_and_directories_in_deployment_archive_folder).to include(*%w(appspec.yml scripts))
 
-  files_in_scripts_folder = directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{deployment_group_id}/#{deployment_id}/deployment-archive/scripts")
-  sample_app_bundle_script_files = directories_and_files_inside(expected_scripts_directory)
+  files_in_scripts_folder = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside("#{InstanceAgent::Config.config[:root_dir]}/#{deployment_group_id}/#{deployment_id}/deployment-archive/scripts")
+  sample_app_bundle_script_files = InstanceAgent::Plugins::CodeDeployPlugin::DeploymentCommandTracker.directories_and_files_inside(expected_scripts_directory)
   expect(files_in_scripts_folder.size).to eq(sample_app_bundle_script_files.size)
   expect(files_in_scripts_folder).to include(*sample_app_bundle_script_files)
 end

features/step_definitions/step_constants.rb

@@ -5,9 +5,18 @@ class StepConstants
   ENV['AWS_REGION'] = Aws.config[:region]
 
   def self.current_aws_account
+    if StepConstants::IS_WINDOWS then StepConstants.configure_windows_certificate end
+
     Aws::STS::Client.new.get_caller_identity.account
   end
 
+  def self.configure_windows_certificate
+    cert_dir = File.expand_path(File.join(File.dirname(__FILE__), '..\..\certs'))
+    Aws.config[:ssl_ca_bundle] = File.join(cert_dir, 'windows-ca-bundle.crt')
+    ENV['AWS_SSL_CA_DIRECTORY'] = File.join(cert_dir, 'windows-ca-bundle.crt')
+    ENV['SSL_CERT_FILE'] = File.join(cert_dir, 'windows-ca-bundle.crt')
+  end
+
   CODEDEPLOY_TEST_PREFIX = "codedeploy-agent-integ-test-" unless defined?(CODEDEPLOY_TEST_PREFIX)
   IS_WINDOWS = (RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) unless defined?(IS_WINDOWS)
   APP_BUNDLE_BUCKET_SUFFIX = IS_WINDOWS ? '-win' : '-linux' unless defined?(APP_BUNDLE_BUCKET_SUFFIX)

lib/aws/codedeploy/local/cli_validator.rb

@@ -52,7 +52,7 @@ def validate(args)
             end
 
             if events.include?('Install') && any_new_revision_event_before_install(events)
-              raise ValidationError.new("The only events that can be specified before Install are #{events_using_previous_successfuly_deployment_revision.push('DownloadBundle').join(',')}. Please fix the order of your specified events: #{args['--events']}")
+              raise ValidationError.new("The only events that can be specified before Install are #{events_using_previous_successfuly_deployment_revision.push('DownloadBundle', 'BeforeInstall').join(',')}. Please fix the order of your specified events: #{args['--events']}")
             end
           end
 
@@ -79,7 +79,7 @@ def events_using_previous_successfuly_deployment_revision
 
         def events_using_new_revision
           InstanceAgent::Plugins::CodeDeployPlugin::HookExecutor::MAPPING_BETWEEN_HOOKS_AND_DEPLOYMENTS.select do |key,value|
-            value != InstanceAgent::Plugins::CodeDeployPlugin::HookExecutor::LAST_SUCCESSFUL_DEPLOYMENT
+            value != InstanceAgent::Plugins::CodeDeployPlugin::HookExecutor::LAST_SUCCESSFUL_DEPLOYMENT && key != 'BeforeInstall'
           end.keys
         end
 

lib/instance_agent/config.rb

@@ -23,6 +23,7 @@ def initialize
         :pid_dir => nil,
         :shared_dir => nil,
         :user => nil,
+        :ongoing_deployment_tracking => 'ongoing-deployment',
         :children => 1,
         :http_read_timeout => 80,
         :instance_service_region => nil,
@@ -31,6 +32,7 @@ def initialize
         :wait_between_runs => 30,
         :wait_after_error => 30,
         :codedeploy_test_profile => 'prod',
+        :kill_agent_max_wait_time_seconds => 7200,
         :on_premises_config_file => '/etc/codedeploy-agent/conf/codedeploy.onpremises.yml',
         :proxy_uri => nil,
         :enable_deployments_log => true

lib/instance_agent/platform/linux_util.rb

@@ -67,6 +67,32 @@ def self.codedeploy_version_file
     def self.fallback_version_file
       "/opt/codedeploy-agent"
     end
+
+     # shelling out the rm folder command to native os in this case linux.
+    def self.delete_dirs_command(dirs_to_delete)
+      log(:debug,"Dirs to delete: #{dirs_to_delete}");
+      for dir in dirs_to_delete do
+        log(:debug,"Deleting dir: #{dir}");
+        delete_folder(dir);  
+      end
+    end
+
+    private 
+    def self.delete_folder (dir)
+      if dir != nil && dir != "/"
+        output = `rm -rf #{dir} 2>&1`
+        exit_status = $?.exitstatus
+        log(:debug, "Command status: #{$?}")
+        log(:debug, "Command output: #{output}")
+        unless exit_status == 0
+          msg = "Error deleting directories: #{exit_status}"
+          log(:error, msg)
+          raise msg
+        end 
+      else
+        log(:debug, "Empty directory or a wrong directory passed,#{dir}");  
+      end
+    end  
 
     private
     def self.execute_tar_command(cmd)
@@ -84,7 +110,7 @@ def self.execute_tar_command(cmd)
         raise msg
       end
     end
-
+   
     private
     def self.log(severity, message)
       raise ArgumentError, "Unknown severity #{severity.inspect}" unless InstanceAgent::Log::SEVERITIES.include?(severity.to_s)