From 28752bbeac7ab7cfbefd7bec4838de04f9bbae32 Mon Sep 17 00:00:00 2001 From: Zara Date: Mon, 1 Oct 2018 08:33:03 -0700 Subject: [PATCH 1/5] Added support in the agent for Ubuntu LTS release("bionic") ships with Ruby 2.5.x. cr https://code.amazon.com/reviews/CR-3383787 --- bin/codedeploy-agent | 2 +- bin/install | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/codedeploy-agent b/bin/codedeploy-agent index a683d0c3..00ef95f2 100755 --- a/bin/codedeploy-agent +++ b/bin/codedeploy-agent @@ -2,7 +2,7 @@ $:.unshift File.join(File.dirname(File.expand_path('..', __FILE__)), 'lib') -ruby_versions = ["2.4", "2.3", "2.2", "2.1", "2.0"] +ruby_versions = ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0"] actual_ruby_version = RUBY_VERSION.split('.').map{|s|s.to_i} left_bound = '2.0.0'.split('.').map{|s|s.to_i} ruby_bin = nil diff --git a/bin/install b/bin/install index ec10fbde..80f4b440 100755 --- a/bin/install +++ b/bin/install @@ -86,7 +86,7 @@ EOF end def supported_ruby_versions - ['2.4', '2.3', '2.2', '2.1', '2.0'] + ['2.5', '2.4', '2.3', '2.2', '2.1', '2.0'] end # check ruby version, only version 2.x works @@ -163,7 +163,7 @@ EOF # change interpreter when symlink /usr/bin/ruby2.x exists, but running with non-supported ruby version actual_ruby_version = RUBY_VERSION.split('.').map{|s|s.to_i} left_bound = '2.0.0'.split('.').map{|s|s.to_i} - right_bound = '2.4.1'.split('.').map{|s|s.to_i} + right_bound = '2.5.1'.split('.').map{|s|s.to_i} if (actual_ruby_version <=> left_bound) < 0 if(!@reexeced) @log.info("The current Ruby version is not 2.x! Restarting the installer with #{ruby_interpreter_path}") From f9788bd2ffe458e82f553272425b2e1c5fbd3ca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20L=C3=BCer?= Date: Thu, 11 Oct 2018 17:36:34 +0000 Subject: [PATCH 2/5] Add use_fips_mode option For Fedramp compliance, we need to enable customers to use Fips vips. By setting this option in the configuration file, they can make the Agent use the ADCS Fips vips. cr https://code.amazon.com/reviews/CR-3463212 --- README.md | 12 +++++- lib/instance_agent/config.rb | 20 ++++++++- test/instance_agent/config_test.rb | 41 ++++++++++++++++++- .../codedeploy/codedeploy_control_test.rb | 35 +++++++++++++++- .../aws/plugins/deploy_control_endpoint.rb | 6 ++- 5 files changed, 108 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ecc94ce4..993944aa 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,17 @@ The integration test creates the following It terminates the test ec2 instance and deletes the CodeDeploy application at the end of each test run. It also terminates any test ec2 instances before starting up the test. -Create your default aws credentials file in the default location (~/.aws/credentials on linux/mac and %USERPROFILE%.awscredentials on windows). Add your AWS access key, secret key, and optionally your session token there. The access key should have permission to create the above mentioned resources. You can also change the default region. To run the integration test execute +Create your default aws credentials file in the default location (~/.aws/credentials on linux/mac and %USERPROFILE%.awscredentials on windows). Add your AWS access key, secret key, and optionally your session token there. The access key should have permission to create the above mentioned resources. You can also change the default region. Note that temporary credentials won't work. + +Sample format of the credentials file: + +``` +[default] +aws_access_key_id= +aws_secret_access_key= +``` + +To run the integration test execute: ``` rake test-integration diff --git a/lib/instance_agent/config.rb b/lib/instance_agent/config.rb index 59a0bc0a..e57bffa3 100644 --- a/lib/instance_agent/config.rb +++ b/lib/instance_agent/config.rb @@ -1,8 +1,12 @@ # encoding: UTF-8 require 'process_manager/config' +require 'set' module InstanceAgent class Config < ProcessManager::Config + + FIPS_ENABLED_REGIONS = Set['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'us-gov-west-1', 'us-gov-east-1'] + def self.init @config = Config.new ProcessManager::Config.instance_variable_set("@config", @config) @@ -11,6 +15,7 @@ def self.init def validate errors = super validate_children(errors) + validate_use_fips_mode(errors) errors end @@ -35,14 +40,25 @@ def initialize :kill_agent_max_wait_time_seconds => 7200, :on_premises_config_file => '/etc/codedeploy-agent/conf/codedeploy.onpremises.yml', :proxy_uri => nil, - :enable_deployments_log => true + :enable_deployments_log => true, + :use_fips_mode => false }) end def validate_children(errors = []) errors << 'children can only be set to 1' unless config[:children] == 1 - errors + end + + def validate_use_fips_mode errors + if config[:use_fips_mode] && ! (FIPS_ENABLED_REGIONS.include? region) + errors << 'use_fips_mode can be set to true only in regions located in the USA' + end end + #Return the region we are currently in + def region + ENV['AWS_REGION'] || InstanceMetadata.region + end + end end diff --git a/test/instance_agent/config_test.rb b/test/instance_agent/config_test.rb index 2b53bbaa..fba404f1 100644 --- a/test/instance_agent/config_test.rb +++ b/test/instance_agent/config_test.rb @@ -31,7 +31,8 @@ class InstanceAgentConfigTest < InstanceAgentTestCase :ongoing_deployment_tracking => 'ongoing-deployment', :proxy_uri => nil, :enable_deployments_log => true, - :kill_agent_max_wait_time_seconds => 7200 + :kill_agent_max_wait_time_seconds => 7200, + :use_fips_mode => false }, InstanceAgent::Config.config) end @@ -41,9 +42,11 @@ class InstanceAgentConfigTest < InstanceAgentTestCase end should 'execute all available validation methods' do + InstanceMetadata.stubs(:region).returns('us-west-1') #without stubbing this, the test will fail in the build fleet because MetadataService is not available there validations = sequence('validation') err = [] InstanceAgent::Config.any_instance.expects(:validate_children).with(err).in_sequence(validations) + InstanceAgent::Config.any_instance.expects(:validate_use_fips_mode).with(err).in_sequence(validations) InstanceAgent::Config.validate_config end @@ -53,6 +56,8 @@ class InstanceAgentConfigTest < InstanceAgentTestCase InstanceAgent::Config.config[:instance_service_region] = 'eu-west-1' InstanceAgent::Config.config[:instance_service_endpoint] = 'api-endpoint.example.com' InstanceAgent::Config.config[:instance_service_port] = 123 + + InstanceMetadata.stubs(:region).returns('us-west-1') #without stubbing this, the test will fail in the build fleet because MetadataService is not available there end should 'validate the children setting' do @@ -65,5 +70,39 @@ class InstanceAgentConfigTest < InstanceAgentTestCase assert InstanceAgent::Config.validate_config.empty?, InstanceAgent::Config.validate_config.inspect end end + + context 'validate use_fips_mode' do + + error = 'use_fips_mode can be set to true only in regions located in the USA' + + should 'error in eu-west-1' do + InstanceAgent::Config.config[:use_fips_mode] = true + ENV['AWS_REGION'] = 'eu-west-1' + assert InstanceAgent::Config.validate_config.include? error + end + + should 'not error in eu-west-1 if not set' do + InstanceAgent::Config.config[:use_fips_mode] = false + ENV['AWS_REGION'] = 'eu-west-1' + assert_false InstanceAgent::Config.validate_config.include? error + end + + should 'not error in us-east-1' do + InstanceAgent::Config.config[:use_fips_mode] = true + ENV['AWS_REGION'] = 'us-east-1' + assert_false InstanceAgent::Config.validate_config.include? error + end + + should 'not error in us-gov-west-1' do + InstanceAgent::Config.config[:use_fips_mode] = true + ENV['AWS_REGION'] = 'us-gov-west-1' + assert_false InstanceAgent::Config.validate_config.include? error + end + + cleanup do + ENV['AWS_REGION'] = nil + end + + end end end diff --git a/test/instance_agent/plugins/codedeploy/codedeploy_control_test.rb b/test/instance_agent/plugins/codedeploy/codedeploy_control_test.rb index 75a0b6bb..2a0efbfb 100644 --- a/test/instance_agent/plugins/codedeploy/codedeploy_control_test.rb +++ b/test/instance_agent/plugins/codedeploy/codedeploy_control_test.rb @@ -11,7 +11,6 @@ class CodeDeployControlTest < InstanceAgentTestCase ENV['AWS_ACCESS_KEY_ID'] = "Test Access Key" ENV['AWS_SECRET_ACCESS_KEY'] = "Test Secret Access Key" ENV['AWS_REGION'] = nil - ENV['AWSDEPLOY_CONTROL_ENDPOINT'] = "https://tempuri" ENV['DEPLOYMENT_CREATOR'] = "User" ENV['DEPLOYMENT_TYPE'] = "IN_PLACE" end @@ -51,6 +50,40 @@ class CodeDeployControlTest < InstanceAgentTestCase } end end + + context "with ADCS endpoint set in an environment variable" do + setup do + ENV['AWS_DEPLOY_CONTROL_ENDPOINT'] = "https://tempuri" + end + + should "use endpoint from environment variable" do + codedeploy_control_client = CodeDeployControl.new :region => "us-west-2" + assert_equal "tempuri", codedeploy_control_client.get_client.config.endpoint.host + end + + cleanup do + ENV['AWS_DEPLOY_CONTROL_ENDPOINT'] = nil + end + end + + context "with use_fips_mode not set" do + should "use non-Fips endpoint" do + codedeploy_control_client = CodeDeployControl.new :region => "us-west-2" + assert_equal "codedeploy-commands.us-west-2.amazonaws.com", codedeploy_control_client.get_client.config.endpoint.host + end + end + + context "with use_fips_mode set" do + setup do + InstanceAgent::Config.config[:use_fips_mode] = true + end + + should "use Fips endpoint" do + codedeploy_control_client = CodeDeployControl.new :region => "us-west-2" + assert_equal "codedeploy-commands-fips.us-west-2.amazonaws.com", codedeploy_control_client.get_client.config.endpoint.host + end + end + end end end diff --git a/vendor/gems/codedeploy-commands-1.0.0/lib/aws/plugins/deploy_control_endpoint.rb b/vendor/gems/codedeploy-commands-1.0.0/lib/aws/plugins/deploy_control_endpoint.rb index 6af525aa..d17f7751 100644 --- a/vendor/gems/codedeploy-commands-1.0.0/lib/aws/plugins/deploy_control_endpoint.rb +++ b/vendor/gems/codedeploy-commands-1.0.0/lib/aws/plugins/deploy_control_endpoint.rb @@ -6,7 +6,11 @@ class DeployControlEndpoint < Seahorse::Client::Plugin option(:endpoint) do |cfg| url = ENV['AWS_DEPLOY_CONTROL_ENDPOINT'] if url.nil? - url = "https://codedeploy-commands.#{cfg.region}.amazonaws.com" + url = "https://codedeploy-commands" + if InstanceAgent::Config.config[:use_fips_mode] + url.concat "-fips" + end + url.concat ".#{cfg.region}.amazonaws.com" if "cn" == cfg.region.split("-")[0] url.concat(".cn") end From 9a5a7ea513593249ff95f2466f56b1cdc6c473c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20L=C3=BCer?= Date: Thu, 18 Oct 2018 21:10:30 +0000 Subject: [PATCH 3/5] Use S3 Fips endpoints To support Fedramp requirements, we connect to S3 using its Fips endpoints if use_fips_mode is set to true --- .../plugins/codedeploy/command_executor.rb | 62 +++++++++++-------- .../codedeploy/command_executor_test.rb | 22 +++++++ 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/lib/instance_agent/plugins/codedeploy/command_executor.rb b/lib/instance_agent/plugins/codedeploy/command_executor.rb index 3d1a1def..4b9524c0 100644 --- a/lib/instance_agent/plugins/codedeploy/command_executor.rb +++ b/lib/instance_agent/plugins/codedeploy/command_executor.rb @@ -238,33 +238,8 @@ def most_recent_install_file_path(deployment_group) private def download_from_s3(deployment_spec, bucket, key, version, etag) log(:debug, "Downloading artifact bundle from bucket '#{bucket}' and key '#{key}', version '#{version}', etag '#{etag}'") - region = ENV['AWS_REGION'] || InstanceMetadata.region - - proxy_uri = nil - if InstanceAgent::Config.config[:proxy_uri] - proxy_uri = URI(InstanceAgent::Config.config[:proxy_uri]) - end - - if InstanceAgent::Config.config[:log_aws_wire] - s3 = Aws::S3::Client.new( - :region => region, - :ssl_ca_directory => ENV['AWS_SSL_CA_DIRECTORY'], - # wire logs might be huge; customers should be careful about turning them on - # allow 1GB of old wire logs in 64MB chunks - :logger => Logger.new( - File.join(InstanceAgent::Config.config[:log_dir], "#{InstanceAgent::Config.config[:program_name]}.aws_wire.log"), - 16, - 64 * 1024 * 1024), - :http_wire_trace => true, - :signature_version => 'v4', - :http_proxy => proxy_uri) - else - s3 = Aws::S3::Client.new( - :region => region, - :ssl_ca_directory => ENV['AWS_SSL_CA_DIRECTORY'], - :signature_version => 'v4', - :http_proxy => proxy_uri) - end + + s3 = Aws::S3::Client.new(s3_options) File.open(artifact_bundle(deployment_spec), 'wb') do |file| @@ -283,6 +258,39 @@ def download_from_s3(deployment_spec, bucket, key, version, etag) log(:debug, "Download complete from bucket #{bucket} and key #{key}") end + public + def s3_options + options = {} + options[:ssl_ca_directory] = ENV['AWS_SSL_CA_DIRECTORY'] + options[:signature_version] = 'v4' + + region = ENV['AWS_REGION'] || InstanceMetadata.region + options[:region] = region + if InstanceAgent::Config.config[:use_fips_mode] + #S3 Fips pseudo-regions are not supported by the SDK yet + #source for the URL: https://aws.amazon.com/compliance/fips/ + options[:endpoint] = "https://s3-fips.#{region}.amazonaws.com" + end + + proxy_uri = nil + if InstanceAgent::Config.config[:proxy_uri] + proxy_uri = URI(InstanceAgent::Config.config[:proxy_uri]) + end + options[:http_proxy] = proxy_uri + + if InstanceAgent::Config.config[:log_aws_wire] + # wire logs might be huge; customers should be careful about turning them on + # allow 1GB of old wire logs in 64MB chunks + options[:logger] = Logger.new( + File.join(InstanceAgent::Config.config[:log_dir], "#{InstanceAgent::Config.config[:program_name]}.aws_wire.log"), + 16, + 64 * 1024 * 1024) + options[:http_wire_trace] = true + end + + options + end + private def download_from_github(deployment_spec, account, repo, commit, anonymous, token) diff --git a/test/instance_agent/plugins/codedeploy/command_executor_test.rb b/test/instance_agent/plugins/codedeploy/command_executor_test.rb index fc10b105..3bd2b474 100644 --- a/test/instance_agent/plugins/codedeploy/command_executor_test.rb +++ b/test/instance_agent/plugins/codedeploy/command_executor_test.rb @@ -316,6 +316,28 @@ def generate_signed_message_for(map) end end + context "when creating S3 options" do + + should "use right region" do + assert_equal 'us-east-1', @command_executor.s3_options[:region] + end + + should "use right signature version" do + assert_equal 'v4', @command_executor.s3_options[:signature_version] + end + + should "use right endpoint when using Fips" do + InstanceAgent::Config.config[:use_fips_mode] = true + assert_equal 'https://s3-fips.us-east-1.amazonaws.com', @command_executor.s3_options[:endpoint] + InstanceAgent::Config.config[:use_fips_mode] = false + end + + should "use no endpoint when not using Fips" do + assert_false @command_executor.s3_options.include? :endpoint + end + + end + context "downloading bundle from S3" do setup do File.expects(:open).with(File.join(@deployment_root_dir, 'bundle.tar'), 'wb').yields(@mock_file) From b05b72805d6963efa7b67230d976f32b4e2c358a Mon Sep 17 00:00:00 2001 From: Dan Mendoza Date: Fri, 2 Nov 2018 14:33:54 -0700 Subject: [PATCH 4/5] Remove procfs dependency to make agent more compatible with other operating systems cr https://code.amazon.com/reviews/CR-3765381 --- .../gems/process_manager-0.0.13/lib/process_manager/master.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/gems/process_manager-0.0.13/lib/process_manager/master.rb b/vendor/gems/process_manager-0.0.13/lib/process_manager/master.rb index 76fbd02d..42366b58 100644 --- a/vendor/gems/process_manager-0.0.13/lib/process_manager/master.rb +++ b/vendor/gems/process_manager-0.0.13/lib/process_manager/master.rb @@ -167,7 +167,7 @@ def process_matcher(pid) if pid == own_pid return false end - File.read("/proc/#{pid}/cmdline").include?("codedeploy-agent: master") + `ps -p #{pid} -o command`.include?("codedeploy-agent: master") end def handle_pid_file From 3bbe54f954ca7d20b84cd198d7d63a189c8ec8e5 Mon Sep 17 00:00:00 2001 From: Zara Date: Wed, 21 Nov 2018 11:49:58 -0800 Subject: [PATCH 5/5] Revert "Added support in the agent for Ubuntu LTS release("bionic") ships with Ruby 2.5.x." This reverts commit 92ad4e3c1a8d92c29974373f8f17f708701fd178. cr https://code.amazon.com/reviews/CR-3966728 --- bin/codedeploy-agent | 2 +- bin/install | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/codedeploy-agent b/bin/codedeploy-agent index 00ef95f2..a683d0c3 100755 --- a/bin/codedeploy-agent +++ b/bin/codedeploy-agent @@ -2,7 +2,7 @@ $:.unshift File.join(File.dirname(File.expand_path('..', __FILE__)), 'lib') -ruby_versions = ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0"] +ruby_versions = ["2.4", "2.3", "2.2", "2.1", "2.0"] actual_ruby_version = RUBY_VERSION.split('.').map{|s|s.to_i} left_bound = '2.0.0'.split('.').map{|s|s.to_i} ruby_bin = nil diff --git a/bin/install b/bin/install index 80f4b440..ec10fbde 100755 --- a/bin/install +++ b/bin/install @@ -86,7 +86,7 @@ EOF end def supported_ruby_versions - ['2.5', '2.4', '2.3', '2.2', '2.1', '2.0'] + ['2.4', '2.3', '2.2', '2.1', '2.0'] end # check ruby version, only version 2.x works @@ -163,7 +163,7 @@ EOF # change interpreter when symlink /usr/bin/ruby2.x exists, but running with non-supported ruby version actual_ruby_version = RUBY_VERSION.split('.').map{|s|s.to_i} left_bound = '2.0.0'.split('.').map{|s|s.to_i} - right_bound = '2.5.1'.split('.').map{|s|s.to_i} + right_bound = '2.4.1'.split('.').map{|s|s.to_i} if (actual_ruby_version <=> left_bound) < 0 if(!@reexeced) @log.info("The current Ruby version is not 2.x! Restarting the installer with #{ruby_interpreter_path}")