knife-ec2 master to knife-ec2 knife-cloud sync

lib/chef/knife/cloud/chefbootstrap/bootstrap_options.rb

@@ -54,13 +54,19 @@ def self.included(includer)
               :short => "-p PORT",
               :long => "--ssh-port PORT",
               :description => "The ssh port",
-              :proc => Proc.new { |key| Chef::Config[:knife][:ssh_port] = key }
+              :proc => Proc.new { |key| Chef::Config[:knife][:ssh_port] = key },
+              :default => "22"
 
             option :ssh_gateway,
               :long => "--ssh-gateway GATEWAY",
-              :description => "The ssh gateway",
+              :description => "The ssh gateway server. Any proxies configured in your ssh config are automatically used by default.",
               :proc => Proc.new { |key| Chef::Config[:knife][:ssh_gateway] = key }
 
+            option :ssh_gateway_identity,
+              :long => "--ssh-gateway-identity IDENTITY_FILE",
+              :description => "The private key for ssh gateway server",
+              :proc => Proc.new { |key| Chef::Config[:knife][:ssh_gateway_identity] = key }
+
             option :forward_agent,
               :long => "--forward-agent",
               :description => "Enable SSH agent forwarding",

lib/chef/knife/cloud/chefbootstrap/ssh_bootstrap_protocol.rb

@@ -21,11 +21,12 @@
 require 'chef/knife/core/windows_bootstrap_context'
 require 'chef/knife/bootstrap'
 
+
 class Chef
   class Knife
     class Cloud
       class SshBootstrapProtocol < BootstrapProtocol
-        
+
         def initialize(config)
           @bootstrap = (config[:image_os_type] == 'linux') ? Chef::Knife::Bootstrap.new : Chef::Knife::BootstrapWindowsSsh.new
           super
@@ -34,10 +35,11 @@ def initialize(config)
         def init_bootstrap_options
           bootstrap.config[:ssh_user] = @config[:ssh_user]
           bootstrap.config[:ssh_password] = @config[:ssh_password]
-          bootstrap.config[:ssh_port] = @config[:ssh_port]
+          bootstrap.config[:ssh_port] = locate_config_value(:ssh_port)
           bootstrap.config[:identity_file] = @config[:identity_file]
           bootstrap.config[:host_key_verify] = @config[:host_key_verify]
           bootstrap.config[:use_sudo] = true unless @config[:ssh_user] == 'root'
+          bootstrap.config[:template_file] =  @config[:template_file]
           bootstrap.config[:ssh_gateway] = locate_config_value(:ssh_gateway)
           bootstrap.config[:forward_agent] = locate_config_value(:forward_agent)
           bootstrap.config[:use_sudo_password] = locate_config_value(:use_sudo_password)
@@ -46,39 +48,131 @@ def init_bootstrap_options
 
         def wait_for_server_ready
           print "\n#{ui.color("Waiting for sshd to host (#{@config[:bootstrap_ip_address]})", :magenta)}"
-          print(".") until tcp_test_ssh(@config[:bootstrap_ip_address]) {
-            sleep @initial_sleep_delay ||= 10
-            puts("done")
-          }
+
+          ssh_gateway = get_ssh_gateway_for(@config[:bootstrap_ip_address])
+
+          # The ssh_gateway & subnet_id are currently supported only in EC2.
+          if ssh_gateway
+            print(".") until tunnel_test_ssh(ssh_gateway, @config[:bootstrap_ip_address]) {
+              @initial_sleep_delay = !!locate_config_value(:subnet_id) ? 40 : 10
+              sleep @initial_sleep_delay
+              puts("done")
+            }
+          else
+            print(".") until tcp_test_ssh(@config[:bootstrap_ip_address], locate_config_value(:ssh_port)) {
+              @initial_sleep_delay = !!locate_config_value(:subnet_id) ? 40 : 10
+              sleep @initial_sleep_delay
+              puts("done")
+            }
+          end
         end
 
-        def tcp_test_ssh(hostname)
-          tcp_socket = TCPSocket.new(hostname, 22)
-          readable = IO.select([tcp_socket], nil, nil, 5)
-          if readable
-            Chef::Log.debug("sshd accepting connections on #{hostname}, banner is #{tcp_socket.gets}")
-            yield
-            true
+        def get_ssh_gateway_for(hostname)
+          if locate_config_value(:ssh_gateway)
+            # The ssh_gateway specified in the knife config (if any) takes
+            # precedence over anything in the SSH configuration
+            Chef::Log.debug("Using ssh gateway #{locate_config_value(:ssh_gateway)} from knife config")
+            locate_config_value(:ssh_gateway)
           else
+            # Next, check if the SSH configuration has a ProxyCommand
+            # directive for this host. If there is one, parse out the
+            # host from the proxy command
+            ssh_proxy = Net::SSH::Config.for(hostname)[:proxy]
+            if ssh_proxy.respond_to?(:command_line_template)
+              # ssh gateway_hostname nc %h %p
+              proxy_pattern = /ssh\s+(\S+)\s+nc/
+              matchdata = proxy_pattern.match(ssh_proxy.command_line_template)
+              if matchdata.nil?
+                Chef::Log.debug("Unable to determine ssh gateway for '#{hostname}' from ssh config template: #{ssh_proxy.command_line_template}")
+                nil
+              else
+                # Return hostname extracted from command line template
+                Chef::Log.debug("Using ssh gateway #{matchdata[1]} from ssh config")
+                matchdata[1]
+              end
+            else
+              # Return nil if we cannot find an ssh_gateway
+              Chef::Log.debug("No ssh gateway found, making a direct connection")
+              nil
+            end
+          end
+        end
+
+        def tcp_test_ssh(hostname, ssh_port)
+          begin
+            tcp_socket = TCPSocket.new(hostname, ssh_port)
+            readable = IO.select([tcp_socket], nil, nil, 5)
+            if readable
+              ssh_banner = tcp_socket.gets
+              if ssh_banner.nil? or ssh_banner.empty?
+                false
+              else
+                Chef::Log.debug("ssh accepting connections on #{hostname}, banner is #{tcp_socket.gets}")
+                yield
+                true
+              end
+            else
+              false
+            end
+          rescue Errno::EPERM, Errno::ETIMEDOUT
+            Chef::Log.debug("ssh timed out: #{hostname}")
+            false
+          rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, IOError
+            Chef::Log.debug("ssh failed to connect: #{hostname}")
+            sleep 2
+            false
+           # This happens on some mobile phone networks
+          rescue Errno::ECONNRESET
+             Chef::Log.debug("ssh reset its connection: #{hostname}")
+            sleep 2
+            false
+          ensure
+            tcp_socket && tcp_socket.close
+          end
+        end
+
+        def tunnel_test_ssh(ssh_gateway, hostname, &block)
+          begin
+            status = false
+            gateway = configure_ssh_gateway(ssh_gateway)
+            gateway.open(hostname, locate_config_value(:ssh_port)) do |local_tunnel_port|
+              status = tcp_test_ssh('localhost', local_tunnel_port, &block)
+            end
+            status
+          rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, IOError
+            sleep 2
+            false
+          rescue Errno::EPERM, Errno::ETIMEDOUT
             false
           end
-        rescue Errno::ETIMEDOUT
-          false
-        rescue Errno::EPERM
-          false
-        rescue Errno::ECONNREFUSED
-          sleep 2
-          false
-        rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH
-          sleep 2
-          false
-        rescue Errno::ENETUNREACH
-          sleep 2
-          false
-        ensure
-          tcp_socket && tcp_socket.close
         end
 
+        def configure_ssh_gateway(ssh_gateway)
+          gw_host, gw_user = ssh_gateway.split('@').reverse
+          gw_host, gw_port = gw_host.split(':')
+          gateway_options = { :port => gw_port || 22 }
+
+          # Load the SSH config for the SSH gateway host.
+          # Set the gateway user if it was not part of the
+          # SSH gateway string, and use any configured
+          # SSH keys.
+          ssh_gateway_config = Net::SSH::Config.for(gw_host)
+          gw_user ||= ssh_gateway_config[:user]
+
+          # Always use the gateway keys from the SSH Config
+          gateway_keys = ssh_gateway_config[:keys]        
+
+          # Use the keys specificed on the command line if available (overrides SSH Config)
+          if locate_config_value(:ssh_gateway_identity)
+            gateway_keys = Array(locate_config_value(:ssh_gateway_identity))
+          end
+
+          unless gateway_keys.nil?
+            gateway_options[:keys] = gateway_keys
+          end
+
+          Net::SSH::Gateway.new(gw_host, gw_user, gateway_options)
+        end
       end
     end
   end

lib/chef/knife/cloud/command.rb

@@ -32,7 +32,7 @@ def run
           begin
             # Set dafult config
             set_default_config
-            
+
             # validate compulsory params
             validate!
 

lib/chef/knife/cloud/fog/service.rb

@@ -93,7 +93,6 @@ def create_server(options = {})
         def delete_server(server_name)
           begin
             server = get_server(server_name)
-
             msg_pair("Instance Name", get_server_name(server))
             msg_pair("Instance ID", server.id)
 

lib/chef/knife/cloud/server/create_command.rb

@@ -51,7 +51,7 @@ def validate_params!
           error_message = ""
           raise CloudExceptions::ValidationError, error_message if errors.each{|e| ui.error(e); error_message = "#{error_message} #{e}."}.any?
         end
-        
+
         def before_exec_command
           begin
             post_connection_validations
@@ -86,7 +86,7 @@ def after_exec_command
             raise e
           rescue => e
             error_message = "Check if --bootstrap-protocol and --image-os-type is correct. #{e.message}"
-            ui.fatal(error_message) 
+            ui.fatal(error_message)
             cleanup_on_failure
             raise e, error_message
           end
@@ -143,7 +143,7 @@ def ssh_override_winrm
             config[:ssh_user] = locate_config_value(:winrm_user)
           end
           # unchanged ssh_port and changed winrm_port, override ssh_port
-          if locate_config_value(:ssh_port).nil? &&
+          if locate_config_value(:ssh_port).eql?(options[:ssh_port][:default]) &&
               !locate_config_value(:winrm_port).eql?(options[:winrm_port][:default])
             config[:ssh_port] = locate_config_value(:winrm_port)
           end

spec/unit/server_create_command_spec.rb

@@ -168,9 +168,9 @@ class ServerCreate < Chef::Knife::Cloud::ServerCreateCommand
 
       it "set ssh_port value by using -p option for ssh bootstrap protocol or linux image" do
         # Currently -p option set config[:winrm_port]
-        # default value of config[:ssh_port] is nil
+        # default value of config[:ssh_port] is 22
         @instance.config[:winrm_port] = "1234"
-        @instance.config[:ssh_port] =  nil
+        @instance.config[:ssh_port] =  "22"
 
         @instance.before_bootstrap
         expect(@instance.config[:ssh_port]).to eq("1234")

spec/unit/ssh_bootstrap_protocol_spec.rb

@@ -76,38 +76,41 @@
   end
 
   describe "#tcp_test_ssh" do
+
     it "return true" do
       tcpSocket = double()
       allow(tcpSocket).to receive(:close).and_return(true)
       allow(tcpSocket).to receive(:gets).and_return(true)
       allow(TCPSocket).to receive(:new).and_return(tcpSocket)
       allow(IO).to receive(:select).and_return(true)
-      expect(@instance.tcp_test_ssh("localhost"){}).to be(true)
+      allow(tcpSocket.gets).to receive(:nil?).and_return(false)
+      allow(tcpSocket.gets).to receive(:empty?).and_return(false)
+      expect(@instance.tcp_test_ssh("localhost","22"){}).to be(true)
     end
 
     it "raise ETIMEDOUT error" do
       allow(TCPSocket).to receive(:new).and_raise(Errno::ETIMEDOUT)
-      expect(@instance.tcp_test_ssh("localhost"){}).to be(false)
+      expect(@instance.tcp_test_ssh("localhost","22"){}).to be(false)
     end
 
     it "raise EPERM error" do
       allow(TCPSocket).to receive(:new).and_raise(Errno::EPERM)
-      expect(@instance.tcp_test_ssh("localhost"){}).to be(false)
+      expect(@instance.tcp_test_ssh("localhost","22"){}).to be(false)
     end
 
     it "raise ECONNREFUSED error" do
       allow(TCPSocket).to receive(:new).and_raise(Errno::ECONNREFUSED)
-      expect(@instance.tcp_test_ssh("localhost"){}).to be(false)
+      expect(@instance.tcp_test_ssh("localhost","22"){}).to be(false)
     end
 
     it "raise EHOSTUNREACH error" do
       allow(TCPSocket).to receive(:new).and_raise(Errno::EHOSTUNREACH)
-      expect(@instance.tcp_test_ssh("localhost"){}).to be(false)
+      expect(@instance.tcp_test_ssh("localhost","22"){}).to be(false)
     end
 
     it "raise ENETUNREACH error" do
       allow(TCPSocket).to receive(:new).and_raise(Errno::ENETUNREACH)
-      expect(@instance.tcp_test_ssh("localhost"){}).to be(false)
+      expect(@instance.tcp_test_ssh("localhost","22"){}).to be(false)
     end
   end
 end