Merge branch 'CHEF-1031'

lib/chef/provider/remote_file.rb

@@ -1,4 +1,5 @@
 #
+# Author:: Jesse Campbell (<hikeit@gmail.com>)
 # Author:: Adam Jacob (<adam@opscode.com>)
 # Copyright:: Copyright (c) 2008 Opscode, Inc.
 # License:: Apache License, Version 2.0
@@ -17,10 +18,9 @@
 #
 
 require 'chef/provider/file'
-require 'chef/rest'
+require 'rest_client'
 require 'uri'
 require 'tempfile'
-require 'net/https'
 
 class Chef
   class Provider
@@ -40,24 +40,12 @@ def action_create
           Chef::Log.debug("#{@new_resource} checksum matches target checksum (#{@new_resource.checksum}) - not updating")
         else
           sources = @new_resource.source
-          source = sources.shift
-          begin
-            rest = Chef::REST.new(source, nil, nil, http_client_opts(source))
-            raw_file = rest.streaming_request(rest.create_url(source), {})
-          rescue SocketError, Errno::ECONNREFUSED, Timeout::Error, Net::HTTPFatalError => e
-            Chef::Log.debug("#{@new_resource} cannot be downloaded from #{source}")
-            if source = sources.shift
-              Chef::Log.debug("#{@new_resource} trying to download from another mirror")
-              retry
-            else
-              raise e
-            end
-          end
+          raw_file, raw_file_source = try_multiple_sources(sources)
           if matches_current_checksum?(raw_file)
             Chef::Log.debug "#{@new_resource} target and source checksums are the same - not updating"
           else
             description = [] 
-            description << "copy file downloaded from #{@new_resource.source} into #{@new_resource.path}"
+            description << "copy file downloaded from #{raw_file_source} into #{@new_resource.path}"
             description << diff_current(raw_file.path)
             converge_by(description) do
               backup_new_resource
@@ -102,38 +90,55 @@ def backup_new_resource
         end
       end
 
-      def source_file(source, current_checksum, &block)
-        if absolute_uri?(source)
-          fetch_from_uri(source, &block)
-        elsif !Chef::Config[:solo]
-          fetch_from_chef_server(source, current_checksum, &block)
-        else
-          fetch_from_local_cookbook(source, &block)
-        end
-      end
+      private
 
-      def http_client_opts(source)
-        opts={}
-        # CHEF-3140
-        # 1. If it's already compressed, trying to compress it more will
-        # probably be counter-productive.
-        # 2. Some servers are misconfigured so that you GET $URL/file.tgz but
-        # they respond with content type of tar and content encoding of gzip,
-        # which tricks Chef::REST into decompressing the response body. In this
-        # case you'd end up with a tar archive (no gzip) named, e.g., foo.tgz,
-        # which is not what you wanted.
-        if @new_resource.path =~ /gz$/ or source =~ /gz$/
-          opts[:disable_gzip] = true
+      # Given an array of source uris, iterate through them until one does not fail
+      def try_multiple_sources(sources)
+        sources = sources.dup
+        source = sources.shift
+        begin
+          uri = URI.parse(source)
+          raw_file = grab_file_from_uri(uri)
+        rescue ArgumentError => e
+          raise e
+        rescue => e
+          if e.is_a?(RestClient::Exception)
+            error = "Request returned #{e.message}"
+          else
+            error = e.to_s
+          end
+          Chef::Log.debug("#{@new_resource} cannot be downloaded from #{source}: #{error}")
+          if source = sources.shift
+            Chef::Log.debug("#{@new_resource} trying to download from another mirror")
+            retry
+          else
+            raise e
+          end
         end
-        opts
+        if uri.userinfo
+          uri.password = "********"
+        end
+        return raw_file, uri.to_s
       end
 
-      private
-
-      def absolute_uri?(source)
-        URI.parse(source).absolute?
-      rescue URI::InvalidURIError
-        false
+      # Given a source uri, return a Tempfile, or a File that acts like a Tempfile (close! method)
+      def grab_file_from_uri(uri)
+        if URI::HTTP === uri
+          #HTTP or HTTPS
+          raw_file = RestClient::Request.execute(:method => :get, :url => uri.to_s, :raw_response => true).file
+        elsif URI::FTP === uri
+          #FTP
+          raw_file = FTP::fetch(uri, @new_resource.ftp_active_mode)
+        elsif uri.scheme == "file"
+          #local/network file
+          raw_file = ::File.new(uri.path, "r")
+          def raw_file.close!
+            self.close
+          end
+        else
+          raise ArgumentError, "Invalid uri. Only http(s), ftp, and file are currently supported"
+        end
+        raw_file
       end
 
     end

lib/chef/provider/remote_file/ftp.rb

@@ -0,0 +1,95 @@
+#
+# Author:: Jesse Campbell (<hikeit@gmail.com>)
+# Copyright:: Copyright (c) 2013 Jesse Campbell
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'uri'
+require 'tempfile'
+require 'net/ftp'
+require 'chef/provider/remote_file'
+
+class Chef
+  class Provider
+    class RemoteFile
+      class FTP
+
+        # Fetches the file at uri using Net::FTP, returning a Tempfile
+        def self.fetch(uri, ftp_active_mode)
+          self.new(uri, ftp_active_mode).fetch()
+        end
+
+        # Parse the uri into instance variables
+        def initialize(uri, ftp_active_mode)
+          @directories, @filename = parse_path(uri.path)
+          @typecode = uri.typecode
+          # Only support ascii and binary types
+          if @typecode && /\A[ai]\z/ !~ @typecode
+            raise ArgumentError, "invalid typecode: #{@typecode.inspect}"
+          end
+          @ftp_active_mode = ftp_active_mode
+          @hostname = uri.hostname
+          @port = uri.port
+          if uri.userinfo
+            @user = URI.unescape(uri.user)
+            @pass = URI.unescape(uri.password)
+          else
+            @user = 'anonymous'
+            @pass = nil
+          end
+        end
+
+        # Fetches using Net::FTP, returns a Tempfile with the content
+        def fetch()
+          tempfile = Tempfile.new(@filename)
+
+          # The access sequence is defined by RFC 1738
+          ftp = Net::FTP.new
+          ftp.connect(@hostname, @port)
+          ftp.passive = !@ftp_active_mode
+          ftp.login(@user, @pass)
+          @directories.each do |cwd|
+            ftp.voidcmd("CWD #{cwd}")
+          end
+          if @typecode
+            ftp.voidcmd("TYPE #{@typecode.upcase}")
+          end
+          ftp.getbinaryfile(@filename, tempfile.path)
+          ftp.close
+
+          tempfile
+        end
+
+				private
+
+        def parse_path(path)
+          path = path.sub(%r{\A/}, '%2F') # re-encode the beginning slash because uri library decodes it.
+          directories = path.split(%r{/}, -1)
+          directories.each {|d|
+            d.gsub!(/%([0-9A-Fa-f][0-9A-Fa-f])/) { [$1].pack("H2") }
+          }
+          unless filename = directories.pop
+            raise ArgumentError, "no filename: #{path.inspect}"
+          end
+          if filename.length == 0 || filename.end_with?( "/" )
+            raise ArgumentError, "no filename: #{path.inspect}"
+          end
+          return directories, filename
+        end
+
+      end
+    end
+  end
+end

lib/chef/providers.rb

@@ -101,5 +101,7 @@
 require 'chef/provider/deploy/revision'
 require 'chef/provider/deploy/timestamped'
 
+require 'chef/provider/remote_file/ftp'
+
 require "chef/provider/lwrp_base"
 require 'chef/provider/registry_key'

lib/chef/resource/remote_file.rb

@@ -32,7 +32,8 @@ def initialize(name, run_context=nil)
         super
         @resource_name = :remote_file
         @action = "create"
-        @source = nil
+        @source = []
+        @ftp_active_mode = false
         @provider = Chef::Provider::RemoteFile
       end
 
@@ -54,6 +55,14 @@ def checksum(args=nil)
         )
       end
 
+      def ftp_active_mode(args=nil)
+        set_or_return(
+          :ftp_active_mode,
+          args,
+          :kind_of => [ TrueClass, FalseClass ]
+        )
+      end
+
       def after_created
         validate_source(@source)
       end

spec/functional/resource/remote_file_spec.rb

@@ -23,14 +23,6 @@
   include_context Chef::Resource::File
 
   let(:file_base) { "remote_file_spec" }
-  let(:source) { 'http://localhost:9000/nyan_cat.png' }
-  let(:expected_content) do
-    content = File.open(File.join(CHEF_SPEC_DATA, 'remote_file', 'nyan_cat.png'), "rb") do |f|
-      f.read
-    end
-    content.force_encoding(Encoding::BINARY) if content.respond_to?(:force_encoding)
-    content
-  end
 
   def create_resource
     node = Chef::Node.new
@@ -71,13 +63,44 @@ def create_resource
         f.read
       end
     }
+    @api.get("/nyan_cat.png.gz", 200, nil, { 'Content-Type' => 'application/gzip', 'Content-Encoding' => 'gzip' } ) {
+      File.open(File.join(CHEF_SPEC_DATA, 'remote_file', 'nyan_cat.png.gz'), "rb") do |f|
+        f.read
+      end
+    }
   end
 
   after(:all) do
     @server.stop
   end
 
-  it_behaves_like "a file resource"
+  context "when using normal encoding" do
+    let(:source) { 'http://localhost:9000/nyan_cat.png' }
+    let(:expected_content) do
+      content = File.open(File.join(CHEF_SPEC_DATA, 'remote_file', 'nyan_cat.png'), "rb") do |f|
+        f.read
+      end
+      content.force_encoding(Encoding::BINARY) if content.respond_to?(:force_encoding)
+      content
+    end
+
+    it_behaves_like "a file resource"
 
-  it_behaves_like "a securable resource with reporting"
+    it_behaves_like "a securable resource with reporting"
+  end
+
+  context "when using gzip encoding" do
+    let(:source) { 'http://localhost:9000/nyan_cat.png.gz' }
+    let(:expected_content) do
+      content = File.open(File.join(CHEF_SPEC_DATA, 'remote_file', 'nyan_cat.png.gz'), "rb") do |f|
+        f.read
+      end
+      content.force_encoding(Encoding::BINARY) if content.respond_to?(:force_encoding)
+      content
+    end
+
+    it_behaves_like "a file resource"
+
+    it_behaves_like "a securable resource with reporting"
+  end
 end

spec/tiny_server.rb

@@ -127,20 +127,20 @@ def clear
       @routes = {GET => [], PUT => [], POST => [], DELETE => []}
     end
 
-    def get(path, response_code, data=nil, &block)
-      @routes[GET] << Route.new(path, Response.new(response_code,data, &block))
+    def get(path, response_code, data=nil, headers=nil, &block)
+      @routes[GET] << Route.new(path, Response.new(response_code, data, headers, &block))
     end
 
-    def put(path, response_code, data=nil, &block)
-      @routes[PUT] << Route.new(path, Response.new(response_code,data, &block))
+    def put(path, response_code, data=nil, headers=nil, &block)
+      @routes[PUT] << Route.new(path, Response.new(response_code, data, headers, &block))
     end
 
-    def post(path, response_code, data=nil, &block)
-      @routes[POST] << Route.new(path, Response.new(response_code,data, &block))
+    def post(path, response_code, data=nil, headers=nil, &block)
+      @routes[POST] << Route.new(path, Response.new(response_code, data, headers, &block))
     end
 
-    def delete(path, response_code, data=nil, &block)
-      @routes[DELETE] << Route.new(path, Response.new(response_code,data, &block))
+    def delete(path, response_code, data=nil, headers=nil, &block)
+      @routes[DELETE] << Route.new(path, Response.new(response_code, data, headers, &block))
     end
 
     def call(env)
@@ -183,14 +183,15 @@ def to_s
   class Response
     HEADERS = {'Content-Type' => 'application/json'}
 
-    def initialize(response_code=200,data=nil, &block)
+    def initialize(response_code=200, data=nil, headers=nil, &block)
       @response_code, @data = response_code, data
+      @response_headers = headers ? HEADERS.merge(headers) : HEADERS
       @block = block_given? ? block : nil
     end
 
     def call
       data = @data || @block.call
-      [@response_code, HEADERS, Array(data)]
+      [@response_code, @response_headers, Array(data)]
     end
 
     def to_s