taking content type and character set into account; instead of incorr…

…ectly using content-encoding

lib/httplog/adapters/ethon.rb

@@ -30,12 +30,13 @@ def perform
 
           # Not sure where the acutal status code is stored - so let's
           # extract it from the response header.
-          status = response_headers.scan(/HTTP\/... (\d{3})/).flatten.first
+          status   = response_headers.scan(/HTTP\/... (\d{3})/).flatten.first
           encoding = response_headers.scan(/Content-Encoding: (\S+)/).flatten.first
+          content_type = response_headers.scan(/Content-Type: (\S+(; charset=\S+)?)/).flatten.first
           HttpLog.log_compact(@action_name, @url, @return_code, bm)
           HttpLog.log_status(status)
           HttpLog.log_benchmark(bm)
-          HttpLog.log_body(response_body, encoding)
+          HttpLog.log_body(response_body, encoding, content_type)
           return_code
         end
       end

lib/httplog/adapters/excon.rb

@@ -55,9 +55,10 @@ def response(datum={})
           datum = orig_response(datum)
         end
         response = datum[:response]
-        encoding = response[:headers] && response[:headers]['Content-Encoding']
+        headers = response[:headers] || {}
+        content_type = 
         HttpLog.log_status(response[:status])
-        HttpLog.log_body(response[:body], encoding)
+        HttpLog.log_body(response[:body], headers['Content-Encoding'], headers['Content-Type'])
         datum
       end
     end

lib/httplog/adapters/http.rb

@@ -18,11 +18,12 @@ def make_request(req, options)
         end
 
         if log_enabled
+          headers = @response.headers
           HttpLog.log_compact(req.verb, req.uri, @response.code, bm)
           HttpLog.log_status(@response.code)
           HttpLog.log_benchmark(bm)
           HttpLog.log_headers(@response.headers.to_h)
-          HttpLog.log_body(@response.body, @response.headers["Content-Encoding"])
+          HttpLog.log_body(@response.body, headers['Content-Encoding'], headers['Content-Type'])
         end
 
         @response

lib/httplog/adapters/httpclient.rb

@@ -23,10 +23,11 @@ def do_get_block(req, proxy, conn, &block)
 
       if log_enabled
         res = conn.pop
+        headers = res.headers
         HttpLog.log_compact(req.header.request_method, req.header.request_uri, res.status_code, bm)
         HttpLog.log_status(res.status_code)
         HttpLog.log_benchmark(bm)
-        HttpLog.log_body(res.body, res.headers["Content-Encoding"])
+        HttpLog.log_body(res.body, headers['Content-Encoding'], headers['Content-Type'])
         conn.push(res)
       end
 

lib/httplog/adapters/net_http.rb

@@ -26,7 +26,7 @@ def request(req, body = nil, &block)
         HttpLog.log_status(@response.code)
         HttpLog.log_benchmark(bm)
         HttpLog.log_headers(@response.each_header.collect)
-        HttpLog.log_body(@response.body, @response["Content-Encoding"])
+        HttpLog.log_body(@response.body, @response['Content-Encoding'], @response['Content-Type'])
       end
 
       @response

lib/httplog/adapters/patron.rb

@@ -16,10 +16,11 @@ def request(action_name, url, headers, options = {})
         end
 
         if log_enabled
+          headers = @response.headers
           HttpLog.log_compact(action_name, url, @response.status, bm)
           HttpLog.log_status(@response.status)
           HttpLog.log_benchmark(bm)
-          HttpLog.log_body(@response.body, @response.headers['Content-Encoding'])
+          HttpLog.log_body(@response.body, headers['Content-Encoding'], headers['Content-Type'])
         end
       end
     end

lib/httplog/http_log.rb

@@ -40,16 +40,12 @@ def url_approved?(url)
       url.to_s.match(options[:url_whitelist_pattern])
     end
 
-    def log(msg, encoding='UTF-8')
+    def log(msg)
       # This builds a hash {0=>:DEBUG, 1=>:INFO, 2=>:WARN, 3=>:ERROR, 4=>:FATAL, 5=>:UNKNOWN}.
       # Courtesy of the delayed_job gem in this commit: 
       # https://github.com/collectiveidea/delayed_job/commit/e7f5aa1ed806e61251bdb77daf25864eeb3aff59
       severities = Hash[*Logger::Severity.constants.enum_for(:each_with_index).collect{ |s, i| [i, s] }.flatten]
       severity = severities[options[:severity]].to_s.downcase
-
-      msg.force_encoding(encoding) rescue msg.force_encoding('UTF-8')
-      msg.encode('UTF-8', :invalid => :replace, :undef => :replace)
-
       options[:logger].send(severity, colorize(LOG_PREFIX + msg))
     end
 
@@ -80,25 +76,35 @@ def log_benchmark(seconds)
       log("Benchmark: #{seconds} seconds")
     end
 
-    def log_body(body, encoding = nil)
+    def log_body(body, encoding = nil, content_type=nil)
       return if options[:compact_log] || !options[:log_response]
+
+      if content_type !~ /text/
+        log("Response: (not showing binary data)")
+        return
+      end
+
       if body.is_a?(Net::ReadAdapter)
         # open-uri wraps the response in a Net::ReadAdapter that defers reading
         # the content, so the reponse body is not available here.
         log("Response: (not available yet)")
-      else
-        if encoding =~ /gzip/
-          sio = StringIO.new( body.to_s )
-          gz = Zlib::GzipReader.new( sio )
-          log("Response:\n#{gz.read}")
-        else
-          log("Response:\n#{body}", encoding)
-        end
+        return
       end
+
+      if encoding =~ /gzip/
+        sio = StringIO.new( body.to_s )
+        gz = Zlib::GzipReader.new( sio )
+        body = gz.read
+      end
+
+      data = utf_encoded(body.to_s, content_type)
+
+      log("Response:\n#{data}")
     end
 
     def log_data(data)
-      return if options[:compact_log] || !options[:log_data]
+      return if options[:compact_log] || !options[:log_data] || data.nil?
+      data = utf_encoded(data)
       log("Data: #{data}")
     end
 
@@ -113,6 +119,10 @@ def colorize(msg)
       msg.send(:colorize, options[:color])
     end
 
-
+    def utf_encoded(data, content_type=nil)
+      charset = content_type.to_s.scan(/; charset=(\S+)/).flatten.first || 'UTF-8'
+      data.force_encoding(charset) rescue data.force_encoding('UTF-8')
+      data.encode('UTF-8', :invalid => :replace, :undef => :replace)
+    end
   end
 end

spec/adapters/http_base_adapter.rb

@@ -9,6 +9,10 @@ def initialize(host, port, path, headers, data, params, protocol = 'http')
     @protocol = protocol
   end
 
+  def logs_data?
+    true
+  end
+
   def parse_uri
     URI.parse("#{@protocol}://#{@host}:#{@port}#{@path}")
   end

spec/adapters/open_uri_adapter.rb

@@ -10,4 +10,8 @@ def expected_response_body
   def self.should_log_headers?
     false
   end
+
+  def logs_data?
+    false
+  end
 end

spec/http_log_spec.rb

@@ -57,6 +57,29 @@
               expect(log).to include(HttpLog::LOG_PREFIX + "Response:#{adapter.expected_response_body}")
             end
           end
+
+          context "with UTF-8 response body" do
+            let(:path) { '/utf8.html' }
+            let(:data) { nil }
+
+            it "works" do
+              adapter.send_get_request
+              expect(log).to include(HttpLog::LOG_PREFIX + "Response:#{adapter.expected_response_body}")
+              if adapter.logs_data?
+                expect(log).to include("    <title>Блог Яндекса</title>")
+              end
+            end
+          end
+
+          context "with binary response body" do
+            let(:path) { '/test.bin' }
+            let(:data) { nil }
+
+            it "doesn't log response" do
+              adapter.send_get_request
+              expect(log).to include(HttpLog::LOG_PREFIX + "Response: (not showing binary data)")
+            end
+          end
         end
 
         if adapter_class.method_defined? :send_post_request

spec/support/test_server.rb

@@ -19,8 +19,9 @@ def call(env)
 
         elsif File.exists?(file)
           headers["Content-Type"] = "application/octet-stream" if File.extname(file) == '.bin'
+          headers["Content-Type"] = "text/html; charset=UTF-8" if path =~ /utf8/
           headers["Content-Encoding"] = "gzip" if File.extname(file) == '.gz'
-          [ 200, headers, File.read(file) ]
+          [ 200, headers, File.binread(file) ]
         else
           [ 404, {'Content-Type' => 'text/plain'}, 'file not found' ]
         end

spec/support/utf8.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Блог Яндекса</title>
+  </head>
+  <body>
+    <h1>This is the test page.</h1>
+  </body>
+</html>