Refactor Processors, fix Yajl compatibility

+ Refactor our SanitizeData processor into three separate processors
+ Change weird Processor namespacing
+ Change weird requires in specs
+ Fix YAJL compatibility by preferring vendored OkJson

lib/raven/base.rb

@@ -10,7 +10,10 @@
 require 'raven/interfaces/exception'
 require 'raven/interfaces/stack_trace'
 require 'raven/interfaces/http'
-require 'raven/processors/sanitizedata'
+require 'raven/processor'
+require 'raven/processor/sanitizedata'
+require 'raven/processor/removecircularreferences'
+require 'raven/processor/utf8conversion'
 
 module Raven
   class << self

lib/raven/configuration.rb

@@ -94,7 +94,7 @@ def initialize
       self.current_environment = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'default'
       self.send_modules = true
       self.excluded_exceptions = IGNORE_DEFAULT
-      self.processors = [Raven::Processor::SanitizeData]
+      self.processors = [Raven::Processor::RemoveCircularReferences, Raven::Processor::UTF8Conversion, Raven::Processor::SanitizeData]
       self.ssl_verification = false
       self.encoding = 'gzip'
       self.timeout = 1

lib/raven/processor.rb

@@ -1,13 +1,13 @@
+require 'json'
+
 module Raven
-  module Processor
-    class Processor
-      def initialize(client)
-        @client = client
-      end
+  class Processor
+    def initialize(client)
+      @client = client
+    end
 
-      def process(data)
-        data
-      end
+    def process(data)
+      data
     end
   end
 end

lib/raven/processor/removecircularreferences.rb

@@ -0,0 +1,26 @@
+module Raven
+  class Processor::RemoveCircularReferences < Processor
+
+    def process(v, visited = [])
+      return "(...)" if visited.include?(v.__id__)
+      visited += [v.__id__]
+      if v.is_a?(Hash)
+        v.reduce({}) { |memo, (k, v_)| memo[k] = process(v_, visited); memo }
+      elsif v.is_a?(Array)
+        v.map { |v_| process(v_, visited) }
+      elsif v.is_a?(String) && json_hash = parse_json_or_nil(v)
+        v.reduce({}) { |memo, (k, v_)| memo[k] = process(v_, visited); memo }.to_json
+      end
+    end
+
+    private
+
+    def parse_json_or_nil(string)
+      begin
+        result = OkJson.decode(string)
+      rescue Raven::OkJson::Error
+        nil
+      end
+    end
+  end
+end

lib/raven/processor/sanitizedata.rb

@@ -0,0 +1,41 @@
+module Raven
+  class Processor::SanitizeData < Processor
+    STRING_MASK = '********'
+    INT_MASK = 0
+    FIELDS_RE = /(authorization|password|passwd|secret|ssn|social(.*)?sec)/i
+    VALUES_RE = /^\d{16}$/
+
+    def process(data)
+      data.merge( data ) {|k, v| sanitize(k, v)}
+    end
+
+    private
+
+    def sanitize(k, v)
+      if v.is_a?(Hash)
+        process(v)
+      elsif v.is_a?(String) && (json_hash = parse_json_or_nil(v))
+        #if this string is actually a json obj, convert and sanitize
+        process(json_hash).to_json
+      elsif v.is_a?(Integer) && (VALUES_RE.match(v.to_s) || FIELDS_RE.match(k))
+        INT_MASK
+      elsif VALUES_RE.match(v.to_s) || FIELDS_RE.match(k)
+        STRING_MASK
+      else
+        v
+      end
+    end
+
+    private
+
+    def parse_json_or_nil(string)
+      begin
+        result = OkJson.decode(string)
+        result.is_a?(String) ? nil : result
+      rescue Raven::OkJson::Error
+        nil
+      end
+    end
+  end
+end
+

lib/raven/processor/utf8conversion.rb

@@ -0,0 +1,32 @@
+module Raven
+  class Processor::UTF8Conversion < Processor
+
+    def process(data)
+      recursively_apply(data) { |obj| clean_invalid_utf8_bytes(obj) }
+    end
+
+    private
+
+    def clean_invalid_utf8_bytes(obj)
+      if obj.respond_to?(:to_utf8)
+        obj.to_utf8
+      elsif obj.respond_to?(:encoding)
+        obj.encode('UTF-16', :invalid => :replace, :undef => :replace, :replace => '').encode('UTF-8')
+      else
+        obj
+      end
+    end
+
+    def recursively_apply(obj, &block)
+      if block_given?
+        if obj.is_a? Array
+          obj.map {|item| recursively_apply item, &block}
+        elsif obj.is_a? Hash
+          obj.merge(obj) {|k, val| recursively_apply val, &block }
+        else
+          block.call obj
+        end
+      end
+    end
+  end
+end

spec/raven/better_attr_accessor_spec.rb

@@ -1,5 +1,4 @@
-require File.expand_path('../../spec_helper', __FILE__)
-require 'raven/better_attr_accessor'
+require 'spec_helper'
 
 describe Raven::BetterAttrAccessor do
 

spec/raven/configuration_spec.rb

@@ -1,5 +1,4 @@
-require File.expand_path('../../spec_helper', __FILE__)
-require 'raven'
+require 'spec_helper'
 
 describe Raven::Configuration do
   before do

spec/raven/event_spec.rb

@@ -1,5 +1,4 @@
-require File.expand_path('../../spec_helper', __FILE__)
-require 'raven'
+require 'spec_helper'
 
 describe Raven::Event do
   before do

spec/raven/integration_spec.rb

@@ -1,7 +1,4 @@
 require 'spec_helper'
-require 'raven'
-require 'raven/error'
-require 'logger'
 
 describe "Integration tests" do
 

spec/raven/logger_spec.rb

@@ -1,5 +1,4 @@
-require File.expand_path('../../spec_helper', __FILE__)
-require 'raven'
+require 'spec_helper'
 
 describe Raven::Logger do
   context 'without a backend logger' do

spec/raven/okjson_spec.rb

@@ -1,5 +1,4 @@
-require File.expand_path('../../spec_helper', __FILE__)
-require 'raven'
+require 'spec_helper'
 
 describe Raven::OkJson do
 

spec/raven/rack_spec.rb

@@ -1,5 +1,4 @@
-require File.expand_path('../../spec_helper', __FILE__)
-require 'raven'
+require 'spec_helper'
 
 describe Raven::Rack do
   it 'should capture exceptions' do

spec/raven/raven_spec.rb

@@ -1,5 +1,4 @@
-require File.expand_path('../../spec_helper', __FILE__)
-require 'raven'
+require 'spec_helper'
 
 describe Raven do
   let(:event) { double("event") }
@@ -87,6 +86,6 @@ def ivars(object)
     end
   end
 
-  
+
 
 end

spec/raven/removecirculareferences_spec.rb

@@ -0,0 +1,23 @@
+# Encoding: utf-8
+
+require 'spec_helper'
+
+describe Raven::Processor::RemoveCircularReferences do
+  before do
+    @client = double("client")
+    @processor = Raven::Processor::RemoveCircularReferences.new(@client)
+  end
+
+  it 'should cleanup circular references' do
+    data = {}
+    data['data'] = data
+    data['ary'] = []
+    data['ary'].push('x' => data['ary'])
+    data['ary2'] = data['ary']
+
+    result = @processor.process(data)
+    expect(result['data']).to eq('(...)')
+    expect(result['ary'].first['x']).to eq('(...)')
+    expect(result['ary2']).not_to eq('(...)')
+  end
+end