Extract heap_dump reader and tools into repo

bin/histo.rb

@@ -0,0 +1,26 @@
+require 'heap_dump'
+require 'heap_dump/diff'
+
+file = ARGV.shift
+after_file = ARGV.shift
+
+if after_file
+  before = Rubinius::HeapDump.open(file)
+  after =  Rubinius::HeapDump.open(after_file)
+
+  diff = Rubinius::HeapDump::Diff.new(before, after)
+
+  histo = diff.histogram
+
+  histo.each_sorted do |name, entry|
+    printf "%10d %30s %d\n", entry.objects, name, entry.bytes
+  end
+else
+  decoder = Rubinius::HeapDump.open(file)
+
+  histo = decoder.all_objects.histogram
+
+  histo.each_sorted do |klass, entry|
+    printf "%10d %30s %d\n", entry.objects, klass.name, entry.bytes
+  end
+end

lib/heap_dump.rb

@@ -0,0 +1,11 @@
+require 'heap_dump/decoder'
+
+module Rubinius
+  module HeapDump
+    def self.open(file)
+      dec = Decoder.new
+      dec.decode(file)
+      return dec
+    end
+  end
+end

lib/heap_dump/decoder.rb

@@ -0,0 +1,194 @@
+require 'heap_dump/dumped_object'
+require 'heap_dump/objects'
+
+module Rubinius
+  module HeapDump
+
+    Reference = Struct.new(:id)
+    XSymbol = Struct.new(:id, :data)
+
+    class XSymbol
+      def ==(other)
+        case other
+        when String
+          data == other
+        else
+          super
+        end
+      end
+
+      def <=>(other)
+        data <=> other.data
+      end
+
+      def to_s
+        data
+      end
+    end
+
+    class Decoder
+
+      Tuple = Struct.new(:objects)
+      Bytes = Struct.new(:data)
+
+      def initialize
+        @symbols = []
+        @objects = []
+        @layouts = []
+      end
+
+      attr_reader :symbols, :objects, :layouts
+
+      def deref(id)
+        if id.kind_of? Reference
+          @objects[id.id]
+        else
+          @objects[id]
+        end
+      end
+
+      def int
+        @f.read(4).unpack("N").first
+      end
+
+      def ints(n)
+        @f.read(4*n).unpack("N#{n}")
+      end
+
+      def short
+        @f.read(2).unpack("n").first
+      end
+
+      def char
+        @f.read(1)[0]
+      end
+
+      def decode_reference
+        subcmd = @f.read(1)[0]
+        case subcmd
+        when ?r
+          obj = Reference.new(int)
+        when ?x
+          ref = int
+          obj = @symbols[ref]
+        when ?s
+          id, sz = ints(2)
+          obj = XSymbol.new(id, @f.read(sz))
+          @symbols[id] = obj
+        when ?f
+          obj = int
+        when ?t
+          obj = []
+          sz = int
+          sz.times do
+            obj << decode_reference
+          end
+
+          obj = Tuple.new(obj)
+        when ?b
+          obj = Bytes.new(@f.read(int))
+        when ?i
+          obj = case char
+                when 0
+                  false
+                when 1
+                  true
+                when 2
+                  nil
+                end
+        else
+          raise "invalid sub code - #{subcmd}, #{subcmd.chr}"
+        end
+
+        return obj
+      end
+
+      def decode(file)
+        File.open(file) do |f|
+          magic = f.read(12)
+          if magic != "RBXHEAPDUMP\0"
+            raise "Invalid file format"
+          end
+
+          @f = f
+
+          version = int
+
+          unless version == 1
+            raise "Invalid version - #{version}"
+          end
+
+          while true
+            str = @f.read(1)
+            break unless str
+            cmd = str[0]
+
+            case cmd
+            when ?s
+              id, sz = ints(2)
+              str = @f.read(sz)
+              @symbols[id] = str
+            when ?o
+              id, bytes, layout, klass = ints(4)
+              ivar_ref = decode_reference
+              syms = short # , syms = @f.read(18).unpack("NNNNn")
+              ivars = []
+
+              syms.times do
+                ivars << decode_reference
+              end
+
+              if @objects[id]
+                raise "redefined object #{id}"
+              end
+
+              @objects[id] = DumpedObject.new(self, id, bytes,
+                                              @layouts[layout],
+                                              Reference.new(klass),
+                                              Reference.new(ivar_ref), ivars)
+            when ?l
+              id, sz = ints(2)
+              syms = ints(sz)
+              @layouts[id] = syms.map { |x| @symbols[x] }
+            when ?- # footer
+              @object = @objects[int].as_module
+              @klass =  @objects[int].as_module
+              @module = @objects[int].as_module
+            else
+              raise "invalid code #{cmd}"
+            end
+          end
+        end
+
+        @included_module = @object.find_class("IncludedModule")
+        @f = nil
+      end
+
+      def Object
+        @object
+      end
+
+      def Class
+        @klass
+      end
+
+      def Module
+        @module
+      end
+
+      def all_objects
+        return Objects.new(@objects)
+      end
+
+      def find_references(id)
+        out = []
+        @objects.each do |o|
+          out << o if o.references?(id)
+        end
+
+        return out
+      end
+    end
+
+  end
+end

lib/heap_dump/diff.rb

@@ -0,0 +1,30 @@
+module Rubinius
+  module HeapDump
+    class Diff
+      def initialize(before, after)
+        @before = before
+        @after = after
+      end
+
+      def histogram
+        b = Histogram.by_class_name @before.all_objects.array
+        a = Histogram.by_class_name @after.all_objects.array
+
+        c = {}
+
+        a.each do |k,e|
+          if prev = b[k]
+            n = e - prev
+            if n.objects != 0
+              c[k] = n
+            end
+          else
+            c[k] = e
+          end
+        end
+
+        Histogram.new(c)
+      end
+    end
+  end
+end

lib/heap_dump/dumped_lookup_table.rb

@@ -0,0 +1,26 @@
+require 'heap_dump/specializer'
+
+module Rubinius
+  module HeapDump
+    class DumpedLookupTable < Specializer
+      def each
+        @object["@values"].data.objects.each do |ref|
+          if ref
+            obj = @object.decoder.deref(ref)
+            while obj
+              yield obj["@key"], obj["@value"]
+
+              obj = obj["@next"]
+            end
+          end
+        end
+      end
+
+      include Enumerable
+
+      def keys
+        map { |k,v| k.data }
+      end
+    end
+  end
+end

lib/heap_dump/dumped_method_table.rb

@@ -0,0 +1,26 @@
+require 'heap_dump/specializer'
+
+module Rubinius
+  module HeapDump
+    class DumpedMethodTable < Specializer
+      def each
+        @object["@values"].data.objects.each do |ref|
+          if ref
+            obj = @object.decoder.deref(ref)
+            while obj
+              yield obj["@name"], obj["@method"], obj["@visibility"]
+
+              obj = obj["@next"]
+            end
+          end
+        end
+      end
+
+      include Enumerable
+
+      def keys
+        map { |k,v| k.data }
+      end
+    end
+  end
+end

lib/heap_dump/dumped_module.rb

@@ -0,0 +1,75 @@
+require 'heap_dump/specializer'
+require 'heap_dump/dumped_lookup_table'
+require 'heap_dump/dumped_method_table'
+
+module Rubinius
+  module HeapDump
+    class DumpedModule < Specializer
+      def name
+        if sym = @object["@module_name"]
+          sym.data
+        end
+      end
+
+      def superclass
+        if obj = @object["@superclass"]
+          return obj.as_module
+        end
+
+        return nil
+      end
+
+      def constant_table
+        @constants ||= DumpedLookupTable.new(@object["@constants"])
+      end
+
+      def constants
+        constant_table.map do |key, val|
+          key.data
+        end
+      end
+
+      def method_table
+        @methods ||= DumpedMethodTable.new(@object["@method_table"])
+      end
+
+      def find_class(name)
+        name = name.to_s
+        constant_table.map do |key, val|
+          return val.as_module if key.data == name
+        end
+
+        nil
+      end
+
+      def constant(name)
+        name = name.to_s
+        constant_table.map do |key, val|
+          if key.data == name
+            if val.dump_kind_of?(@object.decoder.Module)
+              return val.as_module
+            else
+              return val
+            end
+          end
+        end
+
+        nil
+      end
+
+      def all_instances
+        ary = []
+        @object.decoder.objects.each do |obj|
+          ary << obj if obj.class_object.id == @object.id
+        end
+
+        return Objects.new(ary)
+      end
+
+      def metaclass?
+        !@object["@attached_instance"].nil?
+      end
+    end
+
+  end
+end