Stack allocated lvars, GC fixes, compiler changes, oh my!

This is a biggy (too big in fact). It started as a change to allow
arguments to be accessed directly from the stack, and turned into a
monster.

Arguments and some lvars can now be accessed directly from the stack,
making them cheaper to create and use. This turned out to expose
a large number of bugs in the VM related to stack access, as well
as some in the GC.

The big GC change here is that the mark/sweep GC is actually run now,
as opposed to before when it would just allocated more and more memory
(the source of memory issues I suspect).

Rakefile

@@ -24,6 +24,8 @@ if File.exists?(@pc)
   ENV['CORE'] = @pc
 end
 
+@compiler = ENV['COMPILER']
+
 def update_archive(files, archive, dir=nil)
   archive = File.expand_path(ENV['OUTPUT'] || archive)
 
@@ -32,7 +34,11 @@ def update_archive(files, archive, dir=nil)
     cmp = "#{file}c"
     if !newer?(file, cmp)
       changed << cmp
-      system "shotgun/rubinius compile #{file}"
+      if @compiler
+        system "shotgun/rubinius -I#{@compiler} compile #{file}"
+      else
+        system "shotgun/rubinius compile #{file}"
+      end
     elsif !File.exists?(archive)
       changed << cmp
     end
@@ -229,7 +235,9 @@ namespace :build do
       paths << dest
     end
 
-    paths += %w!native/bytecode/rubinius.rb native/bytecode/system_hints.rb!
+    paths += %w!native/bytecode/rubinius.rb 
+                native/bytecode/system_hints.rb
+                native/bytecode/plugins.rb!
 
     update_archive paths, 'runtime/compiler.rba', "native"
   end

kernel/bootstrap/04kernel.rb

@@ -47,7 +47,7 @@ def raise(exc=$!, msg=nil)
       STDERR.puts "Exception: #{exc.message} (#{exc.class})"
     end
 
-    Ruby.asm "push exc\nraise_exc"
+    Ruby.asm "#local exc\nraise_exc"
   end
 
   def at_exit(&block)

kernel/bootstrap/compiled_method.rb

@@ -99,14 +99,14 @@ def activate(recv, args, locals=nil, &prc)
     end
 
     Ruby.asm <<-ASM
-    push args
-    push_array
-    push block
-    push locals
-    push sz
-    push self
-    push recv
-    activate_method
+#local args
+push_array
+#local block
+#local locals
+#local sz
+push self
+#local recv
+activate_method
     ASM
   end
 

kernel/bootstrap/context.rb

@@ -19,8 +19,7 @@ def name     ; @name      ; end
   def module   ; @module    ; end
 
   def self.current
-    cur = nil
-    Ruby.asm "push_context\nset cur\n"
+    cur = Ruby.asm "push_context\n"
     return cur.sender
   end
 

kernel/bootstrap/core.rb

@@ -35,9 +35,9 @@ def undef?
 
   def kind_of?(cls)
     Ruby.asm <<-ASM
-    push cls
-    push self
-    kind_of
+#local cls
+push self
+kind_of
     ASM
   end
 
@@ -88,7 +88,7 @@ def inspect
         end
       end
     else
-      0.step(@__ivars__.size, 2) do |i|
+      0.step(@__ivars__.size - 1, 2) do |i|
         if k = @__ivars__[i]
           v = @__ivars__[i+1]
           if v.object_id == self.object_id # This would be an infinite loop
@@ -107,12 +107,10 @@ def inspect
 
   def respond_to?(meth)
     meth = meth.to_sym
-    cm = nil
-    Ruby.asm <<-ASM
-    push self
-    push meth
-    locate_method
-    set cm
+    cm = Ruby.asm <<-ASM
+push self
+#local meth
+locate_method
     ASM
     !cm.nil?
   end
@@ -121,27 +119,25 @@ def __send__(name, *args, &prc)
     meth = name.to_sym
     count = args.size.to_i
     Ruby.asm <<-ASM
-    push args
-    push_array
-    push self
-    push prc
-    push meth
-    push count
-    set_args
-    send_off_stack
+#local args
+push_array
+push self
+#local prc
+#local meth
+#local count
+set_args
+send_off_stack
     ASM
   end
 
   alias :send :__send__
 
   def method(name)
     meth = name.to_sym
-    cm = nil
-    Ruby.asm <<-ASM
-    push self
-    push meth
-    locate_method
-    set cm
+    cm = Ruby.asm <<-ASM
+push self
+#local meth
+locate_method
     ASM
 
     if cm
@@ -152,8 +148,7 @@ def method(name)
   end
 
   def lambda
-    env = nil
-    Ruby.asm "push_block\nset env\n"
+    env = Ruby.asm "push_block\n"
     unless env
       raise ArgumentError, "Unable to create a Proc if a block is not passed in"
     end

kernel/bootstrap/string.rb

@@ -70,9 +70,7 @@ def <<(other)
       end
     end
 
-    out = nil
-    Ruby.asm "push other\npush self\nstring_append\nset out"
-    return out
+    Ruby.asm "#local other\npush self\nstring_append\n"
   end
   alias :concat :<<
 
@@ -85,9 +83,7 @@ def +(other)
   end
 
   def dup
-    out = nil
-    Ruby.asm "push self\nstring_dup\nset out"
-    return out
+    Ruby.asm "push self\nstring_dup\n"
   end
 
   def substring(start, count)

kernel/core/09class.rb

@@ -1,7 +1,11 @@
 class Class
   def new(*args, &block)
     obj = allocate()
-    obj.initialize(*args, &block)
+    if block
+      obj.initialize(*args, &block)
+    else
+      obj.initialize(*args)
+    end
     obj
   end
 

kernel/core/__loader.rb

@@ -121,13 +121,16 @@
   code = e.code
 rescue Object => e
   begin
+    p e
     puts "An exception has occurred:"
     puts "    #{e.message} (#{e.class})"
     puts "\nBacktrace:"
     puts e.backtrace.show
     code = 1
-  rescue Object => e
-    puts "VERY BROKEN."
+  rescue Object => e2
+    puts "Unable to build backtrace due to errors"
+    puts "Original Exception: #{e.inspect} (#{e.class})"
+    puts "New Exception: #{e2.inspect} (#{e.class})"
     code = 128
   end
 end

kernel/core/string.rb

@@ -26,18 +26,15 @@ def <<(other)
 
     raise TypeError, "can't modify frozen string" if self.frozen?
 
-    out = nil
-    Ruby.asm "push other\npush self\nstring_append\nset out"
-    return out
+    Ruby.asm "#local other\npush self\nstring_append\n"
   end
   alias :concat :<<
 
   #---
   # NOTE: This overwrites String#dup defined in bootstrap
   #+++
   def dup
-    out = nil
-    Ruby.asm "push self\nstring_dup\nset out"
+    out = Ruby.asm "push self\nstring_dup\n"
     out.taint if self.tainted?
     out.freeze if self.frozen?
     return out

lib/bin/describe.rb

@@ -15,9 +15,9 @@
 
 sexp = code.to_sexp
 puts "\nSexp:\n  #{sexp.pretty_inspect}"
-nx = compiler.fully_normalize(sexp, state)
-puts "\nNormalized Sexp:\n  #{nx.pretty_inspect}"
-desc = compiler.compile_as_method(nx, :__eval_script__, state)
+# nx = compiler.fully_normalize(sexp, state)
+# puts "\nNormalized Sexp:\n  #{nx.pretty_inspect}"
+desc = compiler.compile_as_script(sexp, :__eval_script__, state)
 puts "\nAssembly:\n#{desc.assembly}"
 
 puts "Sub methods:"

lib/bytecode/assembler.rb

@@ -205,12 +205,29 @@ def parse_command(kind, args)
         @primitive = args.to_i
       when :arg
         @arguments << args.strip.to_sym
+      when :local
+        name = args.strip.to_sym
+        lv = @state.local(name)
+        unless lv
+          raise "Unknown local '#{name}'"
+        end
+
+        parse_line lv.access_assembly
+      when :set_local
+        name = args.strip.to_sym
+        lv = @state.local(name)
+        unless lv
+          raise "Unknown local '#{name}'"
+        end
+
+        parse_line lv.set_assembly        
       else
         raise "Unknown command '#{kind}'"
       end
     end
 
     def parse_line(line)
+      line.gsub!(/;.*$/, "")
       if m = /^\s*([^\s]*):(.*)/.match(line)
         name = m[1].to_sym
         if @labels.key?(name)
@@ -440,7 +457,7 @@ def parse_operation(*parts)
       if lop = Translations[op]
         op = lop
       end
-
+            
       if Simple.include?(op)
         @output << op
         @current_op += 1
@@ -526,7 +543,10 @@ def parse_operation(*parts)
 
       if Bytecode::InstructionEncoder::OpCodes.include?(op)
         @current_op += 1
-        if Bytecode::InstructionEncoder::IntArg.include?(op)
+        if Bytecode::InstructionEncoder::TwoInt.include?(op)
+          @current_op += 8
+          @output << [op, parts.shift.to_i, parts.shift.to_i]
+        elsif Bytecode::InstructionEncoder::IntArg.include?(op)
           @current_op += 4
           @output << [op, parts.first.to_i]
         else