Reworked building multiple language versions of kernel.

kernel/delta/codeloader.rb

@@ -82,6 +82,22 @@ class << self
       attr_accessor :load_compiled
       attr_accessor :check_version
 
+      # Loads the pre-compiled bytecode compiler. Sets up paths needed by the
+      # compiler to find dependencies like the parser.
+      def load_compiler
+        Rubinius.const_set :COMPILER_PATH, Rubinius::RUNTIME_PATH
+        Rubinius.const_set :PARSER_PATH, "#{Rubinius::RUNTIME_PATH}/melbourne"
+
+        ext_path = "#{Rubinius::LIB_PATH}/ext/melbourne/rbx/melbourne"
+        Rubinius.const_set :PARSER_EXT_PATH, ext_path
+
+        begin
+          require_compiled "#{Rubinius::COMPILER_PATH}/compiler"
+        rescue Rubinius::InvalidRBC => e
+          raise LoadError, "Unable to load the bytecode compiler", e
+        end
+      end
+
       def require_compiled(name, check_version=true)
         new(name).require_compiled(check_version)
       end

kernel/loader.rb

@@ -84,12 +84,7 @@ def system_load_path
       # This conforms more closely to MRI. It is necessary to support
       # paths that mkmf adds when compiling and installing native exts.
       additions = []
-      if Rubinius.ruby19? or Rubinius.ruby20?
-        version_lib = "/19"
-      else
-        version_lib = "/18"
-      end
-      additions << @main_lib + version_lib
+      additions << "#{@main_lib}/#{Rubinius::RUBY_LIB_VERSION}"
       additions << Rubinius::SITE_PATH
       additions << "#{Rubinius::SITE_PATH}/#{Rubinius::CPU}-#{Rubinius::OS}"
       additions << Rubinius::VENDOR_PATH
@@ -509,34 +504,17 @@ def load_paths
     def load_compiler
       @stage = "loading the compiler"
 
-      # This happens before we parse ARGV, so we have to check ARGV ourselves
-      # here.
-
-      rebuild = (ARGV.last == "--rebuild-compiler")
-
       begin
-        CodeLoader.require_compiled "compiler", rebuild ? false : true
-      rescue Rubinius::InvalidRBC => e
-        STDERR.puts "There was an error loading the compiler."
-        STDERR.puts "It appears that your compiler is out of date with the VM."
-        STDERR.puts "\nPlease use 'rbx --rebuild-compiler' or 'rake [install]' to"
-        STDERR.puts "bring the compiler back to date."
-        exit 1
-      end
-
-      if rebuild
-        STDOUT.puts "Rebuilding compiler..."
-        files =
-          ["#{@main_lib}/compiler.rb"] +
-          Dir["#{@main_lib}/compiler/*.rb"] +
-          Dir["#{@main_lib}/compiler/**/*.rb"]
+        CodeLoader.load_compiler
+      rescue LoadError => e
+        STDERR.puts <<-EOM
+Unable to load the bytecode compiler. Please run 'rake' or 'rake install'
+to rebuild the compiler.
 
-        files.each do |file|
-          puts "#{file}"
-          Rubinius.compile_file file, true
-        end
+        EOM
 
-        exit 0
+        e.render
+        exit 1
       end
     end
 

lib/20

@@ -0,0 +1 @@
+19

lib/compiler.rb

@@ -1,15 +1,14 @@
-base = File.expand_path "../", __FILE__
+require Rubinius::PARSER_PATH
 
-require base + "/melbourne"
-
-require base + "/compiler/compiler"
-require base + "/compiler/stages"
-require base + "/compiler/locals"
-require base + "/compiler/ast"
-require base + "/compiler/generator_methods"
-require base + "/compiler/generator"
-require base + "/compiler/iseq"
-require base + "/compiler/opcodes"
-require base + "/compiler/compiled_file"
-require base + "/compiler/evaluator"
-require base + "/compiler/printers"
+path = Rubinius::COMPILER_PATH
+require path + "/compiler/compiler"
+require path + "/compiler/stages"
+require path + "/compiler/locals"
+require path + "/compiler/ast"
+require path + "/compiler/generator_methods"
+require path + "/compiler/generator"
+require path + "/compiler/iseq"
+require path + "/compiler/opcodes"
+require path + "/compiler/compiled_file"
+require path + "/compiler/evaluator"
+require path + "/compiler/printers"

lib/melbourne.rb

@@ -1,14 +1,5 @@
-base = File.expand_path "../", __FILE__
-
-unless Object.const_defined?(:RUBY_ENGINE) && RUBY_ENGINE == "rbx"
-  ext_dir = "ruby"
-  require base + "/mri_bridge"
-else
-  ext_dir = Object.const_get :RUBY_ENGINE
-end
-
-require base + "/ext/melbourne/#{ext_dir}/melbourne"
-require base + "/melbourne/processor"
+require Rubinius::PARSER_EXT_PATH
+require Rubinius::PARSER_PATH + "/processor"
 
 class String
   def to_ast(name="(eval)", line=1)

rakelib/kernel.rake

@@ -42,71 +42,41 @@ rule ".rbc" do |t|
   end
 
   source = t.prerequisites.first
-  puts "RBC #{source}"
+  version = t.name.match(%r[^runtime/(\d+)])[1]
+  puts "RBC #{version.split(//).join('.')} #{source}"
   Rubinius::Compiler.compile source, t.name, 1, [:default, :kernel]
 end
 
 # Collection of all files in the kernel runtime. Modified by
 # various tasks below.
 runtime = FileList["runtime/platform.conf"]
 
-# Names of subdirectories of the runtime/ directory.
-dir_names = [
-  "bootstrap",
-  "platform",
-  "common",
-  "delta"
+# Names of subdirectories of the runtime/ directory based on language
+# version.
+version_names = %w[ 18 19 20 ]
+
+# Names of subdirectories of the language directories.
+dir_names = %w[
+  bootstrap
+  platform
+  common
+  delta
 ]
 
 # Generate file tasks for all kernel and load_order files.
-compiler_signature = "kernel/signature.rb"
-
-def kernel_file_task(runtime, signature, rb, rbc=nil)
-  rbc ||= rb.sub(/^kernel/, "runtime") + "c"
+def file_task(re, runtime, signature, version, rb, rbc)
+  rbc ||= rb.sub(re, "runtime/#{version}") + "c"
 
   file rbc => [rb, signature]
   runtime << rbc
 end
 
-["18", "19", "20"].each do |ver|
-  dir_names.each do |dir|
-    directory(runtime_dir = "runtime/#{dir}")
-    runtime << runtime_dir
-
-    load_order = "runtime/#{dir}/load_order#{ver}.txt"
-    runtime << load_order
-
-    kernel_load_order = "kernel/#{dir}/load_order#{ver}.txt"
-
-    file load_order => kernel_load_order do |t|
-      cp t.prerequisites.first, t.name, :verbose => $verbose
-    end
-
-    kernel_dir  = "kernel/#{dir}/"
-    runtime_dir = "runtime/#{dir}/"
-
-    IO.foreach kernel_load_order do |name|
-      rbc = runtime_dir + name.chomp!
-      rb  = kernel_dir + name.chop
-      kernel_file_task runtime, compiler_signature, rb, rbc
-    end
-  end
+def kernel_file_task(runtime, signature, version, rb, rbc=nil)
+  file_task(/^kernel/, runtime, signature, version, rb, rbc)
 end
 
-[ compiler_signature,
-  "kernel/alpha.rb",
-  "kernel/loader.rb"
-].each do |name|
-  kernel_file_task runtime, compiler_signature, name
-end
-
-# Directories to store the core library runtime files (.rbc's)
-runtime_index = "runtime/index"
-
-file runtime_index do |t|
-  File.open t.name, "w" do |file|
-    file.puts dir_names
-  end
+def compiler_file_task(runtime, signature, version, rb, rbc=nil)
+  file_task(/^lib/, runtime, signature, version, rb, rbc)
 end
 
 # Compile all compiler files during build stage
@@ -121,27 +91,23 @@ compiler_files = FileList[
   "lib/melbourne/**/*.rb"
 ]
 
-compiler_files.each do |rb|
-  rbc = rb + "c"
-
-  file rbc => [rb, compiler_signature]
-  runtime << rbc
-end
-
-parser_ext_files = FileList[
+parser_files = FileList[
   "lib/ext/melbourne/**/*.{c,h}pp",
   "lib/ext/melbourne/grammar18.y",
   "lib/ext/melbourne/grammar19.y",
   "lib/ext/melbourne/lex.c.tab"
 ]
 
+
 # Generate a sha1 of all parser and compiler files to use as
 # as signature in the .rbc files.
-file compiler_signature => compiler_files + parser_ext_files do |t|
+compiler_signature = "kernel/signature.rb"
+
+file compiler_signature => compiler_files + parser_files do |t|
   require 'digest/sha1'
   digest = Digest::SHA1.new
 
-  t.prerequisites.each do |name|
+  (compiler_files + parser_files).each do |name|
     File.open name, "r" do |file|
       while chunk = file.read(1024)
         digest << chunk
@@ -151,33 +117,101 @@ file compiler_signature => compiler_files + parser_ext_files do |t|
 
   # Collapse the digest to a 64bit quantity
   hd = digest.hexdigest
-  hash = hd[0, 16].to_i(16) ^ hd[16,16].to_i(16) ^ hd[32,8].to_i(16)
+  compiler_hash = hd[0, 16].to_i(16) ^ hd[16,16].to_i(16) ^ hd[32,8].to_i(16)
 
-  File.open t.name, "wb" do |file|
+  File.open compiler_signature, "wb" do |file|
     file.puts "# This file is generated by rakelib/kernel.rake. The signature"
     file.puts "# is used to ensure that only current .rbc files are loaded."
     file.puts "#"
-    file.puts "Rubinius::Signature = #{hash}"
+    file.puts "Rubinius::Signature = #{compiler_hash}"
+  end
+end
+
+# Index files for loading a particular version of the kernel.
+version_names.each do |ver|
+  directory(runtime_base_dir = "runtime/#{ver}")
+  runtime << runtime_base_dir
+
+  runtime_index = "#{runtime_base_dir}/index"
+  runtime << runtime_index
+
+  file runtime_index => runtime_base_dir do |t|
+    File.open t.name, "wb" do |file|
+      file.puts dir_names
+    end
   end
 
-  File.open "runtime/signature", "wb" do |file|
-    file.puts hash
+  signature = "runtime/#{ver}/signature"
+  file signature => compiler_signature do |t|
+    require compiler_signature
+    File.open t.name, "wb" do |file|
+      puts "GEN #{t.name}"
+      file.puts Rubinius::Signature
+    end
+  end
+  runtime << signature
+
+  # All the kernel files
+  dir_names.each do |dir|
+    directory(runtime_dir = "runtime/#{ver}/#{dir}")
+    runtime << runtime_dir
+
+    load_order = "runtime/#{ver}/#{dir}/load_order.txt"
+    runtime << load_order
+
+    kernel_load_order = "kernel/#{dir}/load_order#{ver}.txt"
+
+    file load_order => kernel_load_order do |t|
+      cp t.prerequisites.first, t.name, :verbose => $verbose
+    end
+
+    kernel_dir  = "kernel/#{dir}/"
+    runtime_dir = "runtime/#{ver}/#{dir}/"
+
+    IO.foreach kernel_load_order do |name|
+      rbc = runtime_dir + name.chomp!
+      rb  = kernel_dir + name.chop
+      kernel_file_task runtime, compiler_signature, ver, rb, rbc
+    end
+  end
+
+  [ compiler_signature,
+    "kernel/alpha.rb",
+    "kernel/loader.rb"
+  ].each do |name|
+    kernel_file_task runtime, compiler_signature, ver, name
+  end
+
+  compiler_files.map { |f| File.dirname f }.uniq.each do |dir|
+    directory dir
+  end
+
+  compiler_files.each do |name|
+    compiler_file_task runtime, compiler_signature, ver, name
   end
 end
 
 namespace :compiler do
+  signature_path = File.expand_path("../../kernel/signature", __FILE__)
+
   if BUILD_CONFIG[:which_ruby] == :ruby
     melbourne = "lib/ext/melbourne/ruby/melbourne.#{$dlext}"
 
     file melbourne => "extensions:melbourne_mri"
 
+    path = File.expand_path("../../lib", __FILE__)
+    Rubinius::COMPILER_PATH = path
+    Rubinius::PARSER_PATH = "#{path}/melbourne"
+    Rubinius::PARSER_EXT_PATH = "#{path}/ext/melbourne/ruby/melbourne"
+
     task :load => [compiler_signature, melbourne] + compiler_files do
-      require File.expand_path("../../lib/compiler", __FILE__)
-      require File.expand_path("../../kernel/signature", __FILE__)
+      require "#{Rubinius::COMPILER_PATH}/mri_bridge"
+      require "#{Rubinius::COMPILER_PATH}/compiler"
+      require signature_path
     end
   else
     task :load => compiler_signature do
-      require File.expand_path("../../kernel/signature", __FILE__)
+      require signature_path
     end
   end
 end
@@ -187,7 +221,7 @@ task :kernel => 'kernel:build'
 
 namespace :kernel do
   desc "Build all kernel files"
-  task :build => ['compiler:load', runtime_index] + runtime
+  task :build => ['compiler:load'] + runtime
 
   desc "Delete all .rbc files"
   task :clean do

runtime/index

@@ -1,4 +0,0 @@
-bootstrap
-platform
-common
-delta

vm/drivers/cli.cpp

@@ -94,15 +94,12 @@ int main(int argc, char** argv) {
     env.state->print_backtrace();
     return 1;
   } catch(BadKernelFile& e) {
-    std::cout << "ERROR: BadKernelFile: " << e.what() << "\n\n";
-    std::cout << "An invalid kernel file has been detected.\n";
-    std::cout << "This is because the VM is out of sync with the kernel.\n";
-    std::cout << "Please recompile your kernel using:\n";
-    std::cout << "  rake kernel:clean clean\n";
-    std::cout << "  rake\n";
-    std::cout << "\nIf the problem persists, please open an issue at:\n";
+    std::cout << "ERROR: Unable to load: " << e.what() << std::endl << std::endl;
+    std::cout << "Please run the following commands to rebuild:" << std::endl;
+    std::cout << "  rake clean" << std::endl;
+    std::cout << "  rake or rake install" << std::endl << std::endl;
+    std::cout << "If the problem persists, please open an issue at:" << std::endl;
     std::cout << "  http://github.com/evanphx/rubinius\n";
-    std::cout << "\nThanks,\n  Management.\n";
     return 1;
   } catch(VMException &e) {
     std::cout << "Unknown VM exception detected." << std::endl;