diff --git a/actionpack/lib/action_view/template/handlers/builder.rb b/actionpack/lib/action_view/template/handlers/builder.rb
index abe140af0bcde..5f381f7bf09ac 100644
--- a/actionpack/lib/action_view/template/handlers/builder.rb
+++ b/actionpack/lib/action_view/template/handlers/builder.rb
@@ -1,5 +1,3 @@
-require 'builder'
-
 module ActionView
   module TemplateHandlers
     class Builder < TemplateHandler
@@ -8,6 +6,7 @@ class Builder < TemplateHandler
       self.default_format = Mime::XML
 
       def compile(template)
+        require 'builder'
         "xml = ::Builder::XmlMarkup.new(:indent => 2);" +
           "self.output_buffer = xml.target!;" +
           template.source +
diff --git a/actionpack/lib/action_view/template/handlers/erb.rb b/actionpack/lib/action_view/template/handlers/erb.rb
index 21272ef089a23..e3a7d96941d80 100644
--- a/actionpack/lib/action_view/template/handlers/erb.rb
+++ b/actionpack/lib/action_view/template/handlers/erb.rb
@@ -1,4 +1,3 @@
-require 'erb'
 require 'active_support/core_ext/class/attribute_accessors'
 
 module ActionView
@@ -16,6 +15,8 @@ class ERB < TemplateHandler
       self.default_format = Mime::HTML
 
       def compile(template)
+        require 'erb'
+
         magic = $1 if template.source =~ /\A(<%#.*coding:\s*(\S+)\s*-?%>)/
         erb = "#{magic}<% __in_erb_template=true %>#{template.source}"
         ::ERB.new(erb, nil, erb_trim_mode, '@output_buffer').src
diff --git a/activesupport/lib/active_support/cache.rb b/activesupport/lib/active_support/cache.rb
index 192a82e74f0b4..3c422e0252a4a 100644
--- a/activesupport/lib/active_support/cache.rb
+++ b/activesupport/lib/active_support/cache.rb
@@ -2,7 +2,12 @@
 require 'active_support/core_ext/benchmark'
 require 'active_support/core_ext/exception'
 require 'active_support/core_ext/class/attribute_accessors'
-require 'active_support/core_ext' # FIXME: pulling in all to_param extensions
+
+%w(hash nil string time date date_time array big_decimal range object boolean).each do |library|
+  require "active_support/core_ext/#{library}/conversions"
+end
+
+# require 'active_support/core_ext' # FIXME: pulling in all to_param extensions
 
 module ActiveSupport
   # See ActiveSupport::Cache::Store for documentation.
diff --git a/railties/lib/initializer.rb b/railties/lib/initializer.rb
index 68c3576fedb58..cdf2a22c834e0 100644
--- a/railties/lib/initializer.rb
+++ b/railties/lib/initializer.rb
@@ -177,7 +177,8 @@ def self.run(initializer = nil, config = nil)
   # Action Pack, Action Mailer, and Active Resource) are loaded.
   Initializer.default.add :require_frameworks do
     begin
-      require 'active_support/all'
+      require 'active_support'
+      require 'active_support/core_ext/kernel/reporting'
       configuration.frameworks.each { |framework| require(framework.to_s) }
     rescue LoadError => e
       # Re-raise as RuntimeError because Mongrel would swallow LoadError.
@@ -397,12 +398,15 @@ def self.run(initializer = nil, config = nil)
   end
 
   Initializer.default.add :initialize_metal do
-    Rails::Rack::Metal.requested_metals = configuration.metals
-    Rails::Rack::Metal.metal_paths += plugin_loader.engine_metal_paths
+    # TODO: Make Rails and metal work without ActionController
+    if defined?(ActionController)
+      Rails::Rack::Metal.requested_metals = configuration.metals
+      Rails::Rack::Metal.metal_paths += plugin_loader.engine_metal_paths
 
-    configuration.middleware.insert_before(
-      :"ActionDispatch::ParamsParser",
-      Rails::Rack::Metal, :if => Rails::Rack::Metal.metals.any?)
+      configuration.middleware.insert_before(
+        :"ActionDispatch::ParamsParser",
+        Rails::Rack::Metal, :if => Rails::Rack::Metal.metals.any?)
+    end
   end
 
   # Add the load paths used by support functions such as the info controller
diff --git a/railties/lib/rails/configuration.rb b/railties/lib/rails/configuration.rb
index c48c989b18cfa..fdb071fc18d86 100644
--- a/railties/lib/rails/configuration.rb
+++ b/railties/lib/rails/configuration.rb
@@ -93,7 +93,6 @@ def framework_root_path
 
     # TODO: Fix this when there is an application object
     def middleware
-      require 'action_controller'
       ActionController::Dispatcher.middleware
     end