Skip to content
This repository
Browse code

Makes the gem system understand development vs. runtime dependencies [#…

…2195 state:resolved]

The patch also fixes:

* Fixes the chicken/egg problem present in the current gem system when
  gems are defined in the config that are not yet installed.

* Remove the need to have hoe as a dependency of your production app.

* Makes the gem 'unpacking' system a lot less fragile.

Signed-off-by: Matt Jones <al2o3cr@gmail.com>
Signed-off-by: Pratik Naik <pratiknaik@gmail.com>
  • Loading branch information...
commit 99d75a7b02bf430a124b9c3e2515850959d78acf 1 parent 5b751ae
David Dollar authored March 13, 2009 lifo committed March 13, 2009
4  railties/lib/initializer.rb
@@ -301,7 +301,9 @@ def add_gem_load_paths
301 301
     end
302 302
 
303 303
     def load_gems
304  
-      @configuration.gems.each { |gem| gem.load }
  304
+      unless $gems_build_rake_task
  305
+        @configuration.gems.each { |gem| gem.load }
  306
+      end
305 307
     end
306 308
 
307 309
     def check_gem_dependencies
214  railties/lib/rails/gem_dependency.rb
@@ -7,8 +7,8 @@ def self.source_index=(index)
7 7
 end
8 8
 
9 9
 module Rails
10  
-  class GemDependency
11  
-    attr_accessor :lib, :source
  10
+  class GemDependency < Gem::Dependency
  11
+    attr_accessor :lib, :source, :dep
12 12
 
13 13
     def self.unpacked_path
14 14
       @unpacked_path ||= File.join(RAILS_ROOT, 'vendor', 'gems')
@@ -29,18 +29,6 @@ def self.add_frozen_gem_path
29 29
       end
30 30
     end
31 31
 
32  
-    def framework_gem?
33  
-      @@framework_gems.has_key?(name)
34  
-    end
35  
-
36  
-    def vendor_rails?
37  
-      Gem.loaded_specs.has_key?(name) && Gem.loaded_specs[name].loaded_from.empty?
38  
-    end
39  
-
40  
-    def vendor_gem?
41  
-      Gem.loaded_specs.has_key?(name) && Gem.loaded_specs[name].loaded_from.include?(self.class.unpacked_path)
42  
-    end
43  
-
44 32
     def initialize(name, options = {})
45 33
       require 'rubygems' unless Object.const_defined?(:Gem)
46 34
 
@@ -52,10 +40,11 @@ def initialize(name, options = {})
52 40
         req = Gem::Requirement.default
53 41
       end
54 42
 
55  
-      @dep = Gem::Dependency.new(name, req)
56 43
       @lib      = options[:lib]
57 44
       @source   = options[:source]
58 45
       @loaded   = @frozen = @load_paths_added = false
  46
+
  47
+      super(name, req)
59 48
     end
60 49
 
61 50
     def add_load_paths
@@ -65,52 +54,74 @@ def add_load_paths
65 54
         @load_paths_added = @loaded = @frozen = true
66 55
         return
67 56
       end
68  
-      gem @dep
  57
+      gem self
69 58
       @spec = Gem.loaded_specs[name]
70 59
       @frozen = @spec.loaded_from.include?(self.class.unpacked_path) if @spec
71 60
       @load_paths_added = true
72 61
     rescue Gem::LoadError
73 62
     end
74 63
 
75  
-    def dependencies(options = {})
76  
-      return [] if framework_gem? || specification.nil?
77  
-
78  
-      all_dependencies = specification.dependencies.map do |dependency|
  64
+    def dependencies
  65
+      return [] if framework_gem?
  66
+      return [] unless installed?
  67
+      specification.dependencies.reject do |dependency|
  68
+        dependency.type == :development
  69
+      end.map do |dependency|
79 70
         GemDependency.new(dependency.name, :requirement => dependency.version_requirements)
80 71
       end
  72
+    end
81 73
 
82  
-      all_dependencies += all_dependencies.map { |d| d.dependencies(options) }.flatten if options[:flatten]
83  
-      all_dependencies.uniq
  74
+    def specification
  75
+      # code repeated from Gem.activate. Find a matching spec, or the currently loaded version.
  76
+      # error out if loaded version and requested version are incompatible.
  77
+      @spec ||= begin
  78
+        matches = Gem.source_index.search(self)
  79
+        matches << @@framework_gems[name] if framework_gem?
  80
+        if Gem.loaded_specs[name] then
  81
+          # This gem is already loaded.  If the currently loaded gem is not in the
  82
+          # list of candidate gems, then we have a version conflict.
  83
+          existing_spec = Gem.loaded_specs[name]
  84
+          unless matches.any? { |spec| spec.version == existing_spec.version } then
  85
+            raise Gem::Exception,
  86
+                  "can't activate #{@dep}, already activated #{existing_spec.full_name}"
  87
+          end
  88
+          # we're stuck with it, so change to match
  89
+          version_requirements = Gem::Requirement.create("=#{existing_spec.version}")
  90
+          existing_spec
  91
+        else
  92
+          # new load
  93
+          matches.last
  94
+        end
  95
+      end
84 96
     end
85 97
 
86  
-    def gem_dir(base_directory)
87  
-      File.join(base_directory, specification.full_name)
  98
+    def requirement
  99
+      r = version_requirements
  100
+      (r == Gem::Requirement.default) ? nil : r
88 101
     end
89 102
 
90  
-    def spec_filename(base_directory)
91  
-      File.join(gem_dir(base_directory), '.specification')
  103
+    def built?
  104
+      # TODO: If Rubygems ever gives us a way to detect this, we should use it
  105
+      false
92 106
     end
93 107
 
94  
-    def load
95  
-      return if @loaded || @load_paths_added == false
96  
-      require(@lib || name) unless @lib == false
97  
-      @loaded = true
98  
-    rescue LoadError
99  
-      puts $!.to_s
100  
-      $!.backtrace.each { |b| puts b }
  108
+    def framework_gem?
  109
+      @@framework_gems.has_key?(name)
101 110
     end
102 111
 
103  
-    def name
104  
-      @dep.name.to_s
  112
+    def frozen?
  113
+      @frozen ||= vendor_rails? || vendor_gem?
105 114
     end
106 115
 
107  
-    def requirement
108  
-      r = @dep.version_requirements
109  
-      (r == Gem::Requirement.default) ? nil : r
  116
+    def installed?
  117
+      Gem.loaded_specs.keys.include?(name)
110 118
     end
111 119
 
112  
-    def frozen?
113  
-      @frozen ||= vendor_rails? || vendor_gem?
  120
+    def load_paths_added?
  121
+      # always try to add load paths - even if a gem is loaded, it may not
  122
+      # be a compatible version (ie random_gem 0.4 is loaded and a later spec
  123
+      # needs >= 0.5 - gem 'random_gem' will catch this and error out)
  124
+      @load_paths_added
114 125
     end
115 126
 
116 127
     def loaded?
@@ -136,48 +147,49 @@ def loaded?
136 147
       end
137 148
     end
138 149
 
139  
-    def load_paths_added?
140  
-      # always try to add load paths - even if a gem is loaded, it may not
141  
-      # be a compatible version (ie random_gem 0.4 is loaded and a later spec
142  
-      # needs >= 0.5 - gem 'random_gem' will catch this and error out)
143  
-      @load_paths_added
  150
+    def vendor_rails?
  151
+      Gem.loaded_specs.has_key?(name) && Gem.loaded_specs[name].loaded_from.empty?
144 152
     end
145 153
 
146  
-    def install
147  
-      cmd = "#{gem_command} #{install_command.join(' ')}"
148  
-      puts cmd
149  
-      puts %x(#{cmd})
  154
+    def vendor_gem?
  155
+      specification && File.exists?(unpacked_gem_directory)
150 156
     end
151 157
 
152  
-    def unpack_to(directory)
153  
-      return if specification.nil? || File.directory?(gem_dir(directory)) || framework_gem?
154  
-
155  
-      FileUtils.mkdir_p directory
156  
-      Dir.chdir directory do
157  
-        Gem::GemRunner.new.run(unpack_command)
  158
+    def build
  159
+      require 'rails/gem_builder'
  160
+      unless built?
  161
+        return unless File.exists?(unpacked_specification_filename)
  162
+        spec = YAML::load_file(unpacked_specification_filename)
  163
+        Rails::GemBuilder.new(spec, unpacked_gem_directory).build_extensions
  164
+        puts "Built gem: '#{unpacked_gem_directory}'"
158 165
       end
159  
-
160  
-      # Gem.activate changes the spec - get the original
161  
-      real_spec = Gem::Specification.load(specification.loaded_from)
162  
-      write_spec(directory, real_spec)
163  
-
  166
+      dependencies.each { |dep| dep.build }
164 167
     end
165 168
 
166  
-    def write_spec(directory, spec)
167  
-      # copy the gem's specification into GEMDIR/.specification so that
168  
-      # we can access information about the gem on deployment systems
169  
-      # without having the gem installed
170  
-      File.open(spec_filename(directory), 'w') do |file|
171  
-        file.puts spec.to_yaml
  169
+    def install
  170
+      unless installed?
  171
+        cmd = "#{gem_command} #{install_command.join(' ')}"
  172
+        puts cmd
  173
+        puts %x(#{cmd})
172 174
       end
173 175
     end
174 176
 
175  
-    def refresh_spec(directory)
  177
+    def load
  178
+      return if @loaded || @load_paths_added == false
  179
+      require(@lib || name) unless @lib == false
  180
+      @loaded = true
  181
+    rescue LoadError
  182
+      puts $!.to_s
  183
+      $!.backtrace.each { |b| puts b }
  184
+    end
  185
+
  186
+    def refresh
  187
+      Rails::VendorGemSourceIndex.silence_spec_warnings = true
176 188
       real_gems = Gem.source_index.installed_source_index
177 189
       exact_dep = Gem::Dependency.new(name, "= #{specification.version}")
178 190
       matches = real_gems.search(exact_dep)
179 191
       installed_spec = matches.first
180  
-      if File.exist?(File.dirname(spec_filename(directory)))
  192
+      if frozen?
181 193
         if installed_spec
182 194
           # we have a real copy
183 195
           # get a fresh spec - matches should only have one element
@@ -185,11 +197,11 @@ def refresh_spec(directory)
185 197
           # spec is the same as the copy from real_gems - Gem.activate changes
186 198
           # some of the fields
187 199
           real_spec = Gem::Specification.load(matches.first.loaded_from)
188  
-          write_spec(directory, real_spec)
  200
+          write_specification(real_spec)
189 201
           puts "Reloaded specification for #{name} from installed gems."
190 202
         else
191 203
           # the gem isn't installed locally - write out our current specs
192  
-          write_spec(directory, specification)
  204
+          write_specification(specification)
193 205
           puts "Gem #{name} not loaded locally - writing out current spec."
194 206
         end
195 207
       else
@@ -201,40 +213,35 @@ def refresh_spec(directory)
201 213
       end
202 214
     end
203 215
 
204  
-    def ==(other)
205  
-      self.name == other.name && self.requirement == other.requirement
  216
+    def unpack(options={})
  217
+      unless frozen? || framework_gem?
  218
+        FileUtils.mkdir_p unpack_base
  219
+        Dir.chdir unpack_base do
  220
+          Gem::GemRunner.new.run(unpack_command)
  221
+        end
  222
+        # Gem.activate changes the spec - get the original
  223
+        real_spec = Gem::Specification.load(specification.loaded_from)
  224
+        write_specification(real_spec)
  225
+      end
  226
+      dependencies.each { |dep| dep.unpack } if options[:recursive]
206 227
     end
207  
-    alias_method :"eql?", :"=="
208 228
 
209  
-    def hash
210  
-      @dep.hash
  229
+    def write_specification(spec)
  230
+      # copy the gem's specification into GEMDIR/.specification so that
  231
+      # we can access information about the gem on deployment systems
  232
+      # without having the gem installed
  233
+      File.open(unpacked_specification_filename, 'w') do |file|
  234
+        file.puts spec.to_yaml
  235
+      end
211 236
     end
212 237
 
213  
-    def specification
214  
-      # code repeated from Gem.activate. Find a matching spec, or the currently loaded version.
215  
-      # error out if loaded version and requested version are incompatible.
216  
-      @spec ||= begin
217  
-        matches = Gem.source_index.search(@dep)
218  
-        matches << @@framework_gems[name] if framework_gem?
219  
-        if Gem.loaded_specs[name] then
220  
-          # This gem is already loaded.  If the currently loaded gem is not in the
221  
-          # list of candidate gems, then we have a version conflict.
222  
-          existing_spec = Gem.loaded_specs[name]
223  
-          unless matches.any? { |spec| spec.version == existing_spec.version } then
224  
-            raise Gem::Exception,
225  
-                  "can't activate #{@dep}, already activated #{existing_spec.full_name}"
226  
-          end
227  
-          # we're stuck with it, so change to match
228  
-          @dep.version_requirements = Gem::Requirement.create("=#{existing_spec.version}")
229  
-          existing_spec
230  
-        else
231  
-          # new load
232  
-          matches.last
233  
-        end
234  
-      end
  238
+    def ==(other)
  239
+      self.name == other.name && self.requirement == other.requirement
235 240
     end
  241
+    alias_method :"eql?", :"=="
236 242
 
237 243
     private
  244
+
238 245
       def gem_command
239 246
         case RUBY_PLATFORM
240 247
           when /win32/
@@ -258,5 +265,18 @@ def unpack_command
258 265
         cmd << "--version" << "= "+specification.version.to_s if requirement
259 266
         cmd
260 267
       end
  268
+
  269
+      def unpack_base
  270
+        Rails::GemDependency.unpacked_path
  271
+      end
  272
+
  273
+      def unpacked_gem_directory
  274
+        File.join(unpack_base, specification.full_name)
  275
+      end
  276
+
  277
+      def unpacked_specification_filename
  278
+        File.join(unpacked_gem_directory, '.specification')
  279
+      end
  280
+
261 281
   end
262 282
 end
78  railties/lib/tasks/gems.rake
@@ -9,71 +9,57 @@ task :gems => 'gems:base' do
9 9
   puts "R = Framework (loaded before rails starts)"
10 10
 end
11 11
 
12  
-def print_gem_status(gem, indent=1)
13  
-  code = gem.loaded? ? (gem.frozen? ? (gem.framework_gem? ? "R" : "F") : "I") : " "
14  
-  puts "   "*(indent-1)+" - [#{code}] #{gem.name} #{gem.requirement.to_s}"
15  
-  gem.dependencies.each { |g| print_gem_status(g, indent+1)} if gem.loaded?
16  
-end
17  
-
18 12
 namespace :gems do
19 13
   task :base do
20 14
     $gems_rake_task = true
  15
+    require 'rubygems'
  16
+    require 'rubygems/gem_runner'
21 17
     Rake::Task[:environment].invoke
22 18
   end
23 19
 
24 20
   desc "Build any native extensions for unpacked gems"
25 21
   task :build do
26  
-    $gems_rake_task = true
27  
-    require 'rails/gem_builder'
28  
-    Dir[File.join(Rails::GemDependency.unpacked_path, '*')].each do |gem_dir|
29  
-      spec_file = File.join(gem_dir, '.specification')
30  
-      next unless File.exists?(spec_file)
31  
-      specification = YAML::load_file(spec_file)
32  
-      next unless ENV['GEM'].blank? || ENV['GEM'] == specification.name
33  
-      Rails::GemBuilder.new(specification, gem_dir).build_extensions
34  
-      puts "Built gem: '#{gem_dir}'"
35  
-    end
  22
+    $gems_build_rake_task = true
  23
+    Rake::Task['gems:unpack'].invoke
  24
+    current_gems.each &:build
36 25
   end
37 26
 
38  
-  desc "Installs all required gems for this application."
  27
+  desc "Installs all required gems."
39 28
   task :install => :base do
40  
-    require 'rubygems'
41  
-    require 'rubygems/gem_runner'
42  
-    Rails.configuration.gems.each { |gem| gem.install unless gem.loaded? }
  29
+    current_gems.each &:install
43 30
   end
44 31
 
45  
-  desc "Unpacks the specified gem into vendor/gems."
46  
-  task :unpack => :base do
47  
-    require 'rubygems'
48  
-    require 'rubygems/gem_runner'
49  
-    Rails.configuration.gems.each do |gem|
50  
-      next unless ENV['GEM'].blank? || ENV['GEM'] == gem.name
51  
-      gem.unpack_to(Rails::GemDependency.unpacked_path)
52  
-    end
  32
+  desc "Unpacks all required gems into vendor/gems."
  33
+  task :unpack => :install do
  34
+    current_gems.each &:unpack
53 35
   end
54 36
 
55 37
   namespace :unpack do
56  
-    desc "Unpacks the specified gems and its dependencies into vendor/gems"
57  
-    task :dependencies => :unpack do
58  
-      require 'rubygems'
59  
-      require 'rubygems/gem_runner'
60  
-      Rails.configuration.gems.each do |gem|
61  
-        next unless ENV['GEM'].blank? || ENV['GEM'] == gem.name
62  
-        gem.dependencies(:flatten => true).each do |dependency|
63  
-          dependency.unpack_to(Rails::GemDependency.unpacked_path)
64  
-        end
65  
-      end
  38
+    desc "Unpacks all required gems and their dependencies into vendor/gems."
  39
+    task :dependencies => :install do
  40
+      current_gems.each { |gem| gem.unpack(:recursive => true) }
66 41
     end
67 42
   end
68 43
 
69 44
   desc "Regenerate gem specifications in correct format."
70 45
   task :refresh_specs => :base do
71  
-    require 'rubygems'
72  
-    require 'rubygems/gem_runner'
73  
-    Rails::VendorGemSourceIndex.silence_spec_warnings = true
74  
-    Rails.configuration.gems.each do |gem|
75  
-      next unless gem.frozen? && (ENV['GEM'].blank? || ENV['GEM'] == gem.name)
76  
-      gem.refresh_spec(Rails::GemDependency.unpacked_path) if gem.loaded?
77  
-    end
  46
+    current_gems.each &:refresh
  47
+  end
  48
+end
  49
+
  50
+def current_gems
  51
+  gems = Rails.configuration.gems
  52
+  gems = gems.select { |gem| gem.name == ENV['GEM'] } unless ENV['GEM'].blank?
  53
+  gems
  54
+end
  55
+
  56
+def print_gem_status(gem, indent=1)
  57
+  code = case
  58
+    when gem.framework_gem? then 'R'
  59
+    when gem.frozen?        then 'F'
  60
+    when gem.installed?     then 'I'
  61
+    else                         ' '
78 62
   end
79  
-end
  63
+  puts "   "*(indent-1)+" - [#{code}] #{gem.name} #{gem.requirement.to_s}"
  64
+  gem.dependencies.each { |g| print_gem_status(g, indent+1) }
  65
+end
17  railties/test/gem_dependency_test.rb
@@ -46,31 +46,34 @@ def test_gem_with_version_unpack_install_command
46 46
   end
47 47
 
48 48
   def test_gem_adds_load_paths
49  
-    @gem.expects(:gem).with(Gem::Dependency.new(@gem.name, nil))
  49
+    @gem.expects(:gem).with(@gem)
50 50
     @gem.add_load_paths
51 51
   end
52 52
 
53 53
   def test_gem_with_version_adds_load_paths
54  
-    @gem_with_version.expects(:gem).with(Gem::Dependency.new(@gem_with_version.name, @gem_with_version.requirement.to_s))
  54
+    @gem_with_version.expects(:gem).with(@gem_with_version)
55 55
     @gem_with_version.add_load_paths
  56
+    assert @gem_with_version.load_paths_added?
56 57
   end
57 58
 
58 59
   def test_gem_loading
59  
-    @gem.expects(:gem).with(Gem::Dependency.new(@gem.name, nil))
  60
+    @gem.expects(:gem).with(@gem)
60 61
     @gem.expects(:require).with(@gem.name)
61 62
     @gem.add_load_paths
62 63
     @gem.load
  64
+    assert @gem.loaded?
63 65
   end
64 66
 
65 67
   def test_gem_with_lib_loading
66  
-    @gem_with_lib.expects(:gem).with(Gem::Dependency.new(@gem_with_lib.name, nil))
  68
+    @gem_with_lib.expects(:gem).with(@gem_with_lib)
67 69
     @gem_with_lib.expects(:require).with(@gem_with_lib.lib)
68 70
     @gem_with_lib.add_load_paths
69 71
     @gem_with_lib.load
  72
+    assert @gem_with_lib.loaded?
70 73
   end
71 74
 
72 75
   def test_gem_without_lib_loading
73  
-    @gem_without_load.expects(:gem).with(Gem::Dependency.new(@gem_without_load.name, nil))
  76
+    @gem_without_load.expects(:gem).with(@gem_without_load)
74 77
     @gem_without_load.expects(:require).with(@gem_without_load.lib).never
75 78
     @gem_without_load.add_load_paths
76 79
     @gem_without_load.load
@@ -132,8 +135,8 @@ def test_gem_handle_missing_dependencies
132 135
     dummy_gem = Rails::GemDependency.new "dummy-gem-g"
133 136
     dummy_gem.add_load_paths
134 137
     dummy_gem.load
135  
-    assert dummy_gem.loaded?
136  
-    assert_equal 2, dummy_gem.dependencies(:flatten => true).size
  138
+    assert_equal 1, dummy_gem.dependencies.size
  139
+    assert_equal 1, dummy_gem.dependencies.first.dependencies.size
137 140
     assert_nothing_raised do
138 141
       dummy_gem.dependencies.each do |g|
139 142
         g.dependencies
2  railties/test/vendor/gems/dummy-gem-g-1.0.0/.specification
@@ -9,7 +9,7 @@ date: 2008-10-03 00:00:00 -04:00
9 9
 dependencies:
10 10
 - !ruby/object:Gem::Dependency
11 11
   name: dummy-gem-f
12  
-  type: :development
  12
+  type: :runtime
13 13
   version_requirement:
14 14
   version_requirements: !ruby/object:Gem::Requirement
15 15
     requirements:

2 notes on commit 99d75a7

Dean Strelau

You have just made my day. Thanks!

Greg Hurrell

I suspect this commit might have broken vendored gems which are C extensions and haven’t been built yet. This causes “rake gems”, “rake gems:build”, and of course “script/server” to not work. To fix the problem you would run “rake gems:build”, but seeing as that’s broken too, you have to manually copy the built extension into the right place.

For the full details, see:

http://rails.lighthouseapp.com/projects/8994/tickets/2266

Please sign in to comment.
Something went wrong with that request. Please try again.