Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Git stager PoC

  This change adds a naive implementation for Gemfiles containing Git
URLs.

N.B.
  For this to work, platform.yml in BOSH release needs to be configured
to point to the location of the Git binary.

  Test plan:
    - Unit tests passed
    - BVTs passed
    - CCNG with git Gemfile worked.

Change-Id: Ia8ef1a1ad0b4fd0ad65bdecd5be882382b824f1b
  • Loading branch information...
commit 8815bbb63cc0a29cac2446751f428baa463b1101 1 parent 743dd69
Jesse Zhang authored July 11, 2012
4  Gemfile.lock
@@ -21,7 +21,7 @@ GIT
21 21
 PATH
22 22
   remote: .
23 23
   specs:
24  
-    vcap_staging (0.1.61)
  24
+    vcap_staging (0.1.62)
25 25
       nokogiri (>= 1.4.4)
26 26
       rake
27 27
       rspec
@@ -59,7 +59,7 @@ GEM
59 59
       daemons (>= 1.0.9)
60 60
       eventmachine (>= 0.12.6)
61 61
       rack (>= 1.0.0)
62  
-    uuidtools (2.1.2)
  62
+    uuidtools (2.1.3)
63 63
     yajl-ruby (0.8.3)
64 64
 
65 65
 PLATFORMS
3  lib/vcap/staging/plugin/gemfile_support.rb
@@ -30,8 +30,9 @@ def compile_gems
30 30
 
31 31
     app_dir  = File.join(destination_directory, 'app')
32 32
     ruby_cmd = "env -i #{safe_env} #{ruby}"
  33
+    git_cmd = StagingPlugin.platform_config["git_cmd"] || `which git`.strip
33 34
 
34  
-    @task = GemfileTask.new(app_dir, library_version, ruby_cmd, base_dir, @staging_uid, @staging_gid)
  35
+    @task = GemfileTask.new(app_dir, library_version, ruby_cmd, git_cmd, base_dir, @staging_uid, @staging_gid)
35 36
 
36 37
     @task.install
37 38
     @task.install_bundler
135  lib/vcap/staging/plugin/gemfile_task.rb
... ...
@@ -1,10 +1,11 @@
1 1
 require "logger"
2 2
 require "fileutils"
3 3
 require "bundler"
  4
+require "rubygems/installer"
  5
+require "vcap/staging/plugin/gem_cache"
4 6
 
5 7
 class GemfileTask
6  
-
7  
-  def initialize(app_dir, library_version, ruby_cmd, base_dir, uid=nil, gid=nil)
  8
+  def initialize(app_dir, library_version, ruby_cmd, git_cmd, base_dir, uid=nil, gid=nil)
8 9
     @app_dir          = File.expand_path(app_dir)
9 10
     @library_version  = library_version
10 11
     @cache_base_dir   = File.join(base_dir, @library_version)
@@ -12,6 +13,7 @@ def initialize(app_dir, library_version, ruby_cmd, base_dir, uid=nil, gid=nil)
12 13
     FileUtils.mkdir_p(@blessed_gems_dir)
13 14
 
14 15
     @ruby_cmd = ruby_cmd
  16
+    @git_cmd = git_cmd
15 17
     @uid = uid
16 18
     @gid = gid
17 19
 
@@ -25,6 +27,8 @@ def initialize(app_dir, library_version, ruby_cmd, base_dir, uid=nil, gid=nil)
25 27
     @cache = GemCache.new(File.join(@cache_base_dir, "gem_cache"))
26 28
   end
27 29
 
  30
+  attr_reader :git_cmd
  31
+
28 32
   def lockfile_path
29 33
     File.join(@app_dir, "Gemfile.lock")
30 34
   end
@@ -35,6 +39,12 @@ def locked_dependencies
35 39
     @locked = Bundler::LockfileParser.new(lockfile)
36 40
   end
37 41
 
  42
+  def git_gem_specs
  43
+    @git_gem_specs ||= locked_dependencies.specs.select { |s|
  44
+      s.source.is_a? Bundler::Source::Git
  45
+    }
  46
+  end
  47
+
38 48
   # TODO - Inject EM.system-compatible control here.
39 49
   def install
40 50
     install_specs(locked_dependencies.specs)
@@ -48,19 +58,26 @@ def remove_gems_cached_in_app
48 58
   # e.g. ['thin', '1.2.10']
49 59
   def install_gems(gems)
50 60
     gems.each do |(name, version)|
51  
-      install_gem(name, version)
  61
+      install_rubygems_gem(name, version)
52 62
     end
53 63
   end
54 64
 
55 65
   # Each dependency is a Bundler::Spec object
56 66
   def install_specs(specs)
57 67
     specs.each do |spec|
58  
-      install_gem(spec.name, spec.version.version, spec.source)
  68
+      case spec.source
  69
+      when Bundler::Source::Rubygems
  70
+        install_rubygems_gem(spec.name, spec.version.version)
  71
+      when Bundler::Source::Git
  72
+        install_git_gem(spec.name, spec.version.to_s, spec.source)
  73
+      else
  74
+        # TODO: log something
  75
+      end
59 76
     end
60 77
   end
61 78
 
62 79
   def install_bundler
63  
-    install_gem("bundler", "1.0.10")
  80
+    install_rubygems_gem("bundler", "1.0.10")
64 81
   end
65 82
 
66 83
   def install_local_gem(gem_dir, gem_filename, gem_name, gem_version)
@@ -80,7 +97,7 @@ def bundles_gem?(gem_name)
80 97
   end
81 98
 
82 99
   # source is Bundler::Source object, defaults to rubygems
83  
-  def install_gem(name, version, source=nil)
  100
+  def install_rubygems_gem(name, version)
84 101
     gem_filename = gem_filename(name, version)
85 102
 
86 103
     user_gem_path = File.join(@app_dir, "vendor", "cache", gem_filename)
@@ -88,23 +105,101 @@ def install_gem(name, version, source=nil)
88 105
     if File.exists?(user_gem_path)
89 106
       install_gem_from_path(gem_filename, user_gem_path, "user")
90 107
     else
91  
-      if source.kind_of?(Bundler::Source::Git)
92  
-        # Do git stuff
93  
-        raise "Failed installing gem #{gem_filename}: git URLs are not supported"
  108
+      blessed_gem_path = File.join(@blessed_gems_dir, gem_filename)
  109
+      if File.exists?(blessed_gem_path)
  110
+        install_gem_from_path(gem_filename, blessed_gem_path, "blessed")
94 111
       else
95  
-        # assuming Rubygems
96  
-        blessed_gem_path = File.join(@blessed_gems_dir, gem_filename)
97  
-        if File.exists?(blessed_gem_path)
98  
-          install_gem_from_path(gem_filename, blessed_gem_path, "blessed")
99  
-        else
100  
-          @logger.info("Need to fetch #{gem_filename} from RubyGems")
101  
-          Dir.mktmpdir do |tmp_dir|
102  
-            fetched_path = fetch_gem_from_rubygems(name, version, tmp_dir)
103  
-            install_gem_from_path(gem_filename, fetched_path, "fetched")
104  
-            save_blessed_gem(fetched_path)
105  
-          end
  112
+        @logger.info("Need to fetch #{gem_filename} from RubyGems")
  113
+        Dir.mktmpdir do |tmp_dir|
  114
+          fetched_path = fetch_gem_from_rubygems(name, version, tmp_dir)
  115
+          install_gem_from_path(gem_filename, fetched_path, "fetched")
  116
+          save_blessed_gem(fetched_path)
  117
+        end
  118
+      end
  119
+    end
  120
+  end
  121
+
  122
+  # returns a tuple of (dir, gemspec) where dir is the tree hosting the gem
  123
+  # we also assume that the file for gemspec lives directly below dir
  124
+  def git_checkout(tmpdir, uri, revision, gem_name)
  125
+    `#{git_cmd} clone --quiet --no-checkout #{uri} #{tmpdir} && cd #{tmpdir} && #{git_cmd} checkout --quiet #{revision}`
  126
+    if $?.exitstatus != 0
  127
+      raise "Git clone failed"
  128
+    end
  129
+    # FIXME: logger.debug
  130
+    @logger.debug("git revision: %s" % `cd #{tmpdir} && #{git_cmd} rev-parse HEAD`.strip)
  131
+    Dir.glob(File.join(tmpdir, Bundler::Source::Path::DEFAULT_GLOB)).each do |file|
  132
+      # FIXME: ideally we should spawn a new Ruby VM in a clean env
  133
+      # but assuming gemspecs only require files from themselves
  134
+      # clearing $LOAD_PATH seems sufficient
  135
+      gemspec = IO.pipe do |rd, wr|
  136
+        pid = fork do
  137
+          rd.close
  138
+          # duh, people are shelling out in their gemspec
  139
+          ENV["PATH"] = "%s:%s" % [ File.dirname(git_cmd), ENV["PATH"] ]
  140
+          $:.clear
  141
+          gemspec = Bundler.load_gemspec(file)
  142
+          wr.write(gemspec.to_ruby_for_cache)
  143
+          exit!
106 144
         end
  145
+        wr.close
  146
+        spec_as_ruby = rd.read
  147
+        Process.waitpid(pid)
  148
+        eval(spec_as_ruby)
  149
+      end
  150
+      if gemspec && gemspec.name == gem_name
  151
+        # sanitizing the gemspec, removing all dynamism
  152
+        # no more shelling out yo
  153
+        File.open(file, "w") { |f| f.write(gemspec.to_ruby_for_cache) }
  154
+        return [File.dirname(file), gemspec]
  155
+      end
  156
+    end
  157
+    nil
  158
+  end
  159
+
  160
+  # XXX: hax
  161
+  def build_extensions(dir, gemspec)
  162
+    klass = Class.new(Gem::Installer) do
  163
+      def initialize(dir, gemspec)
  164
+        @spec = gemspec
  165
+        @gem_dir = dir
  166
+      end
  167
+    end
  168
+    installer = klass.new(dir, gemspec)
  169
+    installer.build_extensions
  170
+  end
  171
+
  172
+  def git_installation_dir
  173
+    File.join(installation_directory, 'bundler', 'gems')
  174
+  end
  175
+
  176
+  def git_gem_dir(uri, revision)
  177
+    git_scope = "%s-%s" % [ File.basename(uri, '.git'), revision[0, 12] ]
  178
+    File.join(git_installation_dir, git_scope)
  179
+  end
  180
+
  181
+  def copy_git_gem_to_app(dir, uri, revision)
  182
+    raise ArgumentError, [dir,uri,revision].inspect unless dir && uri && revision
  183
+    FileUtils.mkdir_p(git_installation_dir)
  184
+    FileUtils.cp_r(dir, git_gem_dir(uri, revision), :preserve => true)
  185
+  end
  186
+
  187
+  # TODO: cache compilation results
  188
+  def install_git_gem(name, version, source)
  189
+    uri = source.uri
  190
+    revision = source.options["revision"]
  191
+    Dir.mktmpdir do |tmpdir|
  192
+      @logger.info("checking out git repo for #{name} from #{source.options}")
  193
+      checkout_dir, gemspec = git_checkout(
  194
+        tmpdir, uri, revision, name
  195
+      )
  196
+      @logger.info("loaded gemspec: #{gemspec.name}-#{gemspec.version}")
  197
+      unless gemspec.extensions.empty?
  198
+        @logger.info("building extensions for #{gemspec.name}-#{gemspec.version}")
  199
+        build_extensions(checkout_dir, gemspec)
107 200
       end
  201
+      @logger.info("copying git gem #{gemspec.name}-#{gemspec.version} to app")
  202
+      copy_git_gem_to_app(checkout_dir, uri, revision)
108 203
     end
109 204
   end
110 205
 
36  lib/vcap/staging/plugin/git_gem_cache.rb
... ...
@@ -0,0 +1,36 @@
  1
+require "digest/sha1"
  2
+require "fileutils"
  3
+
  4
+class GitGemCache
  5
+
  6
+  def initialize(directory)
  7
+    @directory  = directory
  8
+  end
  9
+
  10
+  def put(revision, gem_name, installed_gem_path)
  11
+    return unless revision && gem_name
  12
+    return unless installed_gem_path && File.directory?(installed_gem_path)
  13
+
  14
+    dst_dir = cached_obj_dir(revision, gem_name)
  15
+
  16
+    # FIXME: use stdlib
  17
+    `cp -a #{installed_gem_path}/* #{dst_dir} && touch #{dst_dir}/.done`
  18
+    return installed_gem_path if $?.exitstatus != 0
  19
+    dst_dir
  20
+  end
  21
+
  22
+  def get(revision, name)
  23
+    return nil unless revision && name
  24
+    dir = cached_obj_dir(revision, name)
  25
+    return nil if !File.exists?(File.join(dir, ".done"))
  26
+    File.directory?(dir) ? dir : nil
  27
+  end
  28
+
  29
+  private
  30
+
  31
+  def cached_obj_dir(revision, name)
  32
+    sha1 = Digest::SHA1.hexdigest("%s %s" % [revision, name])
  33
+    "%s/%s/%s/%s" % [ @directory, sha1[0..1], sha1[2..3], sha1[4..-1] ]
  34
+  end
  35
+
  36
+end
2  lib/vcap/staging/version.rb
... ...
@@ -1,5 +1,5 @@
1 1
 module VCAP
2 2
   module Staging
3  
-    VERSION = '0.1.61'
  3
+    VERSION = '0.1.62'
4 4
   end
5 5
 end
10  spec/fixtures/apps/sinatra_git/native_gem/hello/ext/ext.c
... ...
@@ -0,0 +1,10 @@
  1
+#include <ruby.h>
  2
+
  3
+static VALUE hola(VALUE self) {
  4
+  return rb_str_new2("hola");
  5
+}
  6
+
  7
+void Init_ext() {
  8
+  VALUE klass = rb_define_module("Hello");
  9
+  rb_define_singleton_method(klass, "hola", hola, 0);
  10
+}
3  spec/fixtures/apps/sinatra_git/native_gem/hello/ext/extconf.rb
... ...
@@ -0,0 +1,3 @@
  1
+require "mkmf"
  2
+
  3
+create_makefile("ext")
5  spec/fixtures/apps/sinatra_git/native_gem/hello/hello.gemspec
... ...
@@ -0,0 +1,5 @@
  1
+Gem::Specification.new do |s|
  2
+  s.name = "hello"
  3
+  s.version = "0.0.1"
  4
+  s.extensions = ["ext/extconf.rb"]
  5
+end
3  spec/fixtures/apps/sinatra_git/source/Gemfile
... ...
@@ -0,0 +1,3 @@
  1
+gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => 'e36886a1'
  2
+gem 'eventmachine', :git => 'https://github.com/cloudfoundry/eventmachine.git', :branch => 'release-0.12.11-cf'
  3
+gem "sinatra"
34  spec/fixtures/apps/sinatra_git/source/Gemfile.lock
... ...
@@ -0,0 +1,34 @@
  1
+GIT
  2
+  remote: git://github.com/cloudfoundry/common.git
  3
+  revision: e36886a189b82f880a5aa3e9169712d5d9048a88
  4
+  ref: e36886a1
  5
+  specs:
  6
+    vcap_logging (1.0.1)
  7
+      rake
  8
+
  9
+GIT
  10
+  remote: https://github.com/cloudfoundry/eventmachine.git
  11
+  revision: 2806c630d8631d5dcf9fb2555f665b829052aabe
  12
+  branch: release-0.12.11-cf
  13
+  specs:
  14
+    eventmachine (0.12.11.cloudfoundry.3)
  15
+
  16
+GEM
  17
+  specs:
  18
+    rack (1.4.1)
  19
+    rack-protection (1.2.0)
  20
+      rack
  21
+    rake (0.9.2.2)
  22
+    sinatra (1.3.2)
  23
+      rack (~> 1.3, >= 1.3.6)
  24
+      rack-protection (~> 1.2)
  25
+      tilt (~> 1.3, >= 1.3.3)
  26
+    tilt (1.3.3)
  27
+
  28
+PLATFORMS
  29
+  ruby
  30
+
  31
+DEPENDENCIES
  32
+  eventmachine!
  33
+  sinatra
  34
+  vcap_logging!
5  spec/fixtures/apps/sinatra_git/source/app.rb
... ...
@@ -0,0 +1,5 @@
  1
+require 'sinatra'
  2
+
  3
+get '/hello' do
  4
+  "hello"
  5
+end
14  spec/fixtures/apps/sinatra_git/source/manifest.yml
... ...
@@ -0,0 +1,14 @@
  1
+---
  2
+applications:
  3
+  .:
  4
+    name: frank
  5
+    runtime: ruby19
  6
+    framework:
  7
+      name: sinatra
  8
+      info:
  9
+        mem: 128M
  10
+        description: Sinatra Application
  11
+        exec: ruby app.rb
  12
+    url: ${name}.${target-base}
  13
+    mem: 128M
  14
+    instances: 1
134  spec/unit/gemfile_task_spec.rb
... ...
@@ -0,0 +1,134 @@
  1
+require "fileutils"
  2
+require "tmpdir"
  3
+require "vcap/staging/plugin/gemfile_task"
  4
+
  5
+describe GemfileTask do
  6
+  before :each do
  7
+    @base_dir = Dir.mktmpdir
  8
+    @app_dir_src = File.expand_path("../../fixtures/apps/sinatra_git/source", __FILE__)
  9
+    @app_dir = File.expand_path("source", @base_dir)
  10
+    FileUtils.cp_r(@app_dir_src, @app_dir, :preserve => true)
  11
+
  12
+    # yuck, but better be gross than not testing
  13
+    ruby_cmd = `which ruby`.strip
  14
+    git_cmd = `which git`.strip
  15
+    @task = GemfileTask.new(@app_dir, "1.9.1", ruby_cmd, git_cmd, @base_dir)
  16
+    # The tests will still work without short-circuiting this, but the
  17
+    # turnaround time will become prohibitively long, e.g.  EventMachine
  18
+    # takes 10+ seconds just to compile
  19
+    @task.stub(:build_extensions)
  20
+  end
  21
+
  22
+  after :each do
  23
+    FileUtils.remove_entry_secure(@base_dir)
  24
+  end
  25
+
  26
+  describe "#install" do
  27
+    it "should not download git gems from rubygems" do
  28
+      @task.stub(:install_rubygems_gem) do |name, version|
  29
+        ["vcap_logging", "eventmachine"].should_not include name
  30
+      end
  31
+      @task.stub(:install_git_gem)
  32
+      @task.install
  33
+    end
  34
+
  35
+    it "should find git gems" do
  36
+      @task.locked_dependencies.specs.select { |s|
  37
+        s.source.is_a? Bundler::Source::Git
  38
+      }.map { |s|
  39
+        s.name
  40
+      }.sort.should == ["eventmachine", "vcap_logging"]
  41
+    end
  42
+
  43
+    it "should install git gems" do
  44
+      seen = []
  45
+      @task.stub(:install_rubygems_gem)
  46
+      @task.stub(:install_git_gem) do |name, version, source|
  47
+        seen << [name, version]
  48
+      end
  49
+      @task.install
  50
+      seen.sort.should == [["eventmachine", "0.12.11.cloudfoundry.3"], ["vcap_logging", "1.0.1"]]
  51
+    end
  52
+
  53
+    it "should install git gems in the same location as Bundler" do
  54
+      em_install_path = File.join(@app_dir, "rubygems", "ruby", "1.9.1", "bundler", "gems", "eventmachine-2806c630d863")
  55
+      @task.stub(:install_rubygems_gem)
  56
+      @task.stub(:build_extensions)
  57
+      @task.install
  58
+      File.directory?(em_install_path).should be_true
  59
+    end
  60
+
  61
+    it "should find gems nested in the repo" do
  62
+      logging_gem_install_path = File.join(@app_dir, "rubygems", "ruby", "1.9.1", "bundler", "gems", "common-e36886a189b8")
  63
+      @task.stub(:install_rubygems_gem)
  64
+      @task.stub(:build_extensions)
  65
+      @task.install
  66
+      File.directory?(logging_gem_install_path).should be_true
  67
+      Dir.glob(File.join(logging_gem_install_path, "**/vcap_logging.gemspec")).should_not be_empty
  68
+    end
  69
+  end
  70
+
  71
+  describe "#git_checkout" do
  72
+    it "should check out the right revision" do
  73
+      Dir.mktmpdir do |dir|
  74
+        @task.git_checkout(
  75
+          dir,
  76
+          "git://github.com/cloudfoundry/common.git",
  77
+          "e36886a189b82f880a5aa3e9169712d5d9048a88",
  78
+          "vcap_logging",
  79
+        )
  80
+        File.read(File.join(dir, ".git", "HEAD")).should start_with("e36886a1")
  81
+      end
  82
+    end
  83
+  end
  84
+
  85
+  describe "#build_extensions" do
  86
+    before :each do
  87
+      @gem_checkout_dir = File.expand_path("native_gem", @base_dir)
  88
+      @gem_dir = File.join(@gem_checkout_dir, "hello")
  89
+      FileUtils.cp_r(File.join(@app_dir_src, "../native_gem"), @base_dir, :preserve => true)
  90
+      @native_gemspec = Gem::Specification.new("hello", "0.0.1") do |s|
  91
+        s.extensions = ["ext/extconf.rb"]
  92
+      end
  93
+      @task.unstub(:build_extensions)
  94
+    end
  95
+
  96
+    it "should build native extensions" do
  97
+      @task.build_extensions(@gem_dir, @native_gemspec)
  98
+      expect {
  99
+        require File.join(@gem_dir, "lib/ext")
  100
+        ::Hello.should respond_to(:hola)
  101
+        ::Hello.hola.should == "hola"
  102
+      }.not_to raise_error
  103
+    end
  104
+  end
  105
+
  106
+  describe "#install_git_gem" do
  107
+    before :each do
  108
+      @gem_checkout_dir = File.expand_path("native_gem", @base_dir)
  109
+      @gem_dir = File.join(@gem_checkout_dir, "hello")
  110
+      FileUtils.cp_r(File.join(@app_dir_src, "../native_gem"), @base_dir, :preserve => true)
  111
+      @native_gemspec = Gem::Specification.new("hello", "0.0.1") do |s|
  112
+        s.extensions = ["ext/extconf.rb"]
  113
+      end
  114
+      @task.stub(:git_checkout).and_return([@gem_dir, @native_gemspec])
  115
+      @task.unstub(:build_extensions)
  116
+    end
  117
+
  118
+    it "should call #build_extensions" do
  119
+      @task.should_receive(:build_extensions).with(@gem_dir, @native_gemspec)
  120
+      @task.stub(:copy_git_gem_to_app)
  121
+      @task.install_git_gem("hello", "0.0.1", double.as_null_object)
  122
+    end
  123
+
  124
+    it "should build native extensions" do
  125
+      @task.stub(:copy_git_gem_to_app)
  126
+      @task.install_git_gem("hello", "0.0.1", double.as_null_object)
  127
+      expect {
  128
+        require File.join(@gem_dir, "lib/ext")
  129
+        ::Hello.should respond_to(:hola)
  130
+        ::Hello.hola.should == "hola"
  131
+      }.not_to raise_error
  132
+    end
  133
+  end
  134
+end

Git Notes

review

Code-Review+2: Matt Page <mpage@rbcon.com>
Verified+1: CI Master <cf-ci@rbcon.com>
Submitted-by: Jesse Zhang <jessezhang@vmware.com>
Submitted-at: Thu, 26 Jul 2012 23:12:56 +0000
Reviewed-on: http://reviews.cloudfoundry.org/7284
Project: vcap-staging
Branch: refs/heads/git-support

0 notes on commit 8815bbb

Please sign in to comment.
Something went wrong with that request. Please try again.