Permalink
Browse files

Git stager PoC

  This change adds a naive implementation for Gemfiles containing Git
URLs.

N.B.
  For this to work, platform.yml in BOSH release needs to be configured
to point to the location of the Git binary.

  Test plan:
    - Unit tests passed
    - BVTs passed
    - CCNG with git Gemfile worked.

Change-Id: Ia8ef1a1ad0b4fd0ad65bdecd5be882382b824f1b
  • Loading branch information...
1 parent 743dd69 commit 8815bbb63cc0a29cac2446751f428baa463b1101 @d d committed Jul 12, 2012
View
4 Gemfile.lock
@@ -21,7 +21,7 @@ GIT
PATH
remote: .
specs:
- vcap_staging (0.1.61)
+ vcap_staging (0.1.62)
nokogiri (>= 1.4.4)
rake
rspec
@@ -59,7 +59,7 @@ GEM
daemons (>= 1.0.9)
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
- uuidtools (2.1.2)
+ uuidtools (2.1.3)
yajl-ruby (0.8.3)
PLATFORMS
View
3 lib/vcap/staging/plugin/gemfile_support.rb
@@ -30,8 +30,9 @@ def compile_gems
app_dir = File.join(destination_directory, 'app')
ruby_cmd = "env -i #{safe_env} #{ruby}"
+ git_cmd = StagingPlugin.platform_config["git_cmd"] || `which git`.strip
- @task = GemfileTask.new(app_dir, library_version, ruby_cmd, base_dir, @staging_uid, @staging_gid)
+ @task = GemfileTask.new(app_dir, library_version, ruby_cmd, git_cmd, base_dir, @staging_uid, @staging_gid)
@task.install
@task.install_bundler
View
135 lib/vcap/staging/plugin/gemfile_task.rb
@@ -1,17 +1,19 @@
require "logger"
require "fileutils"
require "bundler"
+require "rubygems/installer"
+require "vcap/staging/plugin/gem_cache"
class GemfileTask
-
- def initialize(app_dir, library_version, ruby_cmd, base_dir, uid=nil, gid=nil)
+ def initialize(app_dir, library_version, ruby_cmd, git_cmd, base_dir, uid=nil, gid=nil)
@app_dir = File.expand_path(app_dir)
@library_version = library_version
@cache_base_dir = File.join(base_dir, @library_version)
@blessed_gems_dir = File.join(@cache_base_dir, "blessed_gems")
FileUtils.mkdir_p(@blessed_gems_dir)
@ruby_cmd = ruby_cmd
+ @git_cmd = git_cmd
@uid = uid
@gid = gid
@@ -25,6 +27,8 @@ def initialize(app_dir, library_version, ruby_cmd, base_dir, uid=nil, gid=nil)
@cache = GemCache.new(File.join(@cache_base_dir, "gem_cache"))
end
+ attr_reader :git_cmd
+
def lockfile_path
File.join(@app_dir, "Gemfile.lock")
end
@@ -35,6 +39,12 @@ def locked_dependencies
@locked = Bundler::LockfileParser.new(lockfile)
end
+ def git_gem_specs
+ @git_gem_specs ||= locked_dependencies.specs.select { |s|
+ s.source.is_a? Bundler::Source::Git
+ }
+ end
+
# TODO - Inject EM.system-compatible control here.
def install
install_specs(locked_dependencies.specs)
@@ -48,19 +58,26 @@ def remove_gems_cached_in_app
# e.g. ['thin', '1.2.10']
def install_gems(gems)
gems.each do |(name, version)|
- install_gem(name, version)
+ install_rubygems_gem(name, version)
end
end
# Each dependency is a Bundler::Spec object
def install_specs(specs)
specs.each do |spec|
- install_gem(spec.name, spec.version.version, spec.source)
+ case spec.source
+ when Bundler::Source::Rubygems
+ install_rubygems_gem(spec.name, spec.version.version)
+ when Bundler::Source::Git
+ install_git_gem(spec.name, spec.version.to_s, spec.source)
+ else
+ # TODO: log something
+ end
end
end
def install_bundler
- install_gem("bundler", "1.0.10")
+ install_rubygems_gem("bundler", "1.0.10")
end
def install_local_gem(gem_dir, gem_filename, gem_name, gem_version)
@@ -80,31 +97,109 @@ def bundles_gem?(gem_name)
end
# source is Bundler::Source object, defaults to rubygems
- def install_gem(name, version, source=nil)
+ def install_rubygems_gem(name, version)
gem_filename = gem_filename(name, version)
user_gem_path = File.join(@app_dir, "vendor", "cache", gem_filename)
if File.exists?(user_gem_path)
install_gem_from_path(gem_filename, user_gem_path, "user")
else
- if source.kind_of?(Bundler::Source::Git)
- # Do git stuff
- raise "Failed installing gem #{gem_filename}: git URLs are not supported"
+ blessed_gem_path = File.join(@blessed_gems_dir, gem_filename)
+ if File.exists?(blessed_gem_path)
+ install_gem_from_path(gem_filename, blessed_gem_path, "blessed")
else
- # assuming Rubygems
- blessed_gem_path = File.join(@blessed_gems_dir, gem_filename)
- if File.exists?(blessed_gem_path)
- install_gem_from_path(gem_filename, blessed_gem_path, "blessed")
- else
- @logger.info("Need to fetch #{gem_filename} from RubyGems")
- Dir.mktmpdir do |tmp_dir|
- fetched_path = fetch_gem_from_rubygems(name, version, tmp_dir)
- install_gem_from_path(gem_filename, fetched_path, "fetched")
- save_blessed_gem(fetched_path)
- end
+ @logger.info("Need to fetch #{gem_filename} from RubyGems")
+ Dir.mktmpdir do |tmp_dir|
+ fetched_path = fetch_gem_from_rubygems(name, version, tmp_dir)
+ install_gem_from_path(gem_filename, fetched_path, "fetched")
+ save_blessed_gem(fetched_path)
+ end
+ end
+ end
+ end
+
+ # returns a tuple of (dir, gemspec) where dir is the tree hosting the gem
+ # we also assume that the file for gemspec lives directly below dir
+ def git_checkout(tmpdir, uri, revision, gem_name)
+ `#{git_cmd} clone --quiet --no-checkout #{uri} #{tmpdir} && cd #{tmpdir} && #{git_cmd} checkout --quiet #{revision}`
+ if $?.exitstatus != 0
+ raise "Git clone failed"
+ end
+ # FIXME: logger.debug
+ @logger.debug("git revision: %s" % `cd #{tmpdir} && #{git_cmd} rev-parse HEAD`.strip)
+ Dir.glob(File.join(tmpdir, Bundler::Source::Path::DEFAULT_GLOB)).each do |file|
+ # FIXME: ideally we should spawn a new Ruby VM in a clean env
+ # but assuming gemspecs only require files from themselves
+ # clearing $LOAD_PATH seems sufficient
+ gemspec = IO.pipe do |rd, wr|
+ pid = fork do
+ rd.close
+ # duh, people are shelling out in their gemspec
+ ENV["PATH"] = "%s:%s" % [ File.dirname(git_cmd), ENV["PATH"] ]
+ $:.clear
+ gemspec = Bundler.load_gemspec(file)
+ wr.write(gemspec.to_ruby_for_cache)
+ exit!
end
+ wr.close
+ spec_as_ruby = rd.read
+ Process.waitpid(pid)
+ eval(spec_as_ruby)
+ end
+ if gemspec && gemspec.name == gem_name
+ # sanitizing the gemspec, removing all dynamism
+ # no more shelling out yo
+ File.open(file, "w") { |f| f.write(gemspec.to_ruby_for_cache) }
+ return [File.dirname(file), gemspec]
+ end
+ end
+ nil
+ end
+
+ # XXX: hax
+ def build_extensions(dir, gemspec)
+ klass = Class.new(Gem::Installer) do
+ def initialize(dir, gemspec)
+ @spec = gemspec
+ @gem_dir = dir
+ end
+ end
+ installer = klass.new(dir, gemspec)
+ installer.build_extensions
+ end
+
+ def git_installation_dir
+ File.join(installation_directory, 'bundler', 'gems')
+ end
+
+ def git_gem_dir(uri, revision)
+ git_scope = "%s-%s" % [ File.basename(uri, '.git'), revision[0, 12] ]
+ File.join(git_installation_dir, git_scope)
+ end
+
+ def copy_git_gem_to_app(dir, uri, revision)
+ raise ArgumentError, [dir,uri,revision].inspect unless dir && uri && revision
+ FileUtils.mkdir_p(git_installation_dir)
+ FileUtils.cp_r(dir, git_gem_dir(uri, revision), :preserve => true)
+ end
+
+ # TODO: cache compilation results
+ def install_git_gem(name, version, source)
+ uri = source.uri
+ revision = source.options["revision"]
+ Dir.mktmpdir do |tmpdir|
+ @logger.info("checking out git repo for #{name} from #{source.options}")
+ checkout_dir, gemspec = git_checkout(
+ tmpdir, uri, revision, name
+ )
+ @logger.info("loaded gemspec: #{gemspec.name}-#{gemspec.version}")
+ unless gemspec.extensions.empty?
+ @logger.info("building extensions for #{gemspec.name}-#{gemspec.version}")
+ build_extensions(checkout_dir, gemspec)
end
+ @logger.info("copying git gem #{gemspec.name}-#{gemspec.version} to app")
+ copy_git_gem_to_app(checkout_dir, uri, revision)
end
end
View
36 lib/vcap/staging/plugin/git_gem_cache.rb
@@ -0,0 +1,36 @@
+require "digest/sha1"
+require "fileutils"
+
+class GitGemCache
+
+ def initialize(directory)
+ @directory = directory
+ end
+
+ def put(revision, gem_name, installed_gem_path)
+ return unless revision && gem_name
+ return unless installed_gem_path && File.directory?(installed_gem_path)
+
+ dst_dir = cached_obj_dir(revision, gem_name)
+
+ # FIXME: use stdlib
+ `cp -a #{installed_gem_path}/* #{dst_dir} && touch #{dst_dir}/.done`
+ return installed_gem_path if $?.exitstatus != 0
+ dst_dir
+ end
+
+ def get(revision, name)
+ return nil unless revision && name
+ dir = cached_obj_dir(revision, name)
+ return nil if !File.exists?(File.join(dir, ".done"))
+ File.directory?(dir) ? dir : nil
+ end
+
+ private
+
+ def cached_obj_dir(revision, name)
+ sha1 = Digest::SHA1.hexdigest("%s %s" % [revision, name])
+ "%s/%s/%s/%s" % [ @directory, sha1[0..1], sha1[2..3], sha1[4..-1] ]
+ end
+
+end
View
2 lib/vcap/staging/version.rb
@@ -1,5 +1,5 @@
module VCAP
module Staging
- VERSION = '0.1.61'
+ VERSION = '0.1.62'
end
end
View
10 spec/fixtures/apps/sinatra_git/native_gem/hello/ext/ext.c
@@ -0,0 +1,10 @@
+#include <ruby.h>
+
+static VALUE hola(VALUE self) {
+ return rb_str_new2("hola");
+}
+
+void Init_ext() {
+ VALUE klass = rb_define_module("Hello");
+ rb_define_singleton_method(klass, "hola", hola, 0);
+}
View
3 spec/fixtures/apps/sinatra_git/native_gem/hello/ext/extconf.rb
@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile("ext")
View
5 spec/fixtures/apps/sinatra_git/native_gem/hello/hello.gemspec
@@ -0,0 +1,5 @@
+Gem::Specification.new do |s|
+ s.name = "hello"
+ s.version = "0.0.1"
+ s.extensions = ["ext/extconf.rb"]
+end
View
3 spec/fixtures/apps/sinatra_git/source/Gemfile
@@ -0,0 +1,3 @@
+gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => 'e36886a1'
+gem 'eventmachine', :git => 'https://github.com/cloudfoundry/eventmachine.git', :branch => 'release-0.12.11-cf'
+gem "sinatra"
View
34 spec/fixtures/apps/sinatra_git/source/Gemfile.lock
@@ -0,0 +1,34 @@
+GIT
+ remote: git://github.com/cloudfoundry/common.git
+ revision: e36886a189b82f880a5aa3e9169712d5d9048a88
+ ref: e36886a1
+ specs:
+ vcap_logging (1.0.1)
+ rake
+
+GIT
+ remote: https://github.com/cloudfoundry/eventmachine.git
+ revision: 2806c630d8631d5dcf9fb2555f665b829052aabe
+ branch: release-0.12.11-cf
+ specs:
+ eventmachine (0.12.11.cloudfoundry.3)
+
+GEM
+ specs:
+ rack (1.4.1)
+ rack-protection (1.2.0)
+ rack
+ rake (0.9.2.2)
+ sinatra (1.3.2)
+ rack (~> 1.3, >= 1.3.6)
+ rack-protection (~> 1.2)
+ tilt (~> 1.3, >= 1.3.3)
+ tilt (1.3.3)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ eventmachine!
+ sinatra
+ vcap_logging!
View
5 spec/fixtures/apps/sinatra_git/source/app.rb
@@ -0,0 +1,5 @@
+require 'sinatra'
+
+get '/hello' do
+ "hello"
+end
View
14 spec/fixtures/apps/sinatra_git/source/manifest.yml
@@ -0,0 +1,14 @@
+---
+applications:
+ .:
+ name: frank
+ runtime: ruby19
+ framework:
+ name: sinatra
+ info:
+ mem: 128M
+ description: Sinatra Application
+ exec: ruby app.rb
+ url: ${name}.${target-base}
+ mem: 128M
+ instances: 1
View
134 spec/unit/gemfile_task_spec.rb
@@ -0,0 +1,134 @@
+require "fileutils"
+require "tmpdir"
+require "vcap/staging/plugin/gemfile_task"
+
+describe GemfileTask do
+ before :each do
+ @base_dir = Dir.mktmpdir
+ @app_dir_src = File.expand_path("../../fixtures/apps/sinatra_git/source", __FILE__)
+ @app_dir = File.expand_path("source", @base_dir)
+ FileUtils.cp_r(@app_dir_src, @app_dir, :preserve => true)
+
+ # yuck, but better be gross than not testing
+ ruby_cmd = `which ruby`.strip
+ git_cmd = `which git`.strip
+ @task = GemfileTask.new(@app_dir, "1.9.1", ruby_cmd, git_cmd, @base_dir)
+ # The tests will still work without short-circuiting this, but the
+ # turnaround time will become prohibitively long, e.g. EventMachine
+ # takes 10+ seconds just to compile
+ @task.stub(:build_extensions)
+ end
+
+ after :each do
+ FileUtils.remove_entry_secure(@base_dir)
+ end
+
+ describe "#install" do
+ it "should not download git gems from rubygems" do
+ @task.stub(:install_rubygems_gem) do |name, version|
+ ["vcap_logging", "eventmachine"].should_not include name
+ end
+ @task.stub(:install_git_gem)
+ @task.install
+ end
+
+ it "should find git gems" do
+ @task.locked_dependencies.specs.select { |s|
+ s.source.is_a? Bundler::Source::Git
+ }.map { |s|
+ s.name
+ }.sort.should == ["eventmachine", "vcap_logging"]
+ end
+
+ it "should install git gems" do
+ seen = []
+ @task.stub(:install_rubygems_gem)
+ @task.stub(:install_git_gem) do |name, version, source|
+ seen << [name, version]
+ end
+ @task.install
+ seen.sort.should == [["eventmachine", "0.12.11.cloudfoundry.3"], ["vcap_logging", "1.0.1"]]
+ end
+
+ it "should install git gems in the same location as Bundler" do
+ em_install_path = File.join(@app_dir, "rubygems", "ruby", "1.9.1", "bundler", "gems", "eventmachine-2806c630d863")
+ @task.stub(:install_rubygems_gem)
+ @task.stub(:build_extensions)
+ @task.install
+ File.directory?(em_install_path).should be_true
+ end
+
+ it "should find gems nested in the repo" do
+ logging_gem_install_path = File.join(@app_dir, "rubygems", "ruby", "1.9.1", "bundler", "gems", "common-e36886a189b8")
+ @task.stub(:install_rubygems_gem)
+ @task.stub(:build_extensions)
+ @task.install
+ File.directory?(logging_gem_install_path).should be_true
+ Dir.glob(File.join(logging_gem_install_path, "**/vcap_logging.gemspec")).should_not be_empty
+ end
+ end
+
+ describe "#git_checkout" do
+ it "should check out the right revision" do
+ Dir.mktmpdir do |dir|
+ @task.git_checkout(
+ dir,
+ "git://github.com/cloudfoundry/common.git",
+ "e36886a189b82f880a5aa3e9169712d5d9048a88",
+ "vcap_logging",
+ )
+ File.read(File.join(dir, ".git", "HEAD")).should start_with("e36886a1")
+ end
+ end
+ end
+
+ describe "#build_extensions" do
+ before :each do
+ @gem_checkout_dir = File.expand_path("native_gem", @base_dir)
+ @gem_dir = File.join(@gem_checkout_dir, "hello")
+ FileUtils.cp_r(File.join(@app_dir_src, "../native_gem"), @base_dir, :preserve => true)
+ @native_gemspec = Gem::Specification.new("hello", "0.0.1") do |s|
+ s.extensions = ["ext/extconf.rb"]
+ end
+ @task.unstub(:build_extensions)
+ end
+
+ it "should build native extensions" do
+ @task.build_extensions(@gem_dir, @native_gemspec)
+ expect {
+ require File.join(@gem_dir, "lib/ext")
+ ::Hello.should respond_to(:hola)
+ ::Hello.hola.should == "hola"
+ }.not_to raise_error
+ end
+ end
+
+ describe "#install_git_gem" do
+ before :each do
+ @gem_checkout_dir = File.expand_path("native_gem", @base_dir)
+ @gem_dir = File.join(@gem_checkout_dir, "hello")
+ FileUtils.cp_r(File.join(@app_dir_src, "../native_gem"), @base_dir, :preserve => true)
+ @native_gemspec = Gem::Specification.new("hello", "0.0.1") do |s|
+ s.extensions = ["ext/extconf.rb"]
+ end
+ @task.stub(:git_checkout).and_return([@gem_dir, @native_gemspec])
+ @task.unstub(:build_extensions)
+ end
+
+ it "should call #build_extensions" do
+ @task.should_receive(:build_extensions).with(@gem_dir, @native_gemspec)
+ @task.stub(:copy_git_gem_to_app)
+ @task.install_git_gem("hello", "0.0.1", double.as_null_object)
+ end
+
+ it "should build native extensions" do
+ @task.stub(:copy_git_gem_to_app)
+ @task.install_git_gem("hello", "0.0.1", double.as_null_object)
+ expect {
+ require File.join(@gem_dir, "lib/ext")
+ ::Hello.should respond_to(:hola)
+ ::Hello.hola.should == "hola"
+ }.not_to raise_error
+ end
+ end
+end

0 comments on commit 8815bbb

Please sign in to comment.