Improve process for adding runtimes and frameworks

- Remove manifest processing in favor of receiving runtime and framework info from CC Change-Id: I77d59b40498c5dd15d98e30f7ba5450a24b1e20f
hsume2 · Sep 5, 2012 · 54c9f26 · 54c9f26
1 parent 3903fca
commit 54c9f26
Show file tree

Hide file tree

Showing 13 changed files with 28 additions and 67 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
 pkg
+vendor/cache
diff --git a/Gemfile b/Gemfile
@@ -7,7 +7,7 @@ gem 'yajl-ruby', '>= 0.7.9'
 
 gem 'vcap_common', '2.0.0', :git => 'https://github.com/cloudfoundry/vcap-common.git', :ref => '5334b662'
 gem 'vcap_logging', '>= 0.1.3', :git => 'https://github.com/cloudfoundry/common.git', :ref => 'b96ec119'
-gem 'vcap_staging', '~> 0.1.62', :git => 'https://github.com/cloudfoundry/vcap-staging.git', :ref => '8d9da6cd'
+gem 'vcap_staging', '~> 0.1.63', :git => 'https://github.com/cloudfoundry/vcap-staging.git', :ref => '1740e4da'
 gem 'vcap-concurrency', '~> 0.1.0', :git => 'https://github.com/cloudfoundry/vcap-concurrency.git', :ref => 'be2bdb9f'
 gem 'stager-client', '~> 0.0.2', :git => 'https://github.com/cloudfoundry/stager-client.git', :ref => 'fdaae720'
 

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -45,10 +45,10 @@ GIT
 
 GIT
   remote: https://github.com/cloudfoundry/vcap-staging.git
-  revision: 8d9da6cd9bb641225a12fcbcbf7a1db9ab06e844
-  ref: 8d9da6cd
+  revision: 1740e4daef7cb988f16375a4ec7b69392c5705ac
+  ref: 1740e4da
   specs:
-    vcap_staging (0.1.62)
+    vcap_staging (0.1.63)
       nokogiri (>= 1.4.4)
       rake
       rspec
@@ -112,6 +112,6 @@ DEPENDENCIES
   vcap-concurrency (~> 0.1.0)!
   vcap_common (= 2.0.0)!
   vcap_logging (>= 0.1.3)!
-  vcap_staging (~> 0.1.62)!
+  vcap_staging (~> 0.1.63)!
   webmock
   yajl-ruby (>= 0.7.9)
diff --git a/lib/vcap/stager.rb b/lib/vcap/stager.rb
@@ -18,9 +18,6 @@ class << self
       def init(config)
         @config = config
         VCAP::Logging.setup_from_config(config[:logging])
-        StagingPlugin.manifest_root = config[:dirs][:manifests]
-        StagingPlugin.load_all_manifests
-        StagingPlugin.validate_configuration!
       end
     end
   end

diff --git a/lib/vcap/stager/config.rb b/lib/vcap/stager/config.rb
@@ -24,7 +24,6 @@ class VCAP::Stager::Config < VCAP::Config
       :queues                => [String],   # List of queues to pull tasks from
       :pid_filename          => String,     # Pid filename to use
       optional(:dirs) => {
-        optional(:manifests) => String,     # Where all of the staging manifests live
         optional(:tmp)       => String,     # Default is /tmp
       },
 
@@ -41,7 +40,6 @@ def self.from_file(*args)
     config = super(*args)
 
     config[:dirs] ||= {}
-    config[:dirs][:manifests] ||= StagingPlugin::DEFAULT_MANIFEST_ROOT
     config[:run_plugin_path]  ||= File.expand_path('../../../../bin/run_plugin', __FILE__)
     config[:ruby_path]        ||= `which ruby`.chomp
 

diff --git a/lib/vcap/stager/server.rb b/lib/vcap/stager/server.rb
@@ -157,11 +157,6 @@ def create_task_config(server_config, user_manager)
       :run_plugin_path => server_config[:run_plugin_path],
       :secure_user_manager => user_manager,
     }
-
-    if server_config[:dirs]
-      task_config[:manifest_root] = server_config[:dirs][:manifests]
-    end
-
     task_config
   end
 end
diff --git a/lib/vcap/stager/task.rb b/lib/vcap/stager/task.rb
@@ -29,7 +29,6 @@ def initialize(request, opts = {})
     @request      = request
     @user_manager = opts[:secure_user_manager]
     @runner       = opts[:runner] || VCAP::Stager::ProcessRunner.new(@logger)
-    @manifest_dir = opts[:manifest_root] || StagingPlugin::DEFAULT_MANIFEST_ROOT
     @ruby_path    = opts[:ruby_path] || "ruby"
     @run_plugin_path = opts[:run_plugin_path] || RUN_PLUGIN_PATH
     @max_staging_duration = opts[:max_staging_duration] || MAX_STAGING_DURATION
@@ -110,8 +109,7 @@ def stage_app(src_dir, dst_dir, task_logger)
     plugin_config = {
       "source_dir"   => src_dir,
       "dest_dir"     => dst_dir,
-      "environment"  => @request["properties"],
-      "manifest_dir" => @manifest_dir,
+      "environment"  => @request["properties"]
     }
 
     secure_user = nil
@@ -128,7 +126,7 @@ def stage_app(src_dir, dst_dir, task_logger)
     StagingPlugin::Config.to_file(plugin_config, plugin_config_file.path)
 
     cmd = [@ruby_path, @run_plugin_path,
-           @request["properties"]["framework"],
+           @request["properties"]["framework_info"]["name"],
            plugin_config_file.path].join(" ")
 
     res = @runner.run_logged(cmd,

diff --git a/spec/fixtures/stager_config.yml.erb b/spec/fixtures/stager_config.yml.erb
@@ -2,8 +2,6 @@
 logging:
   level: debug2
 nats_uri: nats://127.0.0.1:<%= nats_port %>
-dirs:
-  manifests: <%= manifest_dir %>
 max_staging_duration: 120
 max_active_tasks: 10
 secure: false

diff --git a/spec/functional/stager_spec.rb b/spec/functional/stager_spec.rb
@@ -8,9 +8,12 @@
     @task_timeout = ENV['VCAP_TEST_TASK_TIMEOUT'] || 10
     # Set this to true if you want to save the output of each component
     @save_logs = ENV['VCAP_TEST_LOG'] == 'true'
+    ruby18_version =  ENV['VCAP_RUNTIME_RUBY18_VER'] || '1.8.7'
+    ruby18_exec =  ENV['VCAP_RUNTIME_RUBY18'] || '/usr/bin/ruby'
     @app_props = {
-      'framework'   => 'sinatra',
-      'runtime'     => 'ruby18',
+      'framework_info' => {'name' => "sinatra", 'runtimes' => ['ruby18' => {'default' => true}],
+                        'detection' => [{'*.rb' => "require 'sinatra'|require \"sinatra\""}]},
+      'runtime_info'   => {'name' => 'ruby18', 'version' => ruby18_version, 'executable'=> ruby18_exec},
       'services'    => [{}],
       'resources'   => {
         'memory'    => 128,
@@ -26,7 +29,6 @@
     @http_server  = start_http_server(@tmp_dirs[:http], @tmp_dirs)
     @nats_server  = start_nats(@tmp_dirs[:nats])
     @stager       = start_stager(@nats_server.port,
-                                 StagingPlugin.manifest_root,
                                  @tmp_dirs[:stager])
   end
 
@@ -77,8 +79,8 @@ def start_nats(nats_dir)
     nats
   end
 
-  def start_stager(nats_port, manifest_dir, stager_dir)
-    stager = VCAP::Stager::Spec::ForkedStager.new(nats_port, manifest_dir, stager_dir)
+  def start_stager(nats_port, stager_dir)
+    stager = VCAP::Stager::Spec::ForkedStager.new(nats_port, stager_dir)
     ready = false
     NATS.start(:uri => "nats://127.0.0.1:#{nats_port}") do
       EM.add_timer(30) { EM.stop }

diff --git a/spec/functional/task_spec.rb b/spec/functional/task_spec.rb
@@ -37,8 +37,9 @@
 
       # Framework/runtime mismatch. Web.xml will not be found
       request = create_request(app_name,
-                               "framework" => "spring",
-                               "runtime"   => "java")
+                               "framework_info" => {"name" => "spring", "runtimes" => ["java" => {"default" => true}],
+                                               "detection" => [{"*.war" => true}]},
+                               "runtime_info"   => {"name" => "java", "version" => "1.6", "executable"=> "java"})
 
       expect_error(request, /Staging plugin failed/)
     end
@@ -74,11 +75,14 @@ def expect_error(request, matcher)
   end
 
   def create_request(app_name, app_props = {})
+    ruby18_version =  ENV["VCAP_RUNTIME_RUBY18_VER"] || "1.8.7"
+    ruby18_exec =  ENV["VCAP_RUNTIME_RUBY18"] || "/usr/bin/ruby"
     { "download_uri" => DummyHandler.app_download_uri(@http_server, app_name),
       "upload_uri" => DummyHandler.droplet_upload_uri(@http_server, app_name),
       "properties" => {
-        "framework" => "sinatra",
-        "runtime"   => "ruby18",
+        "framework_info" => {"name" => "sinatra", "runtimes" => ["ruby18" => {"default" => true}],
+                        "detection" => [{"*.rb" => "require 'sinatra'|require \"sinatra\""}]},
+        "runtime_info"   => {"name" => "ruby18", "version" => ruby18_version, "executable"=> ruby18_exec},
         "services"  => [{}],
         "resources" => {
           "memory" => 128,

diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
@@ -15,49 +15,19 @@
 # Created as needed, removed at the end of the spec run.
 # Allows us to override staging paths.
 STAGING_TEMP = Dir.mktmpdir
-StagingPlugin.manifest_root = STAGING_TEMP
 
 VCAP::Logging.setup_from_config({:level => :debug2}) if ENV['VCAP_TEST_LOG'] == 'true'
 
 RSpec.configure do |config|
   config.before(:all) do
-    begin
-      VCAP::Subprocess.run("cp -a #{File.join(StagingPlugin::DEFAULT_MANIFEST_ROOT, 'sinatra.yml')} #{STAGING_TEMP}")
-      if ENV["VCAP_RUNTIME_RUBY18"] && ENV["VCAP_RUNTIME_RUBY18_VER"] then
-        sinatra_manifest = {
-          'name' => "sinatra",
-          'runtimes' => [
-            'ruby18' => {
-              'version' => ENV["VCAP_RUNTIME_RUBY18_VER"],
-              'executable' => ENV["VCAP_RUNTIME_RUBY18"],
-            },
-          ],
-          'detection' => [
-            {'*.rb' => "require 'sinatra'|require \"sinatra\""},
-          ],
-        }
-        File.open(File.join(STAGING_TEMP, "sinatra.yml"), "w") do |file|
-          YAML.dump sinatra_manifest, file
-        end
-      end
-    rescue VCAP::SubprocessStatusError => e
-      puts "Unable to copy staging manifests. Permissions problem?"
-      puts "#{e}"
-      puts "STDOUT:"
-      puts e.stdout
-      puts "STDERR"
-      puts e.stderr
-    rescue => e
-      puts "Unable to copy staging manifests. Permissions problem?"
-      puts "#{e}"
-    end
     File.open(File.join(STAGING_TEMP, 'platform.yml'), 'wb') do |f|
       # XXX - This is better than putting the gem cache in one's home
       # dir, but still not ideal. Having the gem cache around greatly
       # speeds up tests (mostly due to rails).
       cache_dir = File.join('/tmp', '.vcap_gems')
       f.print YAML.dump('cache' => cache_dir)
     end
+    ENV['PLATFORM_CONFIG'] = File.join(STAGING_TEMP, 'platform.yml')
   end
 end
 

diff --git a/spec/support/app_fixture.rb b/spec/support/app_fixture.rb
@@ -26,8 +26,8 @@ def initialize(name)
   # You must manually clean up the directory thus created.
   # If called with a block, yields the staged directory as a Pathname, and
   # automatically deletes it when the block returns.
-  def stage(framework, env = {})
-    plugin_klass = StagingPlugin.load_plugin_for(framework)
+  def stage(env = {})
+    plugin_klass = StagingPlugin.load_plugin_for(env[:framework_info]["name"])
     working_dir = Dir.mktmpdir("#{@name}-staged")
     stager = plugin_klass.new(source_dir, working_dir, env)
     stager.stage_application

diff --git a/spec/support/forked_stager.rb b/spec/support/forked_stager.rb
@@ -15,11 +15,10 @@ class VCAP::Stager::Spec::ForkedStager < VCAP::Spec::ForkedComponent::Base
   CONF_TEMPLATE = File.expand_path('../../fixtures/stager_config.yml.erb', __FILE__)
   STAGER_PATH   = File.expand_path('../../../bin/stager', __FILE__)
 
-  attr_reader :log_dir, :nats_port, :manifest_dir, :pid_filename
+  attr_reader :log_dir, :nats_port, :pid_filename
 
-  def initialize(nats_port, manifest_dir, log_dir='/tmp', keep_logs=false)
+  def initialize(nats_port, log_dir='/tmp', keep_logs=false)
     @nats_port    = nats_port
-    @manifest_dir = manifest_dir
     @log_dir      = log_dir
     @conf_path    = File.join(@log_dir, 'stager.conf')
     @pid_filename = File.join(@log_dir, 'stager.pid')
@@ -44,7 +43,6 @@ def write_conf(filename)
     # Wish I could pass these to ERB instead of it reaching into the current scope
     # Investigate using something like liquid templates...
     nats_port    = @nats_port
-    manifest_dir = @manifest_dir
     pid_filename = @pid_filename
     conf = template.result(binding())
     File.open(filename, 'w+') {|f| f.write(conf) }