Add client gem for communicating with stagers

This provides a gem that provides a small rpc interface to stagers.
Although this is a bit of yak shaving, it does clean up the interaction
between the CC and Stagers and paves the way for adding support for allowing
staging plugins to perform actions on behalf of users.

Test plan:
- All existing unit tests pass
- New stager_ipc unit tests pass
- BVTs pass

Change-Id: I18d059cbc044fd628680dbb1bfd8428b4ced0c16

cloud_controller/Gemfile

@@ -14,7 +14,7 @@ gem 'vcap_staging', '= 0.1.22'
 
 # For queuing staging tasks
 gem 'em-hiredis'
-gem 'vcap_stager', '= 0.1.3'
+gem 'vcap_stager_ipc'
 
 # Databases
 gem 'sqlite3'

cloud_controller/Gemfile.lock

@@ -124,7 +124,7 @@ GEM
     ruby-hmac (0.4.0)
     sinatra (1.2.1)
       rack (~> 1.1)
-      tilt (< 2.0, >= 1.2.2)
+      tilt (>= 1.2.2, < 2.0)
     sqlite3 (1.3.3)
     thin (1.2.11)
       daemons (>= 1.0.9)
@@ -137,8 +137,13 @@ GEM
     tzinfo (0.3.26)
     uuidtools (2.1.2)
     vcap_logging (0.1.0)
-    vcap_stager (0.1.3)
-    vcap_staging (0.1.22)
+    vcap_stager_ipc (0.0.1)
+      eventmachine
+      rake
+      vcap_common
+      vcap_logging
+      yajl-ruby
+    vcap_staging (0.1.11)
       nokogiri (>= 1.4.4)
       rake
       rspec
@@ -176,6 +181,6 @@ DEPENDENCIES
   uuidtools
   vcap_common!
   vcap_logging
-  vcap_stager (= 0.1.3)
-  vcap_staging (= 0.1.22)
+  vcap_stager_ipc
+  vcap_staging (= 0.1.11)
   yajl-ruby (>= 0.7.9)

cloud_controller/app/controllers/apps_controller.rb

@@ -1,4 +1,4 @@
-require 'staging_task_manager'
+require 'vcap/stager/ipc'
 
 class AppsController < ApplicationController
   before_filter :require_user, :except => [:download_staged]
@@ -212,25 +212,33 @@ def files
   private
 
   def stage_app(app)
-    task_mgr = StagingTaskManager.new(:logger  => CloudController.logger,
-                                      :timeout => AppConfig[:staging][:max_staging_runtime])
     dl_uri = StagingController.download_app_uri(app)
     ul_hdl = StagingController.create_upload(app)
+    stager_client = VCAP::Stager::Ipc::FiberedNatsClient.new(NATS.client)
 
-    result = task_mgr.run_staging_task(app, dl_uri, ul_hdl.upload_uri)
+    begin
+      result = stager_client.add_task(app.id,
+                                      app.staging_task_properties,
+                                      dl_uri,
+                                      ul_hdl.upload_uri,
+                                      AppConfig[:staging][:max_staging_runtime])
+      StagingTaskLog.new(app.id, result['task_log']).save
+    rescue VCAP::Stager::Ipc::RequestTimeoutError
+      raise CloudError.new(CloudError::APP_STAGING_ERROR, "Timed out waiting for reply from stager")
+    end
 
     # Update run count to be consistent with previous staging code
-    if result.was_success?
+    unless result['error']
       CloudController.logger.debug("Staging task for app_id=#{app.id} succeded.", :tags => [:staging])
-      CloudController.logger.debug1("Details: #{result.task_log}", :tags => [:staging])
+      CloudController.logger.debug1("Details: #{result['task_log']}", :tags => [:staging])
       app.update_staged_package(ul_hdl.upload_path)
       app.package_state = 'STAGED'
       app.update_run_count()
     else
-      CloudController.logger.warn("Staging task for app_id=#{app.id} failed: #{result.error}",
+      CloudController.logger.warn("Staging task for app_id=#{app.id} failed: #{result['error']}",
                                   :tags => [:staging])
-      CloudController.logger.debug1("Details: #{result.task_log}", :tags => [:staging])
-      raise CloudError.new(CloudError::APP_STAGING_ERROR, result.error.to_s)
+      CloudController.logger.debug1("Details: #{result['task_log']}", :tags => [:staging])
+      raise CloudError.new(CloudError::APP_STAGING_ERROR, result['error'])
     end
 
   rescue => e

cloud_controller/app/models/app_manager.rb

@@ -1,5 +1,3 @@
-require 'vcap/stager'
-
 class AppManager
   attr_reader :app
 

cloud_controller/config/cloud_controller.yml

@@ -96,7 +96,7 @@ staging:
   max_staging_runtime:    120 # secs
   # Create a secure environment for staging
   secure: false
-  new_stager_percent: 0
+  new_stager_percent: 100
   auth:
     user: zxsfhgjg
     password: ZNVfdase9

cloud_controller/spec/models/service_spec.rb

@@ -116,6 +116,7 @@
   describe "#is_builtin?" do
     it "should correctly check against AppConfig" do
       svc = Service.new(:label => "foo-bar", :url => "http://www.google.com", :token => "foo")
+      AppConfig[:builtin_services].delete(:foo)
       svc.is_builtin?.should be_false
       AppConfig[:builtin_services][:foo] = true
       svc.is_builtin?.should be_true

stager/Gemfile

@@ -9,7 +9,8 @@ gem 'yajl-ruby', '>= 0.7.9'
 
 gem 'vcap_common'
 gem 'vcap_logging', '>= 0.1.1'
-gem 'vcap_staging', '>= 0.1.22'
+gem 'vcap_staging', '>= 0.1.11'
+gem 'vcap_stager_ipc'
 
 group :test do
   gem 'rspec'

stager/Gemfile.lock

@@ -45,7 +45,13 @@ GEM
       thin
       yajl-ruby
     vcap_logging (0.1.1)
-    vcap_staging (0.1.22)
+    vcap_stager_ipc (0.0.1)
+      eventmachine
+      rake
+      vcap_common
+      vcap_logging
+      yajl-ruby
+    vcap_staging (0.1.11)
       nokogiri (>= 1.4.4)
       rake
       rspec
@@ -69,6 +75,7 @@ DEPENDENCIES
   sinatra
   vcap_common
   vcap_logging (>= 0.1.1)
-  vcap_staging (>= 0.1.22)
+  vcap_stager_ipc
+  vcap_staging (>= 0.1.11)
   webmock
   yajl-ruby (>= 0.7.9)

stager/Rakefile

@@ -1,36 +1,7 @@
 require 'rubygems'
 require 'rake'
-require 'rake/gempackagetask'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
-require 'vcap/stager/version'
-
-GEM_NAME    = 'vcap_stager'
-GEM_VERSION = VCAP::Stager::VERSION
-
-gemspec = Gem::Specification.new do |s|
-  s.name         = GEM_NAME
-  s.version      = GEM_VERSION
-  s.platform     = Gem::Platform::RUBY
-  s.summary      = 'Component responsible for staging apps'
-  s.description  = 'Takes an app package, environment, and services' \
-                   + ' and produces a droplet that is executable by the DEA'
-  s.authors      = ['Matt Page']
-  s.email        = 'mpage@vmware.com'
-  s.homepage     = 'http://www.cloudfoundry.com'
-  s.executables  = []
-  s.bindir       = 'bin'
-  s.require_path = 'lib'
-  s.files        = %w(Rakefile Gemfile) + Dir.glob("{lib,spec,vendor}/**/*")
-end
-
-Rake::GemPackageTask.new(gemspec) do |pkg|
-  pkg.gem_spec = gemspec
-end
-
-task :install => [:package] do
-  sh "gem install --no-ri --no-rdoc pkg/#{GEM_NAME}-#{GEM_VERSION}"
-end
 
 task :spec => ['bundler:install:test'] do
   desc 'Run tests'
@@ -42,11 +13,7 @@ namespace 'bundler' do
     sh('bundle install')
   end
 
-  environments = %w(test development production)
-  environments.each do |env|
-    desc "Install gems for #{env}"
-    task "install:#{env}" do
-      sh("bundle install --local --without #{(environments - [env]).join(' ')}")
-    end
+  task 'install:test' do
+    sh('bundle install --local --without development production')
   end
 end

stager/config/dev.yml

@@ -5,5 +5,5 @@ pid_filename: /var/vcap/sys/run/stager.pid
 nats_uri: nats://127.0.0.1:4222
 max_staging_duration: 120
 max_active_tasks: 10
-queues: ['staging']
+queues: ['vcap.staging.linux']
 secure: false

stager/lib/vcap/stager.rb

@@ -9,7 +9,6 @@
 require 'vcap/stager/task_error'
 require 'vcap/stager/task_logger'
 require 'vcap/stager/task_manager'
-require 'vcap/stager/task_result'
 require 'vcap/stager/server'
 require 'vcap/stager/util'
 require 'vcap/stager/version'