Browse files

Merge branch 'master' into services-r5

Change-Id: I46d25577471fb46f80db51b64b8f5bf46939379b
  • Loading branch information...
2 parents 202a551 + 4d56fed commit 218c19de187d0d094e45d1357bbdbd64fafa8a7e felixhoo committed Sep 18, 2011
Showing with 2,371 additions and 636 deletions.
  1. +3 −0 bin/config/redis-server.conf
  2. +4 −0 bin/stager
  3. +57 −2 bin/vcap
  4. +5 −2 cloud_controller/Gemfile
  5. +16 −2 cloud_controller/Gemfile.lock
  6. +74 −4 cloud_controller/app/controllers/apps_controller.rb
  7. +137 −0 cloud_controller/app/controllers/staging_controller.rb
  8. +22 −0 cloud_controller/app/models/app.rb
  9. +2 −0 cloud_controller/app/models/app_manager.rb
  10. +59 −0 cloud_controller/app/models/staging_task_log.rb
  11. +11 −0 cloud_controller/app/models/user.rb
  12. +9 −1 cloud_controller/config/appconfig.rb
  13. +10 −1 cloud_controller/config/cloud_controller.yml
  14. +1 −0 cloud_controller/config/final_stage/activate.rb
  15. +12 −0 cloud_controller/config/final_stage/redis.rb
  16. +4 −0 cloud_controller/config/routes.rb
  17. +5 −0 cloud_controller/lib/cloud_error.rb
  18. +58 −0 cloud_controller/lib/staging_task_manager.rb
  19. 0 cloud_controller/public/favicon.ico
  20. +74 −0 cloud_controller/spec/controllers/staging_controller_spec.rb
  21. +20 −0 cloud_controller/spec/models/app_spec.rb
  22. +47 −0 cloud_controller/spec/models/staging_task_log_spec.rb
  23. +33 −0 cloud_controller/spec/models/user_spec.rb
  24. +46 −0 cloud_controller/spec/staging/staging_task_manager_spec.rb
  25. BIN cloud_controller/vendor/cache/em-hiredis-0.1.0.gem
  26. BIN cloud_controller/vendor/cache/hiredis-0.3.2.gem
  27. BIN cloud_controller/vendor/cache/rest-client-1.6.7.gem
  28. BIN cloud_controller/vendor/cache/vcap_common-0.99.gem
  29. BIN cloud_controller/vendor/cache/vcap_stager-0.1.3.gem
  30. BIN stager/vendor/cache/vcap_staging-0.1.4.gem → cloud_controller/vendor/cache/vcap_staging-0.1.7.gem
  31. +3 −3 common/Gemfile.lock
  32. +10 −0 common/lib/vcap/json_schema.rb
  33. +1 −1 common/lib/vcap/spec/forked_component/nats_server.rb
  34. +20 −0 common/spec/unit/json_schema_spec.rb
  35. BIN common/vcap_common-0.99.gem
  36. +1 −1 java
  37. +22 −0 setup/cc_proxy.nginx.conf
  38. +73 −0 setup/mime.types
  39. +5 −4 stager/Gemfile
  40. +24 −29 stager/Gemfile.lock
  41. +1 −1 stager/Rakefile
  42. +11 −0 stager/bin/create_secure_users.rb
  43. +29 −0 stager/bin/download_app
  44. +11 −29 stager/bin/stager
  45. +29 −0 stager/bin/upload_droplet
  46. +4 −4 stager/config/dev.yml
  47. +1 −0 stager/config/stager.yml
  48. +6 −16 stager/lib/vcap/stager.rb
  49. +5 −13 stager/lib/vcap/stager/config.rb
  50. +6 −0 stager/lib/vcap/stager/constants.rb
  51. +0 −8 stager/lib/vcap/stager/errors.rb
  52. +119 −0 stager/lib/vcap/stager/secure_user_manager.rb
  53. +112 −0 stager/lib/vcap/stager/server.rb
  54. +215 −128 stager/lib/vcap/stager/task.rb
  55. +59 −0 stager/lib/vcap/stager/task_error.rb
  56. +94 −0 stager/lib/vcap/stager/task_manager.rb
  57. +22 −33 stager/lib/vcap/stager/task_result.rb
  58. +74 −41 stager/lib/vcap/stager/util.rb
  59. +1 −1 stager/lib/vcap/stager/version.rb
  60. +0 −7 stager/spec/fixtures/apps/rails3_gitgems/source/config/initializers/backtrace_silencers.rb
  61. +2 −3 stager/spec/fixtures/stager_config.yml.erb
  62. +21 −32 stager/spec/functional/stager_spec.rb
  63. +3 −2 stager/spec/spec_helper.rb
  64. +0 −47 stager/spec/support/forked_redis_server.rb
  65. +2 −8 stager/spec/support/forked_stager.rb
  66. +37 −0 stager/spec/unit/task_error_spec.rb
  67. +74 −0 stager/spec/unit/task_manager_spec.rb
  68. +18 −36 stager/spec/unit/task_result_spec.rb
  69. +98 −88 stager/spec/unit/task_spec.rb
  70. +62 −16 stager/spec/unit/util_spec.rb
  71. BIN stager/vendor/cache/json-1.5.3.gem
  72. BIN stager/vendor/cache/mime-types-1.16.gem
  73. BIN stager/vendor/cache/nokogiri-1.5.0.gem
  74. BIN stager/vendor/cache/redis-2.2.1.gem
  75. BIN stager/vendor/cache/redis-namespace-1.0.3.gem
  76. BIN stager/vendor/cache/resque-1.17.1.gem
  77. BIN stager/vendor/cache/rest-client-1.6.7.gem
  78. BIN stager/vendor/cache/vcap_common-0.99.gem
  79. BIN cloud_controller/vendor/cache/vcap_staging-0.1.4.gem → stager/vendor/cache/vcap_staging-0.1.7.gem
  80. BIN stager/vendor/cache/vegas-0.1.8.gem
  81. BIN stager/vendor/cache/webmock-1.6.4.gem
  82. BIN stager/vendor/cache/webmock-1.7.4.gem
  83. BIN stager/vendor/cache/yajl-ruby-0.8.2.gem
  84. BIN stager/vendor/cache/yajl-ruby-0.8.3.gem
  85. +1 −9 staging/Gemfile
  86. +16 −8 staging/Gemfile.lock
  87. +2 −24 staging/Rakefile
  88. +10 −0 staging/lib/vcap/staging/plugin/config.rb
  89. +15 −1 staging/lib/vcap/staging/plugin/gemfile_task.rb
  90. +6 −0 staging/lib/vcap/staging/plugin/java_web/plugin.rb
  91. BIN ...va_web/resources/{auto-reconfiguration-0.6.0-BUILD-SNAPSHOT.jar → auto-reconfiguration-0.6.0.jar}
  92. BIN staging/lib/vcap/staging/plugin/java_web/resources/mysql-connector-java-5.1.12-bin.jar
  93. BIN staging/lib/vcap/staging/plugin/java_web/resources/postgresql-9.0-801.jdbc4.jar
  94. BIN staging/lib/vcap/staging/plugin/java_web/resources/tomcat.zip
  95. +41 −3 staging/lib/vcap/staging/plugin/java_web/tomcat.rb
  96. +0 −5 staging/lib/vcap/staging/plugin/node/stage
  97. +0 −7 staging/lib/vcap/staging/plugin/php/stage
  98. +5 −0 staging/lib/vcap/staging/plugin/spring/autostaging_template_spring.xml
  99. +0 −5 staging/lib/vcap/staging/plugin/wsgi/stage
  100. +1 −1 staging/lib/vcap/staging/version.rb
  101. BIN staging/spec/fixtures/apps/spring_context_initializer_foo/source.war
  102. +3 −1 staging/spec/support/staging_spec_helpers.rb
  103. +42 −0 staging/spec/unit/config_spec.rb
  104. +9 −0 staging/spec/unit/grails_spec.rb
  105. +97 −0 staging/spec/unit/java_web_spec.rb
  106. +20 −0 staging/spec/unit/lift_spec.rb
  107. +91 −8 staging/spec/unit/spring_spec.rb
  108. BIN staging/vcap_staging-0.1.7.gem
  109. +27 −0 staging/vcap_staging.gemspec
  110. BIN staging/vendor/cache/json_pure-1.5.3.gem
  111. BIN staging/vendor/cache/json_pure-1.5.4.gem
  112. BIN staging/vendor/cache/logging-1.5.2.gem
  113. BIN staging/vendor/cache/logging-1.6.1.gem
  114. BIN staging/vendor/cache/spruz-0.2.13.gem
  115. BIN staging/vendor/cache/yajl-ruby-0.8.2.gem
  116. BIN staging/vendor/cache/yajl-ruby-0.8.3.gem
  117. +1 −1 tests
View
3 bin/config/redis-server.conf
@@ -0,0 +1,3 @@
+bind 0.0.0.0
+port 5454
+loglevel debug
View
4 bin/stager
@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+# Copyright (c) 2009-2011 VMware, Inc.
+
+exec(File.expand_path("../../stager/bin/stager", __FILE__), *ARGV)
View
59 bin/vcap
@@ -79,7 +79,7 @@ class Component
end
def pid_file
- configuration["pid"] || raise("#{@configuration_path} does not specify location of pid file")
+ configuration["pid"] || configuration['pid_filename'] || raise("#{@configuration_path} does not specify location of pid file")
end
def log_file?
@@ -244,10 +244,64 @@ class NatsServer
end
end
+class RedisServer
+ DEFAULT_CONFIG_PATH = File.expand_path('../config/redis-server.conf', __FILE__)
+
+ def initialize(pid_filename, config_path=DEFAULT_CONFIG_PATH)
+ @config_path = config_path
+ @pid_filename = pid_filename
+ @redis_path = `which redis-server`.chomp
+ unless $? == 0
+ STDERR.puts "Could not find redis-server, exiting.".red
+ exit 1
+ end
+ @pid = read_pidfile
+ end
+
+ def start
+ return if running?
+ @pid = fork do
+ log_file = File.join(TMP, 'redis-server.log')
+ stdout = File.open(log_file, 'a')
+ STDOUT.reopen(stdout)
+ stderr = File.open(log_file, 'a')
+ STDERR.reopen(stderr)
+ exec("#{@redis_path} #{@config_path}")
+ end
+ Process.detach(@pid)
+ File.open(@pid_filename, 'w+') {|f| f.write(@pid) }
+ end
+
+ def stop
+ return unless running?
+ Process.kill('TERM', @pid)
+ @pid = nil
+ end
+
+ private
+
+ def read_pidfile
+ if File.exist?(@pid_filename)
+ File.read(@pid_filename).chomp.to_i
+ else
+ nil
+ end
+ end
+
+ def running?
+ return false unless @pid
+ File.exist?(File.join('/proc', @pid.to_s))
+ end
+end
+
+
module Run
+ DEFAULT_REDIS_PIDFILE = File.join(TMP, 'redis-server.pid')
+
def self.start_init
nats_server = NatsServer.new
nats_server.start_server
+ RedisServer.new(DEFAULT_REDIS_PIDFILE).start
end
def self.start(args)
@@ -259,6 +313,7 @@ module Run
# Only process this if no one else running..
running_components = components([]).select {|c| c.running?}.map{|c| c.name }
return unless running_components.empty?
+ RedisServer.new(DEFAULT_REDIS_PIDFILE).stop
nats_server = NatsServer.new
return unless nats_server.is_running?
nats_server.kill_server
@@ -380,7 +435,7 @@ module Run
private
def self.core
- %w(router cloud_controller dea health_manager)
+ %w(router cloud_controller dea health_manager stager)
end
def self.services
View
7 cloud_controller/Gemfile
@@ -10,9 +10,11 @@ gem 'logging', '>= 1.5.0'
# VCAP common components
gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :path => '../common'
gem 'vcap_logging', :require => ['vcap/logging']
+gem 'vcap_staging', '= 0.1.7'
-# XXX - Vendor once working
-gem 'vcap_staging'
+# For queuing staging tasks
+gem 'em-hiredis'
+gem 'vcap_stager', '= 0.1.3'
# Databases
gem 'sqlite3'
@@ -45,6 +47,7 @@ gem 'bcrypt-ruby', '>= 2.1.4'
gem 'ruby-hmac', :require => 'hmac-sha1'
gem 'SystemTimer', :platforms => :mri_18
gem 'uuidtools'
+gem 'rest-client', '= 1.6.7'
# rspec-rails is outside the 'test' group in order to consistently provide Rake tasks.
gem 'rspec-rails', '>= 2.4.1'
View
18 cloud_controller/Gemfile.lock
@@ -48,6 +48,8 @@ GEM
builder (>= 2.1.2)
daemons (1.1.2)
diff-lcs (1.1.2)
+ em-hiredis (0.1.0)
+ hiredis (~> 0.3.0)
em-http-request (1.0.0.beta.3)
addressable (>= 2.2.3)
em-socksify
@@ -60,6 +62,7 @@ GEM
erubis (2.6.6)
abstract (>= 1.0.0)
eventmachine (0.12.10)
+ hiredis (0.3.2)
http_parser.rb (0.5.1)
i18n (0.5.0)
json_pure (1.5.1)
@@ -103,6 +106,8 @@ GEM
thor (~> 0.14.4)
rake (0.8.7)
rcov (0.9.9)
+ rest-client (1.6.7)
+ mime-types (>= 1.16)
rspec (2.5.0)
rspec-core (~> 2.5.0)
rspec-expectations (~> 2.5.0)
@@ -132,7 +137,13 @@ GEM
tzinfo (0.3.26)
uuidtools (2.1.2)
vcap_logging (0.1.0)
- vcap_staging (0.1.4)
+ vcap_stager (0.1.3)
+ vcap_staging (0.1.7)
+ nokogiri (>= 1.4.4)
+ rake
+ rspec
+ vcap_common
+ yajl-ruby (>= 0.7.9)
yajl-ruby (0.8.2)
PLATFORMS
@@ -142,6 +153,7 @@ DEPENDENCIES
SystemTimer
bcrypt-ruby (>= 2.1.4)
ci_reporter
+ em-hiredis
em-http-request (~> 1.0.0.beta.3)
em-redis
eventmachine (~> 0.12.10)
@@ -154,6 +166,7 @@ DEPENDENCIES
rack-fiber_pool
rails (~> 3.0.5)
rcov
+ rest-client (= 1.6.7)
rspec (>= 2.4.0)
rspec-rails (>= 2.4.1)
ruby-hmac
@@ -163,5 +176,6 @@ DEPENDENCIES
uuidtools
vcap_common!
vcap_logging
- vcap_staging
+ vcap_stager (= 0.1.3)
+ vcap_staging (= 0.1.7)
yajl-ruby (>= 0.7.9)
View
78 cloud_controller/app/controllers/apps_controller.rb
@@ -1,3 +1,5 @@
+require 'staging_task_manager'
+
class AppsController < ApplicationController
before_filter :require_user, :except => [:download_staged]
before_filter :find_app_by_name, :except => [:create, :list, :download_staged]
@@ -83,7 +85,7 @@ def upload
end
def download
- path = @app.package_path
+ path = @app.unstaged_package_path
if path && File.exists?(path)
send_file path
else
@@ -124,7 +126,13 @@ def start_update
error_on_lock_mismatch(@app)
@app.lock_version += 1
manager = AppManager.new(@app)
- manager.stage if @app.needs_staging?
+ if @app.needs_staging?
+ if user.uses_new_stager?
+ stage_app(@app)
+ else
+ manager.stage
+ end
+ end
manager.stop_all
manager.started
render :nothing => true, :status => 204
@@ -164,6 +172,18 @@ def check_update
# GET /apps/:name/instances/:instance_id/files/:path'
def files
+ # XXX - Yuck. This will have to do until we update VMC with a real
+ # way to fetch staging logs.
+ if user.uses_new_stager? && (params[:path] == 'logs/staging.log')
+ log = StagingTaskLog.fetch_fibered(@app.id)
+ if log
+ render :text => log.task_log
+ else
+ render :nothing => true, :status => 404
+ end
+ return
+ end
+
# will Fiber.yield
url, auth = AppManager.new(@app).get_file_url(params[:instance_id], params[:path])
raise CloudError.new(CloudError::APP_FILE_ERROR, params[:path] || '/') unless url
@@ -191,6 +211,48 @@ def files
private
+ def stage_app(app)
+ task_mgr = StagingTaskManager.new(:logger => CloudController.logger,
+ :timeout => AppConfig[:staging][:max_staging_runtime])
+ dl_uri = StagingController.download_app_uri(app)
+ ul_hdl = StagingController.create_upload(app)
+
+ result = task_mgr.run_staging_task(app, dl_uri, ul_hdl.upload_uri)
+
+ # Update run count to be consistent with previous staging code
+ if result.was_success?
+ CloudController.logger.debug("Staging task for app_id=#{app.id} succeded.", :tags => [:staging])
+ CloudController.logger.debug1("Details: #{result.task_log}", :tags => [:staging])
+ app.update_staged_package(ul_hdl.upload_path)
+ app.package_state = 'STAGED'
+ app.update_run_count()
+ else
+ CloudController.logger.warn("Staging task for app_id=#{app.id} failed: #{result.error}",
+ :tags => [:staging])
+ CloudController.logger.debug1("Details: #{result.task_log}", :tags => [:staging])
+ raise CloudError.new(CloudError::APP_STAGING_ERROR, result.error.to_s)
+ end
+
+ rescue => e
+ # It may be the case that upload from the stager will happen sometime in the future.
+ # Mark the upload as completed so that any upload that occurs in the future will fail.
+ if ul_hdl
+ StagingController.complete_upload(ul_hdl)
+ FileUtils.rm_f(ul_hdl.upload_path)
+ end
+ # This is in keeping with the old CC behavior. Instead of starting a single
+ # instance of a broken app (which is effectively stopped after HM flapping logic
+ # is triggered) we stop it explicitly.
+ app.state = 'STOPPED'
+ AppManager.new(app).stopped
+ app.package_state = 'FAILED'
+ app.update_run_count()
+ raise e
+
+ ensure
+ app.save!
+ end
+
def find_app_by_name
# XXX - What do we want semantics to be like for multiple apps w/ same name (possible w/ contribs)
@app = user.apps_owned.find_by_name(params[:name])
@@ -208,7 +270,6 @@ def find_app_by_name
# App from the request params and makes the necessary AppManager calls.
def update_app_from_params(app)
CloudController.logger.debug "app: #{app.id || "nil"} update_from_parms"
-
error_on_lock_mismatch(app)
app.lock_version += 1
@@ -247,7 +308,14 @@ def update_app_from_params(app)
# Process any changes that require action on out part here.
manager = AppManager.new(app)
- manager.stage if app.needs_staging?
+
+ if app.needs_staging?
+ if user.uses_new_stager?
+ stage_app(app)
+ else
+ manager.stage
+ end
+ end
if changed.include?('state')
if app.stopped?
@@ -405,4 +473,6 @@ def check_has_capacity_for?(app, previous_state)
raise CloudError.new(CloudError::ACCOUNT_NOT_ENOUGH_MEMORY, "#{mem_quota}M")
end
end
+
+
end
View
137 cloud_controller/app/controllers/staging_controller.rb
@@ -0,0 +1,137 @@
+require 'uri'
+
+# Handles app downloads and droplet uploads from the stagers.
+#
+class StagingController < ApplicationController
+ skip_before_filter :fetch_user_from_token
+ before_filter :authenticate_stager
+
+ class DropletUploadHandle
+ attr_reader :upload_id, :upload_path, :upload_uri, :app
+
+ def initialize(app)
+ @app = app
+ @upload_id = VCAP.secure_uuid
+ @upload_path = File.join(AppConfig[:directories][:tmpdir],
+ "staged_upload_#{app.id}_#{@upload_id}.tgz")
+ @upload_uri = StagingController.upload_droplet_uri(app, @upload_id)
+ end
+ end
+
+ class << self
+ def upload_droplet_uri(app, upload_id)
+ staging_uri("/staging/droplet/#{app.id}/#{upload_id}")
+ end
+
+ def download_app_uri(app)
+ staging_uri("/staging/app/#{app.id}")
+ end
+
+ def create_upload(app)
+ @uploads ||= {}
+ ret = DropletUploadHandle.new(app)
+ @uploads[ret.upload_id] = ret
+ ret
+ end
+
+ def lookup_upload(upload_id)
+ @uploads ||= {}
+ @uploads[upload_id]
+ end
+
+ def complete_upload(handle)
+ return unless @uploads
+ @uploads.delete(handle.upload_id)
+ end
+
+ private
+
+ def staging_uri(path)
+ uri = URI::HTTP.build(
+ :host => CloudController.bind_address,
+ :port => CloudController.external_port,
+ :userinfo => [AppConfig[:staging][:auth][:user], AppConfig[:staging][:auth][:password]],
+ :path => path
+ )
+ uri.to_s
+ end
+ end
+
+ # Handles a droplet upload from a stager
+ def upload_droplet
+ upload = nil
+ src_path = nil
+ app = App.find_by_id(params[:id])
+ raise CloudError.new(CloudError::APP_NOT_FOUND) unless app
+
+ upload = self.class.lookup_upload(params[:upload_id])
+ unless upload
+ CloudController.logger.error("No upload set for upload_id=#{params[:upload_id]}")
+ raise CloudError.new(CloudError::BAD_REQUEST)
+ end
+
+ if CloudController.use_nginx
+ src_path = params[:droplet_path]
+ else
+ src_path = params[:upload][:droplet].path
+ end
+ unless src_path && File.exist?(src_path)
+ CloudController.logger.error("Uploaded droplet not found at '#{src_path}'")
+ raise CloudError.new(CloudError::BAD_REQUEST)
+ end
+
+ begin
+ CloudController.logger.debug("Renaming staged droplet from '#{src_path}' to '#{upload.upload_path}'")
+ File.rename(src_path, upload.upload_path)
+ rescue => e
+ CloudController.logger.error("Failed uploading staged droplet: #{e}", :tags => [:staging])
+ CloudController.logger.error(e)
+ FileUtils.rm_f(upload.upload_path)
+ raise e
+ end
+ CloudController.logger.debug("Stager (#{request.remote_ip}) uploaded droplet to #{upload.upload_path}",
+ :tags => [:staging])
+ render :nothing => true, :status => 200
+ ensure
+ FileUtils.rm_f(src_path) if src_path
+ self.class.complete_upload(upload) if upload
+ end
+
+ # Handles an app download from a stager
+ def download_app
+ app = App.find_by_id(params[:id])
+ raise CloudError.new(CloudError::APP_NOT_FOUND) unless app
+
+ path = app.unstaged_package_path
+ unless path && File.exists?(path)
+ CloudController.logger.error("Couldn't find package path for app_id=#{app.id} (stager=#{request.remote_ip})", :tags => [:staging])
+ raise CloudError.new(CloudError::APP_NOT_FOUND)
+ end
+ CloudController.logger.debug("Stager (#{request.remote_ip}) requested app_id=#{app.id} @ path=#{path}", :tags => [:staging])
+
+ if path && File.exists?(path)
+ if CloudController.use_nginx
+ response.headers['X-Accel-Redirect'] = '/droplets/' + File.basename(path)
+ render :nothing => true, :status => 200
+ else
+ send_file path
+ end
+ else
+ raise CloudError.new(CloudError::APP_NOT_FOUND)
+ end
+ end
+
+ private
+
+ def authenticate_stager
+ authenticate_or_request_with_http_basic do |user, pass|
+ if (user == AppConfig[:staging][:auth][:user]) && (pass == AppConfig[:staging][:auth][:password])
+ true
+ else
+ CloudController.logger.error("Stager auth failed (user=#{user}, pass=#{pass} from #{request.remote_ip}", :tags => [:auth_failure, :staging])
+ false
+ end
+ end
+ end
+
+end
View
22 cloud_controller/app/models/app.rb
@@ -123,6 +123,15 @@ def staging_environment_data
:resources => resource_requirements }
end
+ def staging_task_properties
+ services = service_bindings(true).map {|sb| sb.for_staging}
+ { :services => services,
+ :framework => framework,
+ :runtime => runtime,
+ :resources => resource_requirements,
+ :environment => environment}
+ end
+
# Returns an array of the URLs that point to this application
def mapped_urls
routes.active.map {|r| r.url}.sort
@@ -511,6 +520,19 @@ def remove_collaborator(user)
collab.destroy if collab
end
+ def update_staged_package(upload_path)
+ self.staged_package_hash = Digest::SHA1.file(upload_path).hexdigest
+ FileUtils.mv(upload_path, self.staged_package_path)
+ end
+
+ def update_run_count
+ if self.staged_package_hash_changed?
+ self.run_count = 0 # reset
+ else
+ self.run_count += 1
+ end
+ end
+
private
# TODO - Remove this when the VMC client has been updated to match our new strings.
View
2 cloud_controller/app/models/app_manager.rb
@@ -1,3 +1,5 @@
+require 'vcap/stager'
+
class AppManager
attr_reader :app
View
59 cloud_controller/app/models/staging_task_log.rb
@@ -0,0 +1,59 @@
+class StagingTaskLog
+ class << self
+ attr_accessor :redis
+
+ def key_for_id(app_id)
+ "staging_task_log:#{app_id}"
+ end
+
+ def fetch(app_id, redis=nil)
+ redis ||= @redis
+ key = key_for_id(app_id)
+ result = redis.get(key)
+ result ? StagingTaskLog.new(app_id, result) : nil
+ end
+
+ def fetch_fibered(app_id, timeout=5, redis=nil)
+ redis ||= @redis
+ f = Fiber.current
+ key = key_for_id(app_id)
+ logger = VCAP::Logging.logger('vcap.stager.task_result.fetch_fibered')
+
+ logger.debug("Fetching result for key '#{key}' from redis")
+
+ get_def = redis.get(key)
+ get_def.timeout(timeout)
+ get_def.errback do |e|
+ e = VCAP::Stager::TaskResultTimeoutError.new("Timed out fetching result") if e == nil
+ logger.error("Failed fetching result for key '#{key}': #{e}")
+ logger.error(e)
+ f.resume([false, e])
+ end
+ get_def.callback do |result|
+ logger.debug("Fetched result for key '#{key}' => '#{result}'")
+ f.resume([true, result])
+ end
+
+ was_success, result = Fiber.yield
+
+ if was_success
+ result ? StagingTaskLog.new(app_id, result) : nil
+ else
+ raise result
+ end
+ end
+ end
+
+ attr_reader :app_id, :task_log
+
+ def initialize(app_id, task_log)
+ @app_id = app_id
+ @task_log = task_log
+ end
+
+ def save(redis=nil)
+ redis ||= self.class.redis
+ key = self.class.key_for_id(@app_id)
+ redis.set(key, @task_log)
+ end
+end
View
11 cloud_controller/app/models/user.rb
@@ -106,4 +106,15 @@ def no_more_apps?
count
end
end
+
+
+ def uses_new_stager?(cfg=AppConfig)
+ stg = cfg[:staging]
+ if (stg[:new_stager_percent] && ((self.id % 100) < stg[:new_stager_percent])) \
+ || (stg[:new_stager_email_regexp] && stg[:new_stager_email_regexp].match(self.email))
+ true
+ else
+ false
+ end
+ end
end
View
10 cloud_controller/config/appconfig.rb
@@ -2,7 +2,6 @@
# Once we know which Rails environment we are in, we can fail fast in production
# mode by checking that flag. This code runs too early to know for sure if
# we are starting in production mode.
-
require 'vcap/common'
require 'vcap/staging/plugin/common'
@@ -179,3 +178,12 @@
$stderr.puts "The supplied password is too short (#{pw_len} bytes), must be at least #{c.key_len} bytes. (Though only the first #{c.key_len} will be used.)"
exit 1
end
+
+if AppConfig[:staging][:new_stager_email_regexp]
+ AppConfig[:staging][:new_stager_email_regexp] = Regexp.new(AppConfig[:staging][:new_stager_email_regexp])
+end
+
+if (AppConfig[:staging][:new_stager_percent] || AppConfig[:staging][:new_stager_email_regexp]) && !AppConfig[:redis]
+ $stderr.puts "You must supply a redis config to use the new stager"
+ exit 1
+end
View
11 cloud_controller/config/cloud_controller.yml
@@ -39,14 +39,19 @@ nginx:
# if log_file is set, it must be a fully-qualified path,
# because the bin/vcap script reads it directly from the file.
logging:
- level: debug
+ level: debug2
# file:
# Settings for the rails logger
rails_logging:
level: debug
# file:
+redis:
+ host: 127.0.0.1
+ port: 5454
+# password:
+
directories:
droplets: /var/vcap/shared/droplets
resources: /var/vcap/shared/resources
@@ -91,6 +96,10 @@ staging:
max_staging_runtime: 120 # secs
# Create a secure environment for staging
secure: false
+ new_stager_percent: 0
+ auth:
+ user: zxsfhgjg
+ password: ZNVfdase9
allow_debug: false
View
1 cloud_controller/config/final_stage/activate.rb
@@ -5,5 +5,6 @@
require dir.join('event_log')
require dir.join('check_database')
require dir.join('message_bus')
+require dir.join('redis')
require dir.join('log_boot_completion')
View
12 cloud_controller/config/final_stage/redis.rb
@@ -0,0 +1,12 @@
+if AppConfig[:redis]
+ EM::Hiredis.logger = CloudController.logger
+
+ # This will be run once the event loop has started
+ EM.next_tick do
+ redis_client = EM::Hiredis::Client.new(AppConfig[:redis][:host],
+ AppConfig[:redis][:port],
+ AppConfig[:redis][:password])
+ redis_client.connect
+ StagingTaskLog.redis = redis_client
+ end
+end
View
4 cloud_controller/config/routes.rb
@@ -27,6 +27,10 @@
get 'apps/:name/update' => 'apps#check_update'
put 'apps/:name/update' => 'apps#start_update'
+ # Stagers interact with the CC via these urls
+ post 'staging/droplet/:id/:upload_id' => 'staging#upload_droplet', :as => :upload_droplet
+ get 'staging/app/:id' => 'staging#download_app', :as => :download_unstaged_app
+
post 'services/v1/offerings' => 'services#create', :as => :service_create
delete 'services/v1/offerings/:label' => 'services#delete', :as => :service_delete, :label => /[^\/]+/
get 'services/v1/offerings/:label/handles' => 'services#list_handles', :as => :service_list_handles, :label => /[^\/]+/
View
5 cloud_controller/lib/cloud_error.rb
@@ -40,6 +40,7 @@ def to_json(options = nil)
APP_INVALID_RUNTIME = [307, HTTP_BAD_REQUEST, "Invalid runtime specification [%s] for framework: '%s'"]
APP_INVALID_FRAMEWORK = [308, HTTP_BAD_REQUEST, "Invalid framework description: '%s'"]
APP_DEBUG_DISALLOWED = [309, HTTP_BAD_REQUEST, "Cloud controller has disallowed debugging."]
+ APP_STAGING_ERROR = [310, HTTP_INTERNAL_SERVER_ERROR, "Staging failed: '%s'"]
# Bits
RESOURCES_UNKNOWN_PACKAGE_TYPE = [400, HTTP_BAD_REQUEST, "Unknown package type requested: \"%\""]
@@ -62,4 +63,8 @@ def to_json(options = nil)
URI_ALREADY_TAKEN = [701, HTTP_BAD_REQUEST, "The URI: \"%s\" has already been taken or reserved"]
URI_NOT_ALLOWED = [702, HTTP_FORBIDDEN, "External URIs are not enabled for this account"]
+ # Staging
+ STAGING_TIMED_OUT = [800, HTTP_INTERNAL_SERVER_ERROR, "Timed out waiting for staging to complete"]
+ STAGING_FAILED = [801, HTTP_INTERNAL_SERVER_ERROR, "Staging failed"]
+
end
View
58 cloud_controller/lib/staging_task_manager.rb
@@ -0,0 +1,58 @@
+require 'nats/client'
+
+require 'vcap/stager/task'
+require 'vcap/stager/task_result'
+
+class StagingTaskManager
+ DEFAULT_TASK_TIMEOUT = 120
+
+ def initialize(opts={})
+ @logger = opts[:logger] || VCAP::Logging.logger('vcap.cc.staging_manager')
+ @nats_conn = opts[:nats_conn] || NATS.client
+ @timeout = opts[:timeout] || DEFAULT_TASK_TIMEOUT
+ end
+
+ # Enqueues a staging task in redis, blocks until task completes or a timeout occurs.
+ #
+ # @param app App The app to be staged
+ # @param dl_uri String URI that the stager can download the app from
+ # @param ul_uri String URI that the stager should upload the staged droplet to
+ #
+ # @return VCAP::Stager::TaskResult
+ def run_staging_task(app, dl_uri, ul_uri)
+ inbox = "cc.staging." + VCAP.secure_uuid
+ nonce = VCAP.secure_uuid
+ f = Fiber.current
+
+ # Wait for notification from the stager
+ exp_timer = nil
+ sid = @nats_conn.subscribe(inbox) do |msg|
+ @logger.debug("Received result from stager on '#{inbox}' : '#{msg}'")
+ @nats_conn.unsubscribe(sid)
+ EM.cancel_timer(exp_timer)
+ f.resume(msg)
+ end
+
+ # Setup timer to expire our request if we don't hear a response from the stager in time
+ exp_timer = EM.add_timer(@timeout) do
+ @logger.warn("Staging timed out for app_id=#{app.id} (timeout=#{@timeout}, unsubscribing from '#{inbox}'",
+ :tags => [:staging])
+ @nats_conn.unsubscribe(sid)
+ f.resume(nil)
+ end
+
+ task = VCAP::Stager::Task.new(app.id, app.staging_task_properties, dl_uri, ul_uri, inbox)
+ task.enqueue('staging')
+ @logger.debug("Enqeued staging task for app_id=#{app.id}.", :tags => [:staging])
+
+ reply = Fiber.yield
+ if reply
+ result = VCAP::Stager::TaskResult.decode(reply)
+ StagingTaskLog.new(app.id, result.task_log).save
+ else
+ result = VCAP::Stager::TaskResult.new(nil, nil, "Timed out waiting for stager's reply.")
+ end
+
+ result
+ end
+end
View
0 cloud_controller/public/favicon.ico
No changes.
View
74 cloud_controller/spec/controllers/staging_controller_spec.rb
@@ -0,0 +1,74 @@
+require 'spec_helper'
+
+describe StagingController do
+ before :all do
+ VCAP::Logging.setup_from_config({'level' => 'debug2'})
+ AppConfig[:staging][:auth] = {
+ :user => 'test',
+ :password => 'test',
+ }
+ @auth = ActionController::HttpAuthentication::Basic.encode_credentials('test', 'test')
+ end
+
+ describe '#upload_droplet' do
+
+ before :each do
+ request.env["HTTP_AUTHORIZATION"] = @auth
+ end
+
+ it 'should return 401 for incorrect credentials' do
+ request.env["HTTP_AUTHORIZATION"] = nil
+ post :upload_droplet, {:id => 1, :upload_id => 'foo'}
+ response.status.should == 401
+ end
+
+ it 'should return 404 for unknown apps' do
+ post :upload_droplet, {:id => 1, :upload_id => 'foo'}
+ response.status.should == 404
+ end
+
+ it 'should return 400 for unknown uploads' do
+ App.stubs(:find_by_id).with(1).returns('test')
+ post :upload_droplet, {:id => 1, :upload_id => 'foo'}
+ response.status.should == 400
+ end
+
+ it 'should rename the uploaded file correctly' do
+ tmpfile = Tempfile.new('test')
+ app, droplet, upload = stub_test_upload(tmpfile)
+ File.expects(:rename).with(droplet.path, upload.upload_path)
+ params = {
+ :id => app.id,
+ :upload_id => upload.upload_id,
+ :upload => {:droplet => droplet}
+ }
+ post :upload_droplet, params
+ response.status.should == 200
+ end
+
+ it 'should clean up the temporary upload' do
+ tmpfile = Tempfile.new('test')
+ app, droplet, upload = stub_test_upload(tmpfile)
+ File.expects(:rename).with(droplet.path, upload.upload_path)
+ FileUtils.expects(:rm_f).with(droplet.path)
+ params = {
+ :id => app.id,
+ :upload_id => upload.upload_id,
+ :upload => {:droplet => droplet}
+ }
+ post :upload_droplet, params
+ response.status.should == 200
+ end
+ end
+
+ def stub_test_upload(file)
+ app = App.new
+ app.id = 1
+ droplet = Rack::Test::UploadedFile.new(file.path)
+ App.stubs(:find_by_id).with(app.id).returns(app)
+ upload = StagingController::DropletUploadHandle.new(app)
+ CloudController.stubs(:use_nginx).returns(false)
+ StagingController.stubs(:lookup_upload).with(upload.upload_id).returns(upload)
+ [app, droplet, upload]
+ end
+end
View
20 cloud_controller/spec/models/app_spec.rb
@@ -36,6 +36,26 @@
end
end
+ describe '#update_run_count' do
+ before :each do
+ @app = App.new
+ end
+
+ it 'resets the run count if the staged package hash changed' do
+ @app.expects(:staged_package_hash_changed?).returns(true)
+ @app.run_count = 5
+ @app.update_run_count()
+ @app.run_count.should == 0
+ end
+
+ it 'increments the run count if the staged package hash did not change' do
+ @app.expects(:staged_package_hash_changed?).returns(false)
+ @app.run_count = 5
+ @app.update_run_count()
+ @app.run_count.should == 6
+ end
+ end
+
def create_user(email, pw)
u = User.new(:email => email)
u.set_and_encrypt_password(pw)
View
47 cloud_controller/spec/models/staging_task_log_spec.rb
@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe StagingTaskLog do
+ before :all do
+ @task_id = 'test_task'
+ @task_log = StagingTaskLog.new(@task_id, 'Hello')
+ @task_key = StagingTaskLog.key_for_id(@task_id)
+ end
+
+ describe '#save' do
+ it 'should set a json encoded blob in redis' do
+ redis_mock = mock()
+ redis_mock.expects(:set).with(@task_key, @task_log.task_log)
+ @task_log.save(redis_mock)
+ end
+
+ it 'should use the static instance of redis if none is provided' do
+ redis_mock = mock()
+ redis_mock.expects(:set).with(@task_key, @task_log.task_log)
+ StagingTaskLog.redis = redis_mock
+ @task_log.save
+ end
+ end
+
+ describe '#fetch' do
+ it 'should fetch and decode an existing task result' do
+ redis_mock = mock()
+ redis_mock.expects(:get).with(@task_key).returns(@task_log.task_log)
+ res = StagingTaskLog.fetch(@task_id, redis_mock)
+ res.should be_instance_of(StagingTaskLog)
+ end
+
+ it 'should return nil if no key exists' do
+ redis_mock = mock()
+ redis_mock.expects(:get).with(@task_key).returns(nil)
+ res = StagingTaskLog.fetch(@task_id, redis_mock)
+ res.should be_nil
+ end
+
+ it 'should use the static instance of redis if none is provided' do
+ redis_mock = mock()
+ redis_mock.expects(:get).with(@task_key).returns(nil)
+ StagingTaskLog.redis = redis_mock
+ res = StagingTaskLog.fetch(@task_id, redis_mock)
+ end
+ end
+end
View
33 cloud_controller/spec/models/user_spec.rb
@@ -94,6 +94,39 @@
end
end
+ describe "#uses_new_stager?" do
+ it 'should return false if no percent is configured in the config' do
+ u = User.new(:email => 'foo@bar.com')
+ u.uses_new_stager?({:staging => {}}).should be_false
+ end
+
+ it 'should correctly identify which users should have the new stager enabled by percent' do
+ u = User.new(:email => 'foo@bar.com')
+ cfg = {:staging => {:new_stager_percent => 2}}
+
+ u.id = 2
+ u.uses_new_stager?(cfg).should be_false
+
+ u.id = 250
+ u.uses_new_stager?(cfg).should be_false
+
+ u.id = 1
+ u.uses_new_stager?(cfg).should be_true
+
+ u.id = 101
+ u.uses_new_stager?(cfg).should be_true
+ end
+
+ it 'should correctly identify which users should have the new stager enabled by email' do
+ u1 = User.new(:email => 'mpage@vmware.com')
+ u2 = User.new(:email => 'bar@foo.com')
+ cfg = {:staging => {:new_stager_email_regexp => Regexp.new('.*@vmware\.com')}}
+
+ u1.uses_new_stager?(cfg).should be_true
+ u2.uses_new_stager?(cfg).should be_false
+ end
+ end
+
def create_user(email, pw)
u = User.new(:email => email)
u.set_and_encrypt_password(pw)
View
46 cloud_controller/spec/staging/staging_task_manager_spec.rb
@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe StagingTaskManager do
+ before :all do
+ # Prevent EM/NATS related initializers from running
+ EM.instance_variable_set(:@next_tick_queue, [])
+ end
+
+ describe '#run_staging_task' do
+ it 'should expire long running tasks' do
+ nats_conn = mock()
+ logger = stub_everything(:logger)
+
+ nats_conn.expects(:subscribe).with(any_parameters())
+ nats_conn.expects(:unsubscribe).with(any_parameters())
+
+ task = stub_everything(:task)
+ VCAP::Stager::Task.expects(:new).with(any_parameters()).returns(task)
+
+ app = create_stub_app(12345)
+ stm = StagingTaskManager.new(
+ :logger => logger,
+ :nats_conn => nats_conn,
+ :timeout => 1
+ )
+
+ res = nil
+ EM.run do
+ Fiber.new do
+ EM.add_timer(2) { EM.stop }
+ res = stm.run_staging_task(app, nil, nil)
+ end.resume
+ end
+
+ res.should be_instance_of(VCAP::Stager::TaskResult)
+ res.was_success?.should be_false
+ end
+ end
+
+ def create_stub_app(id, props={})
+ ret = mock("app_#{id}")
+ ret.stubs(:id).returns(id)
+ ret.stubs(:staging_task_properties).returns(props)
+ ret
+ end
+end
View
BIN cloud_controller/vendor/cache/em-hiredis-0.1.0.gem
Binary file not shown.
View
BIN cloud_controller/vendor/cache/hiredis-0.3.2.gem
Binary file not shown.
View
BIN cloud_controller/vendor/cache/rest-client-1.6.7.gem
Binary file not shown.
View
BIN cloud_controller/vendor/cache/vcap_common-0.99.gem
Binary file not shown.
View
BIN cloud_controller/vendor/cache/vcap_stager-0.1.3.gem
Binary file not shown.
View
BIN stager/vendor/cache/vcap_staging-0.1.4.gem → ...oller/vendor/cache/vcap_staging-0.1.7.gem
Binary file not shown.
View
6 common/Gemfile.lock
@@ -13,7 +13,7 @@ GEM
remote: http://rubygems.org/
specs:
addressable (2.2.4)
- daemons (1.1.2)
+ daemons (1.1.4)
diff-lcs (1.1.2)
em-http-request (1.0.0.beta.3)
addressable (>= 2.2.3)
@@ -26,7 +26,7 @@ GEM
http_parser.rb (0.5.1)
json_pure (1.5.3)
little-plugger (1.1.2)
- logging (1.5.2)
+ logging (1.6.0)
little-plugger (>= 1.1.2)
nats (0.4.10)
daemons (>= 1.1.0)
@@ -46,7 +46,7 @@ GEM
daemons (>= 1.0.9)
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
- yajl-ruby (0.8.2)
+ yajl-ruby (0.8.3)
PLATFORMS
ruby
View
10 common/lib/vcap/json_schema.rb
@@ -60,6 +60,14 @@ def validate(dec_json)
end
end
+ class BoolSchema < BaseSchema
+ def validate(dec_json)
+ unless dec_json.kind_of?(TrueClass) || dec_json.kind_of?(FalseClass)
+ raise TypeError, "Expected instance of TrueClass or FalseClass, got #{dec_json.class}"
+ end
+ end
+ end
+
# Checks that supplied value is an instance of a given class
class TypeSchema < BaseSchema
def initialize(klass)
@@ -169,6 +177,8 @@ def build(&blk)
def parse(schema_def)
case schema_def
+ when VCAP::JsonSchema::BaseSchema
+ schema_def
when Hash
schema = VCAP::JsonSchema::HashSchema.new
for k, v in schema_def
View
2 common/lib/vcap/spec/forked_component/nats_server.rb
@@ -15,7 +15,7 @@ class VCAP::Spec::ForkedComponent::NatsServer < VCAP::Spec::ForkedComponent::Bas
attr_reader :uri, :port, :parsed_uri
def initialize(pid_filename, port, output_basedir='tmp')
- cmd = "ruby -S bundle exec nats-server -p #{port} -P #{pid_filename} -V"
+ cmd = "ruby -S bundle exec nats-server -p #{port} -P #{pid_filename} -V -D"
super(cmd, 'nats', output_basedir, pid_filename)
@port = port
@uri = "nats://127.0.0.1:#{@port}"
View
20 common/spec/unit/json_schema_spec.rb
@@ -1,6 +1,26 @@
# Copyright (c) 2009-2011 VMware, Inc.
require 'spec_helper'
+describe VCAP::JsonSchema::BoolSchema do
+ describe '#validate' do
+ before :all do
+ @schema = VCAP::JsonSchema::BoolSchema.new
+ end
+
+ it 'should not raise an error when supplied instances of TrueClass' do
+ expect { @schema.validate(true) }.to_not raise_error
+ end
+
+ it 'should not raise an error when supplied instances of FalseClass' do
+ expect { @schema.validate(false) }.to_not raise_error
+ end
+
+ it 'should raise an error when supplied instances not of TrueClass or FalseClass' do
+ expect { @schema.validate('zazzle') }.to raise_error(VCAP::JsonSchema::TypeError)
+ end
+ end
+end
+
describe VCAP::JsonSchema::TypeSchema do
describe '#initialize' do
it 'should raise an exception if supplied with non class instance' do
View
BIN common/vcap_common-0.99.gem
Binary file not shown.
2 java
@@ -1 +1 @@
-Subproject commit 455f124987fe5b06eabb8d231bc823f0b2270ede
+Subproject commit 0f1e9027088d325429a124d25e1d5a6e8c16bb9f
View
22 setup/cc_proxy.nginx.conf
@@ -86,6 +86,28 @@ http {
upload_cleanup 400-505;
}
+ # Droplet uploads from the stager should be authenticated
+ location ~ /staging/droplet/ {
+ # Pass along auth header
+ set $auth_header $upstream_http_x_auth;
+ proxy_set_header Authorization $auth_header;
+
+ # Pass altered request body to this location
+ upload_pass @cc_uploads;
+
+ # Store files to this directory
+ upload_store /var/vcap/data/cloud_controller/tmp/staged_droplet_uploads;
+
+ # Allow uploaded files to be read only by user
+ upload_store_access user:r;
+
+ # Set specified fields in request body
+ upload_set_form_field "droplet_path" $upload_tmp_path;
+
+ #on any error, delete uploaded files.
+ upload_cleanup 400-505;
+ }
+
# Pass altered request body to a backend
location @cc_uploads {
proxy_pass http://localhost:9025;
View
73 setup/mime.types
@@ -0,0 +1,73 @@
+types {
+ text/html html htm shtml;
+ text/css css;
+ text/xml xml;
+ image/gif gif;
+ image/jpeg jpeg jpg;
+ application/x-javascript js;
+ application/atom+xml atom;
+ application/rss+xml rss;
+
+ text/mathml mml;
+ text/plain txt;
+ text/vnd.sun.j2me.app-descriptor jad;
+ text/vnd.wap.wml wml;
+ text/x-component htc;
+
+ image/png png;
+ image/tiff tif tiff;
+ image/vnd.wap.wbmp wbmp;
+ image/x-icon ico;
+ image/x-jng jng;
+ image/x-ms-bmp bmp;
+ image/svg+xml svg;
+
+ application/java-archive jar war ear;
+ application/mac-binhex40 hqx;
+ application/msword doc;
+ application/pdf pdf;
+ application/postscript ps eps ai;
+ application/rtf rtf;
+ application/vnd.ms-excel xls;
+ application/vnd.ms-powerpoint ppt;
+ application/vnd.wap.wmlc wmlc;
+ application/vnd.wap.xhtml+xml xhtml;
+ application/vnd.google-earth.kml+xml kml;
+ application/vnd.google-earth.kmz kmz;
+ application/x-7z-compressed 7z;
+ application/x-cocoa cco;
+ application/x-java-archive-diff jardiff;
+ application/x-java-jnlp-file jnlp;
+ application/x-makeself run;
+ application/x-perl pl pm;
+ application/x-pilot prc pdb;
+ application/x-rar-compressed rar;
+ application/x-redhat-package-manager rpm;
+ application/x-sea sea;
+ application/x-shockwave-flash swf;
+ application/x-stuffit sit;
+ application/x-tcl tcl tk;
+ application/x-x509-ca-cert der pem crt;
+ application/x-xpinstall xpi;
+ application/zip zip;
+
+ application/octet-stream bin exe dll;
+ application/octet-stream deb;
+ application/octet-stream dmg;
+ application/octet-stream eot;
+ application/octet-stream iso img;
+ application/octet-stream msi msp msm;
+
+ audio/midi mid midi kar;
+ audio/mpeg mp3;
+ audio/x-realaudio ra;
+
+ video/3gpp 3gpp 3gp;
+ video/mpeg mpeg mpg;
+ video/quicktime mov;
+ video/x-flv flv;
+ video/x-mng mng;
+ video/x-ms-asf asx asf;
+ video/x-ms-wmv wmv;
+ video/x-msvideo avi;
+}
View
9 stager/Gemfile
@@ -1,14 +1,15 @@
source :rubygems
+gem 'eventmachine', '=0.12.10'
gem 'nats'
gem 'rake'
-gem 'redis'
-gem 'resque'
+gem 'logging', '= 1.5.2'
+gem 'rest-client', '= 1.6.7'
gem 'yajl-ruby', '>= 0.7.9'
-gem 'vcap_common', :path => '../common'
+gem 'vcap_common'
gem 'vcap_logging', '>= 0.1.1'
-gem 'vcap_staging'
+gem 'vcap_staging', '>= 0.1.7'
group :test do
gem 'rspec'
View
53 stager/Gemfile.lock
@@ -1,14 +1,3 @@
-PATH
- remote: ../common
- specs:
- vcap_common (0.99)
- eventmachine (~> 0.12.10)
- logging (>= 1.5.0)
- nats
- posix-spawn
- thin
- yajl-ruby
-
GEM
remote: http://rubygems.org/
specs:
@@ -17,26 +6,21 @@ GEM
daemons (1.1.4)
diff-lcs (1.1.2)
eventmachine (0.12.10)
- json (1.5.3)
json_pure (1.5.3)
little-plugger (1.1.2)
logging (1.5.2)
little-plugger (>= 1.1.2)
+ mime-types (1.16)
nats (0.4.10)
daemons (>= 1.1.0)
eventmachine (>= 0.12.10)
json_pure (>= 1.5.1)
+ nokogiri (1.5.0)
posix-spawn (0.3.6)
rack (1.3.2)
rake (0.9.2)
- redis (2.2.1)
- redis-namespace (1.0.3)
- redis (< 3.0.0)
- resque (1.17.1)
- json (>= 1.4.6, < 1.6)
- redis-namespace (~> 1.0.2)
- sinatra (>= 0.9.2)
- vegas (~> 0.1.2)
+ rest-client (1.6.7)
+ mime-types (>= 1.16)
rspec (2.6.0)
rspec-core (~> 2.6.0)
rspec-expectations (~> 2.6.0)
@@ -53,27 +37,38 @@ GEM
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
tilt (1.3.2)
+ vcap_common (0.99)
+ eventmachine (~> 0.12.10)
+ logging (>= 1.5.0)
+ nats
+ posix-spawn
+ thin
+ yajl-ruby
vcap_logging (0.1.1)
- vcap_staging (0.1.4)
- vegas (0.1.8)
- rack (>= 1.0.0)
- webmock (1.6.4)
+ vcap_staging (0.1.7)
+ nokogiri (>= 1.4.4)
+ rake
+ rspec
+ vcap_common
+ yajl-ruby (>= 0.7.9)
+ webmock (1.7.4)
addressable (~> 2.2, > 2.2.5)
crack (>= 0.1.7)
- yajl-ruby (0.8.2)
+ yajl-ruby (0.8.3)
PLATFORMS
ruby
DEPENDENCIES
+ eventmachine (= 0.12.10)
+ logging (= 1.5.2)
nats
rake
- redis
- resque
+ rest-client (= 1.6.7)
rspec
sinatra
- vcap_common!
+ vcap_common
vcap_logging (>= 0.1.1)
- vcap_staging
+ vcap_staging (>= 0.1.7)
webmock
yajl-ruby (>= 0.7.9)
View
2 stager/Rakefile
@@ -21,7 +21,7 @@ gemspec = Gem::Specification.new do |s|
s.executables = []
s.bindir = 'bin'
s.require_path = 'lib'
- s.files = %w(Rakefile) + Dir.glob("{lib,spec,vendor}/**/*")
+ s.files = %w(Rakefile Gemfile) + Dir.glob("{lib,spec,vendor}/**/*")
end
Rake::GemPackageTask.new(gemspec) do |pkg|
View
11 stager/bin/create_secure_users.rb
@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+
+require 'vcap/stager/secure_user_manager'
+
+VCAP::Stager::SecureUserManager.instance.create_secure_users
View
29 stager/bin/download_app
@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+
+
+require 'rubygems'
+require 'bundler/setup'
+
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+
+require 'vcap/stager/util'
+
+# Utility script to download an app package to a known location on disk.
+# Used to keep long running ruby code (the stager process) out of the data path.
+#
+# NB: If this script exits with a non-zero status, the download failed
+#
+
+unless ARGV.length == 2
+ puts "Usage: download_app [url_file] [dst_path]"
+ exit 1
+end
+
+uri_file = ARGV[0]
+dst_path = ARGV[1]
+app_uri = File.read(uri_file)
+
+VCAP::Stager::Util.fetch_zipped_app(app_uri, dst_path)
+
+exit 0
View
40 stager/bin/stager
@@ -6,14 +6,11 @@ require 'bundler/setup'
$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
-require 'eventmachine'
-require 'resque'
-
require 'vcap/common'
require 'vcap/component'
require 'vcap/stager'
-sleep_interval = 5
+sleep_interval = 1
config_file = VCAP::Stager::Config::DEFAULT_CONFIG_PATH
OptionParser.new do |op|
@@ -44,29 +41,14 @@ rescue => e
exit 1
end
-VCAP::Stager.init(config)
+ENV['TMPDIR'] = config[:dirs][:tmp] if config[:dirs] && config[:dirs][:tmp]
-worker = Resque::Worker.new(*config[:queues])
-Thread.new do
- EM.error_handler do |e|
- logger = VCAP::Logging.logger('vcap.stager.component')
- logger.error("EventMachine error: #{e}")
- logger.error(e)
- raise e
- end
-
- NATS.on_error do |e|
- logger = VCAP::Logging.logger('vcap.stager.component')
- logger.error("NATS error: #{e}")
- logger.error(e)
- raise e
- end
-
- NATS.start(:uri => config[:nats_uri]) do
- VCAP::Component.register(:type => 'stager',
- :local_ip => VCAP.local_ip(config[:local_route]),
- :config => config,
- :index => config[:index])
- end
-end.abort_on_exception = true
-worker.work(sleep_interval)
+VCAP::Stager.init(config)
+user_mgr=nil
+if config[:secure]
+ VCAP::Stager::SecureUserManager.instance.setup
+ user_mgr = VCAP::Stager::SecureUserManager.instance
+end
+task_mgr = VCAP::Stager::TaskManager.new(config[:max_active_tasks], user_mgr)
+server = VCAP::Stager::Server.new(config[:nats_uri], task_mgr, config)
+server.run
View
29 stager/bin/upload_droplet
@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+
+
+require 'rubygems'
+require 'bundler/setup'
+
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+
+require 'vcap/stager/util'
+
+# Utility script to upload a droplet from a known location on disk.
+# Used to keep long running ruby code (the stager process) out of the data path.
+#
+# NB: If this script exits with a non-zero status, the upload failed
+#
+
+unless ARGV.length == 2
+ puts "Usage: upload_droplet [url_file] [dst_path]"
+ exit 1
+end
+
+uri_file = ARGV[0]
+src_path = ARGV[1]
+droplet_uri = File.read(uri_file)
+
+VCAP::Stager::Util.upload_droplet(droplet_uri, src_path)
+
+exit 0
View
8 stager/config/dev.yml
@@ -1,9 +1,9 @@
---
logging:
level: debug2
-redis:
- host: 127.0.0.1
-pid_file: /var/vcap/sys/run/stager.pid
+pid_filename: /var/vcap/sys/run/stager.pid
nats_uri: nats://127.0.0.1:4222
max_staging_duration: 120
-queues: ['staging']
+max_active_tasks: 10
+queues: ['staging']
+secure: false
View
1 stager/config/stager.yml
View
22 stager/lib/vcap/stager.rb
@@ -1,10 +1,10 @@
-require 'eventmachine'
-require 'resque'
-
-require 'vcap/stager/errors'
+require 'vcap/stager/constants'
require 'vcap/stager/config'
+require 'vcap/stager/server'
require 'vcap/stager/task'
+require 'vcap/stager/task_error'
require 'vcap/stager/task_logger'
+require 'vcap/stager/task_manager'
require 'vcap/stager/task_result'
require 'vcap/stager/util'
require 'vcap/stager/version'
@@ -20,18 +20,8 @@ def init(config)
StagingPlugin.manifest_root = config[:dirs][:manifests]
StagingPlugin.load_all_manifests
StagingPlugin.validate_configuration!
-
- Resque.redis = Redis.new(config[:redis])
- Resque.redis.namespace = config[:redis][:namespace] if config[:redis][:namespace]
- # Prevents EM from getting into a blind/deaf state after forking.
- # See: https://github.com/eventmachine/eventmachine/issues/213
- Resque.after_fork do |job|
- if EM.reactor_running?
- EM.stop_event_loop
- EM.release_machine
- EM.instance_variable_set('@reactor_running', false)
- end
- end
+ VCAP::Stager::Task.set_defaults(config)
+ VCAP::Stager::Task.set_defaults({:manifest_dir => config[:dirs][:manifests]})
end
end
end
View
18 stager/lib/vcap/stager/config.rb
@@ -1,4 +1,6 @@
require 'vcap/config'
+require 'vcap/json_schema'
+require 'vcap/staging/plugin/common'
module VCAP
module Stager
@@ -16,27 +18,17 @@ class VCAP::Stager::Config < VCAP::Config
optional(:syslog) => String, # Name to associate with syslog messages (should start with 'vcap.')
},
- :redis => {
- :host => String,
- optional(:port) => Integer,
- optional(:password) => String,
- optional(:namespace) => String,
- },
-
:nats_uri => String, # NATS uri of the form nats://<user>:<pass>@<host>:<port>
:max_staging_duration => Integer, # Maximum number of seconds a staging can run
+ :max_active_tasks => Integer, # Maximum number of tasks executing concurrently
:queues => [String], # List of queues to pull tasks from
:pid_filename => String, # Pid filename to use
optional(:dirs) => {
optional(:manifests) => String, # Where all of the staging manifests live
optional(:tmp) => String, # Default is /tmp
},
-
- optional(:secure_user) => { # Drop privs during staging to this user
- :uid => Integer,
- optional(:gid) => Integer,
- },
+ :secure => VCAP::JsonSchema::BoolSchema.new,
optional(:index) => Integer, # Component index (stager-0, stager-1, etc)
optional(:ruby_path) => String, # Full path to the ruby executable that should execute the run plugin script
@@ -49,7 +41,7 @@ def self.from_file(*args)
config = super(*args)
config[:dirs] ||= {}
- config[:dirs][:manifests] ||= File.expand_path('../plugin/manifests', __FILE__)
+ config[:dirs][:manifests] ||= StagingPlugin::DEFAULT_MANIFEST_ROOT
config[:run_plugin_path] ||= File.expand_path('../../../../bin/run_plugin', __FILE__)
config[:ruby_path] ||= `which ruby`.chomp
View
6 stager/lib/vcap/stager/constants.rb
@@ -0,0 +1,6 @@
+module VCAP
+ module Stager
+ BIN_DIR = File.expand_path("../../../../bin", __FILE__)
+ CONFIG_DIR = File.expand_path("../../../../config", __FILE__)
+ end
+end
View
8 stager/lib/vcap/stager/errors.rb
@@ -1,8 +0,0 @@
-module VCAP
- module Stager
- class StagingError < StandardError; end
- class AppDownloadError < StagingError; end
- class DropletUploadError < StagingError; end
- class ResultPublishingError < StagingError; end
- end
-end
View
119 stager/lib/vcap/stager/secure_user_manager.rb
@@ -0,0 +1,119 @@
+# = XXX =
+# It's multiplying! This file needs to die in a fire. It is a duplicate of the DEA secure user code.
+# Soon we will have a more robust implementation of this, and a better container implementation
+# to go with it.
+
+require 'singleton'
+
+require 'vcap/logging'
+
+module VCAP
+ module Stager
+ end
+end
+
+class VCAP::Stager::SecureUserManager
+ include Singleton
+
+ SECURE_USER_STRING = 'vcap-stager-user-'
+ SECURE_USER_GREP = "#{SECURE_USER_STRING}[0-9]\\{1,3\\}"
+ SECURE_USER_PATTERN = /(vcap-stager-user-\d+):[^:]+:(\d+):(\d+)/
+ DEFAULT_SECURE_GROUP = 'vcap-stager'
+ SECURE_UID_BASE = 23000
+ DEFAULT_NUM_SECURE_USERS = 32
+
+ def initialize
+ @logger = VCAP::Logging.logger('vcap.stager.secure_user_manager')
+ unless RUBY_PLATFORM =~ /linux/
+ @logger.fatal("ERROR: Secure mode not supported on this platform.")
+ exit
+ end
+ end
+
+ def setup(logger=nil)
+ @logger ||= logger
+ @logger.info("Grabbing secure users")
+ File.umask(0077)
+ grab_existing_users
+ unless @secure_users.size >= DEFAULT_NUM_SECURE_USERS
+ raise "Don't have enough secure users (#{@secure_users.size}), did you forget to set them up? "
+ end
+ @secure_mode_initialized = true
+ end
+
+ def create_secure_users
+ if Process.uid != 0
+ @logger.fatal "ERROR: Creating secure users requires root priviliges."
+ exit 1
+ end
+
+ @logger.info "Creating initial #{DEFAULT_NUM_SECURE_USERS} secure users"
+ create_default_group
+ (1..DEFAULT_NUM_SECURE_USERS).each do |i|
+ create_secure_user("#{SECURE_USER_STRING + i.to_s}", SECURE_UID_BASE+i)
+ end
+ end
+
+ def checkout_user
+ raise "Did you forget to call setup_secure_mode()?" unless @secure_mode_initialized
+
+ if @secure_users.length > 0
+ ret = @secure_users.pop
+ @logger.debug("Checked out #{ret}")
+ ret
+ else
+ raise "All secure users are currently in use."
+ end
+ end
+
+ def return_user(user)
+ raise "Did you forget to call setup_secure_mode()?" unless @secure_mode_initialized
+ @logger.debug("Returned #{user}")
+ @secure_users << user
+ end
+
+ protected
+
+ def create_default_group
+ # Add in default group
+ system("addgroup --system #{DEFAULT_SECURE_GROUP} > /dev/null 2>&1")
+ end
+
+ def create_secure_user(username, uid = 0)
+ @logger.info("Creating user:#{username} (#{uid})")
+ system("adduser --system --quiet --no-create-home --home '/nonexistent' --uid #{uid} #{username} > /tmp/foo 2>&1")
+ system("usermod -g #{DEFAULT_SECURE_GROUP} #{username} > /dev/null 2>&1")
+
+ info = get_user_info(username)
+
+ { :user => username,
+ :gid => info[:gid].to_i,
+ :uid => info[:uid].to_i,
+ :group => DEFAULT_SECURE_GROUP,
+ }
+ end
+
+ def check_existing_users
+ return `grep -H "#{SECURE_USER_GREP}" /etc/passwd`
+ end
+
+ def grab_existing_users
+ @secure_users = []
+ File.open('/etc/passwd') do |f|
+ while (line = f.gets)
+ if line =~ SECURE_USER_PATTERN
+ @secure_users << { :user => $1, :uid => $2.to_i, :gid => $3.to_i, :group => DEFAULT_SECURE_GROUP}
+ end
+ end
+ end
+ @secure_users
+ end
+
+ def get_user_info(username)
+ info = `id #{username}`
+ ret = {}
+ ret[:uid] = $1 if info =~ /uid=(\d+)/
+ ret[:gid] = $1 if info =~ /gid=(\d+)/
+ ret
+ end
+end
View
112 stager/lib/vcap/stager/server.rb
@@ -0,0 +1,112 @@
+require 'nats/client'
+require 'yajl'
+
+require 'vcap/component'
+require 'vcap/json_schema'
+require 'vcap/logging'
+
+require 'vcap/stager/task'
+require 'vcap/stager/task_manager'
+
+module VCAP
+ module Stager
+ end
+end
+
+class VCAP::Stager::Server
+ class Channel
+ def initialize(nats_conn, subject, &blk)
+ @nats_conn = nats_conn
+ @subject = subject
+ @sid = nil
+ @receiver = blk
+ end
+
+ def open
+ @sid = @nats_conn.subscribe(@subject, :queue => @subject) {|msg| @receiver.call(msg) }
+ end
+
+ def close
+ @nats_conn.unsubscribe(@sid)
+ end
+ end
+
+ def initialize(nats_uri, task_mgr, config={})
+ @nats_uri = nats_uri
+ @nats_conn = nil
+ @task_mgr = nil
+ @channels = []
+ @config = config
+ @task_mgr = task_mgr
+ @logger = VCAP::Logging.logger('vcap.stager.server')
+ end
+
+ def run
+ install_error_handlers()
+ install_signal_handlers()
+ EM.run do
+ @nats_conn = NATS.connect(:uri => @nats_uri) do
+ VCAP::Stager::Task.set_defaults(:nats => @nats_conn)
+ VCAP::Component.register(:type => 'Stager',
+ :index => @config[:index],
+ :host => VCAP.local_ip(@config[:local_route]),
+ :config => @config,
+ :nats => @nats_conn)
+ setup_channels()
+ @task_mgr.varz = VCAP::Component.varz
+ @logger.info("Server running")
+ end
+ end
+ end
+
+ # Stops receiving new tasks, waits for existing tasks to finish, then stops.
+ def shutdown
+ @logger.info("Shutdown initiated, waiting for remaining #{@task_mgr.num_tasks} task(s) to finish")
+ @channels.each {|c| c.close }
+ @task_mgr.on_idle do
+ @logger.info("All tasks completed. Exiting!")
+ EM.stop
+ end
+ end
+
+ private
+
+ def install_error_handlers
+ EM.error_handler do |e|
+ @logger.error("EventMachine error: #{e}")
+ @logger.error(e)
+ raise e
+ end
+
+ NATS.on_error do |e|
+ @logger.error("NATS error: #{e}")
+ @logger.error(e)
+ raise e
+ end
+ end
+
+ def install_signal_handlers
+ trap('USR2') { shutdown() }
+ trap('INT') { shutdown() }
+ end
+
+ def setup_channels
+ for qn in @config[:queues]
+ channel = Channel.new(@nats_conn, "vcap.stager.#{qn}") {|msg| add_task(msg) }
+ channel.open
+ @channels << channel
+ end
+ end
+
+ def add_task(task_msg)
+ begin
+ @logger.debug("Decoding task '#{task_msg}'")
+ task = VCAP::Stager::Task.decode(task_msg)
+ rescue => e
+ @logger.warn("Failed decoding '#{task_msg}': #{e}")
+ @logger.warn(e)
+ return
+ end
+ @task_mgr.add_task(task)
+ end
+end
View
343 stager/lib/vcap/stager/task.rb
@@ -1,45 +1,49 @@
+require 'fiber'
require 'fileutils'
-require 'redis'
-require 'redis-namespace'
+require 'nats/client'
require 'tmpdir'
require 'uri'
+require 'yajl'
+require 'vcap/common'
require 'vcap/logging'
require 'vcap/staging/plugin/common'
-require 'vcap/subprocess'
+require 'vcap/stager/constants'
+require 'vcap/stager/task_error'