Permalink
Browse files

Merge branch 'master' into services-r5

Change-Id: I46d25577471fb46f80db51b64b8f5bf46939379b
  • Loading branch information...
2 parents 202a551 + 4d56fed commit 218c19de187d0d094e45d1357bbdbd64fafa8a7e felixhoo committed Sep 18, 2011
Showing with 2,371 additions and 636 deletions.
  1. +3 −0 bin/config/redis-server.conf
  2. +4 −0 bin/stager
  3. +57 −2 bin/vcap
  4. +5 −2 cloud_controller/Gemfile
  5. +16 −2 cloud_controller/Gemfile.lock
  6. +74 −4 cloud_controller/app/controllers/apps_controller.rb
  7. +137 −0 cloud_controller/app/controllers/staging_controller.rb
  8. +22 −0 cloud_controller/app/models/app.rb
  9. +2 −0 cloud_controller/app/models/app_manager.rb
  10. +59 −0 cloud_controller/app/models/staging_task_log.rb
  11. +11 −0 cloud_controller/app/models/user.rb
  12. +9 −1 cloud_controller/config/appconfig.rb
  13. +10 −1 cloud_controller/config/cloud_controller.yml
  14. +1 −0 cloud_controller/config/final_stage/activate.rb
  15. +12 −0 cloud_controller/config/final_stage/redis.rb
  16. +4 −0 cloud_controller/config/routes.rb
  17. +5 −0 cloud_controller/lib/cloud_error.rb
  18. +58 −0 cloud_controller/lib/staging_task_manager.rb
  19. 0 cloud_controller/public/favicon.ico
  20. +74 −0 cloud_controller/spec/controllers/staging_controller_spec.rb
  21. +20 −0 cloud_controller/spec/models/app_spec.rb
  22. +47 −0 cloud_controller/spec/models/staging_task_log_spec.rb
  23. +33 −0 cloud_controller/spec/models/user_spec.rb
  24. +46 −0 cloud_controller/spec/staging/staging_task_manager_spec.rb
  25. BIN cloud_controller/vendor/cache/em-hiredis-0.1.0.gem
  26. BIN cloud_controller/vendor/cache/hiredis-0.3.2.gem
  27. BIN cloud_controller/vendor/cache/rest-client-1.6.7.gem
  28. BIN cloud_controller/vendor/cache/vcap_common-0.99.gem
  29. BIN cloud_controller/vendor/cache/vcap_stager-0.1.3.gem
  30. BIN stager/vendor/cache/vcap_staging-0.1.4.gem → cloud_controller/vendor/cache/vcap_staging-0.1.7.gem
  31. +3 −3 common/Gemfile.lock
  32. +10 −0 common/lib/vcap/json_schema.rb
  33. +1 −1 common/lib/vcap/spec/forked_component/nats_server.rb
  34. +20 −0 common/spec/unit/json_schema_spec.rb
  35. BIN common/vcap_common-0.99.gem
  36. +1 −1 java
  37. +22 −0 setup/cc_proxy.nginx.conf
  38. +73 −0 setup/mime.types
  39. +5 −4 stager/Gemfile
  40. +24 −29 stager/Gemfile.lock
  41. +1 −1 stager/Rakefile
  42. +11 −0 stager/bin/create_secure_users.rb
  43. +29 −0 stager/bin/download_app
  44. +11 −29 stager/bin/stager
  45. +29 −0 stager/bin/upload_droplet
  46. +4 −4 stager/config/dev.yml
  47. +1 −0 stager/config/stager.yml
  48. +6 −16 stager/lib/vcap/stager.rb
  49. +5 −13 stager/lib/vcap/stager/config.rb
  50. +6 −0 stager/lib/vcap/stager/constants.rb
  51. +0 −8 stager/lib/vcap/stager/errors.rb
  52. +119 −0 stager/lib/vcap/stager/secure_user_manager.rb
  53. +112 −0 stager/lib/vcap/stager/server.rb
  54. +215 −128 stager/lib/vcap/stager/task.rb
  55. +59 −0 stager/lib/vcap/stager/task_error.rb
  56. +94 −0 stager/lib/vcap/stager/task_manager.rb
  57. +22 −33 stager/lib/vcap/stager/task_result.rb
  58. +74 −41 stager/lib/vcap/stager/util.rb
  59. +1 −1 stager/lib/vcap/stager/version.rb
  60. +0 −7 stager/spec/fixtures/apps/rails3_gitgems/source/config/initializers/backtrace_silencers.rb
  61. +2 −3 stager/spec/fixtures/stager_config.yml.erb
  62. +21 −32 stager/spec/functional/stager_spec.rb
  63. +3 −2 stager/spec/spec_helper.rb
  64. +0 −47 stager/spec/support/forked_redis_server.rb
  65. +2 −8 stager/spec/support/forked_stager.rb
  66. +37 −0 stager/spec/unit/task_error_spec.rb
  67. +74 −0 stager/spec/unit/task_manager_spec.rb
  68. +18 −36 stager/spec/unit/task_result_spec.rb
  69. +98 −88 stager/spec/unit/task_spec.rb
  70. +62 −16 stager/spec/unit/util_spec.rb
  71. BIN stager/vendor/cache/json-1.5.3.gem
  72. BIN stager/vendor/cache/mime-types-1.16.gem
  73. BIN stager/vendor/cache/nokogiri-1.5.0.gem
  74. BIN stager/vendor/cache/redis-2.2.1.gem
  75. BIN stager/vendor/cache/redis-namespace-1.0.3.gem
  76. BIN stager/vendor/cache/resque-1.17.1.gem
  77. BIN stager/vendor/cache/rest-client-1.6.7.gem
  78. BIN stager/vendor/cache/vcap_common-0.99.gem
  79. BIN cloud_controller/vendor/cache/vcap_staging-0.1.4.gem → stager/vendor/cache/vcap_staging-0.1.7.gem
  80. BIN stager/vendor/cache/vegas-0.1.8.gem
  81. BIN stager/vendor/cache/webmock-1.6.4.gem
  82. BIN stager/vendor/cache/webmock-1.7.4.gem
  83. BIN stager/vendor/cache/yajl-ruby-0.8.2.gem
  84. BIN stager/vendor/cache/yajl-ruby-0.8.3.gem
  85. +1 −9 staging/Gemfile
  86. +16 −8 staging/Gemfile.lock
  87. +2 −24 staging/Rakefile
  88. +10 −0 staging/lib/vcap/staging/plugin/config.rb
  89. +15 −1 staging/lib/vcap/staging/plugin/gemfile_task.rb
  90. +6 −0 staging/lib/vcap/staging/plugin/java_web/plugin.rb
  91. BIN ...va_web/resources/{auto-reconfiguration-0.6.0-BUILD-SNAPSHOT.jar → auto-reconfiguration-0.6.0.jar}
  92. BIN staging/lib/vcap/staging/plugin/java_web/resources/mysql-connector-java-5.1.12-bin.jar
  93. BIN staging/lib/vcap/staging/plugin/java_web/resources/postgresql-9.0-801.jdbc4.jar
  94. BIN staging/lib/vcap/staging/plugin/java_web/resources/tomcat.zip
  95. +41 −3 staging/lib/vcap/staging/plugin/java_web/tomcat.rb
  96. +0 −5 staging/lib/vcap/staging/plugin/node/stage
  97. +0 −7 staging/lib/vcap/staging/plugin/php/stage
  98. +5 −0 staging/lib/vcap/staging/plugin/spring/autostaging_template_spring.xml
  99. +0 −5 staging/lib/vcap/staging/plugin/wsgi/stage
  100. +1 −1 staging/lib/vcap/staging/version.rb
  101. BIN staging/spec/fixtures/apps/spring_context_initializer_foo/source.war
  102. +3 −1 staging/spec/support/staging_spec_helpers.rb
  103. +42 −0 staging/spec/unit/config_spec.rb
  104. +9 −0 staging/spec/unit/grails_spec.rb
  105. +97 −0 staging/spec/unit/java_web_spec.rb
  106. +20 −0 staging/spec/unit/lift_spec.rb
  107. +91 −8 staging/spec/unit/spring_spec.rb
  108. BIN staging/vcap_staging-0.1.7.gem
  109. +27 −0 staging/vcap_staging.gemspec
  110. BIN staging/vendor/cache/json_pure-1.5.3.gem
  111. BIN staging/vendor/cache/json_pure-1.5.4.gem
  112. BIN staging/vendor/cache/logging-1.5.2.gem
  113. BIN staging/vendor/cache/logging-1.6.1.gem
  114. BIN staging/vendor/cache/spruz-0.2.13.gem
  115. BIN staging/vendor/cache/yajl-ruby-0.8.2.gem
  116. BIN staging/vendor/cache/yajl-ruby-0.8.3.gem
  117. +1 −1 tests
@@ -0,0 +1,3 @@
+bind 0.0.0.0
+port 5454
+loglevel debug
View
@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+# Copyright (c) 2009-2011 VMware, Inc.
+
+exec(File.expand_path("../../stager/bin/stager", __FILE__), *ARGV)
View
@@ -79,7 +79,7 @@ class Component
end
def pid_file
- configuration["pid"] || raise("#{@configuration_path} does not specify location of pid file")
+ configuration["pid"] || configuration['pid_filename'] || raise("#{@configuration_path} does not specify location of pid file")
end
def log_file?
@@ -244,10 +244,64 @@ class NatsServer
end
end
+class RedisServer
+ DEFAULT_CONFIG_PATH = File.expand_path('../config/redis-server.conf', __FILE__)
+
+ def initialize(pid_filename, config_path=DEFAULT_CONFIG_PATH)
+ @config_path = config_path
+ @pid_filename = pid_filename
+ @redis_path = `which redis-server`.chomp
+ unless $? == 0
+ STDERR.puts "Could not find redis-server, exiting.".red
+ exit 1
+ end
+ @pid = read_pidfile
+ end
+
+ def start
+ return if running?
+ @pid = fork do
+ log_file = File.join(TMP, 'redis-server.log')
+ stdout = File.open(log_file, 'a')
+ STDOUT.reopen(stdout)
+ stderr = File.open(log_file, 'a')
+ STDERR.reopen(stderr)
+ exec("#{@redis_path} #{@config_path}")
+ end
+ Process.detach(@pid)
+ File.open(@pid_filename, 'w+') {|f| f.write(@pid) }
+ end
+
+ def stop
+ return unless running?
+ Process.kill('TERM', @pid)
+ @pid = nil
+ end
+
+ private
+
+ def read_pidfile
+ if File.exist?(@pid_filename)
+ File.read(@pid_filename).chomp.to_i
+ else
+ nil
+ end
+ end
+
+ def running?
+ return false unless @pid
+ File.exist?(File.join('/proc', @pid.to_s))
+ end
+end
+
+
module Run
+ DEFAULT_REDIS_PIDFILE = File.join(TMP, 'redis-server.pid')
+
def self.start_init
nats_server = NatsServer.new
nats_server.start_server
+ RedisServer.new(DEFAULT_REDIS_PIDFILE).start
end
def self.start(args)
@@ -259,6 +313,7 @@ module Run
# Only process this if no one else running..
running_components = components([]).select {|c| c.running?}.map{|c| c.name }
return unless running_components.empty?
+ RedisServer.new(DEFAULT_REDIS_PIDFILE).stop
nats_server = NatsServer.new
return unless nats_server.is_running?
nats_server.kill_server
@@ -380,7 +435,7 @@ module Run
private
def self.core
- %w(router cloud_controller dea health_manager)
+ %w(router cloud_controller dea health_manager stager)
end
def self.services
View
@@ -10,9 +10,11 @@ gem 'logging', '>= 1.5.0'
# VCAP common components
gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :path => '../common'
gem 'vcap_logging', :require => ['vcap/logging']
+gem 'vcap_staging', '= 0.1.7'
-# XXX - Vendor once working
-gem 'vcap_staging'
+# For queuing staging tasks
+gem 'em-hiredis'
+gem 'vcap_stager', '= 0.1.3'
# Databases
gem 'sqlite3'
@@ -45,6 +47,7 @@ gem 'bcrypt-ruby', '>= 2.1.4'
gem 'ruby-hmac', :require => 'hmac-sha1'
gem 'SystemTimer', :platforms => :mri_18
gem 'uuidtools'
+gem 'rest-client', '= 1.6.7'
# rspec-rails is outside the 'test' group in order to consistently provide Rake tasks.
gem 'rspec-rails', '>= 2.4.1'
@@ -48,6 +48,8 @@ GEM
builder (>= 2.1.2)
daemons (1.1.2)
diff-lcs (1.1.2)
+ em-hiredis (0.1.0)
+ hiredis (~> 0.3.0)
em-http-request (1.0.0.beta.3)
addressable (>= 2.2.3)
em-socksify
@@ -60,6 +62,7 @@ GEM
erubis (2.6.6)
abstract (>= 1.0.0)
eventmachine (0.12.10)
+ hiredis (0.3.2)
http_parser.rb (0.5.1)
i18n (0.5.0)
json_pure (1.5.1)
@@ -103,6 +106,8 @@ GEM
thor (~> 0.14.4)
rake (0.8.7)
rcov (0.9.9)
+ rest-client (1.6.7)
+ mime-types (>= 1.16)
rspec (2.5.0)
rspec-core (~> 2.5.0)
rspec-expectations (~> 2.5.0)
@@ -132,7 +137,13 @@ GEM
tzinfo (0.3.26)
uuidtools (2.1.2)
vcap_logging (0.1.0)
- vcap_staging (0.1.4)
+ vcap_stager (0.1.3)
+ vcap_staging (0.1.7)
+ nokogiri (>= 1.4.4)
+ rake
+ rspec
+ vcap_common
+ yajl-ruby (>= 0.7.9)
yajl-ruby (0.8.2)
PLATFORMS
@@ -142,6 +153,7 @@ DEPENDENCIES
SystemTimer
bcrypt-ruby (>= 2.1.4)
ci_reporter
+ em-hiredis
em-http-request (~> 1.0.0.beta.3)
em-redis
eventmachine (~> 0.12.10)
@@ -154,6 +166,7 @@ DEPENDENCIES
rack-fiber_pool
rails (~> 3.0.5)
rcov
+ rest-client (= 1.6.7)
rspec (>= 2.4.0)
rspec-rails (>= 2.4.1)
ruby-hmac
@@ -163,5 +176,6 @@ DEPENDENCIES
uuidtools
vcap_common!
vcap_logging
- vcap_staging
+ vcap_stager (= 0.1.3)
+ vcap_staging (= 0.1.7)
yajl-ruby (>= 0.7.9)
@@ -1,3 +1,5 @@
+require 'staging_task_manager'
+
class AppsController < ApplicationController
before_filter :require_user, :except => [:download_staged]
before_filter :find_app_by_name, :except => [:create, :list, :download_staged]
@@ -83,7 +85,7 @@ def upload
end
def download
- path = @app.package_path
+ path = @app.unstaged_package_path
if path && File.exists?(path)
send_file path
else
@@ -124,7 +126,13 @@ def start_update
error_on_lock_mismatch(@app)
@app.lock_version += 1
manager = AppManager.new(@app)
- manager.stage if @app.needs_staging?
+ if @app.needs_staging?
+ if user.uses_new_stager?
+ stage_app(@app)
+ else
+ manager.stage
+ end
+ end
manager.stop_all
manager.started
render :nothing => true, :status => 204
@@ -164,6 +172,18 @@ def check_update
# GET /apps/:name/instances/:instance_id/files/:path'
def files
+ # XXX - Yuck. This will have to do until we update VMC with a real
+ # way to fetch staging logs.
+ if user.uses_new_stager? && (params[:path] == 'logs/staging.log')
+ log = StagingTaskLog.fetch_fibered(@app.id)
+ if log
+ render :text => log.task_log
+ else
+ render :nothing => true, :status => 404
+ end
+ return
+ end
+
# will Fiber.yield
url, auth = AppManager.new(@app).get_file_url(params[:instance_id], params[:path])
raise CloudError.new(CloudError::APP_FILE_ERROR, params[:path] || '/') unless url
@@ -191,6 +211,48 @@ def files
private
+ def stage_app(app)
+ task_mgr = StagingTaskManager.new(:logger => CloudController.logger,
+ :timeout => AppConfig[:staging][:max_staging_runtime])
+ dl_uri = StagingController.download_app_uri(app)
+ ul_hdl = StagingController.create_upload(app)
+
+ result = task_mgr.run_staging_task(app, dl_uri, ul_hdl.upload_uri)
+
+ # Update run count to be consistent with previous staging code
+ if result.was_success?
+ CloudController.logger.debug("Staging task for app_id=#{app.id} succeded.", :tags => [:staging])
+ CloudController.logger.debug1("Details: #{result.task_log}", :tags => [:staging])
+ app.update_staged_package(ul_hdl.upload_path)
+ app.package_state = 'STAGED'
+ app.update_run_count()
+ else
+ CloudController.logger.warn("Staging task for app_id=#{app.id} failed: #{result.error}",
+ :tags => [:staging])
+ CloudController.logger.debug1("Details: #{result.task_log}", :tags => [:staging])
+ raise CloudError.new(CloudError::APP_STAGING_ERROR, result.error.to_s)
+ end
+
+ rescue => e
+ # It may be the case that upload from the stager will happen sometime in the future.
+ # Mark the upload as completed so that any upload that occurs in the future will fail.
+ if ul_hdl
+ StagingController.complete_upload(ul_hdl)
+ FileUtils.rm_f(ul_hdl.upload_path)
+ end
+ # This is in keeping with the old CC behavior. Instead of starting a single
+ # instance of a broken app (which is effectively stopped after HM flapping logic
+ # is triggered) we stop it explicitly.
+ app.state = 'STOPPED'
+ AppManager.new(app).stopped
+ app.package_state = 'FAILED'
+ app.update_run_count()
+ raise e
+
+ ensure
+ app.save!
+ end
+
def find_app_by_name
# XXX - What do we want semantics to be like for multiple apps w/ same name (possible w/ contribs)
@app = user.apps_owned.find_by_name(params[:name])
@@ -208,7 +270,6 @@ def find_app_by_name
# App from the request params and makes the necessary AppManager calls.
def update_app_from_params(app)
CloudController.logger.debug "app: #{app.id || "nil"} update_from_parms"
-
error_on_lock_mismatch(app)
app.lock_version += 1
@@ -247,7 +308,14 @@ def update_app_from_params(app)
# Process any changes that require action on out part here.
manager = AppManager.new(app)
- manager.stage if app.needs_staging?
+
+ if app.needs_staging?
+ if user.uses_new_stager?
+ stage_app(app)
+ else
+ manager.stage
+ end
+ end
if changed.include?('state')
if app.stopped?
@@ -405,4 +473,6 @@ def check_has_capacity_for?(app, previous_state)
raise CloudError.new(CloudError::ACCOUNT_NOT_ENOUGH_MEMORY, "#{mem_quota}M")
end
end
+
+
end
Oops, something went wrong.

0 comments on commit 218c19d

Please sign in to comment.