Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

dea_ng latest.

Change-Id: I9c410b8fecc1bbe656ae52b794621d3c02d3124d
  • Loading branch information...
commit 1b0a18ad45ba68593eae3e9772c275802764a2a4 1 parent 867e326
Tal Garfinkel authored
Showing with 642 additions and 209 deletions.
  1. +18 −10 Gemfile
  2. +75 −49 Gemfile.lock
  3. +17 −18 TODO
  4. +29 −0 config/dea.yml
  5. +12 −6 lib/vcap/dea.rb
  6. +0 −10 lib/vcap/dea/.yardoc/checksums
  7. BIN  lib/vcap/dea/.yardoc/objects/root.dat
  8. BIN  lib/vcap/dea/.yardoc/proxy_types
  9. +7 −19 lib/vcap/dea/app_cache.rb
  10. +32 −0 lib/vcap/dea/config.rb
  11. +4 −0 lib/vcap/dea/convert.rb
  12. +1 −1  lib/vcap/dea/env_builder.rb
  13. +1 −0  lib/vcap/dea/errors.rb
  14. +212 −0 lib/vcap/dea/file_viewer.rb
  15. +190 −73 lib/vcap/dea/handler.rb
  16. +4 −2 lib/vcap/dea/http_util.rb
  17. +0 −1  lib/vcap/dea/message.rb
  18. +18 −16 lib/vcap/dea/server.rb
  19. +17 −4 lib/vcap/dea/warden_env.rb
  20. BIN  vendor/cache/addressable-2.2.6.gem
  21. BIN  vendor/cache/crack-0.1.8.gem
  22. BIN  vendor/cache/daemons-1.1.8.gem
  23. BIN  vendor/cache/diff-lcs-1.1.3.gem
  24. BIN  vendor/cache/em-http-request-0.3.0.gem
  25. BIN  vendor/cache/em-posix-spawn-0.0.2.gem
  26. BIN  vendor/cache/em-warden-client-0.0.1.gem
  27. BIN  vendor/cache/escape_utils-0.2.4.gem
  28. BIN  vendor/cache/eventmachine-0.12.10.gem
  29. BIN  vendor/cache/json_pure-1.6.5.gem
  30. BIN  vendor/cache/little-plugger-1.1.3.gem
  31. BIN  vendor/cache/logging-1.6.2.gem
  32. BIN  vendor/cache/nats-0.4.10.gem
  33. BIN  vendor/cache/posix-spawn-0.3.6.gem
  34. BIN  vendor/cache/rack-1.4.1.gem
  35. BIN  vendor/cache/rake-0.9.2.2.gem
  36. BIN  vendor/cache/rspec-2.8.0.gem
  37. BIN  vendor/cache/rspec-core-2.8.0.gem
  38. BIN  vendor/cache/rspec-expectations-2.8.0.gem
  39. BIN  vendor/cache/rspec-mocks-2.8.0.gem
  40. BIN  vendor/cache/thin-1.3.1.gem
  41. BIN  vendor/cache/vcap_common-1.0.0.gem
  42. BIN  vendor/cache/vcap_logging-0.1.4.gem
  43. BIN  vendor/cache/webmock-1.7.4.gem
  44. BIN  vendor/cache/yajl-ruby-1.1.0.gem
  45. +1 −0  vendor/checkout/em-posix-spawn
  46. +1 −0  vendor/checkout/em-warden-client
  47. +1 −0  vendor/checkout/eventmachine
  48. +1 −0  vendor/checkout/vcap_common
  49. +1 −0  vendor/checkout/vcap_logging
View
28 Gemfile
@@ -1,15 +1,23 @@
-source :rubygems
+source "http://rubygems.org"
-gem 'em-http-request'
-gem 'em-warden-client'
-gem 'em-posix-spawn'
-gem 'nats'
+gem 'bundler', '>= 1.0.10'
+gem 'nats', :require => 'nats/client'
+gem 'eventmachine', :git => 'git://github.com/cloudfoundry/eventmachine.git', :branch => 'release-0.12.11-cf'
+gem 'em-http-request', '~> 1.0.0.beta.3', :require => 'em-http'
-gem 'vcap_common'
-gem 'vcap_logging'
+gem 'em-warden-client', :git => 'git://github.com/cloudfoundry/warden.git', :ref => '1df76f804'
+gem 'em-posix-spawn', :git => 'git://github.com/cloudfoundry/common.git', :ref => '3f6636fca4fc'
+
+gem 'rack', :require => ["rack/utils", "rack/mime"]
+gem 'rake'
+gem 'thin'
+gem 'yajl-ruby', :require => ['yajl', 'yajl/json_gem']
+
+# FIXME: we should use the CF org instead of Jesse's personal repo...
+gem 'vcap_common', '~> 1.0.8', :git => 'git://github.com/cloudfoundry/vcap-common.git', :ref => '9673dced'
+gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => '3f6636fca4fc'
group :test do
- gem 'rake'
- gem 'rspec', '>= 2.0.0'
- gem 'webmock'
+ gem "rspec"
+ gem "ci_reporter"
end
View
124 Gemfile.lock
@@ -1,71 +1,97 @@
+GIT
+ remote: git://github.com/cloudfoundry/common.git
+ revision: 3f6636fca4fcba81c85db74ff647e15ed3743a81
+ ref: 3f6636fca4fc
+ specs:
+ em-posix-spawn (0.1.0)
+ eventmachine
+ posix-spawn
+ vcap_logging (1.0.2)
+ rake
+
+GIT
+ remote: git://github.com/cloudfoundry/eventmachine.git
+ revision: 2806c630d8631d5dcf9fb2555f665b829052aabe
+ branch: release-0.12.11-cf
+ specs:
+ eventmachine (0.12.11.cloudfoundry.3)
+
+GIT
+ remote: git://github.com/cloudfoundry/vcap-common.git
+ revision: 9673dcedf0c2daf46e3592a8f9b30538c5dc7b56
+ ref: 9673dced
+ specs:
+ vcap_common (1.0.8)
+ eventmachine (~> 0.12.11.cloudfoundry.3)
+ nats (~> 0.4.22.beta.8)
+ posix-spawn (~> 0.3.6)
+ thin (~> 1.3.1)
+ yajl-ruby (~> 0.8.3)
+
+GIT
+ remote: git://github.com/cloudfoundry/warden.git
+ revision: 1df76f804578ce7299abf235bd6b64085e523b92
+ ref: 1df76f804
+ specs:
+ em-warden-client (0.0.1)
+ eventmachine
+ yajl-ruby
+
GEM
remote: http://rubygems.org/
specs:
- addressable (2.2.6)
- crack (0.1.8)
+ addressable (2.2.8)
+ builder (3.0.0)
+ ci_reporter (1.7.0)
+ builder (>= 2.1.2)
daemons (1.1.8)
diff-lcs (1.1.3)
- em-http-request (0.3.0)
- addressable (>= 2.0.0)
- escape_utils
- eventmachine (>= 0.12.9)
- em-posix-spawn (0.0.2)
+ em-http-request (1.0.0.beta.3)
+ addressable (>= 2.2.3)
+ em-socksify
eventmachine
- posix-spawn
- em-warden-client (0.0.1)
+ http_parser.rb (>= 0.5.1)
+ em-socksify (0.1.0)
eventmachine
- yajl-ruby
- escape_utils (0.2.4)
- eventmachine (0.12.10)
- json_pure (1.6.5)
- little-plugger (1.1.3)
- logging (1.6.2)
- little-plugger (>= 1.1.3)
- nats (0.4.10)
- daemons (>= 1.1.0)
+ http_parser.rb (0.5.3)
+ json_pure (1.7.3)
+ nats (0.4.24)
+ daemons (>= 1.1.5)
eventmachine (>= 0.12.10)
- json_pure (>= 1.5.1)
+ json_pure (>= 1.7.3)
+ thin (>= 1.3.1)
posix-spawn (0.3.6)
rack (1.4.1)
rake (0.9.2.2)
- rspec (2.8.0)
- rspec-core (~> 2.8.0)
- rspec-expectations (~> 2.8.0)
- rspec-mocks (~> 2.8.0)
- rspec-core (2.8.0)
- rspec-expectations (2.8.0)
- diff-lcs (~> 1.1.2)
- rspec-mocks (2.8.0)
+ rspec (2.10.0)
+ rspec-core (~> 2.10.0)
+ rspec-expectations (~> 2.10.0)
+ rspec-mocks (~> 2.10.0)
+ rspec-core (2.10.1)
+ rspec-expectations (2.10.0)
+ diff-lcs (~> 1.1.3)
+ rspec-mocks (2.10.1)
thin (1.3.1)
daemons (>= 1.0.9)
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
- vcap_common (1.0.0)
- eventmachine (~> 0.12.10)
- logging (>= 1.5.0)
- nats
- posix-spawn
- rake
- thin
- vcap_logging
- yajl-ruby
- vcap_logging (0.1.4)
- rake
- webmock (1.7.4)
- addressable (~> 2.2, > 2.2.5)
- crack (>= 0.1.7)
- yajl-ruby (1.1.0)
+ yajl-ruby (0.8.3)
PLATFORMS
ruby
DEPENDENCIES
- em-http-request
- em-posix-spawn
- em-warden-client
+ bundler (>= 1.0.10)
+ ci_reporter
+ em-http-request (~> 1.0.0.beta.3)
+ em-posix-spawn!
+ em-warden-client!
+ eventmachine!
nats
+ rack
rake
- rspec (>= 2.0.0)
- vcap_common
- vcap_logging
- webmock
+ rspec
+ thin
+ vcap_common (~> 1.0.8)!
+ vcap_logging!
+ yajl-ruby
View
35 TODO
@@ -1,34 +1,36 @@
###Features:
+-audit resource tracking, make sure code is sound.
-fix up dea status handler
-look at how stopped app state gets cleaned up.
--look at how to add crashed app monitoring.
+-finish varz support.
--look into how we do stats monitoring, implement monitoring, finish varz support.
+-should we sanity check how much memory/disk we advertise?
--add resource usage monitoring for memory and disk.
+-add high water mark check for disk usage based on total disk on the machine.
--setup dea file serving using nginx.
- -figure out authentication story.
- -create simple nginx conf.
- -test stand-alone.
- -move into container.
- -test in container.
- -figure out how to package with gem.
+###Bugz:
+-make sure usage stats, and vmc crashes works...exercise application info features.
--should we sanity check how much memory/disk we advertise?
+-bound number of redundant crashed instances we keep around.
-###Bugz:
+-for some reason health manager keeps trying to restart crashed apps.
+
+-apps can get stuck between STARTING and RUNNING if stuff blocks in warden
+under load (demonstrate by running app with many instances > 8).
+
+-get_stats can get called on a bad handle.
+
+-exercise update feature, make sure its working and happ.
--apps can be left in a stopped state...
###Cleanup:
+-make use of logid consistent.
-attend to all XXX
-
###Testing:
-get BVT's running with new DEA.
@@ -46,10 +48,7 @@
###Structure:
- -refactor handler to ease unit testing.
+ -refactor handler to ease unit testing (i.e. make instances a class)
-add types to all untyped exceptions.
-###Packaging/Deployment:
- -add gem spec
- -get dea2 bosh packaging.
View
29 config/dea.yml
@@ -1,15 +1,33 @@
---
+# Base directory for dea, application directories, dea temp files, etc. are all relative to this.
base_dir: /var/vcap.local/dea2
+
+#the pid file, created at startup, used for job managment.
pid_filename: /var/vcap/sys/run/dea2.pid
+
+#where the NATS message bus is located
nats_uri: nats://localhost:4222/
+
+# Local_route is the IP address of a well known server on your network, it
+# is used to choose the right ip address (think of hosts that have multiple nics
+# and IP addresses assigned to them) of the host running the DEA. Default
+# value of nil, should work in most cases.
+local_route:
+
+#where the server that provides access to application home directories (log files, etc.) lives.
+file_viewer_port: 12345
+
logging:
level: debug
#helpful for development
reset_at_startup: true
+#where we expect the warden to live
warden_socket_path: /tmp/warden.sock
+#resource limits - node limits indicate limits for this dea node.
+#app quotas are limits imposed on a per-application basis.
resources:
node_limits:
max_memory: 4096
@@ -19,6 +37,17 @@ resources:
mem_quota: 512
disk_quota: 256
+#use this to specify mount points that will always be included in the container.
+#useful, for example, if you want to just put all your runtimes under one common
+#tree.
+#mounts:
+# - /tmp, /tmp/outside-tmp, ro
+# - /var/vcap, /var/vcap, ro
+
+#allow disabling of mounting runtimes, can be used in combination with mounts
+#to provide an alternate means of supplying runtimes.
+mount_runtimes: true
+
# This is where the execution agent determines its available runtimes.
# version flags are assumed to be '-v' unless noted below.
View
18 lib/vcap/dea.rb
@@ -1,3 +1,5 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'dea'))
+
require 'vcap/common'
require 'vcap/logging'
require 'vcap/component'
@@ -31,16 +33,15 @@ def init(config_file)
def start_server!
VCAP::Logging.setup_from_config(config[:logging])
- @logger = VCAP::Logging.logger('dea')
+ @logger = VCAP::Logging.logger('dea_ng')
@logger.info "Starting VCAP DEA version #{VCAP::Dea::VERSION}, pid: #{Process.pid}."
sub_dirs = %w[tmp droplets db instances]
base_dir = @config[:base_dir]
setup_pidfile
setup_directories(base_dir, sub_dirs)
clean_directories(sub_dirs) if @config[:reset_at_startup]
- params = { :resources => config[:resources],
- :runtimes => config[:runtimes],
- :directories => @directories }
+ params = @config.dup
+ params[:directories] = @directories
nats_uri = @config[:nats_uri]
@logger.info "Using #{nats_uri}."
handler = VCAP::Dea::Handler.new(params, @logger)
@@ -51,10 +52,11 @@ def start_server!
['TERM', 'INT', 'QUIT'].each { |s| trap(s) { @server.shutdown() } }
trap('USR2') { @server.evacuate_apps_then_quit() }
+ trap('USR1') { @logger.error("Got SIGUSR1 - don't know what that means, SIGUSR2 to evactuate apps, SIGINT to shutdown.")}
at_exit { clean_directories(%w[tmp]) } #prevent storage leaks.
-
EventMachine.run {
@server.start
+ handler.start_file_viewer
}
end
@@ -87,10 +89,14 @@ def clean_directories(sub_dirs)
}
end
+ def add_directory(base_dir, name)
+ @directories[name] = File.join(base_dir, name)
+ end
+
def setup_directories(base_dir, sub_dirs)
@directories = Hash.new
sub_dirs.each {|name|
- @directories[name] = File.join(base_dir, name)
+ add_directory(base_dir, name)
}
@directories.each { |name, path|
unless Dir.exists?(path)
View
10 lib/vcap/dea/.yardoc/checksums
@@ -1,10 +0,0 @@
-app.rb df058b42242eb1b83dddb7303ccd9f3ab7351f47
-server.rb 3aac562a527bc7878d42839468cefe42fc344177
-errors.rb f4b83406d6c68f4d4e2cd4cee83e29cb11ef248a
-version.rb 0a4b90331890eb46b5547a7be6c5da4b30fc4715
-handler.rb b256314fbd87a3e33310663adc650d7cb27cf12e
-message.rb 7f16b97150d16d3d4d7e3acb0ff13f178d5685a9
-resources.rb aa46ce42f7f45ad0e69e5b792781bcca0a2018d6
-http_util.rb b51fb23214c5e4b45bf5ec6020108d67ca8718e9
-app_instance.rb 5bb301c67113c4cc37f538732a93b6d8f8731058
-fiber_aware_helpers.rb 70dfdea53f1d6f96bc2aaeaff2e38c5c0f53d149
View
BIN  lib/vcap/dea/.yardoc/objects/root.dat
Binary file not shown
View
BIN  lib/vcap/dea/.yardoc/proxy_types
Binary file not shown
View
26 lib/vcap/dea/app_cache.rb
@@ -42,7 +42,7 @@ def download_droplet(uri, sha1)
raise VCAP::Dea::HandlerError, "Missing download information."
end
- unless droplet_tgz_path = VCAP::Dea::HttpUtil.download(uri)
+ unless droplet_tgz_path = VCAP::Dea::HttpUtil.download(uri, @directories['tmp'])
@logger.warn("Failed downloading droplet from '#{uri}'")
raise VCAP::Dea::HandlerError, "Failed downloading droplet"
end
@@ -55,25 +55,13 @@ def download_droplet(uri, sha1)
raise VCAP::Dea::HandlerError, "SHA1 mismatch"
end
- tmp_dir = Dir.mktmpdir(nil, @directories['tmp'])
droplet_dir = File.join(@directories['droplets'], sha1)
- status, stdout, stderr = sh("tar -C #{tmp_dir} -xzf #{droplet_tgz_path}")
- if status.exitstatus == 0
- @logger.debug("unpacked droplet to #{tmp_dir}.")
- else
- @logger.warn("Failed extracting #{droplet_tgz_path}")
- @logger.warn("STDOUT: #{stdout}")
- @logger.warn("STDERR: #{stderr}")
- raise VCAP::Dea::HandlerError, "Droplet extraction failed"
- end
- File.rename(tmp_dir, droplet_dir)
- @logger.debug "moved droplet to #{droplet_dir}."
- rescue
- defer { FileUtils.rm_rf(tmp_dir) } if tmp_dir
- raise
-
- ensure
- defer { FileUtils.rm_f(droplet_tgz_path) } if droplet_tgz_path
+ droplet_path = File.join(droplet_dir, 'droplet.tgz')
+ FileUtils.mkdir_p(droplet_dir)
+ File.rename(droplet_tgz_path, droplet_path)
+ File.chmod(0744, droplet_path)
+
+ @logger.debug("move droplet to #{droplet_path}")
end
end
View
32 lib/vcap/dea/config.rb
@@ -11,6 +11,8 @@ class VCAP::Dea::Config < VCAP::Config
:base_dir => String, # where all dea stuff lives
:pid_filename => String, # where our pid file lives.
:reset_at_startup => VCAP::JsonSchema::BoolSchema.new, #blow away saved state at startup.
+ #XXX make this optionally nil, but still in schema:local_route => String,
+ :file_viewer_port => Integer,
:resources => {
:node_limits => {
:max_memory => Integer,
@@ -29,7 +31,9 @@ class VCAP::Dea::Config < VCAP::Config
optional(:syslog) => String, # Name to associate with syslog messages (should start with 'vcap.')
},
+ #XXX add support for mounts: to schema
:runtimes => VCAP::JsonSchema::HashSchema.new,
+ :mount_runtimes => VCAP::JsonSchema::BoolSchema.new, #should we mount the runtime?
}
end
@@ -39,9 +43,37 @@ def from_file(*args)
config = super(*args)
normalize_config(config)
validate_runtimes(config[:runtimes])
+ parse_mounts(config)
config
end
+ #XXX add support to config parser for checking sequences.
+ def parse_mounts(config)
+ mounts = config[:mounts] || []
+ new_mounts = []
+ valid_modes = ['ro','rw'].freeze
+ mounts = [] unless mounts
+ mounts.each do |line|
+ puts "line: #{line}"
+ src_path, dst_path, mode = line.split(',').map {|s| s.strip}
+
+ unless Dir.exist?(src_path)
+ puts "Directory #{src_path} in mount line #{line} does not exists!."
+ exit 1
+ end
+ unless dst_path
+ puts "invalid mount line: #{line}. valid syntax is src_path, dst_path, mode"
+ exit 1
+ end
+ unless valid_modes.include? mode
+ puts "invalid mount line: #{line}. mode must be either ro or rw"
+ exit 1
+ end
+ new_mounts.push([src_path, dst_path, mode])
+ end
+ config[:mounts] = new_mounts
+ end
+
def validate_runtimes(runtimes)
if runtimes.nil? || runtimes.empty?
puts("Can't determine application runtimes, exiting")
View
4 lib/vcap/dea/convert.rb
@@ -8,4 +8,8 @@ def MB_to_bytes(mb)
def bytes_to_MB(bytes)
(bytes / (1024*1024)).to_i
end
+
+ def bytes_to_GB(bytes)
+ (bytes / (1024*1024*1024)).to_i
+ end
end
View
2  lib/vcap/dea/env_builder.rb
@@ -59,7 +59,7 @@ def debug_env(instance)
def runtime_env(runtime_name)
env = []
if runtime_name && runtime = @runtimes[runtime_name]
- if re = runtime['environment']
+ if re = runtime[:environment]
re.each { |k,v| env << "#{k}=#{v}"}
end
end
View
1  lib/vcap/dea/errors.rb
@@ -3,5 +3,6 @@ module Dea
class Error < StandardError; end
class HandlerError < Error; end
class WardenError < Error; end
+ class HttpDownLoadError < Error; end
end
end
View
212 lib/vcap/dea/file_viewer.rb
@@ -0,0 +1,212 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require 'thin'
+require 'logger'
+require 'pathname'
+require 'thin'
+require 'vcap/common'
+require 'vcap/logging'
+
+
+# Rack::Directory serves entries below the +root+ given, according to the
+# path info of the Rack request. If a directory is found, the file's contents
+# will be presented in an html based index. If a file is found, the env will
+# be passed to the specified +app+.
+#
+# If +app+ is not specified, a Rack::File of the same +root+ will be used.
+
+module VCAP module Dea end end
+
+module VCAP::Dea
+ class FileViewer
+ attr_reader :auth_info, :ip, :port
+
+ def initialize(ip, port, root_dir, logger = nil)
+ @logger = logger || VCAP::Logging.logger('dea/files')
+ @ip = ip
+ @port = port
+ #XXX maybe refactor this, see also below.
+ #XXX@uri = "http://#{@ip}:#{@port}/instances/",
+ @root_dir = root_dir
+ @auth_info = [VCAP.secure_uuid, VCAP.secure_uuid]
+ end
+
+ # This is for general access to the file system for the staged droplets.
+ def start
+ auth = @auth_info
+ root_dir = @root_dir
+ @file_viewer_server = Thin::Server.new(@ip, @port, :signals => false) do
+ Thin::Logging.silent = true
+ use Rack::Auth::Basic do |username, password|
+ [username, password] == auth
+ end
+ map '/instances' do
+ run VCAP::Dea::Directory.new(root_dir)
+ end
+ end
+ @file_viewer_server.start!
+ @logger.info("File service started, curl --user #{@auth_info[0]}:#{auth_info[1]} #{@ip}:#{@port}")
+ end
+ end
+
+ class FileServer < Rack::File
+ # based on Rack::File, just add the NOFOLLOW flag
+ def each
+ F.open(@path, File::RDONLY | File::NOFOLLOW | File::BINARY) do |file|
+ file.seek(@range.begin)
+ remaining_len = @range.end-@range.begin+1
+ while remaining_len > 0
+ part = file.read([8192, remaining_len].min)
+ break unless part
+ remaining_len -= part.length
+
+ yield part
+ end
+ end
+ end
+ end
+
+ class Directory
+ attr_reader :files
+ attr_accessor :root, :path
+
+ def initialize(root, app = nil)
+ @root = Pathname.new(F.expand_path(root)).realpath
+ @app = app || FileServer.new(@root)
+ end
+
+ def call(env)
+ dup._call(env)
+ end
+
+ F = ::File
+
+ def _call(env)
+ @env = env
+ @script_name = env['SCRIPT_NAME']
+ @path_info = Rack::Utils.unescape(env['PATH_INFO'])
+ @path = F.expand_path(F.join(@root, @path_info))
+
+ if not File.exists? @path
+ return entity_not_found
+ end
+
+ resolve_symlink
+ if forbidden = check_forbidden
+ forbidden
+ else
+ list_path
+ end
+ end
+
+ def resolve_symlink
+ real_path = Pathname.new(@path).realpath.to_s
+ return if real_path == @path
+
+ # Adjust env only if user has access rights to real path
+ app_base = File.join(@root, @path_info.sub(/^\/+/,'').split('/').first)
+ if real_path.start_with?(app_base)
+ m = real_path.match(@root.to_s)
+ return if m.nil?
+ @env['PATH_INFO'] = @path_info = m.post_match
+ @path = real_path
+ end
+ end
+
+ def check_forbidden
+ forbidden = false
+ forbidden = true if @path_info.include? ".."
+ forbidden = true if @path_info =~ /\/.+\/startup$/
+ forbidden = true if @path_info =~ /\/.+\/stop$/
+
+ # breaks BVTs
+ #forbidden = true if @path_info =~ /\/.+\/run\.pid/
+
+ # Any symlink foolishness checked here
+ check_path = @path.sub(/\/\s*$/,'')
+ forbidden = true if (check_path != Pathname.new(@path).realpath.to_s)
+ return unless forbidden
+
+ body = "Not accessible\n"
+ size = Rack::Utils.bytesize(body)
+ return [403, {"Content-Type" => "text/plain",
+ "Content-Length" => size.to_s,
+ "X-Cascade" => "pass"}, [body]]
+ end
+
+ def list_directory
+ @files = []
+ glob = F.join(@path, '*')
+ root = @path_info.sub(/^\/+/,'').split('/').length <= 1
+
+ Dir[glob].sort.each do |node|
+ stat = stat(node)
+ next unless stat
+
+ basename = F.basename(node)
+ # ignore B29 control files, only return defaults
+ next if root && (basename != 'app' && basename != 'logs' && basename != 'tomcat')
+ size = stat.directory? ? '-' : filesize_format(stat.size)
+ basename << '/' if stat.directory?
+ @files << [ basename, size ]
+ end
+
+ return [ 200, {'Content-Type'=>'text/plain'}, self ]
+ end
+
+ def stat(node, max = 10)
+ F.stat(node)
+ rescue Errno::ENOENT, Errno::ELOOP
+ return nil
+ end
+
+ # TODO: add correct response if not readable, not sure if 404 is the best
+ # option
+ def list_path
+ @stat = F.stat(@path)
+
+ if @stat.readable?
+ return @app.call(@env) if @stat.file?
+ return list_directory if @stat.directory?
+ else
+ raise Errno::ENOENT, 'No such file or directory'
+ end
+
+ rescue Errno::ENOENT, Errno::ELOOP
+ return entity_not_found
+ end
+
+ def entity_not_found
+ body = "Entity not found.\n"
+ size = Rack::Utils.bytesize(body)
+ return [404, {"Content-Type" => "text/plain",
+ "Content-Length" => size.to_s,
+ "X-Cascade" => "pass"}, [body]]
+ end
+
+ def each
+ show_path = @path.sub(/^#{@root}/,'')
+ files = @files.map{|f| "%-35s %10s" % f }*"\n"
+ files.each_line{|l| yield l }
+ end
+
+ # Stolen from Ramaze
+
+ FILESIZE_FORMAT = [
+ ['%.1fT', 1 << 40],
+ ['%.1fG', 1 << 30],
+ ['%.1fM', 1 << 20],
+ ['%.1fK', 1 << 10],
+ ]
+
+ def filesize_format(int)
+ FILESIZE_FORMAT.each do |format, size|
+ return format % (int.to_f / size) if int >= size
+ end
+ int.to_s + 'B'
+ end
+ end
+end
+
+#for standalone testing.
+#f = VCAP::Dea::FileViewer.new('172.16.228.141', '6699', '/tmp')
+#f.start_file_viewer
View
263 lib/vcap/dea/handler.rb
@@ -4,6 +4,7 @@
require 'em/warden/client'
require 'tmpdir'
require 'fiber'
+require 'fileutils'
require 'vcap/common'
require 'vcap/logging'
@@ -19,6 +20,7 @@
require 'vcap/dea/app_cache'
require 'vcap/dea/warden_env'
+require 'vcap/dea/file_viewer'
module VCAP module Dea end end
@@ -26,20 +28,27 @@ class VCAP::Dea::Handler
include VCAP::Dea::FiberAwareHelpers
include VCAP::Dea::Convert
- attr_reader :uuid
+ CRASHED_EXPIRATION_TIME = 60 * 60 #make crashed apps expire once an hour.
+
+ attr_reader :local_ip
+ attr_accessor :uuid
def initialize(params, logger = nil)
@logger = logger || Logger.new(STDOUT)
@uuid = nil
- @local_ip = VCAP.local_ip
+ @local_ip = VCAP.local_ip(params[:local_route])
@num_cores = VCAP.num_cores
@logger.info "Local ip: #{@local_ip}."
@logger.info "Using #{@num_cores} cores."
@runtimes = params[:runtimes]
@directories = params[:directories]
+ @global_mounts = params[:mounts]
+ @mount_runtimes = params[:mount_runtimes]
+ @logger.info "Global container mounts: #{@global_mounts.pretty_inspect}."
@logger.info "Supported runtimes: #{@runtimes.keys}."
@varz = {}
@droplets = {}
+ @file_viewer_port = params[:file_viewer_port]
limits = params[:resources][:node_limits]
init_resources = {:memory => limits[:max_memory],
:disk => limits[:max_disk],
@@ -50,28 +59,37 @@ def initialize(params, logger = nil)
@app_cache = VCAP::Dea::AppCache.new(@directories, @logger)
@snapshot = VCAP::Dea::Snapshot.new(@directories['db'], @logger)
- end
-
- def set_uuid(uuid)
- @uuid = uuid
+ @file_viewer = VCAP::Dea::FileViewer.new(@local_ip, @file_viewer_port, @directories['instances'], @logger)
+ @hello_message = {:id => @uuid, :ip => @local_ip, :port => @file_viewer.port, :version => VCAP::Dea::VERSION }.freeze
+ @memory_in_use = 0
end
def fetch_and_update_varz
@varz[:apps_max_memory] = @resource_tracker.max[:memory]
@varz[:apps_reserved_memory] = @resource_tracker.reserved[:memory]
- @varz[:apps_used_memory] = 000 #XXX fill me in
+ @varz[:apps_used_memory] = @memory_in_use
@varz[:num_apps] = @resource_tracker.reserved[:instances]
@varz
end
+ def start_file_viewer
+ @file_viewer.start
+ end
+
def get_advertise
#XXX return nil if !space_available?
+ #XXX should look into adding physical resource limit checks similar to
+ #XXX what matt added to current DEA.
msg = { :id => @uuid,
:available_memory => @resource_tracker.available[:memory],
:runtimes => @runtimes.keys}
msg
end
+ def get_hello
+ @hello_message
+ end
+
def generate_heartbeat(instance)
{
:droplet => instance[:droplet_id],
@@ -94,7 +112,7 @@ def get_heartbeat
heartbeat
end
- def handle_hm_start
+ def handle_hm_start(msg)
msg.respond('dea.heartbeat', get_heartbeat)
end
@@ -130,15 +148,10 @@ def handle_router_start(msg)
end
def handle_status(msg)
- status_msg = {:id => @uuid,
- :ip => @local_ip,
- :version => VCAP::Dea::VERSION,
- :port => 0, #XXX this is just a place holder, not meaningful in DEA2
- }
-
+ status_msg = @hello_message.dup
status_msg[:max_memory] = @resource_tracker.max[:memory]
status_msg[:reserved_memory] = @resource_tracker.reserved[:memory]
- status_msg[:used_memory] = 0 #XXX fill me in once we track usage dynamically
+ status_msg[:used_memory] = @memory_in_use
status_msg[:num_clients] = @resource_tracker.reserved[:instances]
msg.reply(status_msg)
@@ -176,7 +189,7 @@ def delete_instance(instance)
droplet_id = instance[:droplet_id]
instance_id = instance[:instance_id]
instances = lookup_instances(droplet_id)
- if instances.empty?
+ if instances == nil || instances.empty?
@logger.warn("couldn't delete #{droplet_id}:#{instance_id}, instance not found.")
return
end
@@ -198,8 +211,46 @@ def add_instance(instance)
@logger.debug("added droplet/index: #{droplet_id}: #{instance[:instance_index]} to droplet list.")
end
+ def update_cached_resource_usage
+ droplets = @droplets.dup #since we can call yield inside our itterator, use a copy to avoid
+ #conflicting with insertions into the hash.
+ droplets.each_value do |instances|
+ instances.each_value do |instance|
+ update_instance_usage(instance)
+ end
+ end
+ end
+
+ def update_total_resource_usage
+ total = 0
+ @droplets.each_value do |instances|
+ instances.each_value do |instance|
+ total += instance[:cached_usage][:mem]
+ end
+ end
+ @memory_in_use = total
+ end
+
+ def update_instance_usage(instance)
+ #XXX get cpu and disk stats working pending wardens upport
+ cur_usage = { :time => Time.now, :cpu => 0, :mem => 0, :disk => 0 }
+ warden_env = instance[:warden_env]
+ if warden_env
+ stats = warden_env.get_stats
+ if stats
+ cur_usage[:mem] = bytes_to_GB(stats[:mem_usage_B]) #XXX double check units on this.
+ cur_usage[:disk] = stats[:disk_usage_B] #XXX not yet implemented in warden.
+ end
+ end
+ instance[:cached_usage] = cur_usage
+ end
+
+ #Note on states
+ # instance[:state] should either be :RUNNING or :CRASHED most of the time.
+ # :STARTING and :DELETED are ephemeral states, if there are lingering
+ # instances with them in the heartbeat, this is a sure sign of a bug.
def set_instance_state(instance, new_state)
- valid_states = [:RUNNING, :CRASHED, :DELETED].freeze
+ valid_states = [:STARTING, :RUNNING, :CRASHED, :DELETED].freeze
raise VCAP::Dea::HandlerError, "invalid state #{new_state}" unless valid_states.include? new_state
instance[:state] = new_state
instance[:state_timestamp] = Time.now.to_i
@@ -219,13 +270,23 @@ def attach_container(instance, warden_env)
instance[:warden_container_info] = warden_env.get_container_info
end
- def release_container(instance)
- env = instance[:warden_env]
- unless env
- @logger.error "no container attached to instance #{instance[:instance_id]}"
- raise VCAP::Dea::HandlerError, "no container attached to instance, couldn't free"
+ def detach_container(instance)
+ instance.delete(:warden_env)
+ instance.delete(:warden_container_info)
+ end
+
+ def container_attached?(instance)
+ instance.has_key?(:warden_env) && instance[:warden_env] != nil
+ end
+
+ def destroy_container(instance)
+ if not container_attached?(instance)
+ @logger.error "no container attached to instance #{instance[:instance_id]}, couldn't free."
+ else
+ env = instance[:warden_env]
+ env.destroy!
+ detach_container(instance)
end
- env.destroy!
end
def export_mem_quota(instance)
@@ -237,8 +298,8 @@ def export_disk_quota(instance)
end
def setup_network_ports(warden_env, instance, debug, console)
- http_port = warden_env.alloc_network_port
- instance[:port] = http_port
+ app_port = warden_env.alloc_network_port
+ instance[:port] = app_port
if debug
debug_port = warden_env.alloc_network_port
@@ -264,8 +325,29 @@ def droplet_id_index_in_use?(droplet_id, instance_index)
false
end
+ def alloc_instance_dir(instance)
+ base_dir = @directories['instances']
+ instance_dir = File.join(base_dir, instance[:instance_id])
+ FileUtils.mkdir(instance_dir)
+ File.chmod(01777, instance_dir)
+ instance[:instance_dir] = instance_dir
+ end
+
+ def valid_instance_dir?(instance)
+ instance.has_key?(:instance_dir) && instance[:instance_dir] && Dir.exists?(instance[:instance_dir])
+ end
+
+ def free_instance_dir(instance)
+ if valid_instance_dir?(instance)
+ instance_dir = instance[:instance_dir]
+ FileUtils.rm_rf instance_dir, :secure => true
+ else
+ @logger.error("Free instance dir failed")
+ end
+ end
+
def start_instance(msg)
- @logger.debug("DEA received start message: #{msg.details}")
+ #XXX @logger.debug("DEA received start message: #{msg.details}")
instance_id = VCAP.secure_uuid
@@ -285,12 +367,13 @@ def start_instance(msg)
debug = msg.details['debug']
console = msg.details['console']
limits = msg.details['limits']
+ flapping = msg.details['flapping']
@logger.info("Trying to start instance (name: #{name} index:#{instance_index} id: #{droplet_id})")
if droplet_id_index_in_use?(droplet_id, instance_index)
@logger.warn("got start request for existing id:#{droplet_id} index:#{instance_index}")
- raise VCAP::Dea::HandlerError, "duplicate start request"
+#XXX -- something fucked here raise VCAP::Dea::HandlerError, "duplicate start request"
end
unless @runtimes[runtime]
@@ -298,14 +381,13 @@ def start_instance(msg)
raise VCAP::Dea::HandlerError, "Runtime unsupported"
end
- begin
- #XXX check if resource to run app are available and reserve them
- request = {:memory => limits && limits['mem'] ? limits['mem'] : @default_app_quota[:mem_quota],
- :disk => limits && limits['disk'] ? limits['disk'] : @default_app_quota[:disk_quota],
- :instances => 1}
- resources = @resource_tracker.reserve(request)
- raise VCAP::Dea::HandlerError, "Failed to provision resources #{request}." unless resources
+ request = {:memory => limits && limits['mem'] ? limits['mem'] : @default_app_quota[:mem_quota],
+ :disk => limits && limits['disk'] ? limits['disk'] : @default_app_quota[:disk_quota],
+ :instances => 1}
+ resources = @resource_tracker.reserve(request)
+ raise VCAP::Dea::HandlerError, "Failed to provision resources #{request}." unless resources
+ begin
instance = {
:droplet_id => droplet_id,
:instance_id => instance_id,
@@ -320,74 +402,91 @@ def start_instance(msg)
:runtime => runtime,
:framework => framework,
:start => Time.now,
- :state_timestamp => Time.now.to_i,
+ :flapping => flapping ? true : false,
:log_id => "(name=%s app_id=%s instance=%s index=%s)" % [name, droplet_id, instance_id, instance_index],
}
+ update_instance_usage(instance)
+ set_instance_state(instance, :STARTING) #:STARTING state exists so we know that the associated droplet.tgz is
+ #in use and doesn't get removed out from under us.
+ add_instance(instance) #this is now live, needs to be in a consistent state from here on out.
+
#check if bits already in cache, if not download them.
- existing_droplet = @app_cache.has_droplet?(sha1)
- @app_cache.download_droplet(bits_uri, sha1) unless existing_droplet
+ @app_cache.download_droplet(bits_uri, sha1) unless @app_cache.has_droplet?(sha1)
droplet_dir = @app_cache.droplet_dir(sha1)
+ droplet_mnt = [droplet_dir, droplet_dir, 'ro']
runtime_path = @runtimes[runtime][:executable]
runtime_dir = File.dirname(File.dirname(runtime_path))
+ runtime_mnt = [runtime_dir, runtime_dir, 'ro']
+
+ src_home_dir = alloc_instance_dir(instance)
+ dst_home_dir = '/home/vcap'
+ home_dir_mnt = [src_home_dir, dst_home_dir, 'rw']
+
+ mounts = [droplet_mnt, home_dir_mnt]
+ mounts = mounts + @global_mounts
+ mounts << runtime_mnt if @mount_runtimes
+
- mounts = [droplet_dir, runtime_dir]
warden_env = VCAP::Dea::WardenEnv.new(@logger)
warden_env.create_container(mounts, resources)
setup_network_ports(warden_env, instance, debug, console)
+ @logger.debug("warden setup complete for: #{instance_index}")
+
env_builder = VCAP::Dea::EnvBuilder.new(@runtimes, @local_ip, @logger)
new_env = env_builder.setup_instance_env(instance, app_env, services)
- @logger.debug("new_env #{new_env}")
- status, out, err = warden_env.run("cp -r #{droplet_dir}/* .")
- raise VCAP::Dea::HandlerError, "Failed to copy droplet bits:#{out}:#{err}" unless status == 0
+
+ status, out, err = warden_env.run("tar xzf #{droplet_dir}/droplet.tgz")
+ raise VCAP::Dea::HandlerError, "Failed to extract droplet bits:#{out}:#{err}" unless status == 0
#XXX FIXME
#XXX sed lets us delimit our pattern with any charecter, we use @ to avoid frobbing path names
#XXX this could screw us if a runtime path contains @, should do something safer here
- warden_env.run("sed s@%VCAP_LOCAL_RUNTIME%@#{runtime_path}@ < startup > startup.ready; chmod u+x startup.ready")
+ warden_env.run("sed s@%VCAP_LOCAL_RUNTIME%@#{runtime_path}@g < startup > startup.ready; chmod u+x startup.ready")
env_str = new_env.join(" ")
+ @logger.debug("app setup complete for: #{instance_index}")
+
#add to instance list and let it rip.
#XXX think about the order of this and potential failures
+ #XXX potential refactor for more sharing with resume_instance
set_instance_state(instance, :RUNNING)
- add_instance(instance)
- warden_env.spawn("./startup.ready -p #{instance[:port]}")
+ warden_env.spawn("env -i #{env_str} ./startup.ready -p #{instance[:port]}")
+ @logger.debug("spawned: #{instance_index}")
attach_container(instance, warden_env)
+ update_instance_usage(instance)
+ @logger.debug("about to link: #{instance_index}")
result = warden_env.link
- app_exit_handler(instance, result)
+ app_exit_handler(instance, result, msg)
rescue => e
@logger.error "error while provisioning instance #{instance_id}:#{e.message}"
@logger.error e.backtrace.join("\n")
- #XXX delete instance
- #XXX try to use remove_and_clean_up_instance here. and get rid of the !existing droplet stuff.
- @app_cache.purge_droplet!(sha1) if @app_cache.has_droplet?(sha1) and !existing_droplet
- @resource_tracker.release(resources) if resources
- warden_env.destroy! if warden_env
- raise e
+ remove_and_clean_up_instance(instance)
end
end
- def app_exit_handler(instance, result)
+ def app_exit_handler(instance, result, msg)
instance_id = instance[:instance_id]
status, out, err = result
@logger.info("instance #{instance_id} exited,<#{status}, out: #{out}, err: #{err}")
if status != nil && instance[:state] == :RUNNING
set_instance_state(instance, :CRASHED)
set_exit_reason(instance, :CRASHED)
+ send_exited_message(msg, instance) # unplug from routers and health managers.
+ release_container(instance) #XXX should consolidate and simplify exit paths.
end
- #XXX stash log files.
#XXX check status code, deal with app startup failure.
end
- CRASHED_EXPIRATION_TIME = 10 #XXX make this longer for production
def remove_expired_crashed_apps
- @droplets.each_value do |instances|
+ droplets = @droplets.dup #avoid conflicting with new insertions
+ droplets.each_value do |instances|
instances.each_value do |instance|
if instance[:state] == :CRASHED && (Time.now.to_i - instance[:state_timestamp]) > CRASHED_EXPIRATION_TIME
@logger.debug("Crashed instance: #{instance[:instance_id]} has expired.")
@@ -422,28 +521,34 @@ def resume_detached_containers
def resume_instance(instance)
@logger.info("trying to resume instance #{instance[:instance_id]}")
- warden_env = VCAP::Dea::WardenEnv.new(@logger)
- container_info = instance[:warden_container_info]
- warden_env.bind_container(container_info)
- attach_container(instance, warden_env)
begin
- result = warden_env.link
- rescue VCAP::Dea::WardenError => e
- @logger.warn("failed to resume instance #{instance[:instance_id]}:#{e.message}, cleaning up...")
+ warden_env = VCAP::Dea::WardenEnv.new(@logger)
+ container_info = instance[:warden_container_info]
+ warden_env.bind_container(container_info)
+ attach_container(instance, warden_env)
+ update_instance_usage(instance)
+ begin
+ result = warden_env.link
+ rescue VCAP::Dea::WardenError => e
+ @logger.warn("failed to resume instance #{instance[:instance_id]}:#{e.message}, cleaning up...")
+ remove_and_clean_up_instance(instance)
+ return
+ end
+ app_exit_handler(instance, result)
+ rescue => e
+ @logger.error "error while resuming instance #{instance_id}:#{e.message}"
+ @logger.error e.backtrace.join("\n")
remove_and_clean_up_instance(instance)
- return
end
- app_exit_handler(instance, result)
- #XXX add some exception handling here since this won't get caught by
- #XXX the normal event dispatch handler.
end
def handle_stop(msg)
@logger.debug("got stop message #{msg.details.to_s}")
instances = lookup_matching_instances(msg)
- return unless instances
+ return unless instances
instances.each do |instance|
return if instance[:state] == :DELETED
+ @logger.debug("trying to stop #{instance[:log_id]}")
set_instance_state(instance, :DELETED)
set_exit_reason(instance, :STOPPED)
stop_droplet(msg, instance)
@@ -477,9 +582,16 @@ def stop_droplet(msg, instance)
remove_and_clean_up_instance(instance)
end
+ def release_container(instance)
+ if container_attached?(instance)
+ @resource_tracker.release(instance[:resources])
+ destroy_container(instance)
+ end
+ end
+
def remove_and_clean_up_instance(instance)
@logger.debug("removing and cleaning up: #{instance[:instance_id]}")
- @resource_tracker.release(instance[:resources])
+ free_instance_dir(instance)
release_container(instance)
delete_instance(instance)
remove_unused_droplets
@@ -580,9 +692,10 @@ def handle_find_droplet(msg)
:index => instance[:instance_index],
:state => instance[:state],
:state_timestamp => instance[:state_timestamp],
- #XXX :file_uri => "http://#{@local_ip}:#{@file_viewer_port}/droplets/",
- #XXX :credentials => @file_auth,
- #XXX :staged => instance[:staged],
+ #XXX move this computation into the file viewer, figure out whats fucked..
+ :file_uri => "http://#{@file_viewer.ip}:#{@file_viewer.port}/instances/",
+ :credentials => @file_viewer.auth_info,
+ :staged => instance[:instance_id], #instance dir index'ed by instance id
:debug_ip => instance[:debug_ip],
:debug_port => instance[:debug_port],
:console_ip => instance[:console_ip],
@@ -600,7 +713,7 @@ def handle_find_droplet(msg)
#XXX deprecated :fds_quota => instance[:fds_quota],
:cores => @num_cores
}
- #XXX fix me up response[:stats][:usage] = @usage[instance[:pid]].last if @usage[instance[:pid]]
+ response[:stats][:usage] = instance[:cached_usage]
end
msg.reply(response)
end
@@ -622,7 +735,6 @@ def restore_snapshot
recovered = @snapshot.read_snapshot
return unless recovered
@logger.debug "trying to restore snapshot..."
- #XXX add a rescue block to catch errors and log if recovery fails...
recovered.each_pair do |droplet_id, instances|
@droplets[droplet_id.to_i] = instances
instances.each_pair do |instance_id, instance|
@@ -632,9 +744,14 @@ def restore_snapshot
set_instance_state(instance, instance[:state].to_sym)
instance[:resources] = convert_keys_to_symbols(instance[:resources])
instance[:warden_container_info] = convert_keys_to_symbols(instance[:warden_container_info])
+ instance[:warden_env] = nil
instance[:exit_reason] = instance[:exit_reason].to_sym if instance[:exit_reason]
instance[:start] = Time.parse(instance[:start]) if instance[:start]
+ unless valid_instance_dir?(instance)
+ raise VCAP::Dea::HandlerError, "invalid instance dir: #{instance[:instance_dir]}."
+ end
+
unless @resource_tracker.reserve(instance[:resources])
raise VCAP::Dea::HandlerError, "Failed to provision resources #{request}."
end
View
6 lib/vcap/dea/http_util.rb
@@ -1,5 +1,6 @@
require 'em-http'
require 'tempfile'
+require 'errors'
module VCAP
module Dea
@@ -31,9 +32,10 @@ def download(uri, tmp_dir = nil)
nil
end
end
- rescue
+ rescue => e
tmpfile.unlink if tmpfile
- raise
+ raise VCAP::Dea::HttpDownLoadError, "error while downloading uri #{uri}"
+ raise e
end
end
View
1  lib/vcap/dea/message.rb
@@ -25,7 +25,6 @@ def initialize(nats, subj = nil, opts={})
end
def respond(subj, details)
- #XXX more verbose debug message to show what message this is in response to.
@subject = subj
@details = details
send
View
34 lib/vcap/dea/server.rb
@@ -22,19 +22,18 @@ class VCAP::Dea::Server
ADVERTISE_INTERVAL = 5
WARDEN_PING_INTERVAL = 5
VARZ_UPDATE_INTERVAL = 1
- CRASHED_APPS_CLEANUP_INTERVAL = 10 #XXX increase this for production.
+ CRASHED_APPS_CLEANUP_INTERVAL = 30
+ RESOURCE_USAGE_UPDATE_INTERVAL = 5
attr_accessor :handler
def initialize(nats_uri, handler, logger = nil)
+ @logger = logger || Logger.new(STDOUT)
@nats_uri = nats_uri
@handler = handler
@routes = {}
@sids = []
- @logger = logger || Logger.new(STDOUT)
@shutting_down = false
- @local_ip = '127.0.0.1' #XXX should be VCAP.local_ip(config['local_route'])
- @file_viewer_port = 999 #XXX should be config['filer_port']
@logger.debug("server initialized.")
end
@@ -52,7 +51,7 @@ def shutdown
msg = VCAP::Dea::Message.new(@nats)
@logger.info('Starting shutdown..')
Fiber.new { @handler.shutdown(msg) }.resume
- EM.add_timer(0.25) do #allow messages time to get out XXX crank this down
+ NATS.flush do
NATS.stop { EM.stop }
@logger.info('Shutdown complete..')
exit
@@ -60,20 +59,20 @@ def shutdown
end
def ping_warden
- Fiber.new {
+ Fiber.new do
begin
VCAP::Dea::WardenEnv.ping
rescue => e
@logger.error "Warden unreachable, shutting down:#{e.message}"
shutdown
end
- @logger.debug "warden ping - succeeded."
- }.resume
+ end.resume
end
#XXX - minor - refactor these two to remove duplication.
def send_heartbeat
return if @shutting_down
+ @logger.debug("heartbeating...")
msg = @handler.get_heartbeat
if msg
msg = VCAP::Dea::Message.new(@nats, 'dea.heartbeat', :details => msg)
@@ -91,8 +90,7 @@ def send_advertise
end
def send_hello
- hello_msg = {:id => @handler.uuid, :ip => @local_ip, :port => @file_viewer_port,
- :version => VCAP::Dea::VERSION }
+ hello_msg = @handler.get_hello
msg = VCAP::Dea::Message.new(@nats, 'dea.start',
:details => hello_msg)
msg.send
@@ -113,19 +111,21 @@ def setup_error_handling
def register_component
VCAP::Component.register(:type => 'DEA',
- :host => @local_ip,
+ :host => @handler.local_ip,
:index => '0', #XXX fixme
:config => {}, #XXX fixme
:port => 9999, #XXX fixme
:user => 'foo', #XXX fixme
:password => 'foo') #XXX fixme
@uuid = VCAP::Component.uuid
- @handler.set_uuid(@uuid)
+ @handler.uuid = @uuid
end
def resume_detached_containers
- Fiber.new { @handler.restore_snapshot }.resume
- Fiber.new { @handler.resume_detached_containers }.resume
+ Fiber.new {
+ @handler.restore_snapshot
+ @handler.resume_detached_containers
+ }.resume
end
def update_varz
@@ -135,7 +135,7 @@ def update_varz
def start
@logger.info("connecting to nats: #{@nats_uri}")
- @nats = NATS.connect(:uri => @nats_uri) do
+ @nats = NATS.start(:uri => @nats_uri) do
register_component
setup_error_handling
resume_detached_containers
@@ -155,7 +155,9 @@ def setup_periodic_jobs
EM.add_periodic_timer(ADVERTISE_INTERVAL) { send_advertise }
EM.add_periodic_timer(HEARTBEAT_INTERVAL) { send_heartbeat }
EM.add_periodic_timer(VARZ_UPDATE_INTERVAL) { update_varz }
- EM.add_periodic_timer(CRASHED_APPS_CLEANUP_INTERVAL) { Fiber.new { @handler.remove_expired_crashed_apps }.resume }
+ EM.add_periodic_timer(RESOURCE_USAGE_UPDATE_INTERVAL) { Fiber.new { @handler.update_cached_resource_usage}.resume }
+ EM.add_periodic_timer(RESOURCE_USAGE_UPDATE_INTERVAL) { Fiber.new { @handler.update_total_resource_usage}.resume }
+ EM.add_periodic_timer(CRASHED_APPS_CLEANUP_INTERVAL) { Fiber.new { @handler.remove_expired_crashed_apps }.resume }
end
def setup_subscriptions
View
21 lib/vcap/dea/warden_env.rb
@@ -48,18 +48,29 @@ def setup_warden_client
@client.ping
end
+ def get_stats
+ return nil unless @linked
+ handle = fetch_handle
+ client = EM::Warden::FiberAwareClient.new(@@warden_socket_path)
+ client.connect
+ info = client.info(handle)
+ client.disconnect(false) #em will re-use connection
+ stats = info['stats']
+ {:mem_usage_B => stats['mem_usage_B'], :disk_usage_B => stats['disk_usage_B']}
+ end
+
def ping
@client.ping
end
def build_mounts_list(mounts)
mount_list = []
- mounts.each { |path| mount_list.push([path, path, {"mode" => 'ro'}])}
+ mounts.each { |src, dst, mode| mount_list.push([src, dst, {"mode" => mode}])}
mount_list
end
def alloc_network_port
- @client.net(fetch_handle, 'in')
+ @client.net(fetch_handle, 'in')["host_port"]
end
def bind_container(container_info)
@@ -93,6 +104,7 @@ def copy_in(src_path, dst_path)
raise VCAP::Dea::WardenError, "invalid path #{src_path}" if not File.exists?(src_path)
start_time = Time.now
result = @client.copy(handle, 'in', src_path, dst_path)
+ #XXX refactor timing stuff so its not redundant accross methods.
end_time = Time.now
total_time = end_time - start_time
raise VCAP::Dea::WardenError, "copy in failed" unless result == 'ok'
@@ -121,7 +133,8 @@ def run(cmd)
result = @client.run(handle, cmd)
end_time = Time.now
total_time = end_time - start_time
- @logger.debug("run #{cmd}:took (#{total_time}) returned: #{result.to_s}")
+ #XXX log different for now.
+ #@logger.debug("run #{cmd}:took (#{total_time}) returned: #{result.to_s}")
result
end
@@ -141,11 +154,11 @@ def link
begin
@linked = true
result = @client.link(handle, @jobid)
- @linked = false
rescue => e
@logger.warn "error on link - possible warden restart. #{e.message}"
raise VCAP::Dea::WardenError, "link failed"
ensure
+ @linked = false
@client.disconnect(false) if result[0] == nil #drop connection if container has been destroyed.
end
@logger.debug("link returned: #{result}")
View
BIN  vendor/cache/addressable-2.2.6.gem
Binary file not shown
View
BIN  vendor/cache/crack-0.1.8.gem
Binary file not shown
View
BIN  vendor/cache/daemons-1.1.8.gem
Binary file not shown
View
BIN  vendor/cache/diff-lcs-1.1.3.gem
Binary file not shown
View
BIN  vendor/cache/em-http-request-0.3.0.gem
Binary file not shown
View
BIN  vendor/cache/em-posix-spawn-0.0.2.gem
Binary file not shown
View
BIN  vendor/cache/em-warden-client-0.0.1.gem
Binary file not shown
View
BIN  vendor/cache/escape_utils-0.2.4.gem
Binary file not shown
View
BIN  vendor/cache/eventmachine-0.12.10.gem
Binary file not shown
View
BIN  vendor/cache/json_pure-1.6.5.gem
Binary file not shown
View
BIN  vendor/cache/little-plugger-1.1.3.gem
Binary file not shown
View
BIN  vendor/cache/logging-1.6.2.gem
Binary file not shown
View
BIN  vendor/cache/nats-0.4.10.gem
Binary file not shown
View
BIN  vendor/cache/posix-spawn-0.3.6.gem
Binary file not shown
View
BIN  vendor/cache/rack-1.4.1.gem
Binary file not shown
View
BIN  vendor/cache/rake-0.9.2.2.gem
Binary file not shown
View
BIN  vendor/cache/rspec-2.8.0.gem
Binary file not shown
View
BIN  vendor/cache/rspec-core-2.8.0.gem
Binary file not shown
View
BIN  vendor/cache/rspec-expectations-2.8.0.gem
Binary file not shown
View
BIN  vendor/cache/rspec-mocks-2.8.0.gem
Binary file not shown
View
BIN  vendor/cache/thin-1.3.1.gem
Binary file not shown
View
BIN  vendor/cache/vcap_common-1.0.0.gem
Binary file not shown
View
BIN  vendor/cache/vcap_logging-0.1.4.gem
Binary file not shown
View
BIN  vendor/cache/webmock-1.7.4.gem
Binary file not shown
View
BIN  vendor/cache/yajl-ruby-1.1.0.gem
Binary file not shown
1  vendor/checkout/em-posix-spawn
@@ -0,0 +1 @@
+Subproject commit 3f6636fca4fcba81c85db74ff647e15ed3743a81
1  vendor/checkout/em-warden-client
@@ -0,0 +1 @@
+Subproject commit 1df76f804578ce7299abf235bd6b64085e523b92
1  vendor/checkout/eventmachine
@@ -0,0 +1 @@
+Subproject commit 2806c630d8631d5dcf9fb2555f665b829052aabe
1  vendor/checkout/vcap_common
@@ -0,0 +1 @@
+Subproject commit 9673dcedf0c2daf46e3592a8f9b30538c5dc7b56
1  vendor/checkout/vcap_logging
@@ -0,0 +1 @@
+Subproject commit e36886a189b82f880a5aa3e9169712d5d9048a88
Please sign in to comment.
Something went wrong with that request. Please try again.