Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

remove legacy health manager

Change-Id: Ieef2b7b62fde9e450c57ee81c4e02d42f8094a13
  • Loading branch information...
commit b9d937b7fbda6a23287199b31c2481685b6e2ab1 1 parent d6c9388
Bob Nugmanov and Dmitriy Kalinin authored
View
21 health_manager/Gemfile
@@ -1,21 +0,0 @@
-source "http://rubygems.org"
-
-gem 'bundler', '>= 1.0.10'
-gem 'nats', '~> 0.4.24', :require => 'nats/client'
-gem 'eventmachine', :git => 'https://github.com/cloudfoundry/eventmachine.git', :branch => 'release-0.12.11-cf'
-gem 'em-http-request', '~> 1.0.0.beta.3', :require => 'em-http'
-
-gem 'rack', :require => ["rack/utils", "rack/mime"]
-gem 'rake'
-gem 'thin'
-gem 'yajl-ruby', :require => ['yajl', 'yajl/json_gem']
-
-gem 'vcap_common', '>= 1.0.10', :git => 'https://github.com/cloudfoundry/vcap-common.git', :ref => 'cbeb8a17'
-gem "vcap_logging", "~> 1.0.0", :git => 'https://github.com/cloudfoundry/common.git', :ref => 'e36886a1'
-gem 'cf-uaa-client', '~> 1.2', :git => 'https://github.com/cloudfoundry/uaa.git', :ref => '603bb76ce8'
-
-group :test do
- gem "rspec"
- gem "rcov"
- gem "ci_reporter"
-end
View
105 health_manager/Gemfile.lock
@@ -1,105 +0,0 @@
-GIT
- remote: https://github.com/cloudfoundry/common.git
- revision: e36886a189b82f880a5aa3e9169712d5d9048a88
- ref: e36886a1
- specs:
- vcap_logging (1.0.1)
- rake
-
-GIT
- remote: https://github.com/cloudfoundry/eventmachine.git
- revision: 2806c630d8631d5dcf9fb2555f665b829052aabe
- branch: release-0.12.11-cf
- specs:
- eventmachine (0.12.11.cloudfoundry.3)
-
-GIT
- remote: https://github.com/cloudfoundry/uaa.git
- revision: 603bb76ce8e369546e62402248d28ab9e87ef886
- ref: 603bb76ce8
- specs:
- cf-uaa-client (1.2.4)
- em-http-request (>= 1.0.0.beta.3)
- eventmachine
- highline
- launchy
- rest-client
- yajl-ruby
-
-GIT
- remote: https://github.com/cloudfoundry/vcap-common.git
- revision: cbeb8a17539e8b27b7c4fa6f37a3cb6b805b0d03
- ref: cbeb8a17
- specs:
- vcap_common (1.0.12)
- eventmachine (~> 0.12.11.cloudfoundry.3)
- nats (~> 0.4.22.beta.8)
- posix-spawn (~> 0.3.6)
- thin (~> 1.3.1)
- yajl-ruby (~> 0.8.3)
-
-GEM
- remote: http://rubygems.org/
- specs:
- addressable (2.2.6)
- builder (3.0.0)
- ci_reporter (1.6.5)
- builder (>= 2.1.2)
- daemons (1.1.8)
- diff-lcs (1.1.3)
- em-http-request (1.0.0.beta.3)
- addressable (>= 2.2.3)
- em-socksify
- eventmachine
- http_parser.rb (>= 0.5.1)
- em-socksify (0.1.0)
- eventmachine
- highline (1.6.15)
- http_parser.rb (0.5.3)
- json_pure (1.7.3)
- launchy (2.1.0)
- addressable (~> 2.2.6)
- mime-types (1.19)
- nats (0.4.24)
- daemons (>= 1.1.5)
- eventmachine (>= 0.12.10)
- json_pure (>= 1.7.3)
- thin (>= 1.3.1)
- posix-spawn (0.3.6)
- rack (1.4.1)
- rake (0.9.2)
- rcov (0.9.10)
- rest-client (1.6.7)
- mime-types (>= 1.16)
- rspec (2.6.0)
- rspec-core (~> 2.6.0)
- rspec-expectations (~> 2.6.0)
- rspec-mocks (~> 2.6.0)
- rspec-core (2.6.4)
- rspec-expectations (2.6.0)
- diff-lcs (~> 1.1.2)
- rspec-mocks (2.6.0)
- thin (1.3.1)
- daemons (>= 1.0.9)
- eventmachine (>= 0.12.6)
- rack (>= 1.0.0)
- yajl-ruby (0.8.3)
-
-PLATFORMS
- ruby
-
-DEPENDENCIES
- bundler (>= 1.0.10)
- cf-uaa-client (~> 1.2)!
- ci_reporter
- em-http-request (~> 1.0.0.beta.3)
- eventmachine!
- nats (~> 0.4.24)
- rack
- rake
- rcov
- rspec
- thin
- vcap_common (>= 1.0.10)!
- vcap_logging (~> 1.0.0)!
- yajl-ruby
View
1  health_manager/README.md
@@ -0,0 +1 @@
+Please use https://github.com/cloudfoundry/health_manager.
View
30 health_manager/Rakefile
@@ -1,30 +0,0 @@
-ENV["BUNDLE_GEMFILE"] = File.expand_path("../../cloud_controller/Gemfile", __FILE__)
-require 'rspec/core/rake_task'
-require 'ci/reporter/rake/rspec'
-
-ENV['RAILS_ENV'] = 'test'
-ENV['RACK_ENV'] = 'test'
-
-# FIXME - This does not honor the test db set in
-# config/health_manager.yml
-# TODO HACK FAIL
-task "prepare_test_db" do
- cc_root = File.expand_path("../../cloud_controller", __FILE__)
- Dir.chdir(cc_root) do
- ruby "-S rake RAILS_ENV=test db:migrate >/dev/null 2>/dev/null"
- end
-end
-
-reports_dir = File.expand_path("spec_reports")
-
-ENV['CI_REPORTS'] = reports_dir
-
-RSpec::Core::RakeTask.new do |t|
- t.pattern = "spec/**/*_spec.rb"
- t.rspec_opts = ["--format", "documentation", "--colour"]
-end
-
-namespace :ci do
- desc "Run specs producing results for CI"
- task "spec" => ["ci:setup:rspec", "^spec"]
-end
View
7 health_manager/bin/health_manager
@@ -1,7 +0,0 @@
-#!/usr/bin/env ruby
-# Copyright (c) 2009-2011 VMware, Inc.
-
-home = File.join(File.dirname(__FILE__), '/..')
-ENV['BUNDLE_GEMFILE'] = "#{home}/Gemfile"
-require File.join(home, 'lib/health_manager')
-
View
89 health_manager/config/health_manager.yml
@@ -1,89 +0,0 @@
----
-# Local_route is the IP address of a well known server on your network, it
-# is used to choose the right ip address (think of hosts that have multiple nics
-# and IP addresses assigned to them) of the host running the Health Manager. Default
-# value of nil, should work in most cases.
-local_route: 127.0.0.1
-
-# NATS message bus URI
-mbus: nats://localhost:4222/
-logging:
- level: debug
-pid: /var/vcap/sys/run/healthmanager.pid
-
-# This database is shared with the cloud controller.
-database_environment: # replaces database.yml
- production:
- database: cloudcontroller
- host: localhost
- port: 5432
- encoding: utf8
- username: postgres
- password: postgres
- adapter: postgresql
- timeout: 2000
- template: template0 # Required for utf8 encoding
- development:
- adapter: sqlite3
- database: db/cloudcontroller.sqlite3 # sqlite3 paths are relative to CC root.
- encoding: utf8
- test:
- adapter: sqlite3
- database: db/test.sqlite3
- encoding: utf8
-intervals:
- # Interval for collecting statistics about this cloudfoundry instance.
- # Amongst other things, data collected includes number of users, number of
- # applications and memory usage.
- database_scan: 60
- # Time to wait before starting analysis for stopped applications.
- droplet_lost: 30
- # Interval between scans for analysis of applications.
- droplets_analysis: 20
- # An application is deemed to be flapping if it is found to be in a crashed
- # state (after a restart following every crash) for more than "flapping_death"
- # number of times in an interval that is "flapping_timeout" long.
- flapping_death: 1
- flapping_timeout: 500
-
- #once instance is declared flapping, its restarts will be delayed,
- #starting with the value of min_restart_delay but not exceeding the
- #value of max_restart_delay
- min_restart_delay: 60
- max_restart_delay: 480
-
- #if total number of crashes exceeds this value, the attempts to
- #restart a flapping instance are stopped. Use -1 to never give up on
- #restarts
- giveup_crash_number: 4
-
- # Time to wait before trying to restart an application after a crash is
- # detected
- restart_timeout: 20
- # Time to wait before analyzing the state of an application that has been
- # started/restarted
- stable_state: 60
-
- # spindown: stop non-prod apps if they are inactive for a period
- # longer then specified by this parameter. Negative values disable
- # spindown functionality (default).
- # WARNING: only enable this option if your routers are configured to
- # publish activity information, otherwise all non-prod apps will be
- # spundown.
- inactivity_period_for_spindown: -1
-
-#number of start requests send each second (subject to EM timer limitations)
-#default value is 50.
-dequeueing_rate: 50
-
-# allows partitioning the cc pool for the purpose of AB testing,
-# gradual migration, etc. Each partition has a dedicated hm instance.
-cc_partition: default
-
-# Used for /healthz and /vars endpoints. If not provided random
-# values will be generated on component start. Uncomment to use
-# static values.
-#status:
-# port: 34502
-# user: thin
-# password: thin
View
1,017 health_manager/lib/health_manager.rb
@@ -1,1017 +0,0 @@
-# Copyright (c) 2009-2011 VMware, Inc.
-
-module CloudController
- require 'pathname'
- require 'erb'
- require 'yaml'
- require 'fileutils'
- require 'logger'
- require 'optparse'
- require 'set'
- require 'zlib'
-
- def self.root
- @root ||= Pathname.new(File.expand_path('../../../cloud_controller', __FILE__))
- end
-
- def self.lib_dir
- root.join('lib')
- end
-
- # NOTE - Any models that rely on appconfig.yml settings must likewise
- # be initialized by the Health Manager before they are used.
- def self.all_models
- Dir.glob(root.join('app', 'models', '*.rb'))
- end
-
- def self.setup
- $:.unshift(lib_dir.to_s) unless $:.include?(lib_dir.to_s)
- require root.join('config', 'boot')
- require 'active_record'
- require 'active_support/core_ext'
- require 'yajl'
- require 'eventmachine'
- require 'nats/client'
- require 'vcap/common'
- require 'vcap/component'
- require 'vcap/logging'
- require 'vcap/rolling_metric'
- require 'vcap/priority_queue'
- all_models.each {|fn| require(fn)}
-
- # This is needed for comparisons between the last_updated time of an app and the current time
- # Time.zone = :utc
- ActiveRecord::Base.time_zone_aware_attributes = true
- ActiveRecord::Base.default_timezone = :utc
- end
-
- def self.load_yaml(path)
- File.open(path, 'rb') do |fh|
- yaml = ERB.new(fh.read).result(binding)
- return YAML.load(yaml)
- end
- end
-end
-
-CloudController.setup
-
-class HealthManager
- VERSION = 0.98
-
- attr_reader :database_scan, :droplet_lost, :droplets_analysis, :flapping_death, :flapping_timeout
- attr_reader :restart_timeout, :stable_state, :droplets
- attr_reader :request_queue
- attr_reader :spindown_inactive_apps, :inactivity_period_for_spindown
-
- # TODO - Oh these need comments so badly..
- DOWN = 'DOWN'
- STARTED = 'STARTED'
- STOPPED = 'STOPPED'
- CRASHED = 'CRASHED'
- STARTING = 'STARTING'
- RUNNING = 'RUNNING'
- FLAPPING = 'FLAPPING'
- DEA_SHUTDOWN = 'DEA_SHUTDOWN'
- DEA_EVACUATION = 'DEA_EVACUATION'
- APP_STABLE_STATES = Set.new([STARTED, STOPPED])
- RUNNING_STATES = Set.new([STARTING, RUNNING])
- RESTART_REASONS = Set.new([CRASHED, DEA_SHUTDOWN, DEA_EVACUATION])
-
- INFINITE_PRIORITY = 2_000_000_000
-
- DEFAULT_PARTITION = "default"
-
- def self.start(options)
- health_manager = new(options)
- health_manager.run
- health_manager
- end
-
- def initialize(config)
- @config = config
- VCAP::Logging.setup_from_config(config['logging'])
- @logger = VCAP::Logging.logger('hm')
- @database_scan = config['intervals']['database_scan']
- @droplet_lost = config['intervals']['droplet_lost']
- @droplets_analysis = config['intervals']['droplets_analysis'] || 10
- @flapping_death = config['intervals']['flapping_death'] || 1
- @flapping_timeout = config['intervals']['flapping_timeout'] || 500
- @min_restart_delay = config['intervals']['min_restart_delay'] || 60
- @max_restart_delay = config['intervals']['max_restart_delay'] || 480
- @giveup_crash_number = config['intervals']['giveup_crash_number'] || 4 # Use -1 to never give up!
- @restart_timeout = config['intervals']['restart_timeout']
- @stable_state = config['intervals']['stable_state']
- @max_db_reconnect_wait = config['intervals']['max_db_reconnect_wait'] || 300 #up to five minutes by default
- @inactivity_period_for_spindown = config['intervals']['inactivity_period_for_spindown'] || -1
-
- @dequeueing_rate = config['dequeueing_rate'] || 50
- @database_environment = config['database_environment']
-
- @spindown_inactive_apps = @inactivity_period_for_spindown > 0
-
- @cc_partition = config['cc_partition'] || DEFAULT_PARTITION
-
- @droplets = {}
- @pending_restart = {}
- @request_queue = VCAP::PrioritySet.new
-
- configure_database
-
- if config['pid']
- @pid_file = config['pid']
- # Create pid file
- begin
- FileUtils.mkdir_p(File.dirname(@pid_file))
- rescue => e
- @logger.fatal "Can't create pid directory, exiting: #{e}"
- end
- File.open(@pid_file, 'wb') { |f| f.puts "#{Process.pid}" }
- end
- end
-
- def now
- Time.now.to_i
- end
-
- def encode_json(obj = {})
- Yajl::Encoder.encode(obj)
- end
-
- def parse_json(string = '{}')
- Yajl::Parser.parse(string)
- end
-
- def create_droplet_entry
- { :versions => {}, :crashes => {} }
- end
-
- def create_index_entry
- { :last_action => -1, :crashes => 0, :crash_timestamp => -1 }
- end
-
- def create_version_entry
- { :indices => {} }
- end
-
- def run
- @started = now
- NATS.on_error do |e|
- @logger.error("NATS problem, #{e}")
- @logger.error(e)
- exit!
- end
- EM.error_handler do |e|
- @logger.error "Eventmachine problem, #{e}"
- @logger.error(e)
- exit!
- end
-
- NATS.start(:uri => @config['mbus']) do
- configure_timers
- register_as_component
- subscribe_to_messages
- end
- end
-
- # We use the CloudController database configuration
- # if none is specified in our config file.
- # By default, we connect to the development database.
- def configure_database
- env = @config['rails_environment'] || CloudController.environment
- if @database_environment
- config = @database_environment[env]
- else
- # using CloudController db configuration
- config = AppConfig[:database_environment][env]
- end
- logger = Logger.new(STDOUT)
- logger.level = Logger::INFO
- establish_database_connection(config, logger)
- end
-
- def ensure_connected(&block)
- sleep_time = 1
- total_sleep_time = 0
- failure_count = 0
- begin
- yield
- rescue ActiveRecord::StatementInvalid
- # This exception is raised when a connection was previously connected, but
- # upon executing a statement detects that the connection is actually gone.
- # Calling #disconnect! on the connection pool will make it reconnect.
- @logger.warn('Possibly lost db connection, attempting to re-connect')
- ActiveRecord::Base.connection_pool.disconnect!
- retry
- rescue ActiveSupport::Dependencies::Blamable => e
- @logger.warn("Attempting to recover from: #{e}")
- failure_count += 1
- if total_sleep_time < @max_db_reconnect_wait
- @logger.warn("Waiting for #{sleep_time} seconds before re-attempting database operation")
- sleep sleep_time
- total_sleep_time += sleep_time
- sleep_time *= 2 unless sleep_time > 60
- retry
- else
- @logger.error("Unable to reconnect after #{failure_count} attempts over #{total_sleep_time} seconds of waiting, giving up. Error information follows.")
- @logger.error(e)
- exit!
- end
- end
- end
-
- def establish_database_connection(db_config, logger)
- expand_database_path_for_sqlite3(db_config)
- ActiveRecord::Base.establish_connection(db_config)
- ActiveRecord::Base.logger = logger
- logger.debug "Connected to CloudController database"
- end
-
- # If the adapter is sqlite3 and the path is relative, expand it
- # in reference to the CloudController root.
- def expand_database_path_for_sqlite3(db_config)
- if db_config['adapter'] == 'sqlite3'
- db_path = db_config['database']
- unless db_path[0,1] == '/'
- db_path = File.join(CloudController.root, db_path)
- end
- db_config['database'] = File.expand_path(db_path)
- end
- end
-
- def shutdown
- @logger.info('Shutting down.')
- FileUtils.rm_f(@pid_file) if @pid_file
- NATS.stop { EM.stop }
- end
-
- def analyze_app(app_id, droplet_entry, stats)
- update_timestamp = droplet_entry[:last_updated]
- quiescent = (now - update_timestamp) > @stable_state
-
- if @spindown_inactive_apps &&
- !droplet_entry[:prod] &&
- droplet_entry[:state] == STARTED &&
- now - update_timestamp >= @inactivity_period_for_spindown
-
- unless droplet_entry[:last_activity] &&
- now - droplet_entry[:last_activity] < @inactivity_period_for_spindown
- spindown(app_id)
- return
- end
- end
-
- if APP_STABLE_STATES.include?(droplet_entry[:state]) && quiescent
- extra_instances = []
- missing_indices = []
-
- droplet_entry[:crashes].delete_if do |_, crash_entry|
- now - crash_entry[:timestamp] > @droplet_lost
- end
-
- droplet_entry[:versions].delete_if do |version, version_entry|
- version_entry[:indices].delete_if do |index, index_entry|
- if RUNNING_STATES.include?(index_entry[:state]) && now - index_entry[:timestamp] > @droplet_lost
- index_entry[:state] = DOWN
- index_entry[:state_timestamp] = now
- end
-
- reason = nil
- if droplet_entry[:state] == STOPPED
- extra_instance = true
- reason = "Droplet state is STOPPED."
- elsif index >= droplet_entry[:instances]
- extra_instance = true
- reason = "Extra instance. Droplet should have #{droplet_entry[:instances]} instances running."
- elsif version != droplet_entry[:live_version]
- extra_instance = true
- reason = "Live version mismatch. Live version is #{droplet_entry[:live_version]} instance version is #{version}."
- elsif index_entry[:dea_prod] && !droplet_entry[:prod]
- extra_instance = true
- reason = "Prod flag mismatch. app prod flag: #{droplet_entry[:prod]}, dea prod flag: #{index_entry[:dea_prod]}"
- end
-
- if RUNNING_STATES.include?(index_entry[:state]) && extra_instance
- @logger.info("Preparing to stop instance (app_id=#{app_id}, index=#{index}, instance=#{index_entry[:instance]}). Reason: #{reason}")
- extra_instances << index_entry[:instance]
- elsif extra_instance
- # stop tracking extra instances
- true
- end
- end
-
- # delete empty version entries for non live versions
- if version_entry[:indices].empty?
- droplet_entry[:state] == STOPPED || version != droplet_entry[:live_version]
- end
- end
-
- # if sanity check fails, all bets are off, don't treat droplet as STARTED
- if droplet_entry[:state] == STARTED && sanity_check(app_id, droplet_entry)
- live_version_entry = droplet_entry[:versions][droplet_entry[:live_version]] ||= create_version_entry
-
- framework_stats = stats[:frameworks][droplet_entry[:framework]] ||= create_runtime_metrics
- runtime_stats = stats[:runtimes][droplet_entry[:runtime]] ||= create_runtime_metrics
-
- framework_stats[:apps] += 1
- runtime_stats[:apps] += 1
-
- framework_stats[:crashes] += droplet_entry[:crashes].length
- runtime_stats[:crashes] += droplet_entry[:crashes].length
-
- index_entries = live_version_entry[:indices]
- droplet_entry[:instances].times do |index|
- index_entry = index_entries[index]
- unless index_entry
- index_entry = index_entries[index] = create_index_entry
- index_entry[:state] = DOWN
- index_entry[:state_timestamp] = now
- end
-
- if RUNNING_STATES.include?(index_entry[:state])
- stats[:running] += 1
- framework_stats[:running_instances] += 1
- runtime_stats[:running_instances] += 1
- elsif index_entry[:state] == DOWN
- stats[:down] += 1 if index_entry[:state] == DOWN
- framework_stats[:missing_instances] += 1
- runtime_stats[:missing_instances] += 1
- elsif index_entry[:state] == FLAPPING
- framework_stats[:flapping_instances] += 1
- runtime_stats[:flapping_instances] += 1
- end
-
- if index_entry[:state] == FLAPPING && !restart_pending?(app_id, index) && now - index_entry[:last_action] > @restart_timeout
- delay_or_giveup_restart_of_flapping_instance(app_id, index, index_entry, true)
- end
-
- if index_entry[:state] == DOWN && now - index_entry[:last_action] > @restart_timeout
- @logger.info("Preparing to restart instance (app_id=#{app_id}, index=#{index}). Reason: droplet state is STARTED, but instance state is DOWN. index_entry=#{index_entry}")
- index_entry[:last_action] = now
- missing_indices << index
- end
- end
- end
-
- ensure_connected do
- # don't act if we were looking at a stale droplet
- if update_droplet(App.find_by_id(app_id.to_i))
- if missing_indices.any? || extra_instances.any?
- @logger.info("Droplet information is stale for app id #{app_id}, not taking action.")
- @logger.info("(#{missing_indices.length} instances need to be started, #{extra_instances.length} instances need to be stopped.)")
- end
- return
- end
- end
-
- if missing_indices.any?
- @logger.debug {"missing_indices=#{missing_indices} detected for app_id=#{app_id}, entry=#{droplet_entry}"}
- start_instances(app_id, missing_indices)
- end
- if extra_instances.any?
- @logger.debug {"extra_instances=#{extra_instances} detected for app_id=#{app_id}, entry=#{droplet_entry}"}
- stop_instances(app_id, extra_instances)
- end
- end
- end
-
- def sanity_check(app_id, droplet_entry)
- # check for null staged_package_hash
- # the live_version is in format "#{staged_package_hash}-#{run_count}",
- # so when staged_package_hash is null, the live_version starts with '-'
- # see method #droplet_version
- if droplet_entry[:live_version].start_with?('-')
- @logger.warn("Failed sanity check for live_version for app_id=#{app_id}: #{droplet_entry}")
- return false
- end
- true
- end
-
- def prepare_analysis(collect_stats)
- @analysis = {
- :collect_stats => collect_stats,
- :start => Time.now,
- :instances => 0,
- :crashed => 0,
- :stats => {:running => 0, :down => 0, :frameworks => {}, :runtimes => {}},
- :ids => @droplets.keys,
- :current_key_index => 0
- }
- end
-
- def analysis_in_progress?
- @analysis && !@analysis[:complete]
- end
-
- def perform_quantum(id, droplet_entry)
- return if id.nil?
- return if droplet_entry.nil?
-
- analyze_app(id, droplet_entry, @analysis[:stats]) if @analysis[:collect_stats]
- @analysis[:instances] += droplet_entry[:instances]
- @analysis[:crashed] += droplet_entry[:crashes].size if droplet_entry[:crashes]
- end
-
- def perform_and_schedule_next_quantum
-
- if @analysis[:current_key_index] < @analysis[:ids].size
- perform_quantum(
- @analysis[:ids][@analysis[:current_key_index]],
- @droplets[@analysis[:ids][@analysis[:current_key_index]]])
-
- @analysis[:current_key_index] += 1
-
- EM.next_tick {
- perform_and_schedule_next_quantum
- }
- else
- finish_analysis
- end
- end
-
- def finish_analysis
-
- return unless @analysis
- @logger.debug("Analysis complete: #{@analysis.inspect}")
- VCAP::Component.varz[:total_apps] = @droplets.size
- VCAP::Component.varz[:total_instances] = @analysis[:instances]
- VCAP::Component.varz[:crashed_instances] = @analysis[:crashed]
-
- if @analysis[:collect_stats]
- VCAP::Component.varz[:running_instances] = @analysis[:stats][:running]
- VCAP::Component.varz[:down_instances] = @analysis[:stats][:down]
- VCAP::Component.varz[:running][:frameworks] = @analysis[:stats][:frameworks]
- VCAP::Component.varz[:running][:runtimes] = @analysis[:stats][:runtimes]
- @logger.info("Analyzed #{@analysis[:stats][:running]} running and #{@analysis[:stats][:down]} down apps in #{elapsed_time_in_ms(@analysis[:start])}")
- else
- @logger.info("Analyzed #{@droplets.size} apps in #{elapsed_time_in_ms(@analysis[:start])}")
- end
- @analysis[:complete] = true
- end
-
- def analyze_all_apps(collect_stats = true)
-
- return false if analysis_in_progress?
-
- prepare_analysis(collect_stats)
- perform_and_schedule_next_quantum
- return true
- end
-
- def create_runtime_metrics
- {
- :apps => 0,
- :crashes => 0,
- :running_instances => 0,
- :missing_instances => 0,
- :flapping_instances => 0
- }
- end
-
- def create_db_metrics
- {
- :apps => 0,
- :started_apps => 0,
- :instances => 0,
- :started_instances => 0,
- :memory => 0,
- :started_memory => 0
- }
- end
-
- def elapsed_time_in_ms(start)
- elapsed_ms = (Time.now - start) * 1000
- "#{'%.1f' % elapsed_ms}ms"
- end
-
- def partition_match?(value)
- value ||= DEFAULT_PARTITION
- value == @cc_partition
- end
-
- def process_updated_message(message)
- message = parse_json(message)
- return unless partition_match?(message['cc_partition'])
- VCAP::Component.varz[:droplet_updated_msgs_received] += 1
- ensure_connected { update_droplet App.find_by_id(message['droplet'].to_i) }
- end
-
- def process_exited_message(message)
- exit_message = parse_json(message)
- return unless partition_match?(exit_message['cc_partition'])
- VCAP::Component.varz[:droplet_exited_msgs_received] += 1
- droplet_id = exit_message['droplet'].to_s
- version = exit_message['version']
- index = exit_message['index']
- instance = exit_message['instance']
-
- droplet_entry = @droplets[droplet_id]
- index_entry = nil
-
- if droplet_entry
- version_entry = droplet_entry[:versions][version]
- if version_entry
- index_entry = version_entry[:indices][index]
- end
-
- if version == droplet_entry[:live_version] && index >= 0 && index < droplet_entry[:instances]
-
- version_entry = droplet_entry[:versions][version] ||= create_version_entry
-
- index_entry = version_entry[:indices][index] ||= create_index_entry
- index_entry[:instance] ||= instance
-
- if index_entry[:instance] == instance || !RUNNING_STATES.include?(index_entry[:state])
- if RESTART_REASONS.include?(exit_message['reason'])
- if index_entry[:crash_timestamp] > 0 && now - index_entry[:crash_timestamp] > @flapping_timeout
- index_entry[:crashes] = 0
- index_entry[:crash_timestamp] = -1
- end
-
- if exit_message['reason'] == CRASHED
- index_entry[:crashes] += 1
- index_entry[:crash_timestamp] = now
- end
-
- if index_entry[:crashes] > @flapping_death
- @logger.info("Declaring instance flapping, app_id=#{droplet_id}, index=#{index}. Current state: #{index_entry[:state]}") unless index_entry[:state] == FLAPPING
-
- index_entry[:state] = FLAPPING
- index_entry[:state_timestamp] = now
-
- unless restart_pending?(droplet_id, index)
- delay_or_giveup_restart_of_flapping_instance(droplet_id, index, index_entry)
- end
- else
- index_entry[:state] = DOWN
- index_entry[:state_timestamp] = now
- index_entry[:last_action] = now
-
- high_priority = (exit_message['reason'] == DEA_EVACUATION)
-
- @logger.info("Preparing to start instance (app_id=#{droplet_id}, index=#{index}). Reason: Instance exited with reason '#{exit_message['reason']}'.")
- start_instances(droplet_id, [index], high_priority)
- end
- end
- end
- elsif index_entry
- version_entry[:indices].delete(index)
- droplet_entry[:versions].delete(version) if version_entry[:indices].empty?
- end
-
- if exit_message['reason'] == CRASHED
- droplet_entry[:crashes][instance] = {
- :timestamp => now,
- :crash_timestamp => exit_message['crash_timestamp']
- }
- end
- end
-
- droplet_entry # return the droplet that we changed. This allows the spec tests to ensure the behaviour is correct.
- end
-
- def delay_or_giveup_restart_of_flapping_instance(droplet_id, index, index_entry, giveup_quietly = false)
-
- index_entry[:last_action] = now #regardless of whether real action is omitted or delayed, a decision timestamp is needed
-
- if @giveup_crash_number > 0 && index_entry[:crashes] > @giveup_crash_number
- @logger.info("given up on flapping instance (app_id=#{droplet_id}, index=#{index}). " +
- "Number of crashes: #{index_entry[:crashes]}.") unless giveup_quietly
- else
- @pending_restart[droplet_id] ||= {}
- @pending_restart[droplet_id][index] = true
-
- restart_delay = [@max_restart_delay, @min_restart_delay << (index_entry[:crashes] - @flapping_death - 1) ].min
-
- @logger.info("delayed-restarting flapping instance (app_id=#{droplet_id}, index=#{index}). Delay: #{restart_delay}. Number of crashes: #{index_entry[:crashes]}.")
- EM.add_timer(restart_delay) do
- index_entry[:last_action] = now
- start_instances(droplet_id, [index], false, true)
- end
- end
- end
-
- def restart_pending?(droplet_id, index)
- @pending_restart[droplet_id] && @pending_restart[droplet_id][index]
- end
-
- def process_heartbeat_message(message)
- VCAP::Component.varz[:heartbeat_msgs_received] += 1
- result = []
- parsed_message = parse_json(message)
- dea_prod = parsed_message['prod']
- parsed_message['droplets'].each do |heartbeat|
- next unless partition_match?(heartbeat['cc_partition'])
- droplet_id = heartbeat['droplet'].to_s
- instance = heartbeat['instance']
- droplet_entry = @droplets[droplet_id]
- if droplet_entry
- result << droplet_entry
- state = heartbeat['state']
- if RUNNING_STATES.include?(state)
- version_entry = droplet_entry[:versions][heartbeat['version']] ||= create_version_entry
- index_entry = version_entry[:indices][heartbeat['index']] ||= create_index_entry
-
- if index_entry[:state] == RUNNING && index_entry[:instance] != instance
- stop_instances(droplet_id, [instance])
- else
- index_entry[:instance] = instance
- index_entry[:timestamp] = now
- index_entry[:dea_prod] = dea_prod
- index_entry[:state] = state.to_s
- index_entry[:state_timestamp] = heartbeat['state_timestamp']
- end
- elsif state == CRASHED
- droplet_entry[:crashes][instance] = {
- :timestamp => now,
- :crash_timestamp => heartbeat['state_timestamp']
- }
- end
- else
- instance_uptime = now - heartbeat['state_timestamp']
- health_manager_uptime = now - @started
- threshold = @database_scan * 2
-
- if health_manager_uptime > threshold && instance_uptime > threshold
- @logger.info("Stopping unknown app: #{droplet_id}/#{instance}.")
- stop_instances(droplet_id, [instance])
- end
- end
- end
-
- result # return the droplets that we changed. This allows the spec tests to ensure the behaviour is correct.
- end
-
- def process_active_apps_message(message)
- app_list = parse_json(Zlib::Inflate.inflate(message))
- app_list.each do |app_id|
- droplet_entry = @droplets[app_id]
- if droplet_entry
- droplet_entry[:last_activity] = now
- end
- end
- end
-
- def process_health_message(message, reply)
- VCAP::Component.varz[:healthmanager_health_request_msgs_received] += 1
- message_json = parse_json(message)
- droplets = message_json['droplets']
- exchange = message_json['exchange']
- droplets.each do |droplet|
- droplet_id = droplet['droplet'].to_s
-
- droplet_entry = @droplets[droplet_id]
- if droplet_entry
- version = droplet['version']
- version_entry = droplet_entry[:versions][version]
- running = 0
- if version_entry
- version_entry[:indices].each_value do |index_entry|
- running += 1 if index_entry[:state] == RUNNING
- end
- end
-
- response_json = encode_json(:droplet => droplet_id, :version => version, :healthy => running)
- NATS.publish(reply, response_json)
- end
- end
- end
-
- def process_status_message(message, reply)
- VCAP::Component.varz[:healthmanager_status_msgs_received] += 1
- message_json = parse_json(message)
- droplet_id = message_json['droplet'].to_s
- droplet_entry = @droplets[droplet_id]
-
- if droplet_entry
- state = message_json['state']
- if state == FLAPPING
- version = message_json['version']
- result = []
- version_entry = droplet_entry[:versions][version]
- if version_entry
- version_entry[:indices].each do |index, index_entry|
- if index_entry[:state] == FLAPPING
- result << {
- :index => index,
- :since => index_entry[:state_timestamp]
- }
- end
- end
- end
- NATS.publish(reply, encode_json({:indices => result}))
-
- elsif state == CRASHED
- result = []
- droplet_entry[:crashes].each do |instance, crash_entry|
- result << {
- :instance => instance,
- :since => crash_entry[:crash_timestamp]
- }
- end
- NATS.publish(reply, encode_json({:instances => result}))
- end
- end
- end
-
- def update_from_db
- ensure_connected do
- start = Time.now
- old_droplet_ids = Set.new(@droplets.keys)
-
- App.all.each do |droplet|
- old_droplet_ids.delete(droplet.id.to_s)
- update_droplet(droplet)
- end
-
- old_droplet_ids.each {|id| @droplets.delete(id)}
- # TODO - Devise a version of the below that works with vast numbers of apps and users.
- VCAP::Component.varz[:total_users] = User.count
- VCAP::Component.varz[:users] = User.all_email_addresses.map {|e| {:email => e}}
- VCAP::Component.varz[:apps] = App.health_manager_representations
- @logger.info("Database scan took #{elapsed_time_in_ms(start)} and found #{@droplets.size} apps")
-
- start = Time.now
-
- VCAP::Component.varz[:total] = {
- :frameworks => {},
- :runtimes => {}
- }
-
- App.count(:group => ["framework", "runtime", "state"]).each do |grouping, count|
- framework, runtime, state = grouping
-
- framework_stats = VCAP::Component.varz[:total][:frameworks][framework] ||= create_db_metrics
- framework_stats[:apps] += count
- framework_stats[:started_apps] += count if state == "STARTED"
-
- runtime_stats = VCAP::Component.varz[:total][:runtimes][runtime] ||= create_db_metrics
- runtime_stats[:apps] += count
- runtime_stats[:started_apps] += count if state == "STARTED"
- end
-
- App.sum(:instances, :group => ["framework", "runtime", "state"]).each do |grouping, count|
- framework, runtime, state = grouping
-
- framework_stats = VCAP::Component.varz[:total][:frameworks][framework] ||= create_db_metrics
- framework_stats[:instances] += count
- framework_stats[:started_instances] += count if state == "STARTED"
-
- runtime_stats = VCAP::Component.varz[:total][:runtimes][runtime] ||= create_db_metrics
- runtime_stats[:instances] += count
- runtime_stats[:started_instances] += count if state == "STARTED"
- end
-
- App.sum("instances * memory", :group => ["framework", "runtime", "state"]).each do |grouping, count|
- # memory is stored as a string
- count = count.to_i
- framework, runtime, state = grouping
-
- framework_stats = VCAP::Component.varz[:total][:frameworks][framework] ||= create_db_metrics
- framework_stats[:memory] += count
- framework_stats[:started_memory] += count if state == "STARTED"
-
-
- runtime_stats = VCAP::Component.varz[:total][:runtimes][runtime] ||= create_db_metrics
- runtime_stats[:memory] += count
- runtime_stats[:started_memory] += count if state == "STARTED"
- end
-
- @logger.info("Database stat scan took #{elapsed_time_in_ms(start)}")
- end
- end
-
- def droplet_version(droplet)
- "#{droplet.staged_package_hash}-#{droplet.run_count}"
- end
-
- def update_droplet(droplet)
- return true unless droplet
-
- droplet_entry = @droplets[droplet.id.to_s] ||= create_droplet_entry
-
- entry_updated = droplet_entry[:last_updated] != droplet.last_updated
-
- droplet_entry[:instances] = droplet.instances
- droplet_entry[:framework] = droplet.framework
- droplet_entry[:prod] = droplet.prod
- droplet_entry[:runtime] = droplet.runtime
- droplet_entry[:state] = droplet.state.upcase
- droplet_entry[:last_updated] = droplet.last_updated
- droplet_entry[:live_version] = droplet_version(droplet)
-
- entry_updated
- end
-
- def start_instances(droplet_id, indices, high_priority = false, flapping = false)
- droplet_entry = @droplets[droplet_id]
-
- if droplet_entry.nil?
- @logger.info("droplet #{droplet_id} went away, aborting start_instances")
- return
- end
-
- start_message = {
- :droplet => droplet_id,
- :op => :START,
- :last_updated => droplet_entry[:last_updated],
- :version => droplet_entry[:live_version],
- :indices => indices
- }
-
- start_message[:flapping] = true if flapping
-
- if queue_requests?
- queue_request(start_message, high_priority)
- else
- #old behavior: send the message immediately
- publish_start_message(start_message)
- end
- end
-
- def publish_start_message(start_message)
-
- if indices_restarting = @pending_restart[start_message[:droplet]]
- start_message[:indices].each { |i| indices_restarting[i] = false }
- end
-
- @logger.info("Requesting the start of missing instances: #{start_message}")
- NATS.publish("cloudcontrollers.hm.requests.#{@cc_partition}", encode_json(start_message))
- end
-
- def queue_request(message, high_priority)
- #the priority is higher for older items, to de-prioritize flapping items
- priority = now - message[:last_updated]
- priority = 0 if priority < 0 #avoid timezone drama
- priority = INFINITE_PRIORITY if high_priority
- key = message.clone
- key.delete :last_updated
- @logger.info("Queueing priority '#{priority}' request: #{message}, using key: #{key}. Queue size: #{@request_queue.size}")
- @request_queue.insert(message, priority, key)
- end
-
- def stop_instances(droplet_id, instances)
- droplet_entry = @droplets[droplet_id]
- last_updated = droplet_entry ? droplet_entry[:last_updated] : 0
- stop_message = encode_json({
- :droplet => droplet_id,
- :op => :STOP,
- :last_updated => last_updated,
- :instances => instances
- })
- NATS.publish("cloudcontrollers.hm.requests.#{@cc_partition}", stop_message)
- @logger.info("Requesting the stop of extra instances: #{stop_message}")
- end
-
- def spindown(droplet_id)
- @logger.info("spinning down #{droplet_id} due to inactivity")
-
- message = encode_json({
- :droplet => droplet_id,
- :op => :SPINDOWN
- })
-
- NATS.publish("cloudcontrollers.hm.requests.#{@cc_partition}",message)
- end
-
- def configure_timers
- EM.next_tick { update_from_db }
- EM.add_periodic_timer(@database_scan) { update_from_db }
-
- # Do first pass without the individual analysis
- EM.next_tick { analyze_all_apps(collect_stats = false) }
-
- # Start the droplet analysis timer after the droplet lost timeout to make sure all the heartbeats came in.
- EM.add_timer(@droplet_lost) do
- EM.add_periodic_timer(@droplets_analysis) { analyze_all_apps }
- end
-
- if queue_requests?
- EM.add_periodic_timer(1) do
- deque_a_batch_of_requests
- end
- end
- end
-
- def deque_a_batch_of_requests(num_requests=@dequeueing_rate)
- num_requests.times do
- unless @request_queue.empty?
- start_message = @request_queue.remove
- publish_start_message(start_message)
- VCAP::Component.varz[:queue_length] = @request_queue.size
- end
- end
- end
-
- def register_as_component
- status_config = @config['status'] || {}
- VCAP::Component.register(:type => 'HealthManager',
- :host => VCAP.local_ip(@config['local_route']),
- :index => @config['index'],
- :config => @config,
- :port => status_config['port'],
- :user => status_config['user'],
- :password => status_config['password'])
-
- # Initialize VCAP component varzs..
- VCAP::Component.varz[:total_apps] = 0
- VCAP::Component.varz[:total_users] = 0
- VCAP::Component.varz[:total_instances] = 0
-
- # These will get processed after a small delay..
- VCAP::Component.varz[:running_instances] = -1
- VCAP::Component.varz[:crashed_instances] = -1
-
- VCAP::Component.varz[:down_instances] = -1
-
- VCAP::Component.varz[:queue_length] = 0
-
- VCAP::Component.varz[:total] = {
- :frameworks => {},
- :runtimes => {}
- }
-
- VCAP::Component.varz[:running] = {
- :frameworks => {},
- :runtimes => {}
- }
-
- VCAP::Component.varz[:heartbeat_msgs_received] = 0
- VCAP::Component.varz[:droplet_exited_msgs_received] = 0
- VCAP::Component.varz[:droplet_updated_msgs_received] = 0
- VCAP::Component.varz[:healthmanager_status_msgs_received] = 0
- VCAP::Component.varz[:healthmanager_health_request_msgs_received] = 0
- @logger.info("Starting VCAP Health Manager (#{VERSION})")
- end
-
- def subscribe_to_messages
- # Now we have something worth cleaning up at shutdown.
- trap('TERM') { shutdown }
- trap('INT') { shutdown }
-
- NATS.subscribe('dea.heartbeat') do |message|
- @logger.debug { "heartbeat: #{message}" }
- process_heartbeat_message(message)
- end
-
- NATS.subscribe('droplet.exited') do |message|
- @logger.debug { "droplet.exited: #{message}" }
- process_exited_message(message)
- end
-
- NATS.subscribe('droplet.updated') do |message|
- @logger.debug { "droplet.updated: #{message}" }
- process_updated_message(message)
- end
-
- NATS.subscribe('healthmanager.status') do |message, reply|
- @logger.debug { "healthmanager.status: #{message}" }
- process_status_message(message, reply)
- end
-
- NATS.subscribe('healthmanager.health') do |message, reply|
- @logger.debug { "healthmanager.health: #{message}" }
- process_health_message(message, reply)
- end
-
- if @spindown_inactive_apps
- NATS.subscribe('router.active_apps') do |message|
- @logger.debug { "router.active_apps received, message size=#{message.size}" }
- process_active_apps_message(message)
- end
- end
-
- NATS.publish('healthmanager.start')
- end
-
- def queue_requests?
- @dequeueing_rate != 0
- end
-end
-
-if $0 == __FILE__ || File.expand_path($0) == File.expand_path(File.join(File.dirname(__FILE__), '../bin/health_manager'))
-
- config_path = ENV["CLOUD_FOUNDRY_CONFIG_PATH"] || File.join(File.dirname(__FILE__), '../config')
- config_file = File.join(config_path, "health_manager.yml")
- options = OptionParser.new do |opts|
- opts.banner = 'Usage: healthmanager [OPTIONS]'
- opts.on("-c", "--config [ARG]", "Configuration File") do |opt|
- config_file = opt
- end
- opts.on("-h", "--help", "Help") do
- puts opts
- exit
- end
- end
- options.parse!(ARGV.dup)
-
- begin
- config = YAML.load_file(config_file)
- rescue => e
- $stderr.puts "Could not read configuration file: #{e}"
- exit 1
- end
-
- EM.epoll
-
- EM.run { HealthManager.start(config) }
-end
View
48 health_manager/spec/functional/README
@@ -1,48 +0,0 @@
-the plan:
-
-- create a testing "harness" to bring up minimum number of components
- necessary for testing the HM
-
- - implement a ForkedComponent class for HealthManager
-
- - instantiate ForkedComponents for NATS and the newly created HM
-
- - ensure basic connectivity by receiving ping messages
-
- - ensure test instance of CloudController database is created and
- accessible
-
- - populate the test db with minimal data using App model; and
-
-- emulate heartbeat signals to HM
-
-- ensure that control messages that HM sends to CC are appropriate
- - close app unexpectedly and ensure restart message
- - send heartbeats from extra instances and ensure stop message
-
-
-
-QUESTION:
-
-would it make more sense to establish the true state by actually
-bringing up a CC and DEA? That way the DB would populate "truly",
-there'd be no need to emulate it by writing directly into the tables
-
-This would require:
-
-bringing up the CC, the DEA, actually deploying a test app.
-
-
------ FROM PIVOTAL, original story:
-
-Write a set of functional tests that exercise the public "interface"
-and behavior exposed by the health manager.
-
-Some examples of behaviors tested:
-- Restarting apps upon receipt of droplet.exited messages w/ reason "CRASHED".
-
-- Stopping extra instances of an application.
-- Starting missing instances of an application.
-
-We should be able to use this test suite against the new HM with minor
-modification to help verify its correctness.
View
73 health_manager/spec/functional/bus_snoop.rb
@@ -1,73 +0,0 @@
-# Copyright (c) 2009-2011 VMware, Inc.
-
-require 'nats/client'
-require 'uri'
-require 'vcap/common'
-require 'yaml'
-
-class BusSnoop
-
- NATS_URI_DEFAULT = 'nats://localhost:4222/'
-
- def initialize(options = {})
- @uri = options[:uri] || NATS_URI_DEFAULT
- end
-
- def stop
- NATS.stop
- end
-
- def start
- NATS.start :uri => @uri do
- NATS.subscribe(">") do |msg, reply, sub|
-
- if block_given?
- yield msg, reply, sub
- else
- begin
- puts sub
- puts parse_json(msg).to_yaml
-
- rescue Yajl::ParseError => e
-
- puts "ERROR: Yajl::ParseError: #{e}\nSubject: '#{sub}'\nMessage follows\n#{msg}"
- NATS.stop
- end
-
- end
- end
- end
- end
-
-
-end
-
-def parse_json str
- Yajl::Parser.parse(str)
-end
-
-def parse_args(args)
- opts = {}
- opts[:uri] = args.shift unless args.empty?
- opts[:pattern] = args.shift unless args.empty?
- opts
-end
-
-
-#when run on the command line, starts snooping right away using command line args for parameteres
-#otherwise can be used as a library
-if __FILE__ == $0
- options = parse_args($*)
-
- hm_pattern = 'dea\.heartbeat|healthmanager\.(status|health)|droplet\.(exited|updated)|cloudcontrollers\.hm\.requests'
-
- snoop = BusSnoop.new(options)
- re = Regexp.new( options[:pattern] || hm_pattern)
-
- puts "Starting the BusSnoop at #{@uri} listening on '#{re}'"
- snoop.start do |msg, reply, sub|
- next unless re.match sub
- puts sub
- puts parse_json(msg).to_yaml
- end
-end
View
110 health_manager/spec/functional/db_helper.rb
@@ -1,110 +0,0 @@
-class HMExpectedStateHelperDB
-
- def initialize options
- @options = options
- end
-
- def config
- {
- 'adapter' => 'sqlite3',
- 'database' => File.join(@options['run_dir'], 'test.sqlite3'),
- 'encoding' => 'utf8'
- }
- end
-
- def add_user(args)
- user = User.create(args)
- user.save!
- user
- end
-
- def find_user(args)
- User.where(args).first
- end
-
- def add_app(args)
- app = App.create(args)
- app.save!
- app
- end
-
- def make_app_with_owner_and_instance(app_def, user_def)
- app = App.new app_def
- owner = add_user user_def
-
- app.owner = owner
- app.instances = 1
-
- app.save!
- app
- end
-
-
- def find_app(args)
- App.where(args).first
- end
-
- def delete_all
- App.delete_all
- User.delete_all
- end
-
-
- def prepare_tests
-
- [App, User].each {|model| model.reset_column_information}
-
- ActiveRecord::Base.establish_connection config
- ActiveRecord::Migration.verbose = false
-
- ActiveRecord::Schema.define do
-
- create_table "app_collaborations", :force => true do |t|
- t.integer "app_id"
- t.integer "user_id"
- t.datetime "created_at"
- t.datetime "updated_at"
- end
-
- add_index "app_collaborations", ["app_id", "user_id"], :name => "index_app_collaborations_on_app_id_and_user_id", :unique => true
-
- create_table "apps", :force => true do |t|
- t.integer "owner_id"
- t.string "name"
- t.string "framework"
- t.string "runtime"
- t.boolean "prod", :default => false
- t.integer "memory", :default => 256
- t.integer "instances", :default => 0
- t.string "state", :default => "STOPPED"
- t.string "package_state", :default => "PENDING"
- t.string "package_hash"
- t.text "environment_json"
- t.text "metadata"
- t.boolean "external_secret", :default => false
- t.datetime "created_at"
- t.datetime "updated_at"
- t.string "staged_package_hash"
- t.integer "file_descriptors", :default => 256
- t.integer "disk_quota", :default => 2048
- t.integer "lock_version", :default => 0
- t.integer "run_count", :default => 0, :null => false
- end
-
- create_table "users", :force => true do |t|
- t.string "email"
- t.string "crypted_password"
- t.boolean "active", :default => false
- t.datetime "created_at"
- t.datetime "updated_at"
- end
-
- add_index "users", ["email"], :name => "index_users_on_email"
- end
- end
-
- def release_resources
- delete_all
- ActiveRecord::Base.clear_all_connections!
- end
-end
View
194 health_manager/spec/functional/health_manager_spec.rb
@@ -1,194 +0,0 @@
-# Copyright (c) 2009-2011 VMware, Inc.
-require File.join(File.dirname(__FILE__), 'spec_helper')
-
-require 'digest/sha1'
-require 'fileutils'
-require 'nats/client'
-require 'uri'
-require 'vcap/common'
-require 'yaml'
-
-describe 'Health Manager' do
- nats_timeout_path = File.expand_path(File.join(File.dirname(__FILE__), 'nats_timeout'))
- hm_path = File.expand_path(File.join(__FILE__, "../../../bin/health_manager"))
- run_id_ctr = 0
-
- before :all do
- # Base test directory to house invididual run directories
- @test_dir = "/tmp/hm_tests_#{Process.pid}_#{Time.now.to_i}"
- FileUtils.mkdir(@test_dir)
- File.directory?(@test_dir).should be_true
- @run_id = 0
-
- @run_dir = File.join(@test_dir, "run_#{run_id_ctr}")
- create_dir(@run_dir)
-
- @helper = HMExpectedStateHelperDB.new( 'run_dir' => @run_dir)
-
- # NATS
- port = VCAP.grab_ephemeral_port
- pid_file = File.join(@run_dir, 'nats.pid')
- @nats_uri = "nats://localhost:#{port}"
- @nats_server = VCAP::Spec::ForkedComponent::NatsServer.new(pid_file, port, @run_dir)
-
- @nats_server.start
- @nats_server.wait_ready.should be_true
-
- @hm_cfg = {
- 'mbus' => @nats_uri,
- 'local_route' => '127.0.0.1',
- 'intervals' => {
- 'database_scan' => 1, #60
- 'droplet_lost' => 1, #30
- 'droplets_analysis' => 1, #10
- 'flapping_death' => 4, #3
- 'flapping_timeout' => 9, #180
- 'restart_timeout' => 15, #20
- 'stable_state' => -1, #ensures all apps are "quiescent" for the purpose of testing
- 'nats_ping' => 1,
- },
- 'logging' => {'level' => 'warn'},
- 'pid' => File.join(@run_dir, 'health_manager.pid'),
- 'dequeueing_rate' => 0,
- 'rails_environment' => 'test',
- 'database_environment' =>{
- 'test' => @helper.config
- }
- }
- @hm_config_file = File.join(@run_dir, 'health_manager.config')
- File.open(@hm_config_file, 'w') {|f| YAML.dump(@hm_cfg, f) }
- @hm = HealthManagerComponent.
- new("ruby -r#{nats_timeout_path} #{hm_path} -c #{@hm_config_file}",
- @hm_cfg['pid'],@hm_cfg, @run_dir)
-
- @hm.reopen_stdio = false if ENV['HM_NO_STDIO_REOPEN']
-
- @helper.prepare_tests
- receive_message 'healthmanager.start' do
- @hm.start
- end.should_not be_nil
- end
-
- after :all do
- # Cleanup after ourselves
- @nats_server.stop
- @nats_server.running?.should be_false
-
- @hm.stop
- @hm.running?.should be_false
- @helper.release_resources
-
- FileUtils.rm_rf(@test_dir)
- end
-
- describe 'when running' do
- before :each do
- @helper.delete_all
- end
-
- #TODO: test stop duplicate instances
- #TODO: test heartbeat timeout
- #TODO: test respond to health requests
- #TODO: test flapping instances
- #TODO: test wait for droplet to be stable
- #TODO: test that droplets have a chance to restart
-
- #test start missing instances
- it 'should start missing instances' do
- app = nil
- msg = receive_message 'cloudcontrollers.hm.requests.default' do
- #putting enough into Expected State to trigger an instance START request
- app = @helper.make_app_with_owner_and_instance(
- make_app_def( 'to_be_started_app'),
- make_user_def)
- end
- app.should_not be_nil
- msg.should_not be_nil
- msg = parse_json msg
- msg['op'].should == 'START'
- msg['droplet'].should == app.id.to_s
- end
-
- it 'should not start missing instances with empty staged_package_hash' do
- app = nil
- msg = receive_message('cloudcontrollers.hm.requests.default', false) do
-
- app_def = make_app_def('non_sane_app')
- app_def.delete(:staged_package_hash)
-
- app = @helper.make_app_with_owner_and_instance(app_def, make_user_def)
- end
- app.should_not be_nil
- msg.should be_nil
- end
-
- #test restart crashed instances
- it 'should start crashed instances' do
- app = @helper.make_app_with_owner_and_instance(make_app_def('crasher'), make_user_def)
- crash_msg = {
- 'droplet' => app.id.to_s,
- 'cc_partition' => "default",
- 'version' => 0,
- 'instance' => 0,
- 'index' => 0,
- 'reason' => 'CRASHED',
- 'crash_timestamp' => Time.now.to_i
- }
- msg = receive_message 'cloudcontrollers.hm.requests.default' do
- NATS.publish('droplet.exited', crash_msg.to_json)
- end
- msg.should_not be_nil
- msg = parse_json(msg)
- msg['droplet'].should == app.id.to_s
- msg['op'].should == 'START'
- msg['version'].to_i.should == crash_msg['version']
- end
- end
-
- def make_user_def
- { :email => 'boo@boo.com', :crypted_password => 'boo' }
- end
-
- def make_app_def(app_name)
- {
- :name => app_name,
- :framework => 'sinatra',
- :runtime => 'ruby19',
- :state => 'STARTED',
- :package_state => 'STAGED',
- :staged_package_hash => 'abcdefg',
- :package_hash => 'abcdef'
- }
- end
-
- def create_dir(dir)
- File.directory?(dir).should be_false
- FileUtils.mkdir(dir)
- File.directory?(dir).should be_true
- end
-
- def parse_json(str)
- Yajl::Parser.parse(str)
- end
-
- def receive_message(subj, expected = true)
- ret = nil
- timeout = 5
- EM.run do
- EM.add_timer(timeout) do
- puts "TIMEOUT while waiting on #{subj}" if expected
- EM.stop
- end
- NATS.start :uri => @nats_server.uri do
- NATS.subscribe(subj) do |msg|
- ret = msg
- EM.stop
- end
- if block_given?
- NATS.publish('foo') { yield }
- end
- end
- end
- ret
- end
-end
View
11 health_manager/spec/functional/nats_timeout.rb
@@ -1,11 +0,0 @@
-require 'rubygems'
-require 'bundler/setup'
-
-require 'nats/client'
-
-# Short circuit the reconnect time here by just firing disconnect logic
-module NATS
- def unbind
- process_disconnect
- end
-end
View
17 health_manager/spec/functional/spec_helper.rb
@@ -1,17 +0,0 @@
-# Copyright (c) 2009-2011 VMware, Inc.
-require File.join(File.dirname(__FILE__), '..', 'spec_helper')
-
-require File.join(File.dirname(__FILE__), 'db_helper')
-
-require 'socket'
-require 'vcap/common'
-require 'vcap/spec/forked_component/nats_server'
-
-VCAP::Logging.setup_from_config( :level => :debug )
-
-class HealthManagerComponent < VCAP::Spec::ForkedComponent::Base
- def initialize(cmd, pid_filename, config, output_basedir)
- super(cmd,'healthmanager', output_basedir, pid_filename )
- @config = config
- end
-end
View
499 health_manager/spec/health_manager_spec.rb
@@ -1,499 +0,0 @@
-# Copyright (c) 2009-2011 VMware, Inc.
-require 'spec_helper'
-
-#functional tests are now implemented in functional/health_manager_spec.rb
-
-describe HealthManager do
-
- def build_user
- @user = ::User.find_by_email('test@example.com')
- unless @user
- @user = ::User.new(:email => "test@example.com")
- @user.set_and_encrypt_password('HASHEDPASSWORD')
- @user.save!
- end
- @user
- end
-
- def make_db_app_entry(appname)
- app = @user.apps.find_by_name(appname)
- unless app
- app = ::App.new(:name => appname, :owner => @user, :runtime => "ruby19", :framework => "sinatra")
- app.package_hash = random_hash
- app.staged_package_hash = random_hash
- app.state = "STARTED"
- app.package_state = "STAGED"
- app.instances = 3
- app.save!
- end
- app
- end
-
- def make_stats
- { :frameworks => {}, :runtimes => {}, :running => 0, :down => 0 }
- end
-
- def build_app(appname = 'testapp')
- @app = make_db_app_entry(appname)
- @app.set_urls(["http://#{appname}.vcap.me"])
-
- @droplet_entry = {
- :last_updated => @app.last_updated - 2, # take off 2 seconds so it looks 'quiescent'
- :state => 'STARTED',
- :crashes => {},
- :versions => {},
- :live_version => "#{@app.staged_package_hash}-#{@app.run_count}",
- :instances => @app.instances,
- :framework => 'sinatra',
- :runtime => 'ruby19'
- }
- @hm.update_droplet(@app)
- @app
- end
-
- def random_hash(len=40)
- res = ""
- len.times { res << rand(16).to_s(16) }
- res
- end
-
- def build_user_and_app
- build_user
- build_app
- end
-
- def should_publish_to_nats(message, payload)
- NATS.should_receive(:publish).with(message, payload.to_json)
- end
-
- after(:each) do
- VCAP::Logging.reset
- end
-
- after(:all) do
- ::User.destroy_all
- ::App.destroy_all
- end
-
- before(:each) do
-
- @config = {
- 'mbus' => 'nats://localhost:4222/',
- 'logging' => {
- 'level' => ENV['LOG_LEVEL'] || 'warn',
- },
- 'intervals' => {
- 'database_scan' => 1,
- 'droplet_lost' => 300,
- 'droplets_analysis' => 0.5,
- 'flapping_death' => @flapping_death = 2,
- 'min_restart_delay' => @min_restart_delay = 1,
- 'max_restart_delay' => @max_restart_delay = 3,
- 'giveup_crash_number' => @giveup_crash_number = 5,
- 'flapping_timeout' => 5,
- 'restart_timeout' => 2,
- 'stable_state' => -1,
-
- },
- 'dequeueing_rate' => 50,
- 'rails_environment' => 'test',
- 'database_environment' => {
- 'test' => {
- 'adapter' => 'sqlite3',
- 'database' => 'db/test.sqlite3',
- 'encoding' => 'utf8'
- }
- }
- }
-
- @hm = HealthManager.new(@config)
-
- hash = Hash.new {|h,k| h[k] = 0}
- VCAP::Component.stub!(:varz).and_return(hash)
- ::User.destroy_all
- ::App.destroy_all
-
- build_user_and_app
- end
-
- def make_heartbeat_message(options = {})
- options = options.dup #copy before getting all destructive
- indices = options.delete('indices') || [0]
-
- droplets = []
- indices.each do |index|
- droplets << {
- 'droplet' => @app.id.to_s,
- 'cc_partition' => "default",
- 'index' => index,
- 'instance' => "badbeef-#{index}",
- 'state' => 'RUNNING',
- 'version' => @droplet_entry[:live_version],
- 'state_timestamp' => @droplet_entry[:last_updated]
- }.merge(options)
- end
- { 'droplets' => droplets }
- end
-
- def make_crashed_message(options={})
- {
- 'droplet' => @app.id.to_s,
- 'cc_partition' => "default",
- 'version' => "#{@app.staged_package_hash}-#{@app.run_count}",
- 'index' => 0,
- 'instance' => "badbeef-0",
- 'reason' => 'CRASHED',
- 'crash_timestamp' => Time.now.to_i
- }.merge(options)
- end
-
- def make_restart_message(options = {})
- m = {
- 'droplet' => @app.id.to_s,
- 'op' => 'START',
- 'last_updated' => @app.last_updated.to_i,
- 'version' => "#{@app.staged_package_hash}-#{@app.run_count}",
- 'indices' => [0]
- }.merge(options)
- end
-
- def get_live_index(droplet_entry,index)
- droplet_entry[:versions].should_not be_nil
- version_entry = droplet_entry[:versions][@droplet_entry[:live_version]]
- version_entry.should_not be_nil
- version_entry[:indices][index].should_not be_nil
- version_entry[:indices][index]
- end
-
- describe '#perform_quantum' do
- it 'should be resilient to nil arguments' do
- @hm.perform_quantum(nil, nil)
- end
- end
-
- it "should detect instances that are down and send a START request" do
- stats = { :frameworks => {}, :runtimes => {}, :down => 0 }
- should_publish_to_nats "cloudcontrollers.hm.requests.default", {
- 'droplet' => @app.id.to_s,
- 'op' => 'START',
- 'last_updated' => @app.last_updated.to_i,
- 'version' => "#{@app.staged_package_hash}-#{@app.run_count}",
- 'indices' => [0,1,2]
- }
-
- @hm.analyze_app(@app.id.to_s, @droplet_entry, stats)
- @hm.deque_a_batch_of_requests
-
- stats[:down].should == 3
- stats[:frameworks]['sinatra'][:missing_instances].should == 3
- stats[:runtimes]['ruby19'][:missing_instances].should == 3
- end
-
- it "should detect extra instances and send a STOP request" do
- stats = make_stats
- timestamp = Time.now.to_i
- version_entry = { indices: {
- 0 => { :state => 'RUNNING', :timestamp => timestamp, :last_action => @app.last_updated, :instance => '0' },
- 1 => { :state => 'RUNNING', :timestamp => timestamp, :last_action => @app.last_updated, :instance => '1' },
- 2 => { :state => 'RUNNING', :timestamp => timestamp, :last_action => @app.last_updated, :instance => '2' },
- 3 => { :state => 'RUNNING', :timestamp => timestamp, :last_action => @app.last_updated, :instance => '3' }
- }}
- should_publish_to_nats "cloudcontrollers.hm.requests.default", {
- 'droplet' => @app.id.to_s,
- 'op' => 'STOP',
- 'last_updated' => @app.last_updated.to_i,
- 'instances' => [ version_entry[:indices][3][:instance] ]
- }
- @droplet_entry[:versions][@droplet_entry[:live_version]] = version_entry
-
- @hm.analyze_app(@app.id.to_s, @droplet_entry, stats)
-
- stats[:running].should == 3
- stats[:frameworks]['sinatra'][:running_instances].should == 3
- stats[:runtimes]['ruby19'][:running_instances].should == 3
- end
-
- describe "spindown" do
-
- before(:each) do
- # ensure spindown disabled by default
- @hm.spindown_inactive_apps.should be_false
-
- # now create hm with spindown enabled
- @config['intervals']['inactivity_period_for_spindown'] = 2 # spindown after 2 seconds of inactivity
- @hm = HealthManager.new(@config)
- @hm.update_droplet(@app)
- droplet[:last_updated] -= 2 #to satisfy quiscence requirement
-
- @hm.spindown_inactive_apps.should be_true
- end
-
- def droplet
- @hm.droplets[@app.id.to_s]
- end
-
- def activity_message
- Zlib::Deflate.deflate([@app.id.to_s].to_json)
- end
-
- it "should update last_activity timestamp" do
- droplet[:last_activity].should be_nil
- @hm.process_active_apps_message(activity_message)
- droplet[:last_activity].should_not be_nil
- droplet[:last_activity].should <= @hm.now
- end
-
- it "should spindown app with no acitvity at all" do
- should_publish_to_nats('cloudcontrollers.hm.requests.default', {
- :droplet => @app.id.to_s,
- :op => :SPINDOWN
- })
-
- # make hm believe that 3 seconds have elapsed
- @hm.set_now( @hm.now + 3 )
-
- @hm.analyze_app(@app.id.to_s, droplet, make_stats)
- end
-
- it "should not spindown an app with activity" do
-
- 3.times {
- @hm.set_now( @hm.now + 1 )
- @hm.process_active_apps_message(activity_message)
- }
-
- @hm.analyze_app(@app.id.to_s, droplet, make_stats)
- end
-
- it "should not spindown inactive app with 'prod' flag set to true" do
- droplet[:prod] = true
- # make hm believe that 3 seconds have elapsed
- @hm.set_now( @hm.now + 3 )
- @hm.analyze_app(@app.id.to_s, droplet, make_stats)
- end
-
- it "should spindown an app with stale activity" do
- should_publish_to_nats('cloudcontrollers.hm.requests.default', {
- :droplet => @app.id.to_s,
- :op => :SPINDOWN
- })
- @hm.process_active_apps_message(activity_message)
-
- # make hm believe that 3 seconds have elapsed
- @hm.set_now( @hm.now + 3 )
-
- @hm.analyze_app(@app.id.to_s, droplet, make_stats)
- end
- end
-
- it "should update its internal state to reflect heartbeat messages" do
- droplet_entries = @hm.process_heartbeat_message(make_heartbeat_message.to_json)
-
- droplet_entries.size.should == 1
- droplet_entry = droplet_entries[0]
- get_live_index(droplet_entry,0)[:state].should == 'RUNNING'
- end
-
- it "should restart an instance that exits unexpectedly" do
- ensure_non_flapping_restart
- end
-
- it "should exponentially delay restarts for flapping instance" do
- @flapping_death.times {
- ensure_non_flapping_restart
- }
-
- delay = @min_restart_delay
-
- (@giveup_crash_number - @flapping_death).times {
- ensure_flapping_delayed_restart(delay)
- delay *= 2
- delay = @max_restart_delay if delay > @max_restart_delay
- }
- ensure_gaveup_restarting
- end
-
- describe 'cc_partition' do
- it 'should ignore heartbeat with mismatched cc_partition' do
- should_publish_to_nats("cloudcontrollers.hm.requests.default",make_restart_message('indices'=>[1]))
- hb = make_heartbeat_message('indices' => [0,1,2])
-
- hb['droplets'][1]['cc_partition'].should == 'default'
-
- # changing the value will make hm ignore this heartbeat,
- # resulting in restart message being sent
- hb['droplets'][1]['cc_partition'] = 'bogus_partition'
-
- @hm.process_heartbeat_message(hb.to_json)
- @droplet_entry = @hm.droplets[@app.id.to_s]
-
- @hm.analyze_app(@app.id.to_s, @droplet_entry, make_stats)
- @hm.deque_a_batch_of_requests
- end
-
- it 'should interpret absent cc_partition information as "default"' do
- hb = make_heartbeat_message('indices' => [0,1,2])
-
- # remove cc_partition entry for instance 1.
- # the absence of value will be intreted the same as a 'default' value
- # i.e., there will be no restart.
- hb['droplets'][1].delete('cc_partition').should == 'default'
-
- @hm.process_heartbeat_message(hb.to_json)
- @droplet_entry = @hm.droplets[@app.id.to_s]
- @hm.analyze_app(@app.id.to_s, @droplet_entry, make_stats)
- @hm.deque_a_batch_of_requests
- end
- end
-
- it 'should stop instance with mismatched prod flag' do
- stats = make_stats
-
- # this example simulates a non-prod app, with intances 0,2 running
- # on default (non-discriminating) dea, and instance 1
- # inappropriately running on prod-only dea. The instance 1 is
- # then stopped by hm, and restarted elsewhere.
-
- hb02 = make_heartbeat_message('indices' => [0,2])
- @hm.process_heartbeat_message(hb02.to_json)
-
- hb1 = make_heartbeat_message('indices' => [1])
- hb1['prod'] = true # augment heartbeat with dea prod status
- @hm.process_heartbeat_message(hb1.to_json)
-
- @droplet_entry = @hm.droplets[@app.id.to_s]
-
- stoppee_instance = @droplet_entry[:versions].values.first[:indices][1]
-
- stop_message = {
- 'droplet' => @app.id.to_s,
- 'op' => 'STOP',
- 'last_updated' => stoppee_instance[:timestamp],
- 'instances' => [stoppee_instance[:instance]]
- }
- should_publish_to_nats("cloudcontrollers.hm.requests.default", stop_message)
- should_publish_to_nats("cloudcontrollers.hm.requests.default", make_restart_message('indices'=>[1]))
-
- @hm.analyze_app(@app.id.to_s, @droplet_entry, stats)
- @hm.deque_a_batch_of_requests
- end
-
- def ensure_non_flapping_restart
- should_publish_to_nats "cloudcontrollers.hm.requests.default", make_restart_message
- @hm.process_heartbeat_message(make_heartbeat_message.to_json)
- droplet_entry = @hm.process_exited_message(make_crashed_message.to_json)
- @hm.deque_a_batch_of_requests
- get_live_index(droplet_entry,0)[:state].should == 'DOWN'
- @hm.restart_pending?(@app.id.to_s, 0).should be_false # first @flapping_death restarts are immediate.
- end
-
- def ensure_flapping_delayed_restart(delay)
- in_em_with_fiber do |f|
- should_publish_to_nats "cloudcontrollers.hm.requests.default", make_restart_message('flapping' => true)
-
- @hm.process_heartbeat_message(make_heartbeat_message.to_json)
- droplet_entry = @hm.process_exited_message(make_crashed_message.to_json)
-
- get_live_index(droplet_entry,0)[:state].should == 'FLAPPING'
- @hm.restart_pending?(@app.id.to_s, 0).should be_true
-
- # half a second before the delay elapses the restart is still pending
- EM.add_timer(delay - 0.5) do
- @hm.restart_pending?(@app.id.to_s, 0).should be_true
- @hm.deque_a_batch_of_requests
- @hm.restart_pending?(@app.id.to_s, 0).should be_true
- end
-
- # after delay elapses, the pending restart is initiated and is no longer pending
- EM.add_timer(delay + 0.5) do
- @hm.restart_pending?(@app.id.to_s, 0).should be_true
- @hm.deque_a_batch_of_requests
- @hm.restart_pending?(@app.id.to_s, 0).should be_false
- f.resume
- end
- end
- end
-
- def ensure_gaveup_restarting
- @hm.process_heartbeat_message(make_heartbeat_message.to_json)
- droplet_entry = @hm.process_exited_message(make_crashed_message.to_json)
- get_live_index(droplet_entry,0)[:state].should == 'FLAPPING'
- get_live_index(droplet_entry,0)[:crashes].should > @giveup_crash_number
- @hm.restart_pending?(@app.id.to_s, 0).should be_false
- end
-
- def in_em_with_fiber
- in_em do
- Fiber.new {
- yield Fiber.current
- Fiber.yield
- EM.stop
- }.resume
- end
- end
-
- def in_em(timeout = 10)
- EM.run do
- EM.add_timer(timeout) do
- EM.stop
- fail "Failed to complete withing allotted timeout"
- end
- yield
- end
- end
-
- it "should not re-start timer-triggered analysis loop if previous analysis loop is still in progress" do
-
- n=20
- apps = []
-
- n.times { |i|
- apps << make_db_app_entry("test#{i}")
- }
-
- VCAP::Component.varz[:running] = {}
- @hm.update_from_db
-
- in_em do
- @hm.analysis_in_progress?.should be_false
- @hm.analyze_all_apps.should be_true
- @hm.analysis_in_progress?.should be_true
- @hm.analyze_all_apps.should be_false
- EM.stop
- end
- end
-
- it "should have FIFO behavior for DEA_EVACUATION-triggered restarts" do
- apps = []
-
- apps << @app
- apps << build_app('test2')
- apps << build_app('test3')
-
- apps.each do |app|
-
- should_publish_to_nats("cloudcontrollers.hm.requests.default", {
- 'droplet' => app.id.to_s ,
- 'op' => 'START',
- 'last_updated' => app.last_updated.to_i,
- 'version' => "#{app.staged_package_hash}-#{app.run_count}",
- 'indices' => [0]
-
- }).ordered #CRUCIAL
- end
-
-
- apps.each do |app|
- @hm.process_exited_message({
- 'droplet' => app.id.to_s,
- 'cc_partition' => "default",
- 'version' => "#{app.staged_package_hash}-#{app.run_count}",
- 'index' => 0,
- 'instance' => 0,
- 'reason' => 'DEA_EVACUATION',
- 'crash_timestamp' => Time.now.to_i
- }.to_json)
- end
- @hm.deque_a_batch_of_requests
- end
-end
View
56 health_manager/spec/spec_helper.rb
@@ -1,56 +0,0 @@
-# Copyright (c) 2009-2011 VMware, Inc.
-$:.unshift File.join(File.dirname(__FILE__), '..')
-$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
-
-require 'rubygems'
-require 'rspec'
-require 'health_manager'
-require 'fiber'
-
-
-class ::HealthManager
- def now
- @now || Time.now.to_i
- end
-
- def set_now(val)
- @now = val
- end
-end
-
-module Spec
- module Mocks
- module ArgumentMatchers
-
- class JsonStringMatcher
- def initialize(expected)
- @expected = expected
- end
-
- def ==(actual)
- JSON.parse(actual).eql?(@expected)
- end
-
- def description
- @expected.to_json
- end
- end
-
- def json_string(expected)
- JsonStringMatcher.new(expected)
- end
-
- end
- end
-end
-
-module Kernel
- def silence_warnings
- original_verbosity = $VERBOSE
- $VERBOSE = nil
- result = yield
- $VERBOSE = original_verbosity
- return result
- end
-end
-
Please sign in to comment.
Something went wrong with that request. Please try again.