Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

ship service per-instance healthz data over varz

now each service node's varz data has one key "instances", its value is a dict,
the format is as follows,
{
    "instance 1 name":"ok",
    "instance 2 name":"fail"
}
each service node/provisioner healthz data is inherited from base class.

Change-Id: Id9f5400b3d26a135e64e14ced8e9bd0b3cc00dcb
  • Loading branch information...
commit 8b12af491edfea58953cd07e1c80954a9006b22d 1 parent b25c8da
chunjie authored Chunjie committed
View
5 base/lib/base/base.rb
@@ -85,10 +85,6 @@ def update_varz()
}
end
- def update_healthz()
- VCAP::Component.healthz = Yajl::Encoder.encode(healthz_details, :pretty => true, :terminator => "\n")
- end
-
def shutdown()
@logger.info("#{service_description}: Shutting down")
@node_nats.close if @node_nats
@@ -147,7 +143,6 @@ def group_handles_in_json(instances_list, bindings_list, size_limit)
abstract :on_connect_node
abstract :flavor # "Provisioner" or "Node"
abstract :varz_details
- abstract :healthz_details
# Service Provisioner and Node classes must implement the following
# method
View
12 base/lib/base/node.rb
@@ -23,12 +23,12 @@ def initialize(options)
z_interval = options[:z_interval] || 30
EM.add_periodic_timer(z_interval) do
- EM.defer { update_varz; update_healthz }
+ EM.defer { update_varz }
end
# Defer 5 seconds to give service a change to wake up
EM.add_timer(5) do
- EM.defer { update_varz; update_healthz }
+ EM.defer { update_varz }
end
end
@@ -295,14 +295,6 @@ def varz_details()
return announcement
end
- def healthz_details()
- # Service Node subclasses may want to override this method to
- # provide service specific data
- healthz = {
- :self => "ok"
- }
- end
-
def capacity_unit
# subclasses could overwrite this method to re-define
# the capacity unit decreased/increased by provision/unprovision
View
10 base/lib/base/provisioner.rb
@@ -33,12 +33,12 @@ def initialize(options)
z_interval = options[:z_interval] || 30
EM.add_periodic_timer(z_interval) do
- update_varz; update_healthz
+ update_varz
end
# Defer 5 seconds to give service a change to wake up
EM.add_timer(5) do
- update_varz; update_healthz
+ update_varz
end
EM.add_periodic_timer(60) { process_nodes }
@@ -752,12 +752,6 @@ def varz_details()
@logger.warn("Exception at varz_details #{e}")
end
- def healthz_details()
- healthz = {
- :self => "ok"
- }
- end
-
########
# Helpers
########
View
8 base/spec/helper/node_spec_helper.rb
@@ -30,7 +30,7 @@ class NodeTester < VCAP::Services::Base::Node
attr_reader :unbind_count
attr_reader :capacity
attr_accessor :varz_invoked
- attr_accessor :healthz_invoked
+ attr_reader :healthz_ok
SERVICE_NAME = "Test"
ID = "node-1"
def initialize(options)
@@ -47,10 +47,10 @@ def initialize(options)
@unprovision_count = 0
@unbind_count = 0
@varz_invoked = false
- @healthz_invoked = false
@ins_count = options[:ins_count] || 0
@bind_count = options[:bind_count] || 0
@plan = options[:plan] || "free"
+ @healthz_ok = VCAP::Component.healthz
end
def service_name
SERVICE_NAME
@@ -89,10 +89,6 @@ def varz_details
@varz_invoked = true
{}
end
- def healthz_details
- @healthz_invoked = true
- {}
- end
def all_instances_list
generate_ins_list(@ins_count)
View
5 base/spec/helper/provision_spec_helper.rb
@@ -31,7 +31,6 @@ def self.setup_fake_instance(gateway, provisioner, node)
class ProvisionerTester < VCAP::Services::Base::Provisioner
attr_accessor :prov_svcs
attr_accessor :varz_invoked
- attr_accessor :healthz_invoked
attr_accessor :prov_svcs
attr_reader :staging_orphan_instances
attr_reader :staging_orphan_bindings
@@ -56,10 +55,6 @@ def varz_details
@varz_invoked = true
super
end
- def healthz_details
- @healthz_invoked = true
- super
- end
end
class MockGateway
View
15 base/spec/node_spec.rb
@@ -16,7 +16,7 @@
provisioner.got_announcement.should be_true
end
- it "should call varz/healthz" do
+ it "should call varz" do
node = nil
provisioner = nil
EM.run do
@@ -26,7 +26,18 @@
Do.at(12) { EM.stop }
end
node.varz_invoked.should be_true
- node.healthz_invoked.should be_true
+ end
+
+ it "should report healthz ok" do
+ node = nil
+ provisioner = nil
+ EM.run do
+ # start provisioner then node
+ Do.at(0) { provisioner = NodeTests.create_provisioner }
+ Do.at(1) { node = NodeTests.create_node }
+ Do.at(12) { EM.stop }
+ end
+ node.healthz_ok.should == "ok\n"
end
it "should announce on request" do
View
30 base/spec/provision_spec.rb
@@ -294,36 +294,6 @@
prov_svcs_before.should == prov_svcs_after
end
- it "should support healthz" do
- provisioner = nil
- gateway = nil
- node = nil
- prov_svcs_before = nil
- prov_svcs_after = nil
- healthz_invoked_before = nil
- healthz_invoked_after = nil
- EM.run do
- Do.at(0) { provisioner = ProvisionerTests.create_provisioner }
- Do.at(1) { gateway = ProvisionerTests.create_gateway(provisioner) }
- Do.at(2) { node = ProvisionerTests.create_node(1) }
- Do.at(3) { gateway.send_provision_request }
- Do.at(4) { gateway.send_bind_request }
- Do.at(5) {
- prov_svcs_before = Marshal.dump(provisioner.prov_svcs)
- healthz_invoked_before = provisioner.healthz_invoked
- }
- # healthz is invoked 5 seconds after provisioner is created
- Do.at(11) {
- prov_svcs_after = Marshal.dump(provisioner.prov_svcs)
- healthz_invoked_after = provisioner.healthz_invoked
- }
- Do.at(12) { EM.stop }
- end
- healthz_invoked_before.should be_false
- healthz_invoked_after.should be_true
- prov_svcs_before.should == prov_svcs_after
- end
-
it "should allow over provisioning when it is configured so" do
provisioner = nil
gateway = nil
View
28 mongodb/lib/mongodb_service/mongodb_node.rb
@@ -498,26 +498,26 @@ def varz_details
stat['name'] = provisioned_service.name
stats << stat
end
+
+ # Get service instance status
+ provisioned_instances = {}
+ begin
+ ProvisionedService.all.each do |instance|
+ provisioned_instances[instance.name.to_sym] = get_status(instance)
+ end
+ rescue => e
+ @logger.error("Error get instance list: #{e}")
+ end
+
{
:running_services => stats,
:disk => du_hash,
:max_capacity => @max_capacity,
- :available_capacity => @capacity
+ :available_capacity => @capacity,
+ :instances => provisioned_instances
}
end
- def healthz_details
- healthz = {}
- healthz[:self] = "ok"
- ProvisionedService.all.each do |instance|
- healthz[instance.name.to_sym] = get_healthz(instance)
- end
- healthz
- rescue => e
- @logger.warn("Error get healthz details: #{e}")
- {:self => "fail"}
- end
-
def connect_and_auth(instance)
conn = nil
@connection_mutex.synchronize do
@@ -543,7 +543,7 @@ def close_connection(instance)
end
end
- def get_healthz(instance)
+ def get_status(instance)
conn = connect_and_auth(instance)
"ok"
rescue => e
View
12 mongodb/spec/mongodb_provision_spec.rb
@@ -52,7 +52,6 @@
stats = nil
10.times do
stats = @node.varz_details
- @node.healthz_details
end
stats.should_not be_nil
stats[:running_services].length.should > 0
@@ -62,16 +61,7 @@
stats[:disk].should_not be_nil
stats[:max_capacity].should > 0
stats[:available_capacity].should > 0
- EM.stop
- end
- end
-
- it "should return healthz" do
- EM.run do
- stats = @node.healthz_details
- stats.should_not be_nil
- stats[:self].should == "ok"
- stats[@resp['name'].to_sym].should == "ok"
+ stats[:instances].length.should > 0
EM.stop
end
end
View
29 mysql/lib/mysql_service/node.rb
@@ -542,35 +542,22 @@ def varz_details()
varz[:provision_served] = @provision_served
varz[:binding_served] = @binding_served
end
- varz
- rescue => e
- @logger.error("Error during generate varz: #{e}")
- {}
- end
-
- def healthz_details()
- healthz = {:self => "ok"}
- begin
- @pool.with_connection do |connection|
- connection.query("SHOW DATABASES")
- end
- rescue => e
- @logger.error("Error get database list: #{e}")
- healthz[:self] = "fail"
- return healthz
- end
+ # provisioned services status
+ varz[:instances] = {}
begin
ProvisionedService.all.each do |instance|
- healthz[instance.name.to_sym] = get_instance_healthz(instance)
+ varz[:instances][instance.name.to_sym] = get_status(instance)
end
rescue => e
@logger.error("Error get instance list: #{e}")
- healthz[:self] = "fail"
end
- healthz
+ varz
+ rescue => e
+ @logger.error("Error during generate varz: #{e}")
+ {}
end
- def get_instance_healthz(instance)
+ def get_status(instance)
res = "ok"
host, port, socket, root_user, root_pass = %w{host port socket user pass}.map { |opt| @mysql_config[opt] }
begin
View
59 mysql/spec/mysql_node_spec.rb
@@ -559,56 +559,43 @@ class MysqlError
end
end
- it "should report node status in healthz" do
+ it "should report node instance status in varz" do
pending "This test is not capatiable with mysql2 conenction pool."
EM.run do
- healthz = @node.healthz_details()
- healthz[:self].should == "ok"
+ varz = @node.varz_details
+ varz[:instances].each do |name, status|
+ status.shoud == "ok"
+ end
node = VCAP::Services::Mysql::Node.new(@opts)
EM.add_timer(1) do
node.pool.close
- healthz = node.healthz_details()
- healthz[:self].should == "fail"
+ varz = node.varz_details
+ varz[:instances].each do |name, status|
+ status.should == "ok"
+ end
EM.stop
end
end
end
- it "should report correct health status when user modify instance password" do
- EM.run do
- conn = connect_to_mysql(@db)
- a = conn.query("set password for #{@db['user']}@'localhost' = PASSWORD('newpass')")
- healthz = @node.healthz_details()
- healthz[:self].should == "ok"
- healthz[@db['name'].to_sym].should == "password-modified"
- EM.stop
- end
- end
-
- it "should close extra mysql connections after generate healthz" do
- EM.run do
- @node.pool.with_connection do |connection|
- res = connection.query("show processlist")
- conns_before_healthz = res.count
- healthz = @node.healthz_details()
- healthz.keys.size.should >= 2
- res = connection.query("show processlist")
- conns_after_healthz = res.count
- conns_before_healthz.should == conns_after_healthz
- end
- EM.stop
- end
- end
-
- it "should report instance status in healthz" do
+ it "should report instance status in varz" do
EM.run do
- healthz = @node.healthz_details()
+ varz = @node.varz_details()
instance = @db['name']
- healthz[instance.to_sym].should == "ok"
+ varz[:instances].each do |name, value|
+ if name == instance.to_sym
+ value.should == "ok"
+ end
+ end
@node.pool.with_connection do |connection|
connection.query("Drop database #{instance}")
- healthz = @node.healthz_details()
- healthz[instance.to_sym].should == "fail"
+ sleep 1
+ varz = @node.varz_details()
+ varz[:instances].each do |name, value|
+ if name == instance.to_sym
+ value.should == "fail"
+ end
+ end
# restore db so cleanup code doesn't complain.
connection.query("create database #{instance}")
end
View
6 neo4j/spec/spec_helper.rb
@@ -66,8 +66,12 @@ def parse_property(hash, key, type, options = {})
end
end
+def config_base_dir
+ ENV["CLOUD_FOUNDRY_CONFIG_PATH"] || File.join(File.dirname(__FILE__), '..', 'config')
+end
+
def get_node_config()
- config_file = File.join(File.dirname(__FILE__), "../config/neo4j_node.yml")
+ config_file = File.join(config_base_dir, "neo4j_node.yml")
config = YAML.load_file(config_file)
neo4j_server_conf_template = File.join(File.dirname(__FILE__), "../resources/neo4j-server.properties.erb")
neo4j_conf_template = File.join(File.dirname(__FILE__), "../resources/neo4j.properties.erb")
View
29 postgresql/lib/postgresql_service/node.rb
@@ -800,6 +800,15 @@ def varz_details()
# how many provision/binding operations since startup.
varz[:provision_served] = @provision_served
varz[:binding_served] = @binding_served
+ # get instances status
+ varz[:instances] = {}
+ begin
+ Provisionedservice.all.each do |instance|
+ varz[:instances][instance.name.to_sym] = get_status(instance)
+ end
+ rescue => e
+ @logger.error("Error get instance list: #{e}")
+ end
varz
rescue => e
@logger.warn("Error during generate varz: #{e}")
@@ -833,25 +842,7 @@ def get_db_stat
[]
end
- def healthz_details()
- healthz = {:self => "ok"}
- if connection_exception
- @logger.warn("PostgreSQL connection lost, healthz fail.")
- healthz[:self] = "fail"
- return healthz
- end
- begin
- Provisionedservice.all.each do |instance|
- healthz[instance.name.to_sym] = get_instance_healthz(instance)
- end
- rescue => e
- @logger.error("Error get instance list: #{e}")
- healthz[:self] = "fail"
- end
- healthz
- end
-
- def get_instance_healthz(instance)
+ def get_status(instance)
res = 'ok'
host, port = %w{host port}.map { |opt| @postgresql_config[opt] }
begin
View
62 postgresql/spec/postgresql_node_spec.rb
@@ -424,44 +424,23 @@ class PostgresqlError
end
end
- it "should close extra postgresql connections after generate healthz" do
+ it "should report instance status in varz" do
EM.run do
- conns_before = {}
- conns_after = {}
- varz = @node.varz_details
- db_stats = varz[:db_stat]
- db_stats.each do |db|
- conns_before[db[:name]] = db[:active_server_processes]
- end
- self_db = db_stats.find {|d| d[:name] == @db["name"]}
- self_db.should_not be_nil
-
- healthz = @node.healthz_details()
- healthz.keys.size.should >= 2
-
- sleep 0.1
- varz = @node.varz_details
- db_stats = varz[:db_stat]
- db_stats.each do |db|
- conns_after[db[:name]] = db[:active_server_processes]
- end
-
- conns_before.each do |k, v|
- conns_after[k].should == v
- end
- EM.stop
- end
- end
-
- it "should report instance status in healthz" do
- EM.run do
- healthz = @node.healthz_details()
+ varz = @node.varz_details()
instance = @db['name']
- healthz[instance.to_sym].should == "ok"
+ varz[:instances].each do |name, value|
+ if (name == instance.to_sym)
+ value.should == "ok"
+ end
+ end
conn = @node.connection
conn.query("drop database #{instance}")
- healthz = @node.healthz_details()
- healthz[instance.to_sym].should == "fail"
+ varz = @node.varz_details()
+ varz[:instances].each do |name, value|
+ if (name == instance.to_sym)
+ value.should == "fail"
+ end
+ end
# restore db so cleanup code doesn't complain.
conn.query("create database #{instance}")
EM.stop
@@ -833,21 +812,6 @@ class PostgresqlError
varz.should == {}
end
- it "should report node status in healthz" do
- node = nil
- EM.run do
- opts = getNodeTestConfig
- node = VCAP::Services::Postgresql::Node.new(opts)
- sleep 1
- EM.add_timer(0.1) {EM.stop}
- end
- healthz = node.healthz_details()
- healthz[:self].should == "ok"
- node.connection.close
- healthz = node.healthz_details()
- healthz[:self].should == "fail"
- end
-
it "should return node not ready if postgresql server is not connected" do
node = nil
EM.run do
View
18 redis/lib/redis_service/redis_node.rb
@@ -248,24 +248,16 @@ def varz_details
varz[:provisioned_instances] << get_varz(instance)
varz[:provisioned_instances_num] += 1
end
+ varz[:instances] = {}
+ ProvisionedService.all.each do |instance|
+ varz[:instances][instance.name.to_sym] = get_status(instance)
+ end
varz
rescue => e
@logger.warn("Error while getting varz details: #{e}")
{}
end
- def healthz_details
- healthz = {}
- healthz[:self] = "ok"
- ProvisionedService.all.each do |instance|
- healthz[instance.name.to_sym] = get_healthz(instance)
- end
- healthz
- rescue => e
- @logger.warn("Error while getting healthz details: #{e}")
- {:self => "fail"}
- end
-
def start_db
DataMapper.setup(:default, @local_db)
DataMapper::auto_upgrade!
@@ -417,7 +409,7 @@ def gen_credentials(instance)
}
end
- def get_healthz(instance)
+ def get_status(instance)
Timeout::timeout(@redis_timeout) do
redis = Redis.new({:port => instance.port, :password => instance.password})
redis.echo("")
View
11 redis/spec/node_spec.rb
@@ -308,13 +308,14 @@ class Node
end
end
- describe "Node.healthz_details" do
- it "should report healthz details" do
+ describe "Node.health" do
+ it "should report service instances status" do
@credentials = @node.provision(:free)
sleep 1
- healthz = @node.healthz_details
- healthz[:self].should == "ok"
- healthz[@credentials["name"].to_sym].should == "ok"
+ varz = @node.varz_details
+ varz[:instances].each do |name, status|
+ status.should == "ok"
+ end
@node.unprovision(@credentials["name"])
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.