Permalink
Browse files

service versions

1. change service manifest to {:vendor => '', :version => ''}
2. enable ENV vars for each service manifest
3. add service_versions_helper to manage multiple versions for specific service
4. insert specific service cases in @queue, therefore, all cases including
   different service version cases can be run in parallel
5. print failed cases with environment variables
6. since cfoundry cannot support new service version API, therefore, one cfoundry
   monkey patch is committed. And this patch only is required when environment
   variable VCAP_BVT_SERVICES_MANIFEST is set.
7. add VCAP_BVT_SERVICES_VERSIONS=true to README.md

Change-Id: Ic82a82a22d5c493d654dab48fcd9d3c4d777142a
Signed-off-by: Pin Xie <pxie@vmware.com>
Reviewed-on: http://reviews.cloudfoundry.org/7831
Tested-by: CI Master <cf-ci@rbcon.com>
Reviewed-by: Nicholas Kushmerick <nicholask@vmware.com>
Reviewed-by: Michael Zhang <zhangcheng@rbcon.com>
  • Loading branch information...
1 parent 8d09a09 commit af5be1b6958f75e7b8a787ab60d8faab92529665 @pxie pxie committed Jul 25, 2012
View
@@ -55,6 +55,7 @@ Notes:
|SERVICE_BROKER_TOKEN |service broker token |SERVICE_BROKER_TOKEN=<token> |
|SERVICE_BROKER_URL |service broker url |SERVICE_BROKER_URL=http://... |
|VCAP_BVT_LONGEVITY |run testing N times |VCAP_BVT_LONGEVITY=100 |
+|VCAP_BVT_SERVICES_VERSIONS |enable service version cases |VCAP_BVT_SERVICES_VERSIONS=true |
```
2. In order to support parallel running, and administrative test cases, Yeti will ask administrative
View
@@ -46,3 +46,6 @@ module Harness
require "harness/http_response_code"
require "harness/scripts_helper"
require "harness/parallel_helper"
+require "harness/service_versions_helper"
+
+require "harness/cfoundry_monkey_patch" if ENV['VCAP_BVT_SERVICES_VERSIONS']
@@ -0,0 +1,32 @@
+
+module CFoundry::V1
+ class Base < CFoundry::BaseClient
+
+ def system_services
+ get("services", "v1", "offerings", :json => :json)
+ end
+ end
+
+ class Client
+
+ def services
+ services = []
+
+ @base.system_services.each do |type, vendors|
+ vendors.each do |vendor, provider|
+ provider[:core].each do |num, meta|
+ services << Service.new(vendor.to_s, num, meta[:description], type)
+ meta[:supported_versions].delete(num)
+ meta[:supported_versions].each do |v|
+ services <<
+ Service.new(vendor.to_s, v, meta[:description], type)
+ end
+ end
+ end
+ end
+
+ services
+ end
+ end
+
+end
@@ -1,4 +1,5 @@
require 'progressbar'
+require 'harness'
module BVT::Harness
module ParallelHelper
@@ -61,7 +62,21 @@ def run_tests(thread_number, options = {"tags" => "~admin"})
break if i == thread_number
}
puts yellow("threads number: #{thread_number}\n")
+ @queue = Queue.new
parse_case_list(options)
+
+ # if user set VCAP_BVT_SERVICES_VERSIONS=
+ # insert service versions cases in @queue
+ if ENV['VCAP_BVT_SERVICES_VERSIONS']
+ services = BVT::Spec::ServiceVersions.get_tested_services()
+ services.each do |m|
+ m[:versions].each do |v|
+ parse_case_list({"tags" => m[:vendor]},
+ {:vendor => m[:vendor], :version => v})
+ end
+ end
+ end
+
pbar = ProgressBar.new("0/#{@queue.size}", @queue.size, $stdout)
pbar.format_arguments = [:title, :percentage, :bar, :stat]
case_number = 0
@@ -82,7 +97,7 @@ def run_tests(thread_number, options = {"tags" => "~admin"})
if task_output =~ /Failures/
failure_number += 1
- failure_list << [task, parse_failure_log(task_output)]
+ failure_list << [task[:line], parse_failure_log(task_output), task[:envs]]
@lock.synchronize do
$stdout.print "\e[K"
if failure_number == 1
@@ -129,7 +144,6 @@ def run_tests(thread_number, options = {"tags" => "~admin"})
$stdout.print yellow(", #{pending_number} pending") if pending_number > 0
end
$stdout.print "\n"
-
unless failure_list.empty?
$stdout.print "\nFailed examples:\n\n"
failure_list.each_with_index do |log, i|
@@ -138,7 +152,14 @@ def run_tests(thread_number, options = {"tags" => "~admin"})
case_desc = line
break
}
- rerun_cmd = 'rspec .' + log[0].match(/\/spec\/.*_spec\.rb:\d{1,4}/).to_s
+ env_vars = ""
+ if log[2]
+ BVT::Spec::ServiceVersions.set_environment_variables(log[2]).each do |k, v|
+ env_vars += " #{k}=\'#{v}\'"
+ end
+ end
+
+ rerun_cmd = "#{env_vars} " + 'rspec .' + log[0].match(/\/spec\/.*_spec\.rb:\d{1,4}/).to_s
$stdout.print red(rerun_cmd)
$stdout.print cyan(" # #{case_desc}")
end
@@ -194,8 +215,7 @@ def get_case_list
case_list
end
- def parse_case_list(options)
- @queue = Queue.new
+ def parse_case_list(options, envs = nil)
all_case_list = get_case_list
pattern_filter_list = []
tags_filter_list = []
@@ -247,8 +267,9 @@ def parse_case_list(options)
swap(tags_filter_list, i * mod, rails_console_list[i])
end
end
+
tags_filter_list.each { |t|
- @queue << t["line"]
+ @queue << {:line => t["line"], :envs => envs}
}
end
@@ -265,9 +286,11 @@ def run_task(task, user, password)
"YETI_PARALLEL_USER" => user,
"YETI_PARALLEL_USER_PASSWD" => password
}
+ env_extras = env_extras.merge(
+ BVT::Spec::ServiceVersions.set_environment_variables(task[:envs])) if task[:envs]
cmd << ENV.to_hash.merge(env_extras)
- cmd += ["bundle", "exec", "rspec", "--color", task]
+ cmd += ["bundle", "exec", "rspec", "--color", task[:line]]
cmd
output = ""
@@ -6,11 +6,13 @@ module ScriptsHelper
# Service
def create_service(service_manifest, name=nil)
- service_name = name || service_manifest['vendor']
+ service_name = name || service_manifest[:vendor]
require_namespace = name.nil?
service = @session.service(service_name, require_namespace)
unless service.has_vendor?(service_manifest)
- pending("Service: (#{service_manifest['vendor']} #{service_manifest['version']}) " +
+ @session.log.debug("Service: (#{service_manifest[:vendor]} #{service_manifest[:version]}) " +
+ "is not available on target: #{@session.TARGET}")
+ pending("Service: (#{service_manifest[:vendor]} #{service_manifest[:version]}) " +
"is not available on target: #{@session.TARGET}")
end
service.create(service_manifest)
@@ -30,6 +32,8 @@ def create_app(app_name)
if VCAP_BVT_SYSTEM_FRAMEWORKS.has_key?(app.manifest['framework'].to_sym) &&
VCAP_BVT_SYSTEM_RUNTIMES.has_key?(app.manifest['runtime'])
else
+ @session.log.debug("Runtime/Framework: #{app.manifest['runtime']}/#{app.manifest['framework']} " +
+ "is not available on target: #{@session.TARGET}")
pending("Runtime/Framework: #{app.manifest['runtime']}/#{app.manifest['framework']} " +
"is not available on target: #{@session.TARGET}")
end
View
@@ -15,13 +15,13 @@ def inspect
"#<BVT::Harness::Service '#@name'>"
end
# service manifest example
- #{"vendor"=>"mysql", "version"=>"5.1"}
+ #{:vendor=>"mysql", :version=>"5.1"}
def create(service_manifest)
unless has_vendor?(service_manifest)
- @log.error("Service: #{service_manifest['vendor']} #{service_manifest['version']} " +
+ @log.error("Service: #{service_manifest[:vendor]} #{service_manifest[:version]} " +
"is not available on target: #{@session.TARGET}")
- raise RuntimeError, "Service: #{service_manifest['vendor']}" +
- " #{service_manifest['version']} is not available on target: #{@session.TARGET}"
+ raise RuntimeError, "Service: #{service_manifest[:vendor]}" +
+ " #{service_manifest[:version]} is not available on target: #{@session.TARGET}"
end
@log.info("Create Service (#{@service.vendor} #{@service.version}): #{@service.name}")
@@ -54,10 +54,10 @@ def has_vendor?(service_manifest)
match = false
VCAP_BVT_SYSTEM_SERVICES.each do |vendor, meta|
- next unless vendor =~ /#{service_manifest['vendor']}/
+ next unless vendor =~ /#{service_manifest[:vendor]}/
version = meta[:versions].find { |v|
- v =~ /#{service_manifest['version']}/
+ v =~ /#{service_manifest[:version]}/
}
next unless version
@@ -0,0 +1,73 @@
+
+module BVT::Spec
+
+ TESTED_SERVICES = [{:vendor => 'mongodb', :versions => %w(1.8 2.0)}]
+
+
+ MYSQL_MANIFEST = ENV['VCAP_BVT_MYSQL_MANIFEST'] ? eval(ENV['VCAP_BVT_MYSQL_MANIFEST']) :
+ {:vendor=>"mysql", :version=>"5.1"}
+ REDIS_MANIFEST = ENV['VCAP_BVT_REDIS_MANIFEST'] ? eval(ENV['VCAP_BVT_REDIS_MANIFEST']) :
+ {:vendor => "redis", :version=>"2.2"}
+ MONGODB_MANIFEST = ENV['VCAP_BVT_MONGODB_MANIFEST'] ? eval(ENV['VCAP_BVT_MONGODB_MANIFEST']) :
+ {:vendor => "mongodb", :version=>"1.8"}
+ RABBITMQ_MANIFEST = ENV['VCAP_BVT_RABBITMQ_MANIFEST'] ? eval(ENV['VCAP_BVT_RABBITMQ_MANIFEST']) :
+ {:vendor => "rabbitmq", :version=>"2.4"}
+ POSTGRESQL_MANIFEST = ENV['VCAP_BVT_POSTGRESQL_MANIFEST'] ? eval(ENV['VCAP_BVT_POSTGRESQL_MANIFEST']) :
+ {:vendor => "postgresql", :version=>"9.0"}
+ NEO4J_MANIFEST = ENV['VCAP_BVT_NEO4J_MANIFEST'] ? eval(ENV['VCAP_BVT_NEO4J_MANIFEST']) :
+ {:vendor => "neo4j", :version=>"1.4"}
+ VBLOB_MANIFEST = ENV['VCAP_BVT_VBLOB_MANIFEST'] ? eval(ENV['VCAP_BVT_VBLOB_MANIFEST']) :
+ {:vendor => "vblob", :version=>"1.0"}
+ MEMCACHED_MANIFEST = ENV['VCAP_BVT_MEMCACHED_MANIFEST'] ? eval(ENV['VCAP_BVT_MEMCACHED_MANIFEST']) :
+ {:vendor => "memcached",:version=>"1.4"}
+ COUCHDB_MANIFEST = ENV['VCAP_BVT_COUCHDB_MANIFEST'] ? eval(ENV['VCAP_BVT_COUCHDB_MANIFEST']) :
+ {:vendor => "couchdb",:version=>"1.2"}
+ ELASTICSSEARCH_MANIFEST = ENV['VCAP_BVT_ELASTICSSEARCH_MANIFEST'] ? eval(ENV['VCAP_BVT_ELASTICSSEARCH_MANIFEST']) :
+ {:vendor => "elasticsearch", :version=>"0.19"}
+
+ SERVICES_MANIFEST = [MYSQL_MANIFEST, REDIS_MANIFEST, MONGODB_MANIFEST, RABBITMQ_MANIFEST,
+ POSTGRESQL_MANIFEST, NEO4J_MANIFEST, VBLOB_MANIFEST, MEMCACHED_MANIFEST,
+ COUCHDB_MANIFEST, ELASTICSSEARCH_MANIFEST]
+
+ module ServiceVersions
+ module_function
+
+ def get_tested_services()
+ SERVICES_MANIFEST.each do |manifest|
+ TESTED_SERVICES.each do |m|
+ next unless m[:vendor] == manifest[:vendor]
+ m[:versions].delete(manifest[:version])
+ end
+ end
+ TESTED_SERVICES.delete_if {|x| x[:versions].empty?}
+ end
+
+ def set_environment_variables(envs)
+ res = {}
+ case envs[:vendor]
+ when 'mysql'
+ res["VCAP_BVT_MYSQL_MANIFEST"] = envs.to_s
+ when 'postgresql'
+ res["VCAP_BVT_POSTGRESQL_MANIFEST"] = envs.to_s
+ when 'mongodb'
+ res["VCAP_BVT_MONGODB_MANIFEST"] = envs.to_s
+ when 'redis'
+ res["VCAP_BVT_REDIS_MANIFEST"] = envs.to_s
+ when 'rabbitmq'
+ res["VCAP_BVT_RABBITMQ_MANIFEST"] = envs.to_s
+ when 'vblob'
+ res["VCAP_BVT_VBLOB_MANIFEST"] = envs.to_s
+ when 'memcached'
+ res["VCAP_BVT_MEMCACHED_MANIFEST"] = envs.to_s
+ when 'couchdb'
+ res["VCAP_BVT_COUCHDB_MANIFEST"] = envs.to_s
+ when 'elasticsearch'
+ res["VCAP_BVT_ELASTICSEARCH_MANIFEST"] = envs.to_s
+ else
+ raise RuntimeError, "service vendor: #{envs[:vendor]}," +
+ " is not supported to set proper Environment variables. "
+ end
+ res
+ end
+ end
+end
@@ -94,26 +94,17 @@ def reset_counter(app)
it "deploy env_test app" do
app = create_push_app("env_test_app")
-
should_be_there = []
- ["aurora", "redis"].each do |v|
- if BVT::Harness::VCAP_BVT_SYSTEM_SERVICES.has_key?(v)
- # create named service
- myname = "#{v}my-#{v}"
- if v == 'aurora'
- manifest = {"vendor"=>"aurora"}
- end
- if v == 'redis'
- manifest = REDIS_MANIFEST
- end
- # then record for testing against the environment variables
- manifest['name'] = myname
- service = create_service(manifest, myname)
- app.bind(service.name)
- should_be_there << manifest
- end
- end
+ v = "redis"
+ myname = "#{v}my-#{v}"
+ manifest = REDIS_MANIFEST
+
+ # then record for testing against the environment variables
+ manifest[:name] = myname
+ service = create_service(manifest, myname)
+ app.bind(service.name)
+ should_be_there << manifest
services = app.services
services.should_not == nil
@@ -129,7 +120,7 @@ def reset_counter(app)
found = 0
service_list['services'].each do |s|
should_be_there.each do |v|
- if v[:name] == s[:name] && v[:vendor] == s[:vendor]
+ if v[:name] == s['name'] && v[:vendor] == s['vendor']
found += 1
break
end
@@ -5,15 +5,15 @@ module AutoStagingHelper
def verify_service_autostaging(service_manifest, app)
key = "abc"
- data = "#{service_manifest['vendor']}#{key}"
- url = SERVICE_URL_MAPPING[service_manifest['vendor']]
+ data = "#{service_manifest[:vendor]}#{key}"
+ url = SERVICE_URL_MAPPING[service_manifest[:vendor]]
app.get_response(:post, "/service/#{url}/#{key}", data)
app.get_response(:get, "/service/#{url}/#{key}").body_str.should == data
end
def verify_unsupported_client_version(service_manifest, app, data)
key = "connection"
- url = SERVICE_URL_MAPPING_UNSUPPORTED_VERSION[service_manifest['vendor']]
+ url = SERVICE_URL_MAPPING_UNSUPPORTED_VERSION[service_manifest[:vendor]]
app.get_response(:get, "/service/#{url}/#{key}").body_str.should == data
end
@@ -136,8 +136,7 @@ def record_present record, list
response.body_str.should == 'org.hibernate.dialect.MySQLDialect'
end
- it "Start Spring Web Application with no service dependencies",
- :mysql => true do
+ it "Start Spring Web Application with no service dependencies" do
app = create_push_app("auto-reconfig-missing-deps-test-app")
response = app.get_response(:get)
@@ -38,7 +38,7 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
data = "Connectionrefused-UnabletoconnecttoRedison127.0.0.1:6379"
- url = "/service/#{service_manifest['vendor']}/connection"
+ url = "/service/#{service_manifest[:vendor]}/connection"
app.get_response(:get, url).body_str.should == data
end
@@ -50,7 +50,7 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
data = "Connectionrefused-UnabletoconnecttoRedison127.0.0.1:6379"
- url = "/service/#{service_manifest['vendor']}/connection"
+ url = "/service/#{service_manifest[:vendor]}/connection"
app.get_response(:get, url).body_str.should == data
end
end
@@ -34,7 +34,7 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
data = "Connectionrefused-UnabletoconnecttoRedison127.0.0.1:6379"
- res = app.get_response(:get, "/service/#{service_manifest['vendor']}/connection")
+ res = app.get_response(:get, "/service/#{service_manifest[:vendor]}/connection")
res.body_str.should == data
end
@@ -46,7 +46,7 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
data = "Connectionrefused-UnabletoconnecttoRedison127.0.0.1:6379"
- res = app.get_response(:get, "/service/#{service_manifest['vendor']}/connection")
+ res = app.get_response(:get, "/service/#{service_manifest[:vendor]}/connection")
res.body_str.should == data
end
@@ -21,8 +21,8 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
key = "abc"
- data = "#{service_manifest['vendor']}#{key}"
- url = SERVICE_URL_MAPPING[service_manifest['vendor']]
+ data = "#{service_manifest[:vendor]}#{key}"
+ url = SERVICE_URL_MAPPING[service_manifest[:vendor]]
app.get_response(:post, "/service/#{url}/#{key}", data)
app.get_response(:get, "/service/#{url}/#{key}").body_str.should == data
end
Oops, something went wrong.

0 comments on commit af5be1b

Please sign in to comment.