Skip to content
This repository has been archived by the owner on Jan 26, 2022. It is now read-only.

Commit

Permalink
service versions
Browse files Browse the repository at this point in the history 
1. change service manifest to {:vendor => '', :version => ''}
2. enable ENV vars for each service manifest
3. add service_versions_helper to manage multiple versions for specific service
4. insert specific service cases in @Queue, therefore, all cases including
   different service version cases can be run in parallel
5. print failed cases with environment variables
6. since cfoundry cannot support new service version API, therefore, one cfoundry
   monkey patch is committed. And this patch only is required when environment
   variable VCAP_BVT_SERVICES_MANIFEST is set.
7. add VCAP_BVT_SERVICES_VERSIONS=true to README.md

Change-Id: Ic82a82a22d5c493d654dab48fcd9d3c4d777142a
Signed-off-by: Pin Xie <pxie@vmware.com>
Reviewed-on: http://reviews.cloudfoundry.org/7831
Tested-by: CI Master <cf-ci@rbcon.com>
Reviewed-by: Nicholas Kushmerick <nicholask@vmware.com>
Reviewed-by: Michael Zhang <zhangcheng@rbcon.com>
  • Loading branch information
@pxie
pxie committed Aug 2, 2012
1 parent 8d09a09 commit af5be1b
Show file tree
Hide file tree
Showing 19 changed files with 188 additions and 77 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Expand Up @@ -55,6 +55,7 @@ Notes:
|SERVICE_BROKER_TOKEN |service broker token |SERVICE_BROKER_TOKEN=<token> |
|SERVICE_BROKER_URL |service broker url |SERVICE_BROKER_URL=http://... |
|VCAP_BVT_LONGEVITY |run testing N times |VCAP_BVT_LONGEVITY=100 |
|VCAP_BVT_SERVICES_VERSIONS |enable service version cases |VCAP_BVT_SERVICES_VERSIONS=true |
```

2. In order to support parallel running, and administrative test cases, Yeti will ask administrative
Expand Down
3 changes: 3 additions & 0 deletions lib/harness.rb
View file
Expand Up @@ -46,3 +46,6 @@ module Harness
require "harness/http_response_code"
require "harness/scripts_helper"
require "harness/parallel_helper"
require "harness/service_versions_helper"

require "harness/cfoundry_monkey_patch" if ENV['VCAP_BVT_SERVICES_VERSIONS']
32 changes: 32 additions & 0 deletions lib/harness/cfoundry_monkey_patch.rb
View file
@@ -0,0 +1,32 @@

module CFoundry::V1
class Base < CFoundry::BaseClient

def system_services
get("services", "v1", "offerings", :json => :json)
end
end

class Client

def services
services = []

@base.system_services.each do |type, vendors|
vendors.each do |vendor, provider|
provider[:core].each do |num, meta|
services << Service.new(vendor.to_s, num, meta[:description], type)
meta[:supported_versions].delete(num)
meta[:supported_versions].each do |v|
services <<
Service.new(vendor.to_s, v, meta[:description], type)
end
end
end
end

services
end
end

end
37 changes: 30 additions & 7 deletions lib/harness/parallel_helper.rb
View file
@@ -1,4 +1,5 @@
require 'progressbar'
require 'harness'

module BVT::Harness
module ParallelHelper
Expand Down Expand Up @@ -61,7 +62,21 @@ def run_tests(thread_number, options = {"tags" => "~admin"})
break if i == thread_number
}
puts yellow("threads number: #{thread_number}\n")
@queue = Queue.new
parse_case_list(options)

# if user set VCAP_BVT_SERVICES_VERSIONS=
# insert service versions cases in @queue
if ENV['VCAP_BVT_SERVICES_VERSIONS']
services = BVT::Spec::ServiceVersions.get_tested_services()
services.each do |m|
m[:versions].each do |v|
parse_case_list({"tags" => m[:vendor]},
{:vendor => m[:vendor], :version => v})
end
end
end

pbar = ProgressBar.new("0/#{@queue.size}", @queue.size, $stdout)
pbar.format_arguments = [:title, :percentage, :bar, :stat]
case_number = 0
Expand All @@ -82,7 +97,7 @@ def run_tests(thread_number, options = {"tags" => "~admin"})

if task_output =~ /Failures/
failure_number += 1
failure_list << [task, parse_failure_log(task_output)]
failure_list << [task[:line], parse_failure_log(task_output), task[:envs]]
@lock.synchronize do
$stdout.print "\e[K"
if failure_number == 1
Expand Down Expand Up @@ -129,7 +144,6 @@ def run_tests(thread_number, options = {"tags" => "~admin"})
$stdout.print yellow(", #{pending_number} pending") if pending_number > 0
end
$stdout.print "\n"

unless failure_list.empty?
$stdout.print "\nFailed examples:\n\n"
failure_list.each_with_index do |log, i|
Expand All @@ -138,7 +152,14 @@ def run_tests(thread_number, options = {"tags" => "~admin"})
case_desc = line
break
}
rerun_cmd = 'rspec .' + log[0].match(/\/spec\/.*_spec\.rb:\d{1,4}/).to_s
env_vars = ""
if log[2]
BVT::Spec::ServiceVersions.set_environment_variables(log[2]).each do |k, v|
env_vars += " #{k}=\'#{v}\'"
end
end

rerun_cmd = "#{env_vars} " + 'rspec .' + log[0].match(/\/spec\/.*_spec\.rb:\d{1,4}/).to_s
$stdout.print red(rerun_cmd)
$stdout.print cyan(" # #{case_desc}")
end
Expand Down Expand Up @@ -194,8 +215,7 @@ def get_case_list
case_list
end

def parse_case_list(options)
@queue = Queue.new
def parse_case_list(options, envs = nil)
all_case_list = get_case_list
pattern_filter_list = []
tags_filter_list = []
Expand Down Expand Up @@ -247,8 +267,9 @@ def parse_case_list(options)
swap(tags_filter_list, i * mod, rails_console_list[i])
end
end

tags_filter_list.each { |t|
@queue << t["line"]
@queue << {:line => t["line"], :envs => envs}
}
end

Expand All @@ -265,9 +286,11 @@ def run_task(task, user, password)
"YETI_PARALLEL_USER" => user,
"YETI_PARALLEL_USER_PASSWD" => password
}
env_extras = env_extras.merge(
BVT::Spec::ServiceVersions.set_environment_variables(task[:envs])) if task[:envs]

cmd << ENV.to_hash.merge(env_extras)
cmd += ["bundle", "exec", "rspec", "--color", task]
cmd += ["bundle", "exec", "rspec", "--color", task[:line]]
cmd

output = ""
Expand Down
8 changes: 6 additions & 2 deletions lib/harness/scripts_helper.rb
View file
Expand Up @@ -6,11 +6,13 @@ module ScriptsHelper

# Service
def create_service(service_manifest, name=nil)
service_name = name || service_manifest['vendor']
service_name = name || service_manifest[:vendor]
require_namespace = name.nil?
service = @session.service(service_name, require_namespace)
unless service.has_vendor?(service_manifest)
pending("Service: (#{service_manifest['vendor']} #{service_manifest['version']}) " +
@session.log.debug("Service: (#{service_manifest[:vendor]} #{service_manifest[:version]}) " +
"is not available on target: #{@session.TARGET}")
pending("Service: (#{service_manifest[:vendor]} #{service_manifest[:version]}) " +
"is not available on target: #{@session.TARGET}")
end
service.create(service_manifest)
Expand All @@ -30,6 +32,8 @@ def create_app(app_name)
if VCAP_BVT_SYSTEM_FRAMEWORKS.has_key?(app.manifest['framework'].to_sym) &&
VCAP_BVT_SYSTEM_RUNTIMES.has_key?(app.manifest['runtime'])
else
@session.log.debug("Runtime/Framework: #{app.manifest['runtime']}/#{app.manifest['framework']} " +
"is not available on target: #{@session.TARGET}")
pending("Runtime/Framework: #{app.manifest['runtime']}/#{app.manifest['framework']} " +
"is not available on target: #{@session.TARGET}")
end
Expand Down
12 changes: 6 additions & 6 deletions lib/harness/service.rb
View file
Expand Up @@ -15,13 +15,13 @@ def inspect
"#<BVT::Harness::Service '#@name'>"
end
# service manifest example
#{"vendor"=>"mysql", "version"=>"5.1"}
#{:vendor=>"mysql", :version=>"5.1"}
def create(service_manifest)
unless has_vendor?(service_manifest)
@log.error("Service: #{service_manifest['vendor']} #{service_manifest['version']} " +
@log.error("Service: #{service_manifest[:vendor]} #{service_manifest[:version]} " +
"is not available on target: #{@session.TARGET}")
raise RuntimeError, "Service: #{service_manifest['vendor']}" +
" #{service_manifest['version']} is not available on target: #{@session.TARGET}"
raise RuntimeError, "Service: #{service_manifest[:vendor]}" +
" #{service_manifest[:version]} is not available on target: #{@session.TARGET}"
end

@log.info("Create Service (#{@service.vendor} #{@service.version}): #{@service.name}")
Expand Down Expand Up @@ -54,10 +54,10 @@ def has_vendor?(service_manifest)
match = false

VCAP_BVT_SYSTEM_SERVICES.each do |vendor, meta|
next unless vendor =~ /#{service_manifest['vendor']}/
next unless vendor =~ /#{service_manifest[:vendor]}/

version = meta[:versions].find { |v|
v =~ /#{service_manifest['version']}/
v =~ /#{service_manifest[:version]}/
}
next unless version

Expand Down
73 changes: 73 additions & 0 deletions lib/harness/service_versions_helper.rb
View file
@@ -0,0 +1,73 @@

module BVT::Spec

TESTED_SERVICES = [{:vendor => 'mongodb', :versions => %w(1.8 2.0)}]


MYSQL_MANIFEST = ENV['VCAP_BVT_MYSQL_MANIFEST'] ? eval(ENV['VCAP_BVT_MYSQL_MANIFEST']) :
{:vendor=>"mysql", :version=>"5.1"}
REDIS_MANIFEST = ENV['VCAP_BVT_REDIS_MANIFEST'] ? eval(ENV['VCAP_BVT_REDIS_MANIFEST']) :
{:vendor => "redis", :version=>"2.2"}
MONGODB_MANIFEST = ENV['VCAP_BVT_MONGODB_MANIFEST'] ? eval(ENV['VCAP_BVT_MONGODB_MANIFEST']) :
{:vendor => "mongodb", :version=>"1.8"}
RABBITMQ_MANIFEST = ENV['VCAP_BVT_RABBITMQ_MANIFEST'] ? eval(ENV['VCAP_BVT_RABBITMQ_MANIFEST']) :
{:vendor => "rabbitmq", :version=>"2.4"}
POSTGRESQL_MANIFEST = ENV['VCAP_BVT_POSTGRESQL_MANIFEST'] ? eval(ENV['VCAP_BVT_POSTGRESQL_MANIFEST']) :
{:vendor => "postgresql", :version=>"9.0"}
NEO4J_MANIFEST = ENV['VCAP_BVT_NEO4J_MANIFEST'] ? eval(ENV['VCAP_BVT_NEO4J_MANIFEST']) :
{:vendor => "neo4j", :version=>"1.4"}
VBLOB_MANIFEST = ENV['VCAP_BVT_VBLOB_MANIFEST'] ? eval(ENV['VCAP_BVT_VBLOB_MANIFEST']) :
{:vendor => "vblob", :version=>"1.0"}
MEMCACHED_MANIFEST = ENV['VCAP_BVT_MEMCACHED_MANIFEST'] ? eval(ENV['VCAP_BVT_MEMCACHED_MANIFEST']) :
{:vendor => "memcached",:version=>"1.4"}
COUCHDB_MANIFEST = ENV['VCAP_BVT_COUCHDB_MANIFEST'] ? eval(ENV['VCAP_BVT_COUCHDB_MANIFEST']) :
{:vendor => "couchdb",:version=>"1.2"}
ELASTICSSEARCH_MANIFEST = ENV['VCAP_BVT_ELASTICSSEARCH_MANIFEST'] ? eval(ENV['VCAP_BVT_ELASTICSSEARCH_MANIFEST']) :
{:vendor => "elasticsearch", :version=>"0.19"}

SERVICES_MANIFEST = [MYSQL_MANIFEST, REDIS_MANIFEST, MONGODB_MANIFEST, RABBITMQ_MANIFEST,
POSTGRESQL_MANIFEST, NEO4J_MANIFEST, VBLOB_MANIFEST, MEMCACHED_MANIFEST,
COUCHDB_MANIFEST, ELASTICSSEARCH_MANIFEST]

module ServiceVersions
module_function

def get_tested_services()
SERVICES_MANIFEST.each do |manifest|
TESTED_SERVICES.each do |m|
next unless m[:vendor] == manifest[:vendor]
m[:versions].delete(manifest[:version])
end
end
TESTED_SERVICES.delete_if {|x| x[:versions].empty?}
end

def set_environment_variables(envs)
res = {}
case envs[:vendor]
when 'mysql'
res["VCAP_BVT_MYSQL_MANIFEST"] = envs.to_s
when 'postgresql'
res["VCAP_BVT_POSTGRESQL_MANIFEST"] = envs.to_s
when 'mongodb'
res["VCAP_BVT_MONGODB_MANIFEST"] = envs.to_s
when 'redis'
res["VCAP_BVT_REDIS_MANIFEST"] = envs.to_s
when 'rabbitmq'
res["VCAP_BVT_RABBITMQ_MANIFEST"] = envs.to_s
when 'vblob'
res["VCAP_BVT_VBLOB_MANIFEST"] = envs.to_s
when 'memcached'
res["VCAP_BVT_MEMCACHED_MANIFEST"] = envs.to_s
when 'couchdb'
res["VCAP_BVT_COUCHDB_MANIFEST"] = envs.to_s
when 'elasticsearch'
res["VCAP_BVT_ELASTICSEARCH_MANIFEST"] = envs.to_s
else
raise RuntimeError, "service vendor: #{envs[:vendor]}," +
" is not supported to set proper Environment variables. "
end
res
end
end
end
29 changes: 10 additions & 19 deletions spec/app_performance/ruby19_sinatra_spec.rb
View file
Expand Up @@ -94,26 +94,17 @@ def reset_counter(app)

it "deploy env_test app" do
app = create_push_app("env_test_app")

should_be_there = []
["aurora", "redis"].each do |v|
if BVT::Harness::VCAP_BVT_SYSTEM_SERVICES.has_key?(v)
# create named service
myname = "#{v}my-#{v}"
if v == 'aurora'
manifest = {"vendor"=>"aurora"}
end
if v == 'redis'
manifest = REDIS_MANIFEST
end

# then record for testing against the environment variables
manifest['name'] = myname
service = create_service(manifest, myname)
app.bind(service.name)
should_be_there << manifest
end
end
v = "redis"
myname = "#{v}my-#{v}"
manifest = REDIS_MANIFEST

# then record for testing against the environment variables
manifest[:name] = myname
service = create_service(manifest, myname)
app.bind(service.name)
should_be_there << manifest

services = app.services
services.should_not == nil
Expand All @@ -129,7 +120,7 @@ def reset_counter(app)
found = 0
service_list['services'].each do |s|
should_be_there.each do |v|
if v[:name] == s[:name] && v[:vendor] == s[:vendor]
if v[:name] == s['name'] && v[:vendor] == s['vendor']
found += 1
break
end
Expand Down
6 changes: 3 additions & 3 deletions spec/autostaging/autostaging_helper.rb
View file
Expand Up @@ -5,15 +5,15 @@ module AutoStagingHelper

def verify_service_autostaging(service_manifest, app)
key = "abc"
data = "#{service_manifest['vendor']}#{key}"
url = SERVICE_URL_MAPPING[service_manifest['vendor']]
data = "#{service_manifest[:vendor]}#{key}"
url = SERVICE_URL_MAPPING[service_manifest[:vendor]]
app.get_response(:post, "/service/#{url}/#{key}", data)
app.get_response(:get, "/service/#{url}/#{key}").body_str.should == data
end

def verify_unsupported_client_version(service_manifest, app, data)
key = "connection"
url = SERVICE_URL_MAPPING_UNSUPPORTED_VERSION[service_manifest['vendor']]
url = SERVICE_URL_MAPPING_UNSUPPORTED_VERSION[service_manifest[:vendor]]
app.get_response(:get, "/service/#{url}/#{key}").body_str.should == data
end

Expand Down
3 changes: 1 addition & 2 deletions spec/autostaging/java_spring_spec.rb
View file
Expand Up @@ -136,8 +136,7 @@ def record_present record, list
response.body_str.should == 'org.hibernate.dialect.MySQLDialect'
end

it "Start Spring Web Application with no service dependencies",
:mysql => true do
it "Start Spring Web Application with no service dependencies" do
app = create_push_app("auto-reconfig-missing-deps-test-app")

response = app.get_response(:get)
Expand Down
4 changes: 2 additions & 2 deletions spec/autostaging/ruby18_rack_spec.rb
View file
Expand Up @@ -38,7 +38,7 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
data = "Connectionrefused-UnabletoconnecttoRedison127.0.0.1:6379"
url = "/service/#{service_manifest['vendor']}/connection"
url = "/service/#{service_manifest[:vendor]}/connection"
app.get_response(:get, url).body_str.should == data
end

Expand All @@ -50,7 +50,7 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
data = "Connectionrefused-UnabletoconnecttoRedison127.0.0.1:6379"
url = "/service/#{service_manifest['vendor']}/connection"
url = "/service/#{service_manifest[:vendor]}/connection"
app.get_response(:get, url).body_str.should == data
end
end
4 changes: 2 additions & 2 deletions spec/autostaging/ruby18_standalone_spec.rb
View file
Expand Up @@ -34,7 +34,7 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
data = "Connectionrefused-UnabletoconnecttoRedison127.0.0.1:6379"
res = app.get_response(:get, "/service/#{service_manifest['vendor']}/connection")
res = app.get_response(:get, "/service/#{service_manifest[:vendor]}/connection")
res.body_str.should == data
end

Expand All @@ -46,7 +46,7 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
data = "Connectionrefused-UnabletoconnecttoRedison127.0.0.1:6379"
res = app.get_response(:get, "/service/#{service_manifest['vendor']}/connection")
res = app.get_response(:get, "/service/#{service_manifest[:vendor]}/connection")
res.body_str.should == data
end

Expand Down
4 changes: 2 additions & 2 deletions spec/autostaging/ruby19_rack_spec.rb
View file
Expand Up @@ -21,8 +21,8 @@
service_manifest = REDIS_MANIFEST
bind_service(service_manifest, app)
key = "abc"
data = "#{service_manifest['vendor']}#{key}"
url = SERVICE_URL_MAPPING[service_manifest['vendor']]
data = "#{service_manifest[:vendor]}#{key}"
url = SERVICE_URL_MAPPING[service_manifest[:vendor]]
app.get_response(:post, "/service/#{url}/#{key}", data)
app.get_response(:get, "/service/#{url}/#{key}").body_str.should == data
end
Expand Down

0 comments on commit af5be1b

Please sign in to comment.