Skip to content
Browse files

new parallel implimentation

1. case level parallel, no dependency gem
2. default to run in parallel for rake tests
3. support tags/patterns filter, can be used in rake java/ruby/services...

Change-Id: I2341b9e565ce8958668924eb59c8d08848270e96
Signed-off-by: michael zhang <zhangcheng@rbcon.com>
Reviewed-on: http://reviews.cloudfoundry.org/6373
Tested-by: CI Master <cf-ci@rbcon.com>
Reviewed-by: Pin Xie <pxie@vmware.com>
  • Loading branch information...
1 parent 49a1e6e commit 10c769f853afebf0efb24b29257a97c026c97f8c michael zhang committed
Showing with 279 additions and 22 deletions.
  1. +1 −0 .gitignore
  2. +1 −0 Gemfile
  3. +2 −0 Gemfile.lock
  4. +15 −11 Rakefile
  5. +2 −1 lib/harness.rb
  6. +7 −4 lib/harness/cfsession.rb
  7. +241 −0 lib/harness/parallel_helper.rb
  8. +7 −3 lib/harness/rake_helper.rb
  9. +3 −3 spec/spec_helper.rb
View
1 .gitignore
@@ -3,3 +3,4 @@ tools/.build
tools/play-2.0.1
vendor/
!vendor/cache/vcap_logging-1.0.0.gem
+.bundle/
View
1 Gemfile
@@ -12,6 +12,7 @@ gem "yajl-ruby"
gem "nokogiri"
gem "parallel_tests"
gem "fuubar"
+gem "progressbar", "~> 0.11.0"
group :ci do
gem "ci_reporter"
View
2 Gemfile.lock
@@ -26,6 +26,7 @@ GEM
parallel (0.5.16)
parallel_tests (0.8.1)
parallel
+ progressbar (0.11.0)
rake (0.9.2.2)
rest-client (1.6.7)
mime-types (>= 1.16)
@@ -57,6 +58,7 @@ DEPENDENCIES
mongo
nokogiri
parallel_tests
+ progressbar (~> 0.11.0)
rake
rspec
vcap_logging (>= 1.0)
View
26 Rakefile
@@ -6,10 +6,9 @@ task :default => [:help]
desc "List help commands"
task :help do
puts "Usage: rake [command]"
- puts " tests\t\trun all bvts\n" +
- "\t\tOptions:\n" +
- "\t\t VCAP_BVT_PARALLEL=<NUMBER>, run bvts in parallel, "+
- "number range: 1-#{BVT::Harness::VCAP_BVT_PARALLEL_MAX_USERS}"
+ puts " tests\t\trun all bvts in parallel, add [N] to specify threads " +
+ "number(default:10)"
+ puts " serial\trun all bvts in serial"
puts " random\trun all bvts randomly, add [N] to specify a seed"
puts " admin\t\trun admin test cases"
puts " clean\t\tclean up test environment.\n" +
@@ -24,15 +23,20 @@ task :help do
end
desc "Run the Basic Verification Tests"
-task :tests do
+task :serial do
BVT::Harness::RakeHelper.generate_config_file
BVT::Harness::RakeHelper.check_environment
- if ENV['VCAP_BVT_PARALLEL']
- BVT::Harness::ParallelRunner.run_tests
- else
- BVT::Harness::RakeHelper.print_test_config
- sh("rspec --format Fuubar --color spec/ --tag ~admin")
- end
+ BVT::Harness::RakeHelper.print_test_config
+ sh("rspec --format Fuubar --color spec/ --tag ~admin")
+end
+
+desc "run tests in parallel"
+task :tests, :thread_number do |t, args|
+ BVT::Harness::RakeHelper.generate_config_file
+ BVT::Harness::RakeHelper.check_environment
+ threads = 10
+ threads = args[:thread_number].to_i if args[:thread_number]
+ BVT::Harness::ParallelHelper.run_tests(threads)
end
desc "Run all bvts randomly, add [N] to specify a seed"
View
3 lib/harness.rb
@@ -32,7 +32,7 @@ module Harness
VCAP_BVT_ASSETS_STORE_URL = "http://blobs.cloudfoundry.com"
## parallel
- VCAP_BVT_PARALLEL_MAX_USERS = 8
+ VCAP_BVT_PARALLEL_MAX_USERS = 16
VCAP_BVT_PARALLEL_SYNC_FILE = File.join(VCAP_BVT_HOME, "sync.yml")
end
end
@@ -46,3 +46,4 @@ module Harness
require "harness/http_response_code"
require "harness/scripts_helper"
require "harness/parallelrunner"
+require "harness/parallel_helper"
View
11 lib/harness/cfsession.rb
@@ -136,10 +136,13 @@ def get_namespace
def get_login_email(expected_admin = false)
@config = VCAP_BVT_CONFIG
- if ENV['VCAP_BVT_PARALLEL'] && !expected_admin
- user_info = @config['parallel'][VCAP_BVT_PARALLEL_INDEX]
- @config['user']['email'] = user_info['email']
- @config['user']['passwd'] = user_info['passwd']
+ if ENV['YETI_PARALLEL_USER']
+ @config['user']['email'] = ENV['YETI_PARALLEL_USER']
+ @config['user']['passwd'] = ENV['YETI_PARALLEL_USER_PASSWD']
+ #elsif ENV['VCAP_BVT_PARALLEL'] && !expected_admin
+ # user_info = @config['parallel'][VCAP_BVT_PARALLEL_INDEX]
+ # @config['user']['email'] = user_info['email']
+ # @config['user']['passwd'] = user_info['passwd']
end
expected_admin ? @config["admin"]["email"] : @config["user"]["email"]
end
View
241 lib/harness/parallel_helper.rb
@@ -0,0 +1,241 @@
+require 'progressbar'
+
+module BVT::Harness
+ module ParallelHelper
+ include Interactive, ColorHelpers
+
+ SPEC_PATH = File.join(File.dirname(__FILE__), "../../spec/")
+
+ def create_parallel_users
+ user_info = RakeHelper.get_config
+ unless user_info['parallel']
+ unless user_info['admin']
+ puts "please input admin account to create concurrent users"
+ BVT::Harness::RakeHelper::generate_config_file(true)
+ user_info = YAML.load_file(VCAP_BVT_CONFIG_FILE)
+ end
+
+ user_info['parallel'] = []
+ begin
+ session = BVT::Harness::CFSession.new(:admin => true,
+ :email => user_info['admin']['email'],
+ :passwd => user_info['admin']['passwd'],
+ :target => user_info['target'])
+ rescue Exception => e
+ raise RuntimeError, "#{e.to_s}\nPlease input valid admin credential " +
+ "for parallel running"
+ end
+
+ passwd = user_info['user']['passwd']
+ (1..VCAP_BVT_PARALLEL_MAX_USERS).to_a.each do |index|
+ email = "#{index}-#{user_info['user']['email']}"
+ user = session.user(email)
+ user.create(passwd)
+ config = {}
+ config['email'] = user.email
+ config['passwd'] = passwd
+ puts "create user: #{yellow(config['email'])}"
+ user_info['parallel'] << config
+ end
+ File.open(VCAP_BVT_CONFIG_FILE, "w") { |f| f.write YAML.dump(user_info) }
+ end
+ user_info['parallel']
+ end
+
+ def run_tests(thread_number, options = {"tags" => "~admin"})
+ if thread_number > VCAP_BVT_PARALLEL_MAX_USERS
+ puts "threads_number can't be greater than #{VCAP_BVT_PARALLEL_MAX_USERS}"
+ return
+ end
+ t1 = Time.now
+ all_users = create_parallel_users
+ parallel_users = []
+ i = 0
+ all_users.each {|user|
+ parallel_users << user
+ i += 1
+ break if i == thread_number
+ }
+ puts "threads number: #{thread_number}"
+ parse_case_list(options)
+ pbar = ProgressBar.new("Task Progress", @queue.size)
+ pbar.format_arguments = [:title, :percentage, :bar, :stat]
+ case_number = 0
+ failure_number = 0
+ pending_number = 0
+ failure_list = []
+ pending_list = []
+
+ Thread.abort_on_exception = false
+ threads = []
+
+ parallel_users.each do |user|
+ threads << Thread.new do
+ until @queue.empty?
+ task = @queue.pop
+
+ task_output = run_task(task, user['email'], user['passwd'])
+
+ if task_output =~ /Failures/
+ failure_number += 1
+ failure_list << [task, parse_failure_log(task_output)]
+ puts parse_failure_log(task_output)
+ elsif task_output =~ /Pending/
+ pending_number += 1
+ pending_list << [task, parse_pending_log(task_output)]
+ end
+ case_number += 1
+ pbar.inc
+ # add think time when finishing every task
+ sleep 0.1
+ end
+ end
+ # ramp up user threads one by one
+ sleep 0.1
+ end
+
+ threads.each { |t| t.join }
+ pbar.finish
+
+ $stdout.print "\n\n"
+ t2 = Time.now
+ $stdout.print "Finished in #{format_time(t2-t1)}\n"
+ $stdout.print "#{case_number} examples, #{failure_number} failures"
+ $stdout.print ", #{pending_number} pending" if pending_number > 0
+ $stdout.print "\n"
+
+ unless failure_list.empty?
+ $stdout.print "\nFailed examples:\n\n"
+ failure_list.each_with_index do |log, i|
+ case_desc = ''
+ log[1].each_line {|line|
+ case_desc = line
+ break
+ }
+ rerun_cmd = 'rspec .' + log[0].match(/\/spec\/.*_spec\.rb:\d{1,4}/).to_s
+ $stdout.print "#{rerun_cmd} # #{case_desc}"
+ end
+ $stdout.print "\n"
+ end
+ end
+
+ def get_case_list
+ file_list = `grep -rl '' #{SPEC_PATH}`
+ case_list = []
+ file_list.each_line { |filename|
+ unless filename.include? "_spec.rb"
+ next
+ end
+ f = File.read(filename.strip)
+ cases = f.scan(/it [\s\S]*? do/)
+ line_number = 0
+ cases.each { |c|
+ tags = []
+ draft_tags = c.scan(/:([a-zA-Z]+)/)
+ draft_tags.each { |tag|
+ tags << tag[0]
+ }
+ case_desc = c.scan(/it\s+['"](.*?)['"]/)[0][0]
+ i = 0
+ f.each_line { |line|
+ i += 1
+ if i <= line_number && line_number > 0
+ next
+ end
+ if line.include? case_desc
+ case_hash = {"line" => "#{filename.strip}:#{i}", "tags" => tags}
+ case_list << case_hash
+ line_number = i
+ break
+ end
+ }
+ }
+ }
+ case_list
+ end
+
+ def parse_case_list(options)
+ @queue = Queue.new
+ all_case_list = get_case_list
+ pattern_filter_list = []
+ tags_filter_list = []
+
+ if options["pattern"]
+ all_case_list.each { |c|
+ if c["line"].match(options["pattern"])
+ pattern_filter_list << c
+ end
+ }
+ else
+ pattern_filter_list = all_case_list
+ end
+
+ if options["tags"]
+ include_tags = []
+ exclude_tags = []
+ all_tags = options["tags"].split(",")
+ all_tags.each { |tag|
+ if tag.start_with? "~"
+ exclude_tags << tag.gsub("~", "")
+ else
+ include_tags << tag
+ end
+ }
+ pattern_filter_list.each { |c|
+ if (include_tags.length == 0 || (c["tags"] - include_tags).length < c["tags"].length) &&
+ ((c["tags"] - exclude_tags).length == c["tags"].length)
+ tags_filter_list << c
+ end
+ }
+ else
+ tags_filter_list = pattern_filter_list
+ end
+ tags_filter_list.each { |t|
+ @queue << t["line"]
+ }
+ end
+
+ def run_task(task, user, password)
+ cmd = [] # Preparing command for popen
+
+ env_extras = {
+ "YETI_PARALLEL_USER" => user,
+ "YETI_PARALLEL_USER_PASSWD" => password
+ }
+
+ cmd << ENV.to_hash.merge(env_extras)
+ cmd += ["bundle", "exec", "rspec", task]
+ cmd
+
+ output = ""
+
+ IO.popen(cmd, :err => [:child, :out]) do |io|
+ output << io.read
+ end
+
+ output
+ end
+
+ def format_time(t)
+ time_str = ''
+ time_str += (t / 3600).to_i.to_s + " hours " if t > 3600
+ time_str += (t % 3600 / 60).to_i.to_s + " minutes " if t > 60
+ time_str += (t % 60).to_f.round(2).to_s + " seconds"
+ time_str
+ end
+
+ def parse_failure_log(str)
+ index1 = str.index('1) ')
+ index2 = str.index('Finished in')
+ str.slice(index1+3..index2-1).strip
+ end
+
+ def parse_pending_log(str)
+ index1 = str.index('Pending:')
+ index2 = str.index('Finished in')
+ str.slice(index1+8..index2-1).strip
+ end
+
+ extend self
+ end
+end
View
10 lib/harness/rake_helper.rb
@@ -42,6 +42,10 @@ def check_environment
profile[:frameworks] = client.system_frameworks
profile[:script_hash] = get_script_git_hash
File.open(VCAP_BVT_PROFILE_FILE, "w") { |f| f.write YAML.dump(profile) }
+
+ # clear parallel env
+ ENV.delete('YETI_PARALLEL_USER')
+ ENV.delete('YETI_PARALLEL_USER_PASSWD')
end
def check_network_connection
@@ -131,9 +135,9 @@ def print_test_config
puts yellow("\n\nBVT is starting...")
puts "target: \t#{yellow(@config['target'])}"
puts "admin user: \t#{yellow(@config['admin']['email'])}" if @config['admin']
- unless ENV['VCAP_BVT_PARALLEL']
- puts "normal user: \t#{yellow(@config['user']['email'])}"
- end
+ #unless ENV['VCAP_BVT_PARALLEL']
+ puts "normal user: \t#{yellow(@config['user']['email'])}"
+ #end
end
def get_config
View
6 spec/spec_helper.rb
@@ -123,9 +123,9 @@ def log_case_begin_end(flag)
include BVT::Harness::ParallelRunner
config.before(:suite) do
- if ENV['VCAP_BVT_PARALLEL']
- BVT::Harness::VCAP_BVT_PARALLEL_INDEX = increase_sync_index
- end
+ #if ENV['VCAP_BVT_PARALLEL']
+ # BVT::Harness::VCAP_BVT_PARALLEL_INDEX = increase_sync_index
+ #end
BVT::Harness::VCAP_BVT_CONFIG = BVT::Harness::RakeHelper.get_config
profile = YAML.load_file(BVT::Harness::VCAP_BVT_PROFILE_FILE)
BVT::Harness::VCAP_BVT_SYSTEM_FRAMEWORKS = profile[:frameworks]

0 comments on commit 10c769f

Please sign in to comment.
Something went wrong with that request. Please try again.