changed how subtests are handled completely

README.md

@@ -122,3 +122,16 @@ assuming that /tmp/goog4 has the right data, with this command:
 
     bin/yaml_generate.rb "-f" "/tmp/goog4" "-x" "//div[@id='ires']" "-a" "text" "-H" "basic_search_em_data" "-R" "/em[^/]*$"
 
+Altering Tests From The Command Line
+------------------------------------
+
+If you have a complicated test matrix, writing seperate tests for
+each possible case can get very annoying, thus it's possible to
+change the main data set (called $yaml_data in the code) from the
+command line:
+
+    bin/runtest.rb -c 'server_url=http://www.google.co.jp/'
+
+This will run the tests with a different server_url in $yaml_data,
+which will make things break pretty badly.  :)  You can use as many
+-c options as you like, and the format is just x=y

bin/runtest.rb

@@ -11,22 +11,20 @@
 
 opts = GetoptLong.new(
   [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-  [ '--port', '-p', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--browser', '-b', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--server', '-s', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--experiment', '-e', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--release', '-r', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--skip-sections', '-S', GetoptLong::OPTIONAL_ARGUMENT ],
-  [ '--debug', '-D', GetoptLong::OPTIONAL_ARGUMENT ]
+  [ '--port', '-p', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--browser', '-b', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--test-set', '-t', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--skip-sections', '-s', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--debug', '-D', GetoptLong::NO_ARGUMENT ]
 )
 
 $config = {}
 $config['port'] = 4444
 $config['browser'] = 'firefox'
-$config['server'] = nil
-$config['release'] = nil
+$config['test_set'] = nil
+$config['config'] = {}
 $config['skip'] = nil
-$config['experiment'] = nil
 $config['debug'] = false
 
 # Override defaults with values from config file
@@ -41,11 +39,9 @@ def usage
 
   --port/-p		port to reach selenium on; default: #{$config['port']}
   --browser/-b		browser; default: #{$config['browser']}
-  --server/-s		server to test against; default: #{$config['server']}
-  --experiment/-e	experiment to run; default: #{$config['experiment']}
-  --skip-sections/-S	sections to skip; example: "Heatmap,Histogram"; see the yaml/000*.yaml files for section lists; default: #{$config['skip'].to_s}
-  --release/-r		software release to expect; this is an artificial string only used here; default: #{$config['release']}
-                        available choices: #{$config['releases'].join(', ')}
+  --config/-c		directly overrides values in $yaml_data; format is "foo=bar"; default: #{$config['config']}
+  --test-set/-t         test set to run; correspondes to a yaml/000*.yaml file generally; default: #{$config['test_set']}
+  --skip-sections/-s	tests to skip; example: "BasicSearch,Lucky"; see the yaml/000*.yaml files for section lists (same as the file names without ".rb" in tests/); default: #{$config['skip'].to_s}
   --debug/-D		debug mode; default #{$config['debug']}
 
   EOF
@@ -63,26 +59,24 @@ def usage
       $config['port'] = arg
     when '--browser'
       $config['browser'] = arg
-    when '--server'
-      $config['server'] = arg
-    when '--experiment'
-      $config['experiment'] = arg
-    when '--release'
-      $config['release'] = arg
+    when '--test-set'
+      $config['test_set'] = arg
+    when '--config'
+      $config['config'][arg.split(/=/, 2)[0]] = arg.split(/=/, 2)[1]
     when '--skip-sections'
       $config['skip'] = arg
   end
 end
 
 ENV['PORT']=$config['port'].to_s
 ENV['BROWSER']=$config['browser'].to_s
-ENV['SERVER']=$config['server'].to_s
-ENV['EXPERIMENT']=$config['experiment'].to_s
-ENV['RELEASE']=$config['release'].to_s
 ENV['SKIP']=$config['skip'].to_s
+ENV['TEST_SET']=$config['test_set'].to_s
+ENV['CONFIG']=YAML::dump($config['config'])
 ENV['DEBUG']=$config['debug'].to_s
 
 if $config['debug']
+  # print "env: "+ENV.inspect+"\n"
   exec( "rlwrap", "rspec", "#{$0}/../wrapper.rb" )
 else
   exec( "rspec", "#{$0}/../wrapper.rb" )

bin/wrapper.rb

@@ -22,9 +22,8 @@
 end
 $port = ENV['PORT']
 $browser = ENV['BROWSER']
-$server = ENV['SERVER']
-$experiment = ENV['EXPERIMENT']
-$release = ENV['RELEASE']
+$test_set = ENV['TEST_SET']
+$extra_config = YAML::load(ENV['CONFIG'])
 
 def check_conditions(conditions)
   conditions.each do |condition|
@@ -34,23 +33,19 @@ def check_conditions(conditions)
 end
 
 def check_condition(condition)
-  $debug and puts "In check_condition: #{condition.inspect}\n"
-  if condition[0] == 'experiment'
-    return condition[1] == $experiment
-  end
-  if condition[0] == 'server'
-    return condition[1] == $server
-  end
-  if condition[0] == 'release'
-    return condition[1] == $release
-  end
+  value=$yaml_data[condition[0]] == condition[1]
+  $debug and puts "In check_condition: #{condition.inspect}, which is #{value.inspect}; #{condition[0]}, #{$yaml_data[condition[0]]} == #{condition[1]}\n"
+  return $yaml_data[condition[0]] == condition[1]
 end
 
 require 'GeneralSeleniumUtility.rb'
 include GeneralSeleniumUtility
 
 # Load our own config
 $yaml_data = load_yaml_data("yaml/wrapper.yaml")
+$yaml_data['test_set'] = $test_set
+$yaml_data['port'] = $port
+$yaml_data['browser'] = $browser
 if $yaml_data['wrapper_modules']
   $yaml_data['wrapper_modules'].each do |module_name|
     $debug and print "adding #{module_name}.rb\n"
@@ -108,6 +103,9 @@ def check_condition(condition)
   end
 end
 
+# Now load in any special config
+$yaml_data.merge!($extra_config)
+
 describe "wrapper" do
   include GeneralSeleniumUtility
   if $yaml_data['wrapper_modules']
@@ -122,31 +120,17 @@ def check_condition(condition)
     @debug = $debug
     @port = $port
     @browser = $browser
-    @server = $server
-    @experiment = $experiment
-    @release = $release
+    @test_set = $test_set
     @yaml_data = $yaml_data
 
-    @server_url = $yaml_data['server_url']
-
     selenium_setup
 
   end
 
   $yaml_data['sections'].each do |section|
-    realsection = section
-    if Dir.glob("tests/#{section}_#{$release}.rb").length > 0
-      realsection = "#{section}_#{$release}"
-    end
-    if Dir.glob("tests/#{section}_#{$experiment}.rb").length > 0
-      realsection = "#{section}_#{$experiment}"
-    end
-    if Dir.glob("tests/#{section}_#{$experiment}_#{$release}.rb").length > 0
-      realsection = "#{section}_#{$experiment}_#{$release}"
-    end
-    print "Running tests from #{realsection}.rb\n"
-    require "tests/#{realsection}.rb"
-    include Module.const_get(realsection)
+    print "Running tests from #{section}.rb\n"
+    require "tests/#{section}.rb"
+    include Module.const_get(section)
   end
 
   after(:all) do

lib/GeneralSeleniumUtility.rb

@@ -95,6 +95,15 @@ def load_yaml_data( file )
     return yaml_data
   end
 
+  #****************
+  # Go to the page url given by the yaml_data key
+  #****************
+  def go_to(yaml_data_key)
+    @driver.navigate.to $yaml_data[yaml_data_key]
+    quiesce
+    @driver.current_url.should == $yaml_data[yaml_data_key]
+  end
+
   #****************
   # Finds the given element, checks its name, and types data into
   # the element

lib/GoogleUtility.rb

@@ -1,7 +1,10 @@
 module GoogleUtility
-  def click_logo
+  def click_irrelevant
     quiesce
-    no_move_click_raw(:id, 'hplogo', 'img')
+    # Clicks the more link, twice
+    no_move_click_raw(:id, 'gbztms1', 'span')
+    quiesce
+    no_move_click_raw(:id, 'gbztms1', 'span')
     quiesce
   end
 end

tests/BasicSearch.rb

@@ -3,7 +3,7 @@
 share_as :BasicSearch do
   describe 'run a basic search on Google' do
     it "should load the main page" do
-      @driver.navigate.to @server_url
+      go_to('server_url')
     end
 
     it 'should take the text input' do
@@ -13,7 +13,7 @@
     end
 
     it 'should click elsewhere to close the javascripty bits' do
-      click_logo
+      click_irrelevant
     end
 
     it 'should click on search and load the new page' do

tests/Lucky.rb

@@ -3,15 +3,15 @@
 share_as :Lucky do
   describe 'run a feeling lucky search on Google' do
     it "should load the main page" do
-      @driver.navigate.to @server_url
+      go_to('server_url')
     end
 
     it 'should take the text input' do
       check_element_send_keys('lucky_search_monkeys')
     end
 
     it 'should click elsewhere to close the javascripty bits' do
-      click_logo
+      click_irrelevant
     end
 
     it %q{should click on "I'm Feeling Lucky" and load the new page} do

tests/Setup.rb

@@ -9,7 +9,7 @@
     end
 
     it "should load the setup page" do
-      @driver.navigate.to "http://www.google.com/preferences?hl=en"
+      go_to('setup_url')
     end
 
     it "should change to no instant predictions" do

tests/Translate.rb

@@ -3,7 +3,7 @@
 share_as :Translate do
   describe 'go to the google translate page via the main site' do
     it "should load the main page" do
-      @driver.navigate.to @server_url
+      go_to('server_url')
     end
 
     it 'should click on the More link' do

yaml/000_all.yaml

@@ -1,7 +1,7 @@
 ---
 conditions:
   -
-    - experiment
+    - test_set
     - basic
 
 sections:

yaml/100_basic_prod.yaml

@@ -1,9 +1,6 @@
 ---
 conditions:
   -
-    - server
-    - prod
-  -
-    - experiment
+    - test_set
     - basic
-server_url: "http://www.google.com"
+server_url: "http://www.google.com/"

yaml/BasicSearch_000_basic.yaml

@@ -1,7 +1,7 @@
 ---
 conditions:
   -
-    - experiment
+    - test_set
     - basic
 
 basic_search_click_search:

yaml/Lucky_000_basic.yaml

@@ -1,7 +1,7 @@
 ---
 conditions:
   -
-    - experiment
+    - test_set
     - basic
 
 lucky_click_lucky:

yaml/Setup_000_basic.yaml

@@ -1,7 +1,7 @@
 ---
 conditions:
   -
-    - experiment
+    - test_set
     - basic
 
 setup_instant_predictions_div1: - id
@@ -14,3 +14,5 @@ setup_click_save: - xpath
                   - "//div[text()='Save']"
                   - "div"
                   - "^http://www.google.com/$"
+
+setup_url: "http://www.google.com/preferences?hl=en"

yaml/Translate_000_basic.yaml

@@ -1,7 +1,7 @@
 ---
 conditions:
   -
-    - experiment
+    - test_set
     - basic
 
 translate_more_link: - id

yaml/runtest.yaml

@@ -1,5 +1,2 @@
 browser: firefox
-server: prod
-release: all
-experiment: basic
-releases: - all
+test_set: basic