Merge pull request #229 from alphagov/ab-testing-test

Add feature tests for A/B test example page

config/cucumber.yml

@@ -1,4 +1,4 @@
 # Options specific to a GOV.UK environment
 integration: -t ~@pending -t ~@notintegration
-staging: -t ~@pending
+staging: -t ~@pending -t ~@notstaging
 production: -t ~@pending

features/ab_testing.feature

@@ -0,0 +1,27 @@
+# Tests for the example A/B testing page. Even though these just test an example
+# page, they're still a useful smoke test of real A/B experiments, because they
+# all A/B tests rely on CDN config to assign users into buckets and handle A/B
+# cookies correctly. So a failure in these tests indicates that all A/B tests
+# may be broken.
+#
+# These tests are only run against Production because they rely on the CDN to
+# manipulate the HTTP request and response headers.
+Feature: A/B Testing
+
+  @low @notintegration @notstaging
+  Scenario: check we end up in all buckets
+    Given there is an AB test setup
+    And I am testing through the full stack
+    When multiple new users visit "/help/ab-testing"
+    Then we have shown them all versions of the AB test
+
+  @low @notintegration @notstaging
+  Scenario: check that an A/B test works
+    Given there is an AB test setup
+    And I am testing through the full stack
+    And I do not have any AB testing cookies set
+    When I visit "/help/ab-testing"
+    Then I am assigned to a test bucket
+    And I can see the bucket I am assigned to
+    And the bucket is reported to Google Analytics
+    And I stay on the same bucket when I keep visiting "/help/ab-testing"

features/step_definitions/ab_testing_steps.rb

@@ -0,0 +1,58 @@
+Given(/^there is an AB test setup$/) do
+  # Empty step.
+  # We assume that there is always an A/B test set up on the example A/B test
+  # page. If this is not true, the A/B smoke tests will fail, so we should fix
+  # or delete them as appropriate.
+end
+
+Given(/^I do not have any AB testing cookies set$/) do
+  # Empty step.
+  # By default, no cookies are set.
+end
+
+Then(/^we have shown them all versions of the AB test$/) do
+  buckets = @responses.map { |r| ab_bucket(r.body) }.to_set
+  buckets.should == (Set.new ["A", "B"])
+end
+
+Then(/^I am assigned to a test bucket$/) do
+  ab_cookies = @response.headers[:set_cookie]
+    .select { |h| h.start_with? "ABTest-Example=" }
+  assert_equal 1, ab_cookies.length,
+    "Expected response to set exactly one A/B test cookie"
+
+  ab_cookie = ab_cookies[0]
+  @ab_cookie_value = /ABTest-Example=([^;]*);/.match(ab_cookie)[1]
+
+  assert @ab_cookie_value == "A" || @ab_cookie_value == "B",
+    "Expected A/B cookie to have value 'A' or 'B' but got '#{@ab_cookie_value}'"
+
+  assert ab_cookie.include?("expires="), "A/B cookie has no expiry time"
+end
+
+Then(/^I can see the bucket I am assigned to$/) do
+  bucket = ab_bucket(@response.body)
+  assert bucket == "A" || bucket == "B",
+    "Expected A/B bucket to be 'A' or 'B', but got '#{bucket}'"
+
+  # Store bucket so that subsequent responses can be compared to the original
+  @original_bucket = bucket
+end
+
+Then(/^the bucket is reported to Google Analytics$/) do
+  # TODO: Implement once we know how we're reporting to GA. Is it sufficient to
+  # check the meta tag which is inspected by GA to report the page view?
+end
+
+Then(/^I stay on the same bucket when I keep visiting "(.*?)"$/) do |path|
+  20.times do
+    request_options = default_request_options.merge(cookies: {"ABTest-Example": @ab_cookie_value})
+    response = get_request("#{@host}#{path}", request_options)
+
+    ab_bucket(response.body).should == @original_bucket
+  end
+end
+
+def ab_bucket page
+  Nokogiri::HTML.parse(page).css(".ab-example-group").text.strip
+end

features/step_definitions/smokey_steps.rb

@@ -59,6 +59,13 @@
   }
 end
 
+When(/^multiple new users visit "(.*?)"$/) do |path|
+  @responses = []
+  20.times do
+    @responses << get_request("#{@host}#{path}", default_request_options)
+  end
+end
+
 When /^I visit a non-existent page$/ do
   @response = get_request("#{@host}/404", default_request_options.merge(return_response_on_error: true))
 end

features/support/http_requests.rb

@@ -78,6 +78,9 @@ def do_http_request(url, method = :get, options = {}, &block)
   if options[:host_header]
     headers["Host"] = options[:host_header]
   end
+  if options[:cookies]
+    headers["Cookie"] = options[:cookies].map { |k, v| "#{k}=#{v}" }.join("; ")
+  end
   rate_limit_token = ENV['RATE_LIMIT_TOKEN']
   if rate_limit_token
     headers["Rate-Limit-Token"] = rate_limit_token