Skip to content
Permalink
Browse files

Merge branch 'reduce-time-taken-for-regression-tests'

This reduces the time taken to run the regression tests by only running them
for a Smart Answer when its source files have changed.

I store a hash of the contents of the Smart Answer source files and check those
to determine whether to run the regression tests for that Smart Answer. You can
override this by setting the RUN_REGRESSION_TESTS environment variable to
'true' to run all regression tests, or '<flow-name>' to run the regression
tests for a single flow.

I'm automatically generating a set of source files but these won't always be
comprehensive. Some Smart Answers use calculators and external data files that
will have to be listed manually.
  • Loading branch information...
chrisroos committed Jun 1, 2015
2 parents 578d7a9 + 108da45 commit 458e7029229b11f003a85ee8ea46572660df4917
@@ -133,6 +133,12 @@ Test a single Smartdown flow by running:

* Go to Step 3, add the new responses and continue through the steps up to Step 9.

10. Generate a yaml file containing the set of source files that this Smart Answer depends upon. The script will automatically take the ruby flow file, locale file and erb templates into account. You just need to supply it with the location of any additional files required by the Smart Answer (e.g. calculators and data files). This data is used to determine whether to run the regression tests based on whether the source files have changed.

$ rails r script/generate-checksums-for-smart-answer.rb <name-of-smart-answer> <path/to/additional/files>

11. Commit the generated yaml file to git.

## Issues/todos

Please see the [github issues](https://github.com/alphagov/smart-answers/issues) page.
@@ -0,0 +1,59 @@
class SmartAnswerFiles
def initialize(flow_name, *additional_files_paths)
@flow_name = flow_name
@additional_files_paths = additional_files_paths.map do |path|
Pathname.new(path)
end
end

def paths
relative_paths.map(&:to_s).uniq
end

private

def relative_paths
all_paths.collect do |path|
path.relative_path_from(Rails.root)
end
end

def all_paths
[
flow_path,
locale_path,
questions_and_responses_test_data_path,
responses_and_expected_results_test_data_path
] + erb_template_paths + additional_files_absolute_paths
end

def erb_template_directory
Rails.root.join('lib', 'smart_answer_flows', @flow_name)
end

def erb_template_paths
Dir[erb_template_directory.join('*.erb')].collect do |path|
Pathname.new(path)
end
end

def additional_files_absolute_paths
@additional_files_paths.map(&:realpath)
end

def flow_path
Rails.root.join('lib', 'smart_answer_flows', "#{@flow_name}.rb")
end

def locale_path
Rails.root.join('lib', 'smart_answer_flows', 'locales', 'en', "#{@flow_name}.yml")
end

def questions_and_responses_test_data_path
SmartAnswerTestHelper.new(@flow_name).question_and_responses_path
end

def responses_and_expected_results_test_data_path
SmartAnswerTestHelper.new(@flow_name).responses_and_expected_results_path
end
end
@@ -0,0 +1,29 @@
class SmartAnswerHasher
class FileNotFound < StandardError; end

def initialize(flow_file_paths)
@flow_file_paths = flow_file_paths
ensure_all_files_exist
calculate_checksum_data
end

def write_checksum_data(io)
io.puts(@checksum_data.to_yaml)
end

private

def ensure_all_files_exist
@flow_file_paths.each do |path|
raise FileNotFound unless File.exists?(path)
end
end

def calculate_checksum_data
@checksum_data = @flow_file_paths.inject({}) do |hash, file_path|
file_content = File.read(file_path)
hash[file_path] = Digest::MD5.hexdigest(file_content)
hash
end
end
end
@@ -15,6 +15,37 @@ def initialize(flow_name)
@flow_name = flow_name
end

def files_checksum_path
data_path.join(files_checksum_filename)
end

def write_files_checksum(hasher)
File.open(files_checksum_path, 'w') do |file|
hasher.write_checksum_data(file)
end
end

def read_files_checksums
files_checksums_yaml = File.read(files_checksum_path)
YAML.load(files_checksums_yaml)
end

def files_checksum_data_exists?
File.exists?(files_checksum_path)
end

def run_regression_tests?
explicitly_run_all_regression_tests? ||
explicitly_run_this_regression_test? ||
files_checksum_data_needs_updating?
end

def files_checksum_data_needs_updating?
!files_checksum_data_exists? ||
source_files_have_changed? ||
source_files_have_been_added?
end

def question_and_responses_path
data_path.join(question_and_responses_filename)
end
@@ -63,6 +94,10 @@ def delete_saved_output_files

private

def files_checksum_filename
"#{@flow_name}-files.yml"
end

def question_and_responses_filename
"#{@flow_name}-questions-and-responses.yml"
end
@@ -78,4 +113,30 @@ def data_path
def artefacts_path
self.class.artefacts_path
end

def source_files_have_changed?
checksum_data = read_files_checksums
changed_files = checksum_data.select do |path, expected_checksum|
content = File.read(path)
actual_checksum = Digest::MD5.hexdigest(content)
expected_checksum != actual_checksum
end
changed_files.any?
end

def explicitly_run_this_regression_test?
ENV['RUN_REGRESSION_TESTS'] == @flow_name
end

def explicitly_run_all_regression_tests?
ENV['RUN_REGRESSION_TESTS'] == 'true'
end

def source_files_have_been_added?
checksum_data = read_files_checksums
known_smart_answer_files = checksum_data.keys
detected_smart_answer_files = SmartAnswerFiles.new(@flow_name)
unknown_files = detected_smart_answer_files.paths - known_smart_answer_files
unknown_files.any?
end
end
@@ -0,0 +1,21 @@
unless flow_name = ARGV.shift
puts "Usage: #{__FILE__} <flow-name> <additional-flow-file-paths>"
exit 1
end

flow_helper = SmartAnswerTestHelper.new(flow_name)

existing_additional_flow_file_paths = []
if flow_helper.files_checksum_data_exists?
existing_checksum_data = flow_helper.read_files_checksums
existing_additional_flow_file_paths = existing_checksum_data.keys
end

additional_flow_file_paths = ARGV + existing_additional_flow_file_paths
flow_files = SmartAnswerFiles.new(flow_name, *additional_flow_file_paths)

hasher = SmartAnswerHasher.new(flow_files.paths)

flow_helper.write_files_checksum(hasher)

puts "Checksum data written to #{flow_helper.files_checksum_path}"
@@ -0,0 +1,9 @@
---
lib/smart_answer_flows/additional-commodity-code.rb: d7f38b15cd1c6e000f0cdc5f562f3912
lib/smart_answer_flows/locales/en/additional-commodity-code.yml: 1bd6a7da656f81f10992dcbae147f9b5
test/data/additional-commodity-code-questions-and-responses.yml: f2149cbfa6ec8c5ead572f0a89542a79
test/data/additional-commodity-code-responses-and-expected-results.yml: af16012af254608b78a5b2953e742efd
lib/smart_answer_flows/additional-commodity-code/commodity_code_result_body.govspeak.erb: bf2f9f34f2146c74bdf08bf4fef64875
lib/smart_answer_flows/additional-commodity-code/commodity_code_result_title.txt.erb: b4147ea8166904c0ab01c7f4063692bd
lib/smart_answer/calculators/commodity_code_calculator.rb: e0aba9021dacb17e95d60337bdbed247
lib/data/commodity_codes_data.yml: e18434a8a7b02f644e69ebe45f518772
@@ -0,0 +1,8 @@
---
lib/smart_answer_flows/am-i-getting-minimum-wage.rb: a5f9ccf8dfe0c6835ef605aa1c5cbacc
lib/smart_answer_flows/locales/en/am-i-getting-minimum-wage.yml: 285c621a77cb625d08a440a9b74aa85e
test/data/am-i-getting-minimum-wage-questions-and-responses.yml: 173bbd0ad46b728bdd6498fa71585ddc
test/data/am-i-getting-minimum-wage-responses-and-expected-results.yml: 7f3e7ba3ea38a0013556638836f0af2d
lib/smart_answer_flows/shared_logic/minimum_wage.rb: 0266518e125cf1fd950e8b081c71274f
lib/smart_answer/calculators/minimum_wage_calculator.rb: 1aca36bb1b33abf4cbef8171d9f13f3c
lib/data/minimum_wage_data.yml: 55c71c9ff8252c17a679dad888ad5ad7
@@ -0,0 +1,6 @@
---
lib/smart_answer_flows/apply-tier-4-visa.rb: 9883575c2342dd95963da6bc4d06c5f0
lib/smart_answer_flows/locales/en/apply-tier-4-visa.yml: 54401da7ebae8a1c9e00864bbce54e4c
test/data/apply-tier-4-visa-questions-and-responses.yml: fe87a99e4337e45586806181742f0b8f
test/data/apply-tier-4-visa-responses-and-expected-results.yml: 0a4f632cf6eee9a243f896392cd2ea24
lib/data/apply_tier_4_visa_data.yml: 671371971f6024d6d8b95f422e7985f6
@@ -0,0 +1,6 @@
---
lib/smart_answer_flows/state-pension-through-partner.rb: fde6fc823fc6f523af9ffa6be793f0cc
lib/smart_answer_flows/locales/en/state-pension-through-partner.yml: 741b1a6be2aeed88b3745e9351020ff0
test/data/state-pension-through-partner-questions-and-responses.yml: 34f2b5f7ac286bb3eea690564339051b
test/data/state-pension-through-partner-responses-and-expected-results.yml: 5cf58a9a4d447c348f9821d5d61468eb
lib/data/rates/state_pension.yml: 7e97e88500fb698038c49332b48b82bf
@@ -0,0 +1,8 @@
---
lib/smart_answer_flows/state-pension-topup.rb: 005abed8a3e630d74fc5cd14bf12ea75
lib/smart_answer_flows/locales/en/state-pension-topup.yml: 1c97f6d53c415cd376e99a0ed812d811
test/data/state-pension-topup-questions-and-responses.yml: d3997e715e206c38ea8a0783b07b2350
test/data/state-pension-topup-responses-and-expected-results.yml: ff3557145d39c5b2d10c31b599ee493f
lib/smart_answer/calculators/state_pension_topup_calculator.rb: 2481f46c212f185572def0aa2c3f3e0a
lib/smart_answer/calculators/state_pension_topup_data_query.rb: 9a87f13dd72f0d0518b7261e2dafe32f
lib/data/pension_top_up_data.yml: 9059d16c14713405d406e5aefa81ea46
@@ -0,0 +1,8 @@
---
lib/smart_answer_flows/student-finance-calculator.rb: 11a0a54f5635c6a438ee55e7206c7091
lib/smart_answer_flows/locales/en/student-finance-calculator.yml: 6100f316b4a031ee1a3c46ff377c6ab6
test/data/student-finance-calculator-questions-and-responses.yml: 7f7a6293dd5c37756c89ea6b075ffd45
test/data/student-finance-calculator-responses-and-expected-results.yml: 7680ed7e0f6205784af3c766bbdcf23e
lib/smart_answer_flows/student-finance-calculator/outcome_eu_students_body.govspeak.erb: 09baaca179dba1dc7fd0415800775503
lib/smart_answer_flows/student-finance-calculator/outcome_uk_all_students_body.govspeak.erb: 9a2911d77e4e1fc153584d1d1a99af81
lib/smart_answer_flows/student-finance-calculator/outcome_uk_full_time_students_body.govspeak.erb: 3ccbc53c77f0c94f1fef64048f56496e
@@ -0,0 +1,5 @@
---
lib/smart_answer_flows/towing-rules.rb: 76c99d0ebe9a26d686563ce4b9512a54
lib/smart_answer_flows/locales/en/towing-rules.yml: 65c5f6133bc15d0c77db3ee4bc30de28
test/data/towing-rules-questions-and-responses.yml: 09c8f43d5353fa6248658a27982ea5a2
test/data/towing-rules-responses-and-expected-results.yml: 943926704f214c45404673391ed47bc8
@@ -0,0 +1,21 @@
---
lib/smart_answer_flows/vat-payment-deadlines.rb: 0b42593ff72d99cc3fb1c31d4851c5bb
lib/smart_answer_flows/locales/en/vat-payment-deadlines.yml: 96a0aaa62f2db7169db4ed5f1caeff06
test/data/vat-payment-deadlines-questions-and-responses.yml: a9c89cff2686881257fe52b750032268
test/data/vat-payment-deadlines-responses-and-expected-results.yml: 929c9a22dbd6775ad15330e52e860b65
lib/smart_answer_flows/vat-payment-deadlines/result_bacs_direct_credit_body.govspeak.erb: 9995803d5fee6c5eea80b26e2b92d0ec
lib/smart_answer_flows/vat-payment-deadlines/result_bacs_direct_credit_title.txt.erb: 961e10029948fddff8ac69bedccc82e7
lib/smart_answer_flows/vat-payment-deadlines/result_bank_giro_body.govspeak.erb: d8289868c853367534a7213cd385ac95
lib/smart_answer_flows/vat-payment-deadlines/result_bank_giro_title.txt.erb: f35872c2a3cc02f4f865406669d7ead9
lib/smart_answer_flows/vat-payment-deadlines/result_chaps_body.govspeak.erb: 5af9cb960941eaa01e931859a7435ae4
lib/smart_answer_flows/vat-payment-deadlines/result_chaps_title.txt.erb: 25003d01d6838a1d10932d5ad0f429c6
lib/smart_answer_flows/vat-payment-deadlines/result_cheque_body.govspeak.erb: ae3edd3773de9b1f3ca89c4fefc4e717
lib/smart_answer_flows/vat-payment-deadlines/result_cheque_title.txt.erb: a1c7ea8e502742cd1d47e5850babdd1e
lib/smart_answer_flows/vat-payment-deadlines/result_direct_debit_body.govspeak.erb: b31a27a94a4f2492494c61d82b71a472
lib/smart_answer_flows/vat-payment-deadlines/result_direct_debit_title.txt.erb: 685ca88fd09cd9fd9c2d9ea4986206de
lib/smart_answer_flows/vat-payment-deadlines/result_online_debit_credit_card_body.govspeak.erb: 984a346cec592ac49416d5d18a0f3385
lib/smart_answer_flows/vat-payment-deadlines/result_online_debit_credit_card_title.txt.erb: e2373fabdebcd5d8f46cd5ee788de957
lib/smart_answer_flows/vat-payment-deadlines/result_online_telephone_banking_body.govspeak.erb: 7bbe4994c9d83eab75ec771bab920514
lib/smart_answer_flows/vat-payment-deadlines/result_online_telephone_banking_title.txt.erb: 10cc2b1911ad78b0d3688060372eab90
lib/smart_answer/calculators/vat_payment_deadlines.rb: e1e95cce94754c307eb521a439f49e3a
lib/working_days.rb: cb61b2a6ae8644542cd7574f5b7383c8
@@ -14,6 +14,9 @@ class SmartAnswerResponsesAndExpectedResultsTest < ActionController::TestCase
flow_name = filename[/(.*)-responses-and-expected-results/, 1]

smart_answer_helper = SmartAnswerTestHelper.new(flow_name)

next unless smart_answer_helper.run_regression_tests?

smart_answer_helper.delete_saved_output_files
responses_and_expected_results = smart_answer_helper.read_responses_and_expected_results

@@ -28,6 +31,20 @@ class SmartAnswerResponsesAndExpectedResultsTest < ActionController::TestCase
Timecop.return
end

should "have checksum data" do
message = []
message << "Expected #{smart_answer_helper.files_checksum_path} to exist"
message << "Use the generate-checksums-for-smart-answer script to create it"
assert_equal true, smart_answer_helper.files_checksum_data_exists?, message.join('. ')
end

should "have up to date checksum data" do
message = []
message << "Expected #{smart_answer_helper.files_checksum_path} to contain up to date data"
message << "Use the generate-checksums-for-smart-answer script to update it"
assert_equal false, smart_answer_helper.files_checksum_data_needs_updating?, message.join('. ')
end

responses_and_expected_results.each do |responses_and_expected_node|
responses = responses_and_expected_node[:responses]
outcome_node = responses_and_expected_node[:outcome_node]

0 comments on commit 458e702

Please sign in to comment.
You can’t perform that action at this time.