diff --git a/admin/tool/behat/cli/init.php b/admin/tool/behat/cli/init.php index 4517a7e31b79c..aa08c4daebc4b 100644 --- a/admin/tool/behat/cli/init.php +++ b/admin/tool/behat/cli/init.php @@ -47,11 +47,13 @@ 'help' => false, 'fromrun' => 1, 'torun' => 0, + 'themesuitewithallfeatures' => false, ), array( 'j' => 'parallel', 'm' => 'maxruns', 'h' => 'help', + 'a' => 'themesuitewithallfeatures', ) ); @@ -68,6 +70,8 @@ --fromrun Execute run starting from (Used for parallel runs on different vms) --torun Execute run till (Used for parallel runs on different vms) +-a, --themesuitewithallfeatures Theme suite do not include core features by default. If you want to run core features + with theme, then use this option. -h, --help Print out this help Example from Moodle root directory: @@ -95,6 +99,11 @@ } } +$themesuitewithallfeatures = ''; +if ($options['themesuitewithallfeatures']) { + $themesuitewithallfeatures = '--themesuitewithallfeatures="true"'; +} + // Changing the cwd to admin/tool/behat/cli. $cwd = getcwd(); $output = null; @@ -104,7 +113,7 @@ // Check whether the behat test environment needs to be updated. chdir(__DIR__); -exec("php $utilfile --diag $paralleloption", $output, $code); +exec("php $utilfile --diag $paralleloption $themesuitewithallfeatures", $output, $code); if ($code == 0) { echo "Behat test environment already installed\n"; @@ -112,7 +121,7 @@ } else if ($code == BEHAT_EXITCODE_INSTALL) { // Behat and dependencies are installed and we need to install the test site. chdir(__DIR__); - passthru("php $utilfile --install $paralleloption", $code); + passthru("php $utilfile --install $paralleloption $themesuitewithallfeatures", $code); if ($code != 0) { chdir($cwd); exit($code); @@ -121,14 +130,14 @@ } else if ($code == BEHAT_EXITCODE_REINSTALL) { // Test site data is outdated. chdir(__DIR__); - passthru("php $utilfile --drop $paralleloption", $code); + passthru("php $utilfile --drop $paralleloption $themesuitewithallfeatures", $code); if ($code != 0) { chdir($cwd); exit($code); } chdir(__DIR__); - passthru("php $utilfile --install $paralleloption", $code); + passthru("php $utilfile --install $paralleloption $themesuitewithallfeatures", $code); if ($code != 0) { chdir($cwd); exit($code); @@ -143,7 +152,7 @@ // Enable editing mode according to config.php vars. chdir(__DIR__); -passthru("php $utilfile --enable $paralleloption", $code); +passthru("php $utilfile --enable $paralleloption $themesuitewithallfeatures", $code); if ($code != 0) { echo "Error enabling site" . PHP_EOL; chdir($cwd); diff --git a/admin/tool/behat/cli/run.php b/admin/tool/behat/cli/run.php index f8a37e77523e0..92b992f4a291b 100644 --- a/admin/tool/behat/cli/run.php +++ b/admin/tool/behat/cli/run.php @@ -51,15 +51,18 @@ 'tags' => '', 'profile' => '', 'feature' => '', + 'suite' => '', 'fromrun' => 1, 'torun' => 0, 'single-run' => false, + 'themesuitewithallfeatures' => false, ), array( 'h' => 'help', 't' => 'tags', 'p' => 'profile', 's' => 'single-run', + 'a' => 'themesuitewithallfeatures', ) ); @@ -73,10 +76,13 @@ Options: --BEHAT_OPTION Any combination of behat option specified in http://behat.readthedocs.org/en/v2.5/guides/6.cli.html --feature Only execute specified feature file (Absolute path of feature file). +--suite Specified theme scenarios will be executed. --replace Replace args string with run process number, useful for output. --fromrun Execute run starting from (Used for parallel runs on different vms) --torun Execute run till (Used for parallel runs on different vms) +-a, --themesuitewithallfeatures Theme suite do not include core features by default. If you want to run core features + with theme, then use this option. -h, --help Print out this help Example from Moodle root directory: @@ -101,9 +107,6 @@ if (extension_loaded('pcntl')) { $disabled = explode(',', ini_get('disable_functions')); if (!in_array('pcntl_signal', $disabled)) { - // Handle interrupts on PHP7. - declare(ticks = 1); - pcntl_signal(SIGTERM, "signal_handler"); pcntl_signal(SIGINT, "signal_handler"); } @@ -144,6 +147,11 @@ $extraopts[] = '--tags="' . $tags . '"'; } +// Add suite option if specified. +if ($options['suite']) { + $extraopts[] = '--suite="' . $options['suite'] . '"'; +} + // Feature should be added to last, for behat command. if ($options['feature']) { $extraopts[] = $options['feature']; @@ -170,6 +178,8 @@ // Update config file if tags defined. if ($tags) { + define('ABORT_AFTER_CONFIG_CANCEL', true); + require("$CFG->dirroot/lib/setup.php"); // Hack to set proper dataroot and wwwroot. $behatdataroot = $CFG->behat_dataroot; $behatwwwroot = $CFG->behat_wwwroot; @@ -186,7 +196,7 @@ } else { $CFG->behat_dataroot = $behatdataroot . $i; } - behat_config_manager::update_config_file('', true, $tags); + behat_config_manager::update_config_file('', true, $tags, $options['themesuitewithallfeatures'], $parallelrun); } $CFG->behat_dataroot = $behatdataroot; $CFG->behat_wwwroot = $behatwwwroot; diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index a952efce07df6..06d2bf737e130 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -56,11 +56,13 @@ 'updatesteps' => false, 'fromrun' => 1, 'torun' => 0, + 'themesuitewithallfeatures' => false, ), array( 'h' => 'help', 'j' => 'parallel', - 'm' => 'maxruns' + 'm' => 'maxruns', + 'a' => 'themesuitewithallfeatures', ) ); @@ -78,9 +80,11 @@ --disable Disables test environment --diag Get behat test environment status code --updatesteps Update feature step file. + -j, --parallel Number of parallel behat run operation -m, --maxruns Max parallel processes to be executed at one time. - +-a, --themesuitewithallfeatures Theme suite do not include core features by default. If you want to run core features + with theme, then use this option. -h, --help Print out this help Example from Moodle root directory: @@ -175,12 +179,15 @@ } else if ($options['updatesteps']) { // Rewrite config file to ensure we have all the features covered. if (empty($options['parallel'])) { - behat_config_manager::update_config_file(); + behat_config_manager::update_config_file('', true, '', $options['themesuitewithallfeatures'], false, false); } else { // Update config file, ensuring we have up-to-date behat.yml. for ($i = $options['fromrun']; $i <= $options['torun']; $i++) { $CFG->behatrunprocess = $i; - behat_config_manager::update_config_file(); + + // Update config file for each run. + behat_config_manager::update_config_file('', true, '', $options['themesuitewithallfeatures'], + $options['parallel'], $i); } unset($CFG->behatrunprocess); } diff --git a/admin/tool/behat/cli/util_single_run.php b/admin/tool/behat/cli/util_single_run.php index 1991707223343..c0fc3fa4761fc 100644 --- a/admin/tool/behat/cli/util_single_run.php +++ b/admin/tool/behat/cli/util_single_run.php @@ -40,16 +40,18 @@ 'help' => false, 'install' => false, 'parallel' => 0, - 'run' => '', + 'run' => 0, 'drop' => false, 'enable' => false, 'disable' => false, 'diag' => false, 'tags' => '', 'updatesteps' => false, + 'themesuitewithallfeatures' => false, ), array( - 'h' => 'help' + 'h' => 'help', + 'a' => 'themesuitewithallfeatures', ) ); @@ -72,6 +74,8 @@ --diag Get behat test environment status code --updatesteps Update feature step file. +-a, --themesuitewithallfeatures Theme suite do not include core features by default. If you want to run core features + with theme, then use this option. -h, --help Print out this help Example from Moodle root directory: @@ -130,15 +134,22 @@ require_once($CFG->libdir . '/behat/classes/behat_config_manager.php'); // Ensure run option is <= parallel run installed. +$run = 0; +$parallel = 0; if ($options['run']) { + $run = $options['run']; + // If parallel option is not passed, then try get it form config. if (!$options['parallel']) { - $options['parallel'] = behat_config_manager::get_parallel_test_runs(); + $parallel = behat_config_manager::get_parallel_test_runs(); + } else { + $parallel = $options['parallel']; } - if (empty($options['parallel']) || $options['run'] > $options['parallel']) { - echo "Parallel runs can't be more then ".$options['parallel'].PHP_EOL; + + if (empty($parallel) || $run > $parallel) { + echo "Parallel runs can't be more then ".$parallel.PHP_EOL; exit(1); } - $CFG->behatrunprocess = $options['run']; + $CFG->behatrunprocess = $run; } // Run command (only one per time). @@ -146,7 +157,7 @@ behat_util::install_site(); // This is only displayed once for parallel install. - if (empty($options['run'])) { + if (empty($run)) { mtrace("Acceptance tests site installed"); } @@ -155,30 +166,30 @@ test_lock::acquire('behat'); behat_util::drop_site(); // This is only displayed once for parallel install. - if (empty($options['run'])) { + if (empty($run)) { mtrace("Acceptance tests site dropped"); } } else if ($options['enable']) { - if (!empty($options['parallel'])) { + if (!empty($parallel)) { // Save parallel site info for enable and install options. $filepath = behat_config_manager::get_parallel_test_file_path(); - if (!file_put_contents($filepath, $options['parallel'])) { + if (!file_put_contents($filepath, $parallel)) { behat_error(BEHAT_EXITCODE_PERMISSIONS, 'File ' . $filepath . ' can not be created'); } } // Enable test mode. - behat_util::start_test_mode(); + behat_util::start_test_mode($options['themesuitewithallfeatures'], $parallel, $run); // This is only displayed once for parallel install. - if (empty($options['run'])) { + if (empty($run)) { // Notify user that 2.5 profile has been converted to 3.5. if (behat_config_manager::$autoprofileconversion) { mtrace("2.5 behat profile detected, automatically converted to current 3.x format"); } - $runtestscommand = behat_command::get_behat_command(true, !empty($options['run'])); + $runtestscommand = behat_command::get_behat_command(true, !empty($run)); $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); mtrace("Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use: " . PHP_EOL . @@ -188,7 +199,7 @@ } else if ($options['disable']) { behat_util::stop_test_mode(); // This is only displayed once for parallel install. - if (empty($options['run'])) { + if (empty($run)) { mtrace("Acceptance tests environment disabled"); } diff --git a/admin/tool/behat/index.php b/admin/tool/behat/index.php index 5e71bdff59d23..f3b1ac001258a 100644 --- a/admin/tool/behat/index.php +++ b/admin/tool/behat/index.php @@ -45,7 +45,8 @@ $componentswithsteps = array('' => get_string('allavailablesteps', 'tool_behat')); // Complete the components list with the moodle steps definitions. -$components = behat_config_manager::get_components_steps_definitions(); +$behatconfig = new behat_config_util(); +$components = $behatconfig->get_components_contexts(); if ($components) { foreach ($components as $component => $filepath) { // TODO Use a class static attribute instead of the class name. diff --git a/admin/tool/behat/tests/manager_test.php b/admin/tool/behat/tests/manager_test.php index 0f24f38621dd3..ae052e5cbc754 100644 --- a/admin/tool/behat/tests/manager_test.php +++ b/admin/tool/behat/tests/manager_test.php @@ -70,6 +70,7 @@ public function test_merge_configs() { ); $array = testable_behat_config_manager::merge_config($array1, $array2); + $this->assertDebuggingCalled("Use of merge_config is deprecated, please see behat_config_util"); // Overrides are applied. $this->assertEquals('OVERRIDDEN1', $array['simple']); @@ -95,6 +96,7 @@ public function test_merge_configs() { ); $array = testable_behat_config_manager::merge_config($array1, $array2); + $this->assertDebuggingCalled("Use of merge_config is deprecated, please see behat_config_util"); // Overrides applied. $this->assertNotEmpty($array['simple']); @@ -138,6 +140,7 @@ public function test_config_file_contents() { ); $contents = testable_behat_config_manager::get_config_file_contents($features, $stepsdefinitions); + $this->assertDebuggingCalled("Use of get_config_file_contents is deprecated, please see behat_config_util"); // YAML decides when is is necessary to wrap strings between single quotes, so not controlled // values like paths should not be asserted including the key name as they would depend on the diff --git a/admin/tool/behat/tests/manager_util_test.php b/admin/tool/behat/tests/manager_util_test.php new file mode 100644 index 0000000000000..9bc67fbe7a043 --- /dev/null +++ b/admin/tool/behat/tests/manager_util_test.php @@ -0,0 +1,394 @@ +. + +/** + * Unit tests for behat manager. + * + * @package tool_behat + * @copyright 2016 Rajesh Taneja + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->dirroot . '/' . $CFG->admin .'/tool/behat/locallib.php'); +require_once($CFG->libdir . '/behat/classes/util.php'); +require_once($CFG->libdir . '/behat/classes/behat_config_manager.php'); + +/** + * Behat manager tests. + * + * @package tool_behat + * @copyright 2016 Rajesh Taneja + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tool_behat_manager_util_testcase extends advanced_testcase { + + /** + * @var array core features. + */ + private $corefeatures = array( + 'feedback_editpdf_behat_test1' => '/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature', + 'feedback_file_behat_test2' => "C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature", + 'moodle_login_behat_test3' => "C:\\test\\moodle/login/tests/behat/behat_test3.feature", + ); + + + /** + * @var array theme features. + */ + private $themefeatures = array( + 'behat_themetest1_core_behat_tests_testtheme_theme' => '/test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature', + 'behat_themetest2_mod_assign_behat_tests_testtheme_theme' => "C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature", + 'behat_themetest3_behat_tests_testtheme_theme_moodle' => "C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature", + ); + + /** + * @var array core contexts. + */ + private $corecontexts = array( + 'behat_context1' => '/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_context1.php', + 'behat_context2' => "C:\\test\\moodle\\blocks\\comments\\tests\\behat\\behat_context2.php", + 'behat_context3' => "C:\\test\\moodle/lib/editor/atto/tests/behat/behat_context3.php", + ); + + /** + * @var array Theme contexts for test. + */ + private $themecontexts = array( + 'behat_theme_testtheme_behat_context1' => + '/test/moodle/theme/testtheme/tests/behat/mod_assign/behat_theme_testtheme_behat_context1.php', + 'behat_theme_testtheme_behat_context2' => + "C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\block_comments\\behat_theme_testtheme_behat_context2.php", + 'behat_theme_testtheme_behat_context3' => + "C:\\test\\moodle/theme/testtheme/tests/behat/editor_atto/behat_theme_testtheme_behat_context3.php" + ); + + /** + * Keep instance of behat_config_util mock object. + * + * @var null + */ + private $behatconfigutil = null; + + /** + * Test setup. + */ + public function setUp() { + $this->resetAfterTest(true); + $mockbuilder = $this->getMockBuilder('behat_config_util'); + $mockbuilder->setMethods(array('get_behat_features_for_theme', 'get_behat_contexts_for_theme', + 'get_list_of_themes', 'get_overridden_theme_contexts')); + + $this->behatconfigutil = $mockbuilder->getMock(); + + // List of themes is const for test. + $this->behatconfigutil->expects($this->any()) + ->method('get_list_of_themes') + ->will($this->returnValue(array('testtheme'))); + + $this->behatconfigutil->expects($this->any()) + ->method('get_behat_contexts_for_theme') + ->with($this->equalTo('testtheme')) + ->will($this->returnValue(array_keys($this->themecontexts))); + + } + + /** + * Behat config for single run. + * + */ + public function test_get_config_file_contents_with_single_run() { + global $CFG; + + $CFG->behat_wwwroot = 'http://example.com/behat'; + + $behatconfigutil = $this->behatconfigutil; + + // No theme feature exists. + $behatconfigutil->expects($this->once()) + ->method('get_behat_features_for_theme') + ->with($this->anything()) + ->will($this->returnValue(array(array(), array()))); + + $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts); + + $expectedconfigwithfeatures = "default: + formatters: + moodle_progress: + output_styles: + comment: + - magenta + suites: + default: + paths: + - /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature + - 'C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature' + - 'C:\\test\\moodle/login/tests/behat/behat_test3.feature' + contexts: + - behat_context1 + - behat_context2 + - behat_context3 + testtheme: + paths: { } + contexts: + - behat_theme_testtheme_behat_context1 + - behat_theme_testtheme_behat_context2 + - behat_theme_testtheme_behat_context3 + extensions: + Behat\\MinkExtension: + base_url: 'http://example.com/behat' + goutte: null + selenium2: + wd_host: 'http://localhost:4444/wd/hub' +"; + + $this->assertContains($expectedconfigwithfeatures, $config); + + $expectedstepdefinitions = "steps_definitions: + behat_context1: /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_context1.php + behat_context2: 'C:\\test\\moodle\\blocks\\comments\\tests\\behat\\behat_context2.php' + behat_context3: 'C:\\test\\moodle/lib/editor/atto/tests/behat/behat_context3.php' +"; + $this->assertContains($expectedstepdefinitions, $config); + } + + /** + * Behat config for parallel run. + */ + public function test_get_config_file_contents_with_parallel_run() { + global $CFG; + + $CFG->behat_wwwroot = 'http://example.com/behat'; + $behatconfigutil = $this->behatconfigutil; + + // No theme feature exists. + $behatconfigutil->expects($this->any()) + ->method('get_behat_features_for_theme') + ->with($this->anything()) + ->will($this->returnValue(array(array(), array()))); + + $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts, '', 3, 1); + + // First run. + $this->assertContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature', + $config); + $this->assertNotContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature', + $config); + $this->assertNotContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature', + $config); + + // Second run. + $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts, '', 3, 2); + + $this->assertNotContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature', + $config); + $this->assertContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature', + $config); + $this->assertNotContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature', + $config); + + $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts, '', 3, 3); + + $this->assertNotContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature', + $config); + $this->assertNotContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature', + $config); + $this->assertContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature', + $config); + } + + /** + * Behat config with theme features. + */ + public function test_get_config_file_contents_with_theme_features() { + global $CFG; + + $behatconfigutil = $this->behatconfigutil; + + $suitefeatures = array_merge($this->corefeatures, $this->themefeatures); + $themefeatures = $this->themefeatures; + $behatconfigutil->expects($this->once()) + ->method('get_behat_features_for_theme') + ->with($this->equalTo('testtheme')) + ->will($this->returnValue(array(array(), $themefeatures))); + + $behatconfigutil->expects($this->once()) + ->method('get_behat_contexts_for_theme') + ->with($this->equalTo('testtheme')) + ->will($this->returnValue(array_keys($this->themecontexts))); + + $behatconfigutil->expects($this->once()) + ->method('get_overridden_theme_contexts') + ->will($this->returnValue($this->themecontexts)); + $behatconfigutil->set_theme_suite_to_include_core_features(true); + + $CFG->behat_wwwroot = 'http://example.com/behat'; + $config = $behatconfigutil->get_config_file_contents($suitefeatures, $this->corecontexts); + + $expectedconfigwithfeatures = "default: + formatters: + moodle_progress: + output_styles: + comment: + - magenta + suites: + default: + paths: + - /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature + - 'C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature' + - 'C:\\test\\moodle/login/tests/behat/behat_test3.feature' + contexts: + - behat_context1 + - behat_context2 + - behat_context3 + testtheme: + paths: + - /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature + - 'C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature' + - 'C:\\test\\moodle/login/tests/behat/behat_test3.feature' + - /test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature + - 'C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature' + - 'C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature' + contexts: + - behat_theme_testtheme_behat_context1 + - behat_theme_testtheme_behat_context2 + - behat_theme_testtheme_behat_context3 + extensions: + Behat\\MinkExtension: + base_url: 'http://example.com/behat' + goutte: null + selenium2: + wd_host: 'http://localhost:4444/wd/hub' +"; + $this->assertContains($expectedconfigwithfeatures, $config); + + $expectedstepdefinitions = "steps_definitions: + behat_context1: /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_context1.php + behat_context2: 'C:\\test\\moodle\\blocks\\comments\\tests\\behat\\behat_context2.php' + behat_context3: 'C:\\test\\moodle/lib/editor/atto/tests/behat/behat_context3.php' + behat_theme_testtheme_behat_context1: /test/moodle/theme/testtheme/tests/behat/mod_assign/behat_theme_testtheme_behat_context1.php + behat_theme_testtheme_behat_context2: 'C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\block_comments\\behat_theme_testtheme_behat_context2.php' + behat_theme_testtheme_behat_context3: 'C:\\test\\moodle/theme/testtheme/tests/behat/editor_atto/behat_theme_testtheme_behat_context3.php'"; + + $this->assertContains($expectedstepdefinitions, $config); + } + + /** + * Behat config for parallel run. + */ + public function test_get_config_file_contents_with_theme_and_parallel_run() { + global $CFG; + + $CFG->behat_wwwroot = 'http://example.com/behat'; + + $behatconfigutil = $this->behatconfigutil; + + $features = array_merge($this->corefeatures, $this->themefeatures); + $themefeatures = $this->themefeatures; + $behatconfigutil->expects($this->atLeastOnce()) + ->method('get_behat_features_for_theme') + ->with($this->equalTo('testtheme')) + ->will($this->returnValue(array(array(), $themefeatures))); + + $behatconfigutil->expects($this->atLeastOnce()) + ->method('get_behat_contexts_for_theme') + ->with($this->equalTo('testtheme')) + ->will($this->returnValue(array_keys($this->themecontexts))); + + $CFG->behat_wwwroot = 'http://example.com/behat'; + + $behatconfigutil->set_theme_suite_to_include_core_features(false); + + $config = $behatconfigutil->get_config_file_contents($features, $this->themecontexts, '', 3, 1); + + // First run. + $this->assertContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature', + $config); + $this->assertNotContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature', + $config); + $this->assertNotContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature', + $config); + // Theme suite features. + $this->assertContains('/test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature', + $config); + $this->assertNotContains('C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature', + $config); + $this->assertNotContains('C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature', + $config); + + // Second run. + $config = $behatconfigutil->get_config_file_contents($features, $this->themecontexts, '', 3, 2); + $this->assertNotContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature', + $config); + $this->assertContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature', + $config); + $this->assertNotContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature', + $config); + // Theme suite features. + $this->assertNotContains('/test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature', + $config); + $this->assertContains('C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature', + $config); + $this->assertNotContains('C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature', + $config); + } + + /** + * Test if clean features key and path is returned. + * @dataProvider clean_features_path_list + */ + public function test_get_clean_feature_key_and_path($featurepath, $key, $cleanfeaturepath) { + global $CFG; + + // This is a hack so directory name is correctly detected in tests. + $CFG->dirroot = 'C:'; + + $behatconfigutil = $this->behatconfigutil; + // Fix expected directory path for OS. + $cleanfeaturepath = str_replace('\\', DIRECTORY_SEPARATOR, $cleanfeaturepath); + $cleanfeaturepath = str_replace('/', DIRECTORY_SEPARATOR, $cleanfeaturepath); + + if (testing_is_cygwin()) { + $featurepath = str_replace('\\', '/', $cleanfeaturepath); + } + + list($retkey, $retcleanfeaturepath) = $behatconfigutil->get_clean_feature_key_and_path($featurepath); + + $this->assertEquals($key, $retkey); + $this->assertEquals($cleanfeaturepath, $retcleanfeaturepath); + } + + public function clean_features_path_list() { + return array( + ['/home/test/this/that/test/behat/mod_assign.feature', 'mod_assign_behat_test_that_this_test', '/home/test/this/that/test/behat/mod_assign.feature'], + ['/home/this/that/test/behat/mod_assign.feature', 'mod_assign_behat_test_that_this_home', '/home/this/that/test/behat/mod_assign.feature'], + ['/home/that/test/behat/mod_assign.feature', 'mod_assign_behat_test_that_home', '/home/that/test/behat/mod_assign.feature'], + ['/home/test/behat/mod_assign.feature', 'mod_assign_behat_test_home', '/home/test/behat/mod_assign.feature'], + ['mod_assign.feature', 'mod_assign', 'mod_assign.feature'], + ['C:\test\this\that\test\behat\mod_assign.feature', 'mod_assign_behat_test_that_this_test', 'C:\test\this\that\test\behat\mod_assign.feature'], + ['C:\this\that\test\behat\mod_assign.feature', 'mod_assign_behat_test_that_this', 'C:\this\that\test\behat\mod_assign.feature'], + ['C:\that\test\behat\mod_assign.feature', 'mod_assign_behat_test_that', 'C:\that\test\behat\mod_assign.feature'], + ['C:\test\behat\mod_assign.feature', 'mod_assign_behat_test', 'C:\test\behat\mod_assign.feature'], + ['C:\mod_assign.feature', 'mod_assign', 'C:\mod_assign.feature'], + ); + } +} +// @codeCoverageIgnoreEnd diff --git a/lib/behat/classes/behat_config_manager.php b/lib/behat/classes/behat_config_manager.php index 54049f5aeb2ef..6b237f2a2041b 100644 --- a/lib/behat/classes/behat_config_manager.php +++ b/lib/behat/classes/behat_config_manager.php @@ -25,9 +25,7 @@ defined('MOODLE_INTERNAL') || die(); -require_once(__DIR__ . '/../lib.php'); -require_once(__DIR__ . '/behat_command.php'); -require_once(__DIR__ . '/../../testing/classes/tests_finder.php'); +require_once(__DIR__ . '/behat_config_util.php'); /** * Behat configuration manager @@ -47,6 +45,24 @@ class behat_config_manager { */ public static $autoprofileconversion = false; + /** + * @var behat_config_util keep object of behat_config_util for use. + */ + public static $behatconfigutil = null; + + /** + * Returns behat_config_util. + * + * @return behat_config_util + */ + private static function get_behat_config_util() { + if (!self::$behatconfigutil) { + self::$behatconfigutil = new behat_config_util(); + } + + return self::$behatconfigutil; + } + /** * Updates a config file * @@ -59,9 +75,14 @@ class behat_config_manager { * @param string $component Restricts the obtained steps definitions to the specified component * @param string $testsrunner If the config file will be used to run tests * @param string $tags features files including tags. + * @param bool $themesuitewithallfeatures if only theme specific features need to be included in the suite. + * @param int $parallelruns number of parallel runs. + * @param int $run current run for which config needs to be updated. * @return void */ - public static function update_config_file($component = '', $testsrunner = true, $tags = '') { + public static function update_config_file($component = '', $testsrunner = true, $tags = '', + $themesuitewithallfeatures = false, $parallelruns = 0, $run = 0) { + global $CFG; // Behat must have a separate behat.yml to have access to the whole set of features and steps definitions. @@ -72,49 +93,36 @@ public static function update_config_file($component = '', $testsrunner = true, $configfilepath = self::get_steps_list_config_filepath(); } - // Gets all the components with features. - $features = array(); - $components = tests_finder::get_components_with_tests('features'); - if ($components) { - foreach ($components as $componentname => $path) { - $path = self::clean_path($path) . self::get_behat_tests_path(); - if (empty($featurespaths[$path]) && file_exists($path)) { - - // Standarizes separator (some dirs. comes with OS-dependant separator). - $uniquekey = str_replace('\\', '/', $path); - $featurespaths[$uniquekey] = $path; - } - } - foreach ($featurespaths as $path) { - $additional = glob("$path/*.feature"); - $features = array_merge($features, $additional); - } - } + $behatconfigutil = self::get_behat_config_util(); + $behatconfigutil->set_theme_suite_to_include_core_features($themesuitewithallfeatures); + $behatconfigutil->set_tag_for_feature_filter($tags); - // Optionally include features from additional directories. - if (!empty($CFG->behat_additionalfeatures)) { - $features = array_merge($features, array_map("realpath", $CFG->behat_additionalfeatures)); + // Gets all the components with features, if running the tests otherwise not required. + $features = array(); + if ($testsrunner) { + $features = $behatconfigutil->get_components_features(); } // Gets all the components with steps definitions. - $stepsdefinitions = array(); - $steps = self::get_components_steps_definitions(); - if ($steps) { - foreach ($steps as $key => $filepath) { - if ($component == '' || $component === $key) { - $stepsdefinitions[$key] = $filepath; - } - } - } - + $stepsdefinitions = $behatconfigutil->get_components_contexts($component); // We don't want the deprecated steps definitions here. if (!$testsrunner) { unset($stepsdefinitions['behat_deprecated']); } + // Get current run. + if (empty($run) && ($run !== false) && !empty($CFG->behatrunprocess)) { + $run = $CFG->behatrunprocess; + } + + // Get number of parallel runs if not passed. + if (empty($parallelruns) && ($parallelruns !== false)) { + $parallelruns = self::get_parallel_test_runs(); + } + // Behat config file specifing the main context class, // the required Behat extensions and Moodle test wwwroot. - $contents = self::get_config_file_contents(self::get_features_with_tags($features, $tags), $stepsdefinitions); + $contents = $behatconfigutil->get_config_file_contents($features, $stepsdefinitions, $tags, $parallelruns, $run); // Stores the file. if (!file_put_contents($configfilepath, $contents)) { @@ -129,55 +137,13 @@ public static function update_config_file($component = '', $testsrunner = true, * @param array $features set of feature files. * @param string $tags list of tags (currently support && only.) * @return array filtered list of feature files with tags. + * @deprecated since 3.2 MDL-55072 - please use behat_config_util.php + * @todo MDL-55365 This will be deleted in Moodle 3.6. */ public static function get_features_with_tags($features, $tags) { - if (empty($tags)) { - return $features; - } - $newfeaturelist = array(); - // Split tags in and and or. - $tags = explode('&&', $tags); - $andtags = array(); - $ortags = array(); - foreach ($tags as $tag) { - // Explode all tags seperated by , and add it to ortags. - $ortags = array_merge($ortags, explode(',', $tag)); - // And tags will be the first one before comma(,). - $andtags[] = preg_replace('/,.*/', '', $tag); - } - foreach ($features as $featurefile) { - $contents = file_get_contents($featurefile); - $includefeature = true; - foreach ($andtags as $tag) { - // If negitive tag, then ensure it don't exist. - if (strpos($tag, '~') !== false) { - $tag = substr($tag, 1); - if ($contents && strpos($contents, $tag) !== false) { - $includefeature = false; - break; - } - } else if ($contents && strpos($contents, $tag) === false) { - $includefeature = false; - break; - } - } - - // If feature not included then check or tags. - if (!$includefeature && !empty($ortags)) { - foreach ($ortags as $tag) { - if ($contents && (strpos($tag, '~') === false) && (strpos($contents, $tag) !== false)) { - $includefeature = true; - break; - } - } - } - - if ($includefeature) { - $newfeaturelist[] = $featurefile; - } - } - return $newfeaturelist; + debugging('Use of get_features_with_tags is deprecated, please see behat_config_util', DEBUG_DEVELOPER); + return self::get_behat_config_util()->filtered_features_with_tags($features, $tags); } /** @@ -189,32 +155,14 @@ public static function get_features_with_tags($features, $tags) { * it from the steps definitions web interface * * @return array + * @deprecated since 3.2 MDL-55072 - please use behat_config_util.php + * @todo MDL-55365 This will be deleted in Moodle 3.6. */ public static function get_components_steps_definitions() { - $components = tests_finder::get_components_with_tests('stepsdefinitions'); - if (!$components) { - return false; - } - - $stepsdefinitions = array(); - foreach ($components as $componentname => $componentpath) { - $componentpath = self::clean_path($componentpath); - - if (!file_exists($componentpath . self::get_behat_tests_path())) { - continue; - } - $diriterator = new DirectoryIterator($componentpath . self::get_behat_tests_path()); - $regite = new RegexIterator($diriterator, '|behat_.*\.php$|'); - - // All behat_*.php inside behat_config_manager::get_behat_tests_path() are added as steps definitions files. - foreach ($regite as $file) { - $key = $file->getBasename('.php'); - $stepsdefinitions[$key] = $file->getPathname(); - } - } - - return $stepsdefinitions; + debugging('Use of get_components_steps_definitions is deprecated, please see behat_config_util::get_components_contexts', + DEBUG_DEVELOPER); + return self::get_behat_config_util()->get_components_contexts(); } /** @@ -361,91 +309,13 @@ public final static function create_parallel_site_links($fromrun, $torun) { * @param array $features The system feature files * @param array $stepsdefinitions The system steps definitions * @return string + * @deprecated since 3.2 MDL-55072 - please use behat_config_util.php + * @todo MDL-55365 This will be deleted in Moodle 3.6. */ protected static function get_config_file_contents($features, $stepsdefinitions) { - global $CFG; - - // We require here when we are sure behat dependencies are available. - require_once($CFG->dirroot . '/vendor/autoload.php'); - - $selenium2wdhost = array('wd_host' => 'http://localhost:4444/wd/hub'); - - $parallelruns = self::get_parallel_test_runs(); - // If parallel run, then only divide features. - if (!empty($CFG->behatrunprocess) && !empty($parallelruns)) { - // Attempt to split into weighted buckets using timing information, if available. - if ($alloc = self::profile_guided_allocate($features, max(1, $parallelruns), $CFG->behatrunprocess)) { - $features = $alloc; - } else { - // Divide the list of feature files amongst the parallel runners. - srand(crc32(floor(time() / 3600 / 24) . var_export($features, true))); - shuffle($features); - // Pull out the features for just this worker. - if (count($features)) { - $features = array_chunk($features, ceil(count($features) / max(1, $parallelruns))); - // Check if there is any feature file for this process. - if (!empty($features[$CFG->behatrunprocess - 1])) { - $features = $features[$CFG->behatrunprocess - 1]; - } else { - $features = null; - } - } - } - // Set proper selenium2 wd_host if defined. - if (!empty($CFG->behat_parallel_run[$CFG->behatrunprocess - 1]['wd_host'])) { - $selenium2wdhost = array('wd_host' => $CFG->behat_parallel_run[$CFG->behatrunprocess - 1]['wd_host']); - } - } - - // It is possible that it has no value as we don't require a full behat setup to list the step definitions. - if (empty($CFG->behat_wwwroot)) { - $CFG->behat_wwwroot = 'http://itwillnotbeused.com'; - } - - // Comments use black color, so failure path is not visible. Using color other then black/white is safer. - // https://github.com/Behat/Behat/pull/628. - $config = array( - 'default' => array( - 'formatters' => array( - 'moodle_progress' => array( - 'output_styles' => array( - 'comment' => array('magenta')) - ) - ), - 'suites' => array( - 'default' => array( - 'paths' => $features, - 'contexts' => array_keys($stepsdefinitions) - ) - ), - 'extensions' => array( - 'Behat\MinkExtension' => array( - 'base_url' => $CFG->behat_wwwroot, - 'goutte' => null, - 'selenium2' => $selenium2wdhost - ), - 'Moodle\BehatExtension' => array( - 'moodledirroot' => $CFG->dirroot, - 'steps_definitions' => $stepsdefinitions - ) - ) - ) - ); - - // In case user defined overrides respect them over our default ones. - if (!empty($CFG->behat_config)) { - foreach ($CFG->behat_config as $profile => $values) { - $config = self::merge_config($config, self::merge_behat_config($profile, $values)); - } - } - // Check for Moodle custom ones. - if (!empty($CFG->behat_profiles) && is_array($CFG->behat_profiles)) { - foreach ($CFG->behat_profiles as $profile => $values) { - $config = self::merge_config($config, self::get_behat_profile($profile, $values)); - } - } - return Symfony\Component\Yaml\Yaml::dump($config, 10, 2); + debugging('Use of get_config_file_contents is deprecated, please see behat_config_util', DEBUG_DEVELOPER); + return self::get_behat_config_util()->get_config_file_contents($features, $stepsdefinitions); } /** @@ -454,41 +324,13 @@ protected static function get_config_file_contents($features, $stepsdefinitions) * @param string $profile profile name * @param array $values values for profile * @return array + * @deprecated since 3.2 MDL-55072 - please use behat_config_util.php + * @todo MDL-55365 This will be deleted in Moodle 3.6. */ protected static function merge_behat_config($profile, $values) { - // Only add profile which are compatible with Behat 3.x - // Just check if any of Bheat 2.5 config is set. Not checking for 3.x as it might have some other configs - // Like : rerun_cache etc. - if (!isset($values['filters']['tags']) && !isset($values['extensions']['Behat\MinkExtension\Extension'])) { - return array($profile => $values); - } - - // Parse 2.5 format and get related values. - $oldconfigvalues = array(); - if (isset($values['extensions']['Behat\MinkExtension\Extension'])) { - $extensionvalues = $values['extensions']['Behat\MinkExtension\Extension']; - if (isset($extensionvalues['selenium2']['browser'])) { - $oldconfigvalues['browser'] = $extensionvalues['selenium2']['browser']; - } - if (isset($extensionvalues['selenium2']['wd_host'])) { - $oldconfigvalues['wd_host'] = $extensionvalues['selenium2']['wd_host']; - } - if (isset($extensionvalues['capabilities'])) { - $oldconfigvalues['capabilities'] = $extensionvalues['capabilities']; - } - } - if (isset($values['filters']['tags'])) { - $oldconfigvalues['tags'] = $values['filters']['tags']; - } - - if (!empty($oldconfigvalues)) { - self::$autoprofileconversion = true; - return self::get_behat_profile($profile, $oldconfigvalues); - } - - // If nothing set above then return empty array. - return array(); + debugging('Use of merge_behat_config is deprecated, please see behat_config_util', DEBUG_DEVELOPER); + self::get_behat_config_util()->get_behat_config_for_profile($profile, $values); } /** @@ -569,64 +411,8 @@ protected static function get_behat_profile($profile, $values) { */ protected static function profile_guided_allocate($features, $nbuckets, $instance) { - $behattimingfile = defined('BEHAT_FEATURE_TIMING_FILE') && - @filesize(BEHAT_FEATURE_TIMING_FILE) ? BEHAT_FEATURE_TIMING_FILE : false; - - if (!$behattimingfile || !$behattimingdata = @json_decode(file_get_contents($behattimingfile), true)) { - // No data available, fall back to relying on steps data. - $stepfile = ""; - if (defined('BEHAT_FEATURE_STEP_FILE') && BEHAT_FEATURE_STEP_FILE) { - $stepfile = BEHAT_FEATURE_STEP_FILE; - } - // We should never get this. But in case we can't do this then fall back on simple splitting. - if (empty($stepfile) || !$behattimingdata = @json_decode(file_get_contents($stepfile), true)) { - return false; - } - } - - arsort($behattimingdata); // Ensure most expensive is first. - - $realroot = realpath(__DIR__.'/../../../').'/'; - $defaultweight = array_sum($behattimingdata) / count($behattimingdata); - $weights = array_fill(0, $nbuckets, 0); - $buckets = array_fill(0, $nbuckets, array()); - $totalweight = 0; - - // Re-key the features list to match timing data. - foreach ($features as $k => $file) { - $key = str_replace($realroot, '', $file); - $features[$key] = $file; - unset($features[$k]); - if (!isset($behattimingdata[$key])) { - $behattimingdata[$key] = $defaultweight; - } - } - - // Sort features by known weights; largest ones should be allocated first. - $behattimingorder = array(); - foreach ($features as $key => $file) { - $behattimingorder[$key] = $behattimingdata[$key]; - } - arsort($behattimingorder); - - // Finally, add each feature one by one to the lightest bucket. - foreach ($behattimingorder as $key => $weight) { - $file = $features[$key]; - $lightbucket = array_search(min($weights), $weights); - $weights[$lightbucket] += $weight; - $buckets[$lightbucket][] = $file; - $totalweight += $weight; - } - - if ($totalweight && !defined('BEHAT_DISABLE_HISTOGRAM') && $instance == $nbuckets) { - echo "Bucket weightings:\n"; - foreach ($weights as $k => $weight) { - echo $k + 1 . ": " . str_repeat('*', 70 * $nbuckets * $weight / $totalweight) . PHP_EOL; - } - } - - // Return the features for this worker. - return $buckets[$instance - 1]; + debugging('Use of profile_guided_allocate is deprecated, please see behat_config_util', DEBUG_DEVELOPER); + return self::get_behat_config_util()->profile_guided_allocate($features, $nbuckets, $instance); } /** @@ -637,34 +423,13 @@ protected static function profile_guided_allocate($features, $nbuckets, $instanc * @param mixed $config The node of the default config * @param mixed $localconfig The node of the local config * @return mixed The merge result + * @deprecated since 3.2 MDL-55072 - please use behat_config_util.php + * @todo MDL-55365 This will be deleted in Moodle 3.6. */ protected static function merge_config($config, $localconfig) { - if (!is_array($config) && !is_array($localconfig)) { - return $localconfig; - } - - // Local overrides also deeper default values. - if (is_array($config) && !is_array($localconfig)) { - return $localconfig; - } - - foreach ($localconfig as $key => $value) { - - // If defaults are not as deep as local values let locals override. - if (!is_array($config)) { - unset($config); - } - - // Add the param if it doesn't exists or merge branches. - if (empty($config[$key])) { - $config[$key] = $value; - } else { - $config[$key] = self::merge_config($config[$key], $localconfig[$key]); - } - } - - return $config; + debugging('Use of merge_config is deprecated, please see behat_config_util', DEBUG_DEVELOPER); + return self::get_behat_config_util()->merge_config($config, $localconfig); } /** @@ -673,28 +438,25 @@ protected static function merge_config($config, $localconfig) { * @see tests_finder::get_all_directories_with_tests() it returns the path including /tests/ * @param string $path * @return string The string without the last /tests part + * @deprecated since 3.2 MDL-55072 - please use behat_config_util.php + * @todo MDL-55365 This will be deleted in Moodle 3.6. */ protected final static function clean_path($path) { - $path = rtrim($path, DIRECTORY_SEPARATOR); - - $parttoremove = DIRECTORY_SEPARATOR . 'tests'; - - $substr = substr($path, strlen($path) - strlen($parttoremove)); - if ($substr == $parttoremove) { - $path = substr($path, 0, strlen($path) - strlen($parttoremove)); - } - - return rtrim($path, DIRECTORY_SEPARATOR); + debugging('Use of clean_path is deprecated, please see behat_config_util', DEBUG_DEVELOPER); + return self::get_behat_config_util()->clean_path($path); } /** * The relative path where components stores their behat tests * * @return string + * @deprecated since 3.2 MDL-55072 - please use behat_config_util.php + * @todo MDL-55365 This will be deleted in Moodle 3.6. */ protected final static function get_behat_tests_path() { - return DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR . 'behat'; + debugging('Use of get_behat_tests_path is deprecated, please see behat_config_util', DEBUG_DEVELOPER); + return self::get_behat_config_util()->get_behat_tests_path(); } } diff --git a/lib/behat/classes/behat_config_util.php b/lib/behat/classes/behat_config_util.php new file mode 100644 index 0000000000000..37cef52aa8569 --- /dev/null +++ b/lib/behat/classes/behat_config_util.php @@ -0,0 +1,1228 @@ +. + +/** + * Utils to set Behat config + * + * @package core + * @copyright 2016 Rajesh Taneja + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once(__DIR__ . '/../lib.php'); +require_once(__DIR__ . '/behat_command.php'); +require_once(__DIR__ . '/../../testing/classes/tests_finder.php'); + +/** + * Behat configuration manager + * + * Creates/updates Behat config files getting tests + * and steps from Moodle codebase + * + * @package core + * @copyright 2016 Rajesh Taneja + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class behat_config_util { + + /** + * @var array list of features in core. + */ + private $features; + + /** + * @var array list of contexts in core. + */ + private $contexts; + + /** + * @var array list of theme specific contexts. + */ + private $themecontexts; + + /** + * @var array list of overridden theme contexts. + */ + private $overriddenthemescontexts; + + /** + * @var array list of components with tests. + */ + private $componentswithtests; + + /** + * @var bool keep track of theme to return suite with all core features included or not. + */ + private $themesuitewithallfeatures = false; + + /** + * @var string filter features which have tags. + */ + private $tags = ''; + + /** + * @var int number of parallel runs. + */ + private $parallelruns = 0; + + /** + * @var int current run. + */ + private $currentrun = 0; + + /** + * Set value for theme suite to include all core features. This should be used if your want all core features to be + * run with theme. + * + * @param bool $val + */ + public function set_theme_suite_to_include_core_features($val) { + $this->themesuitewithallfeatures = $val; + } + + /** + * Set the value for tags, so features which are returned will be using filtered by this. + * + * @param string $tags + */ + public function set_tag_for_feature_filter($tags) { + $this->tags = $tags; + } + + /** + * Set parallel run to be used for generating config. + * + * @param int $parallelruns number of parallel runs. + * @param int $currentrun current run + */ + public function set_parallel_run($parallelruns, $currentrun) { + + if ($parallelruns < $currentrun) { + behat_error(BEHAT_EXITCODE_REQUIREMENT, + 'Parallel runs('.$parallelruns.') should be more then current run('.$currentrun.')'); + } + + $this->parallelruns = $parallelruns; + $this->currentrun = $currentrun; + } + + /** + * Return parallel runs + * + * @return int number of parallel runs. + */ + public function get_number_of_parallel_run() { + // Get number of parallel runs if not passed. + if (empty($this->parallelruns) && ($this->parallelruns !== false)) { + $this->parallelruns = behat_config_manager::get_parallel_test_runs(); + } + + return $this->parallelruns; + } + + /** + * Return current run + * + * @return int current run. + */ + public function get_current_run() { + global $CFG; + + // Get number of parallel runs if not passed. + if (empty($this->currentrun) && ($this->currentrun !== false) && !empty($CFG->behatrunprocess)) { + $this->currentrun = $CFG->behatrunprocess; + } + + return $this->currentrun; + } + + /** + * Return list of features. + * + * @param string $tags tags. + * @return array + */ + public function get_components_features($tags = '') { + global $CFG; + + // If we already have a list created then just return that, as it's up-to-date. + // If tags are passed then it's a new filter of features we need. + if (!empty($this->features) && empty($tags)) { + return $this->features; + } + + // Gets all the components with features. + $features = array(); + $featurespaths = array(); + $components = $this->get_components_with_tests(); + + if ($components) { + foreach ($components as $componentname => $path) { + $path = $this->clean_path($path) . self::get_behat_tests_path(); + if (empty($featurespaths[$path]) && file_exists($path)) { + list($key, $featurepath) = $this->get_clean_feature_key_and_path($path); + $featurespaths[$key] = $featurepath; + } + } + foreach ($featurespaths as $path) { + $additional = glob("$path/*.feature"); + + $additionalfeatures = array(); + foreach ($additional as $featurepath) { + list($key, $path) = $this->get_clean_feature_key_and_path($featurepath); + $additionalfeatures[$key] = $path; + } + + $features = array_merge($features, $additionalfeatures); + } + } + + // Optionally include features from additional directories. + if (!empty($CFG->behat_additionalfeatures)) { + $additional = array_map("realpath", $CFG->behat_additionalfeatures); + $additionalfeatures = array(); + foreach ($additional as $featurepath) { + list($key, $path) = $this->get_clean_feature_key_and_path($featurepath); + $additionalfeatures[$key] = $path; + } + $features = array_merge($features, $additionalfeatures); + } + + $this->features = $features; + + return $this->filtered_features_with_tags($features, $tags); + } + + /** + * Return feature key for featurepath + * + * @param string $featurepath + * @return array key and featurepath. + */ + public function get_clean_feature_key_and_path($featurepath) { + global $CFG; + + // Fix directory path. + $featurepath = str_replace('\\', DIRECTORY_SEPARATOR, $featurepath); + $featurepath = str_replace('/', DIRECTORY_SEPARATOR, $featurepath); + + if (testing_is_cygwin()) { + $featurepath = str_replace('\\', '/', $featurepath); + } + + $key = basename($featurepath, '.feature'); + + // Get relative path. + $featuredirname = str_replace($CFG->dirroot . DIRECTORY_SEPARATOR , '', $featurepath); + // Get 5 levels of feature path to ensure we have a unique key. + for ($i = 0; $i < 5; $i++) { + if (($featuredirname = dirname($featuredirname)) && $featuredirname !== '.') { + if ($basename = basename($featuredirname)) { + $key .= '_' . $basename; + } + } + } + + return array($key, $featurepath); + } + + /** + * Get component contexts. + * + * @param string $component component name. + * @return array + */ + private function get_component_contexts($component) { + + if (empty($component)) { + return $this->contexts; + } + + $componentcontexts = array(); + foreach ($this->contexts as $key => $path) { + if ($component == '' || $component === $key) { + $componentcontexts[$key] = $path; + } + } + + return $componentcontexts; + } + + /** + * Gets the list of Moodle behat contexts + * + * Class name as a key and the filepath as value + * + * Externalized from update_config_file() to use + * it from the steps definitions web interface + * + * @param string $component Restricts the obtained steps definitions to the specified component + * @return array + */ + public function get_components_contexts($component = '') { + + // If we already have a list created then just return that, as it's up-to-date. + if (!empty($this->contexts)) { + return $this->get_component_contexts($component); + } + + $components = $this->get_components_with_tests(); + + $this->contexts = array(); + foreach ($components as $componentname => $componentpath) { + $componentpath = self::clean_path($componentpath); + + if (!file_exists($componentpath . self::get_behat_tests_path())) { + continue; + } + $diriterator = new DirectoryIterator($componentpath . self::get_behat_tests_path()); + $regite = new RegexIterator($diriterator, '|behat_.*\.php$|'); + + // All behat_*.php inside self::get_behat_tests_path() are added as steps definitions files. + foreach ($regite as $file) { + $key = $file->getBasename('.php'); + $this->contexts[$key] = $file->getPathname(); + } + } + + return $this->get_component_contexts($component); + } + + /** + * Behat config file specifing the main context class, + * the required Behat extensions and Moodle test wwwroot. + * + * @param array $features The system feature files + * @param array $contexts The system steps definitions + * @param string $tags filter features with specified tags. + * @param int $parallelruns number of parallel runs. + * @param int $currentrun current run for which config file is needed. + * @return string + */ + public function get_config_file_contents($features = '', $contexts = '', $tags = '', $parallelruns = 0, $currentrun = 0) { + global $CFG; + + // Set current run and parallel run. + if (!empty($parallelruns) && !empty($currentrun)) { + $this->set_parallel_run($parallelruns, $currentrun); + } + + // If tags defined then use them. This is for BC. + if (!empty($tags)) { + $this->set_tag_for_feature_filter($tags); + } + + // If features not passed then get it. Empty array means we don't need to include features. + if (empty($features) && !is_array($features)) { + $features = $this->get_components_features(); + } else { + $this->features = $features; + } + + // If stepdefinitions not passed then get the list. + if (empty($contexts)) { + $this->get_components_contexts(); + } else { + $this->contexts = $contexts; + } + + // We require here when we are sure behat dependencies are available. + require_once($CFG->dirroot . '/vendor/autoload.php'); + + $config = $this->build_config(); + + $config = $this->merge_behat_config($config); + + $config = $this->merge_behat_profiles($config); + + return Symfony\Component\Yaml\Yaml::dump($config, 10, 2); + } + + /** + * Search feature files for set of tags. + * + * @param array $features set of feature files. + * @param string $tags list of tags (currently support && only.) + * @return array filtered list of feature files with tags. + */ + public function filtered_features_with_tags($features = '', $tags = '') { + + // This is for BC. Features if not passed then we already have a list in this object. + if (empty($features)) { + $features = $this->features; + } + + // If no tags defined then return full list. + if (empty($tags) && empty($this->tags)) { + return $features; + } + + // If no tags passed by the caller, then it's already set. + if (empty($tags)) { + $tags = $this->tags; + } + + $newfeaturelist = array(); + // Split tags in and and or. + $tags = explode('&&', $tags); + $andtags = array(); + $ortags = array(); + foreach ($tags as $tag) { + // Explode all tags seperated by , and add it to ortags. + $ortags = array_merge($ortags, explode(',', $tag)); + // And tags will be the first one before comma(,). + $andtags[] = preg_replace('/,.*/', '', $tag); + } + + foreach ($features as $featurefile) { + $contents = file_get_contents($featurefile); + $includefeature = true; + foreach ($andtags as $tag) { + // If negitive tag, then ensure it don't exist. + if (strpos($tag, '~') !== false) { + $tag = substr($tag, 1); + if ($contents && strpos($contents, $tag) !== false) { + $includefeature = false; + break; + } + } else if ($contents && strpos($contents, $tag) === false) { + $includefeature = false; + break; + } + } + + // If feature not included then check or tags. + if (!$includefeature && !empty($ortags)) { + foreach ($ortags as $tag) { + if ($contents && (strpos($tag, '~') === false) && (strpos($contents, $tag) !== false)) { + $includefeature = true; + break; + } + } + } + + if ($includefeature) { + $newfeaturelist[] = $featurefile; + } + } + return $newfeaturelist; + } + + /** + * Build config for behat.yml. + * + * @param int $parallelruns how many parallel runs feature needs to be divided. + * @param int $currentrun current run for which features should be returned. + * @return array + */ + protected function build_config($parallelruns = 0, $currentrun = 0) { + global $CFG; + + if (!empty($parallelruns) && !empty($currentrun)) { + $this->set_parallel_run($parallelruns, $currentrun); + } else { + $currentrun = $this->get_current_run(); + $parallelruns = $this->get_number_of_parallel_run(); + } + + $selenium2wdhost = array('wd_host' => 'http://localhost:4444/wd/hub'); + // If parallel run, then set wd_host if specified. + if (!empty($currentrun) && !empty($parallelruns)) { + // Set proper selenium2 wd_host if defined. + if (!empty($CFG->behat_parallel_run[$currentrun - 1]['wd_host'])) { + $selenium2wdhost = array('wd_host' => $CFG->behat_parallel_run[$currentrun - 1]['wd_host']); + } + } + + // It is possible that it has no value as we don't require a full behat setup to list the step definitions. + if (empty($CFG->behat_wwwroot)) { + $CFG->behat_wwwroot = 'http://itwillnotbeused.com'; + } + + $suites = $this->get_behat_suites($parallelruns, $currentrun); + + $overriddenthemescontexts = $this->get_overridden_theme_contexts(); + if (!empty($overriddenthemescontexts)) { + $allcontexts = array_merge($this->contexts, $overriddenthemescontexts); + } else { + $allcontexts = $this->contexts; + } + + // Comments use black color, so failure path is not visible. Using color other then black/white is safer. + // https://github.com/Behat/Behat/pull/628. + $config = array( + 'default' => array( + 'formatters' => array( + 'moodle_progress' => array( + 'output_styles' => array( + 'comment' => array('magenta')) + ) + ), + 'suites' => $suites, + 'extensions' => array( + 'Behat\MinkExtension' => array( + 'base_url' => $CFG->behat_wwwroot, + 'goutte' => null, + 'selenium2' => $selenium2wdhost + ), + 'Moodle\BehatExtension' => array( + 'moodledirroot' => $CFG->dirroot, + 'steps_definitions' => $allcontexts, + ) + ) + ) + ); + + return $config; + } + + /** + * Divide features between the runs and return list. + * + * @param array $features list of features to be divided. + * @param int $parallelruns how many parallel runs feature needs to be divided. + * @param int $currentrun current run for which features should be returned. + * @return array + */ + protected function get_features_for_the_run($features, $parallelruns, $currentrun) { + + // If no features are passed then just return. + if (empty($features)) { + return $features; + } + + $allocatedfeatures = $features; + + // If parallel run, then only divide features. + if (!empty($currentrun) && !empty($parallelruns)) { + // Attempt to split into weighted buckets using timing information, if available. + if ($alloc = $this->profile_guided_allocate($features, max(1, $parallelruns), $currentrun)) { + $allocatedfeatures = $alloc; + } else { + // Divide the list of feature files amongst the parallel runners. + // Pull out the features for just this worker. + if (count($features)) { + $features = array_chunk($features, ceil(count($features) / max(1, $parallelruns))); + + // Check if there is any feature file for this process. + if (!empty($features[$currentrun - 1])) { + $allocatedfeatures = $features[$currentrun - 1]; + } else { + $allocatedfeatures = array(); + } + } + } + } + + return $allocatedfeatures; + } + + /** + * Parse $CFG->behat_profile and return the array with required config structure for behat.yml. + * + * $CFG->behat_profiles = array( + * 'profile' = array( + * 'browser' => 'firefox', + * 'tags' => '@javascript', + * 'wd_host' => 'http://127.0.0.1:4444/wd/hub', + * 'capabilities' => array( + * 'platform' => 'Linux', + * 'version' => 44 + * ) + * ) + * ); + * + * @param string $profile profile name + * @param array $values values for profile. + * @return array + */ + protected function get_behat_profile($profile, $values) { + // Values should be an array. + if (!is_array($values)) { + return array(); + } + + // Check suite values. + $behatprofilesuites = array(); + // Fill tags information. + if (isset($values['tags'])) { + $behatprofilesuites = array( + 'suites' => array( + 'default' => array( + 'filters' => array( + 'tags' => $values['tags'], + ) + ) + ) + ); + } + + // Selenium2 config values. + $behatprofileextension = array(); + $seleniumconfig = array(); + if (isset($values['browser'])) { + $seleniumconfig['browser'] = $values['browser']; + } + if (isset($values['wd_host'])) { + $seleniumconfig['wd_host'] = $values['wd_host']; + } + if (isset($values['capabilities'])) { + $seleniumconfig['capabilities'] = $values['capabilities']; + } + if (!empty($seleniumconfig)) { + $behatprofileextension = array( + 'extensions' => array( + 'Behat\MinkExtension' => array( + 'selenium2' => $seleniumconfig, + ) + ) + ); + } + + return array($profile => array_merge($behatprofilesuites, $behatprofileextension)); + } + + /** + * Attempt to split feature list into fairish buckets using timing information, if available. + * Simply add each one to lightest buckets until all files allocated. + * PGA = Profile Guided Allocation. I made it up just now. + * CAUTION: workers must agree on allocation, do not be random anywhere! + * + * @param array $features Behat feature files array + * @param int $nbuckets Number of buckets to divide into + * @param int $instance Index number of this instance + * @return array|bool Feature files array, sorted into allocations + */ + public function profile_guided_allocate($features, $nbuckets, $instance) { + + $behattimingfile = defined('BEHAT_FEATURE_TIMING_FILE') && + @filesize(BEHAT_FEATURE_TIMING_FILE) ? BEHAT_FEATURE_TIMING_FILE : false; + + if (!$behattimingfile || !$behattimingdata = @json_decode(file_get_contents($behattimingfile), true)) { + // No data available, fall back to relying on steps data. + $stepfile = ""; + if (defined('BEHAT_FEATURE_STEP_FILE') && BEHAT_FEATURE_STEP_FILE) { + $stepfile = BEHAT_FEATURE_STEP_FILE; + } + // We should never get this. But in case we can't do this then fall back on simple splitting. + if (empty($stepfile) || !$behattimingdata = @json_decode(file_get_contents($stepfile), true)) { + return false; + } + } + + arsort($behattimingdata); // Ensure most expensive is first. + + $realroot = realpath(__DIR__.'/../../../').'/'; + $defaultweight = array_sum($behattimingdata) / count($behattimingdata); + $weights = array_fill(0, $nbuckets, 0); + $buckets = array_fill(0, $nbuckets, array()); + $totalweight = 0; + + // Re-key the features list to match timing data. + foreach ($features as $k => $file) { + $key = str_replace($realroot, '', $file); + $features[$key] = $file; + unset($features[$k]); + if (!isset($behattimingdata[$key])) { + $behattimingdata[$key] = $defaultweight; + } + } + + // Sort features by known weights; largest ones should be allocated first. + $behattimingorder = array(); + foreach ($features as $key => $file) { + $behattimingorder[$key] = $behattimingdata[$key]; + } + arsort($behattimingorder); + + // Finally, add each feature one by one to the lightest bucket. + foreach ($behattimingorder as $key => $weight) { + $file = $features[$key]; + $lightbucket = array_search(min($weights), $weights); + $weights[$lightbucket] += $weight; + $buckets[$lightbucket][] = $file; + $totalweight += $weight; + } + + if ($totalweight && !defined('BEHAT_DISABLE_HISTOGRAM') && $instance == $nbuckets && !defined('PHPUNIT_TEST')) { + echo "Bucket weightings:\n"; + foreach ($weights as $k => $weight) { + echo $k + 1 . ": " . str_repeat('*', 70 * $nbuckets * $weight / $totalweight) . PHP_EOL; + } + } + + // Return the features for this worker. + return $buckets[$instance - 1]; + } + + /** + * Overrides default config with local config values + * + * array_merge does not merge completely the array's values + * + * @param mixed $config The node of the default config + * @param mixed $localconfig The node of the local config + * @return mixed The merge result + */ + public function merge_config($config, $localconfig) { + + if (!is_array($config) && !is_array($localconfig)) { + return $localconfig; + } + + // Local overrides also deeper default values. + if (is_array($config) && !is_array($localconfig)) { + return $localconfig; + } + + foreach ($localconfig as $key => $value) { + + // If defaults are not as deep as local values let locals override. + if (!is_array($config)) { + unset($config); + } + + // Add the param if it doesn't exists or merge branches. + if (empty($config[$key])) { + $config[$key] = $value; + } else { + $config[$key] = $this->merge_config($config[$key], $localconfig[$key]); + } + } + + return $config; + } + + /** + * Merges $CFG->behat_config with the one passed. + * + * @param array $config existing config. + * @return array merged config with $CFG->behat_config + */ + public function merge_behat_config($config) { + global $CFG; + + // In case user defined overrides respect them over our default ones. + if (!empty($CFG->behat_config)) { + foreach ($CFG->behat_config as $profile => $values) { + $config = $this->merge_config($config, $this->get_behat_config_for_profile($profile, $values)); + } + } + + return $config; + } + + /** + * Parse $CFG->behat_config and return the array with required config structure for behat.yml + * + * @param string $profile profile name + * @param array $values values for profile + * @return array + */ + public function get_behat_config_for_profile($profile, $values) { + // Only add profile which are compatible with Behat 3.x + // Just check if any of Bheat 2.5 config is set. Not checking for 3.x as it might have some other configs + // Like : rerun_cache etc. + if (!isset($values['filters']['tags']) && !isset($values['extensions']['Behat\MinkExtension\Extension'])) { + return array($profile => $values); + } + + // Parse 2.5 format and get related values. + $oldconfigvalues = array(); + if (isset($values['extensions']['Behat\MinkExtension\Extension'])) { + $extensionvalues = $values['extensions']['Behat\MinkExtension\Extension']; + if (isset($extensionvalues['selenium2']['browser'])) { + $oldconfigvalues['browser'] = $extensionvalues['selenium2']['browser']; + } + if (isset($extensionvalues['selenium2']['wd_host'])) { + $oldconfigvalues['wd_host'] = $extensionvalues['selenium2']['wd_host']; + } + if (isset($extensionvalues['capabilities'])) { + $oldconfigvalues['capabilities'] = $extensionvalues['capabilities']; + } + } + + if (isset($values['filters']['tags'])) { + $oldconfigvalues['tags'] = $values['filters']['tags']; + } + + if (!empty($oldconfigvalues)) { + behat_config_manager::$autoprofileconversion = true; + return $this->get_behat_profile($profile, $oldconfigvalues); + } + + // If nothing set above then return empty array. + return array(); + } + + /** + * Merges $CFG->behat_profiles with the one passed. + * + * @param array $config existing config. + * @return array merged config with $CFG->behat_profiles + */ + public function merge_behat_profiles($config) { + global $CFG; + + // Check for Moodle custom ones. + if (!empty($CFG->behat_profiles) && is_array($CFG->behat_profiles)) { + foreach ($CFG->behat_profiles as $profile => $values) { + $config = $this->merge_config($config, $this->get_behat_profile($profile, $values)); + } + } + + return $config; + } + + /** + * Cleans the path returned by get_components_with_tests() to standarize it + * + * @see tests_finder::get_all_directories_with_tests() it returns the path including /tests/ + * @param string $path + * @return string The string without the last /tests part + */ + public final function clean_path($path) { + + $path = rtrim($path, DIRECTORY_SEPARATOR); + + $parttoremove = DIRECTORY_SEPARATOR . 'tests'; + + $substr = substr($path, strlen($path) - strlen($parttoremove)); + if ($substr == $parttoremove) { + $path = substr($path, 0, strlen($path) - strlen($parttoremove)); + } + + return rtrim($path, DIRECTORY_SEPARATOR); + } + + /** + * The relative path where components stores their behat tests + * + * @return string + */ + public static final function get_behat_tests_path() { + return DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR . 'behat'; + } + + /** + * Return context name of behat_theme selector to use. + * + * @param string $themename name of the theme. + * @param bool $includeclass if class should be included. + * @return string + */ + public static final function get_behat_theme_selector_override_classname($themename, $includeclass = false) { + global $CFG; + + $overridebehatclassname = 'behat_theme_'.$themename.'_behat_selectors'; + + if ($includeclass) { + $themeoverrideselector = $CFG->dirroot . DIRECTORY_SEPARATOR . 'theme' . DIRECTORY_SEPARATOR . $themename . + self::get_behat_tests_path() . DIRECTORY_SEPARATOR . $overridebehatclassname . '.php'; + + if (file_exists($themeoverrideselector)) { + require_once($themeoverrideselector); + } + } + + return $overridebehatclassname; + } + + /** + * List of components which contain behat context or features. + * + * @return array + */ + private function get_components_with_tests() { + if (empty($this->componentswithtests)) { + $this->componentswithtests = tests_finder::get_components_with_tests('behat'); + } + + return $this->componentswithtests; + } + + /** + * Remove list of blacklisted features from the feature list. + * + * @param array $features list of original features. + * @param array|string $blacklist list of features which needs to be removed. + * @return array features - blacklisted features. + */ + protected function remove_blacklisted_features_from_list($features, $blacklist) { + + // If no blacklist passed then return. + if (empty($blacklist)) { + return $features; + } + + // If there is no feature in suite then just return what was passed. + if (empty($features)) { + return $features; + } + + if (!is_array($blacklist)) { + $blacklist = array($blacklist); + } + + // Remove blacklisted features. + foreach ($blacklist as $blacklistpath) { + + list($key, $featurepath) = $this->get_clean_feature_key_and_path($blacklistpath); + + if (isset($features[$key])) { + $features[$key] = null; + unset($features[$key]); + } else if (empty($this->tags)) { + // If tags not set, then ensure we have a blacklisted feature in core. Else, let user know that + // blacklisted feature is invalid. + $featurestocheck = $this->get_components_features(); + if (!isset($featurestocheck[$key]) && !defined('PHPUNIT_TEST')) { + behat_error(BEHAT_EXITCODE_REQUIREMENT, 'Blacklisted feature "' . $blacklistpath . '" not found.'); + } + } + } + + return $features; + } + + /** + * Return list of behat suites. Multiple suites are returned if theme + * overrides default step definitions/features. + * + * @param int $parallelruns number of parallel runs + * @param int $currentrun current run. + * @return array list of suites. + */ + protected function get_behat_suites($parallelruns = 0, $currentrun = 0) { + $features = $this->get_components_features(); + $contexts = $this->get_components_contexts(); + + // Get number of parallel runs and current run. + if (!empty($parallelruns) && !empty($currentrun)) { + $this->set_parallel_run($parallelruns, $currentrun); + } else { + $parallelruns = $this->get_number_of_parallel_run(); + $currentrun = $this->get_current_run();; + } + + $blacklistedfeatures = array(); + $themefeatures = array(); + $themecontexts = array(); + + $themes = $this->get_list_of_themes(); + + // Create list of theme suite features and contexts. + foreach ($themes as $theme) { + // Get theme features. + list($blacklistedfeatures[$theme], $themefeatures[$theme]) = $this->get_behat_features_for_theme($theme); + + $themecontexts[$theme] = $this->get_behat_contexts_for_theme($theme); + } + + // Remove list of theme features for default suite, as default suite should not run theme specific features. + foreach ($themefeatures as $removethemefeatures) { + $features = $this->remove_blacklisted_features_from_list($features, $removethemefeatures); + } + + // Return sub-set of features if parallel run. + $featuresforrun = $this->get_features_for_the_run($features, $parallelruns, $currentrun); + + // Default suite. + $suites = array( + 'default' => array( + 'paths' => array_values($featuresforrun), + 'contexts' => array_keys($contexts), + ) + ); + + // Set suite for each theme. + foreach ($themes as $theme) { + // Get list of features which will be included in theme. + // If theme suite with all features is set, then we want all core features to be part of theme suite. + if ($this->themesuitewithallfeatures) { + // If there is no theme specific feature. Then it's just core features. + if (empty($themefeatures[$theme])) { + $themesuitefeatures = $features; + } else { + $themesuitefeatures = array_merge($features, $themefeatures[$theme]); + } + } else { + $themesuitefeatures = $themefeatures[$theme]; + } + + // Remove blacklisted features. + $themesuitefeatures = $this->remove_blacklisted_features_from_list($themesuitefeatures, $blacklistedfeatures[$theme]); + + // Return sub-set of features if parallel run. + $themesuitefeatures = $this->get_features_for_the_run($themesuitefeatures, $parallelruns, $currentrun); + + // Add suite no matter what. If there is no feature in suite then it will just exist successfully with no + // scenarios. But if we don't set this then the user has to know which run doesn't have suite and which run do. + $suites = array_merge($suites, array( + $theme => array( + 'paths' => array_values($themesuitefeatures), + 'contexts' => array_values($themecontexts[$theme]), + ) + )); + } + + return $suites; + } + + /** + * Return list of themes which can be set in moodle. + * + * @return array list of themes with tests. + */ + protected function get_list_of_themes() { + $selectablethemes = array(); + + // Get all themes installed on site. + $themes = core_component::get_plugin_list('theme'); + ksort($themes); + + foreach ($themes as $themename => $themedir) { + // Load the theme config. + try { + $theme = theme_config::load($themename); + } catch (Exception $e) { + // Bad theme, just skip it for now. + continue; + } + if ($themename !== $theme->name) { + // Obsoleted or broken theme, just skip for now. + continue; + } + if ($theme->hidefromselector) { + // The theme doesn't want to be shown in the theme selector and as theme + // designer mode is switched off we will respect that decision. + continue; + } + if ($themename == theme_config::DEFAULT_THEME) { + // Don't include default theme, as default suite will be running with this theme. + continue; + } + $selectablethemes[] = $themename; + } + + return $selectablethemes; + } + + /** + * Returns all the directories having overridden tests. + * + * @param string $theme name of theme + * @param string $testtype The kind of test we are looking for + * @return array all directories having tests + */ + protected function get_test_directories_overridden_for_theme($theme, $testtype) { + global $CFG; + + $testtypes = array( + 'contexts' => '|behat_.*\.php$|', + 'features' => '|.*\.feature$|', + ); + $themetestdir = "/theme/" . $theme . '/tests/behat'; + $themetestdirfullpath = $CFG->dirroot . $themetestdir; + + // If test directory doesn't exist then return. + if (!is_dir($themetestdirfullpath)) { + return array(); + } + + $directoriestosearch = glob($themetestdirfullpath . DIRECTORY_SEPARATOR . '*' , GLOB_ONLYDIR); + + // Include theme directory to find tests. + $dirs[realpath($themetestdirfullpath)] = trim(str_replace('/', '_', $themetestdir), '_'); + + // Search for tests in valid directories. + foreach ($directoriestosearch as $dir) { + $dirite = new RecursiveDirectoryIterator($dir); + $iteite = new RecursiveIteratorIterator($dirite); + $regexp = $testtypes[$testtype]; + $regite = new RegexIterator($iteite, $regexp); + foreach ($regite as $path => $element) { + $key = dirname($path); + $value = trim(str_replace(DIRECTORY_SEPARATOR, '_', str_replace($CFG->dirroot, '', $key)), '_'); + $dirs[$key] = $value; + } + } + ksort($dirs); + + return array_flip($dirs); + } + + /** + * Return blacklisted contexts or features for a theme, as defined in blacklist.json. + * + * @param string $theme themename + * @param string $testtype test type (contexts|features) + * @return array list of blacklisted contexts or features + */ + protected function get_blacklisted_tests_for_theme($theme, $testtype) { + global $CFG; + + $themetestpath = $CFG->dirroot . DIRECTORY_SEPARATOR . "theme" . DIRECTORY_SEPARATOR . $theme . + self::get_behat_tests_path(); + + if (file_exists($themetestpath . DIRECTORY_SEPARATOR . 'blacklist.json')) { + // Blacklist file exist. Leave it for last to clear the feature and contexts. + $blacklisttests = @json_decode(file_get_contents($themetestpath . DIRECTORY_SEPARATOR . 'blacklist.json'), true); + if (empty($blacklisttests)) { + behat_error(BEHAT_EXITCODE_REQUIREMENT, $themetestpath . DIRECTORY_SEPARATOR . 'blacklist.json is empty'); + } + + // If features or contexts not defined then no problem. + if (!isset($blacklisttests[$testtype])) { + $blacklisttests[$testtype] = array(); + } + return $blacklisttests[$testtype]; + } + + return array(); + } + + /** + * Return list of features and step definitions in theme. + * + * @param string $theme theme name + * @param string $testtype test type, either features or contexts + * @return array list of contexts $contexts or $features + */ + protected function get_tests_for_theme($theme, $testtype) { + + $tests = array(); + $testtypes = array( + 'contexts' => '|behat_.*\.php$|', + 'features' => '|.*\.feature$|', + ); + + // Get all the directories having overridden tests. + $directories = $this->get_test_directories_overridden_for_theme($theme, $testtype); + + // Get overridden test contexts. + foreach ($directories as $dirpath) { + // All behat_*.php inside overridden directory. + $diriterator = new DirectoryIterator($dirpath); + $regite = new RegexIterator($diriterator, $testtypes[$testtype]); + + // All behat_*.php inside behat_config_manager::get_behat_tests_path() are added as steps definitions files. + foreach ($regite as $file) { + $key = $file->getBasename('.php'); + $tests[$key] = $file->getPathname(); + } + } + + return $tests; + } + + /** + * Return list of blacklisted behat features for theme and features defined by theme only. + * + * @param string $theme theme name. + * @return array ($themeblacklistfeatures, $themefeatures) + */ + protected function get_behat_features_for_theme($theme) { + + // Get list of features defined by theme. + $themefeatures = $this->get_tests_for_theme($theme, 'features'); + $themeblacklistfeatures = $this->get_blacklisted_tests_for_theme($theme, 'features'); + + // If tags are specified then we just want features with specified tags. + if (!empty($this->tags)) { + if (!empty($themefeatures)) { + $themefeatures = $this->filtered_features_with_tags($themefeatures); + } + } + + return array($themeblacklistfeatures, $themefeatures); + } + + /** + * Return list of contexts overridden by themes. + * + * @return array. + */ + protected function get_overridden_theme_contexts() { + if (empty($this->overriddenthemescontexts)) { + $this->overriddenthemescontexts = array(); + } + + return $this->overriddenthemescontexts; + } + + /** + * Return list of behat contexts for theme and update $this->stepdefinitions list. + * + * @param string $theme theme name. + * @return array list($features, $contexts) + */ + protected function get_behat_contexts_for_theme($theme) { + + // If we already have this list then just return. This will not change by run. + if (!empty($this->themecontexts[$theme])) { + return $this->themecontexts[$theme]; + } + + if (empty($this->overriddenthemescontexts)) { + $this->overriddenthemescontexts = array(); + } + + $contexts = $this->get_components_contexts(); + + // Create list of contexts used by theme suite. + $overriddencontexts = $this->get_tests_for_theme($theme, 'contexts'); + $blacklistedcontexts = $this->get_blacklisted_tests_for_theme($theme, 'contexts'); + + // Theme suite will use all core contexts, except the one overridden by theme. + $themesuitecontexts = $contexts; + foreach ($overriddencontexts as $context => $path) { + + // If a context in theme starts with behat_theme_{themename}_behat_* then it's overriding core context. + if (preg_match('/^behat_theme_'.$theme.'_(\w+)$/', $context, $match)) { + + if (!empty($themesuitecontexts[$match[1]])) { + unset($themesuitecontexts[$match[1]]); + } + + // Add this to the list of overridden paths, so it can be added to final contexts list for class resolver. + $this->overriddenthemescontexts[$context] = $path; + } + + // Don't include behat_selectors. + if ($context === self::get_behat_theme_selector_override_classname($theme)) { + continue; + } + + // Add theme specific contexts with suffix to steps definitions. + $themesuitecontexts[$context] = $path; + } + + // Remove blacklisted contexts. + foreach ($blacklistedcontexts as $blacklistpath) { + $blacklistcontext = basename($blacklistpath, '.php'); + + unset($themesuitecontexts[$blacklistcontext]); + } + + // We are only interested in the class name of context. + $this->themecontexts[$theme] = array_keys($themesuitecontexts); + + return $this->themecontexts[$theme]; + } +} \ No newline at end of file diff --git a/lib/behat/classes/behat_context_helper.php b/lib/behat/classes/behat_context_helper.php index 47d7fb007395f..99641cd629b71 100644 --- a/lib/behat/classes/behat_context_helper.php +++ b/lib/behat/classes/behat_context_helper.php @@ -55,8 +55,22 @@ class behat_context_helper { * * @param Environment $environment * @return void + * @deprecated since 3.2 MDL-55072 - please use behat_context_helper::set_environment() + * @todo MDL-55365 This will be deleted in Moodle 3.6. */ public static function set_session(Environment $environment) { + debugging('set_session is deprecated. Please use set_environment instead.', DEBUG_DEVELOPER); + + self::set_environment($environment); + } + + /** + * Sets behat environment. + * + * @param Environment $environment + * @return void + */ + public static function set_environment(Environment $environment) { self::$environment = $environment; } @@ -67,17 +81,27 @@ public static function set_session(Environment $environment) { * that uses direct API calls; steps returning step chains * can not be executed like this. * - * @throws coding_exception + * @throws Behat\Behat\Context\Exception\ContextNotFoundException * @param string $classname Context identifier (the class name). * @return behat_base */ public static function get($classname) { - if (!$subcontext = self::$environment->getContext($classname)) { - throw coding_exception('The required "' . $classname . '" class does not exist'); + $suitename = self::$environment->getSuite()->getName(); + $overridencontextname = 'behat_theme_'.$suitename.'_'.$classname; + + // Check if overridden context class exists. + if ($suitename !== 'default') { + try { + $subcontext = self::$environment->getContext($overridencontextname); + return $subcontext; + } catch (Behat\Behat\Context\Exception\ContextNotFoundException $e) { + // If context not found then it's not overridden. + } } - return $subcontext; + // Get the actual context. + return self::$environment->getContext($classname); } /** diff --git a/lib/behat/classes/behat_selectors.php b/lib/behat/classes/behat_selectors.php index a528b70a4f9cf..90f328b6719a6 100644 --- a/lib/behat/classes/behat_selectors.php +++ b/lib/behat/classes/behat_selectors.php @@ -23,7 +23,6 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ -defined('MOODLE_INTERNAL') || die(); /** * Moodle selectors manager. diff --git a/lib/behat/classes/util.php b/lib/behat/classes/util.php index a24ab5be816f3..ff70009194281 100644 --- a/lib/behat/classes/util.php +++ b/lib/behat/classes/util.php @@ -215,10 +215,13 @@ protected static function test_environment_problem() { * * Stores a file in dataroot/behat to allow Moodle to switch * to the test environment when using cli-server. + * @param bool $themesuitewithallfeatures if only theme specific features need to be included in the suite. + * @param int $parallelruns number of parallel runs. + * @param int $run current run. * @throws coding_exception * @return void */ - public static function start_test_mode() { + public static function start_test_mode($themesuitewithallfeatures = false, $parallelruns = 0, $run = 0) { global $CFG; if (!defined('BEHAT_UTIL')) { @@ -234,7 +237,7 @@ public static function start_test_mode() { self::test_environment_problem(); // Updates all the Moodle features and steps definitions. - behat_config_manager::update_config_file(); + behat_config_manager::update_config_file('', true, '', $themesuitewithallfeatures, $parallelruns, $run); if (self::is_test_mode_enabled()) { return; diff --git a/lib/testing/classes/tests_finder.php b/lib/testing/classes/tests_finder.php index 946ec23134669..b93ef233fd007 100644 --- a/lib/testing/classes/tests_finder.php +++ b/lib/testing/classes/tests_finder.php @@ -197,6 +197,9 @@ private static function get_regexp($testtype) { case 'stepsdefinitions': $regexp = '|'.$sep.'tests'.$sep.'behat'.$sep.'behat_.*\.php$|'; break; + case 'behat': + $regexp = '!'.$sep.'tests'.$sep.'behat'.$sep.'(.*\.feature)|(behat_.*\.php)$!'; + break; } return $regexp; diff --git a/lib/tests/behat/behat_hooks.php b/lib/tests/behat/behat_hooks.php index 1f25c1ab975ed..90600cbcbe576 100644 --- a/lib/tests/behat/behat_hooks.php +++ b/lib/tests/behat/behat_hooks.php @@ -96,6 +96,13 @@ class behat_hooks extends behat_base { */ protected static $timings = array(); + /** + * Keeps track of current running suite name. + * + * @var string current running suite name + */ + protected static $runningsuite = ''; + /** * Hook to capture BeforeSuite event so as to give access to moodle codebase. * This will try and catch any exception and exists if anything fails. @@ -128,15 +135,19 @@ public static function before_suite(BeforeSuiteScope $scope) { // Defined only when the behat CLI command is running, the moodle init setup process will // read this value and switch to $CFG->behat_dataroot and $CFG->behat_prefix instead of // the normal site. - define('BEHAT_TEST', 1); + if (!defined('BEHAT_TEST')) { + define('BEHAT_TEST', 1); + } + + if (!defined('CLI_SCRIPT')) { + define('CLI_SCRIPT', 1); + } - define('CLI_SCRIPT', 1); // With BEHAT_TEST we will be using $CFG->behat_* instead of $CFG->dataroot, $CFG->prefix and $CFG->wwwroot. require_once(__DIR__ . '/../../../config.php'); // Now that we are MOODLE_INTERNAL. require_once(__DIR__ . '/../../behat/classes/behat_command.php'); - require_once(__DIR__ . '/../../behat/classes/behat_selectors.php'); require_once(__DIR__ . '/../../behat/classes/behat_context_helper.php'); require_once(__DIR__ . '/../../behat/classes/util.php'); require_once(__DIR__ . '/../../testing/classes/test_lock.php'); @@ -264,7 +275,7 @@ public function before_scenario_hook(BeforeScenarioScope $scope) { * @throws behat_stop_exception If here we are not using the test database it should be because of a coding error */ public function before_scenario(BeforeScenarioScope $scope) { - global $DB, $SESSION, $CFG; + global $DB, $CFG; // As many checks as we can. if (!defined('BEHAT_TEST') || @@ -290,10 +301,18 @@ public function before_scenario(BeforeScenarioScope $scope) { throw new behat_stop_exception($e->getMessage()); } - // We need the Mink session to do it and we do it only before the first scenario. - if (self::is_first_scenario()) { - behat_selectors::register_moodle_selectors($session); - behat_context_helper::set_session($scope->getEnvironment()); + $suitename = $scope->getSuite()->getName(); + + // Register behat selectors for theme, if suite is changed. We do it for every suite change. + if ($suitename !== self::$runningsuite) { + behat_context_helper::set_environment($scope->getEnvironment()); + + // We need the Mink session to do it and we do it only before the first scenario. + $behatselectorclass = behat_config_util::get_behat_theme_selector_override_classname($suitename, true); + if (class_exists($behatselectorclass)) { + $behatselectorclass = new $behatselectorclass(); + $behatselectorclass::register_moodle_selectors($session); + } } // Reset mink session between the scenarios. @@ -317,6 +336,12 @@ public function before_scenario(BeforeScenarioScope $scope) { } } + // Set the theme if not default. + if ($suitename !== "default") { + set_config('theme', $suitename); + self::$runningsuite = $suitename; + } + // Start always in the the homepage. try { // Let's be conservative as we never know when new upstream issues will affect us. @@ -325,7 +350,6 @@ public function before_scenario(BeforeScenarioScope $scope) { throw new behat_stop_exception($e->getMessage()); } - // Checking that the root path is a Moodle test site. if (self::is_first_scenario()) { $notestsiteexception = new behat_stop_exception('The base URL (' . $CFG->wwwroot . ') is not a behat test site, ' . @@ -334,6 +358,7 @@ public function before_scenario(BeforeScenarioScope $scope) { self::$initprocessesfinished = true; } + // Run all test with medium (1024x768) screen size, to avoid responsive problems. $this->resize_window('medium'); } diff --git a/lib/upgrade.txt b/lib/upgrade.txt index 10a5207d648d7..e0cd70f784d27 100644 --- a/lib/upgrade.txt +++ b/lib/upgrade.txt @@ -55,6 +55,21 @@ information provided here is intended especially for developers. * New functions to support deprecation of events have been added to the base event. See MDL-46214 for further details. * A new function `get_name_with_info` has been added to the base event. This function adds information about event deprecations and should be used where this information is relevant. +* Following api's have been deprecated in behat_config_manager, please use behat_config_util instead. + - get_features_with_tags + - get_components_steps_definitions + - get_config_file_contents + - merge_behat_config + - get_behat_profile + - profile_guided_allocate + - merge_config + - clean_path + - get_behat_tests_path +* behat_util::start_test_mode() accepts 3 options now: + - 1. Theme sute with all features: If behat should initialise theme suite with all core features. + - 2. Parallel runs: How many parallel runs will be running. + - 3. Run: Which process behat should be initialise for. +* behat_context_helper::set_session() has been deprecated, please use behat_context_helper::set_environment() instead. === 3.1 ===