MDL-63977 Behat: Allow Behat testing of the Moodle mobile app

This change allows you to write and run Behat tests that cover the
mobile app. These should have the @app tag. They will be run in the
Chrome browser using an Ionic server on the local machine.

See config-dist.php for configuration settings, or full docs here:
https://docs.moodle.org/dev/Acceptance_testing_for_the_mobile_app

admin/tool/behat/lang/en/tool_behat.php

@@ -24,6 +24,7 @@
 
 $string['aim'] = 'This administration tool helps developers and test writers to create .feature files describing Moodle\'s functionalities and run them automatically. Step definitions available for use in .feature files are listed below.';
 $string['allavailablesteps'] = 'All available step definitions';
+$string['errorapproot'] = '$CFG->behat_approot is not pointing to a valid Moodle Mobile developer install.';
 $string['errorbehatcommand'] = 'Error running behat CLI command. Try running "{$a} --help" manually from CLI to find out more about the problem.';
 $string['errorcomposer'] = 'Composer dependencies are not installed.';
 $string['errordataroot'] = '$CFG->behat_dataroot is not set or is invalid.';

config-dist.php

@@ -868,6 +868,13 @@
 // Example:
 //   define('BEHAT_DISABLE_HISTOGRAM', true);
 //
+// Mobile app Behat testing requires this option, pointing to a developer Moodle Mobile directory:
+//   $CFG->behat_approot = '/where/I/keep/my/git/checkouts/moodlemobile2';
+//
+// The following option can be used to indicate a running Ionic server (otherwise Behat will start
+// one automatically for each test run, which is convenient but takes ages):
+//   $CFG->behat_ionicaddress = 'http://localhost:8100';
+//
 //=========================================================================
 // 12. DEVELOPER DATA GENERATOR
 //=========================================================================

course/tests/behat/app_courselist.feature

@@ -0,0 +1,91 @@
+@core @core_course @app @javascript
+Feature: Test course list shown on app start tab
+  In order to select a course
+  As a student
+  I need to see the correct list of courses
+
+  Background:
+    Given the following "courses" exist:
+      | fullname | shortname |
+      | Course 1 | C1        |
+      | Course 2 | C2        |
+    And the following "users" exist:
+      | username |
+      | student1 |
+      | student2 |
+    And the following "course enrolments" exist:
+      | user     | course | role    |
+      | student1 | C1     | student |
+      | student2 | C1     | student |
+      | student2 | C2     | student |
+
+  Scenario: Student is registered on one course
+    When I enter the app
+    And I log in as "student1" in the app
+    Then I should see "Course 1"
+    And I should not see "Course 2"
+
+  Scenario: Student is registered on two courses (shortnames not displayed)
+    When I enter the app
+    And I log in as "student2" in the app
+    Then I should see "Course 1"
+    And I should see "Course 2"
+    And I should not see "C1"
+    And I should not see "C2"
+
+  Scenario: Student is registered on two courses (shortnames displayed)
+    Given the following config values are set as admin:
+      | courselistshortnames | 1 |
+    When I enter the app
+    And I log in as "student2" in the app
+    Then I should see "Course 1"
+    And I should see "Course 2"
+    And I should see "C1"
+    And I should see "C2"
+
+  Scenario: Student uses course list to enter course, then leaves it again
+    When I enter the app
+    And I log in as "student2" in the app
+    And I press "Course 2" in the app
+    Then the header should be "Course 2" in the app
+    And I press the back button in the app
+    Then the header should be "Acceptance test site" in the app
+
+  Scenario: Student uses filter feature to reduce course list
+    Given the following config values are set as admin:
+      | courselistshortnames | 1 |
+    And the following "courses" exist:
+      | fullname | shortname |
+      | Frog 3   | C3        |
+      | Frog 4   | C4        |
+      | Course 5 | C5        |
+      | Toad 6   | C6        |
+    And the following "course enrolments" exist:
+      | user     | course | role    |
+      | student2 | C3     | student |
+      | student2 | C4     | student |
+      | student2 | C5     | student |
+      | student2 | C6     | student |
+    When I enter the app
+    And I log in as "student2" in the app
+    Then I should see "C1"
+    And I should see "C2"
+    And I should see "C3"
+    And I should see "C4"
+    And I should see "C5"
+    And I should see "C6"
+    And I press "Filter my courses" in the app
+    And I set the field "Filter my courses" to "fr" in the app
+    Then I should not see "C1"
+    And I should not see "C2"
+    And I should see "C3"
+    And I should see "C4"
+    And I should not see "C5"
+    And I should not see "C6"
+    And I press "Filter my courses" in the app
+    Then I should see "C1"
+    And I should see "C2"
+    And I should see "C3"
+    And I should see "C4"
+    And I should see "C5"
+    And I should see "C6"

lib/behat/classes/behat_command.php

@@ -219,6 +219,12 @@ public static function behat_setup_problem() {
             return BEHAT_EXITCODE_CONFIG;
         }
 
+        // If app config is supplied, check the value is correct.
+        if (!empty($CFG->behat_approot) && !file_exists($CFG->behat_approot . '/ionic.config.json')) {
+            self::output_msg(get_string('errorapproot', 'tool_behat'));
+            return BEHAT_EXITCODE_CONFIG;
+        }
+
         return 0;
     }
 

lib/behat/classes/behat_config_util.php

@@ -620,6 +620,42 @@ protected function get_behat_profile($profile, $values) {
 
         // Check suite values.
         $behatprofilesuites = array();
+
+        // Automatically set tags information to skip app testing if necessary. We skip app testing
+        // if the browser is not Chrome. (Note: We also skip if it's not configured, but that is
+        // done on the theme/suite level.)
+        if (empty($values['browser']) || $values['browser'] !== 'chrome') {
+            if (!empty($values['tags'])) {
+                $values['tags'] .= ' && ~@app';
+            } else {
+                $values['tags'] = '~@app';
+            }
+        }
+
+        // Automatically add Chrome command line option to skip the prompt about allowing file
+        // storage - needed for mobile app testing (won't hurt for everything else either).
+        if (!empty($values['browser']) && $values['browser'] === 'chrome') {
+            if (!isset($values['capabilities'])) {
+                $values['capabilities'] = [];
+            }
+            if (!isset($values['capabilities']['chrome'])) {
+                $values['capabilities']['chrome'] = [];
+            }
+            if (!isset($values['capabilities']['chrome']['switches'])) {
+                $values['capabilities']['chrome']['switches'] = [];
+            }
+            $values['capabilities']['chrome']['switches'][] = '--unlimited-storage';
+
+            // If the mobile app is enabled, check its version and add appropriate tags.
+            if ($mobiletags = $this->get_mobile_version_tags()) {
+                if (!empty($values['tags'])) {
+                    $values['tags'] .= ' && ' . $mobiletags;
+                } else {
+                    $values['tags'] = $mobiletags;
+                }
+            }
+        }
+
         // Fill tags information.
         if (isset($values['tags'])) {
             $behatprofilesuites = array(
@@ -658,6 +694,84 @@ protected function get_behat_profile($profile, $values) {
         return array($profile => array_merge($behatprofilesuites, $behatprofileextension));
     }
 
+    /**
+     * Gets version tags to use for the mobile app.
+     *
+     * This is based on the current mobile app version (from its package.json) and all known
+     * mobile app versions (based on the list appversions.json in the lib/behat directory).
+     *
+     * @return string List of tags or '' if not supporting mobile
+     */
+    protected function get_mobile_version_tags() : string {
+        global $CFG;
+
+        if (empty($CFG->behat_approot)) {
+            return '';
+        }
+
+        // Get app version.
+        $jsonpath = $CFG->behat_approot . '/package.json';
+        $json = @file_get_contents($jsonpath);
+        if (!$json) {
+            throw new coding_exception('Unable to load app version from ' . $jsonpath);
+        }
+        $package = json_decode($json);
+        if ($package === null) {
+            throw new coding_exception('Invalid app package data in ' . $jsonpath);
+        }
+        $installedversion = $package->version;
+
+        // Read all feature files to check which mobile tags are used. (Note: This could be cached
+        // but ideally, it is the sort of thing that really ought to be refreshed by doing a new
+        // Behat init. Also, at time of coding it only takes 0.3 seconds and only if app enabled.)
+        $usedtags = [];
+        foreach ($this->features as $filepath) {
+            $feature = file_get_contents($filepath);
+            // This may incorrectly detect versions used e.g. in a comment or something, but it
+            // doesn't do much harm if we have extra ones.
+            if (preg_match_all('~@app_(?:from|upto)(?:[0-9]+(?:\.[0-9]+)*)~', $feature, $matches)) {
+                foreach ($matches[0] as $tag) {
+                    // Store as key in array so we don't get duplicates.
+                    $usedtags[$tag] = true;
+                }
+            }
+        }
+
+        // Set up relevant tags for each version.
+        $tags = [];
+        foreach ($usedtags as $usedtag => $ignored) {
+            if (!preg_match('~^@app_(from|upto)([0-9]+(?:\.[0-9]+)*)$~', $usedtag, $matches)) {
+                throw new coding_exception('Unexpected tag format');
+            }
+            $direction = $matches[1];
+            $version = $matches[2];
+
+            switch (version_compare($installedversion, $version)) {
+                case -1:
+                    // Installed version OLDER than the one being considered, so do not
+                    // include any scenarios that only run from the considered version up.
+                    if ($direction === 'from') {
+                        $tags[] = '~app_from' . $version;
+                    }
+                    break;
+
+                case 0:
+                    // Installed version EQUAL to the one being considered - no tags need
+                    // excluding.
+                    break;
+
+                case 1:
+                    // Installed version NEWER than the one being considered, so do not
+                    // include any scenarios that only run up to that version.
+                    if ($direction === 'upto') {
+                        $tags[] = '~app_upto' . $version;
+                    }
+                    break;
+            }
+        }
+        return join(' && ', $tags);
+    }
+
     /**
      * Attempt to split feature list into fairish buckets using timing information, if available.
      * Simply add each one to lightest buckets until all files allocated.
@@ -1237,12 +1351,19 @@ protected function get_tests_for_theme($theme, $testtype) {
      * @return array ($blacklistfeatures, $blacklisttags, $features)
      */
     protected function get_behat_features_for_theme($theme) {
+        global $CFG;
 
         // Get list of features defined by theme.
         $themefeatures = $this->get_tests_for_theme($theme, 'features');
         $themeblacklistfeatures = $this->get_blacklisted_tests_for_theme($theme, 'features');
         $themeblacklisttags = $this->get_blacklisted_tests_for_theme($theme, 'tags');
 
+        // Mobile app tests are not theme-specific, so run only for the default theme (and if
+        // configured).
+        if (empty($CFG->behat_approot) || $theme !== $this->get_default_theme()) {
+            $themeblacklisttags[] = '@app';
+        }
+
         // Clean feature key and path.
         $features = array();
         $blacklistfeatures = array();