DO NOT MERGE: Test stability of t/ui/ui/25-developer_mode.t

.circleci/config.yml

@@ -86,8 +86,6 @@ aliases:
   - &make_test
     name: "Running unit tests"
     command: |
-      # downgrade chromedriver from problematic version to last good version, see https://progress.opensuse.org/issues/93453
-      rpm -q chromedriver | grep -v 91.0.4472.77 || sudo zypper -n in --oldpackage chromedriver-90.0.4430.212-lp152.2.95.1 chromium-90.0.4430.212
       export COVERAGE=1
       export COVERDB_SUFFIX="_${CIRCLE_JOB}"
       make test-$CIRCLE_JOB
@@ -203,7 +201,7 @@ jobs:
             fi
       - save_cache: *save_fullstack_cache
 
-  checkstyle: &test-template
+  ui: &test-template
     docker:
       - <<: *base
     steps:
@@ -222,45 +220,6 @@ jobs:
       - store_test_results: *store_test_results
       - store_artifacts: *store_artifacts
 
-  fullstack: &test-fullstack-template
-    docker:
-      - <<: *base
-    steps:
-      - checkout
-      - run: *chown_hack_for_cache
-      - run: *chown_hack_for_cache_fullstack
-      - restore_cache: *restore_cache
-      - restore_cache: *restore_fullstack_cache
-      - run: *check_cache
-      - store_artifacts: *store_logs
-      - run: *install_cached_packages
-      - run: *build_autoinst
-      - run: *test_junit
-      - run: *make_test
-      - persist_to_workspace:
-          root: .
-          paths:
-            - cover_db_*
-      - store_test_results: *store_test_results
-      - store_artifacts: *store_artifacts
-  t:
-    <<: *test-template
-
-  heavy:
-    <<: *test-template
-
-  ui:
-    <<: *test-template
-
-  api:
-    <<: *test-template
-
-  unstable:
-    <<: *test-template
-
-  fullstack-unstable:
-    <<: *test-fullstack-template
-
   codecov:
     docker:
       - <<: *base
@@ -319,37 +278,12 @@ workflows:
   test:
     jobs:
       - cache
-      - checkstyle: &requires
+      - ui: &requires
           requires:
             - cache
-      - t:
-          <<: *requires
-      - heavy:
-          <<: *requires
-      - api:
-          <<: *requires
-      - ui:
-          <<: *requires
-      # put unstable tests in unstable_tests.txt and uncomment to handle with retries
-      - unstable:
-          requires:
-            - cache
-      - cache.fullstack:
-          <<: *requires
-      - fullstack: &requires_fullstack
-          requires:
-            - cache.fullstack
-      - fullstack-unstable:
-          <<: *requires_fullstack
       - codecov:
           requires:
-            - t
-            - heavy
-            - api
             - ui
-            - unstable
-            - fullstack
-            - fullstack-unstable
       - build-docs:
           <<: *requires
 

Makefile

@@ -156,7 +156,7 @@ test-heavy:
 
 .PHONY: test-ui
 test-ui:
-	$(MAKE) test-with-database TIMEOUT_M=25 PROVE_ARGS="$$HARNESS t/ui/*.t" GLOBIGNORE="t/*tidy*:t/*compile*:$(unstables)"
+	$(MAKE) test-with-database RETRY=200 STABILITY_TEST=1 TIMEOUT_M=100000 PROVE_ARGS="$$HARNESS t/ui/25-developer_mode.t"
 
 .PHONY: test-api
 test-api:

assets/assetpack.def

@@ -161,6 +161,7 @@
 < javascripts/test_result.js
 < javascripts/needlediff.js
 < javascripts/running.js
+< javascripts/disable_status_updates.js [mode==test]
 
 ! job_next_previous.js
 < javascripts/job_next_previous.js

assets/javascripts/disable_status_updates.js

@@ -0,0 +1 @@
+window.enableStatusUpdates = (parseQueryParams().status_updates || []).every(p => Number.parseInt(p) !== 0);

assets/javascripts/running.js

@@ -184,6 +184,11 @@ function sendCommand(command) {
 }
 
 function updateStatus() {
+    // prevent status updates when window.enableStatusUpdates is set by test environment
+    if (window.enableStatusUpdates !== undefined && !window.enableStatusUpdates) {
+        return;
+    }
+
     $.ajax(testStatus.status_url).done(function(status) {
         updateTestStatus(status);
         // continue polling for job state updates until the job state is done

t/lib/OpenQA/SeleniumTest.pm

@@ -2,6 +2,7 @@ package OpenQA::SeleniumTest;
 
 use Test::Most;
 
+use Mojo::Base -signatures;
 use base 'Exporter';
 
 require OpenQA::Test::Database;
@@ -12,7 +13,8 @@ our @EXPORT = qw($drivermissing check_driver_modules enable_timeout
   wait_for_ajax_and_animations
   open_new_tab mock_js_functions element_visible element_hidden
   element_not_present javascript_console_has_no_warnings_or_errors
-  wait_until wait_until_element_gone wait_for_element);
+  wait_until wait_until_element_gone wait_for_element
+  element_prop element_prop_by_selector map_elements);
 
 use Data::Dump 'pp';
 use IPC::Run qw(start);
@@ -307,6 +309,20 @@ sub element_not_present {
     is(scalar @elements, 0, $selector . ' not present');
 }
 
+# returns an element's property
+# note: Workaround for not relying on the functions Selenium::Remote::WebElement::get_value() and is_selected()
+#       because they ceased to work in some cases with chromedriver 91.0.4472.77. (Whether the functions work or
+#       not likely depends on how the property is populated.)
+sub element_prop ($element_id, $property = 'value') {
+    return $_driver->execute_script("return document.getElementById('$element_id').$property;");
+}
+sub element_prop_by_selector ($element_selector, $property = 'value') {
+    return $_driver->execute_script("return document.querySelector('$element_selector').$property;");
+}
+sub map_elements ($selector, $mapping) {
+    return $_driver->execute_script("return Array.from(document.querySelectorAll('$selector')).map(e => [$mapping]);");
+}
+
 sub wait_until {
     my ($check_function, $check_description, $timeout, $check_interval) = @_;
     $timeout        //= 100;
@@ -361,7 +377,7 @@ sub wait_for_element {
     return $element;
 }
 
-sub kill_driver() {
+sub kill_driver {
     return unless $startingpid && $$ == $startingpid;
     if ($_driver) {
         $_driver->quit();

t/ui/01-list.t

@@ -359,9 +359,9 @@ $driver->get('/tests?match=staging_e');
 wait_for_ajax();
 
 @jobs = map { $_->get_attribute('id') } @{$driver->find_elements('tbody tr', 'css')};
-is_deeply(\@jobs, ['', '', 'job_99926'], '1 job matching');
-# note: the first 2 empty IDs are the 'not found' row of the data tables for running and
-#       scheduled jobs
+ok !$jobs[0], 'no running job matching';
+ok !$jobs[1], 'no scheduled job matching';
+is $jobs[2], 'job_99926', 'exactly one finished job matching';
 
 $driver->get('/tests');
 wait_for_ajax();

t/ui/10-tests_overview.t

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2020 SUSE LLC
+# Copyright (C) 2014-2021 SUSE LLC
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -328,7 +328,7 @@ subtest "filtering by machine" => sub {
 
         is($driver->find_element('#content tbody .name span')->get_text(), 'kde@uefi', 'Test suite name is shown');
         $driver->find_element('#filter-panel .card-header')->click();
-        is($driver->find_element('#filter-machine')->get_value(), 'uefi', 'machine text is correct.');
+        is(element_prop('filter-machine'), 'uefi', 'machine text is correct');
 
         $driver->find_element('#filter-machine')->clear();
         $driver->find_element('#filter-machine')->send_keys('64bit,uefi');