Multiple Dogtail tests improvements

- use unittest.skipIf and execute via nosetests_root.sh
- remove support for anacondaArgs because it was not used
- screenshots are placed in the correct test results directory
- logs are collected on test failures too
- cleanup mounted filesystems and update anaconda-cleanup to
use the new BlockDev loop plugin along the way

.gitignore

@@ -72,6 +72,7 @@ py-compile
 stamp-h1
 tags
 test-driver
+tests/autogui-results-*/
 tests/*/*/*.log
 tests/*/*/*.trs
 tests/*/*.log

scripts/anaconda-cleanup

@@ -41,6 +41,7 @@ pyanaconda.anaconda_log.init()
 from blivet.blivet import StorageDiscoveryConfig
 from blivet.devicetree import DeviceTree
 from blivet import devicelibs
+from gi.repository import BlockDev as blockdev
 
 storage_config = StorageDiscoveryConfig()
 
@@ -56,7 +57,7 @@ for dev in os.listdir(sys_class_block):
         continue
 
     loop = os.listdir("%s/%s/slaves" % (sys_class_block, dev))[0].strip()
-    storage_config.disk_images[name] = devicelibs.loop.get_backing_file(loop)
+    storage_config.disk_images[name] = blockdev.loop_get_backing_file(loop)
 
 if not image_install:
     live_install = live_install or os.path.exists("/dev/live")
@@ -87,4 +88,3 @@ for name in devicetree.disk_images.keys():
     device = devicetree.get_device_by_name(name)
     device.deactivate(recursive=True)
 os.system("udevadm control --env=ANACONDA=0")
-

tests/Makefile.am

@@ -53,7 +53,6 @@ dist_check_SCRIPTS = $(srcdir)/glade/*.py \
 		     gettext_tests/canary_tests.sh \
 		     storage/run_storage_tests.py \
 		     install/run_install_test.sh \
-		     run_gui_tests.sh \
 		     $(srcdir)/gui/*.py \
 		     $(srcdir)/storage/cases/*.py \
 		     $(srcdir)/*_tests/*.py

tests/gui/base.py

@@ -23,6 +23,7 @@
 import copy
 import glob
 import shutil
+import selinux
 import subprocess
 import tempfile
 import traceback
@@ -32,7 +33,7 @@
 from dogtail.config import config as dogtail_config
 from dogtail.predicate import GenericPredicate
 from dogtail.tree import SearchError, root
-from dogtail.utils import doDelay, screenshot
+from dogtail.utils import doDelay, isA11yEnabled, screenshot
 from nose.plugins.multiprocess import TimedOutException
 
 class UITestCase(unittest.TestCase):
@@ -69,12 +70,17 @@ class UITestCase(unittest.TestCase):
          which is used instead of runTest for error handling purposes.
     """
 
+    suite_name = None
+
     def update_scratch_dir(self, name=""):
         """ Update the directory where Dogtail saves screenshots. """
         path = os.path.join(testconfig.config.get("resultsdir", ""), name)
         if not path.endswith("/"):
             path += "/"
 
+        if not os.path.isdir(path):
+            os.makedirs(path)
+
         dogtail_config.load({"scratchDir": path})
 
     ###
@@ -86,7 +92,7 @@ def runTest(self):
            anaconda should a test fail.  Subclasses should not override this.
            See the documentation for _run.
         """
-        self.update_scratch_dir()
+        self.update_scratch_dir(self.suite_name)
 
         try:
             self._run()
@@ -240,6 +246,10 @@ def exit_spoke(self, hubName="INSTALLATION SUMMARY", node=None):
         self.check_window_displayed(hubName)
 
 
+@unittest.skipIf(os.geteuid() != 0, "GUI tests must be run as root")
+@unittest.skipIf(os.environ.get("DISPLAY", "") == "", "DISPLAY must be defined")
+@unittest.skipIf(selinux.security_getenforce() == 1, "SELinux must be disabled or in Permissive mode, see rhbz#1276376")
+@unittest.skipIf(not isA11yEnabled(), "Assistive Technologies are disabled")
 class DogtailTestCase(unittest.TestCase):
     """A subclass that defines all the parameters for starting a local
        copy of anaconda, inspecting results, and managing temporary data!
@@ -275,7 +285,7 @@ def __init__(self, methodName='runTest'):
         self.suite = unittest.TestSuite()
         for test in self.tests:
             T = test()
-            T.update_scratch_dir(self.name)
+            T.suite_name = self.name
             self.suite.addTest(T)
         self.test_result = None
 
@@ -351,11 +361,11 @@ def die(self, terminate=False):
             if terminate:
                 self._proc.terminate()
 
-            # tests will click the Reboot or Quit button which will shutdown anaconda
-            # we need to make sure /mnt/sysimage/* are unmounted and device mapper devices
-            # are removed before starting the next test. ATM I have no idea if pyanaconda
-            # provides an API for this so wait for a while instead!
-            doDelay(30)
+            # Tests will click the Reboot or Quit button which will shutdown anaconda.
+            # We need to make sure /mnt/sysimage/* are unmounted and device mapper devices
+            # are removed before starting the next test.
+            subprocess.call(["%s/scripts/anaconda-cleanup" % os.environ.get("top_srcdir", "")],
+                            stderr=subprocess.STDOUT)
 
             self._proc.kill()
             self._proc = None
@@ -389,7 +399,6 @@ def runTest(self):
         args = ["%s/anaconda.py" % os.environ.get("top_srcdir", ""), "-G"]
         for drive in self._drivePaths.values():
             args += ["--image", drive]
-        args += testconfig.config.get("anacondaArgs", "").strip('"')
 
         # Save a reference to the running anaconda process so we can later kill
         # it if necessary.  For now, the only reason we'd want to kill it is
@@ -401,7 +410,7 @@ def runTest(self):
             self.test_result = unittest.TextTestRunner(verbosity=2, failfast=True).run(self.suite)
             if not self.test_result.wasSuccessful():
                 raise AssertionError('Dogtail tests failed')
-        except TimedOutException:
+        except (TimedOutException, AssertionError):
             self.die(True)
             self.collect_logs()
             self.cleanup()

tests/nosetests_root.sh

@@ -4,3 +4,14 @@
 
 srcdir=${srcdir:=$(dirname "$0")}
 exec ${srcdir}/nosetests.sh ${srcdir}/pyanaconda_tests/user_create_test.py
+
+if ! rpm -q python3-nose-testconfig &> /dev/null; then
+    echo "python3-nose-testconfig is not available; exiting."
+    exit 99
+fi
+
+export LC_ALL=C # translations confuse Dogtail
+
+exec ${srcdir}/nosetests.sh -s --nologcapture --process-timeout 1200 \
+    --tc=resultsdir:$(readlink -f autogui-results-$(date +"%Y%m%d_%H%M%S")) \
+    ${srcdir}/gui/test_*.py