NXDRIVE-335: Use virtualenv for fonctional tests jobs

Several tree refactor has been made:
    - rename all requirements files to something unified (requirements-XXX.txt)
    - move tests folder from nuxeo-drive-client/nxdrive/tests to nuxeo-drive-client/tests

Bigs changes are:
    - we download and compile (if needed) our own Python, PyQt and dependencies versions
    - we do not use host softwares anymore (only basic ones like gcc or 7zip)

The process is the same on each platform, but we achieved it in different ways:
    - GNU/Linux and macOS: we use pyenv and compile what we need
    - Windows: we simply install Python version locally
The title is wrong, we do not use virtualenv as it is a mess on Windows and it happens to be a hell on other OS too.
Pyenv is the way to go and works like a charm, it is completely independant from the host.

For Jenkins jobs, we created 2 files in tools:
    - Jenkinsfile: to checkout, setup and launch the tests suite on all platforms
    - Jenkinsfile-release: to checkout and build releases on all platforms (GNU/Linux is not complete for now, but the process works)

Also:
    - Optimizations of imports
    - PEP8 changes here and there

ftest/itests.xml

@@ -14,7 +14,7 @@
 
   <!-- Configure nuxeo-drive marketplace package installation -->
   <property name="nuxeo.drive.mp.filename" value="nuxeo-drive.zip" />
-  <property name="mp.install" value="nuxeo-jsf-ui,${out.dir}/${nuxeo.drive.mp.filename},nuxeo-platform-importer" />
+  <property name="mp.install" value="${out.dir}/${nuxeo.drive.mp.filename}" />
 
   <target name="fetch-nuxeo-drive-mp">
     <exec executable="python" failonerror="true">
@@ -40,7 +40,6 @@
       <arg value="--base-folder=../nuxeo-drive-client" />
       <arg value="--work-folder=${out.dir}" />
       <arg value="test" />
-      <arg value="--msi-folder=../dist" />
       <arg value="--server-url=${env.SERVER_URL}" />
       <arg value="--engine=${env.ENGINE}" />
     </exec>

nuxeo-drive-client/nxdrive/commandline.py

@@ -99,20 +99,20 @@ def make_cli_parser(self, add_subparsers=True):
         )
         common_parser.add_argument(
             "--locale",
-            help=("Select the default language")
+            help="Select the default language"
         )
         common_parser.add_argument(
             "--force-locale",
-            help=("Force the language")
+            help="Force the language"
         )
         common_parser.add_argument(
             "--update-site-url",
             default=DEFAULT_UPDATE_SITE_URL,
-            help=("Website for client auto-update")
+            help="Website for client auto-update"
         )
         common_parser.add_argument(
             "--beta-update-site-url",
-            help=("Website for client beta auto-update")
+            help="Website for client beta auto-update"
         )
         common_parser.add_argument(
             "--debug", default=False, action="store_true",
@@ -308,28 +308,6 @@ def make_cli_parser(self, add_subparsers=True):
             "--file", default="",
             help="File path.")
 
-        # embedded test runner base on nose:
-        test_parser = subparsers.add_parser(
-            'test',
-            help='Run the Nuxeo Drive test suite.',
-            parents=[common_parser],
-        )
-        test_parser.set_defaults(command='test')
-        test_parser.add_argument(
-            "-w", help="Nose working directory.")
-        test_parser.add_argument(
-            "--nologcapture", default=False, action="store_true",
-            help="Disable nose logging capture plugin.")
-        test_parser.add_argument(
-            "--logging-format", help="Format of noestests log capture statements."
-            " Uses the same format as used by standard logging handlers.")
-        test_parser.add_argument(
-            "--with-coverage", default=False, action="store_true",
-            help="Compute coverage report.")
-        test_parser.add_argument(
-            "--with-profile", default=False, action="store_true",
-            help="Compute profiling report.")
-
         return parser
     """Command Line Interface handler: parse options and execute operation"""
 
@@ -402,7 +380,6 @@ def load_config(self, parser):
         else:
             parser.set_defaults(**AbstractOSIntegration.get(None).get_system_configuration())
 
-
     def _configure_logger(self, options):
         """Configure the logging framework from the provided options"""
         # Ensure the log folder exists
@@ -453,14 +430,12 @@ def handle(self, argv):
         self.log.debug("Command line: argv=%r, options=%r",
                        ' '.join(argv), options)
 
-        if command != 'test' and command != 'uninstall':
+        if command != 'uninstall':
             # Install utility to help debugging segmentation faults
             self._install_faulthandler(options)
 
-        if command != 'test':
-            # Initialize a manager for this process, except for the tests
-            # as they initialize their own
-            self.manager = self.get_manager(options)
+        # Initialize a manager for this process
+        self.manager = self.get_manager(options)
 
         # Find the command to execute based on the
         handler = getattr(self, command, None)
@@ -471,7 +446,7 @@ def handle(self, argv):
         try:
             return handler(options)
         except Exception, e:
-            if command == 'test' or options.debug:
+            if options.debug:
                 # Make it possible to use the postmortem debugger
                 raise
             else:
@@ -576,104 +551,6 @@ def unbind_root(self, options):
         self.log.error('No engine registered for local folder %s', options.local_folder)
         return 1
 
-    def test(self, options):
-        import nose
-        # Monkeypatch nose usage message as it's complicated to include
-        # the missing text resource in the frozen binary package
-        nose.core.TestProgram.usage = lambda cls: ""
-        argv = [
-            '',
-            '--verbose',
-            '--stop',
-        ]
-
-        if options.w:
-            argv += [
-                '-w',
-                options.w,
-            ]
-
-        if options.nologcapture:
-            argv += [
-                '--nologcapture',
-            ]
-
-        if options.logging_format:
-            argv += [
-                '--logging-format',
-                options.logging_format,
-            ]
-
-        if options.with_coverage:
-            argv += [
-                '--with-coverage',
-                '--cover-package=nxdrive',
-                '--cover-html',
-                '--cover-html-dir=coverage',
-            ]
-
-        if options.with_profile:
-            argv += [
-                '--with-profile',
-                '--profile-restrict=nxdrive',
-            ]
-        if "TESTS" not in os.environ or os.environ["TESTS"].strip() == "":
-            # List the test modules explicitly as recursive discovery is broken
-            # when the app is frozen.
-            argv += [
-                "nxdrive.tests.test_bind_server",
-                "nxdrive.tests.test_blacklist_queue",
-                "nxdrive.tests.test_bulk_remote_changes",
-                "nxdrive.tests.test_commandline",
-                "nxdrive.tests.test_conflicts",
-                "nxdrive.tests.test_copy",
-                "nxdrive.tests.test_direct_edit",
-                "nxdrive.tests.test_encoding",
-                "nxdrive.tests.test_engine_dao",
-                "nxdrive.tests.test_concurrent_synchronization",
-                "nxdrive.tests.test_integration_local_root_deletion",
-                "nxdrive.tests.test_local_client",
-                "nxdrive.tests.test_local_copy_paste",
-                "nxdrive.tests.test_local_create_folders",
-                "nxdrive.tests.test_local_deletion",
-                "nxdrive.tests.test_local_filter",
-                "nxdrive.tests.test_local_move_and_rename",
-                "nxdrive.tests.test_local_move_folders",
-                "nxdrive.tests.test_local_paste",
-                "nxdrive.tests.test_local_storage_issue",
-                "nxdrive.tests.test_local_share_move_folders",
-                "nxdrive.tests.test_long_path",
-                "nxdrive.tests.test_manager_dao",
-                "nxdrive.tests.test_model_filters",
-                "nxdrive.tests.test_multiple_files",
-                "nxdrive.tests.test_permission_hierarchy",
-                "nxdrive.tests.test_readonly",
-                "nxdrive.tests.test_reinit_database",
-                "nxdrive.tests.test_remote_changes",
-                "nxdrive.tests.test_remote_deletion",
-                "nxdrive.tests.test_remote_document_client",
-                "nxdrive.tests.test_remote_file_system_client",
-                "nxdrive.tests.test_remote_move_and_rename",
-                "nxdrive.tests.test_report",
-                "nxdrive.tests.test_security_updates",
-                "nxdrive.tests.test_shared_folders",
-                "nxdrive.tests.test_sync_roots",
-                "nxdrive.tests.test_synchronization",
-                "nxdrive.tests.test_synchronization_dedup",
-                "nxdrive.tests.test_synchronization_dedup_case_sensitive",
-                "nxdrive.tests.test_synchronization_suspend",
-                "nxdrive.tests.test_translator",
-                "nxdrive.tests.test_updater",
-                "nxdrive.tests.test_utils",
-                "nxdrive.tests.test_versioning",
-                "nxdrive.tests.test_volume",
-                "nxdrive.tests.test_watchers",
-                "nxdrive.tests.test_windows",
-            ]
-        else:
-            argv.extend(['nxdrive.tests.' + test for test in os.environ["TESTS"].strip().split(",")])
-        return 0 if nose.run(argv=argv) else 1
-
     def _install_faulthandler(self, options):
         """Utility to help debug segfaults"""
         try:

nuxeo-drive-client/nxdrive/tests/common.py

@@ -1,20 +1,17 @@
 """Common test utilities"""
-import sys
-import os
-import unittest
-import tempfile
 import hashlib
+import os
 import shutil
+import sys
+import tempfile
+import unittest
+from os.path import dirname
 
 import nxdrive
-from nxdrive.utils import safe_long_path
-from nxdrive.client import RemoteDocumentClient
-from nxdrive.client import RemoteFileSystemClient
-from nxdrive.client import RestAPIClient
-from nxdrive.client import LocalClient
+from nxdrive.client import LocalClient, RemoteDocumentClient, RemoteFileSystemClient, RestAPIClient
 from nxdrive.client.common import BaseClient
-from nxdrive.logging_config import configure
-from nxdrive.logging_config import get_logger
+from nxdrive.logging_config import configure, get_logger
+from nxdrive.utils import safe_long_path
 
 WindowsError = None
 try:
@@ -99,6 +96,9 @@ def clean_dir(_dir):
 class IntegrationTestCase(unittest.TestCase):
 
     def setUp(self):
+        # Save the current path for test files
+        self.location = dirname(__file__)
+
         # Check the Nuxeo server test environment
         self.nuxeo_url = os.environ.get('NXDRIVE_TEST_NUXEO_URL')
         self.admin_user = os.environ.get('NXDRIVE_TEST_USER')
@@ -228,7 +228,11 @@ def setUp(self):
     def tearDown(self):
         # Don't need to revoke tokens for the file system remote clients
         # since they use the same users as the remote document clients
-        self.root_remote_client.execute("NuxeoDrive.TearDownIntegrationTests")
+        try:
+            self.root_remote_client.execute("NuxeoDrive.TearDownIntegrationTests")
+        except AttributeError:
+            # If a test did not have enough time, failed early, `root_remote_client` could be inexistant. Just ignore.
+            pass
 
         clean_dir(self.upload_tmp_dir)
         clean_dir(self.local_test_folder_1)

nuxeo-drive-client/nxdrive/tests/common_unit_test.py

@@ -1,27 +1,23 @@
 """Common test utilities"""
 import os
-import unittest
 import sys
 import tempfile
-
-from nxdrive.client import RemoteDocumentClient
-from nxdrive.client import RemoteFileSystemClient
-from nxdrive.client import LocalClient
-from nxdrive.client import RestAPIClient
-from nxdrive.osi import AbstractOSIntegration
-from nxdrive.manager import Manager
-from nxdrive.logging_config import configure
-from nxdrive.logging_config import get_logger
-from nxdrive.tests.common import TEST_WORKSPACE_PATH
-from nxdrive.tests.common import TEST_DEFAULT_DELAY
-from nxdrive.tests.common import clean_dir
-from nxdrive.wui.translator import Translator
-from nxdrive import __version__
-from PyQt4 import QtCore
+import unittest
 from functools import wraps
+from os.path import dirname
 from threading import Thread
 from time import sleep
 
+from PyQt4 import QtCore
+
+from nxdrive import __version__
+from nxdrive.client import LocalClient, RemoteDocumentClient, RemoteFileSystemClient, RestAPIClient
+from nxdrive.logging_config import configure, get_logger
+from nxdrive.manager import Manager
+from nxdrive.osi import AbstractOSIntegration
+from nxdrive.tests.common import TEST_DEFAULT_DELAY, TEST_WORKSPACE_PATH, clean_dir
+from nxdrive.wui.translator import Translator
+
 if 'DRIVE_YAPPI' in os.environ:
     import yappi
 
@@ -122,31 +118,19 @@ def callable(*args, **kwargs):
 
 
 def configure_logger():
-    configure(
-        console_level='TRACE',
-        command_name='test',
-        force_configure=True,
-    )
+    configure(console_level='TRACE',
+              command_name='test',
+              force_configure=True)
 
 # Configure test logger
 configure_logger()
 log = get_logger(__name__)
 
 
-class TestThread(QtCore.QThread):
-    def __init__(self, method, method_arg):
-        super(TestThread, self).__init__()
-        self._method = method
-        self._method_arg = method_arg
-
-    def run(self):
-        self._method(self._method_arg)
-
-
-class TestQApplication(QtCore.QCoreApplication):
+class StubQApplication(QtCore.QCoreApplication):
 
     def __init__(self, argv, test_case):
-        super(TestQApplication, self).__init__(argv)
+        super(StubQApplication, self).__init__(argv)
         self._test = test_case
 
     @QtCore.pyqtSlot()
@@ -185,6 +169,9 @@ def no_remote_changes_found(self,):
 class UnitTestCase(unittest.TestCase):
 
     def setUpServer(self, server_profile=None):
+        # Save the current path for test files
+        self.location = dirname(__file__)
+
         # Long timeout for the root client that is responsible for the test
         # environment set: this client is doing the first query on the Nuxeo
         # server and might need to wait for a long time without failing for
@@ -510,7 +497,7 @@ def run(self, result=None):
         if hasattr(testMethod, '_repeat'):
             repeat = testMethod._repeat
         while repeat > 0:
-            self.app = TestQApplication([], self)
+            self.app = StubQApplication([], self)
             self.setUpApp()
             self.result = result