Create separate base dir for each CR service in functional test

This fixes the functional tests failing on Windows due to can't write to the same file from multiple processes on NTFS.

appveyor.yml

@@ -0,0 +1,36 @@
+environment:
+  global:
+# SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+# /E:ON and /V:ON options are not enabled in the batch script intepreter
+# See: http://stackoverflow.com/a/13751649/163740
+    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
+
+  matrix:
+    - PYTHON: "C:\\Python34"
+      PYTHON_VERSION: "3.4.x"
+      PYTHON_ARCH: "64"
+
+install:
+  # Install Python (from the official .msi of http://python.org) and pip when
+  # not already installed.
+  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+
+  # Prepend newly installed Python to the PATH of this build (this cannot be
+  # done from inside the powershell script as it would require to restart
+  # the parent CMD process).
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+
+  # Check that we have the expected version and architecture for Python
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+
+  # Install the build dependencies of the project. If some dependencies contain
+  # compiled extensions and are not provided as pre-built wheel packages,
+  # pip will build them from source using the MSVC compiler matching the
+  # target Python version and architecture
+  - "%CMD_IN_ENV% pip install -r C:\\projects\\clusterrunner\\requirements.txt"
+
+build: false
+
+test_script:
+  - "%CMD_IN_ENV% nosetest C:\\projects\\clusterrunner\\test"

test/framework/functional/base_functional_test_case.py

@@ -6,8 +6,6 @@
 from unittest import TestCase
 
 from app.util import log
-from app.util.conf.base_config_loader import BASE_CONFIG_FILE_SECTION
-from app.util.conf.config_file import ConfigFile
 from app.util.process_utils import is_windows
 from app.util.secret import Secret
 from test.framework.functional.fs_item import Directory
@@ -25,19 +23,8 @@ def setUp(self):
         log.configure_logging('DEBUG')
 
         Secret.set('testsecret')
-        self.test_app_base_dir = tempfile.TemporaryDirectory()
 
-        self.test_conf_file_path = self._create_test_config_file({
-            'secret': Secret.get(),
-            'base_directory': self.test_app_base_dir.name,
-            # Set the max log file size to a low value so that we cause  at least one rollover during the test.
-            'max_log_file_size': 1024 * 5,
-        })
-
-        self.cluster = FunctionalTestCluster(
-            conf_file_path=self.test_conf_file_path,
-            verbose=self._get_test_verbosity(),
-        )
+        self.cluster = FunctionalTestCluster(verbose=self._get_test_verbosity())
 
     def _create_test_config_file(self, conf_values_to_set=None):
         """
@@ -62,10 +49,6 @@ def _create_test_config_file(self, conf_values_to_set=None):
         return test_conf_file_path
 
     def tearDown(self):
-        # Clean up files created during this test
-        with suppress(FileNotFoundError):
-            os.remove(self.test_conf_file_path)
-
         # Give the cluster a bit of extra time to finish working (before forcefully killing it and failing the test)
         with suppress(TestClusterTimeoutError):
             self.cluster.block_until_build_queue_empty(timeout=5)
@@ -87,7 +70,8 @@ def tearDown(self):
                     ),
                 )
         # Remove the temp dir. This will delete the log files, so should be run after cluster shuts down.
-        self.test_app_base_dir.cleanup()
+        self.cluster.master_app_base_dir.cleanup()
+        [slave_app_base_dir.cleanup() for slave_app_base_dir in self.cluster.slaves_app_base_dirs]
 
     def _get_test_verbosity(self):
         """
@@ -167,7 +151,7 @@ def assert_build_artifact_contents_match_expected(self, build_id, expected_build
         :param expected_build_artifact_contents: A list of FSItems corresponding to the expected artifact dir contents
         :type expected_build_artifact_contents: list[FSItem]
         """
-        build_artifacts_dir_path = os.path.join(self.test_app_base_dir.name, 'results', 'master', str(build_id))
+        build_artifacts_dir_path = os.path.join(self.cluster.master_app_base_dir.name, 'results', 'master', str(build_id))
         self.assert_directory_contents_match_expected(build_artifacts_dir_path, expected_build_artifact_contents)
 
     def assert_directory_contents_match_expected(self, dir_path, expected_dir_contents):

test/framework/functional/functional_test_cluster.py

@@ -1,13 +1,19 @@
+import requests
+
 from contextlib import suppress
 import functools
+import os
 from os.path import dirname, join, realpath
-import requests
 from subprocess import DEVNULL, Popen
 import sys
+import shutil
 import tempfile
 
 from app.client.cluster_api_client import ClusterMasterAPIClient, ClusterSlaveAPIClient
 from app.util import log, poll, process_utils
+from app.util.conf.base_config_loader import BASE_CONFIG_FILE_SECTION
+from app.util.conf.config_file import ConfigFile
+from app.util.secret import Secret
 
 
 class FunctionalTestCluster(object):
@@ -18,14 +24,11 @@ class FunctionalTestCluster(object):
     _MASTER_PORT = 43000
     _SLAVE_START_PORT = 43001
 
-    def __init__(self, conf_file_path, verbose=False):
+    def __init__(self, verbose=False):
         """
-        :param conf_file_path: The path to the clusterrunner.conf file that this cluster's services should use
-        :type conf_file_path: str
         :param verbose: If true, output from the master and slave processes is allowed to pass through to stdout.
         :type verbose: bool
         """
-        self._conf_file_path = conf_file_path
         self._verbose = verbose
         self._logger = log.get_logger(__name__)
 
@@ -36,8 +39,47 @@ def __init__(self, conf_file_path, verbose=False):
         self._slave_eventlog_names = []
         self._next_slave_port = self._SLAVE_START_PORT
 
-        clusterrunner_repo_dir = dirname(dirname(dirname(dirname(realpath(__file__)))))
-        self._app_executable = join(clusterrunner_repo_dir, 'main.py')
+        self._clusterrunner_repo_dir = dirname(dirname(dirname(dirname(realpath(__file__)))))
+        self._app_executable = join(self._clusterrunner_repo_dir, 'main.py')
+
+        self._master_app_base_dir = None
+        self._slaves_app_base_dirs = []
+
+    @property
+    def master_app_base_dir(self):
+        return self._master_app_base_dir
+
+    @property
+    def slaves_app_base_dirs(self):
+        return self._slaves_app_base_dirs
+
+    def _create_test_config_file(self, base_dir_sys_path):
+        """
+        Create a temporary conf file just for this test.
+
+        :param base_dir_sys_path: Sys path of the base app dir
+        :type base_dir_sys_path: unicode
+
+        :return: The path to the conf file
+        :rtype: str
+        """
+        # Copy default conf file to tmp location
+        self._conf_template_path = join(self._clusterrunner_repo_dir, 'conf', 'default_clusterrunner.conf')
+        test_conf_file_path = tempfile.NamedTemporaryFile().name
+        shutil.copy(self._conf_template_path, test_conf_file_path)
+        os.chmod(test_conf_file_path, ConfigFile.CONFIG_FILE_MODE)
+        conf_file = ConfigFile(test_conf_file_path)
+
+        # Set custom conf file values for this test
+        conf_values_to_set = {
+            'secret': Secret.get(),
+            'base_directory': base_dir_sys_path,
+            'max_log_file_size': 1024 * 5,
+        }
+        for conf_key, conf_value in conf_values_to_set.items():
+            conf_file.write_value(conf_key, conf_value, BASE_CONFIG_FILE_SECTION)
+
+        return test_conf_file_path
 
     def start_master(self):
         """
@@ -95,14 +137,16 @@ def _start_master_process(self):
             popen_kwargs.update({'stdout': DEVNULL, 'stderr': DEVNULL})  # hide output of master process
 
         self._master_eventlog_name = tempfile.NamedTemporaryFile(delete=False).name
+        self._master_app_base_dir = tempfile.TemporaryDirectory()
+        master_config_file_path = self._create_test_config_file(self._master_app_base_dir.name)
         master_hostname = 'localhost'
         master_cmd = [
             sys.executable,
             self._app_executable,
             'master',
             '--port', str(self._MASTER_PORT),
             '--eventlog-file', self._master_eventlog_name,
-            '--config-file', self._conf_file_path,
+            '--config-file', master_config_file_path,
         ]
 
         # Don't use shell=True in the Popen here; the kill command might only kill "sh -c", not the actual process.
@@ -150,6 +194,9 @@ def _start_slave_processes(self, num_slaves, num_executors_per_slave):
 
             slave_eventlog = tempfile.NamedTemporaryFile().name  # each slave writes to its own file to avoid collision
             self._slave_eventlog_names.append(slave_eventlog)
+            slave_base_app_dir = tempfile.TemporaryDirectory()
+            self._slaves_app_base_dirs.append(slave_base_app_dir)
+            slave_config_file_path = self._create_test_config_file(slave_base_app_dir.name)
 
             slave_cmd = [
                 sys.executable,
@@ -159,7 +206,7 @@ def _start_slave_processes(self, num_slaves, num_executors_per_slave):
                 '--num-executors', str(num_executors_per_slave),
                 '--master-url', '{}:{}'.format(self.master.host, self.master.port),
                 '--eventlog-file', slave_eventlog,
-                '--config-file', self._conf_file_path,
+                '--config-file', slave_config_file_path,
             ]
 
             # Don't use shell=True in the Popen here; the kill command may only kill "sh -c", not the actual process.

test/integration/common/test_console_output.py

@@ -15,17 +15,19 @@ def setUpClass(cls):
             console_output.append('line_' + str(i))
 
         # Completed console output file.
-        _, cls._completed_console_output_file_path = mkstemp()
+        cls._completed_console_output_fd, cls._completed_console_output_file_path = mkstemp()
         with open(cls._completed_console_output_file_path, 'w') as f:
             f.write("\n".join(console_output) + "\n")
 
         # Incomplete console output file (still being written to, as there is no newline at the end).
-        _, cls._incomplete_console_output_file_path = mkstemp()
+        cls._incomplete_console_output_fd, cls._incomplete_console_output_file_path = mkstemp()
         with open(cls._incomplete_console_output_file_path, 'w') as f:
             f.write("\n".join(console_output))
 
     @classmethod
     def tearDownClass(cls):
+        os.close(cls._completed_console_output_fd)
+        os.close(cls._incomplete_console_output_fd)
         os.remove(cls._completed_console_output_file_path)
         os.remove(cls._incomplete_console_output_file_path)