Skip cloning from master if 'get_project_from_master' is not set

app/project_type/git.py

@@ -137,14 +137,15 @@ def slave_param_overrides(self):
         """
         param_overrides = super().slave_param_overrides()
 
-        # We modify the repo url so the slave clones or fetches from the master directly. This should be faster than
-        # cloning/fetching from the original git remote.
-        master_repo_url = 'ssh://{}{}'.format(Configuration['hostname'], self._repo_directory)
-        param_overrides['url'] = master_repo_url  # This causes the slave to clone directly from the master.
-
-        # The user-specified branch is overwritten with a locally created ref so that slaves working on a job can
-        # continue to fetch the same HEAD, even if the master resets the user-specified branch for another build.
-        param_overrides['branch'] = self._local_ref
+        if Configuration['get_project_from_master']:
+            # We modify the repo url so the slave clones or fetches from the master directly. This should be faster than
+            # cloning/fetching from the original git remote.
+            master_repo_url = 'ssh://{}{}'.format(Configuration['hostname'], self._repo_directory)
+            param_overrides['url'] = master_repo_url  # This causes the slave to clone directly from the master.
+
+            # The user-specified branch is overwritten with a locally created ref so that slaves working on a job can
+            # continue to fetch the same HEAD, even if the master resets the user-specified branch for another build.
+            param_overrides['branch'] = self._local_ref
 
         return param_overrides
 

app/util/conf/base_config_loader.py

@@ -87,6 +87,11 @@ def configure_defaults(self, conf):
         conf.set('git_askpass_exe', join(bin_dir, 'git_askpass.sh'))
         conf.set('git_ssh_exe', join(bin_dir, 'git_ssh.sh'))
 
+        # How slaves get the project
+        # Slaves would get the project from master if set to True. Otherwise it would just get the project in
+        # the same way how the master gets the project.
+        conf.set('get_project_from_master', False)
+
     def configure_postload(self, conf):
         """
         After the clusterrunner.conf file has been loaded, generate the paths which descend from the base_directory
@@ -131,6 +136,7 @@ def _get_config_file_whitelisted_keys(self):
             'eventlog_filename',
             'git_strict_host_key_checking',
             'cors_allowed_origins_regex',
+            'get_project_from_master',
         ]
 
     def _load_section_from_config_file(self, config, config_filename, section):

conf/default_clusterrunner.conf

@@ -26,6 +26,9 @@
 ## CORS support - a regex to match against allowed API request origins, or None to disable CORS
 # cors_allowed_origins_regex = None
 
+## Should the slaves get the project from master or not
+# get_project_from_master = False
+
 [master]
 ## The port the master service will run on
 # port = 43000

test/functional/test_cluster_basic.py

@@ -1,6 +1,5 @@
 import os
 import tempfile
-from unittest import skip
 
 from genty import genty, genty_dataset
 
@@ -46,8 +45,6 @@ def test_basic_directory_configs_end_to_end(self, test_job_config):
         self.assert_directory_contents_match_expected(
             dir_path=project_dir.name, expected_dir_contents=test_job_config.expected_project_dir_contents)
 
-    # todo: Skipping for now since this fails on Travis-CI due to the slave being unable to ssh into the master.
-    @skip
     def test_git_type_demo_project_config(self):
         master = self.cluster.start_master()
         self.cluster.start_slave(num_executors_per_slave=10)

test/unit/project_type/test_git.py

@@ -1,4 +1,6 @@
+import os
 from subprocess import Popen
+from unittest import skipIf
 from unittest.mock import ANY, call, MagicMock, Mock
 
 from genty import genty, genty_dataset
@@ -158,11 +160,13 @@ def test_execute_git_command_auto_sets_strict_host_option_correctly(self, strict
         self.assertIn(expected_call, popen_mock.call_args_list, 'Executed git command should include the correct '
                                                                 'option for StrictHostKeyChecking.')
 
+    @skipIf(os.name == 'nt', 'Skipping test for cloning repo from master on Windows')
     def test_slave_param_overrides_returns_expected(self):
+        Configuration['get_project_from_master'] = True
+        Configuration['repo_directory'] = '/repo-directory'
         self._patch_popen({
             'git rev-parse FETCH_HEAD': _FakePopenResult(stdout='deadbee123\n')
         })
-        Configuration['repo_directory'] = '/repo-directory'
 
         git = Git(url='http://original-user-specified-url.test/repo-path/repo-name')
         git.fetch_project()
@@ -175,6 +179,14 @@ def test_slave_param_overrides_returns_expected(self):
         self.assertEqual(expected_overrides, actual_overrides, 'Slave param overrides from Git object should match'
                                                                'expected.')
 
+    def test_slave_param_overrides_returns_empty_dict_when_get_project_from_master_is_disabled(self):
+        Configuration['get_project_from_master'] = False
+
+        git = Git(url='http://original-user-specified-url.test/repo-path/repo-name')
+        actual_overrides = git.slave_param_overrides()
+
+        self.assertEqual({}, actual_overrides)
+
     def _patch_popen(self, command_to_result_map=None):
         """
         Mock out calls to Popen to inject fake results for specific command strings.