Have the master always fetch and clone deep, but slave shallow.

Turns out you cannot fetch from a shallow repo in the older versions
of git.

app/project_type/git.py

@@ -1,11 +1,11 @@
 import os
+import shutil
 from urllib.parse import urlparse
 
 from app.project_type.project_type import ProjectType
 from app.util import fs, log
 from app.util.conf.configuration import Configuration
 
-
 class Git(ProjectType):
     """
     Example API call to invoke a git-type build.
@@ -148,22 +148,33 @@ def _fetch_project(self):
         """
         Clones the project if necessary, fetches from the remote repo and resets to the requested commit
         """
+        # If shallow_clones is set to True, then we need to specify the --depth=1 argument to all git fetch
+        # and clone invocations.
+        git_clone_fetch_depth_arg = ''
+        if Configuration['shallow_clones']:
+            git_clone_fetch_depth_arg = '--depth=1'
+
+        existing_repo_is_shallow = os.path.isfile(os.path.join(self._repo_directory, '.git', 'shallow'))
+
+        # If we disable shallow clones, but the existing repo is shallow, we must re-clone non-shallowly.
+        if not Configuration['shallow_clones'] and existing_repo_is_shallow and os.path.exists(self._repo_directory):
+            shutil.rmtree(self._repo_directory)
+            fs.create_dir(self._repo_directory, self.DIRECTORY_PERMISSIONS)
+
         # Clone the repo if it doesn't exist
         try:
             self._execute_git_command_in_repo_and_raise_on_failure('rev-parse')  # rev-parse succeeds if repo exists
         except RuntimeError:
             self._logger.notice('No valid repo in "{}". Cloning fresh from "{}".', self._repo_directory, self._url)
-            # Clone with --depth=1 for shallow clones in order to improve performance.
             self._execute_git_command_in_repo_and_raise_on_failure(
-                git_command='clone --depth=1 {} {}'. format(self._url, self._repo_directory),
+                git_command='clone {} {} {}'. format(git_clone_fetch_depth_arg, self._url, self._repo_directory),
                 error_msg='Could not clone repo.'
             )
 
         # Must add the --update-head-ok in the scenario that the current branch of the working directory
         # is equal to self._branch, otherwise the git fetch will exit with a non-zero exit code.
-        # Must add the --depth=1 in order to have shallow fetches in order to improve performance.
         self._execute_git_command_in_repo_and_raise_on_failure(
-            git_command='fetch --depth=1 --update-head-ok {} {}'.format(self._remote, self._branch),
+            git_command='fetch {} --update-head-ok {} {}'.format(git_clone_fetch_depth_arg, self._remote, self._branch),
             error_msg='Could not fetch specified branch "{}" from remote "{}".'.format(self._branch, self._remote)
         )
 

app/util/conf/base_config_loader.py

@@ -92,6 +92,10 @@ def configure_defaults(self, conf):
         # the same way how the master gets the project.
         conf.set('get_project_from_master', True)
 
+        # Should we have shallow or full clones of the repository?
+        # The master must have full clones, as slaves fetch from the master, and one cannot fetch from a shallow clone.
+        conf.set('shallow_clones', False)
+
     def configure_postload(self, conf):
         """
         After the clusterrunner.conf file has been loaded, generate the paths which descend from the base_directory

app/util/conf/master_config_loader.py

@@ -14,11 +14,10 @@ def configure_defaults(self, conf):
         :type conf: Configuration
         """
         super().configure_defaults(conf)
-
         conf.set('port', 43000)
-
         conf.set('log_filename', 'clusterrunner_master.log')
         conf.set('eventlog_filename', 'eventlog_master.log')
+        conf.set('shallow_clones', False)
 
     def configure_postload(self, conf):
         """

app/util/conf/slave_config_loader.py

@@ -13,15 +13,13 @@ def configure_defaults(self, conf):
         :type conf: Configuration
         """
         super().configure_defaults(conf)
-
         conf.set('port', 43001)
         conf.set('num_executors', 1)
-
         conf.set('log_filename', 'clusterrunner_slave.log')
         conf.set('eventlog_filename', 'eventlog_slave.log')
-
         conf.set('master_hostname', 'localhost')
         conf.set('master_port', 43000)
+        conf.set('shallow_clones', True)
 
     def configure_postload(self, conf):
         """