Issue #818: do shallow clone to speedup download

checkstyle-tester/README.md

@@ -72,6 +72,16 @@ would add `--extraMvnRegressionOptions "-Dmaven.site.skip=true"` to `diff.groovy
  (optional, default is false). This option is
  used for files that are not compilable or that Checkstyle cannot parse.
 
+**useShallowClone** (h) - this option tells `diff.groovy` to use shallow clone for repositories
+ defined in the file
+ specified for the `--listOfProjects (-l)` argument (optional, default is false). This option is
+ convenient for cases when internet is not stable and it is better to clone minimal required sources.
+ Shallow cloning cannot be used for projects that reference SHA commit and, by default,
+ fallback to using normal cloning. ATTENTION: if you change tag/branch in config for project
+ that previously cloned shallowly there will be error to checkout to new tag/branch
+ (Example: `fatal: Could not parse object 'c2ac5b90a467aedb04b52ae50a99e83207d847b3'.`), to resovle
+ problem you need to remove impacted project folder(s) from repositories directory.
+
 ## Outputs
 
 When the script finishes its work the following directory structure will be created

checkstyle-tester/diff.groovy

@@ -68,6 +68,8 @@ def getCliOptions(args) {
             + 'config file (required if baseConfig and patchConfig are not secified)')
         g(longOpt: 'allowExcludes', required: false, 'Whether to allow excludes specified in the list of ' \
             + 'projects (optional, default is false)')
+        h(longOpt: 'useShallowClone', required: false, 'Whether to use shallow clones for repositories ' \
+            + '(optional, default is false)')
         l(longOpt: 'listOfProjects', args: 1, required: true, argName: 'path',
             'Path to file which contains projects to test on (required)')
         s(longOpt: 'shortFilePaths', required: false, 'Whether to save report file paths' \
@@ -264,6 +266,7 @@ def generateCheckstyleReport(cfg) {
     def checkstyleConfig = cfg.checkstyleCfg
     def checkstyleVersion = cfg.checkstyleVersion
     def allowExcludes = cfg.allowExcludes
+    def useShallowClone = cfg.useShallowClone
     def listOfProjectsFile = new File(cfg.listOfProjects)
     def projects = listOfProjectsFile.readLines()
     def extraMvnRegressionOptions = cfg.extraMvnRegressionOptions
@@ -292,7 +295,11 @@ def generateCheckstyleReport(cfg) {
                 if (repoType == 'local') {
                     copyDir(repoUrl, getOsSpecificPath("$srcDir", "$repoName"))
                 } else {
-                    cloneRepository(repoName, repoType, repoUrl, commitId, reposDir)
+                    if (useShallowClone && !isGitSha(commitId)) {
+                        cloneRepositoryShallowly(repoName, repoType, repoUrl, commitId, reposDir)
+                    } else {
+                        cloneRepository(repoName, repoType, repoUrl, commitId, reposDir)
+                    }
                     copyDir(getOsSpecificPath("$reposDir", "$repoName"), getOsSpecificPath("$srcDir", "$repoName"))
                 }
                 runMavenExecution(srcDir, excludes, checkstyleConfig,
@@ -352,16 +359,41 @@ def getCloneCmd(repoType, repoUrl, srcDestinationDir) {
     return cloneCmd
 }
 
+def getCloneShallowCmd(repoType, repoUrl, srcDestinationDir, branchOrTag) {
+    def cloneCmd = ''
+    switch (repoType) {
+        case 'git':
+            cloneCmd = "git clone --depth 1 --branch $branchOrTag $repoUrl $srcDestinationDir"
+            break
+        default:
+            throw new IllegalArgumentException("Error! Unknown $repoType repository.")
+    }
+    return cloneCmd
+}
+
+def cloneRepositoryShallowly(repoName, repoType, repoUrl, commitId, srcDir) {
+    def srcDestinationDir = getOsSpecificPath("$srcDir", "$repoName")
+    if (!Files.exists(Paths.get(srcDestinationDir))) {
+        def cloneCmd = getCloneShallowCmd(repoType, repoUrl, srcDestinationDir, commitId)
+        println "Cloning $repoType repository '$repoName' to $srcDestinationDir folder ..."
+        println "Cloning command: $cloneCmd"
+        executeCmdWithRetry(cloneCmd)
+        println "Cloning $repoType repository '$repoName' - completed\n"
+    }
+    println "$repoName is synchronized"
+}
+
 def cloneRepository(repoName, repoType, repoUrl, commitId, srcDir) {
     def srcDestinationDir = getOsSpecificPath("$srcDir", "$repoName")
     if (!Files.exists(Paths.get(srcDestinationDir))) {
         def cloneCmd = getCloneCmd(repoType, repoUrl, srcDestinationDir)
         println "Cloning $repoType repository '$repoName' to $srcDestinationDir folder ..."
+        println "Cloning command: $cloneCmd"
         executeCmdWithRetry(cloneCmd)
         println "Cloning $repoType repository '$repoName' - completed\n"
     }
 
-    if (commitId && commitId != '') {
+    if (isGitSha(commitId)) {
         def lastCommitSha = getLastProjectCommitSha(repoType, srcDestinationDir)
         def commitIdSha = getCommitSha(commitId, repoType, srcDestinationDir)
         if (lastCommitSha != commitIdSha) {
@@ -373,6 +405,11 @@ def cloneRepository(repoName, repoType, repoUrl, commitId, srcDir) {
     println "$repoName is synchronized"
 }
 
+// it is not very accurate match, but in case of mismatch we will do full clone
+def isGitSha(value) {
+    return value ==~ /[0-9a-f]{5,40}/
+}
+
 def executeCmdWithRetry(cmd, dir = new File("").getAbsoluteFile(), retry = 5) {
     def osSpecificCmd = getOsSpecificCmd(cmd)
     def left = retry
@@ -755,6 +792,7 @@ class Config {
     def checkstyleVersion
     def sevntuVersion
     def allowExcludes
+    def useShallowClone
 
     Config(cliOptions) {
         if (cliOptions.localGitRepo) {
@@ -767,6 +805,7 @@ class Config {
 
         checkstyleVersion = cliOptions.checkstyleVersion
         allowExcludes = cliOptions.allowExcludes
+        useShallowClone = cliOptions.useShallowClone
 
         mode = cliOptions.mode
         if (!mode) {
@@ -815,6 +854,7 @@ class Config {
             destDir: tmpMasterReportsDir,
             extraMvnRegressionOptions: extraMvnRegressionOptions,
             allowExcludes:allowExcludes,
+            useShallowClone: useShallowClone,
         ]
     }
 
@@ -827,6 +867,7 @@ class Config {
             destDir: tmpPatchReportsDir,
             extraMvnRegressionOptions: extraMvnRegressionOptions,
             allowExcludes: allowExcludes,
+            useShallowClone: useShallowClone,
         ]
     }
 
@@ -840,6 +881,7 @@ class Config {
             shortFilePaths: shortFilePaths,
             mode: mode,
             allowExcludes: allowExcludes,
+            useShallowClone: useShallowClone,
         ]
     }
 }