From b4cd30a3e65cae7234ad45d777eba5de53f8ab7b Mon Sep 17 00:00:00 2001 From: Roman Ivanov Date: Sat, 2 Dec 2023 17:31:50 -0800 Subject: [PATCH] Issue #818: do shallow clone to speedup download --- checkstyle-tester/README.md | 10 ++++++++ checkstyle-tester/diff.groovy | 46 +++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/checkstyle-tester/README.md b/checkstyle-tester/README.md index 18f0cda0..909fe06a 100644 --- a/checkstyle-tester/README.md +++ b/checkstyle-tester/README.md @@ -72,6 +72,16 @@ would add `--extraMvnRegressionOptions "-Dmaven.site.skip=true"` to `diff.groovy (optional, default is false). This option is used for files that are not compilable or that Checkstyle cannot parse. +**useShallowClone** (h) - this option tells `diff.groovy` to use shallow clone for repositories + defined in the file + specified for the `--listOfProjects (-l)` argument (optional, default is false). This option is + convenient for cases when internet is not stable and it is better to clone minimal required sources. + Shallow cloning cannot be used for projects that reference SHA commit and, by default, + fallback to using normal cloning. ATTENTION: if you change tag/branch in config for project + that previously cloned shallowly there will be error to checkout to new tag/branch + (Example: `fatal: Could not parse object 'c2ac5b90a467aedb04b52ae50a99e83207d847b3'.`), to resovle + problem you need to remove impacted project folder(s) from repositories directory. + ## Outputs When the script finishes its work the following directory structure will be created diff --git a/checkstyle-tester/diff.groovy b/checkstyle-tester/diff.groovy index 2e6a1289..28968a4b 100644 --- a/checkstyle-tester/diff.groovy +++ b/checkstyle-tester/diff.groovy @@ -68,6 +68,8 @@ def getCliOptions(args) { + 'config file (required if baseConfig and patchConfig are not secified)') g(longOpt: 'allowExcludes', required: false, 'Whether to allow excludes specified in the list of ' \ + 'projects (optional, default is false)') + h(longOpt: 'useShallowClone', required: false, 'Whether to use shallow clones for repositories ' \ + + '(optional, default is false)') l(longOpt: 'listOfProjects', args: 1, required: true, argName: 'path', 'Path to file which contains projects to test on (required)') s(longOpt: 'shortFilePaths', required: false, 'Whether to save report file paths' \ @@ -264,6 +266,7 @@ def generateCheckstyleReport(cfg) { def checkstyleConfig = cfg.checkstyleCfg def checkstyleVersion = cfg.checkstyleVersion def allowExcludes = cfg.allowExcludes + def useShallowClone = cfg.useShallowClone def listOfProjectsFile = new File(cfg.listOfProjects) def projects = listOfProjectsFile.readLines() def extraMvnRegressionOptions = cfg.extraMvnRegressionOptions @@ -292,7 +295,11 @@ def generateCheckstyleReport(cfg) { if (repoType == 'local') { copyDir(repoUrl, getOsSpecificPath("$srcDir", "$repoName")) } else { - cloneRepository(repoName, repoType, repoUrl, commitId, reposDir) + if (useShallowClone && !isGitSha(commitId)) { + cloneRepositoryShallowly(repoName, repoType, repoUrl, commitId, reposDir) + } else { + cloneRepository(repoName, repoType, repoUrl, commitId, reposDir) + } copyDir(getOsSpecificPath("$reposDir", "$repoName"), getOsSpecificPath("$srcDir", "$repoName")) } runMavenExecution(srcDir, excludes, checkstyleConfig, @@ -352,16 +359,41 @@ def getCloneCmd(repoType, repoUrl, srcDestinationDir) { return cloneCmd } +def getCloneShallowCmd(repoType, repoUrl, srcDestinationDir, branchOrTag) { + def cloneCmd = '' + switch (repoType) { + case 'git': + cloneCmd = "git clone --depth 1 --branch $branchOrTag $repoUrl $srcDestinationDir" + break + default: + throw new IllegalArgumentException("Error! Unknown $repoType repository.") + } + return cloneCmd +} + +def cloneRepositoryShallowly(repoName, repoType, repoUrl, commitId, srcDir) { + def srcDestinationDir = getOsSpecificPath("$srcDir", "$repoName") + if (!Files.exists(Paths.get(srcDestinationDir))) { + def cloneCmd = getCloneShallowCmd(repoType, repoUrl, srcDestinationDir, commitId) + println "Cloning $repoType repository '$repoName' to $srcDestinationDir folder ..." + println "Cloning command: $cloneCmd" + executeCmdWithRetry(cloneCmd) + println "Cloning $repoType repository '$repoName' - completed\n" + } + println "$repoName is synchronized" +} + def cloneRepository(repoName, repoType, repoUrl, commitId, srcDir) { def srcDestinationDir = getOsSpecificPath("$srcDir", "$repoName") if (!Files.exists(Paths.get(srcDestinationDir))) { def cloneCmd = getCloneCmd(repoType, repoUrl, srcDestinationDir) println "Cloning $repoType repository '$repoName' to $srcDestinationDir folder ..." + println "Cloning command: $cloneCmd" executeCmdWithRetry(cloneCmd) println "Cloning $repoType repository '$repoName' - completed\n" } - if (commitId && commitId != '') { + if (isGitSha(commitId)) { def lastCommitSha = getLastProjectCommitSha(repoType, srcDestinationDir) def commitIdSha = getCommitSha(commitId, repoType, srcDestinationDir) if (lastCommitSha != commitIdSha) { @@ -373,6 +405,11 @@ def cloneRepository(repoName, repoType, repoUrl, commitId, srcDir) { println "$repoName is synchronized" } +// it is not very accurate match, but in case of mismatch we will do full clone +def isGitSha(value) { + return value ==~ /[0-9a-f]{5,40}/ +} + def executeCmdWithRetry(cmd, dir = new File("").getAbsoluteFile(), retry = 5) { def osSpecificCmd = getOsSpecificCmd(cmd) def left = retry @@ -755,6 +792,7 @@ class Config { def checkstyleVersion def sevntuVersion def allowExcludes + def useShallowClone Config(cliOptions) { if (cliOptions.localGitRepo) { @@ -767,6 +805,7 @@ class Config { checkstyleVersion = cliOptions.checkstyleVersion allowExcludes = cliOptions.allowExcludes + useShallowClone = cliOptions.useShallowClone mode = cliOptions.mode if (!mode) { @@ -815,6 +854,7 @@ class Config { destDir: tmpMasterReportsDir, extraMvnRegressionOptions: extraMvnRegressionOptions, allowExcludes:allowExcludes, + useShallowClone: useShallowClone, ] } @@ -827,6 +867,7 @@ class Config { destDir: tmpPatchReportsDir, extraMvnRegressionOptions: extraMvnRegressionOptions, allowExcludes: allowExcludes, + useShallowClone: useShallowClone, ] } @@ -840,6 +881,7 @@ class Config { shortFilePaths: shortFilePaths, mode: mode, allowExcludes: allowExcludes, + useShallowClone: useShallowClone, ] } }