fix(submodules): add support for private submodules (#8164)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

lib/manager/git-submodules/__fixtures__/.gitmodules.5

@@ -6,4 +6,4 @@
 	url = https://github.com/renovatebot/pro.git
 [submodule "renovate-config"]
 	path = deps/renovate-config
-	url = https://github.com/renovatebot/renovate-config.git
+	url = git@github.com:renovatebot/renovate-config.git

lib/manager/git-submodules/extract.spec.ts

@@ -54,6 +54,9 @@ describe('lib/manager/gitsubmodules/extract', () => {
       expect(res.deps).toHaveLength(1);
       res = await extractPackageFile('', '.gitmodules.5', { localDir });
       expect(res.deps).toHaveLength(3);
+      expect(res.deps[2].registryUrls[0]).toEqual(
+        'https://github.com/renovatebot/renovate-config.git'
+      );
     });
   });
 });

lib/manager/git-submodules/extract.ts

@@ -4,6 +4,8 @@ import upath from 'upath';
 
 import * as datasourceGitSubmodules from '../../datasource/git-submodules';
 import { logger } from '../../logger';
+import { getHttpUrl } from '../../util/git';
+import * as hostRules from '../../util/host-rules';
 import { ManagerConfig, PackageFile } from '../common';
 
 type GitModule = {
@@ -109,14 +111,19 @@ export default async function extractPackageFile(
             .replace(/^[-+]/, '')
             .split(/\s/);
           const subModuleUrl = await getUrl(git, gitModulesPath, name);
+          // hostRules only understands HTTP URLs
+          // Find HTTP URL, then apply token
+          let httpSubModuleUrl = getHttpUrl(subModuleUrl);
+          const hostRule = hostRules.find({ url: httpSubModuleUrl });
+          httpSubModuleUrl = getHttpUrl(subModuleUrl, hostRule?.token);
           const submoduleBranch = await getBranch(
             gitModulesPath,
             name,
-            subModuleUrl
+            httpSubModuleUrl
           );
           return {
             depName: path,
-            registryUrls: [subModuleUrl, submoduleBranch],
+            registryUrls: [httpSubModuleUrl, submoduleBranch],
             currentValue,
             currentDigest: currentValue,
           };

lib/util/git/index.ts

@@ -1,5 +1,6 @@
 import URL from 'url';
 import fs from 'fs-extra';
+import GitUrlParse from 'git-url-parse';
 import Git, {
   DiffResult as DiffResult_,
   ResetMode,
@@ -693,3 +694,9 @@ export function getUrl({
     pathname: repository + '.git',
   });
 }
+
+export function getHttpUrl(url: string, token?: string): string {
+  const parsedUrl = GitUrlParse(url);
+  parsedUrl.token = token;
+  return parsedUrl.toString('https');
+}

package.json

@@ -137,6 +137,7 @@
     "fast-safe-stringify": "2.0.7",
     "find-up": "5.0.0",
     "fs-extra": "9.0.1",
+    "git-url-parse": "11.4.3",
     "github-url-from-git": "1.5.0",
     "global-agent": "2.1.12",
     "got": "11.8.1",
@@ -203,6 +204,7 @@
     "@types/conventional-commits-detector": "1.0.0",
     "@types/eslint": "7.2.6",
     "@types/fs-extra": "9.0.5",
+    "@types/git-url-parse": "9.0.0",
     "@types/github-url-from-git": "1.5.0",
     "@types/global-agent": "2.1.0",
     "@types/ini": "1.3.30",

yarn.lock

@@ -1592,6 +1592,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/git-url-parse@9.0.0":
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/@types/git-url-parse/-/git-url-parse-9.0.0.tgz#aac1315a44fa4ed5a52c3820f6c3c2fb79cbd12d"
+  integrity sha512-kA2RxBT/r/ZuDDKwMl+vFWn1Z0lfm1/Ik6Qb91wnSzyzCDa/fkM8gIOq6ruB7xfr37n6Mj5dyivileUVKsidlg==
+
 "@types/github-url-from-git@1.5.0":
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/@types/github-url-from-git/-/github-url-from-git-1.5.0.tgz#cfb8fda460cf22402d4465e1caee6f9063a30fdf"
@@ -4815,6 +4820,21 @@ git-raw-commits@^2.0.0:
     split2 "^2.0.0"
     through2 "^4.0.0"
 
+git-up@^4.0.0:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/git-up/-/git-up-4.0.2.tgz#10c3d731051b366dc19d3df454bfca3f77913a7c"
+  integrity sha512-kbuvus1dWQB2sSW4cbfTeGpCMd8ge9jx9RKnhXhuJ7tnvT+NIrTVfYZxjtflZddQYcmdOTlkAcjmx7bor+15AQ==
+  dependencies:
+    is-ssh "^1.3.0"
+    parse-url "^5.0.0"
+
+git-url-parse@11.4.3:
+  version "11.4.3"
+  resolved "https://registry.yarnpkg.com/git-url-parse/-/git-url-parse-11.4.3.tgz#1610284edf1f14964180f5b3399ec68b692cfd87"
+  integrity sha512-LZTTk0nqJnKN48YRtOpR8H5SEfp1oM2tls90NuZmBxN95PnCvmuXGzqQ4QmVirBgKx2KPYfPGteX3/raWjKenQ==
+  dependencies:
+    git-up "^4.0.0"
+
 github-url-from-git@1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/github-url-from-git/-/github-url-from-git-1.5.0.tgz#f985fedcc0a9aa579dc88d7aff068d55cc6251a0"
@@ -5649,6 +5669,13 @@ is-retry-allowed@^1.0.0:
   resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4"
   integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==
 
+is-ssh@^1.3.0:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/is-ssh/-/is-ssh-1.3.2.tgz#a4b82ab63d73976fd8263cceee27f99a88bdae2b"
+  integrity sha512-elEw0/0c2UscLrNG+OAorbP539E3rhliKPg+hDMWN9VwrDXfYK+4PBEykDPfxlYYtQvl84TascnQyobfQLHEhQ==
+  dependencies:
+    protocols "^1.1.0"
+
 is-stream@^1.0.0, is-stream@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
@@ -7581,6 +7608,11 @@ normalize-path@^3.0.0, normalize-path@~3.0.0:
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
   integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
 
+normalize-url@^3.3.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559"
+  integrity sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==
+
 normalize-url@^4.1.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.0.tgz#453354087e6ca96957bd8f5baf753f5982142129"
@@ -8271,6 +8303,24 @@ parse-passwd@^1.0.0:
   resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
   integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=
 
+parse-path@^4.0.0:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/parse-path/-/parse-path-4.0.2.tgz#ef14f0d3d77bae8dd4bc66563a4c151aac9e65aa"
+  integrity sha512-HSqVz6iuXSiL8C1ku5Gl1Z5cwDd9Wo0q8CoffdAghP6bz8pJa1tcMC+m4N+z6VAS8QdksnIGq1TB6EgR4vPR6w==
+  dependencies:
+    is-ssh "^1.3.0"
+    protocols "^1.4.0"
+
+parse-url@^5.0.0:
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/parse-url/-/parse-url-5.0.2.tgz#856a3be1fcdf78dc93fc8b3791f169072d898b59"
+  integrity sha512-Czj+GIit4cdWtxo3ISZCvLiUjErSo0iI3wJ+q9Oi3QuMYTI6OZu+7cewMWZ+C1YAnKhYTk6/TLuhIgCypLthPA==
+  dependencies:
+    is-ssh "^1.3.0"
+    normalize-url "^3.3.0"
+    parse-path "^4.0.0"
+    protocols "^1.4.0"
+
 parse5@5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178"
@@ -8565,6 +8615,11 @@ proto-list@~1.2.1:
   resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849"
   integrity sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=
 
+protocols@^1.1.0, protocols@^1.4.0:
+  version "1.4.8"
+  resolved "https://registry.yarnpkg.com/protocols/-/protocols-1.4.8.tgz#48eea2d8f58d9644a4a32caae5d5db290a075ce8"
+  integrity sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg==
+
 protoduck@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/protoduck/-/protoduck-5.0.1.tgz#03c3659ca18007b69a50fd82a7ebcc516261151f"