feat(helm): improve subdirectory support (#19181)

lib/modules/datasource/helm/__fixtures__/sample.yaml

@@ -89,6 +89,18 @@ entries:
     urls:
     - pgadmin4-1.7.2.tgz
     version: 1.7.2
+  private-chart-github:
+    - description: Some private chart from a self-hosted GitHub Enterprise server
+      home: https://github.example.com/some-org/charts/tree/master/private-chart
+      urls:
+        - https://charts.example.com/private-chart-1.2.3.tgz
+      version: 1.2.3
+  private-chart-gitlab:
+    - description: Some private chart from a self-hosted GitLab server
+      home: https://gitlab.example.com/some/group/charts/-/tree/master/private-chart
+      urls:
+        - https://charts.example.com/private-chart-1.2.3.tgz
+      version: 1.2.3
   dummy:
   - home:
     urls:

lib/modules/datasource/helm/common.spec.ts

@@ -11,11 +11,13 @@ const repo = load(Fixtures.get('sample.yaml'), {
 describe('modules/datasource/helm/common', () => {
   describe('findSourceUrl', () => {
     test.each`
-      input         | output
-      ${'airflow'}  | ${{ sourceUrl: 'https://github.com/bitnami/charts', sourceDirectory: 'bitnami/airflow' }}
-      ${'coredns'}  | ${{ sourceUrl: 'https://github.com/coredns/helm', sourceDirectory: undefined }}
-      ${'pgadmin4'} | ${{ sourceUrl: 'https://github.com/rowanruseler/helm-charts', sourceDirectory: undefined }}
-      ${'dummy'}    | ${{}}
+      input                     | output
+      ${'airflow'}              | ${'https://github.com/bitnami/charts/tree/master/bitnami/airflow'}
+      ${'coredns'}              | ${'https://github.com/coredns/helm'}
+      ${'pgadmin4'}             | ${'https://github.com/rowanruseler/helm-charts'}
+      ${'private-chart-github'} | ${'https://github.example.com/some-org/charts/tree/master/private-chart'}
+      ${'private-chart-gitlab'} | ${'https://gitlab.example.com/some/group/charts/-/tree/master/private-chart'}
+      ${'dummy'}                | ${null}
     `(
       '$input -> $output',
       ({ input, output }: { input: string; output: string }) => {

lib/modules/datasource/helm/common.ts

@@ -1,45 +1,43 @@
+import { detectPlatform } from '../../../util/common';
+import { parseGitUrl } from '../../../util/git/url';
 import { regEx } from '../../../util/regex';
-import type { HelmRelease, RepoSource } from './types';
+import type { HelmRelease } from './types';
 
 const chartRepo = regEx(/charts?|helm|helm-charts/i);
-const githubUrl = regEx(
-  /^(?<url>https:\/\/github\.com\/[^/]+\/(?<repo>[^/]+))(:?\/|\/tree\/[^/]+\/(?<path>.+))?$/
-);
 const githubRelease = regEx(
   /^(https:\/\/github\.com\/[^/]+\/[^/]+)\/releases\//
 );
 
-export function findSourceUrl(release: HelmRelease): RepoSource {
+function isPossibleChartRepo(url: string): boolean {
+  if (detectPlatform(url) === null) {
+    return false;
+  }
+
+  const parsed = parseGitUrl(url);
+  return chartRepo.test(parsed.name);
+}
+
+export function findSourceUrl(release: HelmRelease): string | null {
   // it's a github release :)
   const releaseMatch = githubRelease.exec(release.urls[0]);
   if (releaseMatch) {
-    return { sourceUrl: releaseMatch[1] };
+    return releaseMatch[1];
   }
 
-  if (release.home) {
-    const githubUrlMatch = githubUrl.exec(release.home);
-    if (githubUrlMatch?.groups && chartRepo.test(githubUrlMatch?.groups.repo)) {
-      return {
-        sourceUrl: githubUrlMatch.groups.url,
-        sourceDirectory: githubUrlMatch.groups.path,
-      };
-    }
+  if (release.home && isPossibleChartRepo(release.home)) {
+    return release.home;
   }
 
   if (!release.sources?.length) {
-    return {};
+    return null;
   }
 
   for (const url of release.sources) {
-    const githubUrlMatch = githubUrl.exec(url);
-    if (githubUrlMatch?.groups && chartRepo.test(githubUrlMatch?.groups.repo)) {
-      return {
-        sourceUrl: githubUrlMatch.groups.url,
-        sourceDirectory: githubUrlMatch.groups.path,
-      };
+    if (isPossibleChartRepo(url)) {
+      return url;
     }
   }
 
   // fallback
-  return { sourceUrl: release.sources[0] };
+  return release.sources[0];
 }

lib/modules/datasource/helm/index.ts

@@ -61,11 +61,10 @@ export class HelmDatasource extends Datasource {
       const result: HelmRepositoryData = {};
       for (const [name, releases] of Object.entries(doc.entries)) {
         const latestRelease = releases[0];
-        const { sourceUrl, sourceDirectory } = findSourceUrl(latestRelease);
+        const sourceUrl = findSourceUrl(latestRelease);
         result[name] = {
           homepage: latestRelease.home,
           sourceUrl,
-          sourceDirectory,
           releases: releases.map((release) => ({
             version: release.version,
             releaseTimestamp: release.created ?? null,

lib/modules/datasource/helm/types.ts

@@ -13,8 +13,3 @@ export interface HelmRepository {
 }
 
 export type HelmRepositoryData = Record<string, ReleaseResult>;
-
-export interface RepoSource {
-  sourceUrl?: string;
-  sourceDirectory?: string;
-}

lib/modules/datasource/metadata.spec.ts

@@ -1,3 +1,4 @@
+import { HelmDatasource } from './helm';
 import { MavenDatasource } from './maven';
 import {
   addMetaData,
@@ -77,6 +78,65 @@ describe('modules/datasource/metadata', () => {
     });
   });
 
+  test.each`
+    sourceUrl                                                                  | expectedSourceUrl                            | expectedSourceDirectory
+    ${'https://github.com/bitnami/charts/tree/master/bitnami/kube-prometheus'} | ${'https://github.com/bitnami/charts'}       | ${'bitnami/kube-prometheus'}
+    ${'https://gitlab.com/group/sub-group/repo/tree/main/some/path'}           | ${'https://gitlab.com/group/sub-group/repo'} | ${'some/path'}
+    ${'https://gitlab.com/group/sub-group/repo/-/tree/main/some/path'}         | ${'https://gitlab.com/group/sub-group/repo'} | ${'some/path'}
+    ${'https://github.example.com/org/repo/tree/main/foo/bar/baz'}             | ${'https://github.example.com/org/repo'}     | ${'foo/bar/baz'}
+  `(
+    'Should split the sourceDirectory out of sourceUrl for known platforms: $sourceUrl -> ($expectedSourceUrl, $expectedSourceDirectory)',
+    ({ sourceUrl, expectedSourceUrl, expectedSourceDirectory }) => {
+      const dep: ReleaseResult = { sourceUrl, releases: [] };
+      const datasource = HelmDatasource.id;
+      const packageName = 'some-chart';
+
+      addMetaData(dep, datasource, packageName);
+      expect(dep).toMatchObject({
+        sourceUrl: expectedSourceUrl,
+      });
+    }
+  );
+
+  test.each`
+    sourceUrl
+    ${'https://github.com/bitnami'}
+    ${'https://github.com/bitnami/charts'}
+    ${'https://gitlab.com/group'}
+    ${'https://gitlab.com/group/repo'}
+    ${'https://gitlab.com/group/sub-group/repo'}
+    ${'https://github.example.com/org/repo'}
+    ${'https://unknown-platform.com/some/repo/files/foo/bar'}
+  `(
+    'Should not split a sourceDirectory when one cannot be detected $sourceUrl',
+    ({ sourceUrl }) => {
+      const dep: ReleaseResult = { sourceUrl, releases: [] };
+      const datasource = HelmDatasource.id;
+      const packageName = 'some-chart';
+
+      addMetaData(dep, datasource, packageName);
+      expect(dep.sourceDirectory).toBeUndefined();
+      expect(dep).toMatchObject({ sourceUrl });
+    }
+  );
+
+  it('Should not overwrite any existing sourceDirectory', () => {
+    const dep: ReleaseResult = {
+      sourceUrl:
+        'https://github.com/neutrinojs/neutrino/tree/master/packages/react',
+      sourceDirectory: 'packages/foo',
+      releases: [],
+    };
+    const datasource = NpmDatasource.id;
+    const packageName = '@neutrinojs/react';
+
+    addMetaData(dep, datasource, packageName);
+    expect(dep).toMatchObject({
+      sourceUrl: 'https://github.com/neutrinojs/neutrino',
+      sourceDirectory: 'packages/foo',
+    });
+  });
+
   it('Should massage github sourceUrls', () => {
     const dep: ReleaseResult = {
       sourceUrl: 'https://some.github.com/repo',

lib/modules/datasource/metadata.ts

@@ -2,6 +2,7 @@ import is from '@sindresorhus/is';
 import parse from 'github-url-from-git';
 import { DateTime } from 'luxon';
 import { detectPlatform } from '../../util/common';
+import { parseGitUrl } from '../../util/git/url';
 import * as hostRules from '../../util/host-rules';
 import { regEx } from '../../util/regex';
 import { parseUrl, trimTrailingSlash, validateUrl } from '../../util/url';
@@ -131,6 +132,18 @@ export function addMetaData(
     dep.sourceUrl = manualSourceUrl;
   }
 
+  if (dep.sourceUrl && !dep.sourceDirectory) {
+    try {
+      const parsed = parseGitUrl(dep.sourceUrl);
+      if (parsed.filepathtype === 'tree' && parsed.filepath !== '') {
+        dep.sourceUrl = parsed.toString();
+        dep.sourceDirectory = parsed.filepath;
+      }
+    } catch (err) {
+      // ignore invalid urls
+    }
+  }
+
   if (
     !dep.sourceUrl &&
     dep.changelogUrl &&

lib/modules/datasource/npm/get.spec.ts

@@ -259,8 +259,10 @@ describe('modules/datasource/npm/get', () => {
     const registryUrl = resolveRegistryUrl('@neutrinojs/react');
     const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
 
-    expect(dep?.sourceUrl).toBe('https://github.com/neutrinojs/neutrino');
-    expect(dep?.sourceDirectory).toBe('packages/react');
+    expect(dep?.sourceUrl).toBe(
+      'https://github.com/neutrinojs/neutrino/tree/master/packages/react'
+    );
+    expect(dep?.sourceDirectory).toBeUndefined();
 
     expect(httpMock.getTrace()).toMatchInlineSnapshot(`
       [
@@ -296,8 +298,10 @@ describe('modules/datasource/npm/get', () => {
     const registryUrl = resolveRegistryUrl('@neutrinojs/react');
     const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
 
-    expect(dep?.sourceUrl).toBe('https://github.com/neutrinojs/neutrino');
-    expect(dep?.sourceDirectory).toBe('packages/react');
+    expect(dep?.sourceUrl).toBe(
+      'https://github.com/neutrinojs/neutrino/tree/master/packages/react'
+    );
+    expect(dep?.sourceDirectory).toBeUndefined();
   });
 
   it('handles mixed sourceUrls in releases', async () => {
@@ -398,7 +402,9 @@ describe('modules/datasource/npm/get', () => {
     const registryUrl = resolveRegistryUrl('@neutrinojs/react');
     const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
 
-    expect(dep?.sourceUrl).toBe('https://github.com/neutrinojs/neutrino');
+    expect(dep?.sourceUrl).toBe(
+      'https://github.com/neutrinojs/neutrino/tree/master/packages/react'
+    );
     expect(dep?.sourceDirectory).toBe('packages/foo');
 
     expect(httpMock.getTrace()).toMatchInlineSnapshot(`

lib/modules/datasource/npm/get.ts

@@ -41,19 +41,6 @@ function getPackageSource(repository: any): PackageSource {
     if (is.nonEmptyString(repository.directory)) {
       res.sourceDirectory = repository.directory;
     }
-    // TODO: types (#7154)
-    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-    const sourceUrlCopy = `${res.sourceUrl}`;
-    const sourceUrlSplit: string[] = sourceUrlCopy.split('/');
-    if (sourceUrlSplit.length > 7 && sourceUrlSplit[2] === 'github.com') {
-      // Massage the repository URL for non-compliant strings for github (see issue #4610)
-      // Remove the non-compliant segments of path, so the URL looks like "<scheme>://<domain>/<vendor>/<repo>"
-      // and add directory to the repository
-      res.sourceUrl = sourceUrlSplit.slice(0, 5).join('/');
-      res.sourceDirectory ||= sourceUrlSplit
-        .slice(7, sourceUrlSplit.length)
-        .join('/');
-    }
   }
   return res;
 }