feat(npm): support custom registryUrls (#14622)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

lib/config/presets/npm/index.ts

@@ -1,5 +1,8 @@
 import { logger } from '../../../logger';
-import { resolvePackage } from '../../../modules/datasource/npm/npmrc';
+import {
+  resolvePackageUrl,
+  resolveRegistryUrl,
+} from '../../../modules/datasource/npm/npmrc';
 import type { NpmResponse } from '../../../modules/datasource/npm/types';
 import { Http } from '../../../util/http';
 import type { Preset, PresetConfig } from '../types';
@@ -19,7 +22,8 @@ export async function getPreset({
 }: PresetConfig): Promise<Preset> {
   let dep;
   try {
-    const { packageUrl } = resolvePackage(pkg);
+    const registryUrl = resolveRegistryUrl(pkg);
+    const packageUrl = resolvePackageUrl(registryUrl, pkg);
     // istanbul ignore if
     if (!packageUrl.startsWith('https://registry.npmjs.org/')) {
       logger.warn(

lib/modules/datasource/index.ts

@@ -11,6 +11,8 @@ import { trimTrailingSlash } from '../../util/url';
 import * as allVersioning from '../versioning';
 import datasources from './api';
 import { addMetaData } from './metadata';
+import { setNpmrc } from './npm';
+import { resolveRegistryUrl } from './npm/npmrc';
 import type {
   DatasourceApi,
   DigestConfig,
@@ -239,15 +241,24 @@ async function fetchReleases(
   config: GetReleasesInternalConfig
 ): Promise<ReleaseResult | null> {
   const { datasource: datasourceName } = config;
+  let { registryUrls } = config;
   if (!datasourceName || getDatasourceFor(datasourceName) === undefined) {
     logger.warn('Unknown datasource: ' + datasourceName);
     return null;
   }
+  if (datasourceName === 'npm') {
+    if (is.string(config.npmrc)) {
+      setNpmrc(config.npmrc);
+    }
+    if (!is.nonEmptyArray(registryUrls)) {
+      registryUrls = [resolveRegistryUrl(config.packageName)];
+    }
+  }
   const datasource = getDatasourceFor(datasourceName);
-  const registryUrls = resolveRegistryUrls(
+  registryUrls = resolveRegistryUrls(
     datasource,
     config.defaultRegistryUrls,
-    config.registryUrls
+    registryUrls
   );
   let dep: ReleaseResult = null;
   const registryStrategy = datasource.registryStrategy || 'hunt';

lib/modules/datasource/npm/__snapshots__/index.spec.ts.snap

@@ -19,7 +19,7 @@ Array [
 exports[`modules/datasource/npm/index should fetch package info from custom registry 1`] = `
 Object {
   "name": "foobar",
-  "registryUrl": "https://npm.mycustomregistry.com/",
+  "registryUrl": "https://npm.mycustomregistry.com",
   "releases": Array [
     Object {
       "releaseTimestamp": "2018-05-06T05:21:53.000Z",
@@ -561,7 +561,7 @@ Array [
 exports[`modules/datasource/npm/index should use host rules by baseUrl if provided 1`] = `
 Object {
   "name": "foobar",
-  "registryUrl": "https://npm.mycustomregistry.com/_packaging/mycustomregistry/npm/registry/",
+  "registryUrl": "https://npm.mycustomregistry.com/_packaging/mycustomregistry/npm/registry",
   "releases": Array [
     Object {
       "releaseTimestamp": "2018-05-06T05:21:53.000Z",
@@ -600,7 +600,7 @@ Array [
 exports[`modules/datasource/npm/index should use host rules by hostName if provided 1`] = `
 Object {
   "name": "foobar",
-  "registryUrl": "https://npm.mycustomregistry.com/",
+  "registryUrl": "https://npm.mycustomregistry.com",
   "releases": Array [
     Object {
       "releaseTimestamp": "2018-05-06T05:21:53.000Z",

lib/modules/datasource/npm/get.spec.ts

@@ -3,7 +3,7 @@ import { ExternalHostError } from '../../../types/errors/external-host-error';
 import * as hostRules from '../../../util/host-rules';
 import { Http } from '../../../util/http';
 import { getDependency, resetMemCache } from './get';
-import { setNpmrc } from './npmrc';
+import { resolveRegistryUrl, setNpmrc } from './npmrc';
 
 function getPath(s = ''): string {
   const [x] = s.split('\n');
@@ -44,7 +44,8 @@ describe('modules/datasource/npm/get', () => {
         .reply(200, { name: '@myco/test' });
 
       setNpmrc(npmrc);
-      await getDependency(http, '@myco/test');
+      const registryUrl = resolveRegistryUrl('@myco/test');
+      await getDependency(http, registryUrl, '@myco/test');
 
       const trace = httpMock.getTrace();
       expect(trace[0].headers.authorization).toBe('Bearer XXX');
@@ -76,7 +77,8 @@ describe('modules/datasource/npm/get', () => {
         .get(getPath(npmrc))
         .reply(200, { name: '@myco/test' });
       setNpmrc(npmrc);
-      await getDependency(http, '@myco/test');
+      const registryUrl = resolveRegistryUrl('@myco/test');
+      await getDependency(http, registryUrl, '@myco/test');
 
       const trace = httpMock.getTrace();
       expect(trace[0].headers.authorization).toBe('Basic dGVzdDp0ZXN0');
@@ -99,7 +101,8 @@ describe('modules/datasource/npm/get', () => {
         .get(getPath(npmrc))
         .reply(200, { name: '@myco/test' });
       setNpmrc(npmrc);
-      await getDependency(http, '@myco/test');
+      const registryUrl = resolveRegistryUrl('@myco/test');
+      await getDependency(http, registryUrl, '@myco/test');
 
       const trace = httpMock.getTrace();
       expect(trace[0].headers.authorization).toBeUndefined();
@@ -125,7 +128,8 @@ describe('modules/datasource/npm/get', () => {
       .get(getPath(npmrc))
       .reply(200, { name: '@myco/test' });
     setNpmrc(npmrc);
-    await getDependency(http, '@myco/test');
+    const registryUrl = resolveRegistryUrl('@myco/test');
+    await getDependency(http, registryUrl, '@myco/test');
     expect(httpMock.getTrace()).toMatchSnapshot();
   });
 
@@ -146,7 +150,8 @@ describe('modules/datasource/npm/get', () => {
       .get('/renovate')
       .reply(200, { name: 'renovate' });
     setNpmrc(npmrc);
-    await getDependency(http, 'renovate');
+    const registryUrl = resolveRegistryUrl('renovate');
+    await getDependency(http, registryUrl, 'renovate');
     expect(httpMock.getTrace()).toMatchSnapshot();
   });
 
@@ -168,7 +173,8 @@ describe('modules/datasource/npm/get', () => {
       .get('/renovate')
       .reply(200, { name: 'renovate' });
     setNpmrc(npmrc);
-    await getDependency(http, 'renovate');
+    const registryUrl = resolveRegistryUrl('renovate');
+    await getDependency(http, registryUrl, 'renovate');
     expect(httpMock.getTrace()).toMatchSnapshot();
   });
 
@@ -181,7 +187,8 @@ describe('modules/datasource/npm/get', () => {
       .scope('https://test.org')
       .get('/none')
       .reply(200, { name: '@myco/test' });
-    expect(await getDependency(http, 'none')).toBeNull();
+    let registryUrl = resolveRegistryUrl('none');
+    expect(await getDependency(http, registryUrl, 'none')).toBeNull();
 
     httpMock
       .scope('https://test.org')
@@ -192,7 +199,8 @@ describe('modules/datasource/npm/get', () => {
         versions: { '1.0.0': {} },
         'dist-tags': { latest: '1.0.0' },
       });
-    expect(await getDependency(http, '@myco/test')).toBeDefined();
+    registryUrl = resolveRegistryUrl('@myco/test');
+    expect(await getDependency(http, registryUrl, '@myco/test')).toBeDefined();
 
     httpMock
       .scope('https://test.org')
@@ -202,34 +210,40 @@ describe('modules/datasource/npm/get', () => {
         versions: { '1.0.0': {} },
         'dist-tags': { latest: '1.0.0' },
       });
-    expect(await getDependency(http, '@myco/test2')).toBeDefined();
+    registryUrl = resolveRegistryUrl('@myco/test2');
+    expect(await getDependency(http, registryUrl, '@myco/test2')).toBeDefined();
 
     httpMock.scope('https://test.org').get('/error-401').reply(401);
-    expect(await getDependency(http, 'error-401')).toBeNull();
+    registryUrl = resolveRegistryUrl('error-401');
+    expect(await getDependency(http, registryUrl, 'error-401')).toBeNull();
 
     httpMock.scope('https://test.org').get('/error-402').reply(402);
-    expect(await getDependency(http, 'error-402')).toBeNull();
+    registryUrl = resolveRegistryUrl('error-402');
+    expect(await getDependency(http, registryUrl, 'error-402')).toBeNull();
 
     httpMock.scope('https://test.org').get('/error-404').reply(404);
-    expect(await getDependency(http, 'error-404')).toBeNull();
+    registryUrl = resolveRegistryUrl('error-404');
+    expect(await getDependency(http, registryUrl, 'error-404')).toBeNull();
 
     httpMock.scope('https://test.org').get('/error4').reply(200, null);
-    expect(await getDependency(http, 'error4')).toBeNull();
+    registryUrl = resolveRegistryUrl('error4');
+    expect(await getDependency(http, registryUrl, 'error4')).toBeNull();
 
     setNpmrc();
     httpMock
       .scope('https://registry.npmjs.org')
       .get('/npm-parse-error')
       .reply(200, 'not-a-json');
-    await expect(getDependency(http, 'npm-parse-error')).rejects.toThrow(
-      ExternalHostError
-    );
+    registryUrl = resolveRegistryUrl('npm-parse-error');
+    await expect(
+      getDependency(http, registryUrl, 'npm-parse-error')
+    ).rejects.toThrow(ExternalHostError);
 
     httpMock
       .scope('https://registry.npmjs.org')
       .get('/npm-error-402')
       .reply(402);
-    expect(await getDependency(http, 'npm-error-402')).toBeNull();
+    expect(await getDependency(http, registryUrl, 'npm-error-402')).toBeNull();
 
     expect(httpMock.getTrace()).toMatchSnapshot();
   });
@@ -249,8 +263,8 @@ describe('modules/datasource/npm/get', () => {
         versions: { '1.0.0': {} },
         'dist-tags': { latest: '1.0.0' },
       });
-
-    const dep = await getDependency(http, '@neutrinojs/react');
+    const registryUrl = resolveRegistryUrl('@neutrinojs/react');
+    const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
 
     expect(dep.sourceUrl).toBe('https://github.com/neutrinojs/neutrino');
     expect(dep.sourceDirectory).toBe('packages/react');
@@ -300,8 +314,8 @@ describe('modules/datasource/npm/get', () => {
         },
         'dist-tags': { latest: '2.0.0' },
       });
-
-    const dep = await getDependency(http, 'vue');
+    const registryUrl = resolveRegistryUrl('vue');
+    const dep = await getDependency(http, registryUrl, 'vue');
 
     expect(dep.sourceUrl).toBe('https://github.com/vuejs/vue.git');
     expect(dep.releases[0].sourceUrl).toBeUndefined();
@@ -326,8 +340,8 @@ describe('modules/datasource/npm/get', () => {
         versions: { '1.0.0': {} },
         'dist-tags': { latest: '1.0.0' },
       });
-
-    const dep = await getDependency(http, '@neutrinojs/react');
+    const registryUrl = resolveRegistryUrl('@neutrinojs/react');
+    const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
 
     expect(dep.sourceUrl).toBe('https://github.com/neutrinojs/neutrino');
     expect(dep.sourceDirectory).toBe('packages/foo');
@@ -364,8 +378,8 @@ describe('modules/datasource/npm/get', () => {
         versions: { '1.0.0': {} },
         'dist-tags': { latest: '1.0.0' },
       });
-
-    const dep = await getDependency(http, '@neutrinojs/react');
+    const registryUrl = resolveRegistryUrl('@neutrinojs/react');
+    const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
 
     expect(dep.sourceUrl).toBe(
       'https://bitbucket.org/neutrinojs/neutrino/tree/master/packages/react'

lib/modules/datasource/npm/get.ts

@@ -4,8 +4,8 @@ import { logger } from '../../../logger';
 import { ExternalHostError } from '../../../types/errors/external-host-error';
 import * as packageCache from '../../../util/cache/package';
 import type { Http } from '../../../util/http';
+import { joinUrlParts } from '../../../util/url';
 import { id } from './common';
-import { resolvePackage } from './npmrc';
 import type { NpmDependency, NpmRelease, NpmResponse } from './types';
 
 let memcache: Record<string, string> = {};
@@ -52,6 +52,7 @@ function getPackageSource(repository: any): PackageSource {
 
 export async function getDependency(
   http: Http,
+  registryUrl: string,
   packageName: string
 ): Promise<NpmDependency | null> {
   logger.trace(`npm.getDependency(${packageName})`);
@@ -62,7 +63,7 @@ export async function getDependency(
     return JSON.parse(memcache[packageName]) as NpmDependency;
   }
 
-  const { packageUrl, registryUrl } = resolvePackage(packageName);
+  const packageUrl = joinUrlParts(registryUrl, packageName.replace('/', '%2F'));
 
   // Now check the persistent cache
   const cacheNamespace = 'datasource-npm';

lib/modules/datasource/npm/index.ts

@@ -1,20 +1,18 @@
-import is from '@sindresorhus/is';
 import * as npmVersioning from '../../versioning/npm';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 import { id } from './common';
 import { getDependency } from './get';
-import { setNpmrc } from './npmrc';
 
 export { resetMemCache, resetCache } from './get';
 export { setNpmrc } from './npmrc';
 
-export const customRegistrySupport = false;
-
 export class NpmDatasource extends Datasource {
   static readonly id = id;
 
-  override readonly customRegistrySupport = false;
+  override readonly customRegistrySupport = true;
+
+  override readonly registryStrategy = 'first';
 
   override readonly defaultVersioning = npmVersioning.id;
 
@@ -24,12 +22,9 @@ export class NpmDatasource extends Datasource {
 
   async getReleases({
     packageName,
-    npmrc,
+    registryUrl,
   }: GetReleasesConfig): Promise<ReleaseResult | null> {
-    if (is.string(npmrc)) {
-      setNpmrc(npmrc);
-    }
-    const res = await getDependency(this.http, packageName);
+    const res = await getDependency(this.http, registryUrl, packageName);
     if (res) {
       res.tags = res['dist-tags'];
       delete res['dist-tags'];

lib/modules/datasource/npm/npmrc.ts

@@ -10,7 +10,7 @@ import { regEx } from '../../../util/regex';
 import { fromBase64 } from '../../../util/string';
 import { ensureTrailingSlash, validateUrl } from '../../../util/url';
 import { defaultRegistryUrls } from './common';
-import type { NpmrcRules, PackageResolution } from './types';
+import type { NpmrcRules } from './types';
 
 let npmrc: Record<string, any> = {};
 let npmrcRaw = '';
@@ -179,11 +179,12 @@ export function resolveRegistryUrl(packageName: string): string {
   return registryUrl;
 }
 
-export function resolvePackage(packageName: string): PackageResolution {
-  const registryUrl = resolveRegistryUrl(packageName);
-  const packageUrl = url.resolve(
+export function resolvePackageUrl(
+  registryUrl: string,
+  packageName: string
+): string {
+  return url.resolve(
     ensureTrailingSlash(registryUrl),
     encodeURIComponent(packageName).replace(regEx(/^%40/), '@')
   );
-  return { packageUrl, registryUrl };
 }

lib/modules/datasource/npm/types.ts

@@ -47,8 +47,3 @@ export interface NpmDependency extends ReleaseResult {
 }
 
 export type Npmrc = Record<string, any>;
-
-export interface PackageResolution {
-  packageUrl: string;
-  registryUrl: string;
-}

lib/modules/datasource/types.ts

@@ -18,7 +18,6 @@ export interface DigestConfig {
 }
 
 export interface GetReleasesConfig {
-  npmrc?: string;
   packageName: string;
   registryUrl?: string;
 }