Merge branch 'master' into feat/host-rules-abort

.gitignore

@@ -10,7 +10,7 @@
 /package-lock.json
 *.pyc
 renovate-0.0.0-semantic-release.tgz
-/e2e/node_modules
+/test/e2e/node_modules
 .eslintcache
 junit.xml
 /test-results

lib/config-validator.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 // istanbul ignore file
 import { readFileSync } from 'fs-extra';
+import JSON5 from 'json5';
 import { configFileNames } from './config/app-strings';
 import { RenovateConfig } from './config/common';
 import { getConfig } from './config/file';
@@ -19,13 +20,13 @@ async function validate(
   const res = await validateConfig(massageConfig(config), isPreset);
   if (res.errors.length) {
     console.log(
-      `${desc} contains errors:\n\n${JSON.stringify(res.errors, null, 2)}`
+      `${desc} contains errors:\n\n${JSON5.stringify(res.errors, null, 2)}`
     );
     returnVal = 1;
   }
   if (res.warnings.length) {
     console.log(
-      `${desc} contains warnings:\n\n${JSON.stringify(res.warnings, null, 2)}`
+      `${desc} contains warnings:\n\n${JSON5.stringify(res.warnings, null, 2)}`
     );
     returnVal = 1;
   }
@@ -44,18 +45,18 @@ type PackageJson = {
       const rawContent = readFileSync(file, 'utf8');
       console.log(`Validating ${file}`);
       try {
-        const jsonContent = JSON.parse(rawContent) as PackageJson;
+        const jsonContent = JSON5.parse(rawContent) as PackageJson;
         await validate(file, jsonContent);
       } catch (err) {
-        console.log(`${file} is not valid Renovate config`);
+        console.log(`${file} is not valid Renovate config`, err);
         returnVal = 1;
       }
     } catch (err) {
       // file does not exist
     }
   }
   try {
-    const pkgJson = JSON.parse(
+    const pkgJson = JSON5.parse(
       readFileSync('package.json', 'utf8')
     ) as PackageJson;
     if (pkgJson.renovate) {

lib/config/presets/github/index.ts

@@ -1,6 +1,6 @@
-import { PLATFORM_FAILURE } from '../../../constants/error-messages';
 import { PLATFORM_TYPE_GITHUB } from '../../../constants/platforms';
 import { logger } from '../../../logger';
+import { ExternalHostError } from '../../../types/error';
 import { Http, HttpOptions } from '../../../util/http';
 import { Preset, PresetConfig } from '../common';
 import { PRESET_DEP_NOT_FOUND, fetchPreset } from '../util';
@@ -27,7 +27,7 @@ export async function fetchJSONFile(
     res = await http.getJson(url, opts);
   } catch (err) {
     // istanbul ignore if: not testable with nock
-    if (err.message === PLATFORM_FAILURE) {
+    if (err instanceof ExternalHostError) {
       throw err;
     }
     logger.debug(

lib/config/presets/gitlab/__snapshots__/index.spec.ts.snap

@@ -70,7 +70,7 @@ Array [
 ]
 `;
 
-exports[`config/presets/gitlab/index getPreset() throws platform-failure 1`] = `
+exports[`config/presets/gitlab/index getPreset() throws EXTERNAL_HOST_ERROR 1`] = `
 Array [
   Object {
     "headers": Object {

lib/config/presets/gitlab/index.spec.ts

@@ -1,6 +1,6 @@
 import * as httpMock from '../../../../test/httpMock';
 import { getName } from '../../../../test/util';
-import { PLATFORM_FAILURE } from '../../../constants/error-messages';
+import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages';
 import { PRESET_DEP_NOT_FOUND } from '../util';
 import * as gitlab from '.';
 
@@ -18,14 +18,14 @@ describe(getName(__filename), () => {
   });
 
   describe('getPreset()', () => {
-    it('throws platform-failure', async () => {
+    it('throws EXTERNAL_HOST_ERROR', async () => {
       httpMock.scope(gitlabApiHost).get(`${basePath}/branches`).reply(500);
       await expect(
         gitlab.getPreset({
           packageName: 'some/repo',
           presetName: 'non-default',
         })
-      ).rejects.toThrow(PLATFORM_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
 

lib/config/presets/gitlab/index.ts

@@ -1,5 +1,5 @@
-import { PLATFORM_FAILURE } from '../../../constants/error-messages';
 import { logger } from '../../../logger';
+import { ExternalHostError } from '../../../types/error';
 import type { GitLabBranch } from '../../../types/platform/gitlab';
 import { GitlabHttp } from '../../../util/http/gitlab';
 import { Preset, PresetConfig } from '../common';
@@ -42,7 +42,7 @@ export async function fetchJSONFile(
     const url = `${endpoint}projects/${urlEncodedRepo}/repository/files/${urlEncodedPkgName}/raw?ref=${defautlBranchName}`;
     return (await gitlabApi.getJson<Preset>(url)).body;
   } catch (err) {
-    if (err.message === PLATFORM_FAILURE) {
+    if (err instanceof ExternalHostError) {
       throw err;
     }
     logger.debug(

lib/config/presets/index.ts

@@ -1,10 +1,7 @@
 import is from '@sindresorhus/is';
-import {
-  CONFIG_VALIDATION,
-  DATASOURCE_FAILURE,
-  PLATFORM_FAILURE,
-} from '../../constants/error-messages';
+import { CONFIG_VALIDATION } from '../../constants/error-messages';
 import { logger } from '../../logger';
+import { ExternalHostError } from '../../types/error';
 import { regEx } from '../../util/regex';
 import { RenovateConfig } from '../common';
 import * as massage from '../massage';
@@ -207,10 +204,7 @@ export async function resolveConfigPresets(
         } catch (err) {
           logger.debug({ preset, err }, 'Preset fetch error');
           // istanbul ignore if
-          if (
-            err.message === PLATFORM_FAILURE ||
-            err.message === DATASOURCE_FAILURE
-          ) {
+          if (err instanceof ExternalHostError) {
             throw err;
           }
           const error = new Error(CONFIG_VALIDATION);

lib/constants/error-messages.ts

@@ -5,7 +5,6 @@ export const SYSTEM_INSUFFICIENT_MEMORY = 'out-of-memory';
 // Platform Error
 export const PLATFORM_AUTHENTICATION_ERROR = 'authentication-error';
 export const PLATFORM_BAD_CREDENTIALS = 'bad-credentials';
-export const PLATFORM_FAILURE = 'platform-failure';
 export const PLATFORM_GPG_FAILED = 'gpg-failed';
 export const PLATFORM_INTEGRATION_UNAUTHORIZED = 'integration-unauthorized';
 export const PLATFORM_NOT_FOUND = 'platform-not-found';
@@ -34,8 +33,8 @@ export const REPOSITORY_UNINITIATED = 'uninitiated';
 export const MANAGER_LOCKFILE_ERROR = 'lockfile-error';
 export const MANAGER_NO_PACKAGE_FILES = 'no-package-files';
 
-// Datasource error
-export const DATASOURCE_FAILURE = 'registry-failure';
+// Host error
+export const EXTERNAL_HOST_ERROR = 'external-host-error';
 
 // Worker Error
 export const WORKER_FILE_UPDATE_FAILED = 'update-failure';

lib/datasource/__snapshots__/index.spec.ts.snap

@@ -22,3 +22,18 @@ Object {
   "sourceUrl": "https://github.com/nodejs/node",
 }
 `;
+
+exports[`datasource/index merges registries and returns success 1`] = `
+Object {
+  "releases": Array [
+    Object {
+      "version": "1.0.0",
+    },
+    Object {
+      "version": "1.1.0",
+    },
+  ],
+}
+`;
+
+exports[`datasource/index warns if multiple registryUrls for registryStrategy=first 1`] = `null`;

lib/datasource/cdnjs/index.spec.ts

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import { getPkgReleases } from '..';
 import * as httpMock from '../../../test/httpMock';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
 import { id as datasource } from '.';
 
 let res1 = fs.readFileSync(
@@ -36,14 +36,14 @@ describe('datasource/cdnjs', () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(200, null);
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('throws for error', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error');
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('returns null for 404', async () => {
@@ -70,28 +70,28 @@ describe('datasource/cdnjs', () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(401);
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('throws for 429', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(429);
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('throws for 5xx', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(502);
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('returns null for unknown error', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error');
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })
-      ).rejects.toThrow(DATASOURCE_FAILURE);
+      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
     it('processes real data', async () => {

lib/datasource/cdnjs/index.ts

@@ -1,7 +1,8 @@
 import { logger } from '../../logger';
+import { ExternalHostError } from '../../types/error';
 import { Http } from '../../util/http';
 import { CachePromise, cacheAble } from '../cache';
-import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
+import { GetReleasesConfig, ReleaseResult } from '../common';
 
 export const id = 'cdnjs';
 
@@ -60,7 +61,7 @@ export async function getReleases({
       logger.debug({ library }, 'cdnjs library not found');
       return null;
     }
-    // Throw a DatasourceError for all other types of errors
-    throw new DatasourceError(err);
+    // Throw an ExternalHostError for all other types of errors
+    throw new ExternalHostError(err);
   }
 }

lib/datasource/common.ts

@@ -1,13 +1,13 @@
-import { DATASOURCE_FAILURE } from '../constants/error-messages';
-
 export interface Config {
   datasource?: string;
   depName?: string;
   lookupName?: string;
   registryUrls?: string[];
 }
 
-export type DigestConfig = Config;
+export interface DigestConfig extends Config {
+  registryUrl?: string;
+}
 
 interface ReleasesConfigBase {
   compatibility?: Record<string, string>;
@@ -17,6 +17,7 @@ interface ReleasesConfigBase {
 
 export interface GetReleasesConfig extends ReleasesConfigBase {
   lookupName: string;
+  registryUrl?: string;
 }
 
 export interface GetPkgReleasesConfig extends ReleasesConfigBase {
@@ -65,6 +66,7 @@ export interface ReleaseResult {
   sourceUrl?: string;
   tags?: Record<string, string>;
   versions?: any;
+  registryUrl?: string;
 }
 
 export interface Datasource {
@@ -74,19 +76,5 @@ export interface Datasource {
   defaultRegistryUrls?: string[];
   appendRegistryUrls?: string[];
   defaultConfig?: object;
-}
-
-export class DatasourceError extends Error {
-  err: Error;
-
-  datasource?: string;
-
-  lookupName?: string;
-
-  constructor(err: Error) {
-    super(DATASOURCE_FAILURE);
-    // Set the prototype explicitly: https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
-    Object.setPrototypeOf(this, DatasourceError.prototype);
-    this.err = err;
-  }
+  registryStrategy?: 'first' | 'hunt' | 'merge';
 }

lib/datasource/crate/__snapshots__/index.spec.ts.snap

@@ -764,7 +764,7 @@ Array [
 ]
 `;
 
-exports[`datasource/crate getReleases throws for 5xx 1`] = `[Error: registry-failure]`;
+exports[`datasource/crate getReleases throws for 5xx 1`] = `[Error: external-host-error]`;
 
 exports[`datasource/crate getReleases throws for 5xx 2`] = `
 Array [

lib/datasource/crate/index.ts

@@ -1,12 +1,8 @@
 import { logger } from '../../logger';
+import { ExternalHostError } from '../../types/error';
 import * as globalCache from '../../util/cache/global';
 import { Http } from '../../util/http';
-import {
-  DatasourceError,
-  GetReleasesConfig,
-  Release,
-  ReleaseResult,
-} from '../common';
+import { GetReleasesConfig, Release, ReleaseResult } from '../common';
 
 export const id = 'crate';
 
@@ -105,7 +101,7 @@ export async function getReleases({
       err.statusCode === 429 ||
       (err.statusCode >= 500 && err.statusCode < 600)
     ) {
-      throw new DatasourceError(err);
+      throw new ExternalHostError(err);
     }
     logger.warn({ err, lookupName }, 'crates.io lookup failure: Unknown error');
     return null;

lib/datasource/dart/__snapshots__/index.spec.ts.snap

@@ -144,7 +144,7 @@ Array [
 ]
 `;
 
-exports[`datasource/dart getReleases throws for 5xx 1`] = `[Error: registry-failure]`;
+exports[`datasource/dart getReleases throws for 5xx 1`] = `[Error: external-host-error]`;
 
 exports[`datasource/dart getReleases throws for 5xx 2`] = `
 Array [

lib/datasource/dart/index.ts

@@ -1,6 +1,7 @@
 import { logger } from '../../logger';
+import { ExternalHostError } from '../../types/error';
 import { Http, HttpResponse } from '../../util/http';
-import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
+import { GetReleasesConfig, ReleaseResult } from '../common';
 
 export const id = 'dart';
 
@@ -33,7 +34,7 @@ export async function getReleases({
       err.statusCode === 429 ||
       (err.statusCode >= 500 && err.statusCode < 600)
     ) {
-      throw new DatasourceError(err);
+      throw new ExternalHostError(err);
     }
     logger.warn(
       { err, lookupName },