From b5e344b449e00290fbb7584f8d2969837b05986a Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 13 Feb 2020 13:29:55 +0100 Subject: [PATCH] feat: DatasourceError (#5475) Adds centralized handling and logging of datasource failures. --- lib/datasource/cargo/index.ts | 11 +++--- lib/datasource/cdnjs/index.ts | 6 ++-- lib/datasource/common.ts | 15 ++++++++ lib/datasource/dart/index.ts | 6 ++-- lib/datasource/docker/index.ts | 39 +++++++-------------- lib/datasource/gradle-version/index.ts | 10 ++++-- lib/datasource/helm/index.ts | 6 ++-- lib/datasource/hex/index.ts | 6 ++-- lib/datasource/index.ts | 21 ++++++++--- lib/datasource/maven/util.ts | 4 +-- lib/datasource/npm/get.ts | 15 ++------ lib/datasource/packagist/index.ts | 16 ++++----- lib/datasource/ruby-version/index.ts | 10 ++---- lib/datasource/rubygems/get-rubygems-org.ts | 8 ++--- lib/datasource/rubygems/get.ts | 5 ++- 15 files changed, 86 insertions(+), 92 deletions(-) diff --git a/lib/datasource/cargo/index.ts b/lib/datasource/cargo/index.ts index cc8d68005cc41c..8dcc7df8fa1255 100644 --- a/lib/datasource/cargo/index.ts +++ b/lib/datasource/cargo/index.ts @@ -1,7 +1,11 @@ import { logger } from '../../logger'; import got from '../../util/got'; -import { PkgReleaseConfig, ReleaseResult, Release } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { + DatasourceError, + PkgReleaseConfig, + ReleaseResult, + Release, +} from '../common'; import { DATASOURCE_CARGO } from '../../constants/data-binary-source'; export async function getPkgReleases({ @@ -103,8 +107,7 @@ export async function getPkgReleases({ err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - logger.warn({ lookupName, err }, `cargo crates.io registry failure`); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } logger.warn( { err, lookupName }, diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts index 22c8758f42ccf1..7cdb260f924783 100644 --- a/lib/datasource/cdnjs/index.ts +++ b/lib/datasource/cdnjs/index.ts @@ -1,7 +1,6 @@ import { logger } from '../../logger'; import got from '../../util/got'; -import { ReleaseResult, PkgReleaseConfig } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common'; import { DATASOURCE_CDNJS } from '../../constants/data-binary-source'; interface CdnjsAsset { @@ -74,8 +73,7 @@ export async function getPkgReleases({ err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - logger.warn({ lookupName, err }, `CDNJS registry failure`); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } if (err.statusCode === 401) { diff --git a/lib/datasource/common.ts b/lib/datasource/common.ts index ee585ff359fa18..f979f01e4eb866 100644 --- a/lib/datasource/common.ts +++ b/lib/datasource/common.ts @@ -1,3 +1,5 @@ +import { DATASOURCE_FAILURE } from '../constants/error-messages'; + export interface Config { datasource?: string; depName?: string; @@ -50,3 +52,16 @@ export interface Datasource { getPreset?(packageName: string, presetName?: string): Promise; getPkgReleases(config: PkgReleaseConfig): Promise; } + +export class DatasourceError extends Error { + err: Error; + + datasource?: string; + + lookupName?: string; + + constructor(err: Error) { + super(DATASOURCE_FAILURE); + this.err = err; + } +} diff --git a/lib/datasource/dart/index.ts b/lib/datasource/dart/index.ts index 3d7933743ba835..5c585720da3f42 100644 --- a/lib/datasource/dart/index.ts +++ b/lib/datasource/dart/index.ts @@ -1,7 +1,6 @@ import got from '../../util/got'; import { logger } from '../../logger'; -import { ReleaseResult, PkgReleaseConfig } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common'; export async function getPkgReleases({ lookupName, @@ -34,8 +33,7 @@ export async function getPkgReleases({ err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - logger.warn({ lookupName, err }, `pub.dartlang.org registry failure`); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } logger.warn( { err, lookupName }, diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index 46284f9a2475c8..34959f6d1edd46 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -12,9 +12,8 @@ import AWS from 'aws-sdk'; import { logger } from '../../logger'; import got from '../../util/got'; import * as hostRules from '../../util/host-rules'; -import { PkgReleaseConfig, ReleaseResult } from '../common'; +import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common'; import { GotResponse } from '../../platform'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; import { DATASOURCE_DOCKER } from '../../constants/data-binary-source'; // TODO: add got typings when available @@ -168,17 +167,13 @@ async function getAuthHeaders( return null; } if (err.name === 'RequestError' && registry.endsWith('docker.io')) { - logger.debug({ err }, 'err'); - logger.info('Docker registry error: RequestError'); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } if (err.statusCode === 429 && registry.endsWith('docker.io')) { - logger.warn({ err }, 'docker registry failure: too many requests'); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } if (err.statusCode >= 500 && err.statusCode < 600) { - logger.warn({ err }, 'docker registry failure: internal error'); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } logger.warn( { registry, dockerRepository: repository, err }, @@ -218,7 +213,7 @@ async function getManifestResponse( }); return manifestResponse; } catch (err) /* istanbul ignore next */ { - if (err.message === DATASOURCE_FAILURE) { + if (err instanceof DatasourceError) { throw err; } if (err.statusCode === 401) { @@ -242,20 +237,10 @@ async function getManifestResponse( return null; } if (err.statusCode === 429 && registry.endsWith('docker.io')) { - logger.warn({ err }, 'docker registry failure: too many requests'); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } if (err.statusCode >= 500 && err.statusCode < 600) { - logger.info( - { - err, - registry, - dockerRepository: repository, - tag, - }, - 'docker registry failure: internal error' - ); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } if (err.code === 'ETIMEDOUT') { logger.info( @@ -319,7 +304,7 @@ export async function getDigest( await renovateCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); return digest; } catch (err) /* istanbul ignore next */ { - if (err.message === DATASOURCE_FAILURE) { + if (err instanceof DatasourceError) { throw err; } logger.info( @@ -374,7 +359,7 @@ async function getTags( await renovateCache.set(cacheNamespace, cacheKey, tags, cacheMinutes); return tags; } catch (err) /* istanbul ignore next */ { - if (err.message === DATASOURCE_FAILURE) { + if (err instanceof DatasourceError) { throw err; } logger.debug( @@ -401,14 +386,14 @@ async function getTags( { registry, dockerRepository: repository, err }, 'docker registry failure: too many requests' ); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } if (err.statusCode >= 500 && err.statusCode < 600) { logger.warn( { registry, dockerRepository: repository, err }, 'docker registry failure: internal error' ); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } if (err.code === 'ETIMEDOUT') { logger.info( @@ -538,7 +523,7 @@ async function getLabels( await renovateCache.set(cacheNamespace, cacheKey, labels, cacheMinutes); return labels; } catch (err) { - if (err.message === DATASOURCE_FAILURE) { + if (err instanceof DatasourceError) { throw err; } if (err.statusCode === 401) { diff --git a/lib/datasource/gradle-version/index.ts b/lib/datasource/gradle-version/index.ts index 5c0006f27189b3..df2b1bd494ffe3 100644 --- a/lib/datasource/gradle-version/index.ts +++ b/lib/datasource/gradle-version/index.ts @@ -2,8 +2,12 @@ import { coerce } from 'semver'; import is from '@sindresorhus/is'; import { logger } from '../../logger'; import got from '../../util/got'; -import { PkgReleaseConfig, ReleaseResult, Release } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { + DatasourceError, + PkgReleaseConfig, + ReleaseResult, + Release, +} from '../common'; const GradleVersionsServiceUrl = 'https://services.gradle.org/versions/all'; @@ -49,7 +53,7 @@ export async function getPkgReleases({ if (!(err.statusCode === 404 || err.code === 'ENOTFOUND')) { logger.warn({ err }, 'Gradle release lookup failure: Unknown error'); } - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } }) ); diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts index 874e8bf162673e..cd262dd5fa16d1 100644 --- a/lib/datasource/helm/index.ts +++ b/lib/datasource/helm/index.ts @@ -1,7 +1,6 @@ import yaml from 'js-yaml'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; -import { PkgReleaseConfig, ReleaseResult } from '../common'; +import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common'; import got from '../../util/got'; import { logger } from '../../logger'; @@ -35,8 +34,7 @@ export async function getRepositoryData( err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - logger.warn({ err }, `${repository} server error`); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } // istanbul ignore if if (err.name === 'UnsupportedProtocolError') { diff --git a/lib/datasource/hex/index.ts b/lib/datasource/hex/index.ts index 5837d45f05516d..a55a8959dcd89d 100644 --- a/lib/datasource/hex/index.ts +++ b/lib/datasource/hex/index.ts @@ -1,7 +1,6 @@ import { logger } from '../../logger'; import got from '../../util/got'; -import { ReleaseResult, PkgReleaseConfig } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common'; import { DATASOURCE_HEX } from '../../constants/data-binary-source'; interface HexRelease { @@ -66,8 +65,7 @@ export async function getPkgReleases({ err.statusCode === 429 || (err.statusCode >= 500 && err.statusCode < 600) ) { - logger.warn({ lookupName, err }, `hex.pm registry failure`); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } if (err.statusCode === 401) { diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 9f3612683952cf..c91526c5313857 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -5,6 +5,7 @@ import * as versioning from '../versioning'; import { Datasource, + DatasourceError, PkgReleaseConfig, Release, ReleaseResult, @@ -80,10 +81,22 @@ function getRawReleases( export async function getPkgReleases( config: PkgReleaseConfig ): Promise { - const res = await getRawReleases({ - ...config, - lookupName: config.lookupName || config.depName, - }); + const { datasource } = config; + const lookupName = config.lookupName || config.depName; + let res; + try { + res = await getRawReleases({ + ...config, + lookupName, + }); + } catch (e) /* istanbul ignore next */ { + if (e instanceof DatasourceError) { + logger.warn({ datasource, lookupName, err: e.err }, 'Datasource failure'); + e.datasource = datasource; + e.lookupName = lookupName; + } + throw e; + } if (!res) { return res; } diff --git a/lib/datasource/maven/util.ts b/lib/datasource/maven/util.ts index 5ee395ab674242..9b9e90881034dc 100644 --- a/lib/datasource/maven/util.ts +++ b/lib/datasource/maven/util.ts @@ -1,8 +1,8 @@ import url from 'url'; import got from '../../util/got'; import { logger } from '../../logger'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; import { DATASOURCE_MAVEN } from '../../constants/data-binary-source'; +import { DatasourceError } from '../common'; function isMavenCentral(pkgUrl: url.URL | string): boolean { return ( @@ -74,7 +74,7 @@ export async function downloadHttpProtocol( } else if (isTemporalError(err)) { logger.info({ failedUrl, err }, 'Temporary error'); if (isMavenCentral(pkgUrl)) { - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } } else if (isConnectionError(err)) { // istanbul ignore next diff --git a/lib/datasource/npm/get.ts b/lib/datasource/npm/get.ts index 8f51eecb27ad14..a0d7fd074d4804 100644 --- a/lib/datasource/npm/get.ts +++ b/lib/datasource/npm/get.ts @@ -10,8 +10,7 @@ import { logger } from '../../logger'; import got, { GotJSONOptions } from '../../util/got'; import { maskToken } from '../../util/mask'; import { getNpmrc } from './npmrc'; -import { Release, ReleaseResult } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { DatasourceError, Release, ReleaseResult } from '../common'; import { DATASOURCE_NPM } from '../../constants/data-binary-source'; let memcache = {}; @@ -238,17 +237,7 @@ export async function getDependency( await delay(5000); return getDependency(name, retries - 1); } - logger.warn( - { - err, - errorCodes: err.gotOptions?.retry?.errorCodes, - statusCodes: err.gotOptions?.retry?.statusCodes, - regUrl, - depName: name, - }, - 'npm registry failure' - ); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } // istanbul ignore next return null; diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts index ec0432dba105c0..f607e103be22b0 100644 --- a/lib/datasource/packagist/index.ts +++ b/lib/datasource/packagist/index.ts @@ -7,8 +7,7 @@ import { logger } from '../../logger'; import got, { GotJSONOptions } from '../../util/got'; import * as hostRules from '../../util/host-rules'; -import { PkgReleaseConfig, ReleaseResult } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common'; import { DATASOURCE_PACKAGIST } from '../../constants/data-binary-source'; function getHostOpts(url: string): GotJSONOptions { @@ -289,12 +288,13 @@ async function packageLookup( }); return null; } - if ( - (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') && - err.host === 'packagist.org' - ) { - logger.info('Packagist.org timeout'); - throw new Error(DATASOURCE_FAILURE); + if (err.host === 'packagist.org') { + if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') { + throw new DatasourceError(err); + } + if (err.statusCode && err.statusCode >= 500 && err.statusCode < 600) { + throw new DatasourceError(err); + } } logger.warn({ err, name }, 'packagist registry failure: Unknown error'); return null; diff --git a/lib/datasource/ruby-version/index.ts b/lib/datasource/ruby-version/index.ts index 39154e7fe1344b..fb9f0304f2e20f 100644 --- a/lib/datasource/ruby-version/index.ts +++ b/lib/datasource/ruby-version/index.ts @@ -1,10 +1,8 @@ import { parse } from 'node-html-parser'; -import { logger } from '../../logger'; import got from '../../util/got'; import { isVersion } from '../../versioning/ruby'; -import { PkgReleaseConfig, ReleaseResult } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common'; const rubyVersionsUrl = 'https://www.ruby-lang.org/en/downloads/releases/'; @@ -48,10 +46,6 @@ export async function getPkgReleases( await renovateCache.set(cacheNamespace, 'all', res, 15); return res; } catch (err) { - if (err && (err.statusCode === 404 || err.code === 'ENOTFOUND')) { - throw new Error(DATASOURCE_FAILURE); - } - logger.warn({ err }, 'Ruby release lookup failure: Unknown error'); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } } diff --git a/lib/datasource/rubygems/get-rubygems-org.ts b/lib/datasource/rubygems/get-rubygems-org.ts index b415d7bbc10170..52615c4a4ff4e4 100644 --- a/lib/datasource/rubygems/get-rubygems-org.ts +++ b/lib/datasource/rubygems/get-rubygems-org.ts @@ -1,7 +1,6 @@ import got from '../../util/got'; import { logger } from '../../logger'; -import { ReleaseResult } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { DatasourceError, ReleaseResult } from '../common'; let lastSync = new Date('2000-01-01'); let packageReleases: Record = Object.create(null); // Because we might need a "constructor" key @@ -24,10 +23,11 @@ async function updateRubyGemsVersions(): Promise { newLines = (await got(url, options)).body; } catch (err) /* istanbul ignore next */ { if (err.statusCode !== 416) { - logger.warn({ err }, 'Rubygems error - resetting cache'); contentLength = 0; packageReleases = Object.create(null); // Because we might need a "constructor" key - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError( + new Error('Rubygems fetch error - need to reset cache') + ); } logger.debug('Rubygems: No update'); lastSync = new Date(); diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts index 86ba96d379ff79..1445415b84b0e1 100644 --- a/lib/datasource/rubygems/get.ts +++ b/lib/datasource/rubygems/get.ts @@ -4,8 +4,7 @@ import got from '../../util/got'; import { maskToken } from '../../util/mask'; import retriable from './retriable'; import { UNAUTHORIZED, FORBIDDEN, NOT_FOUND } from './errors'; -import { ReleaseResult } from '../common'; -import { DATASOURCE_FAILURE } from '../../constants/error-messages'; +import { DatasourceError, ReleaseResult } from '../common'; import { DATASOURCE_RUBYGEMS } from '../../constants/data-binary-source'; const INFO_PATH = '/api/v1/gems'; @@ -31,7 +30,7 @@ const processError = ({ err, ...rest }): null => { break; default: logger.debug(data, 'RubyGems lookup failure'); - throw new Error(DATASOURCE_FAILURE); + throw new DatasourceError(err); } return null; };