Skip to content

Commit

Permalink
feat: DatasourceError (#5475)
Browse files Browse the repository at this point in the history 
Adds centralized handling and logging of datasource failures.
  • Loading branch information
@rarkins
rarkins committed Feb 13, 2020
1 parent 30ee09c commit b5e344b
Show file tree
Hide file tree
Showing 15 changed files with 86 additions and 92 deletions.
11 changes: 7 additions & 4 deletions lib/datasource/cargo/index.ts
View file
@@ -1,7 +1,11 @@
import { logger } from '../../logger';
import got from '../../util/got';
import { PkgReleaseConfig, ReleaseResult, Release } from '../common';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import {
DatasourceError,
PkgReleaseConfig,
ReleaseResult,
Release,
} from '../common';
import { DATASOURCE_CARGO } from '../../constants/data-binary-source';

export async function getPkgReleases({
Expand Down Expand Up @@ -103,8 +107,7 @@ export async function getPkgReleases({
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
logger.warn({ lookupName, err }, `cargo crates.io registry failure`);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
logger.warn(
{ err, lookupName },
Expand Down
6 changes: 2 additions & 4 deletions lib/datasource/cdnjs/index.ts
View file
@@ -1,7 +1,6 @@
import { logger } from '../../logger';
import got from '../../util/got';
import { ReleaseResult, PkgReleaseConfig } from '../common';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common';
import { DATASOURCE_CDNJS } from '../../constants/data-binary-source';

interface CdnjsAsset {
Expand Down Expand Up @@ -74,8 +73,7 @@ export async function getPkgReleases({
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
logger.warn({ lookupName, err }, `CDNJS registry failure`);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}

if (err.statusCode === 401) {
Expand Down
15 changes: 15 additions & 0 deletions lib/datasource/common.ts
View file
@@ -1,3 +1,5 @@
import { DATASOURCE_FAILURE } from '../constants/error-messages';

export interface Config {
datasource?: string;
depName?: string;
Expand Down Expand Up @@ -50,3 +52,16 @@ export interface Datasource {
getPreset?(packageName: string, presetName?: string): Promise<Preset>;
getPkgReleases(config: PkgReleaseConfig): Promise<ReleaseResult | null>;
}

export class DatasourceError extends Error {
err: Error;

datasource?: string;

lookupName?: string;

constructor(err: Error) {
super(DATASOURCE_FAILURE);
this.err = err;
}
}
6 changes: 2 additions & 4 deletions lib/datasource/dart/index.ts
View file
@@ -1,7 +1,6 @@
import got from '../../util/got';
import { logger } from '../../logger';
import { ReleaseResult, PkgReleaseConfig } from '../common';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common';

export async function getPkgReleases({
lookupName,
Expand Down Expand Up @@ -34,8 +33,7 @@ export async function getPkgReleases({
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
logger.warn({ lookupName, err }, `pub.dartlang.org registry failure`);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
logger.warn(
{ err, lookupName },
Expand Down
39 changes: 12 additions & 27 deletions lib/datasource/docker/index.ts
View file
Expand Up @@ -12,9 +12,8 @@ import AWS from 'aws-sdk';
import { logger } from '../../logger';
import got from '../../util/got';
import * as hostRules from '../../util/host-rules';
import { PkgReleaseConfig, ReleaseResult } from '../common';
import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common';
import { GotResponse } from '../../platform';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DATASOURCE_DOCKER } from '../../constants/data-binary-source';

// TODO: add got typings when available
Expand Down Expand Up @@ -168,17 +167,13 @@ async function getAuthHeaders(
return null;
}
if (err.name === 'RequestError' && registry.endsWith('docker.io')) {
logger.debug({ err }, 'err');
logger.info('Docker registry error: RequestError');
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
if (err.statusCode === 429 && registry.endsWith('docker.io')) {
logger.warn({ err }, 'docker registry failure: too many requests');
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
if (err.statusCode >= 500 && err.statusCode < 600) {
logger.warn({ err }, 'docker registry failure: internal error');
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
logger.warn(
{ registry, dockerRepository: repository, err },
Expand Down Expand Up @@ -218,7 +213,7 @@ async function getManifestResponse(
});
return manifestResponse;
} catch (err) /* istanbul ignore next */ {
if (err.message === DATASOURCE_FAILURE) {
if (err instanceof DatasourceError) {
throw err;
}
if (err.statusCode === 401) {
Expand All @@ -242,20 +237,10 @@ async function getManifestResponse(
return null;
}
if (err.statusCode === 429 && registry.endsWith('docker.io')) {
logger.warn({ err }, 'docker registry failure: too many requests');
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
if (err.statusCode >= 500 && err.statusCode < 600) {
logger.info(
{
err,
registry,
dockerRepository: repository,
tag,
},
'docker registry failure: internal error'
);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
if (err.code === 'ETIMEDOUT') {
logger.info(
Expand Down Expand Up @@ -319,7 +304,7 @@ export async function getDigest(
await renovateCache.set(cacheNamespace, cacheKey, digest, cacheMinutes);
return digest;
} catch (err) /* istanbul ignore next */ {
if (err.message === DATASOURCE_FAILURE) {
if (err instanceof DatasourceError) {
throw err;
}
logger.info(
Expand Down Expand Up @@ -374,7 +359,7 @@ async function getTags(
await renovateCache.set(cacheNamespace, cacheKey, tags, cacheMinutes);
return tags;
} catch (err) /* istanbul ignore next */ {
if (err.message === DATASOURCE_FAILURE) {
if (err instanceof DatasourceError) {
throw err;
}
logger.debug(
Expand All @@ -401,14 +386,14 @@ async function getTags(
{ registry, dockerRepository: repository, err },
'docker registry failure: too many requests'
);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
if (err.statusCode >= 500 && err.statusCode < 600) {
logger.warn(
{ registry, dockerRepository: repository, err },
'docker registry failure: internal error'
);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
if (err.code === 'ETIMEDOUT') {
logger.info(
Expand Down Expand Up @@ -538,7 +523,7 @@ async function getLabels(
await renovateCache.set(cacheNamespace, cacheKey, labels, cacheMinutes);
return labels;
} catch (err) {
if (err.message === DATASOURCE_FAILURE) {
if (err instanceof DatasourceError) {
throw err;
}
if (err.statusCode === 401) {
Expand Down
10 changes: 7 additions & 3 deletions lib/datasource/gradle-version/index.ts
View file
Expand Up @@ -2,8 +2,12 @@ import { coerce } from 'semver';
import is from '@sindresorhus/is';
import { logger } from '../../logger';
import got from '../../util/got';
import { PkgReleaseConfig, ReleaseResult, Release } from '../common';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import {
DatasourceError,
PkgReleaseConfig,
ReleaseResult,
Release,
} from '../common';

const GradleVersionsServiceUrl = 'https://services.gradle.org/versions/all';

Expand Down Expand Up @@ -49,7 +53,7 @@ export async function getPkgReleases({
if (!(err.statusCode === 404 || err.code === 'ENOTFOUND')) {
logger.warn({ err }, 'Gradle release lookup failure: Unknown error');
}
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
})
);
Expand Down
6 changes: 2 additions & 4 deletions lib/datasource/helm/index.ts
View file
@@ -1,7 +1,6 @@
import yaml from 'js-yaml';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';

import { PkgReleaseConfig, ReleaseResult } from '../common';
import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common';
import got from '../../util/got';
import { logger } from '../../logger';

Expand Down Expand Up @@ -35,8 +34,7 @@ export async function getRepositoryData(
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
logger.warn({ err }, `${repository} server error`);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
// istanbul ignore if
if (err.name === 'UnsupportedProtocolError') {
Expand Down
6 changes: 2 additions & 4 deletions lib/datasource/hex/index.ts
View file
@@ -1,7 +1,6 @@
import { logger } from '../../logger';
import got from '../../util/got';
import { ReleaseResult, PkgReleaseConfig } from '../common';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common';
import { DATASOURCE_HEX } from '../../constants/data-binary-source';

interface HexRelease {
Expand Down Expand Up @@ -66,8 +65,7 @@ export async function getPkgReleases({
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
logger.warn({ lookupName, err }, `hex.pm registry failure`);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}

if (err.statusCode === 401) {
Expand Down
21 changes: 17 additions & 4 deletions lib/datasource/index.ts
View file
Expand Up @@ -5,6 +5,7 @@ import * as versioning from '../versioning';

import {
Datasource,
DatasourceError,
PkgReleaseConfig,
Release,
ReleaseResult,
Expand Down Expand Up @@ -80,10 +81,22 @@ function getRawReleases(
export async function getPkgReleases(
config: PkgReleaseConfig
): Promise<ReleaseResult | null> {
const res = await getRawReleases({
...config,
lookupName: config.lookupName || config.depName,
});
const { datasource } = config;
const lookupName = config.lookupName || config.depName;
let res;
try {
res = await getRawReleases({
...config,
lookupName,
});
} catch (e) /* istanbul ignore next */ {
if (e instanceof DatasourceError) {
logger.warn({ datasource, lookupName, err: e.err }, 'Datasource failure');
e.datasource = datasource;
e.lookupName = lookupName;
}
throw e;
}
if (!res) {
return res;
}
Expand Down
4 changes: 2 additions & 2 deletions lib/datasource/maven/util.ts
View file
@@ -1,8 +1,8 @@
import url from 'url';
import got from '../../util/got';
import { logger } from '../../logger';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DATASOURCE_MAVEN } from '../../constants/data-binary-source';
import { DatasourceError } from '../common';

function isMavenCentral(pkgUrl: url.URL | string): boolean {
return (
Expand Down Expand Up @@ -74,7 +74,7 @@ export async function downloadHttpProtocol(
} else if (isTemporalError(err)) {
logger.info({ failedUrl, err }, 'Temporary error');
if (isMavenCentral(pkgUrl)) {
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
} else if (isConnectionError(err)) {
// istanbul ignore next
Expand Down
15 changes: 2 additions & 13 deletions lib/datasource/npm/get.ts
View file
Expand Up @@ -10,8 +10,7 @@ import { logger } from '../../logger';
import got, { GotJSONOptions } from '../../util/got';
import { maskToken } from '../../util/mask';
import { getNpmrc } from './npmrc';
import { Release, ReleaseResult } from '../common';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DatasourceError, Release, ReleaseResult } from '../common';
import { DATASOURCE_NPM } from '../../constants/data-binary-source';

let memcache = {};
Expand Down Expand Up @@ -238,17 +237,7 @@ export async function getDependency(
await delay(5000);
return getDependency(name, retries - 1);
}
logger.warn(
{
err,
errorCodes: err.gotOptions?.retry?.errorCodes,
statusCodes: err.gotOptions?.retry?.statusCodes,
regUrl,
depName: name,
},
'npm registry failure'
);
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
// istanbul ignore next
return null;
Expand Down
16 changes: 8 additions & 8 deletions lib/datasource/packagist/index.ts
View file
Expand Up @@ -7,8 +7,7 @@ import { logger } from '../../logger';

import got, { GotJSONOptions } from '../../util/got';
import * as hostRules from '../../util/host-rules';
import { PkgReleaseConfig, ReleaseResult } from '../common';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common';
import { DATASOURCE_PACKAGIST } from '../../constants/data-binary-source';

function getHostOpts(url: string): GotJSONOptions {
Expand Down Expand Up @@ -289,12 +288,13 @@ async function packageLookup(
});
return null;
}
if (
(err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') &&
err.host === 'packagist.org'
) {
logger.info('Packagist.org timeout');
throw new Error(DATASOURCE_FAILURE);
if (err.host === 'packagist.org') {
if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') {
throw new DatasourceError(err);
}
if (err.statusCode && err.statusCode >= 500 && err.statusCode < 600) {
throw new DatasourceError(err);
}
}
logger.warn({ err, name }, 'packagist registry failure: Unknown error');
return null;
Expand Down
10 changes: 2 additions & 8 deletions lib/datasource/ruby-version/index.ts
View file
@@ -1,10 +1,8 @@
import { parse } from 'node-html-parser';
import { logger } from '../../logger';

import got from '../../util/got';
import { isVersion } from '../../versioning/ruby';
import { PkgReleaseConfig, ReleaseResult } from '../common';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common';

const rubyVersionsUrl = 'https://www.ruby-lang.org/en/downloads/releases/';

Expand Down Expand Up @@ -48,10 +46,6 @@ export async function getPkgReleases(
await renovateCache.set(cacheNamespace, 'all', res, 15);
return res;
} catch (err) {
if (err && (err.statusCode === 404 || err.code === 'ENOTFOUND')) {
throw new Error(DATASOURCE_FAILURE);
}
logger.warn({ err }, 'Ruby release lookup failure: Unknown error');
throw new Error(DATASOURCE_FAILURE);
throw new DatasourceError(err);
}
}

0 comments on commit b5e344b

Please sign in to comment.