From b5e344b449e00290fbb7584f8d2969837b05986a Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Thu, 13 Feb 2020 13:29:55 +0100
Subject: [PATCH] feat: DatasourceError (#5475)
Adds centralized handling and logging of datasource failures.
---
lib/datasource/cargo/index.ts | 11 +++---
lib/datasource/cdnjs/index.ts | 6 ++--
lib/datasource/common.ts | 15 ++++++++
lib/datasource/dart/index.ts | 6 ++--
lib/datasource/docker/index.ts | 39 +++++++--------------
lib/datasource/gradle-version/index.ts | 10 ++++--
lib/datasource/helm/index.ts | 6 ++--
lib/datasource/hex/index.ts | 6 ++--
lib/datasource/index.ts | 21 ++++++++---
lib/datasource/maven/util.ts | 4 +--
lib/datasource/npm/get.ts | 15 ++------
lib/datasource/packagist/index.ts | 16 ++++-----
lib/datasource/ruby-version/index.ts | 10 ++----
lib/datasource/rubygems/get-rubygems-org.ts | 8 ++---
lib/datasource/rubygems/get.ts | 5 ++-
15 files changed, 86 insertions(+), 92 deletions(-)
diff --git a/lib/datasource/cargo/index.ts b/lib/datasource/cargo/index.ts
index cc8d68005c..8dcc7df8fa 100644
--- a/lib/datasource/cargo/index.ts
+++ b/lib/datasource/cargo/index.ts
@@ -1,7 +1,11 @@
import { logger } from '../../logger';
import got from '../../util/got';
-import { PkgReleaseConfig, ReleaseResult, Release } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import {
+ DatasourceError,
+ PkgReleaseConfig,
+ ReleaseResult,
+ Release,
+} from '../common';
import { DATASOURCE_CARGO } from '../../constants/data-binary-source';
export async function getPkgReleases({
@@ -103,8 +107,7 @@ export async function getPkgReleases({
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
- logger.warn({ lookupName, err }, `cargo crates.io registry failure`);
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
logger.warn(
{ err, lookupName },
diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts
index 22c8758f42..7cdb260f92 100644
--- a/lib/datasource/cdnjs/index.ts
+++ b/lib/datasource/cdnjs/index.ts
@@ -1,7 +1,6 @@
import { logger } from '../../logger';
import got from '../../util/got';
-import { ReleaseResult, PkgReleaseConfig } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common';
import { DATASOURCE_CDNJS } from '../../constants/data-binary-source';
interface CdnjsAsset {
@@ -74,8 +73,7 @@ export async function getPkgReleases({
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
- logger.warn({ lookupName, err }, `CDNJS registry failure`);
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
if (err.statusCode === 401) {
diff --git a/lib/datasource/common.ts b/lib/datasource/common.ts
index ee585ff359..f979f01e4e 100644
--- a/lib/datasource/common.ts
+++ b/lib/datasource/common.ts
@@ -1,3 +1,5 @@
+import { DATASOURCE_FAILURE } from '../constants/error-messages';
+
export interface Config {
datasource?: string;
depName?: string;
@@ -50,3 +52,16 @@ export interface Datasource {
getPreset?(packageName: string, presetName?: string): Promise<Preset>;
getPkgReleases(config: PkgReleaseConfig): Promise<ReleaseResult | null>;
}
+
+export class DatasourceError extends Error {
+ err: Error;
+
+ datasource?: string;
+
+ lookupName?: string;
+
+ constructor(err: Error) {
+ super(DATASOURCE_FAILURE);
+ this.err = err;
+ }
+}
diff --git a/lib/datasource/dart/index.ts b/lib/datasource/dart/index.ts
index 3d7933743b..5c585720da 100644
--- a/lib/datasource/dart/index.ts
+++ b/lib/datasource/dart/index.ts
@@ -1,7 +1,6 @@
import got from '../../util/got';
import { logger } from '../../logger';
-import { ReleaseResult, PkgReleaseConfig } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common';
export async function getPkgReleases({
lookupName,
@@ -34,8 +33,7 @@ export async function getPkgReleases({
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
- logger.warn({ lookupName, err }, `pub.dartlang.org registry failure`);
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
logger.warn(
{ err, lookupName },
diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts
index 46284f9a24..34959f6d1e 100644
--- a/lib/datasource/docker/index.ts
+++ b/lib/datasource/docker/index.ts
@@ -12,9 +12,8 @@ import AWS from 'aws-sdk';
import { logger } from '../../logger';
import got from '../../util/got';
import * as hostRules from '../../util/host-rules';
-import { PkgReleaseConfig, ReleaseResult } from '../common';
+import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common';
import { GotResponse } from '../../platform';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DATASOURCE_DOCKER } from '../../constants/data-binary-source';
// TODO: add got typings when available
@@ -168,17 +167,13 @@ async function getAuthHeaders(
return null;
}
if (err.name === 'RequestError' && registry.endsWith('docker.io')) {
- logger.debug({ err }, 'err');
- logger.info('Docker registry error: RequestError');
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
if (err.statusCode === 429 && registry.endsWith('docker.io')) {
- logger.warn({ err }, 'docker registry failure: too many requests');
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
if (err.statusCode >= 500 && err.statusCode < 600) {
- logger.warn({ err }, 'docker registry failure: internal error');
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
logger.warn(
{ registry, dockerRepository: repository, err },
@@ -218,7 +213,7 @@ async function getManifestResponse(
});
return manifestResponse;
} catch (err) /* istanbul ignore next */ {
- if (err.message === DATASOURCE_FAILURE) {
+ if (err instanceof DatasourceError) {
throw err;
}
if (err.statusCode === 401) {
@@ -242,20 +237,10 @@ async function getManifestResponse(
return null;
}
if (err.statusCode === 429 && registry.endsWith('docker.io')) {
- logger.warn({ err }, 'docker registry failure: too many requests');
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
if (err.statusCode >= 500 && err.statusCode < 600) {
- logger.info(
- {
- err,
- registry,
- dockerRepository: repository,
- tag,
- },
- 'docker registry failure: internal error'
- );
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
if (err.code === 'ETIMEDOUT') {
logger.info(
@@ -319,7 +304,7 @@ export async function getDigest(
await renovateCache.set(cacheNamespace, cacheKey, digest, cacheMinutes);
return digest;
} catch (err) /* istanbul ignore next */ {
- if (err.message === DATASOURCE_FAILURE) {
+ if (err instanceof DatasourceError) {
throw err;
}
logger.info(
@@ -374,7 +359,7 @@ async function getTags(
await renovateCache.set(cacheNamespace, cacheKey, tags, cacheMinutes);
return tags;
} catch (err) /* istanbul ignore next */ {
- if (err.message === DATASOURCE_FAILURE) {
+ if (err instanceof DatasourceError) {
throw err;
}
logger.debug(
@@ -401,14 +386,14 @@ async function getTags(
{ registry, dockerRepository: repository, err },
'docker registry failure: too many requests'
);
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
if (err.statusCode >= 500 && err.statusCode < 600) {
logger.warn(
{ registry, dockerRepository: repository, err },
'docker registry failure: internal error'
);
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
if (err.code === 'ETIMEDOUT') {
logger.info(
@@ -538,7 +523,7 @@ async function getLabels(
await renovateCache.set(cacheNamespace, cacheKey, labels, cacheMinutes);
return labels;
} catch (err) {
- if (err.message === DATASOURCE_FAILURE) {
+ if (err instanceof DatasourceError) {
throw err;
}
if (err.statusCode === 401) {
diff --git a/lib/datasource/gradle-version/index.ts b/lib/datasource/gradle-version/index.ts
index 5c0006f271..df2b1bd494 100644
--- a/lib/datasource/gradle-version/index.ts
+++ b/lib/datasource/gradle-version/index.ts
@@ -2,8 +2,12 @@ import { coerce } from 'semver';
import is from '@sindresorhus/is';
import { logger } from '../../logger';
import got from '../../util/got';
-import { PkgReleaseConfig, ReleaseResult, Release } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import {
+ DatasourceError,
+ PkgReleaseConfig,
+ ReleaseResult,
+ Release,
+} from '../common';
const GradleVersionsServiceUrl = 'https://services.gradle.org/versions/all';
@@ -49,7 +53,7 @@ export async function getPkgReleases({
if (!(err.statusCode === 404 || err.code === 'ENOTFOUND')) {
logger.warn({ err }, 'Gradle release lookup failure: Unknown error');
}
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
})
);
diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts
index 874e8bf162..cd262dd5fa 100644
--- a/lib/datasource/helm/index.ts
+++ b/lib/datasource/helm/index.ts
@@ -1,7 +1,6 @@
import yaml from 'js-yaml';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
-import { PkgReleaseConfig, ReleaseResult } from '../common';
+import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common';
import got from '../../util/got';
import { logger } from '../../logger';
@@ -35,8 +34,7 @@ export async function getRepositoryData(
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
- logger.warn({ err }, `${repository} server error`);
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
// istanbul ignore if
if (err.name === 'UnsupportedProtocolError') {
diff --git a/lib/datasource/hex/index.ts b/lib/datasource/hex/index.ts
index 5837d45f05..a55a8959dc 100644
--- a/lib/datasource/hex/index.ts
+++ b/lib/datasource/hex/index.ts
@@ -1,7 +1,6 @@
import { logger } from '../../logger';
import got from '../../util/got';
-import { ReleaseResult, PkgReleaseConfig } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { DatasourceError, ReleaseResult, PkgReleaseConfig } from '../common';
import { DATASOURCE_HEX } from '../../constants/data-binary-source';
interface HexRelease {
@@ -66,8 +65,7 @@ export async function getPkgReleases({
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
- logger.warn({ lookupName, err }, `hex.pm registry failure`);
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
if (err.statusCode === 401) {
diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts
index 9f36126839..c91526c531 100644
--- a/lib/datasource/index.ts
+++ b/lib/datasource/index.ts
@@ -5,6 +5,7 @@ import * as versioning from '../versioning';
import {
Datasource,
+ DatasourceError,
PkgReleaseConfig,
Release,
ReleaseResult,
@@ -80,10 +81,22 @@ function getRawReleases(
export async function getPkgReleases(
config: PkgReleaseConfig
): Promise<ReleaseResult | null> {
- const res = await getRawReleases({
- ...config,
- lookupName: config.lookupName || config.depName,
- });
+ const { datasource } = config;
+ const lookupName = config.lookupName || config.depName;
+ let res;
+ try {
+ res = await getRawReleases({
+ ...config,
+ lookupName,
+ });
+ } catch (e) /* istanbul ignore next */ {
+ if (e instanceof DatasourceError) {
+ logger.warn({ datasource, lookupName, err: e.err }, 'Datasource failure');
+ e.datasource = datasource;
+ e.lookupName = lookupName;
+ }
+ throw e;
+ }
if (!res) {
return res;
}
diff --git a/lib/datasource/maven/util.ts b/lib/datasource/maven/util.ts
index 5ee395ab67..9b9e908810 100644
--- a/lib/datasource/maven/util.ts
+++ b/lib/datasource/maven/util.ts
@@ -1,8 +1,8 @@
import url from 'url';
import got from '../../util/got';
import { logger } from '../../logger';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import { DATASOURCE_MAVEN } from '../../constants/data-binary-source';
+import { DatasourceError } from '../common';
function isMavenCentral(pkgUrl: url.URL | string): boolean {
return (
@@ -74,7 +74,7 @@ export async function downloadHttpProtocol(
} else if (isTemporalError(err)) {
logger.info({ failedUrl, err }, 'Temporary error');
if (isMavenCentral(pkgUrl)) {
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
} else if (isConnectionError(err)) {
// istanbul ignore next
diff --git a/lib/datasource/npm/get.ts b/lib/datasource/npm/get.ts
index 8f51eecb27..a0d7fd074d 100644
--- a/lib/datasource/npm/get.ts
+++ b/lib/datasource/npm/get.ts
@@ -10,8 +10,7 @@ import { logger } from '../../logger';
import got, { GotJSONOptions } from '../../util/got';
import { maskToken } from '../../util/mask';
import { getNpmrc } from './npmrc';
-import { Release, ReleaseResult } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { DatasourceError, Release, ReleaseResult } from '../common';
import { DATASOURCE_NPM } from '../../constants/data-binary-source';
let memcache = {};
@@ -238,17 +237,7 @@ export async function getDependency(
await delay(5000);
return getDependency(name, retries - 1);
}
- logger.warn(
- {
- err,
- errorCodes: err.gotOptions?.retry?.errorCodes,
- statusCodes: err.gotOptions?.retry?.statusCodes,
- regUrl,
- depName: name,
- },
- 'npm registry failure'
- );
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
// istanbul ignore next
return null;
diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts
index ec0432dba1..f607e103be 100644
--- a/lib/datasource/packagist/index.ts
+++ b/lib/datasource/packagist/index.ts
@@ -7,8 +7,7 @@ import { logger } from '../../logger';
import got, { GotJSONOptions } from '../../util/got';
import * as hostRules from '../../util/host-rules';
-import { PkgReleaseConfig, ReleaseResult } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common';
import { DATASOURCE_PACKAGIST } from '../../constants/data-binary-source';
function getHostOpts(url: string): GotJSONOptions {
@@ -289,12 +288,13 @@ async function packageLookup(
});
return null;
}
- if (
- (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') &&
- err.host === 'packagist.org'
- ) {
- logger.info('Packagist.org timeout');
- throw new Error(DATASOURCE_FAILURE);
+ if (err.host === 'packagist.org') {
+ if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') {
+ throw new DatasourceError(err);
+ }
+ if (err.statusCode && err.statusCode >= 500 && err.statusCode < 600) {
+ throw new DatasourceError(err);
+ }
}
logger.warn({ err, name }, 'packagist registry failure: Unknown error');
return null;
diff --git a/lib/datasource/ruby-version/index.ts b/lib/datasource/ruby-version/index.ts
index 39154e7fe1..fb9f0304f2 100644
--- a/lib/datasource/ruby-version/index.ts
+++ b/lib/datasource/ruby-version/index.ts
@@ -1,10 +1,8 @@
import { parse } from 'node-html-parser';
-import { logger } from '../../logger';
import got from '../../util/got';
import { isVersion } from '../../versioning/ruby';
-import { PkgReleaseConfig, ReleaseResult } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { DatasourceError, PkgReleaseConfig, ReleaseResult } from '../common';
const rubyVersionsUrl = 'https://www.ruby-lang.org/en/downloads/releases/';
@@ -48,10 +46,6 @@ export async function getPkgReleases(
await renovateCache.set(cacheNamespace, 'all', res, 15);
return res;
} catch (err) {
- if (err && (err.statusCode === 404 || err.code === 'ENOTFOUND')) {
- throw new Error(DATASOURCE_FAILURE);
- }
- logger.warn({ err }, 'Ruby release lookup failure: Unknown error');
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
}
diff --git a/lib/datasource/rubygems/get-rubygems-org.ts b/lib/datasource/rubygems/get-rubygems-org.ts
index b415d7bbc1..52615c4a4f 100644
--- a/lib/datasource/rubygems/get-rubygems-org.ts
+++ b/lib/datasource/rubygems/get-rubygems-org.ts
@@ -1,7 +1,6 @@
import got from '../../util/got';
import { logger } from '../../logger';
-import { ReleaseResult } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { DatasourceError, ReleaseResult } from '../common';
let lastSync = new Date('2000-01-01');
let packageReleases: Record<string, string[]> = Object.create(null); // Because we might need a "constructor" key
@@ -24,10 +23,11 @@ async function updateRubyGemsVersions(): Promise<void> {
newLines = (await got(url, options)).body;
} catch (err) /* istanbul ignore next */ {
if (err.statusCode !== 416) {
- logger.warn({ err }, 'Rubygems error - resetting cache');
contentLength = 0;
packageReleases = Object.create(null); // Because we might need a "constructor" key
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(
+ new Error('Rubygems fetch error - need to reset cache')
+ );
}
logger.debug('Rubygems: No update');
lastSync = new Date();
diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts
index 86ba96d379..1445415b84 100644
--- a/lib/datasource/rubygems/get.ts
+++ b/lib/datasource/rubygems/get.ts
@@ -4,8 +4,7 @@ import got from '../../util/got';
import { maskToken } from '../../util/mask';
import retriable from './retriable';
import { UNAUTHORIZED, FORBIDDEN, NOT_FOUND } from './errors';
-import { ReleaseResult } from '../common';
-import { DATASOURCE_FAILURE } from '../../constants/error-messages';
+import { DatasourceError, ReleaseResult } from '../common';
import { DATASOURCE_RUBYGEMS } from '../../constants/data-binary-source';
const INFO_PATH = '/api/v1/gems';
@@ -31,7 +30,7 @@ const processError = ({ err, ...rest }): null => {
break;
default:
logger.debug(data, 'RubyGems lookup failure');
- throw new Error(DATASOURCE_FAILURE);
+ throw new DatasourceError(err);
}
return null;
};
--
GitLab