From c30a4b0b54599b5d7cf07f3cc9c6bb6bdd14d6ba Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Fri, 16 Feb 2024 17:06:11 -0300
Subject: [PATCH] feat(logger): Log level remapping (#26951)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
docs/usage/configuration-options.md | 56 ++++++++++++++++++++++
lib/config/options/index.ts | 26 ++++++++++
lib/config/types.ts | 6 ++-
lib/logger/index.ts | 30 ++++++++++--
lib/logger/remap.spec.ts | 72 ++++++++++++++++++++++++++++
lib/logger/remap.ts | 68 ++++++++++++++++++++++++++
lib/logger/types.ts | 7 ++-
lib/workers/global/index.ts | 3 ++
lib/workers/global/initialize.ts | 2 +
lib/workers/repository/index.ts | 2 +
lib/workers/repository/init/index.ts | 2 +
11 files changed, 267 insertions(+), 7 deletions(-)
create mode 100644 lib/logger/remap.spec.ts
create mode 100644 lib/logger/remap.ts
diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
index 0efc6ec42e..a84fe56d30 100644
--- a/docs/usage/configuration-options.md
+++ b/docs/usage/configuration-options.md
@@ -1796,6 +1796,21 @@ You can configure a different maximum value in seconds using `maxRetryAfter`:
}
```
+### newLogLevel
+
+For log level remapping, `newLogLevel` will set for the particular log message:
+
+```json
+{
+ "logLevelRemap": [
+ {
+ "matchMessage": "/Error executing maven wrapper update command/",
+ "newLogLevel": "warn"
+ }
+ ]
+}
+```
+
### dnsCache
Enable got [dnsCache](https://github.com/sindresorhus/got/blob/v11.5.2/readme.md#dnsCache) support.
@@ -2155,6 +2170,27 @@ To enable `lockFileMaintenance` add this to your configuration:
To reduce "noise" in the repository, Renovate performs `lockFileMaintenance` `"before 4am on monday"`, i.e. to achieve once-per-week semantics.
Depending on its running schedule, Renovate may run a few times within that time window - even possibly updating the lock file more than once - but it hopefully leaves enough time for tests to run and automerge to apply, if configured.
+## logLevelRemap
+
+This option allows you to remap log levels for specific messages.
+
+Be careful with remapping `warn` or `error` messages to lower log levels, as it may hide important information.
+
+```json
+{
+ "logLevelRemap": [
+ {
+ "matchMessage": "/^pip-compile:/",
+ "newLogLevel": "info"
+ },
+ {
+ "matchMessage": "Package lookup error",
+ "newLogLevel": "warn"
+ }
+ ]
+}
+```
+
## major
Add to this object if you wish to define rules that apply only to major updates.
@@ -2588,6 +2624,26 @@ Use this field to restrict rules to a particular package manager. e.g.
For the full list of available managers, see the [Supported Managers](modules/manager/index.md#supported-managers) documentation.
+### matchMessage
+
+For log level remapping, use this field to match against the particular log messages.
+You can match based on any of the following:
+
+- an exact match string (e.g. `This is the string`)
+- a minimatch pattern (e.g. `This*`)
+- a regex pattern (e.g. `/^This/`)
+
+```json
+{
+ "logLevelRemap": [
+ {
+ "matchMessage": "Manager explicitly enabled*",
+ "newLogLevel": "warn"
+ }
+ ]
+}
+```
+
### matchDatasources
Use this field to restrict rules to a particular datasource. e.g.
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index 214d2ec7ca..0c3203e440 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -2818,6 +2818,32 @@ const options: RenovateOptions[] = [
cli: false,
env: false,
},
+ {
+ name: 'logLevelRemap',
+ description: 'Remap log levels to different levels.',
+ type: 'array',
+ subType: 'object',
+ stage: 'repository',
+ cli: false,
+ env: false,
+ },
+ {
+ name: 'matchMessage',
+ description: 'Regex/minimatch expression to match against log message.',
+ type: 'string',
+ parents: ['logLevelRemap'],
+ cli: false,
+ env: false,
+ },
+ {
+ name: 'newLogLevel',
+ description: 'New log level to use if matchMessage matches.',
+ type: 'string',
+ allowedValues: ['trace', 'debug', 'info', 'warn', 'error', 'fatal'],
+ parents: ['logLevelRemap'],
+ cli: false,
+ env: false,
+ },
];
export function getOptions(): RenovateOptions[] {
diff --git a/lib/config/types.ts b/lib/config/types.ts
index c18f8ace77..dc9a690db4 100644
--- a/lib/config/types.ts
+++ b/lib/config/types.ts
@@ -1,5 +1,6 @@
import type { LogLevel } from 'bunyan';
import type { PlatformId } from '../constants';
+import type { LogLevelRemap } from '../logger/types';
import type { CustomManager } from '../modules/manager/custom/types';
import type { HostRule } from '../types';
import type { GitNoVerifyOption } from '../util/git/types';
@@ -275,6 +276,8 @@ export interface RenovateConfig
customizeDashboard?: Record<string, string>;
statusCheckNames?: Record<StatusCheckKey, string | null>;
+
+ logLevelRemap?: LogLevelRemap[];
}
const CustomDatasourceFormats = ['json', 'plain', 'yaml', 'html'] as const;
@@ -374,7 +377,8 @@ export type AllowedParents =
| 'customDatasources'
| 'hostRules'
| 'postUpgradeTasks'
- | 'packageRules';
+ | 'packageRules'
+ | 'logLevelRemap';
export interface RenovateOptionBase {
/**
* If true, the option can only be configured by people with access to the Renovate instance.
diff --git a/lib/logger/index.ts b/lib/logger/index.ts
index 7e90ab4d84..54cb4ebf4e 100644
--- a/lib/logger/index.ts
+++ b/lib/logger/index.ts
@@ -6,6 +6,7 @@ import configSerializer from './config-serializer';
import errSerializer from './err-serializer';
import { once, reset as onceReset } from './once';
import { RenovateStream } from './pretty-stdout';
+import { getRemappedLevel } from './remap';
import type { BunyanRecord, Logger } from './types';
import { ProblemStream, validateLogLevel, withSanitizer } from './utils';
@@ -61,20 +62,39 @@ const bunyanLogger = bunyan.createLogger({
].map(withSanitizer),
});
-const logFactory =
- (level: bunyan.LogLevelString) =>
- (p1: any, p2: any): void => {
+const logFactory = (
+ _level: bunyan.LogLevelString,
+): ((p1: unknown, p2: unknown) => void) => {
+ return (p1: any, p2: any): void => {
+ let level = _level;
if (p2) {
// meta and msg provided
- bunyanLogger[level]({ logContext, ...curMeta, ...p1 }, p2);
+ const msg = p2;
+ const meta: Record<string, unknown> = { logContext, ...curMeta, ...p1 };
+ const remappedLevel = getRemappedLevel(msg);
+ // istanbul ignore if: not testable
+ if (remappedLevel) {
+ meta.oldLevel = level;
+ level = remappedLevel;
+ }
+ bunyanLogger[level](meta, msg);
} else if (is.string(p1)) {
// only message provided
- bunyanLogger[level]({ logContext, ...curMeta }, p1);
+ const msg = p1;
+ const meta: Record<string, unknown> = { logContext, ...curMeta };
+ const remappedLevel = getRemappedLevel(msg);
+ // istanbul ignore if: not testable
+ if (remappedLevel) {
+ meta.oldLevel = level;
+ level = remappedLevel;
+ }
+ bunyanLogger[level](meta, msg);
} else {
// only meta provided
bunyanLogger[level]({ logContext, ...curMeta, ...p1 });
}
};
+};
const loggerLevels: bunyan.LogLevelString[] = [
'trace',
diff --git a/lib/logger/remap.spec.ts b/lib/logger/remap.spec.ts
new file mode 100644
index 0000000000..3e7608971e
--- /dev/null
+++ b/lib/logger/remap.spec.ts
@@ -0,0 +1,72 @@
+import {
+ getRemappedLevel,
+ resetGlobalLogLevelRemaps,
+ resetRepositoryLogLevelRemaps,
+ setGlobalLogLevelRemaps,
+ setRepositoryLogLevelRemaps,
+} from './remap';
+
+describe('logger/remap', () => {
+ afterEach(() => {
+ resetRepositoryLogLevelRemaps();
+ resetGlobalLogLevelRemaps();
+ });
+
+ it('returns null if no remaps are set', () => {
+ setGlobalLogLevelRemaps(undefined);
+ setRepositoryLogLevelRemaps(undefined);
+
+ const res = getRemappedLevel('foo');
+
+ expect(res).toBeNull();
+ });
+
+ it('performs global remaps', () => {
+ setGlobalLogLevelRemaps([{ matchMessage: '*foo*', newLogLevel: 'error' }]);
+ setRepositoryLogLevelRemaps(undefined);
+
+ const res = getRemappedLevel('foo');
+
+ expect(res).toBe('error');
+ });
+
+ it('performs repository-level remaps', () => {
+ setGlobalLogLevelRemaps(undefined);
+ setRepositoryLogLevelRemaps([
+ { matchMessage: '*bar*', newLogLevel: 'error' },
+ ]);
+
+ const res = getRemappedLevel('bar');
+
+ expect(res).toBe('error');
+ });
+
+ it('prioritizes repository-level remaps over global remaps', () => {
+ setGlobalLogLevelRemaps([{ matchMessage: '*foo*', newLogLevel: 'error' }]);
+ setRepositoryLogLevelRemaps([
+ { matchMessage: '*bar*', newLogLevel: 'warn' },
+ ]);
+
+ const res = getRemappedLevel('foobar');
+
+ expect(res).toBe('warn');
+ });
+
+ it('supports regex patterns', () => {
+ setGlobalLogLevelRemaps([{ matchMessage: '/foo/', newLogLevel: 'error' }]);
+ setRepositoryLogLevelRemaps(undefined);
+
+ const res = getRemappedLevel('foo');
+
+ expect(res).toBe('error');
+ });
+
+ it('does not match against invalid regex patterns', () => {
+ setGlobalLogLevelRemaps([{ matchMessage: '/(/', newLogLevel: 'error' }]);
+ setRepositoryLogLevelRemaps(undefined);
+
+ const res = getRemappedLevel('()');
+
+ expect(res).toBeNull();
+ });
+});
diff --git a/lib/logger/remap.ts b/lib/logger/remap.ts
new file mode 100644
index 0000000000..57920e2677
--- /dev/null
+++ b/lib/logger/remap.ts
@@ -0,0 +1,68 @@
+import type { LogLevelString } from 'bunyan';
+import {
+ StringMatchPredicate,
+ makeRegexOrMinimatchPredicate,
+} from '../util/string-match';
+import type { LogLevelRemap } from './types';
+
+let globalRemaps: LogLevelRemap[] | undefined;
+let repositoryRemaps: LogLevelRemap[] | undefined;
+
+let matcherCache = new WeakMap<LogLevelRemap, StringMatchPredicate>();
+
+function match(remap: LogLevelRemap, input: string): boolean {
+ const { matchMessage: pattern } = remap;
+ let matchFn = matcherCache.get(remap);
+ if (!matchFn) {
+ matchFn = makeRegexOrMinimatchPredicate(pattern) ?? (() => false);
+ matcherCache.set(remap, matchFn);
+ }
+
+ return matchFn(input);
+}
+
+export function getRemappedLevel(msg: string): LogLevelString | null {
+ if (repositoryRemaps) {
+ for (const remap of repositoryRemaps) {
+ if (match(remap, msg)) {
+ return remap.newLogLevel;
+ }
+ }
+ }
+
+ if (globalRemaps) {
+ for (const remap of globalRemaps) {
+ if (match(remap, msg)) {
+ return remap.newLogLevel;
+ }
+ }
+ }
+
+ return null;
+}
+
+function resetMatcherCache(): void {
+ matcherCache = new WeakMap();
+}
+
+export function setGlobalLogLevelRemaps(
+ remaps: LogLevelRemap[] | undefined,
+): void {
+ globalRemaps = remaps;
+}
+
+export function resetGlobalLogLevelRemaps(): void {
+ globalRemaps = undefined;
+ resetMatcherCache();
+}
+
+export function setRepositoryLogLevelRemaps(
+ remaps: LogLevelRemap[] | undefined,
+): void {
+ repositoryRemaps = remaps;
+}
+
+export function resetRepositoryLogLevelRemaps(): void {
+ repositoryRemaps = undefined;
+ resetMatcherCache();
+}
diff --git a/lib/logger/types.ts b/lib/logger/types.ts
index 26fe16a73a..6880cc0f0c 100644
--- a/lib/logger/types.ts
+++ b/lib/logger/types.ts
@@ -1,5 +1,5 @@
import type { Stream } from 'node:stream';
-import type { LogLevel } from 'bunyan';
+import type { LogLevel, LogLevelString } from 'bunyan';
export interface LogError {
level: LogLevel;
@@ -40,3 +40,8 @@ export type BunyanStream = (NodeJS.WritableStream | Stream) & {
cb: (err?: Error | null) => void,
) => void;
};
+
+export interface LogLevelRemap {
+ matchMessage: string;
+ newLogLevel: LogLevelString;
+}
diff --git a/lib/workers/global/index.ts b/lib/workers/global/index.ts
index e2c2a83a23..cd285c06ee 100644
--- a/lib/workers/global/index.ts
+++ b/lib/workers/global/index.ts
@@ -17,6 +17,7 @@ import { CONFIG_PRESETS_INVALID } from '../../constants/error-messages';
import { pkg } from '../../expose.cjs';
import { instrument } from '../../instrumentation';
import { getProblems, logger, setMeta } from '../../logger';
+import { setGlobalLogLevelRemaps } from '../../logger/remap';
import * as hostRules from '../../util/host-rules';
import * as queue from '../../util/http/queue';
import * as throttle from '../../util/http/throttle';
@@ -158,6 +159,8 @@ export async function start(): Promise<number> {
// validate secrets. Will throw and abort if invalid
validateConfigSecrets(config);
+
+ setGlobalLogLevelRemaps(config.logLevelRemap);
});
// autodiscover repositories (needs to come after platform initialization)
diff --git a/lib/workers/global/initialize.ts b/lib/workers/global/initialize.ts
index 41af7f9a5a..47c780b4ea 100644
--- a/lib/workers/global/initialize.ts
+++ b/lib/workers/global/initialize.ts
@@ -4,6 +4,7 @@ import upath from 'upath';
import { applySecretsToConfig } from '../../config/secrets';
import type { AllConfig, RenovateConfig } from '../../config/types';
import { logger } from '../../logger';
+import { resetGlobalLogLevelRemaps } from '../../logger/remap';
import { initPlatform } from '../../modules/platform';
import * as packageCache from '../../util/cache/package';
import { setEmojiConfig } from '../../util/emoji';
@@ -93,4 +94,5 @@ export async function globalInitialize(
export async function globalFinalize(config: RenovateConfig): Promise<void> {
await packageCache.cleanup(config);
+ resetGlobalLogLevelRemaps();
}
diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts
index 6e775996ce..fa2294a020 100644
--- a/lib/workers/repository/index.ts
+++ b/lib/workers/repository/index.ts
@@ -10,6 +10,7 @@ import {
import { pkg } from '../../expose.cjs';
import { instrument } from '../../instrumentation';
import { logger, setMeta } from '../../logger';
+import { resetRepositoryLogLevelRemaps } from '../../logger/remap';
import { removeDanglingContainers } from '../../util/exec/docker';
import { deleteLocalFile, privateCacheDir } from '../../util/fs';
import { isCloned } from '../../util/git';
@@ -129,6 +130,7 @@ export async function renovateRepository(
clearDnsCache();
const cloned = isCloned();
logger.info({ cloned, durationMs: splits.total }, 'Repository finished');
+ resetRepositoryLogLevelRemaps();
return repoResult;
}
diff --git a/lib/workers/repository/init/index.ts b/lib/workers/repository/init/index.ts
index 374b1ec8c8..81f05826c0 100644
--- a/lib/workers/repository/init/index.ts
+++ b/lib/workers/repository/init/index.ts
@@ -2,6 +2,7 @@ import { GlobalConfig } from '../../../config/global';
import { applySecretsToConfig } from '../../../config/secrets';
import type { RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
+import { setRepositoryLogLevelRemaps } from '../../../logger/remap';
import { platform } from '../../../modules/platform';
import { clone } from '../../../util/clone';
import { cloneSubmodules, setUserRepoConfig } from '../../../util/git';
@@ -50,6 +51,7 @@ export async function initRepo(
config = await initApis(config);
await initializeCaches(config as WorkerPlatformConfig);
config = await getRepoConfig(config);
+ setRepositoryLogLevelRemaps(config.logLevelRemap);
checkIfConfigured(config);
warnOnUnsupportedOptions(config);
config = applySecretsToConfig(config);
--
GitLab