From 481aa216b22a5fd5c26a1bda8aafe3c5b41e719f Mon Sep 17 00:00:00 2001
From: Sebastian Poxhofer <secustor@users.noreply.github.com>
Date: Sun, 17 Mar 2024 10:22:42 +0100
Subject: [PATCH] feat(instrumentation/reporting): add report option (#26087)
Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
docs/usage/self-hosted-configuration.md | 19 ++
lib/config/options/index.ts | 18 ++
lib/config/types.ts | 2 +
lib/config/validation.spec.ts | 50 +++++
lib/config/validation.ts | 15 ++
lib/instrumentation/reporting.spec.ts | 177 ++++++++++++++++++
lib/instrumentation/reporting.ts | 97 ++++++++++
lib/instrumentation/types.ts | 11 ++
lib/util/cache/repository/types.ts | 2 +-
lib/util/fs/index.spec.ts | 9 +
lib/util/fs/index.ts | 7 +
lib/workers/global/index.ts | 3 +
.../finalize/repository-statistics.ts | 2 +
lib/workers/repository/index.ts | 7 +-
test/s3.ts | 4 +
15 files changed, 421 insertions(+), 2 deletions(-)
create mode 100644 lib/instrumentation/reporting.spec.ts
create mode 100644 lib/instrumentation/reporting.ts
create mode 100644 test/s3.ts
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index cfbfbd8604..5dcbf4f77d 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -909,6 +909,25 @@ For TLS/SSL-enabled connections, use rediss prefix
Example URL structure: `rediss://[[username]:[password]]@localhost:6379/0`.
+## reportPath
+
+`reportPath` describes the location where the report is written to.
+
+If [`reportType`](#reporttype) is set to `file`, then set `reportPath` to a filepath.
+For example: `/foo/bar.json`.
+
+If the value `s3` is used in [`reportType`](#reporttype), then use a S3 URI.
+For example: `s3://bucket-name/key-name`.
+
+## reportType
+
+Defines how the report is exposed:
+
+- `<unset>` If unset, no report will be provided, though the debug logs will still have partial information of the report
+- `logging` The report will be printed as part of the log messages on `INFO` level
+- `file` The report will be written to a path provided by [`reportPath`](#reportpath)
+- `s3` The report is pushed to an S3 bucket defined by [`reportPath`](#reportpath). This option reuses [`RENOVATE_X_S3_ENDPOINT`](./self-hosted-experimental.md#renovatexs3endpoint) and [`RENOVATE_X_S3_PATH_STYLE`](./self-hosted-experimental.md#renovatexs3pathstyle)
+
## repositories
Elements in the `repositories` array can be an object if you wish to define more settings:
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index 72eecec44c..1e07029e2b 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -299,6 +299,24 @@ const options: RenovateOptions[] = [
stage: 'repository',
default: 'local',
},
+ {
+ name: 'reportType',
+ description: 'Set how, or if, reports should be generated.',
+ globalOnly: true,
+ type: 'string',
+ default: null,
+ experimental: true,
+ allowedValues: ['logging', 'file', 's3'],
+ },
+ {
+ name: 'reportPath',
+ description:
+ 'Path to where the file should be written. In case of `s3` this has to be a full S3 URI.',
+ globalOnly: true,
+ type: 'string',
+ default: null,
+ experimental: true,
+ },
{
name: 'force',
description:
diff --git a/lib/config/types.ts b/lib/config/types.ts
index ae3c2364ed..06ea517e84 100644
--- a/lib/config/types.ts
+++ b/lib/config/types.ts
@@ -215,6 +215,8 @@ export interface RenovateConfig
AssigneesAndReviewersConfig,
ConfigMigration,
Record<string, unknown> {
+ reportPath?: string;
+ reportType?: 'logging' | 'file' | 's3' | null;
depName?: string;
baseBranches?: string[];
commitBody?: string;
diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts
index b92743957a..9d675b5b7c 100644
--- a/lib/config/validation.spec.ts
+++ b/lib/config/validation.spec.ts
@@ -1620,5 +1620,55 @@ describe('config/validation', () => {
expect(warnings).toHaveLength(0);
expect(errors).toHaveLength(0);
});
+
+ it('fails for missing reportPath if reportType is "s3"', async () => {
+ const config: RenovateConfig = {
+ reportType: 's3',
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ 'global',
+ config,
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toHaveLength(1);
+ });
+
+ it('validates reportPath if reportType is "s3"', async () => {
+ const config: RenovateConfig = {
+ reportType: 's3',
+ reportPath: 's3://bucket-name/key-name',
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ 'global',
+ config,
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toHaveLength(0);
+ });
+
+ it('fails for missing reportPath if reportType is "file"', async () => {
+ const config: RenovateConfig = {
+ reportType: 'file',
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ 'global',
+ config,
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toHaveLength(1);
+ });
+
+ it('validates reportPath if reportType is "file"', async () => {
+ const config: RenovateConfig = {
+ reportType: 'file',
+ reportPath: './report.json',
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ 'global',
+ config,
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toHaveLength(0);
+ });
});
});
diff --git a/lib/config/validation.ts b/lib/config/validation.ts
index f555df8fd6..17c67b5692 100644
--- a/lib/config/validation.ts
+++ b/lib/config/validation.ts
@@ -170,7 +170,9 @@ export async function validateConfig(
val,
optionTypes[key],
warnings,
+ errors,
currentPath,
+ config,
);
continue;
} else {
@@ -828,7 +830,9 @@ async function validateGlobalConfig(
val: unknown,
type: string,
warnings: ValidationMessage[],
+ errors: ValidationMessage[],
currentPath: string | undefined,
+ config: RenovateConfig,
): Promise<void> {
if (val !== null) {
if (type === 'string') {
@@ -882,6 +886,17 @@ async function validateGlobalConfig(
message: `Invalid value \`${val}\` for \`${currentPath}\`. The allowed values are ${['default', 'ssh', 'endpoint'].join(', ')}.`,
});
}
+
+ if (
+ key === 'reportType' &&
+ ['s3', 'file'].includes(val) &&
+ !is.string(config.reportPath)
+ ) {
+ errors.push({
+ topic: 'Configuration Error',
+ message: `reportType '${val}' requires a configured reportPath`,
+ });
+ }
} else {
warnings.push({
topic: 'Configuration Error',
diff --git a/lib/instrumentation/reporting.spec.ts b/lib/instrumentation/reporting.spec.ts
new file mode 100644
index 0000000000..8705ac618b
--- /dev/null
+++ b/lib/instrumentation/reporting.spec.ts
@@ -0,0 +1,177 @@
+import type { S3Client } from '@aws-sdk/client-s3';
+import { mockDeep } from 'jest-mock-extended';
+import { s3 } from '../../test/s3';
+import { fs, logger } from '../../test/util';
+import type { RenovateConfig } from '../config/types';
+import type { PackageFile } from '../modules/manager/types';
+import type { BranchCache } from '../util/cache/repository/types';
+import {
+ addBranchStats,
+ addExtractionStats,
+ exportStats,
+ getReport,
+} from './reporting';
+
+jest.mock('../util/fs', () => mockDeep());
+jest.mock('../util/s3', () => mockDeep());
+
+describe('instrumentation/reporting', () => {
+ const branchInformation: Partial<BranchCache>[] = [
+ {
+ branchName: 'a-branch-name',
+ prNo: 20,
+ upgrades: [
+ {
+ currentVersion: '21.1.1',
+ currentValue: 'v21.1.1',
+ newVersion: '22.0.0',
+ newValue: 'v22.0.0',
+ },
+ ],
+ },
+ ];
+ const packageFiles: Record<string, PackageFile[]> = {
+ terraform: [
+ {
+ packageFile: 'terraform/versions.tf',
+ deps: [
+ {
+ currentValue: 'v21.1.1',
+ currentVersion: '4.4.3',
+ updates: [
+ {
+ bucket: 'non-major',
+ newVersion: '4.7.0',
+ newValue: '~> 4.7.0',
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ };
+
+ const expectedReport = {
+ repositories: {
+ 'myOrg/myRepo': {
+ branches: branchInformation,
+ packageFiles,
+ },
+ },
+ };
+
+ it('return empty report if no stats have been added', () => {
+ const config = {};
+ addBranchStats(config, []);
+ addExtractionStats(config, {
+ branchList: [],
+ branches: [],
+ packageFiles: {},
+ });
+
+ expect(getReport()).toEqual({
+ repositories: {},
+ });
+ });
+
+ it('return report if reportType is set to logging', () => {
+ const config: RenovateConfig = {
+ repository: 'myOrg/myRepo',
+ reportType: 'logging',
+ };
+
+ addBranchStats(config, branchInformation);
+ addExtractionStats(config, { branchList: [], branches: [], packageFiles });
+
+ expect(getReport()).toEqual(expectedReport);
+ });
+
+ it('log report if reportType is set to logging', async () => {
+ const config: RenovateConfig = {
+ repository: 'myOrg/myRepo',
+ reportType: 'logging',
+ };
+
+ addBranchStats(config, branchInformation);
+ addExtractionStats(config, { branchList: [], branches: [], packageFiles });
+
+ await exportStats(config);
+ expect(logger.logger.info).toHaveBeenCalledWith(
+ { report: expectedReport },
+ 'Printing report',
+ );
+ });
+
+ it('write report if reportType is set to file', async () => {
+ const config: RenovateConfig = {
+ repository: 'myOrg/myRepo',
+ reportType: 'file',
+ reportPath: './report.json',
+ };
+
+ addBranchStats(config, branchInformation);
+ addExtractionStats(config, { branchList: [], branches: [], packageFiles });
+
+ await exportStats(config);
+ expect(fs.writeSystemFile).toHaveBeenCalledWith(
+ config.reportPath,
+ JSON.stringify(expectedReport),
+ );
+ });
+
+ it('send report to an S3 bucket if reportType is s3', async () => {
+ const mockClient = mockDeep<S3Client>();
+ s3.parseS3Url.mockReturnValue({ Bucket: 'bucket-name', Key: 'key-name' });
+ // @ts-expect-error TS2589
+ s3.getS3Client.mockReturnValue(mockClient);
+
+ const config: RenovateConfig = {
+ repository: 'myOrg/myRepo',
+ reportType: 's3',
+ reportPath: 's3://bucket-name/key-name',
+ };
+
+ addBranchStats(config, branchInformation);
+ addExtractionStats(config, { branchList: [], branches: [], packageFiles });
+
+ await exportStats(config);
+ expect(mockClient.send.mock.calls[0][0]).toMatchObject({
+ input: {
+ Body: JSON.stringify(expectedReport),
+ },
+ });
+ });
+
+ it('handle failed parsing of S3 url', async () => {
+ s3.parseS3Url.mockReturnValue(null);
+
+ const config: RenovateConfig = {
+ repository: 'myOrg/myRepo',
+ reportType: 's3',
+ reportPath: 'aPath',
+ };
+
+ addBranchStats(config, branchInformation);
+ addExtractionStats(config, { branchList: [], branches: [], packageFiles });
+
+ await exportStats(config);
+ expect(logger.logger.warn).toHaveBeenCalledWith(
+ { reportPath: config.reportPath },
+ 'Failed to parse s3 URL',
+ );
+ });
+
+ it('catch exception', async () => {
+ const config: RenovateConfig = {
+ repository: 'myOrg/myRepo',
+ reportType: 'file',
+ reportPath: './report.json',
+ };
+
+ addBranchStats(config, branchInformation);
+ addExtractionStats(config, { branchList: [], branches: [], packageFiles });
+
+ fs.writeSystemFile.mockRejectedValue(null);
+ await expect(exportStats(config)).toResolve();
+ });
+});
diff --git a/lib/instrumentation/reporting.ts b/lib/instrumentation/reporting.ts
new file mode 100644
index 0000000000..2167212ffb
--- /dev/null
+++ b/lib/instrumentation/reporting.ts
@@ -0,0 +1,97 @@
+import { PutObjectCommand, PutObjectCommandInput } from '@aws-sdk/client-s3';
+import is from '@sindresorhus/is';
+import type { RenovateConfig } from '../config/types';
+import { logger } from '../logger';
+import type { BranchCache } from '../util/cache/repository/types';
+import { writeSystemFile } from '../util/fs';
+import { getS3Client, parseS3Url } from '../util/s3';
+import type { ExtractResult } from '../workers/repository/process/extract-update';
+import type { Report } from './types';
+
+const report: Report = {
+ repositories: {},
+};
+
+export function addBranchStats(
+ config: RenovateConfig,
+ branchesInformation: Partial<BranchCache>[],
+): void {
+ if (is.nullOrUndefined(config.reportType)) {
+ return;
+ }
+
+ coerceRepo(config.repository!);
+ report.repositories[config.repository!].branches = branchesInformation;
+}
+
+export function addExtractionStats(
+ config: RenovateConfig,
+ extractResult: ExtractResult,
+): void {
+ if (is.nullOrUndefined(config.reportType)) {
+ return;
+ }
+
+ coerceRepo(config.repository!);
+ report.repositories[config.repository!].packageFiles =
+ extractResult.packageFiles;
+}
+
+export async function exportStats(config: RenovateConfig): Promise<void> {
+ try {
+ if (is.nullOrUndefined(config.reportType)) {
+ return;
+ }
+
+ if (config.reportType === 'logging') {
+ logger.info({ report }, 'Printing report');
+ return;
+ }
+
+ if (config.reportType === 'file') {
+ const path = config.reportPath!;
+ await writeSystemFile(path, JSON.stringify(report));
+ logger.debug({ path }, 'Writing report');
+ return;
+ }
+
+ if (config.reportType === 's3') {
+ const s3Url = parseS3Url(config.reportPath!);
+ if (is.nullOrUndefined(s3Url)) {
+ logger.warn(
+ { reportPath: config.reportPath },
+ 'Failed to parse s3 URL',
+ );
+ return;
+ }
+
+ const s3Params: PutObjectCommandInput = {
+ Bucket: s3Url.Bucket,
+ Key: s3Url.Key,
+ Body: JSON.stringify(report),
+ ContentType: 'application/json',
+ };
+
+ const client = getS3Client();
+ const command = new PutObjectCommand(s3Params);
+ await client.send(command);
+ }
+ } catch (err) {
+ logger.warn({ err }, 'Reporting.exportStats() - failure');
+ }
+}
+
+export function getReport(): Report {
+ return structuredClone(report);
+}
+
+function coerceRepo(repository: string): void {
+ if (!is.undefined(report.repositories[repository])) {
+ return;
+ }
+
+ report.repositories[repository] = {
+ branches: [],
+ packageFiles: {},
+ };
+}
diff --git a/lib/instrumentation/types.ts b/lib/instrumentation/types.ts
index a753ecb56d..aa29b4ebdb 100644
--- a/lib/instrumentation/types.ts
+++ b/lib/instrumentation/types.ts
@@ -1,4 +1,6 @@
import type { Attributes, SpanKind } from '@opentelemetry/api';
+import type { PackageFile } from '../modules/manager/types';
+import type { BranchCache } from '../util/cache/repository/types';
/**
* The instrumentation decorator parameters.
@@ -24,3 +26,12 @@ export interface SpanParameters {
*/
kind?: SpanKind;
}
+
+export interface Report {
+ repositories: Record<string, RepoReport>;
+}
+
+interface RepoReport {
+ branches: Partial<BranchCache>[];
+ packageFiles: Record<string, PackageFile[]>;
+}
diff --git a/lib/util/cache/repository/types.ts b/lib/util/cache/repository/types.ts
index 4e52dc617b..7859e03cfa 100644
--- a/lib/util/cache/repository/types.ts
+++ b/lib/util/cache/repository/types.ts
@@ -83,7 +83,7 @@ export interface BranchCache {
*/
branchName: string;
/**
- * Whether the update branch is behind base branh
+ * Whether the update branch is behind base branch
*/
isBehindBase?: boolean;
/**
diff --git a/lib/util/fs/index.spec.ts b/lib/util/fs/index.spec.ts
index 13c1757168..f8e71562ab 100644
--- a/lib/util/fs/index.spec.ts
+++ b/lib/util/fs/index.spec.ts
@@ -33,6 +33,7 @@ import {
rmCache,
statLocalFile,
writeLocalFile,
+ writeSystemFile,
} from '.';
jest.mock('../exec/env');
@@ -480,6 +481,14 @@ describe('util/fs/index', () => {
});
});
+ describe('writeSystemFile', () => {
+ it('writes file', async () => {
+ const path = `${tmpDir}/file.txt`;
+ await writeSystemFile(path, 'foobar');
+ expect(await readSystemFile(path)).toEqual(Buffer.from('foobar'));
+ });
+ });
+
describe('getLocalFiles', () => {
it('reads list of files from local fs', async () => {
const fileContentMap = {
diff --git a/lib/util/fs/index.ts b/lib/util/fs/index.ts
index 0d34ea3a09..14930c03b7 100644
--- a/lib/util/fs/index.ts
+++ b/lib/util/fs/index.ts
@@ -302,6 +302,13 @@ export function readSystemFile(
return encoding ? fs.readFile(fileName, encoding) : fs.readFile(fileName);
}
+export async function writeSystemFile(
+ fileName: string,
+ data: string | Buffer,
+): Promise<void> {
+ await fs.outputFile(fileName, data);
+}
+
export async function getLocalFiles(
fileNames: string[],
): Promise<Record<string, string | null>> {
diff --git a/lib/workers/global/index.ts b/lib/workers/global/index.ts
index cd285c06ee..e9b4f5abc6 100644
--- a/lib/workers/global/index.ts
+++ b/lib/workers/global/index.ts
@@ -16,6 +16,7 @@ import type {
import { CONFIG_PRESETS_INVALID } from '../../constants/error-messages';
import { pkg } from '../../expose.cjs';
import { instrument } from '../../instrumentation';
+import { exportStats } from '../../instrumentation/reporting';
import { getProblems, logger, setMeta } from '../../logger';
import { setGlobalLogLevelRemaps } from '../../logger/remap';
import * as hostRules from '../../util/host-rules';
@@ -210,6 +211,8 @@ export async function start(): Promise<number> {
},
);
}
+
+ await exportStats(config);
} catch (err) /* istanbul ignore next */ {
if (err.message.startsWith('Init: ')) {
logger.fatal(err.message.substring(6));
diff --git a/lib/workers/repository/finalize/repository-statistics.ts b/lib/workers/repository/finalize/repository-statistics.ts
index 28c795a662..36d450c929 100644
--- a/lib/workers/repository/finalize/repository-statistics.ts
+++ b/lib/workers/repository/finalize/repository-statistics.ts
@@ -1,4 +1,5 @@
import type { RenovateConfig } from '../../../config/types';
+import { addBranchStats } from '../../../instrumentation/reporting';
import { logger } from '../../../logger';
import type { Pr } from '../../../modules/platform';
import { getCache, isCacheModified } from '../../../util/cache/repository';
@@ -152,6 +153,7 @@ export function runBranchSummary(config: RenovateConfig): void {
if (branches?.length) {
const branchesInformation = filterDependencyDashboardData(branches);
+ addBranchStats(config, branchesInformation);
logger.debug({ branchesInformation }, 'branches info extended');
}
}
diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts
index fa7fa880e8..f9d30bcf78 100644
--- a/lib/workers/repository/index.ts
+++ b/lib/workers/repository/index.ts
@@ -9,6 +9,7 @@ import {
} from '../../constants/error-messages';
import { pkg } from '../../expose.cjs';
import { instrument } from '../../instrumentation';
+import { addExtractionStats } from '../../instrumentation/reporting';
import { logger, setMeta } from '../../logger';
import { resetRepositoryLogLevelRemaps } from '../../logger/remap';
import { removeDanglingContainers } from '../../util/exec/docker';
@@ -64,9 +65,13 @@ export async function renovateRepository(
config.repoIsOnboarded! ||
!OnboardingState.onboardingCacheValid ||
OnboardingState.prUpdateRequested;
- const { branches, branchList, packageFiles } = performExtract
+ const extractResult = performExtract
? await instrument('extract', () => extractDependencies(config))
: emptyExtract(config);
+ addExtractionStats(config, extractResult);
+
+ const { branches, branchList, packageFiles } = extractResult;
+
if (config.semanticCommits === 'auto') {
config.semanticCommits = await detectSemanticCommits();
}
diff --git a/test/s3.ts b/test/s3.ts
new file mode 100644
index 0000000000..85f94122e7
--- /dev/null
+++ b/test/s3.ts
@@ -0,0 +1,4 @@
+import { jest } from '@jest/globals';
+import * as _s3 from '../lib/util/s3';
+
+export const s3 = jest.mocked(_s3);
--
GitLab