From da4ee8b8741491ba85981f55708e89ac812f0fb4 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Tue, 8 Oct 2024 11:11:15 +0200
Subject: [PATCH] feat(jsonata): validation, caching, better logging (#31832)
---
lib/config/validation.spec.ts | 9 ++++++
lib/config/validation.ts | 13 +++++++-
lib/modules/datasource/custom/index.spec.ts | 33 ++++++++++++++++++---
lib/modules/datasource/custom/index.ts | 19 ++++++++----
lib/util/jsonata.spec.ts | 13 ++++++++
lib/util/jsonata.ts | 21 +++++++++++++
6 files changed, 98 insertions(+), 10 deletions(-)
create mode 100644 lib/util/jsonata.spec.ts
create mode 100644 lib/util/jsonata.ts
diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts
index 4eb07aa8ca..e7dc924ca5 100644
--- a/lib/config/validation.spec.ts
+++ b/lib/config/validation.spec.ts
@@ -306,10 +306,19 @@ describe('config/validation', () => {
defaultRegistryUrlTemplate: [],
transformTemplates: [{}],
},
+ bar: {
+ description: 'foo',
+ defaultRegistryUrlTemplate: 'bar',
+ transformTemplates: ['foo = "bar"', 'bar[0'],
+ },
},
} as any;
const { errors } = await configValidation.validateConfig('repo', config);
expect(errors).toMatchObject([
+ {
+ message:
+ 'Invalid JSONata expression for customDatasources: Expected "]" before end of expression',
+ },
{
message:
'Invalid `customDatasources.defaultRegistryUrlTemplate` configuration: is a string',
diff --git a/lib/config/validation.ts b/lib/config/validation.ts
index fdcf487a20..b6218b84be 100644
--- a/lib/config/validation.ts
+++ b/lib/config/validation.ts
@@ -8,6 +8,7 @@ import type {
} from '../modules/manager/custom/regex/types';
import type { CustomManager } from '../modules/manager/custom/types';
import type { HostRule } from '../types';
+import { getExpression } from '../util/jsonata';
import { regEx } from '../util/regex';
import {
getRegexPredicate,
@@ -745,7 +746,17 @@ export async function validateConfig(
message: `Invalid \`${currentPath}.${subKey}\` configuration: key is not allowed`,
});
} else if (subKey === 'transformTemplates') {
- if (!is.array(subValue, is.string)) {
+ if (is.array(subValue, is.string)) {
+ for (const expression of subValue) {
+ const res = getExpression(expression);
+ if (res instanceof Error) {
+ errors.push({
+ topic: 'Configuration Error',
+ message: `Invalid JSONata expression for ${currentPath}: ${res.message}`,
+ });
+ }
+ }
+ } else {
errors.push({
topic: 'Configuration Error',
message: `Invalid \`${currentPath}.${subKey}\` configuration: is not an array of string`,
diff --git a/lib/modules/datasource/custom/index.spec.ts b/lib/modules/datasource/custom/index.spec.ts
index 2eb0f46212..62574bdf77 100644
--- a/lib/modules/datasource/custom/index.spec.ts
+++ b/lib/modules/datasource/custom/index.spec.ts
@@ -229,7 +229,7 @@ describe('modules/datasource/custom/index', () => {
expect(result).toEqual(expected);
});
- it('returns null if transformation using jsonata rules fail', async () => {
+ it('returns null if transformation compilation using jsonata fails', async () => {
httpMock
.scope('https://example.com')
.get('/v1')
@@ -248,9 +248,34 @@ describe('modules/datasource/custom/index', () => {
},
});
expect(result).toBeNull();
- expect(logger.debug).toHaveBeenCalledWith(
- { err: expect.any(Object), transformTemplate: '$[.name = "Alice" and' },
- 'Error while transforming response',
+ expect(logger.once.warn).toHaveBeenCalledWith(
+ { errorMessage: 'The symbol "." cannot be used as a unary operator' },
+ 'Invalid JSONata expression: $[.name = "Alice" and',
+ );
+ });
+
+ it('returns null if jsonata expression evaluation fails', async () => {
+ httpMock
+ .scope('https://example.com')
+ .get('/v1')
+ .reply(200, '1.0.0 \n2.0.0 \n 3.0.0 ', {
+ 'Content-Type': 'text/plain',
+ });
+ const result = await getPkgReleases({
+ datasource: `${CustomDatasource.id}.foo`,
+ packageName: 'myPackage',
+ customDatasources: {
+ foo: {
+ defaultRegistryUrlTemplate: 'https://example.com/v1',
+ transformTemplates: ['$notafunction()'],
+ format: 'plain',
+ },
+ },
+ });
+ expect(result).toBeNull();
+ expect(logger.once.warn).toHaveBeenCalledWith(
+ { err: expect.any(Object) },
+ 'Error while evaluating JSONata expression: $notafunction()',
);
});
diff --git a/lib/modules/datasource/custom/index.ts b/lib/modules/datasource/custom/index.ts
index 0ecdc11bc8..372d15e97a 100644
--- a/lib/modules/datasource/custom/index.ts
+++ b/lib/modules/datasource/custom/index.ts
@@ -1,6 +1,6 @@
import is from '@sindresorhus/is';
-import jsonata from 'jsonata';
import { logger } from '../../../logger';
+import { getExpression } from '../../../util/jsonata';
import { Datasource } from '../datasource';
import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
import { fetchers } from './formats';
@@ -46,13 +46,22 @@ export class CustomDatasource extends Datasource {
logger.trace({ data }, `Custom manager fetcher '${format}' returned data.`);
for (const transformTemplate of transformTemplates) {
+ const expression = getExpression(transformTemplate);
+
+ if (expression instanceof Error) {
+ logger.once.warn(
+ { errorMessage: expression.message },
+ `Invalid JSONata expression: ${transformTemplate}`,
+ );
+ return null;
+ }
+
try {
- const expression = jsonata(transformTemplate);
data = await expression.evaluate(data);
} catch (err) {
- logger.debug(
- { err, transformTemplate },
- 'Error while transforming response',
+ logger.once.warn(
+ { err },
+ `Error while evaluating JSONata expression: ${transformTemplate}`,
);
return null;
}
diff --git a/lib/util/jsonata.spec.ts b/lib/util/jsonata.spec.ts
new file mode 100644
index 0000000000..f5a6651637
--- /dev/null
+++ b/lib/util/jsonata.spec.ts
@@ -0,0 +1,13 @@
+import { getExpression } from './jsonata';
+
+describe('util/jsonata', () => {
+ describe('getExpression', () => {
+ it('should return an expression', () => {
+ expect(getExpression('foo')).not.toBeInstanceOf(Error);
+ });
+
+ it('should return an error', () => {
+ expect(getExpression('foo[')).toBeInstanceOf(Error);
+ });
+ });
+});
diff --git a/lib/util/jsonata.ts b/lib/util/jsonata.ts
new file mode 100644
index 0000000000..2811f4e407
--- /dev/null
+++ b/lib/util/jsonata.ts
@@ -0,0 +1,21 @@
+import jsonata from 'jsonata';
+import * as memCache from './cache/memory';
+import { toSha256 } from './hash';
+
+export function getExpression(input: string): jsonata.Expression | Error {
+ const cacheKey = `jsonata:${toSha256(input)}`;
+ const cachedExpression = memCache.get<jsonata.Expression | Error>(cacheKey);
+ // istanbul ignore if: cannot test
+ if (cachedExpression) {
+ return cachedExpression;
+ }
+ let result: jsonata.Expression | Error;
+ try {
+ result = jsonata(input);
+ } catch (err) {
+ // JSONata errors aren't detected as TypeOf Error
+ result = new Error(err.message);
+ }
+ memCache.set(cacheKey, result);
+ return result;
+}
--
GitLab