From fb72204a28e664d6dd3e58bda7fc33f533e45bd7 Mon Sep 17 00:00:00 2001
From: Dan Ports <danports@gmail.com>
Date: Fri, 26 Aug 2022 04:44:53 -0700
Subject: [PATCH] feat(datasource): New datasource that pulls database engine
versions available for use on AWS RDS (#17345)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
lib/modules/datasource/api.ts | 2 +
lib/modules/datasource/aws-rds/index.spec.ts | 157 +++++++++++++++++++
lib/modules/datasource/aws-rds/index.ts | 43 +++++
lib/modules/datasource/aws-rds/readme.md | 78 +++++++++
package.json | 1 +
yarn.lock | 55 +++++++
6 files changed, 336 insertions(+)
create mode 100644 lib/modules/datasource/aws-rds/index.spec.ts
create mode 100644 lib/modules/datasource/aws-rds/index.ts
create mode 100644 lib/modules/datasource/aws-rds/readme.md
diff --git a/lib/modules/datasource/api.ts b/lib/modules/datasource/api.ts
index 4bb76b0b3c..130107de4c 100644
--- a/lib/modules/datasource/api.ts
+++ b/lib/modules/datasource/api.ts
@@ -1,6 +1,7 @@
import { AdoptiumJavaDatasource } from './adoptium-java';
import { ArtifactoryDatasource } from './artifactory';
import { AwsMachineImageDataSource } from './aws-machine-image';
+import { AwsRdsDataSource } from './aws-rds';
import { AzurePipelinesTasksDatasource } from './azure-pipelines-tasks';
import { BitBucketTagsDatasource } from './bitbucket-tags';
import { CdnJsDatasource } from './cdnjs';
@@ -52,6 +53,7 @@ export default api;
api.set(AdoptiumJavaDatasource.id, new AdoptiumJavaDatasource());
api.set(ArtifactoryDatasource.id, new ArtifactoryDatasource());
api.set(AwsMachineImageDataSource.id, new AwsMachineImageDataSource());
+api.set(AwsRdsDataSource.id, new AwsRdsDataSource());
api.set(AzurePipelinesTasksDatasource.id, new AzurePipelinesTasksDatasource());
api.set(BitBucketTagsDatasource.id, new BitBucketTagsDatasource());
api.set(CdnJsDatasource.id, new CdnJsDatasource());
diff --git a/lib/modules/datasource/aws-rds/index.spec.ts b/lib/modules/datasource/aws-rds/index.spec.ts
new file mode 100644
index 0000000000..8a5015e689
--- /dev/null
+++ b/lib/modules/datasource/aws-rds/index.spec.ts
@@ -0,0 +1,157 @@
+import {
+ DBEngineVersion,
+ DescribeDBEngineVersionsCommand,
+ DescribeDBEngineVersionsCommandOutput,
+ RDSClient,
+} from '@aws-sdk/client-rds';
+import { mockClient } from 'aws-sdk-client-mock';
+import { getPkgReleases } from '..';
+import { AwsRdsDataSource } from '.';
+
+const rdsMock = mockClient(RDSClient);
+
+const version1: DBEngineVersion = {
+ Engine: 'mysql',
+ EngineVersion: '8.0.26',
+ DBParameterGroupFamily: 'mysql8.0',
+ DBEngineDescription: 'MySQL Community Edition',
+ DBEngineVersionDescription: 'MySQL 8.0.26',
+ ValidUpgradeTarget: [
+ {
+ Engine: 'mysql',
+ EngineVersion: '8.0.27',
+ Description: 'MySQL 8.0.27',
+ AutoUpgrade: false,
+ IsMajorVersionUpgrade: false,
+ },
+ {
+ Engine: 'mysql',
+ EngineVersion: '8.0.28',
+ Description: 'MySQL 8.0.28',
+ AutoUpgrade: false,
+ IsMajorVersionUpgrade: false,
+ },
+ ],
+ ExportableLogTypes: ['audit', 'error', 'general', 'slowquery'],
+ SupportsLogExportsToCloudwatchLogs: true,
+ SupportsReadReplica: true,
+ SupportedFeatureNames: [],
+ Status: 'available',
+ SupportsParallelQuery: false,
+ SupportsGlobalDatabases: false,
+ MajorEngineVersion: '8.0',
+ SupportsBabelfish: false,
+};
+
+const version2: DBEngineVersion = {
+ Engine: 'mysql',
+ EngineVersion: '8.0.27',
+ DBParameterGroupFamily: 'mysql8.0',
+ DBEngineDescription: 'MySQL Community Edition',
+ DBEngineVersionDescription: 'MySQL 8.0.27',
+ ValidUpgradeTarget: [
+ {
+ Engine: 'mysql',
+ EngineVersion: '8.0.28',
+ Description: 'MySQL 8.0.28',
+ AutoUpgrade: false,
+ IsMajorVersionUpgrade: false,
+ },
+ ],
+ ExportableLogTypes: ['audit', 'error', 'general', 'slowquery'],
+ SupportsLogExportsToCloudwatchLogs: true,
+ SupportsReadReplica: true,
+ SupportedFeatureNames: [],
+ Status: 'deprecated',
+ SupportsParallelQuery: false,
+ SupportsGlobalDatabases: false,
+ MajorEngineVersion: '8.0',
+ SupportsBabelfish: false,
+};
+
+const version3: DBEngineVersion = {
+ Engine: 'mysql',
+ EngineVersion: '8.0.28',
+ DBParameterGroupFamily: 'mysql8.0',
+ DBEngineDescription: 'MySQL Community Edition',
+ DBEngineVersionDescription: 'MySQL 8.0.28',
+ ValidUpgradeTarget: [],
+ ExportableLogTypes: ['audit', 'error', 'general', 'slowquery'],
+ SupportsLogExportsToCloudwatchLogs: true,
+ SupportsReadReplica: true,
+ SupportedFeatureNames: [],
+ Status: 'available',
+ SupportsParallelQuery: false,
+ SupportsGlobalDatabases: false,
+ MajorEngineVersion: '8.0',
+ SupportsBabelfish: false,
+};
+
+function mockDescribeVersionsCommand(
+ result: DescribeDBEngineVersionsCommandOutput
+): void {
+ rdsMock.on(DescribeDBEngineVersionsCommand).resolves(result);
+}
+
+describe('modules/datasource/aws-rds/index', () => {
+ beforeEach(() => rdsMock.reset());
+
+ describe('getPkgReleases()', () => {
+ it('without returned versions', async () => {
+ mockDescribeVersionsCommand({
+ $metadata: {},
+ });
+ const res = await getPkgReleases({
+ datasource: AwsRdsDataSource.id,
+ depName: '[{"Name":"engine","Values":["mysql"]}]',
+ });
+ expect(res).toBeNull();
+ });
+
+ it('with one deprecated version', async () => {
+ mockDescribeVersionsCommand({
+ $metadata: {},
+ DBEngineVersions: [version2],
+ });
+ const res = await getPkgReleases({
+ datasource: AwsRdsDataSource.id,
+ depName: '[{"Name":"engine","Values":["mysql"]}]',
+ });
+ expect(res).toStrictEqual({
+ releases: [
+ {
+ isDeprecated: true,
+ version: version2.EngineVersion,
+ },
+ ],
+ });
+ });
+
+ it('with 3 matching versions', async () => {
+ mockDescribeVersionsCommand({
+ $metadata: {},
+ DBEngineVersions: [version1, version2, version3],
+ });
+ const res = await getPkgReleases({
+ datasource: AwsRdsDataSource.id,
+ depName: '[{"Name":"engine","Values":["mysql"]}]',
+ });
+ expect(res).toStrictEqual({
+ releases: [
+ {
+ isDeprecated: false,
+ version: version1.EngineVersion,
+ },
+ {
+ isDeprecated: true,
+ version: version2.EngineVersion,
+ },
+ {
+ isDeprecated: false,
+ version: version3.EngineVersion,
+ },
+ ],
+ });
+ });
+ });
+});
diff --git a/lib/modules/datasource/aws-rds/index.ts b/lib/modules/datasource/aws-rds/index.ts
new file mode 100644
index 0000000000..1ad325f5e3
--- /dev/null
+++ b/lib/modules/datasource/aws-rds/index.ts
@@ -0,0 +1,43 @@
+import {
+ DescribeDBEngineVersionsCommand,
+ RDSClient,
+} from '@aws-sdk/client-rds';
+import { cache } from '../../../util/cache/package/decorator';
+import { Lazy } from '../../../util/lazy';
+import { Datasource } from '../datasource';
+import type { GetReleasesConfig, ReleaseResult } from '../types';
+
+export class AwsRdsDataSource extends Datasource {
+ static readonly id = 'aws-rds';
+
+ override readonly caching = true;
+
+ private readonly rds: Lazy<RDSClient>;
+
+ constructor() {
+ super(AwsRdsDataSource.id);
+ this.rds = new Lazy(() => new RDSClient({}));
+ }
+
+ @cache({
+ namespace: `datasource-${AwsRdsDataSource.id}`,
+ key: ({ packageName }: GetReleasesConfig) => `getReleases:${packageName}`,
+ })
+ async getReleases({
+ packageName: serializedFilter,
+ }: GetReleasesConfig): Promise<ReleaseResult | null> {
+ const cmd = new DescribeDBEngineVersionsCommand({
+ Filters: JSON.parse(serializedFilter),
+ });
+ const response = await this.rds.getValue().send(cmd);
+ const versions = response.DBEngineVersions ?? [];
+ return {
+ releases: versions
+ .filter((version) => version.EngineVersion)
+ .map((version) => ({
+ version: version.EngineVersion!,
+ isDeprecated: version.Status === 'deprecated',
+ })),
+ };
+ }
+}
diff --git a/lib/modules/datasource/aws-rds/readme.md b/lib/modules/datasource/aws-rds/readme.md
new file mode 100644
index 0000000000..fa074ab260
--- /dev/null
+++ b/lib/modules/datasource/aws-rds/readme.md
@@ -0,0 +1,78 @@
+This datasource returns the database engine versions available for use on [AWS RDS](https://aws.amazon.com/rds/) via the AWS API.
+Generally speaking, all publicly released database versions are available for use on RDS.
+However, new versions may not be available on RDS for a few weeks or months after their release while AWS tests them.
+In addition, AWS may pull existing versions if serious problems arise during their use.
+
+**AWS API configuration**
+
+Since the datasource uses the AWS SDK for JavaScript, you can configure it like other AWS Tools.
+You can use common AWS configuration options, for example:
+
+- Set the region via the `AWS_REGION` environment variable or your `~/.aws/config` file
+- Provide credentials via the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables or your `~/.aws/credentials` file
+- Select the profile to use via `AWS_PROFILE` environment variable
+
+Read the [AWS Developer Guide - Configuring the SDK for JavaScript](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/configuring-the-jssdk.html) for more information on these configuration options.
+
+The minimal IAM privileges required for this datasource are:
+
+```json
+{
+ "Sid": "AllowDBEngineVersionLookup",
+ "Effect": "Allow",
+ "Action": ["rds:DescribeDBEngineVersions"],
+ "Resource": "*"
+}
+```
+
+Read the [AWS RDS IAM reference](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonrds.html) for more information.
+
+**Usage**
+
+Because Renovate has no manager for the AWS RDS datasource, you need to help Renovate by configuring the regex manager to identify the RDS dependencies you want updated.
+
+When configuring the regex manager, you have to pass a [filter](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-rds/interfaces/describedbengineversionscommandinput.html#filters) as minified JSON as the `packageName`.
+For example:
+
+```yaml
+# Getting the latest supported MySQL 5.7 version from RDS as a filter would look like:
+
+[
+ {
+ "Name": "engine",
+ "Values": [ "mysql" ]
+ },
+ {
+ "Name": "engine-version",
+ "Values": [ "5.7" ]
+ }
+]
+
+# In order to use it with this datasource, you have to minify it:
+
+[{"Name":"engine","Values":["mysql"]},{"Name":"engine-version","Values":["5.7"]}]
+```
+
+Here's an example of using the regex manager to configure this datasource:
+
+```json
+{
+ "regexManagers": [
+ {
+ "fileMatch": ["\\.yaml$"],
+ "matchStrings": [
+ ".*amiFilter=(?<lookupName>.+?)[ ]*\n[ ]*(?<depName>[a-zA-Z0-9-_:]*)[ ]*?:[ ]*?[\"|']?(?<currentValue>[.\\d]+)[\"|']?.*"
+ ],
+ "datasourceTemplate": "aws-rds"
+ }
+ ]
+}
+```
+
+The configuration above matches every YAML file, and recognizes these lines:
+
+```yaml
+spec:
+ # amiFilter=[{"Name":"engine","Values":["mysql"]},{"Name":"engine-version","Values":["5.7"]}]
+ engineVersion: 5.7.34
+```
diff --git a/package.json b/package.json
index f256fa6adf..f6d2accc14 100644
--- a/package.json
+++ b/package.json
@@ -138,6 +138,7 @@
"dependencies": {
"@aws-sdk/client-ec2": "3.112.0",
"@aws-sdk/client-ecr": "3.112.0",
+ "@aws-sdk/client-rds": "3.112.0",
"@aws-sdk/client-s3": "3.113.0",
"@breejs/later": "4.1.0",
"@cheap-glitch/mi-cron": "1.0.1",
diff --git a/yarn.lock b/yarn.lock
index f6512004a2..038737f046 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -226,6 +226,50 @@
"@aws-sdk/util-waiter" "3.110.0"
tslib "^2.3.1"
+"@aws-sdk/client-rds@3.112.0":
+ version "3.112.0"
+ resolved "https://registry.yarnpkg.com/@aws-sdk/client-rds/-/client-rds-3.112.0.tgz#3f9d32b20c5d7bf1d0d738d7c17f3d7e70418d91"
+ integrity sha512-A8Q+QTjaV2y5gK+3YGE6D1PLzcCswZPgQv5HjR2u6gw6WlzDu0iYTTFSLcDG8aW7TyS9r3Ugjj2kMYVJy8FIJA==
+ dependencies:
+ "@aws-crypto/sha256-browser" "2.0.0"
+ "@aws-crypto/sha256-js" "2.0.0"
+ "@aws-sdk/client-sts" "3.112.0"
+ "@aws-sdk/config-resolver" "3.110.0"
+ "@aws-sdk/credential-provider-node" "3.112.0"
+ "@aws-sdk/fetch-http-handler" "3.110.0"
+ "@aws-sdk/hash-node" "3.110.0"
+ "@aws-sdk/invalid-dependency" "3.110.0"
+ "@aws-sdk/middleware-content-length" "3.110.0"
+ "@aws-sdk/middleware-host-header" "3.110.0"
+ "@aws-sdk/middleware-logger" "3.110.0"
+ "@aws-sdk/middleware-recursion-detection" "3.110.0"
+ "@aws-sdk/middleware-retry" "3.110.0"
+ "@aws-sdk/middleware-sdk-rds" "3.110.0"
+ "@aws-sdk/middleware-serde" "3.110.0"
+ "@aws-sdk/middleware-signing" "3.110.0"
+ "@aws-sdk/middleware-stack" "3.110.0"
+ "@aws-sdk/middleware-user-agent" "3.110.0"
+ "@aws-sdk/node-config-provider" "3.110.0"
+ "@aws-sdk/node-http-handler" "3.110.0"
+ "@aws-sdk/protocol-http" "3.110.0"
+ "@aws-sdk/smithy-client" "3.110.0"
+ "@aws-sdk/types" "3.110.0"
+ "@aws-sdk/url-parser" "3.110.0"
+ "@aws-sdk/util-base64-browser" "3.109.0"
+ "@aws-sdk/util-base64-node" "3.55.0"
+ "@aws-sdk/util-body-length-browser" "3.55.0"
+ "@aws-sdk/util-body-length-node" "3.55.0"
+ "@aws-sdk/util-defaults-mode-browser" "3.110.0"
+ "@aws-sdk/util-defaults-mode-node" "3.110.0"
+ "@aws-sdk/util-user-agent-browser" "3.110.0"
+ "@aws-sdk/util-user-agent-node" "3.110.0"
+ "@aws-sdk/util-utf8-browser" "3.109.0"
+ "@aws-sdk/util-utf8-node" "3.109.0"
+ "@aws-sdk/util-waiter" "3.110.0"
+ entities "2.2.0"
+ fast-xml-parser "3.19.0"
+ tslib "^2.3.1"
+
"@aws-sdk/client-s3@3.113.0":
version "3.113.0"
resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.113.0.tgz#fc416139a1d1f5a8e07ac6f3d453042c6d84c4f9"
@@ -664,6 +708,17 @@
"@aws-sdk/util-format-url" "3.110.0"
tslib "^2.3.1"
+"@aws-sdk/middleware-sdk-rds@3.110.0":
+ version "3.110.0"
+ resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-rds/-/middleware-sdk-rds-3.110.0.tgz#ada55b2f7fee4f2291d96c6563cfa166db29f345"
+ integrity sha512-NM5fwxoWZtKhy0cMMgptp2FkmUkHlFkWNpi6m5NDJDq5AP+aCaZR0fBT6HEF5kR3DRZm1Mp+mFa4pRELjETP2Q==
+ dependencies:
+ "@aws-sdk/protocol-http" "3.110.0"
+ "@aws-sdk/signature-v4" "3.110.0"
+ "@aws-sdk/types" "3.110.0"
+ "@aws-sdk/util-format-url" "3.110.0"
+ tslib "^2.3.1"
+
"@aws-sdk/middleware-sdk-s3@3.110.0":
version "3.110.0"
resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.110.0.tgz#069603d33fbc349661facb0aaa131a95263e1b88"
--
GitLab