From bc83d69d651d964b5867f858d717c68b67c18a58 Mon Sep 17 00:00:00 2001
From: Sebastian Poxhofer <secustor@users.noreply.github.com>
Date: Fri, 30 Jul 2021 13:24:59 +0200
Subject: [PATCH] feat(terraform): implement lockfile support for Terraform
community providers (#10619)
---
.../telmate-proxmox-versions-response.json | 102 ++++++++
.../__snapshots__/index.spec.ts.snap | 221 ++++++++++++++++
.../terraform-provider/index.spec.ts | 134 +++++++++-
lib/datasource/terraform-provider/index.ts | 126 +++++++++
lib/datasource/terraform-provider/types.ts | 18 ++
.../lockfile/__snapshots__/index.spec.ts.snap | 136 +++++++++-
.../lockfile/__snapshots__/util.spec.ts.snap | 6 +-
lib/manager/terraform/lockfile/hash.spec.ts | 41 ++-
lib/manager/terraform/lockfile/hash.ts | 242 +++++++-----------
lib/manager/terraform/lockfile/index.spec.ts | 126 ++++++++-
lib/manager/terraform/lockfile/index.ts | 120 +++++----
lib/manager/terraform/lockfile/util.ts | 14 +-
12 files changed, 1072 insertions(+), 214 deletions(-)
create mode 100644 lib/datasource/terraform-provider/__fixtures__/telmate-proxmox-versions-response.json
diff --git a/lib/datasource/terraform-provider/__fixtures__/telmate-proxmox-versions-response.json b/lib/datasource/terraform-provider/__fixtures__/telmate-proxmox-versions-response.json
new file mode 100644
index 0000000000..3894a5337e
--- /dev/null
+++ b/lib/datasource/terraform-provider/__fixtures__/telmate-proxmox-versions-response.json
@@ -0,0 +1,102 @@
+{
+ "id": "Telmate/proxmox",
+ "versions": [
+ {
+ "version": "2.6.1",
+ "protocols": [
+ "5.0"
+ ],
+ "platforms": [
+ {
+ "os": "darwin",
+ "arch": "arm64"
+ },
+ {
+ "os": "linux",
+ "arch": "amd64"
+ },
+ {
+ "os": "linux",
+ "arch": "arm"
+ },
+ {
+ "os": "windows",
+ "arch": "amd64"
+ }
+ ]
+ },
+ {
+ "version": "2.6.2-pre",
+ "protocols": [
+ "5.0"
+ ],
+ "platforms": [
+ {
+ "os": "darwin",
+ "arch": "arm64"
+ },
+ {
+ "os": "linux",
+ "arch": "amd64"
+ },
+ {
+ "os": "linux",
+ "arch": "arm"
+ },
+ {
+ "os": "windows",
+ "arch": "amd64"
+ }
+ ]
+ },
+ {
+ "version": "2.6.2",
+ "protocols": [
+ "5.0"
+ ],
+ "platforms": [
+ {
+ "os": "darwin",
+ "arch": "arm64"
+ },
+ {
+ "os": "linux",
+ "arch": "amd64"
+ },
+ {
+ "os": "linux",
+ "arch": "arm"
+ },
+ {
+ "os": "windows",
+ "arch": "amd64"
+ }
+ ]
+ },
+ {
+ "version": "2.7.1",
+ "protocols": [
+ "5.0"
+ ],
+ "platforms": [
+ {
+ "os": "darwin",
+ "arch": "arm64"
+ },
+ {
+ "os": "linux",
+ "arch": "amd64"
+ },
+ {
+ "os": "linux",
+ "arch": "arm"
+ },
+ {
+ "os": "windows",
+ "arch": "amd64"
+ }
+ ]
+ }
+ ],
+ "warnings": null
+}
diff --git a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap
index 77ce27defe..dda7a2b4a9 100644
--- a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap
@@ -1,5 +1,226 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
+exports[`datasource/terraform-provider/index getBuilds processes real data 1`] = `
+Array [
+ Object {
+ "arch": "arm64",
+ "download_url": "https://downloads.example.com/proxmox",
+ "filename": "aFileName.zip",
+ "name": "Telmate/proxmox",
+ "os": "darwin",
+ "url": "https://downloads.example.com/proxmox",
+ "version": "2.6.1",
+ },
+ Object {
+ "arch": "amd64",
+ "download_url": "https://downloads.example.com/proxmox",
+ "filename": "aFileName.zip",
+ "name": "Telmate/proxmox",
+ "os": "linux",
+ "url": "https://downloads.example.com/proxmox",
+ "version": "2.6.1",
+ },
+ Object {
+ "arch": "arm",
+ "download_url": "https://downloads.example.com/proxmox",
+ "filename": "aFileName.zip",
+ "name": "Telmate/proxmox",
+ "os": "linux",
+ "url": "https://downloads.example.com/proxmox",
+ "version": "2.6.1",
+ },
+ Object {
+ "arch": "amd64",
+ "download_url": "https://downloads.example.com/proxmox",
+ "filename": "aFileName.zip",
+ "name": "Telmate/proxmox",
+ "os": "windows",
+ "url": "https://downloads.example.com/proxmox",
+ "version": "2.6.1",
+ },
+]
+`;
+
+exports[`datasource/terraform-provider/index getBuilds processes real data 2`] = `
+Array [
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/.well-known/terraform.json",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/versions",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/2.6.1/download/darwin/arm64",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/2.6.1/download/linux/amd64",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/2.6.1/download/linux/arm",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/2.6.1/download/windows/amd64",
+ },
+]
+`;
+
+exports[`datasource/terraform-provider/index getBuilds return null if the retrieval of a single build fails 1`] = `
+Array [
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/.well-known/terraform.json",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/versions",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/2.6.1/download/darwin/arm64",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/2.6.1/download/linux/amd64",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/2.6.1/download/linux/arm",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/2.6.1/download/windows/amd64",
+ },
+]
+`;
+
+exports[`datasource/terraform-provider/index getBuilds returns null for empty result 1`] = `
+Array [
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/.well-known/terraform.json",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm/versions",
+ },
+]
+`;
+
+exports[`datasource/terraform-provider/index getBuilds returns null if a version is requested which is not available 1`] = `
+Array [
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/.well-known/terraform.json",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate, br",
+ "host": "registry.terraform.io",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/Telmate/proxmox/versions",
+ },
+]
+`;
+
exports[`datasource/terraform-provider/index getReleases processes data with alternative backend 1`] = `
Object {
"registryUrl": "https://releases.hashicorp.com",
diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts
index afde1cffb3..836e57be2a 100644
--- a/lib/datasource/terraform-provider/index.spec.ts
+++ b/lib/datasource/terraform-provider/index.spec.ts
@@ -6,6 +6,9 @@ import { TerraformProviderDatasource } from '.';
const consulData: any = loadFixture('azurerm-provider.json');
const hashicorpReleases: any = loadFixture('releaseBackendIndex.json');
const serviceDiscoveryResult: any = loadFixture('service-discovery.json');
+const telmateProxmocVersions: any = loadFixture(
+ 'telmate-proxmox-versions-response.json'
+);
const terraformProviderDatasource = new TerraformProviderDatasource();
const primaryUrl = terraformProviderDatasource.defaultRegistryUrls[0];
@@ -13,10 +16,6 @@ const secondaryUrl = terraformProviderDatasource.defaultRegistryUrls[1];
describe(getName(), () => {
describe('getReleases', () => {
- beforeEach(() => {
- jest.clearAllMocks();
- });
-
it('returns null for empty result', async () => {
httpMock
.scope(primaryUrl)
@@ -151,4 +150,131 @@ describe(getName(), () => {
expect(httpMock.getTrace()).toMatchSnapshot();
});
});
+ describe('getBuilds', () => {
+ it('returns null for empty result', async () => {
+ httpMock
+ .scope(primaryUrl)
+ .get('/v1/providers/hashicorp/azurerm/versions')
+ .reply(200, {})
+ .get('/.well-known/terraform.json')
+ .reply(200, serviceDiscoveryResult);
+
+ const result = await terraformProviderDatasource.getBuilds(
+ terraformProviderDatasource.defaultRegistryUrls[0],
+ 'hashicorp/azurerm',
+ '2.50.0'
+ );
+ expect(result).toBeNull();
+ expect(httpMock.getTrace()).toMatchSnapshot();
+ });
+
+ it('returns null for non hashicorp dependency and releases.hashicorp.com registryUrl', async () => {
+ const result = await terraformProviderDatasource.getBuilds(
+ terraformProviderDatasource.defaultRegistryUrls[1],
+ 'test/azurerm',
+ '2.50.0'
+ );
+ expect(result).toBeNull();
+ });
+
+ it('returns null if a version is requested which is not available', async () => {
+ httpMock
+ .scope(primaryUrl)
+ .get('/v1/providers/Telmate/proxmox/versions')
+ .reply(200, telmateProxmocVersions)
+ .get('/.well-known/terraform.json')
+ .reply(200, serviceDiscoveryResult);
+ const result = await terraformProviderDatasource.getBuilds(
+ terraformProviderDatasource.defaultRegistryUrls[0],
+ 'Telmate/proxmox',
+ '2.8.0'
+ );
+ expect(result).toBeNull();
+ expect(httpMock.getTrace()).toMatchSnapshot();
+ });
+
+ it('processes real data', async () => {
+ httpMock
+ .scope(primaryUrl)
+ .get('/v1/providers/Telmate/proxmox/versions')
+ .reply(200, telmateProxmocVersions)
+ .get('/.well-known/terraform.json')
+ .reply(200, serviceDiscoveryResult)
+ .get('/v1/providers/Telmate/proxmox/2.6.1/download/darwin/arm64')
+ .reply(200, {
+ os: 'darwin',
+ arch: 'arm64',
+ filename: 'aFileName.zip',
+ download_url: 'https://downloads.example.com/proxmox',
+ })
+ .get('/v1/providers/Telmate/proxmox/2.6.1/download/linux/amd64')
+ .reply(200, {
+ os: 'linux',
+ arch: 'amd64',
+ filename: 'aFileName.zip',
+ download_url: 'https://downloads.example.com/proxmox',
+ })
+ .get('/v1/providers/Telmate/proxmox/2.6.1/download/linux/arm')
+ .reply(200, {
+ os: 'linux',
+ arch: 'arm',
+ filename: 'aFileName.zip',
+ download_url: 'https://downloads.example.com/proxmox',
+ })
+ .get('/v1/providers/Telmate/proxmox/2.6.1/download/windows/amd64')
+ .reply(200, {
+ os: 'windows',
+ arch: 'amd64',
+ filename: 'aFileName.zip',
+ download_url: 'https://downloads.example.com/proxmox',
+ });
+ const res = await terraformProviderDatasource.getBuilds(
+ terraformProviderDatasource.defaultRegistryUrls[0],
+ 'Telmate/proxmox',
+ '2.6.1'
+ );
+ expect(res).toMatchSnapshot();
+ expect(res).not.toBeNull();
+ expect(httpMock.getTrace()).toMatchSnapshot();
+ });
+
+ it('return null if the retrieval of a single build fails', async () => {
+ httpMock
+ .scope(primaryUrl)
+ .get('/v1/providers/Telmate/proxmox/versions')
+ .reply(200, telmateProxmocVersions)
+ .get('/.well-known/terraform.json')
+ .reply(200, serviceDiscoveryResult)
+ .get('/v1/providers/Telmate/proxmox/2.6.1/download/darwin/arm64')
+ .reply(200, {
+ os: 'darwin',
+ arch: 'arm64',
+ filename: 'aFileName.zip',
+ download_url: 'https://downloads.example.com/proxmox',
+ })
+ .get('/v1/providers/Telmate/proxmox/2.6.1/download/linux/amd64')
+ .reply(200, {
+ os: 'linux',
+ arch: 'amd64',
+ filename: 'aFileName.zip',
+ download_url: 'https://downloads.example.com/proxmox',
+ })
+ .get('/v1/providers/Telmate/proxmox/2.6.1/download/linux/arm')
+ .reply(200, {
+ os: 'linux',
+ arch: 'arm',
+ filename: 'aFileName.zip',
+ download_url: 'https://downloads.example.com/proxmox',
+ })
+ .get('/v1/providers/Telmate/proxmox/2.6.1/download/windows/amd64')
+ .reply(404);
+ const res = await terraformProviderDatasource.getBuilds(
+ terraformProviderDatasource.defaultRegistryUrls[0],
+ 'Telmate/proxmox',
+ '2.6.1'
+ );
+ expect(res).toBeNull();
+ expect(httpMock.getTrace()).toMatchSnapshot();
+ });
+ });
});
diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts
index e03b488bec..4bd19dbd0a 100644
--- a/lib/datasource/terraform-provider/index.ts
+++ b/lib/datasource/terraform-provider/index.ts
@@ -1,12 +1,18 @@
+import pMap from 'p-map';
import { logger } from '../../logger';
+import { ExternalHostError } from '../../types/errors/external-host-error';
import { cache } from '../../util/cache/package/decorator';
import { parseUrl } from '../../util/url';
import * as hashicorpVersioning from '../../versioning/hashicorp';
import { TerraformDatasource } from '../terraform-module/base';
import type { GetReleasesConfig, ReleaseResult } from '../types';
import type {
+ TerraformBuild,
TerraformProvider,
TerraformProviderReleaseBackend,
+ TerraformRegistryBuildResponse,
+ TerraformRegistryVersions,
+ VersionDetailResponse,
} from './types';
export class TerraformProviderDatasource extends TerraformDatasource {
@@ -17,6 +23,8 @@ export class TerraformProviderDatasource extends TerraformDatasource {
'https://releases.hashicorp.com',
];
+ static repositoryRegex = /^hashicorp\/(?<lookupName>\S+)$/;
+
constructor() {
super(TerraformProviderDatasource.id);
}
@@ -116,4 +124,122 @@ export class TerraformProviderDatasource extends TerraformDatasource {
logger.trace({ dep }, 'dep');
return dep;
}
+
+ @cache({
+ namespace: `datasource-${TerraformProviderDatasource.id}-builds`,
+ key: (registryURL: string, repository: string, version: string) =>
+ `${registryURL}/${repository}/${version}`,
+ })
+ async getBuilds(
+ registryURL: string,
+ repository: string,
+ version: string
+ ): Promise<TerraformBuild[]> {
+ if (registryURL === TerraformProviderDatasource.defaultRegistryUrls[1]) {
+ // check if registryURL === secondary backend
+ const repositoryRegexResult =
+ TerraformProviderDatasource.repositoryRegex.exec(repository);
+ if (!repositoryRegexResult) {
+ // non hashicorp builds are not supported with releases.hashicorp.com
+ return null;
+ }
+ const lookupName = repositoryRegexResult.groups.lookupName;
+ const backendLookUpName = `terraform-provider-${lookupName}`;
+ let versionReleaseBackend: VersionDetailResponse;
+ try {
+ versionReleaseBackend = await this.getReleaseBackendIndex(
+ backendLookUpName,
+ version
+ );
+ } catch (err) {
+ /* istanbul ignore next */
+ if (err instanceof ExternalHostError) {
+ throw err;
+ }
+ logger.debug(
+ { err, backendLookUpName, version },
+ `Failed to retrieve builds for ${backendLookUpName} ${version}`
+ );
+ return null;
+ }
+ return versionReleaseBackend.builds;
+ }
+
+ // check public or private Terraform registry
+ const serviceDiscovery = await this.getTerraformServiceDiscoveryResult(
+ registryURL
+ );
+ if (!serviceDiscovery) {
+ logger.trace(`Failed to retrieve service discovery from ${registryURL}`);
+ return null;
+ }
+ const backendURL = `${registryURL}${serviceDiscovery['providers.v1']}${repository}`;
+ const versionsResponse = (
+ await this.http.getJson<TerraformRegistryVersions>(
+ `${backendURL}/versions`
+ )
+ ).body;
+ if (!versionsResponse.versions) {
+ logger.trace(`Failed to retrieve version list for ${backendURL}`);
+ return null;
+ }
+ const builds = versionsResponse.versions.find(
+ (value) => value.version === version
+ );
+ if (!builds) {
+ logger.trace(
+ `No builds found for ${repository}:${version} on ${registryURL}`
+ );
+ return null;
+ }
+ const result = await pMap(
+ builds.platforms,
+ async (platform) => {
+ const buildURL = `${backendURL}/${version}/download/${platform.os}/${platform.arch}`;
+ try {
+ const res = (
+ await this.http.getJson<TerraformRegistryBuildResponse>(buildURL)
+ ).body;
+ const newBuild: TerraformBuild = {
+ name: repository,
+ url: res.download_url,
+ version,
+ ...res,
+ };
+ return newBuild;
+ } catch (err) {
+ /* istanbul ignore next */
+ if (err instanceof ExternalHostError) {
+ throw err;
+ }
+ logger.debug({ err, url: buildURL }, 'Failed to retrieve build');
+ return null;
+ }
+ },
+ { concurrency: 4 }
+ );
+
+ // if any of the requests to build details have failed, return null
+ if (result.some((value) => Boolean(value) === false)) {
+ return null;
+ }
+
+ return result;
+ }
+
+ @cache({
+ namespace: `datasource-${TerraformProviderDatasource.id}-releaseBackendIndex`,
+ key: (backendLookUpName: string, version: string) =>
+ `${backendLookUpName}/${version}`,
+ })
+ async getReleaseBackendIndex(
+ backendLookUpName: string,
+ version: string
+ ): Promise<VersionDetailResponse> {
+ return (
+ await this.http.getJson<VersionDetailResponse>(
+ `${TerraformProviderDatasource.defaultRegistryUrls[1]}/${backendLookUpName}/${version}/index.json`
+ )
+ ).body;
+ }
}
diff --git a/lib/datasource/terraform-provider/types.ts b/lib/datasource/terraform-provider/types.ts
index e2a739c48d..63c90097e0 100644
--- a/lib/datasource/terraform-provider/types.ts
+++ b/lib/datasource/terraform-provider/types.ts
@@ -32,3 +32,21 @@ export type TerraformProviderReleaseBackend = Record<
>;
export type VersionsReleaseBackend = Record<string, VersionDetailResponse>;
+
+export interface TerraformRegistryVersions {
+ id: string;
+ versions: {
+ version: string;
+ platforms: {
+ os: string;
+ arch: string;
+ }[];
+ }[];
+}
+
+export interface TerraformRegistryBuildResponse {
+ os: string;
+ arch: string;
+ filename: string;
+ download_url: string;
+}
diff --git a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap
index 5061dc2f11..f6af2a1e5c 100644
--- a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap
+++ b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap
@@ -48,10 +48,72 @@ provider \\"registry.terraform.io/hashicorp/random\\" {
exports[`manager/terraform/lockfile/index do full lock file maintenance 2`] = `
Array [
Array [
+ "https://registry.terraform.io",
"hashicorp/azurerm",
"2.56.0",
],
Array [
+ "https://registry.terraform.io",
+ "hashicorp/random",
+ "2.2.2",
+ ],
+]
+`;
+
+exports[`manager/terraform/lockfile/index do full lock file maintenance with lockfile in subfolder 1`] = `
+Object {
+ "contents": "# This file is maintained automatically by \\"terraform init\\".
+# Manual edits may be lost in future updates.
+
+provider \\"registry.terraform.io/hashicorp/aws\\" {
+ version = \\"3.0.0\\"
+ constraints = \\"3.0.0\\"
+ hashes = [
+ \\"h1:ULKfwySvQ4pDhy027ryRhLxDhg640wsojYc+7NHMFBU=\\",
+ \\"zh:25294510ae9c250502f2e37ac32b01017439735f098f82a1728772427626a2fd\\",
+ \\"zh:3b723e7772d47bd8cc11bea6e5d3e0b5e1df8398c0e7aaf510e3a8a54e0f1874\\",
+ \\"zh:4b7b73b86f4a0705d5d2a7f1d3ad3279706bdb3957a48f4a389c36918fba838e\\",
+ \\"zh:9e26cdc3be97e3001c253c0ca28c5c8ff2d5476373ca1beb849f3f3957ce7f1a\\",
+ \\"zh:9e73cf1304bf57968d3048d70c0b766d41497430a2a9a7a718a196f3a385106a\\",
+ \\"zh:a30b5b66facfbb2b02814e4cd33ca9899f9ade5bbf478f78c41d2fe789f0582a\\",
+ \\"zh:b06fb5da094db41cb5e430c95c988b73f32695e9f90f25499e926842dbd21b21\\",
+ \\"zh:c5a4ff607e9e9edee3fcd6d6666241fb532adf88ea1fe24f2aa1eb36845b3ca3\\",
+ \\"zh:df568a69087831c1780fac4395630a2cfb3cdf67b7dffbfe16bd78c64770bb75\\",
+ \\"zh:fce1b69dd673aace19508640b0b9b7eb1ef7e746d76cb846b49e7d52e0f5fb7e\\",
+ ]
+}
+
+provider \\"registry.terraform.io/hashicorp/azurerm\\" {
+ version = \\"2.56.0\\"
+ constraints = \\"~> 2.50\\"
+ hashes = [
+ \\"h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=\\",
+ \\"h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=\\",
+ ]
+}
+
+provider \\"registry.terraform.io/hashicorp/random\\" {
+ version = \\"2.2.2\\"
+ constraints = \\"~> 2.2\\"
+ hashes = [
+ \\"h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=\\",
+ \\"h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=\\",
+ ]
+}
+",
+ "name": "subfolder/.terraform.lock.hcl",
+}
+`;
+
+exports[`manager/terraform/lockfile/index do full lock file maintenance with lockfile in subfolder 2`] = `
+Array [
+ Array [
+ "https://registry.terraform.io",
+ "hashicorp/azurerm",
+ "2.56.0",
+ ],
+ Array [
+ "https://registry.terraform.io",
"hashicorp/random",
"2.2.2",
],
@@ -63,16 +125,83 @@ exports[`manager/terraform/lockfile/index do full lock file maintenance without
exports[`manager/terraform/lockfile/index return null if hashing fails 1`] = `
Array [
Array [
+ "https://registry.terraform.io",
"hashicorp/azurerm",
"2.56.0",
],
Array [
+ "https://registry.terraform.io",
"hashicorp/random",
"2.2.2",
],
]
`;
+exports[`manager/terraform/lockfile/index update single dependency in subfolder 1`] = `
+Object {
+ "contents": "# This file is maintained automatically by \\"terraform init\\".
+# Manual edits may be lost in future updates.
+
+provider \\"registry.terraform.io/hashicorp/aws\\" {
+ version = \\"3.0.0\\"
+ constraints = \\"3.0.0\\"
+ hashes = [
+ \\"h1:ULKfwySvQ4pDhy027ryRhLxDhg640wsojYc+7NHMFBU=\\",
+ \\"zh:25294510ae9c250502f2e37ac32b01017439735f098f82a1728772427626a2fd\\",
+ \\"zh:3b723e7772d47bd8cc11bea6e5d3e0b5e1df8398c0e7aaf510e3a8a54e0f1874\\",
+ \\"zh:4b7b73b86f4a0705d5d2a7f1d3ad3279706bdb3957a48f4a389c36918fba838e\\",
+ \\"zh:9e26cdc3be97e3001c253c0ca28c5c8ff2d5476373ca1beb849f3f3957ce7f1a\\",
+ \\"zh:9e73cf1304bf57968d3048d70c0b766d41497430a2a9a7a718a196f3a385106a\\",
+ \\"zh:a30b5b66facfbb2b02814e4cd33ca9899f9ade5bbf478f78c41d2fe789f0582a\\",
+ \\"zh:b06fb5da094db41cb5e430c95c988b73f32695e9f90f25499e926842dbd21b21\\",
+ \\"zh:c5a4ff607e9e9edee3fcd6d6666241fb532adf88ea1fe24f2aa1eb36845b3ca3\\",
+ \\"zh:df568a69087831c1780fac4395630a2cfb3cdf67b7dffbfe16bd78c64770bb75\\",
+ \\"zh:fce1b69dd673aace19508640b0b9b7eb1ef7e746d76cb846b49e7d52e0f5fb7e\\",
+ ]
+}
+
+provider \\"registry.terraform.io/hashicorp/azurerm\\" {
+ version = \\"2.50.0\\"
+ constraints = \\"~> 2.50\\"
+ hashes = [
+ \\"h1:Vr6WUm88s9hXGkyVjHtHsP2Jmc2ypQXn6ww7dXtvk1M=\\",
+ \\"zh:0c0688d5a743248f8646d39eb3645a4ac19fd7523ba1b47072fa3fb03b92b1b0\\",
+ \\"zh:2beb3a55ee970f87a9292ae96d57134be8a03d0566117e7be0fe0d9c1267e4ea\\",
+ \\"zh:38091b463fbafe5756420ce34c87845c2a391fec0cded27bdcbbca28febad382\\",
+ \\"zh:4ba455da3b37ba8f8b03ff2781121d9c54d0bd8afd76dfe67593011c475dd73f\\",
+ \\"zh:5d32b9ed871b3c3b774dc69f1fe14cdf7c1fd63d12bb5f21aad4bfbf75e5ee3d\\",
+ \\"zh:6c80cf90a3fc1e17d9caf67cc558c2ff91f8b25e29fdf00942f67711895be5c0\\",
+ \\"zh:c0a53e3165407999d10de7aaa983485d42797433c60b5775791ae299121279ed\\",
+ \\"zh:dab51d6d76041505aeebf20111febe8616ec465ca31dfb7901f5f5c23a5af095\\",
+ \\"zh:e1ad6399f6a6d799002206ee4cb7b794dbb2533b8c3c14502a4419955ec96bff\\",
+ \\"zh:e98f1d178d1e111b3f3449e27d305ce263071226fad3d86272e1bd161c26fd43\\",
+ \\"zh:eb76ec000c9c49a0bf730370c8880f671597bc01f7b7401ab301df7124c049ec\\",
+ ]
+}
+
+provider \\"registry.terraform.io/hashicorp/random\\" {
+ version = \\"3.1.0\\"
+ constraints = \\"~> 3.0\\"
+ hashes = [
+ \\"h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=\\",
+ \\"h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=\\",
+ ]
+}
+",
+ "name": "test/.terraform.lock.hcl",
+}
+`;
+
+exports[`manager/terraform/lockfile/index update single dependency in subfolder 2`] = `
+Array [
+ Array [
+ "https://registry.terraform.io",
+ "hashicorp/random",
+ "3.1.0",
+ ],
+]
+`;
+
exports[`manager/terraform/lockfile/index update single dependency with exact constraint 1`] = `
Object {
"contents": "# This file is maintained automatically by \\"terraform init\\".
@@ -133,6 +262,7 @@ provider \\"registry.terraform.io/hashicorp/random\\" {
exports[`manager/terraform/lockfile/index update single dependency with exact constraint 2`] = `
Array [
Array [
+ "https://registry.terraform.io",
"hashicorp/aws",
"3.36.0",
],
@@ -197,13 +327,14 @@ provider \\"registry.terraform.io/hashicorp/random\\" {
exports[`manager/terraform/lockfile/index update single dependency with range constraint and major update 2`] = `
Array [
Array [
+ "https://registry.terraform.io",
"hashicorp/random",
"3.1.0",
],
]
`;
-exports[`manager/terraform/lockfile/index update single dependency with range constraint and minor update 1`] = `
+exports[`manager/terraform/lockfile/index update single dependency with range constraint and minor update from private registry 1`] = `
Object {
"contents": "# This file is maintained automatically by \\"terraform init\\".
# Manual edits may be lost in future updates.
@@ -259,9 +390,10 @@ provider \\"registry.terraform.io/hashicorp/random\\" {
}
`;
-exports[`manager/terraform/lockfile/index update single dependency with range constraint and minor update 2`] = `
+exports[`manager/terraform/lockfile/index update single dependency with range constraint and minor update from private registry 2`] = `
Array [
Array [
+ "https://registry.example.com",
"hashicorp/azurerm",
"2.56.0",
],
diff --git a/lib/manager/terraform/lockfile/__snapshots__/util.spec.ts.snap b/lib/manager/terraform/lockfile/__snapshots__/util.spec.ts.snap
index c17268a2dc..419a1311fd 100644
--- a/lib/manager/terraform/lockfile/__snapshots__/util.spec.ts.snap
+++ b/lib/manager/terraform/lockfile/__snapshots__/util.spec.ts.snap
@@ -30,7 +30,7 @@ Array [
"version": 1,
},
"lookupName": "hashicorp/aws",
- "registryUrl": "registry.terraform.io",
+ "registryUrl": "https://registry.terraform.io",
"version": "3.0.0",
},
Object {
@@ -62,7 +62,7 @@ Array [
"version": 1,
},
"lookupName": "hashicorp/azurerm",
- "registryUrl": "registry.terraform.io",
+ "registryUrl": "https://registry.terraform.io",
"version": "2.50.0",
},
Object {
@@ -95,7 +95,7 @@ Array [
"version": 1,
},
"lookupName": "hashicorp/random",
- "registryUrl": "registry.terraform.io",
+ "registryUrl": "https://registry.terraform.io",
"version": "2.2.1",
},
]
diff --git a/lib/manager/terraform/lockfile/hash.spec.ts b/lib/manager/terraform/lockfile/hash.spec.ts
index 45a1b5f635..a7ef4e7360 100644
--- a/lib/manager/terraform/lockfile/hash.spec.ts
+++ b/lib/manager/terraform/lockfile/hash.spec.ts
@@ -10,7 +10,7 @@ import {
import { setAdminConfig } from '../../../config/admin';
import { TerraformProviderDatasource } from '../../../datasource/terraform-provider';
import { Logger } from '../../../logger/types';
-import { createHashes } from './hash';
+import { TerraformProviderHash } from './hash';
const releaseBackendUrl = TerraformProviderDatasource.defaultRegistryUrls[1];
const releaseBackendAzurerm = loadFixture('releaseBackendAzurerm_2_56_0.json');
@@ -29,8 +29,16 @@ describe(getName(), () => {
afterAll(() => cacheDir.cleanup());
- it('returns null if a non hashicorp release is found ', async () => {
- const result = await createHashes('test/gitlab', '2.56.0');
+ it('returns null if getBuilds returns null', async () => {
+ httpMock
+ .scope('https://example.com')
+ .get('/.well-known/terraform.json')
+ .reply(200, '');
+ const result = await TerraformProviderHash.createHashes(
+ 'https://example.com',
+ 'test/gitlab',
+ '2.56.0'
+ );
expect(result).toBeNull();
});
@@ -40,7 +48,11 @@ describe(getName(), () => {
.get('/terraform-provider-azurerm/2.59.0/index.json')
.reply(403, '');
- const result = await createHashes('hashicorp/azurerm', '2.59.0');
+ const result = await TerraformProviderHash.createHashes(
+ 'https://releases.hashicorp.com',
+ 'hashicorp/azurerm',
+ '2.59.0'
+ );
expect(result).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
@@ -51,7 +63,11 @@ describe(getName(), () => {
.get('/terraform-provider-azurerm/2.56.0/index.json')
.replyWithError('');
- const result = await createHashes('hashicorp/azurerm', '2.56.0');
+ const result = await TerraformProviderHash.createHashes(
+ 'https://releases.hashicorp.com',
+ 'hashicorp/azurerm',
+ '2.56.0'
+ );
expect(result).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
@@ -76,8 +92,13 @@ describe(getName(), () => {
)
.reply(200, readStreamDarwin);
- const result = await createHashes('hashicorp/azurerm', '2.56.0');
- expect(result).toBeNull();
+ await expect(
+ TerraformProviderHash.createHashes(
+ 'https://releases.hashicorp.com',
+ 'hashicorp/azurerm',
+ '2.56.0'
+ )
+ ).rejects.toThrow();
expect(httpMock.getTrace()).toMatchSnapshot();
});
@@ -101,7 +122,11 @@ describe(getName(), () => {
)
.reply(200, readStreamDarwin);
- const result = await createHashes('hashicorp/azurerm', '2.56.0');
+ const result = await TerraformProviderHash.createHashes(
+ 'https://releases.hashicorp.com',
+ 'hashicorp/azurerm',
+ '2.56.0'
+ );
expect(log.error.mock.calls).toMatchSnapshot();
expect(result).not.toBeNull();
expect(result).toBeArrayOfSize(2);
diff --git a/lib/manager/terraform/lockfile/hash.ts b/lib/manager/terraform/lockfile/hash.ts
index 71e4101163..fe39aaedaf 100644
--- a/lib/manager/terraform/lockfile/hash.ts
+++ b/lib/manager/terraform/lockfile/hash.ts
@@ -3,168 +3,124 @@ import extract from 'extract-zip';
import pMap from 'p-map';
import { join } from 'upath';
import { TerraformProviderDatasource } from '../../../datasource/terraform-provider';
-import type {
- TerraformBuild,
- VersionDetailResponse,
-} from '../../../datasource/terraform-provider/types';
+import type { TerraformBuild } from '../../../datasource/terraform-provider/types';
import { logger } from '../../../logger';
-import * as packageCache from '../../../util/cache/package';
+import { cache } from '../../../util/cache/package/decorator';
import * as fs from '../../../util/fs';
import { ensureCacheDir } from '../../../util/fs';
import { Http } from '../../../util/http';
-import { repositoryRegex } from './util';
-
-const http = new Http(TerraformProviderDatasource.id);
-const hashCacheTTL = 10080; // in seconds == 1 week
-
-export async function hashFiles(files: string[]): Promise<string> {
- const rootHash = crypto.createHash('sha256');
-
- for (const file of files) {
- // build for every file a line looking like "aaaaaaaaaaaaaaa file.txt\n"
- const hash = crypto.createHash('sha256');
-
- // a sha256sum displayed as lowercase hex string to root hash
- const fileBuffer = await fs.readFile(file);
- hash.update(fileBuffer);
- hash.end();
- const data = hash.read();
- rootHash.update(data.toString('hex'));
-
- // add double space, the filename and a new line char
- rootHash.update(' ');
- const fileName = file.replace(/^.*[\\/]/, '');
- rootHash.update(fileName);
- rootHash.update('\n');
+
+export class TerraformProviderHash {
+ static http = new Http(TerraformProviderDatasource.id);
+
+ static terraformDatasource = new TerraformProviderDatasource();
+
+ static hashCacheTTL = 10080; // in minutes == 1 week
+
+ private static async hashFiles(files: string[]): Promise<string> {
+ const rootHash = crypto.createHash('sha256');
+
+ for (const file of files) {
+ // build for every file a line looking like "aaaaaaaaaaaaaaa file.txt\n"
+ const hash = crypto.createHash('sha256');
+
+ // a sha256sum displayed as lowercase hex string to root hash
+ const fileBuffer = await fs.readFile(file);
+ hash.update(fileBuffer);
+ hash.end();
+ const data = hash.read();
+ rootHash.update(data.toString('hex'));
+
+ // add double space, the filename and a new line char
+ rootHash.update(' ');
+ const fileName = file.replace(/^.*[\\/]/, '');
+ rootHash.update(fileName);
+ rootHash.update('\n');
+ }
+
+ rootHash.end();
+ const rootData = rootHash.read();
+ const result: string = rootData.toString('base64');
+ return result;
}
- rootHash.end();
- const rootData = rootHash.read();
- const result: string = rootData.toString('base64');
- return result;
-}
+ static async hashOfZipContent(
+ zipFilePath: string,
+ extractPath: string
+ ): Promise<string> {
+ await extract(zipFilePath, { dir: extractPath });
+ const files = await fs.readdir(extractPath);
+ // the h1 hashing algorithms requires that the files are sorted by filename
+ const sortedFiles = files.sort((a, b) => a.localeCompare(b));
+ const filesWithPath = sortedFiles.map((file) => `${extractPath}/${file}`);
-export async function hashOfZipContent(
- zipFilePath: string,
- extractPath: string
-): Promise<string> {
- await extract(zipFilePath, { dir: extractPath });
- const files = await fs.readdir(extractPath);
- // the h1 hashing algorithms requires that the files are sorted by filename
- const sortedFiles = files.sort((a, b) => a.localeCompare(b));
- const filesWithPath = sortedFiles.map((file) => `${extractPath}/${file}`);
+ const result = await TerraformProviderHash.hashFiles(filesWithPath);
- const result = await hashFiles(filesWithPath);
+ // delete extracted files
+ await fs.rm(extractPath, { recursive: true });
- // delete extracted files
- await fs.rm(extractPath, { recursive: true });
+ return result;
+ }
- return result;
-}
+ @cache({
+ namespace: `datasource-${TerraformProviderDatasource.id}-build-hashes`,
+ key: (build: TerraformBuild) => build.url,
+ ttlMinutes: TerraformProviderHash.hashCacheTTL,
+ })
+ static async calculateSingleHash(
+ build: TerraformBuild,
+ cacheDir: string
+ ): Promise<string> {
+ const downloadFileName = join(cacheDir, build.filename);
+ const extractPath = join(cacheDir, 'extract', build.filename);
+ logger.trace(
+ `Downloading archive and generating hash for ${build.name}-${build.version}...`
+ );
+ const readStream = TerraformProviderHash.http.stream(build.url);
+ const writeStream = fs.createWriteStream(downloadFileName);
-async function getReleaseBackendIndex(
- backendLookUpName: string,
- version: string
-): Promise<VersionDetailResponse> {
- return (
- await http.getJson<VersionDetailResponse>(
- `${TerraformProviderDatasource.defaultRegistryUrls[1]}/${backendLookUpName}/${version}/index.json`
- )
- ).body;
-}
+ try {
+ await fs.pipeline(readStream, writeStream);
-export async function calculateHashes(
- builds: TerraformBuild[]
-): Promise<string[]> {
- const cacheDir = await ensureCacheDir('./others/terraform');
-
- // for each build download ZIP, extract content and generate hash for all containing files
- const hashes = await pMap(
- builds,
- async (build) => {
- const downloadFileName = join(cacheDir, build.filename);
- const extractPath = join(cacheDir, 'extract', build.filename);
+ const hash = await this.hashOfZipContent(downloadFileName, extractPath);
logger.trace(
- `Downloading archive and generating hash for ${build.name}-${build.version}...`
+ { hash },
+ `Generated hash for ${build.name}-${build.version}`
);
- const readStream = http.stream(build.url);
- const writeStream = fs.createWriteStream(downloadFileName);
-
- let hash = null;
- try {
- await fs.pipeline(readStream, writeStream);
-
- hash = await hashOfZipContent(downloadFileName, extractPath);
- logger.trace(
- { hash },
- `Generated hash for ${build.name}-${build.version}`
- );
- } catch (err) {
- /* istanbul ignore next */
- logger.error({ err, build }, 'write stream error');
- } finally {
- // delete zip file
- await fs.unlink(downloadFileName);
- }
return hash;
- },
- { concurrency: 4 } // allow to look up 4 builds for this version in parallel
- );
- return hashes;
-}
-
-export async function createHashes(
- repository: string,
- version: string
-): Promise<string[]> {
- // check cache for hashes
- const repositoryRegexResult = repositoryRegex.exec(repository);
- if (!repositoryRegexResult) {
- // non hashicorp builds are not supported at the moment
- return null;
+ } finally {
+ // delete zip file
+ await fs.unlink(downloadFileName);
+ }
}
- const lookupName = repositoryRegexResult.groups.lookupName;
- const backendLookUpName = `terraform-provider-${lookupName}`;
-
- const cacheKey = `${TerraformProviderDatasource.defaultRegistryUrls[1]}/${repository}/${lookupName}-${version}`;
- const cachedRelease = await packageCache.get<string[]>(
- 'terraform-provider-release',
- cacheKey
- );
- // istanbul ignore if
- if (cachedRelease) {
- return cachedRelease;
- }
- let versionReleaseBackend: VersionDetailResponse;
- try {
- versionReleaseBackend = await getReleaseBackendIndex(
- backendLookUpName,
- version
- );
- } catch (err) {
- logger.debug(
- { err, backendLookUpName, version },
- `Failed to retrieve builds for ${backendLookUpName} ${version}`
+
+ static async calculateHashes(builds: TerraformBuild[]): Promise<string[]> {
+ const cacheDir = await ensureCacheDir('./others/terraform');
+
+ // for each build download ZIP, extract content and generate hash for all containing files
+ return pMap(
+ builds,
+ (build) => this.calculateSingleHash(build, cacheDir),
+ { concurrency: 4 } // allow to look up 4 builds for this version in parallel
);
- return null;
}
- const builds = versionReleaseBackend.builds;
- const hashes = await calculateHashes(builds);
+ static async createHashes(
+ registryURL: string,
+ repository: string,
+ version: string
+ ): Promise<string[]> {
+ const builds = await TerraformProviderHash.terraformDatasource.getBuilds(
+ registryURL,
+ repository,
+ version
+ );
+ if (!builds) {
+ return null;
+ }
+ const hashes = await TerraformProviderHash.calculateHashes(builds);
- // if a hash could not be produced skip caching and return null
- if (hashes.some((value) => value == null)) {
- return null;
+ // sorting the hash alphabetically as terraform does this as well
+ return hashes.sort().map((hash) => `h1:${hash}`);
}
-
- // sorting the hash alphabetically as terraform does this as well
- const sortedHashes = hashes.sort().map((hash) => `h1:${hash}`);
- // save to cache
- await packageCache.set(
- 'terraform-provider-release',
- cacheKey,
- sortedHashes,
- hashCacheTTL
- );
- return sortedHashes;
}
diff --git a/lib/manager/terraform/lockfile/index.spec.ts b/lib/manager/terraform/lockfile/index.spec.ts
index aff2f7710b..919ee97815 100644
--- a/lib/manager/terraform/lockfile/index.spec.ts
+++ b/lib/manager/terraform/lockfile/index.spec.ts
@@ -3,7 +3,7 @@ import { fs, getName, loadFixture, mocked } from '../../../../test/util';
import { setAdminConfig } from '../../../config/admin';
import { getPkgReleases } from '../../../datasource';
import type { UpdateArtifactsConfig } from '../../types';
-import * as hash from './hash';
+import { TerraformProviderHash } from './hash';
import { updateArtifacts } from './index';
// auto-mock fs
@@ -23,7 +23,7 @@ const adminConfig = {
const validLockfile = loadFixture('validLockfile.hcl');
-const mockHash = mocked(hash).createHashes;
+const mockHash = mocked(TerraformProviderHash).createHashes;
const mockGetPkgReleases = getPkgReleases as jest.MockedFunction<
typeof getPkgReleases
>;
@@ -71,6 +71,7 @@ describe(getName(), () => {
it('update single dependency with exact constraint', async () => {
fs.readLocalFile.mockResolvedValueOnce(validLockfile as any);
+ fs.getSiblingFileName.mockReturnValueOnce('.terraform.lock.hcl');
mockHash.mockResolvedValueOnce([
'h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=',
@@ -101,8 +102,9 @@ describe(getName(), () => {
expect(mockHash.mock.calls).toMatchSnapshot();
});
- it('update single dependency with range constraint and minor update', async () => {
+ it('update single dependency with range constraint and minor update from private registry', async () => {
fs.readLocalFile.mockResolvedValueOnce(validLockfile as any);
+ fs.getSiblingFileName.mockReturnValueOnce('.terraform.lock.hcl');
mockHash.mockResolvedValueOnce([
'h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=',
@@ -120,7 +122,13 @@ describe(getName(), () => {
const result = await updateArtifacts({
packageFileName: 'main.tf',
- updatedDeps: [{ depName: 'azurerm', lookupName: 'azurerm' }],
+ updatedDeps: [
+ {
+ depName: 'azurerm',
+ lookupName: 'azurerm',
+ registryUrls: ['https://registry.example.com'],
+ },
+ ],
newPackageFileContent: '',
config: localConfig,
});
@@ -135,6 +143,7 @@ describe(getName(), () => {
it('update single dependency with range constraint and major update', async () => {
fs.readLocalFile.mockResolvedValueOnce(validLockfile as any);
+ fs.getSiblingFileName.mockReturnValueOnce('.terraform.lock.hcl');
mockHash.mockResolvedValueOnce([
'h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=',
@@ -165,8 +174,117 @@ describe(getName(), () => {
expect(mockHash.mock.calls).toMatchSnapshot();
});
+ it('update single dependency in subfolder', async () => {
+ fs.readLocalFile.mockResolvedValueOnce(validLockfile as any);
+ fs.getSiblingFileName.mockReturnValueOnce('test/.terraform.lock.hcl');
+
+ mockHash.mockResolvedValueOnce([
+ 'h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=',
+ 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=',
+ ]);
+
+ const localConfig: UpdateArtifactsConfig = {
+ updateType: 'major',
+ newVersion: '3.1.0',
+ newValue: '~> 3.0',
+ ...config,
+ };
+
+ process.env.RENOVATE_X_TERRAFORM_LOCK_FILE = 'test';
+
+ const result = await updateArtifacts({
+ packageFileName: 'test/main.tf',
+ updatedDeps: [{ depName: 'random', lookupName: 'hashicorp/random' }],
+ newPackageFileContent: '',
+ config: localConfig,
+ });
+ expect(result).not.toBeNull();
+ expect(result).toBeArrayOfSize(1);
+ expect(result[0].file).not.toBeNull();
+ expect(result[0].file).toMatchSnapshot();
+
+ expect(mockHash.mock.calls).toBeArrayOfSize(1);
+ expect(mockHash.mock.calls).toMatchSnapshot();
+ });
+
it('do full lock file maintenance', async () => {
fs.readLocalFile.mockResolvedValueOnce(validLockfile as any);
+ fs.getSiblingFileName.mockReturnValueOnce('.terraform.lock.hcl');
+
+ mockGetPkgReleases
+ .mockResolvedValueOnce({
+ // aws
+ releases: [
+ {
+ version: '2.30.0',
+ },
+ {
+ version: '3.0.0',
+ },
+ {
+ version: '3.36.0',
+ },
+ ],
+ })
+ .mockResolvedValueOnce({
+ // azurerm
+ releases: [
+ {
+ version: '2.50.0',
+ },
+ {
+ version: '2.55.0',
+ },
+ {
+ version: '2.56.0',
+ },
+ ],
+ })
+ .mockResolvedValueOnce({
+ // random
+ releases: [
+ {
+ version: '2.2.1',
+ },
+ {
+ version: '2.2.2',
+ },
+ {
+ version: '3.0.0',
+ },
+ ],
+ });
+ mockHash.mockResolvedValue([
+ 'h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=',
+ 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=',
+ ]);
+
+ const localConfig: UpdateArtifactsConfig = {
+ updateType: 'lockFileMaintenance',
+ ...config,
+ };
+
+ process.env.RENOVATE_X_TERRAFORM_LOCK_FILE = 'test';
+
+ const result = await updateArtifacts({
+ packageFileName: '',
+ updatedDeps: [],
+ newPackageFileContent: '',
+ config: localConfig,
+ });
+ expect(result).not.toBeNull();
+ expect(result).toBeArrayOfSize(1);
+
+ result.forEach((value) => expect(value.file).not.toBeNull());
+ result.forEach((value) => expect(value.file).toMatchSnapshot());
+
+ expect(mockHash.mock.calls).toBeArrayOfSize(2);
+ expect(mockHash.mock.calls).toMatchSnapshot();
+ });
+
+ it('do full lock file maintenance with lockfile in subfolder', async () => {
+ fs.readLocalFile.mockResolvedValueOnce(validLockfile as any);
+ fs.getSiblingFileName.mockReturnValueOnce('subfolder/.terraform.lock.hcl');
mockGetPkgReleases
.mockResolvedValueOnce({
diff --git a/lib/manager/terraform/lockfile/index.ts b/lib/manager/terraform/lockfile/index.ts
index e39f0b3169..975f73166c 100644
--- a/lib/manager/terraform/lockfile/index.ts
+++ b/lib/manager/terraform/lockfile/index.ts
@@ -1,12 +1,14 @@
import pMap from 'p-map';
import { GetPkgReleasesConfig, getPkgReleases } from '../../../datasource';
+import { TerraformProviderDatasource } from '../../../datasource/terraform-provider';
import { logger } from '../../../logger';
import { get as getVersioning } from '../../../versioning';
import type { UpdateArtifact, UpdateArtifactsResult } from '../../types';
-import { createHashes } from './hash';
+import { TerraformProviderHash } from './hash';
import type { ProviderLock, ProviderLockUpdate } from './types';
import {
extractLocks,
+ findLockFile,
isPinnedVersion,
readLockFile,
writeLockUpdates,
@@ -38,7 +40,11 @@ async function updateAllLocks(
const update: ProviderLockUpdate = {
newVersion,
newConstraint: lock.constraints,
- newHashes: await createHashes(lock.lookupName, newVersion),
+ newHashes: await TerraformProviderHash.createHashes(
+ lock.registryUrl,
+ lock.lookupName,
+ newVersion
+ ),
...lock,
};
return update;
@@ -64,49 +70,75 @@ export async function updateArtifacts({
return null;
}
- const lockFileContent = await readLockFile(packageFileName);
- if (!lockFileContent) {
- logger.debug('No .terraform.lock.hcl found');
- return null;
- }
- const locks = extractLocks(lockFileContent);
- if (!locks) {
- logger.debug('No Locks in .terraform.lock.hcl found');
- return null;
- }
+ const lockFilePath = findLockFile(packageFileName);
+ try {
+ const lockFileContent = await readLockFile(lockFilePath);
+ if (!lockFileContent) {
+ logger.debug('No .terraform.lock.hcl found');
+ return null;
+ }
+ const locks = extractLocks(lockFileContent);
+ if (!locks) {
+ logger.debug('No Locks in .terraform.lock.hcl found');
+ return null;
+ }
- const updates: ProviderLockUpdate[] = [];
- if (config.updateType === 'lockFileMaintenance') {
- // update all locks in the file during maintenance --> only update version in constraints
- const maintenanceUpdates = await updateAllLocks(locks);
- updates.push(...maintenanceUpdates);
- } else {
- // update only specific locks but with constrain updates
- const lookupName = updatedDeps[0].lookupName;
- const repository = lookupName.includes('/')
- ? lookupName
- : `hashicorp/${lookupName}`;
- const newConstraint = isPinnedVersion(config.newValue)
- ? config.newVersion
- : config.newValue;
- const updateLock = locks.find((value) => value.lookupName === repository);
- const update: ProviderLockUpdate = {
- newVersion: config.newVersion,
- newConstraint,
- newHashes: await createHashes(repository, config.newVersion),
- ...updateLock,
- };
- updates.push(update);
- }
+ const updates: ProviderLockUpdate[] = [];
+ if (config.updateType === 'lockFileMaintenance') {
+ // update all locks in the file during maintenance --> only update version in constraints
+ const maintenanceUpdates = await updateAllLocks(locks);
+ updates.push(...maintenanceUpdates);
+ } else {
+ // update only specific locks but with constrain updates
+ const dep = updatedDeps[0];
- // if no updates have been found or there are failed hashes abort
- if (
- updates.length === 0 ||
- updates.some((value) => value.newHashes == null)
- ) {
- return null;
- }
+ const lookupName = dep.lookupName ?? dep.depName;
- const res = writeLockUpdates(updates, lockFileContent);
- return res ? [res] : null;
+ // handle cases like `Telmate/proxmox`
+ const massagedLookupName = lookupName.toLowerCase();
+
+ const repository = massagedLookupName.includes('/')
+ ? massagedLookupName
+ : `hashicorp/${massagedLookupName}`;
+ const registryUrl = dep.registryUrls
+ ? dep.registryUrls[0]
+ : TerraformProviderDatasource.defaultRegistryUrls[0];
+ const newConstraint = isPinnedVersion(config.newValue)
+ ? config.newVersion
+ : config.newValue;
+ const updateLock = locks.find((value) => value.lookupName === repository);
+ const update: ProviderLockUpdate = {
+ newVersion: config.newVersion,
+ newConstraint,
+ newHashes: await TerraformProviderHash.createHashes(
+ registryUrl,
+ repository,
+ config.newVersion
+ ),
+ ...updateLock,
+ };
+ updates.push(update);
+ }
+
+ // if no updates have been found or there are failed hashes abort
+ if (
+ updates.length === 0 ||
+ updates.some((value) => value.newHashes == null)
+ ) {
+ return null;
+ }
+
+ const res = writeLockUpdates(updates, lockFilePath, lockFileContent);
+ return res ? [res] : null;
+ } catch (err) {
+ /* istanbul ignore next */
+ return [
+ {
+ artifactError: {
+ lockFile: lockFilePath,
+ stderr: err.message,
+ },
+ },
+ ];
+ }
}
diff --git a/lib/manager/terraform/lockfile/util.ts b/lib/manager/terraform/lockfile/util.ts
index 4cb5443998..47039cf696 100644
--- a/lib/manager/terraform/lockfile/util.ts
+++ b/lib/manager/terraform/lockfile/util.ts
@@ -8,8 +8,6 @@ import type {
ProviderSlice,
} from './types';
-export const repositoryRegex = /^hashicorp\/(?<lookupName>\S+)$/;
-
const providerStartLineRegex =
/^provider "(?<registryUrl>[^/]*)\/(?<namespace>[^/]*)\/(?<depName>[^/]*)"/;
const versionLineRegex =
@@ -20,8 +18,11 @@ const hashLineRegex = /^(?<prefix>\s*")(?<hash>[^"]+)(?<suffix>",.*)$/;
const lockFile = '.terraform.lock.hcl';
-export function readLockFile(packageFilePath: string): Promise<string> {
- const lockFilePath = getSiblingFileName(packageFilePath, lockFile);
+export function findLockFile(packageFilePath: string): string {
+ return getSiblingFileName(packageFilePath, lockFile);
+}
+
+export function readLockFile(lockFilePath: string): Promise<string> {
return readLocalFile(lockFilePath, 'utf8');
}
@@ -104,7 +105,7 @@ export function extractLocks(lockFileContent: string): ProviderLock[] {
const lock: ProviderLock = {
lookupName,
- registryUrl,
+ registryUrl: `https://${registryUrl}`,
version,
constraints,
hashes,
@@ -126,6 +127,7 @@ export function isPinnedVersion(value: string): boolean {
export function writeLockUpdates(
updates: ProviderLockUpdate[],
+ lockFilePath: string,
oldLockFileContent: string
): UpdateArtifactsResult {
const lines = oldLockFileContent.split('\n');
@@ -202,7 +204,7 @@ export function writeLockUpdates(
return {
file: {
- name: lockFile,
+ name: lockFilePath,
contents: newContent,
},
};
--
GitLab