From fc85d654e9017c731b209e7f34aaade7f439e0c8 Mon Sep 17 00:00:00 2001
From: Sebastian Poxhofer <secustor@users.noreply.github.com>
Date: Fri, 12 Jan 2024 07:00:19 +0100
Subject: [PATCH] fix(manager/terraform): handle separate shasum files (#26607)
---
.../manager/terraform/lockfile/hash.spec.ts | 105 ++++++++++++++++++
.../manager/terraform/lockfile/hash.ts | 23 +++-
lib/util/array.ts | 4 +
3 files changed, 126 insertions(+), 6 deletions(-)
diff --git a/lib/modules/manager/terraform/lockfile/hash.spec.ts b/lib/modules/manager/terraform/lockfile/hash.spec.ts
index 0dd528599b..b0e82c83b7 100644
--- a/lib/modules/manager/terraform/lockfile/hash.spec.ts
+++ b/lib/modules/manager/terraform/lockfile/hash.spec.ts
@@ -206,6 +206,111 @@ describe('modules/manager/terraform/lockfile/hash', () => {
]);
});
+ it('full walkthrough with different shasum per build', async () => {
+ const readStreamLinux = createReadStream(
+ 'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip',
+ );
+ const readStreamDarwin = createReadStream(
+ 'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip',
+ );
+ httpMock
+ .scope(terraformCloudReleaseBackendUrl)
+ .get('/.well-known/terraform.json')
+ .reply(200, terraformCloudSDCJson)
+ .get('/v1/providers/gravitational/teleport/versions')
+ .reply(
+ 200,
+ JSON.stringify({
+ id: 'gravitational/teleport',
+ versions: [
+ {
+ version: '14.3.1',
+ protocols: ['5.0'],
+ platforms: [
+ {
+ os: 'linux',
+ arch: 'amd64',
+ },
+ {
+ os: 'darwin',
+ arch: 'amd64',
+ },
+ ],
+ },
+ {
+ version: '1.33.0',
+ protocols: ['4.0', '5.0'],
+ platforms: [
+ {
+ os: 'linux',
+ arch: 'amd64',
+ },
+ {
+ os: 'darwin',
+ arch: 'amd64',
+ },
+ ],
+ },
+ ],
+ warnings: null,
+ }),
+ )
+ .get('/v1/providers/gravitational/teleport/14.3.1/download/linux/amd64')
+ .reply(200, {
+ os: 'linux',
+ arch: 'amd64',
+ filename: 'terraform-provider-teleport-v14.3.1-linux-amd64-bin.zip',
+ shasums_url:
+ 'https://terraform.releases.teleport.dev/store/terraform-provider-teleport-v14.3.1-linux-amd64-bin.zip.sums',
+ download_url:
+ 'https://terraform.releases.teleport.dev/store/terraform-provider-teleport-v14.3.1-linux-amd64-bin.zip',
+ })
+ .get('/v1/providers/gravitational/teleport/14.3.1/download/darwin/amd64')
+ .reply(200, {
+ os: 'darwin',
+ arch: 'amd64',
+ filename: 'terraform-provider-teleport-v14.3.1-darwin-amd64-bin.zip',
+ shasums_url:
+ 'https://terraform.releases.teleport.dev/store/terraform-provider-teleport-v14.3.1-darwin-amd64-bin.zip.sums',
+ download_url:
+ 'https://terraform.releases.teleport.dev/store/terraform-provider-teleport-v14.3.1-darwin-amd64-bin.zip',
+ });
+
+ httpMock
+ .scope('https://terraform.releases.teleport.dev')
+ .get(
+ '/store/terraform-provider-teleport-v14.3.1-linux-amd64-bin.zip.sums',
+ )
+ .reply(
+ 200,
+ '1d47d00730fab764bddb6d548fed7e124739b0bcebb9f3b3c6aa247de55fb804 terraform-provider-teleport-v14.3.1-linux-amd64-bin.zip',
+ )
+ .get('/store/terraform-provider-teleport-v14.3.1-linux-amd64-bin.zip')
+ .reply(200, readStreamLinux)
+ .get(
+ '/store/terraform-provider-teleport-v14.3.1-darwin-amd64-bin.zip.sums',
+ )
+ .reply(
+ 200,
+ '29bff92b4375a35a7729248b3bc5db8991ca1b9ba640fc25b13700e12f99c195 terraform-provider-teleport-v14.3.1-darwin-amd64-bin.zip',
+ )
+ .get('/store/terraform-provider-teleport-v14.3.1-darwin-amd64-bin.zip')
+ .reply(200, readStreamDarwin);
+
+ const result = await TerraformProviderHash.createHashes(
+ 'https://registry.terraform.io',
+ 'gravitational/teleport',
+ '14.3.1',
+ );
+ expect(log.error.mock.calls).toBeEmptyArray();
+ expect(result).toMatchObject([
+ 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=',
+ 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=',
+ 'zh:1d47d00730fab764bddb6d548fed7e124739b0bcebb9f3b3c6aa247de55fb804',
+ 'zh:29bff92b4375a35a7729248b3bc5db8991ca1b9ba640fc25b13700e12f99c195',
+ ]);
+ });
+
it('full walkthrough without ziphashes available', async () => {
const readStreamLinux = createReadStream(
'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip',
diff --git a/lib/modules/manager/terraform/lockfile/hash.ts b/lib/modules/manager/terraform/lockfile/hash.ts
index e6f9fdddc7..5736ed33fe 100644
--- a/lib/modules/manager/terraform/lockfile/hash.ts
+++ b/lib/modules/manager/terraform/lockfile/hash.ts
@@ -2,6 +2,11 @@ import crypto from 'node:crypto';
import extract from 'extract-zip';
import upath from 'upath';
import { logger } from '../../../../logger';
+import {
+ coerceArray,
+ deduplicateArray,
+ isNotNullOrUndefined,
+} from '../../../../util/array';
import { cache } from '../../../../util/cache/package/decorator';
import * as fs from '../../../../util/fs';
import { ensureCacheDir } from '../../../../util/fs';
@@ -115,12 +120,18 @@ export class TerraformProviderHash {
return null;
}
- let zhHashes: string[] = [];
- if (builds.length > 0 && builds[0].shasums_url) {
- zhHashes =
- (await TerraformProviderHash.terraformDatasource.getZipHashes(
- builds[0].shasums_url,
- )) ?? [];
+ // check if the publisher uses one shasum file for all builds or separate ones
+ // we deduplicate to reduce the number of API calls
+ const shaUrls = deduplicateArray(
+ builds.map((build) => build.shasums_url).filter(isNotNullOrUndefined),
+ );
+
+ const zhHashes: string[] = [];
+ for (const shaUrl of shaUrls) {
+ const hashes =
+ await TerraformProviderHash.terraformDatasource.getZipHashes(shaUrl);
+
+ zhHashes.push(...coerceArray(hashes));
}
const h1Hashes =
diff --git a/lib/util/array.ts b/lib/util/array.ts
index 87fc4c59ac..41b2ad0679 100644
--- a/lib/util/array.ts
+++ b/lib/util/array.ts
@@ -28,3 +28,7 @@ export function isNotNullOrUndefined<T>(
export function toArray<T>(value: T | T[]): T[] {
return is.array(value) ? value : [value];
}
+
+export function deduplicateArray<T>(array: T[]): T[] {
+ return Array.from(new Set(array));
+}
--
GitLab