From 713e35e88f30629689643f8bcf70cdd8ed21bbcd Mon Sep 17 00:00:00 2001
From: Michael Kriese <michael.kriese@visualon.de>
Date: Wed, 16 Jun 2021 16:19:14 +0200
Subject: [PATCH] fix(manager): optimize lockfile cache handling (#10463)
---
.../lockfile/__snapshots__/index.spec.ts.snap | 7 ------
lib/manager/terraform/lockfile/hash.spec.ts | 25 +++++++++++++------
lib/manager/terraform/lockfile/hash.ts | 16 ++++++------
lib/manager/terraform/lockfile/index.spec.ts | 6 ++---
lib/manager/terraform/lockfile/index.ts | 22 +++++-----------
lib/manager/terraform/lockfile/util.ts | 12 ++++++++-
6 files changed, 45 insertions(+), 43 deletions(-)
diff --git a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap
index 22c1e15be8..5061dc2f11 100644
--- a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap
+++ b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap
@@ -50,12 +50,10 @@ Array [
Array [
"hashicorp/azurerm",
"2.56.0",
- "/tmp/renovate/cache",
],
Array [
"hashicorp/random",
"2.2.2",
- "/tmp/renovate/cache",
],
]
`;
@@ -67,12 +65,10 @@ Array [
Array [
"hashicorp/azurerm",
"2.56.0",
- "/tmp/renovate/cache",
],
Array [
"hashicorp/random",
"2.2.2",
- "/tmp/renovate/cache",
],
]
`;
@@ -139,7 +135,6 @@ Array [
Array [
"hashicorp/aws",
"3.36.0",
- "/tmp/renovate/cache",
],
]
`;
@@ -204,7 +199,6 @@ Array [
Array [
"hashicorp/random",
"3.1.0",
- "/tmp/renovate/cache",
],
]
`;
@@ -270,7 +264,6 @@ Array [
Array [
"hashicorp/azurerm",
"2.56.0",
- "/tmp/renovate/cache",
],
]
`;
diff --git a/lib/manager/terraform/lockfile/hash.spec.ts b/lib/manager/terraform/lockfile/hash.spec.ts
index 8c17ee4c20..a2c9e0b9b4 100644
--- a/lib/manager/terraform/lockfile/hash.spec.ts
+++ b/lib/manager/terraform/lockfile/hash.spec.ts
@@ -1,18 +1,27 @@
import { createReadStream } from 'fs';
-import { join } from 'upath';
+import { DirectoryResult, dir } from 'tmp-promise';
import * as httpMock from '../../../../test/http-mock';
import { getFixturePath, getName, loadFixture } from '../../../../test/util';
+import { setAdminConfig } from '../../../config/admin';
import { TerraformProviderDatasource } from '../../../datasource/terraform-provider';
-import createHashes from './hash';
+import { createHashes } from './hash';
const terraformProviderDatasource = new TerraformProviderDatasource();
const releaseBackendUrl = terraformProviderDatasource.defaultRegistryUrls[1];
const releaseBackendAzurerm = loadFixture('releaseBackendAzurerm_2_56_0.json');
-const cacheDir = join('/tmp/renovate/cache');
describe(getName(), () => {
+ let cacheDir: DirectoryResult;
+
+ beforeAll(async () => {
+ cacheDir = await dir({ unsafeCleanup: true });
+ setAdminConfig({ cacheDir: cacheDir.path });
+ });
+
+ afterAll(() => cacheDir.cleanup());
+
it('returns null if a non hashicorp release is found ', async () => {
- const result = await createHashes('test/gitlab', '2.56.0', cacheDir);
+ const result = await createHashes('test/gitlab', '2.56.0');
expect(result).toBeNull();
});
@@ -22,7 +31,7 @@ describe(getName(), () => {
.get('/terraform-provider-azurerm/2.59.0/index.json')
.reply(403, '');
- const result = await createHashes('hashicorp/azurerm', '2.59.0', cacheDir);
+ const result = await createHashes('hashicorp/azurerm', '2.59.0');
expect(result).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
@@ -33,7 +42,7 @@ describe(getName(), () => {
.get('/terraform-provider-azurerm/2.56.0/index.json')
.replyWithError('');
- const result = await createHashes('hashicorp/azurerm', '2.56.0', cacheDir);
+ const result = await createHashes('hashicorp/azurerm', '2.56.0');
expect(result).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
@@ -58,7 +67,7 @@ describe(getName(), () => {
)
.reply(200, readStreamDarwin);
- const result = await createHashes('hashicorp/azurerm', '2.56.0', cacheDir);
+ const result = await createHashes('hashicorp/azurerm', '2.56.0');
expect(result).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
@@ -83,7 +92,7 @@ describe(getName(), () => {
)
.reply(200, readStreamDarwin);
- const result = await createHashes('hashicorp/azurerm', '2.56.0', cacheDir);
+ const result = await createHashes('hashicorp/azurerm', '2.56.0');
expect(result).not.toBeNull();
expect(result).toBeArrayOfSize(2);
expect(result).toMatchSnapshot();
diff --git a/lib/manager/terraform/lockfile/hash.ts b/lib/manager/terraform/lockfile/hash.ts
index f659a60fee..15c3f72f82 100644
--- a/lib/manager/terraform/lockfile/hash.ts
+++ b/lib/manager/terraform/lockfile/hash.ts
@@ -11,7 +11,7 @@ import { logger } from '../../../logger';
import * as packageCache from '../../../util/cache/package';
import * as fs from '../../../util/fs';
import { Http } from '../../../util/http';
-import { repositoryRegex } from './util';
+import { getCacheDir, repositoryRegex } from './util';
const http = new Http(TerraformProviderDatasource.id);
const hashCacheTTL = 10080; // in seconds == 1 week
@@ -74,9 +74,10 @@ async function getReleaseBackendIndex(
}
export async function calculateHashes(
- builds: TerraformBuild[],
- cacheDir: string
+ builds: TerraformBuild[]
): Promise<string[]> {
+ const cacheDir = await getCacheDir();
+
// for each build download ZIP, extract content and generate hash for all containing files
const hashes = await pMap(
builds,
@@ -112,10 +113,9 @@ export async function calculateHashes(
return hashes;
}
-export default async function createHashes(
+export async function createHashes(
repository: string,
- version: string,
- cacheDir: string
+ version: string
): Promise<string[]> {
// check cache for hashes
const repositoryRegexResult = repositoryRegex.exec(repository);
@@ -135,7 +135,7 @@ export default async function createHashes(
if (cachedRelease) {
return cachedRelease;
}
- let versionReleaseBackend;
+ let versionReleaseBackend: VersionDetailResponse;
try {
versionReleaseBackend = await getReleaseBackendIndex(
backendLookUpName,
@@ -150,7 +150,7 @@ export default async function createHashes(
}
const builds = versionReleaseBackend.builds;
- const hashes = await calculateHashes(builds, cacheDir);
+ const hashes = await calculateHashes(builds);
// if a hash could not be produced skip caching and return null
if (hashes.some((value) => value == null)) {
diff --git a/lib/manager/terraform/lockfile/index.spec.ts b/lib/manager/terraform/lockfile/index.spec.ts
index b2dbc4f8be..b896895868 100644
--- a/lib/manager/terraform/lockfile/index.spec.ts
+++ b/lib/manager/terraform/lockfile/index.spec.ts
@@ -1,9 +1,9 @@
import { join } from 'upath';
-import { fs, getName, loadFixture } from '../../../../test/util';
+import { fs, getName, loadFixture, mocked } from '../../../../test/util';
import { setAdminConfig } from '../../../config/admin';
import { getPkgReleases } from '../../../datasource';
import type { UpdateArtifactsConfig } from '../../types';
-import hash from './hash';
+import * as hash from './hash';
import { updateArtifacts } from './index';
// auto-mock fs
@@ -23,7 +23,7 @@ const adminConfig = {
const validLockfile = loadFixture('validLockfile.hcl');
-const mockHash = hash as jest.MockedFunction<typeof hash>;
+const mockHash = mocked(hash).createHashes;
const mockGetPkgReleases = getPkgReleases as jest.MockedFunction<
typeof getPkgReleases
>;
diff --git a/lib/manager/terraform/lockfile/index.ts b/lib/manager/terraform/lockfile/index.ts
index deb30c23fc..2fb8b5a6e8 100644
--- a/lib/manager/terraform/lockfile/index.ts
+++ b/lib/manager/terraform/lockfile/index.ts
@@ -1,14 +1,9 @@
import pMap from 'p-map';
-import { getAdminConfig } from '../../../config/admin';
import { GetPkgReleasesConfig, getPkgReleases } from '../../../datasource';
import { logger } from '../../../logger';
import { get as getVersioning } from '../../../versioning';
-import type {
- UpdateArtifact,
- UpdateArtifactsConfig,
- UpdateArtifactsResult,
-} from '../../types';
-import hash from './hash';
+import type { UpdateArtifact, UpdateArtifactsResult } from '../../types';
+import { createHashes } from './hash';
import type { ProviderLock, ProviderLockUpdate } from './types';
import {
extractLocks,
@@ -18,11 +13,8 @@ import {
} from './util';
async function updateAllLocks(
- locks: ProviderLock[],
- config: UpdateArtifactsConfig
+ locks: ProviderLock[]
): Promise<ProviderLockUpdate[]> {
- const { cacheDir } = getAdminConfig();
-
const updates = await pMap(
locks,
async (lock) => {
@@ -46,7 +38,7 @@ async function updateAllLocks(
const update: ProviderLockUpdate = {
newVersion,
newConstraint: lock.constraints,
- newHashes: await hash(lock.lookupName, newVersion, cacheDir),
+ newHashes: await createHashes(lock.lookupName, newVersion),
...lock,
};
return update;
@@ -72,8 +64,6 @@ export async function updateArtifacts({
return null;
}
- const { cacheDir } = getAdminConfig();
-
const lockFileContent = await readLockFile(packageFileName);
if (!lockFileContent) {
logger.debug('No .terraform.lock.hcl found');
@@ -88,7 +78,7 @@ export async function updateArtifacts({
const updates: ProviderLockUpdate[] = [];
if (config.updateType === 'lockFileMaintenance') {
// update all locks in the file during maintenance --> only update version in constraints
- const maintenanceUpdates = await updateAllLocks(locks, config);
+ const maintenanceUpdates = await updateAllLocks(locks);
updates.push(...maintenanceUpdates);
} else {
// update only specific locks but with constrain updates
@@ -103,7 +93,7 @@ export async function updateArtifacts({
const update: ProviderLockUpdate = {
newVersion: config.newVersion,
newConstraint,
- newHashes: await hash(repository, config.newVersion, cacheDir),
+ newHashes: await createHashes(repository, config.newVersion),
...updateLock,
};
updates.push(update);
diff --git a/lib/manager/terraform/lockfile/util.ts b/lib/manager/terraform/lockfile/util.ts
index 4cb5443998..ae2bfcbcbd 100644
--- a/lib/manager/terraform/lockfile/util.ts
+++ b/lib/manager/terraform/lockfile/util.ts
@@ -1,4 +1,7 @@
-import { getSiblingFileName, readLocalFile } from '../../../util/fs';
+import { join } from 'upath';
+import { getAdminConfig } from '../../../config/admin';
+import { logger } from '../../../logger';
+import { ensureDir, getSiblingFileName, readLocalFile } from '../../../util/fs';
import { get as getVersioning } from '../../../versioning';
import type { UpdateArtifactsResult } from '../../types';
import type {
@@ -207,3 +210,10 @@ export function writeLockUpdates(
},
};
}
+
+export async function getCacheDir(): Promise<string> {
+ const cacheDir = join(getAdminConfig().cacheDir, './others/terraform');
+ await ensureDir(cacheDir);
+ logger.debug(`Using terraform cache: ${cacheDir}`);
+ return cacheDir;
+}
--
GitLab