From 04620d71a8650e742e2eb3bbbf0e778b63554f1d Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Fri, 12 Nov 2021 08:19:23 +0100
Subject: [PATCH] feat: evaluate buildpack constraints in exec (#12609)
---
lib/manager/composer/artifacts.ts | 11 ++--
lib/manager/composer/utils.spec.ts | 53 +---------------
lib/manager/composer/utils.ts | 40 ------------
lib/util/exec/buildpack.spec.ts | 99 ++++++++++++++++++++++++++++++
lib/util/exec/buildpack.ts | 75 ++++++++++++++++++++++
lib/util/exec/index.ts | 9 ++-
lib/util/exec/types.ts | 10 +++
7 files changed, 199 insertions(+), 98 deletions(-)
create mode 100644 lib/util/exec/buildpack.spec.ts
create mode 100644 lib/util/exec/buildpack.ts
create mode 100644 lib/util/exec/types.ts
diff --git a/lib/manager/composer/artifacts.ts b/lib/manager/composer/artifacts.ts
index 929d64b588..27a5378f62 100644
--- a/lib/manager/composer/artifacts.ts
+++ b/lib/manager/composer/artifacts.ts
@@ -8,6 +8,7 @@ import {
import * as datasourcePackagist from '../../datasource/packagist';
import { logger } from '../../logger';
import { ExecOptions, exec } from '../../util/exec';
+import type { ToolConstraint } from '../../util/exec/types';
import {
ensureCacheDir,
ensureLocalDir,
@@ -25,7 +26,6 @@ import {
composerVersioningId,
extractContraints,
getComposerArguments,
- getComposerConstraint,
getPhpConstraint,
} from './utils';
@@ -102,9 +102,10 @@ export async function updateArtifacts({
...config.constraints,
};
- const preCommands: string[] = [
- `install-tool composer ${await getComposerConstraint(constraints)}`,
- ];
+ const composerToolConstraint: ToolConstraint = {
+ toolName: 'composer',
+ constraint: constraints.composer,
+ };
const execOptions: ExecOptions = {
cwdFile: packageFileName,
@@ -112,8 +113,8 @@ export async function updateArtifacts({
COMPOSER_CACHE_DIR: await ensureCacheDir('composer'),
COMPOSER_AUTH: getAuthJson(),
},
+ toolConstraints: [composerToolConstraint],
docker: {
- preCommands,
image: 'php',
tagConstraint: getPhpConstraint(constraints),
tagScheme: composerVersioningId,
diff --git a/lib/manager/composer/utils.spec.ts b/lib/manager/composer/utils.spec.ts
index edeab17528..6970039a65 100644
--- a/lib/manager/composer/utils.spec.ts
+++ b/lib/manager/composer/utils.spec.ts
@@ -1,58 +1,7 @@
-import { mocked } from '../../../test/util';
import { setGlobalConfig } from '../../config/global';
-import * as _datasource from '../../datasource';
-import {
- extractContraints,
- getComposerArguments,
- getComposerConstraint,
-} from './utils';
-
-jest.mock('../../../lib/datasource');
-
-const datasource = mocked(_datasource);
+import { extractContraints, getComposerArguments } from './utils';
describe('manager/composer/utils', () => {
- describe('getComposerConstraint', () => {
- beforeEach(() => {
- datasource.getPkgReleases.mockResolvedValueOnce({
- releases: [
- { version: '1.0.0' },
- { version: '1.1.0' },
- { version: '1.3.0' },
- { version: '2.0.14' },
- { version: '2.1.0' },
- ],
- });
- });
- it('returns from config', async () => {
- expect(await getComposerConstraint({ composer: '1.1.0' })).toBe('1.1.0');
- });
-
- it('returns from latest', async () => {
- expect(await getComposerConstraint({})).toBe('2.1.0');
- });
-
- it('throws no releases', async () => {
- datasource.getPkgReleases.mockReset();
- datasource.getPkgReleases.mockResolvedValueOnce({
- releases: [],
- });
- await expect(getComposerConstraint({})).rejects.toThrow(
- 'No composer releases found.'
- );
- });
-
- it('throws no compatible releases', async () => {
- datasource.getPkgReleases.mockReset();
- datasource.getPkgReleases.mockResolvedValueOnce({
- releases: [{ version: '1.2.3' }],
- });
- await expect(
- getComposerConstraint({ composer: '^3.1.0' })
- ).rejects.toThrow('No compatible composer releases found.');
- });
- });
-
describe('extractContraints', () => {
it('returns from require', () => {
expect(
diff --git a/lib/manager/composer/utils.ts b/lib/manager/composer/utils.ts
index 57b837dc70..68684d9ba7 100644
--- a/lib/manager/composer/utils.ts
+++ b/lib/manager/composer/utils.ts
@@ -1,6 +1,5 @@
import { quote } from 'shlex';
import { getGlobalConfig } from '../../config/global';
-import { getPkgReleases } from '../../datasource';
import { logger } from '../../logger';
import { api, id as composerVersioningId } from '../../versioning/composer';
import type { UpdateArtifactsConfig } from '../types';
@@ -29,45 +28,6 @@ export function getComposerArguments(config: UpdateArtifactsConfig): string {
return args;
}
-export async function getComposerConstraint(
- constraints: Record<string, string>
-): Promise<string> {
- const { composer } = constraints;
-
- if (api.isSingleVersion(composer)) {
- logger.debug(
- { version: composer },
- 'Using composer constraint from config'
- );
- return composer;
- }
-
- const release = await getPkgReleases({
- depName: 'composer/composer',
- datasource: 'github-releases',
- versioning: composerVersioningId,
- });
-
- if (!release?.releases?.length) {
- throw new Error('No composer releases found.');
- }
- let versions = release.releases.map((r) => r.version);
-
- if (composer) {
- versions = versions.filter(
- (v) => api.isValid(v) && api.matches(v, composer)
- );
- }
-
- if (!versions.length) {
- throw new Error('No compatible composer releases found.');
- }
-
- const version = versions.pop();
- logger.debug({ range: composer, version }, 'Using composer constraint');
- return version;
-}
-
export function getPhpConstraint(constraints: Record<string, string>): string {
const { php } = constraints;
diff --git a/lib/util/exec/buildpack.spec.ts b/lib/util/exec/buildpack.spec.ts
new file mode 100644
index 0000000000..1a47e3f3c4
--- /dev/null
+++ b/lib/util/exec/buildpack.spec.ts
@@ -0,0 +1,99 @@
+import { mocked } from '../../../test/util';
+import * as _datasource from '../../datasource';
+import { generateInstallCommands, resolveConstraint } from './buildpack';
+import type { ToolConstraint } from './types';
+
+jest.mock('../../../lib/datasource');
+
+const datasource = mocked(_datasource);
+
+describe('util/exec/buildpack', () => {
+ describe('resolveConstraint()', () => {
+ beforeEach(() => {
+ datasource.getPkgReleases.mockResolvedValueOnce({
+ releases: [
+ { version: '1.0.0' },
+ { version: '1.1.0' },
+ { version: '1.3.0' },
+ { version: '2.0.14' },
+ { version: '2.1.0' },
+ ],
+ });
+ });
+ it('returns from config', async () => {
+ expect(
+ await resolveConstraint({ toolName: 'composer', constraint: '1.1.0' })
+ ).toBe('1.1.0');
+ });
+
+ it('returns from latest', async () => {
+ expect(await resolveConstraint({ toolName: 'composer' })).toBe('2.1.0');
+ });
+
+ it('throws for unknown tools', async () => {
+ datasource.getPkgReleases.mockReset();
+ datasource.getPkgReleases.mockResolvedValueOnce({
+ releases: [],
+ });
+ await expect(resolveConstraint({ toolName: 'whoops' })).rejects.toThrow(
+ 'Invalid tool to install: whoops'
+ );
+ });
+
+ it('throws no releases', async () => {
+ datasource.getPkgReleases.mockReset();
+ datasource.getPkgReleases.mockResolvedValueOnce({
+ releases: [],
+ });
+ await expect(resolveConstraint({ toolName: 'composer' })).rejects.toThrow(
+ 'No tool releases found.'
+ );
+ });
+
+ it('falls back to latest version if no compatible release', async () => {
+ datasource.getPkgReleases.mockReset();
+ datasource.getPkgReleases.mockResolvedValueOnce({
+ releases: [{ version: '1.2.3' }],
+ });
+ expect(
+ await resolveConstraint({ toolName: 'composer', constraint: '^3.1.0' })
+ ).toBe('1.2.3');
+ });
+
+ it('falls back to latest version if invalid constraint', async () => {
+ datasource.getPkgReleases.mockReset();
+ datasource.getPkgReleases.mockResolvedValueOnce({
+ releases: [{ version: '1.2.3' }],
+ });
+ expect(
+ await resolveConstraint({ toolName: 'composer', constraint: 'whoops' })
+ ).toBe('1.2.3');
+ });
+ });
+ describe('generateInstallCommands()', () => {
+ beforeEach(() => {
+ datasource.getPkgReleases.mockResolvedValueOnce({
+ releases: [
+ { version: '1.0.0' },
+ { version: '1.1.0' },
+ { version: '1.3.0' },
+ { version: '2.0.14' },
+ { version: '2.1.0' },
+ ],
+ });
+ });
+ it('returns install commands', async () => {
+ const toolConstraints: ToolConstraint[] = [
+ {
+ toolName: 'composer',
+ },
+ ];
+ expect(await generateInstallCommands(toolConstraints))
+ .toMatchInlineSnapshot(`
+ Array [
+ "install-tool composer 2.1.0",
+ ]
+ `);
+ });
+ });
+});
diff --git a/lib/util/exec/buildpack.ts b/lib/util/exec/buildpack.ts
new file mode 100644
index 0000000000..ae47f44e8c
--- /dev/null
+++ b/lib/util/exec/buildpack.ts
@@ -0,0 +1,75 @@
+import { quote } from 'shlex';
+import { getPkgReleases } from '../../datasource';
+import { logger } from '../../logger';
+import * as allVersioning from '../../versioning';
+import { id as composerVersioningId } from '../../versioning/composer';
+import type { ToolConfig, ToolConstraint } from './types';
+
+const allToolConfig: Record<string, ToolConfig> = {
+ composer: {
+ datasource: 'github-releases',
+ depName: 'composer/composer',
+ versioning: composerVersioningId,
+ },
+};
+
+export async function resolveConstraint(
+ toolConstraint: ToolConstraint
+): Promise<string> {
+ const { toolName } = toolConstraint;
+ const toolConfig = allToolConfig[toolName];
+ if (!toolConfig) {
+ throw new Error(`Invalid tool to install: ${toolName}`);
+ }
+
+ const versioning = allVersioning.get(toolConfig.versioning);
+ let constraint = toolConstraint.constraint;
+ if (constraint) {
+ if (versioning.isValid(constraint)) {
+ if (versioning.isSingleVersion(constraint)) {
+ return constraint;
+ }
+ } else {
+ logger.warn({ toolName, constraint }, 'Invalid tool constraint');
+ constraint = undefined;
+ }
+ }
+
+ const pkgReleases = await getPkgReleases(toolConfig);
+ if (!pkgReleases?.releases?.length) {
+ throw new Error('No tool releases found.');
+ }
+
+ const allVersions = pkgReleases.releases.map((r) => r.version);
+ const matchingVersions = allVersions.filter(
+ (v) => !constraint || versioning.matches(v, constraint)
+ );
+
+ if (matchingVersions.length) {
+ const resolvedVersion = matchingVersions.pop();
+ logger.debug({ toolName, constraint, resolvedVersion }, 'Resolved version');
+ return resolvedVersion;
+ }
+ const latestVersion = allVersions.filter((v) => versioning.isStable(v)).pop();
+ logger.warn(
+ { toolName, constraint, latestVersion },
+ 'No matching tool versions found for constraint - using latest version'
+ );
+ return latestVersion;
+}
+
+export async function generateInstallCommands(
+ toolConstraints: ToolConstraint[]
+): Promise<string[]> {
+ const installCommands = [];
+ if (toolConstraints?.length) {
+ for (const toolConstraint of toolConstraints) {
+ const toolVersion = await resolveConstraint(toolConstraint);
+ const installCommand = `install-tool ${toolConstraint.toolName} ${quote(
+ toolVersion
+ )}`;
+ installCommands.push(installCommand);
+ }
+ }
+ return installCommands;
+}
diff --git a/lib/util/exec/index.ts b/lib/util/exec/index.ts
index b086f22b1b..c093e0b059 100644
--- a/lib/util/exec/index.ts
+++ b/lib/util/exec/index.ts
@@ -3,6 +3,7 @@ import { dirname, join } from 'upath';
import { getGlobalConfig } from '../../config/global';
import { TEMPORARY_ERROR } from '../../constants/error-messages';
import { logger } from '../../logger';
+import { generateInstallCommands } from './buildpack';
import {
DockerOptions,
ExecResult,
@@ -12,6 +13,7 @@ import {
} from './common';
import { generateDockerCommand, removeDockerContainer } from './docker';
import { getChildProcessEnv } from './env';
+import type { ToolConstraint } from './types';
type ExtraEnv<T = unknown> = Record<string, T>;
@@ -19,6 +21,7 @@ export interface ExecOptions extends ChildProcessExecOptions {
cwdFile?: string;
extraEnv?: Opt<ExtraEnv>;
docker?: Opt<DockerOptions>;
+ toolConstraints?: Opt<ToolConstraint[]>;
}
function getChildEnv({
@@ -69,6 +72,7 @@ function getRawExecOptions(opts: ExecOptions): RawExecOptions {
delete execOptions.extraEnv;
delete execOptions.docker;
delete execOptions.cwdFile;
+ delete execOptions.toolConstraints;
const childEnv = getChildEnv(opts);
const cwd = getCwd(opts);
@@ -113,7 +117,10 @@ async function prepareRawExec(
const envVars = dockerEnvVars(extraEnv, childEnv);
const cwd = getCwd(opts);
const dockerOptions: DockerOptions = { ...docker, cwd, envVars };
-
+ dockerOptions.preCommands = [
+ ...(await generateInstallCommands(opts.toolConstraints)),
+ ...(dockerOptions.preCommands || []),
+ ];
const dockerCommand = await generateDockerCommand(
rawCommands,
dockerOptions
diff --git a/lib/util/exec/types.ts b/lib/util/exec/types.ts
new file mode 100644
index 0000000000..65b015083d
--- /dev/null
+++ b/lib/util/exec/types.ts
@@ -0,0 +1,10 @@
+export interface ToolConstraint {
+ toolName: string;
+ constraint?: string;
+}
+
+export interface ToolConfig {
+ datasource: string;
+ depName: string;
+ versioning: string;
+}
--
GitLab