From 78aa91aca8f3702853828d691381d0effb5b69d9 Mon Sep 17 00:00:00 2001
From: RahulGautamSingh <rahultesnik@gmail.com>
Date: Thu, 27 Apr 2023 10:36:07 +0530
Subject: [PATCH] feat(pnpm): get locked version from `pnpm-lock.yaml` (#21480)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
.../lockfile-parsing/pnpm-lock.yaml | 23 +++
.../npm/extract/locked-versions.spec.ts | 76 ++++++---
.../manager/npm/extract/locked-versions.ts | 23 ++-
lib/modules/manager/npm/extract/pnpm.spec.ts | 86 ++++++++++
lib/modules/manager/npm/extract/pnpm.ts | 156 +++++++++++++++++-
lib/modules/manager/npm/post-update/types.ts | 3 +-
6 files changed, 336 insertions(+), 31 deletions(-)
create mode 100644 lib/modules/manager/npm/__fixtures__/lockfile-parsing/pnpm-lock.yaml
diff --git a/lib/modules/manager/npm/__fixtures__/lockfile-parsing/pnpm-lock.yaml b/lib/modules/manager/npm/__fixtures__/lockfile-parsing/pnpm-lock.yaml
new file mode 100644
index 0000000000..d8e3e77aaf
--- /dev/null
+++ b/lib/modules/manager/npm/__fixtures__/lockfile-parsing/pnpm-lock.yaml
@@ -0,0 +1,23 @@
+lockfileVersion: '6.0'
+
+dependencies:
+ xmldoc:
+ specifier: 1.1.0
+ version: 1.1.0
+
+packages:
+
+ /sax@1.2.4:
+ resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==}
+ dev: false
+
+ /xmldoc@1.1.0:
+ resolution: {integrity: sha512-5CEmEtW6IeVMEHSIxchhwpwJKnpFFsCOl9J3R2trVPcMsT7loE7jwT/q1Zwzlk3MetuiyCAdpA699gq0E4fgdw==}
+ dependencies:
+ sax: 1.2.4
+ dev: false
+
+ /sux-1.2.4: #invlaid
+ resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==}
+ dev: false
+
diff --git a/lib/modules/manager/npm/extract/locked-versions.spec.ts b/lib/modules/manager/npm/extract/locked-versions.spec.ts
index 8a9ff6b1f6..6f411397e7 100644
--- a/lib/modules/manager/npm/extract/locked-versions.spec.ts
+++ b/lib/modules/manager/npm/extract/locked-versions.spec.ts
@@ -2,13 +2,13 @@ import type { PackageFile } from '../../types';
import type { NpmManagerData } from '../types';
import { getLockedVersions } from './locked-versions';
-/** @type any */
const npm = require('./npm');
-/** @type any */
+const pnpm = require('./pnpm');
const yarn = require('./yarn');
jest.mock('./npm');
jest.mock('./yarn');
+jest.mock('./pnpm');
describe('modules/manager/npm/extract/locked-versions', () => {
describe('.getLockedVersions()', () => {
@@ -17,7 +17,10 @@ describe('modules/manager/npm/extract/locked-versions', () => {
): PackageFile<NpmManagerData>[] {
return [
{
- managerData: { npmLock: 'package-lock.json', yarnLock: 'yarn.lock' },
+ managerData: {
+ npmLock: 'package-lock.json',
+ yarnLock: 'yarn.lock',
+ },
extractedConstraints: {},
deps: [
{ depName: 'a', currentValue: '1.0.0' },
@@ -485,32 +488,51 @@ describe('modules/manager/npm/extract/locked-versions', () => {
},
]);
});
+ });
- it('ignores pnpm', async () => {
- const packageFiles = [
- {
- managerData: {
- pnpmShrinkwrap: 'pnpm-lock.yaml',
- },
- deps: [
- { depName: 'a', currentValue: '1.0.0' },
- { depName: 'b', currentValue: '2.0.0' },
- ],
- packageFile: 'some-file',
+ it('uses pnpm-lock', async () => {
+ pnpm.getPnpmLock.mockReturnValue({
+ lockedVersions: {
+ a: '1.0.0',
+ b: '2.0.0',
+ c: '3.0.0',
+ },
+ lockfileVersion: 6.0,
+ });
+ const packageFiles = [
+ {
+ managerData: {
+ pnpmShrinkwrap: 'pnpm-lock.yaml',
},
- ];
- await getLockedVersions(packageFiles);
- expect(packageFiles).toEqual([
- {
- deps: [
- { currentValue: '1.0.0', depName: 'a' },
- { currentValue: '2.0.0', depName: 'b' },
- ],
- lockFiles: ['pnpm-lock.yaml'],
- managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml' },
- packageFile: 'some-file',
+ extractedConstraints: {
+ pnpm: '>=6.0.0',
},
- ]);
- });
+ deps: [
+ {
+ depName: 'a',
+ currentValue: '1.0.0',
+ },
+ {
+ depName: 'b',
+ currentValue: '2.0.0',
+ },
+ ],
+ packageFile: 'some-file',
+ },
+ ];
+ pnpm.getConstraints.mockReturnValue('>=6.0.0 >=8');
+ await getLockedVersions(packageFiles);
+ expect(packageFiles).toEqual([
+ {
+ extractedConstraints: { pnpm: '>=6.0.0 >=8' },
+ deps: [
+ { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' },
+ { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' },
+ ],
+ lockFiles: ['pnpm-lock.yaml'],
+ managerData: { pnpmShrinkwrap: 'pnpm-lock.yaml' },
+ packageFile: 'some-file',
+ },
+ ]);
});
});
diff --git a/lib/modules/manager/npm/extract/locked-versions.ts b/lib/modules/manager/npm/extract/locked-versions.ts
index 6c3fa3e7a4..dde7884b90 100644
--- a/lib/modules/manager/npm/extract/locked-versions.ts
+++ b/lib/modules/manager/npm/extract/locked-versions.ts
@@ -3,6 +3,7 @@ import { logger } from '../../../../logger';
import type { PackageFile } from '../../types';
import type { NpmManagerData } from '../types';
import { getNpmLock } from './npm';
+import { getConstraints, getPnpmLock } from './pnpm';
import type { LockFile } from './types';
import { getYarnLock } from './yarn';
@@ -19,7 +20,7 @@ export async function getLockedVersions(
logger.trace('Found yarnLock');
lockFiles.push(yarnLock);
if (!lockFileCache[yarnLock]) {
- logger.trace('Retrieving/parsing ' + yarnLock);
+ logger.trace(`Retrieving/parsing ${yarnLock}`);
lockFileCache[yarnLock] = await getYarnLock(yarnLock);
}
const { lockfileVersion, isYarn1 } = lockFileCache[yarnLock];
@@ -87,8 +88,26 @@ export async function getLockedVersions(
)!;
}
} else if (pnpmShrinkwrap) {
- logger.debug('TODO: implement pnpm-lock.yaml parsing of lockVersion');
+ logger.debug('Found pnpm lock-file');
lockFiles.push(pnpmShrinkwrap);
+ if (!lockFileCache[pnpmShrinkwrap]) {
+ logger.trace(`Retrieving/parsing ${pnpmShrinkwrap}`);
+ lockFileCache[pnpmShrinkwrap] = await getPnpmLock(pnpmShrinkwrap);
+ }
+ const { lockfileVersion } = lockFileCache[pnpmShrinkwrap];
+ if (lockfileVersion) {
+ packageFile.extractedConstraints!.pnpm = getConstraints(
+ lockfileVersion,
+ packageFile.extractedConstraints!.pnpm
+ );
+ }
+
+ for (const dep of packageFile.deps) {
+ // TODO: types (#7154)
+ dep.lockedVersion = semver.valid(
+ lockFileCache[pnpmShrinkwrap].lockedVersions[dep.depName!]
+ )!;
+ }
}
if (lockFiles.length) {
packageFile.lockFiles = lockFiles;
diff --git a/lib/modules/manager/npm/extract/pnpm.spec.ts b/lib/modules/manager/npm/extract/pnpm.spec.ts
index af0a9fef89..6174595759 100644
--- a/lib/modules/manager/npm/extract/pnpm.spec.ts
+++ b/lib/modules/manager/npm/extract/pnpm.spec.ts
@@ -1,4 +1,5 @@
import yaml from 'js-yaml';
+import { Fixtures } from '../../../../../test/fixtures';
import { getFixturePath, logger } from '../../../../../test/util';
import { GlobalConfig } from '../../../../config/global';
import * as fs from '../../../../util/fs';
@@ -6,6 +7,8 @@ import {
detectPnpmWorkspaces,
extractPnpmFilters,
findPnpmWorkspace,
+ getConstraints,
+ getPnpmLock,
} from './pnpm';
describe('modules/manager/npm/extract/pnpm', () => {
@@ -227,4 +230,87 @@ describe('modules/manager/npm/extract/pnpm', () => {
).toBeUndefined();
});
});
+
+ describe('getConstraints()', () => {
+ // no constraints
+ it.each([
+ [6.0, undefined, '>=8'],
+ [5.4, undefined, '>=7 <8'],
+ [5.3, undefined, '>=6 <7'],
+ [5.2, undefined, '>=5.10.0 <6'],
+ [5.1, undefined, '>=3.5.0 <5.9.3'],
+ [5.0, undefined, '>=3 <3.5.0'],
+ ])('adds constraints for %f', (lockfileVersion, constraints, expected) => {
+ expect(getConstraints(lockfileVersion, constraints)).toBe(expected);
+ });
+
+ // constraints present
+ it.each([
+ [6.0, '>=8.2.0', '>=8.2.0'],
+ [6.0, '>=7', '>=7 >=8'],
+
+ [5.4, '^7.2.0', '^7.2.0'],
+ [5.4, '<7.2.0', '<7.2.0 >=7'],
+ [5.4, '>7.2.0', '>7.2.0 <8'],
+ [5.4, '>=6', '>=6 >=7 <8'],
+
+ [5.3, '^6.0.0', '^6.0.0'],
+ [5.3, '<6.2.0', '<6.2.0 >=6'],
+ [5.3, '>6.2.0', '>6.2.0 <7'],
+ [5.3, '>=5', '>=5 >=6 <7'],
+
+ [5.2, '5.10.0', '5.10.0'],
+ [5.2, '>5.0.0 <5.18.0', '>5.0.0 <5.18.0 >=5.10.0'],
+ [5.2, '>5.10.0', '>5.10.0 <6'],
+ [5.2, '>=5', '>=5 >=5.10.0 <6'],
+
+ [5.1, '^4.0.0', '^4.0.0'],
+ [5.1, '<4', '<4 >=3.5.0'],
+ [5.1, '>=4', '>=4 <5.9.3'],
+ [5.1, '>=3', '>=3 >=3.5.0 <5.9.3'],
+
+ [5.0, '3.1.0', '3.1.0'],
+ [5.0, '^3.0.0', '^3.0.0 <3.5.0'],
+ [5.0, '>=3', '>=3 <3.5.0'],
+ [5.0, '>=2', '>=2 >=3 <3.5.0'],
+ ])('adds constraints for %f', (lockfileVersion, constraints, expected) => {
+ expect(getConstraints(lockfileVersion, constraints)).toBe(expected);
+ });
+ });
+
+ describe('.getPnpmLock()', () => {
+ const readLocalFile = jest.spyOn(fs, 'readLocalFile');
+
+ it('returns empty if failed to parse', async () => {
+ readLocalFile.mockResolvedValueOnce(undefined as never);
+ const res = await getPnpmLock('package.json');
+ expect(Object.keys(res.lockedVersions)).toHaveLength(0);
+ });
+
+ it('extracts', async () => {
+ const plocktest1Lock = Fixtures.get('pnpm-monorepo/pnpm-lock.yaml', '..');
+ readLocalFile.mockResolvedValueOnce(plocktest1Lock);
+ const res = await getPnpmLock('package.json');
+ expect(Object.keys(res.lockedVersions)).toHaveLength(8);
+ });
+
+ it('logs when packagePath is invalid', async () => {
+ const plocktest1Lock = Fixtures.get(
+ 'lockfile-parsing/pnpm-lock.yaml',
+ '..'
+ );
+ readLocalFile.mockResolvedValueOnce(plocktest1Lock);
+ const res = await getPnpmLock('package.json');
+ expect(Object.keys(res.lockedVersions)).toHaveLength(2);
+ expect(logger.logger.trace).toHaveBeenLastCalledWith(
+ 'Invalid package path /sux-1.2.4'
+ );
+ });
+
+ it('returns empty if no deps', async () => {
+ readLocalFile.mockResolvedValueOnce('{}');
+ const res = await getPnpmLock('package.json');
+ expect(Object.keys(res.lockedVersions)).toHaveLength(0);
+ });
+ });
});
diff --git a/lib/modules/manager/npm/extract/pnpm.ts b/lib/modules/manager/npm/extract/pnpm.ts
index 1f566d1e8b..18e48b8185 100644
--- a/lib/modules/manager/npm/extract/pnpm.ts
+++ b/lib/modules/manager/npm/extract/pnpm.ts
@@ -1,6 +1,7 @@
import is from '@sindresorhus/is';
import { findPackages } from 'find-packages';
import { load } from 'js-yaml';
+import semver from 'semver';
import upath from 'upath';
import { GlobalConfig } from '../../../../config/global';
import { logger } from '../../../../logger';
@@ -10,9 +11,15 @@ import {
localPathExists,
readLocalFile,
} from '../../../../util/fs';
+import { regEx } from '../../../../util/regex';
import type { PackageFile } from '../../types';
+import type { PnpmLockFile } from '../post-update/types';
import type { NpmManagerData } from '../types';
-import type { PnpmWorkspaceFile } from './types';
+import type { LockFile, PnpmWorkspaceFile } from './types';
+
+function isPnpmLockfile(obj: any): obj is PnpmLockFile {
+ return is.plainObject(obj) && 'lockfileVersion' in obj;
+}
export async function extractPnpmFilters(
fileName: string
@@ -135,3 +142,150 @@ export async function detectPnpmWorkspaces(
}
}
}
+
+export async function getPnpmLock(filePath: string): Promise<LockFile> {
+ try {
+ const pnpmLockRaw = await readLocalFile(filePath, 'utf8');
+ if (!pnpmLockRaw) {
+ throw new Error('Unable to read pnpm-lock.yaml');
+ }
+
+ const lockParsed = load(pnpmLockRaw);
+ if (!isPnpmLockfile(lockParsed)) {
+ throw new Error('Invalid or empty lockfile');
+ }
+ logger.trace({ lockParsed }, 'pnpm lockfile parsed');
+
+ // field lockfileVersion is type string in lockfileVersion = 6 and type number in < 6
+ const lockfileVersion: number = is.number(lockParsed.lockfileVersion)
+ ? lockParsed.lockfileVersion
+ : parseFloat(lockParsed.lockfileVersion);
+
+ const lockedVersions: Record<string, string> = {};
+ const packagePathRegex = regEx(
+ /^\/(?<packageName>.+)(?:@|\/)(?<version>[^/@]+)$/
+ ); // eg. "/<packageName>(@|/)<version>"
+
+ for (const packagePath of Object.keys(lockParsed.packages ?? {})) {
+ const result = packagePath.match(packagePathRegex);
+ if (!result?.groups) {
+ logger.trace(`Invalid package path ${packagePath}`);
+ continue;
+ }
+
+ const packageName = result.groups.packageName;
+ const version = result.groups.version;
+ logger.trace({
+ packagePath,
+ packageName,
+ version,
+ });
+ lockedVersions[packageName] = version;
+ }
+ return {
+ lockedVersions,
+ lockfileVersion,
+ };
+ } catch (err) {
+ logger.debug({ filePath, err }, 'Warning: Exception parsing pnpm lockfile');
+ return { lockedVersions: {} };
+ }
+}
+
+export function getConstraints(
+ lockfileVersion: number,
+ constraints?: string
+): string {
+ let newConstraints = constraints;
+
+ // find matching lockfileVersion and use its constraints
+ // if no match found use lockfileVersion 5
+ // lockfileVersion 5 is the minimum version required to generate the pnpm-lock.yaml file
+ const { lowerBound, upperBound, lowerConstraint, upperConstraint } =
+ lockToPnpmVersionMapping.find(
+ (m) => m.lockfileVersion === lockfileVersion
+ ) ?? {
+ lockfileVersion: 5.0,
+ lowerBound: '2.24.0',
+ upperBound: '3.5.0',
+ lowerConstraint: '>=3',
+ upperConstraint: '<3.5.0',
+ };
+
+ // inorder to ensure that the constraint doesn't allow any pnpm versions that can't generate the extracted lockfileVersion
+ // compare the current constraint to the lowerBound and upperBound of the lockfileVersion
+ // if the current constraint is not comaptible, add the lowerConstraint and upperConstraint, whichever is needed
+ if (newConstraints) {
+ // if constraint satisfies versions lower than lowerBound add the lowerConstraint to narrow the range
+ if (semver.satisfies(lowerBound, newConstraints)) {
+ newConstraints += ` ${lowerConstraint}`;
+ }
+
+ // if constraint satisfies versions higher than upperBound add the upperConstraint to narrow the range
+ if (
+ upperBound &&
+ upperConstraint &&
+ semver.satisfies(upperBound, newConstraints)
+ ) {
+ newConstraints += ` ${upperConstraint}`;
+ }
+ }
+ // if no constraint is present, add the lowerConstraint and upperConstraint corresponding to the lockfileVersion
+ else {
+ newConstraints = `${lowerConstraint}${
+ upperConstraint ? ` ${upperConstraint}` : ''
+ }`;
+ }
+
+ return newConstraints;
+}
+
+/**
+ pnpm lockfiles have corresponding version numbers called "lockfileVersion"
+ each lockfileVersion can only be generated by a certain pnpm version ranges
+ eg. lockfileVersion: 5.4 can only be generated by pnpm version >=7 && <8
+ official list can be found here : https:github.com/pnpm/spec/tree/master/lockfile
+ we use the mapping present below to find the compatible pnpm version range for a given lockfileVersion
+
+ the various terms used in the mapping are explained below:
+ lowerConstriant : lowest pnpm version that can generate the lockfileVersion
+ upperConstraint : highest pnpm version that can generate the lockfileVersion
+ lowerBound : highest pnpm version that is less than the lowerConstraint
+ upperBound : lowest pnpm version that is greater than upperConstraint
+
+ For handling future lockfileVersions, we need to:
+ 1. add a upperBound and upperConstraint to the current lastest lockfileVersion
+ 2. add an object for the new lockfileVersion with lowerBound and lowerConstraint
+ */
+
+const lockToPnpmVersionMapping = [
+ { lockfileVersion: 6.0, lowerBound: '7.32.0', lowerConstraint: '>=8' },
+ {
+ lockfileVersion: 5.4,
+ lowerBound: '6.35.1',
+ upperBound: '8.0.0',
+ lowerConstraint: '>=7',
+ upperConstraint: '<8',
+ },
+ {
+ lockfileVersion: 5.3,
+ lowerBound: '5.18.10',
+ upperBound: '7.0.0',
+ lowerConstraint: '>=6',
+ upperConstraint: '<7',
+ },
+ {
+ lockfileVersion: 5.2,
+ lowerBound: '5.9.3',
+ upperBound: '5.18.10',
+ lowerConstraint: '>=5.10.0',
+ upperConstraint: '<6',
+ },
+ {
+ lockfileVersion: 5.1,
+ lowerBound: '3.4.1',
+ upperBound: '5.9.3',
+ lowerConstraint: '>=3.5.0',
+ upperConstraint: '<5.9.3',
+ },
+];
diff --git a/lib/modules/manager/npm/post-update/types.ts b/lib/modules/manager/npm/post-update/types.ts
index 56b50b78cc..a134a951e7 100644
--- a/lib/modules/manager/npm/post-update/types.ts
+++ b/lib/modules/manager/npm/post-update/types.ts
@@ -31,7 +31,8 @@ export interface GenerateLockFileResult {
}
export interface PnpmLockFile {
- lockfileVersion?: number;
+ lockfileVersion: number | string;
+ packages?: Record<string, unknown>;
}
export interface YarnRcYmlFile {
--
GitLab