From 7118404981e3e0b7c7d9893cde80c7102fc8e60e Mon Sep 17 00:00:00 2001 From: Rhys Arkins <rhys@arkins.net> Date: Sat, 20 Feb 2021 14:22:50 +0100 Subject: [PATCH] feat: matchFiles + lockFiles (#8783) --- docs/usage/configuration-options.md | 2 +- .../__snapshots__/extract.spec.ts.snap | 21 ++++ lib/manager/bundler/extract.ts | 1 + lib/manager/cargo/extract.ts | 10 +- lib/manager/cocoapods/extract.spec.ts | 9 +- lib/manager/cocoapods/extract.ts | 15 ++- lib/manager/common.ts | 1 + .../__snapshots__/extract.spec.ts.snap | 6 + lib/manager/composer/extract.ts | 1 + lib/manager/helmv3/extract.spec.ts | 44 +++---- lib/manager/helmv3/extract.ts | 12 +- .../mix/__snapshots__/extract.spec.ts.snap | 118 +++++++++--------- lib/manager/mix/extract.spec.ts | 10 +- lib/manager/mix/extract.ts | 14 ++- .../locked-versions.spec.ts.snap | 21 ++++ lib/manager/npm/extract/locked-versions.ts | 7 ++ lib/manager/nuget/extract.ts | 10 +- .../pipenv/__snapshots__/extract.spec.ts.snap | 12 ++ lib/manager/pipenv/extract.spec.ts | 68 +++++----- lib/manager/pipenv/extract.ts | 11 +- lib/manager/poetry/extract.ts | 21 +++- lib/util/cache/repository/index.ts | 2 +- lib/util/package-rules.spec.ts | 14 +++ lib/util/package-rules.ts | 5 +- 24 files changed, 298 insertions(+), 137 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 9c2d3b0589..54588ccecd 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -1288,7 +1288,7 @@ Use the syntax `!/ /` like the following: ### matchFiles -Renovate will compare `matchFiles` for an exact match against the dependency's package file. +Renovate will compare `matchFiles` for an exact match against the dependency's package file or lock file. For example the following would match `package.json` but not `package/frontend/package.json`: diff --git a/lib/manager/bundler/__snapshots__/extract.spec.ts.snap b/lib/manager/bundler/__snapshots__/extract.spec.ts.snap index 8390b3f01d..d7fbf03b7a 100644 --- a/lib/manager/bundler/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/bundler/__snapshots__/extract.spec.ts.snap @@ -143,6 +143,9 @@ Object { "skipReason": "no-version", }, ], + "lockFiles": Array [ + "Gemfile.lock", + ], "registryUrls": Array [ "https://rubygems.org", ], @@ -1379,6 +1382,9 @@ Object { "skipReason": "no-version", }, ], + "lockFiles": Array [ + "Gemfile.lock", + ], "registryUrls": Array [ "https://rubygems.org", ], @@ -1440,6 +1446,9 @@ Object { "skipReason": "no-version", }, ], + "lockFiles": Array [ + "Gemfile.lock", + ], "registryUrls": Array [ "https://rubygems.org", ], @@ -2138,6 +2147,9 @@ Object { }, }, ], + "lockFiles": Array [ + "Gemfile.lock", + ], "registryUrls": Array [ "https://rubygems.org", ], @@ -4684,6 +4696,9 @@ Object { }, }, ], + "lockFiles": Array [ + "Gemfile.lock", + ], "registryUrls": Array [ "https://rubygems.org", ], @@ -4716,6 +4731,9 @@ Object { ], }, ], + "lockFiles": Array [ + "Gemfile.lock", + ], "registryUrls": Array [], } `; @@ -4749,6 +4767,9 @@ Object { "skipReason": "no-version", }, ], + "lockFiles": Array [ + "Gemfile.lock", + ], "registryUrls": Array [], } `; diff --git a/lib/manager/bundler/extract.ts b/lib/manager/bundler/extract.ts index 1d9e4308ce..649fb97696 100644 --- a/lib/manager/bundler/extract.ts +++ b/lib/manager/bundler/extract.ts @@ -185,6 +185,7 @@ export async function extractPackageFile( const lockContent = await readLocalFile(gemfileLock, 'utf8'); if (lockContent) { logger.debug({ packageFile: fileName }, 'Found Gemfile.lock file'); + res.lockFiles = [gemfileLock]; const lockedEntries = extractLockFileEntries(lockContent); for (const dep of res.deps) { const lockedDepValue = lockedEntries.get(dep.depName); diff --git a/lib/manager/cargo/extract.ts b/lib/manager/cargo/extract.ts index 052858d717..890a411e20 100644 --- a/lib/manager/cargo/extract.ts +++ b/lib/manager/cargo/extract.ts @@ -2,7 +2,7 @@ import { parse } from '@iarna/toml'; import * as datasourceCrate from '../../datasource/crate'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; -import { readLocalFile } from '../../util/fs'; +import { findLocalSiblingOrParent, readLocalFile } from '../../util/fs'; import { ExtractConfig, PackageDependency, PackageFile } from '../common'; import { CargoConfig, @@ -186,5 +186,11 @@ export async function extractPackageFile( if (!deps.length) { return null; } - return { deps }; + const lockFileName = await findLocalSiblingOrParent(fileName, 'Cargo.lock'); + const res: PackageFile = { deps }; + // istanbul ignore if + if (lockFileName) { + res.lockFiles = [lockFileName]; + } + return res; } diff --git a/lib/manager/cocoapods/extract.spec.ts b/lib/manager/cocoapods/extract.spec.ts index 0a23a936eb..f61af71467 100644 --- a/lib/manager/cocoapods/extract.spec.ts +++ b/lib/manager/cocoapods/extract.spec.ts @@ -15,11 +15,14 @@ const complexPodfile = fs.readFileSync( describe('lib/manager/cocoapods/extract', () => { describe('extractPackageFile()', () => { - it('extracts all dependencies', () => { - const simpleResult = extractPackageFile(simplePodfile).deps; + it('extracts all dependencies', async () => { + const simpleResult = (await extractPackageFile(simplePodfile, 'Podfile')) + .deps; expect(simpleResult).toMatchSnapshot(); - const complexResult = extractPackageFile(complexPodfile).deps; + const complexResult = ( + await extractPackageFile(complexPodfile, 'Podfile') + ).deps; expect(complexResult).toMatchSnapshot(); }); }); diff --git a/lib/manager/cocoapods/extract.ts b/lib/manager/cocoapods/extract.ts index 3db79255bb..36e9abf2a7 100644 --- a/lib/manager/cocoapods/extract.ts +++ b/lib/manager/cocoapods/extract.ts @@ -2,6 +2,7 @@ import * as datasourceGithubTags from '../../datasource/github-tags'; import * as datasourcePod from '../../datasource/pod'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; +import { getSiblingFileName, localPathExists } from '../../util/fs'; import { PackageDependency, PackageFile } from '../common'; const regexMappings = [ @@ -75,7 +76,10 @@ export function gitDep(parsedLine: ParsedLine): PackageDependency | null { return null; } -export function extractPackageFile(content: string): PackageFile | null { +export async function extractPackageFile( + content: string, + fileName: string +): Promise<PackageFile | null> { logger.trace('cocoapods.extractPackageFile()'); const deps: PackageDependency[] = []; const lines: string[] = content.split('\n'); @@ -137,6 +141,11 @@ export function extractPackageFile(content: string): PackageFile | null { deps.push(dep); } } - - return deps.length ? { deps } : null; + const res: PackageFile = { deps }; + const lockFile = getSiblingFileName(fileName, 'Podfile.lock'); + // istanbul ignore if + if (await localPathExists(lockFile)) { + res.lockFiles = [lockFile]; + } + return res; } diff --git a/lib/manager/common.ts b/lib/manager/common.ts index d937b48c82..79ca23b90d 100644 --- a/lib/manager/common.ts +++ b/lib/manager/common.ts @@ -74,6 +74,7 @@ export interface NpmLockFiles { pnpmShrinkwrap?: string; npmLock?: string; lernaDir?: string; + lockFiles?: string[]; } export interface PackageFile<T = Record<string, any>> diff --git a/lib/manager/composer/__snapshots__/extract.spec.ts.snap b/lib/manager/composer/__snapshots__/extract.spec.ts.snap index e6dc9b02d2..2f6ef4ba87 100644 --- a/lib/manager/composer/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/composer/__snapshots__/extract.spec.ts.snap @@ -208,6 +208,9 @@ Object { "depType": "require-dev", }, ], + "lockFiles": Array [ + "composer.lock", + ], } `; @@ -557,6 +560,9 @@ Object { "lookupName": "git@my-git.example:my-git-repo", }, ], + "lockFiles": Array [ + "composer.lock", + ], "registryUrls": Array [ "https://wpackagist.org", "https://packagist.org", diff --git a/lib/manager/composer/extract.ts b/lib/manager/composer/extract.ts index 302eb427dc..c678e957d6 100644 --- a/lib/manager/composer/extract.ts +++ b/lib/manager/composer/extract.ts @@ -95,6 +95,7 @@ export async function extractPackageFile( let lockParsed: ComposerLock; if (lockContents) { logger.debug({ packageFile: fileName }, 'Found composer lock file'); + res.lockFiles = [lockfilePath]; try { lockParsed = JSON.parse(lockContents) as ComposerLock; } catch (err) /* istanbul ignore next */ { diff --git a/lib/manager/helmv3/extract.spec.ts b/lib/manager/helmv3/extract.spec.ts index 769f61408c..c4e402ead3 100644 --- a/lib/manager/helmv3/extract.spec.ts +++ b/lib/manager/helmv3/extract.spec.ts @@ -9,7 +9,7 @@ describe('lib/manager/helm-requirements/extract', () => { jest.resetAllMocks(); fs.readLocalFile = jest.fn(); }); - it('skips invalid registry urls', () => { + it('skips invalid registry urls', async () => { const content = ` apiVersion: v2 appVersion: "1.0" @@ -27,7 +27,7 @@ describe('lib/manager/helm-requirements/extract', () => { version: 0.8.1 `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, @@ -36,7 +36,7 @@ describe('lib/manager/helm-requirements/extract', () => { expect(result).toMatchSnapshot(); expect(result.deps.every((dep) => dep.skipReason)).toEqual(true); }); - it('parses simple Chart.yaml correctly', () => { + it('parses simple Chart.yaml correctly', async () => { const content = ` apiVersion: v2 appVersion: "1.0" @@ -54,7 +54,7 @@ describe('lib/manager/helm-requirements/extract', () => { condition: postgresql.enabled `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, @@ -62,7 +62,7 @@ describe('lib/manager/helm-requirements/extract', () => { expect(result).not.toBeNull(); expect(result).toMatchSnapshot(); }); - it('resolves aliased registry urls', () => { + it('resolves aliased registry urls', async () => { const content = ` apiVersion: v2 appVersion: "1.0" @@ -75,7 +75,7 @@ describe('lib/manager/helm-requirements/extract', () => { repository: '@placeholder' `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { placeholder: 'https://my-registry.gcr.io/', }, @@ -84,21 +84,21 @@ describe('lib/manager/helm-requirements/extract', () => { expect(result).toMatchSnapshot(); expect(result.deps.every((dep) => dep.skipReason)).toEqual(false); }); - it("doesn't fail if Chart.yaml is invalid", () => { + it("doesn't fail if Chart.yaml is invalid", async () => { const content = ` Invalid Chart.yaml content. arr: [ `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); }); - it('skips local dependencies', () => { + it('skips local dependencies', async () => { const content = ` apiVersion: v2 appVersion: "1.0" @@ -114,7 +114,7 @@ describe('lib/manager/helm-requirements/extract', () => { repository: file:///some/local/path/ `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, @@ -122,7 +122,7 @@ describe('lib/manager/helm-requirements/extract', () => { expect(result).not.toBeNull(); expect(result).toMatchSnapshot(); }); - it('returns null if no dependencies key', () => { + it('returns null if no dependencies key', async () => { fs.readLocalFile.mockResolvedValueOnce(` `); const content = ` @@ -134,14 +134,14 @@ describe('lib/manager/helm-requirements/extract', () => { hello: world `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); }); - it('returns null if dependencies are an empty list', () => { + it('returns null if dependencies are an empty list', async () => { fs.readLocalFile.mockResolvedValueOnce(` `); const content = ` @@ -153,14 +153,14 @@ describe('lib/manager/helm-requirements/extract', () => { dependencies: [] `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); }); - it('returns null if dependencies key is invalid', () => { + it('returns null if dependencies key is invalid', async () => { const content = ` apiVersion: v2 appVersion: "1.0" @@ -172,24 +172,24 @@ describe('lib/manager/helm-requirements/extract', () => { [ `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); }); - it('returns null if Chart.yaml is empty', () => { + it('returns null if Chart.yaml is empty', async () => { const content = ''; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); }); - it('returns null if Chart.yaml uses an unsupported apiVersion', () => { + it('returns null if Chart.yaml uses an unsupported apiVersion', async () => { const content = ` apiVersion: v1 appVersion: "1.0" @@ -198,14 +198,14 @@ describe('lib/manager/helm-requirements/extract', () => { version: 0.1.0 `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); }); - it('returns null if name and version are missing for all dependencies', () => { + it('returns null if name and version are missing for all dependencies', async () => { const content = ` apiVersion: v2 appVersion: "1.0" @@ -218,7 +218,7 @@ describe('lib/manager/helm-requirements/extract', () => { alias: "test" `; const fileName = 'Chart.yaml'; - const result = extractPackageFile(content, fileName, { + const result = await extractPackageFile(content, fileName, { aliases: { stable: 'https://charts.helm.sh/stable', }, diff --git a/lib/manager/helmv3/extract.ts b/lib/manager/helmv3/extract.ts index 4f5c270998..0f08dc3815 100644 --- a/lib/manager/helmv3/extract.ts +++ b/lib/manager/helmv3/extract.ts @@ -3,13 +3,14 @@ import yaml from 'js-yaml'; import * as datasourceHelm from '../../datasource/helm'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; +import { getSiblingFileName, localPathExists } from '../../util/fs'; import { ExtractConfig, PackageDependency, PackageFile } from '../common'; -export function extractPackageFile( +export async function extractPackageFile( content: string, fileName: string, config: ExtractConfig -): PackageFile | null { +): Promise<PackageFile | null> { let chart: { apiVersion: string; name: string; @@ -82,10 +83,15 @@ export function extractPackageFile( } return res; }); - const res = { + const res: PackageFile = { deps, datasource: datasourceHelm.id, packageFileVersion, }; + const lockFileName = getSiblingFileName(fileName, 'Chart.lock'); + // istanbul ignore if + if (await localPathExists(lockFileName)) { + res.lockFiles = [lockFileName]; + } return res; } diff --git a/lib/manager/mix/__snapshots__/extract.spec.ts.snap b/lib/manager/mix/__snapshots__/extract.spec.ts.snap index ea0aecc3cf..6f84c81f02 100644 --- a/lib/manager/mix/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/mix/__snapshots__/extract.spec.ts.snap @@ -1,71 +1,73 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`lib/manager/mix/extract extractPackageFile() extracts all dependencies 1`] = ` -Array [ - Object { - "currentValue": "~> 0.8.1", - "datasource": "hex", - "depName": "postgrex", - "lookupName": "postgrex", - "managerData": Object { - "lineNumber": 18, +Object { + "deps": Array [ + Object { + "currentValue": "~> 0.8.1", + "datasource": "hex", + "depName": "postgrex", + "lookupName": "postgrex", + "managerData": Object { + "lineNumber": 18, + }, }, - }, - Object { - "currentValue": ">2.1.0 or <=3.0.0", - "datasource": "hex", - "depName": "ecto", - "lookupName": "ecto", - "managerData": Object { - "lineNumber": 19, + Object { + "currentValue": ">2.1.0 or <=3.0.0", + "datasource": "hex", + "depName": "ecto", + "lookupName": "ecto", + "managerData": Object { + "lineNumber": 19, + }, }, - }, - Object { - "currentValue": "ninenines/cowboy", - "datasource": "github", - "depName": "cowboy", - "managerData": Object { - "lineNumber": 20, + Object { + "currentValue": "ninenines/cowboy", + "datasource": "github", + "depName": "cowboy", + "managerData": Object { + "lineNumber": 20, + }, + "skipReason": "non-hex depTypes", }, - "skipReason": "non-hex depTypes", - }, - Object { - "currentValue": "~> 1.0", - "datasource": "hex", - "depName": "secret", - "lookupName": "secret:acme", - "managerData": Object { - "lineNumber": 21, + Object { + "currentValue": "~> 1.0", + "datasource": "hex", + "depName": "secret", + "lookupName": "secret:acme", + "managerData": Object { + "lineNumber": 21, + }, }, - }, - Object { - "currentValue": ">2.1.0 and <=3.0.0", - "datasource": "hex", - "depName": "ex_doc", - "lookupName": "ex_doc", - "managerData": Object { - "lineNumber": 22, + Object { + "currentValue": ">2.1.0 and <=3.0.0", + "datasource": "hex", + "depName": "ex_doc", + "lookupName": "ex_doc", + "managerData": Object { + "lineNumber": 22, + }, }, - }, - Object { - "currentValue": ">= 1.0.0", - "datasource": "hex", - "depName": "jason", - "lookupName": "jason", - "managerData": Object { - "lineNumber": 24, + Object { + "currentValue": ">= 1.0.0", + "datasource": "hex", + "depName": "jason", + "lookupName": "jason", + "managerData": Object { + "lineNumber": 24, + }, }, - }, - Object { - "currentValue": "~> 1.0", - "datasource": "hex", - "depName": "jason", - "lookupName": "jason", - "managerData": Object { - "lineNumber": 24, + Object { + "currentValue": "~> 1.0", + "datasource": "hex", + "depName": "jason", + "lookupName": "jason", + "managerData": Object { + "lineNumber": 24, + }, }, - }, -] + ], +} `; exports[`lib/manager/mix/extract extractPackageFile() returns empty for invalid dependency file 1`] = ` diff --git a/lib/manager/mix/extract.spec.ts b/lib/manager/mix/extract.spec.ts index e5f4d3562f..3c5c0e6f2e 100644 --- a/lib/manager/mix/extract.spec.ts +++ b/lib/manager/mix/extract.spec.ts @@ -9,11 +9,13 @@ const sample = fs.readFileSync( describe('lib/manager/mix/extract', () => { describe('extractPackageFile()', () => { - it('returns empty for invalid dependency file', () => { - expect(extractPackageFile('nothing here')).toMatchSnapshot(); + it('returns empty for invalid dependency file', async () => { + expect( + await extractPackageFile('nothing here', 'mix.exs') + ).toMatchSnapshot(); }); - it('extracts all dependencies', () => { - const res = extractPackageFile(sample).deps; + it('extracts all dependencies', async () => { + const res = await extractPackageFile(sample, 'mix.exs'); expect(res).toMatchSnapshot(); }); }); diff --git a/lib/manager/mix/extract.ts b/lib/manager/mix/extract.ts index 0026185d73..e2b07de4aa 100644 --- a/lib/manager/mix/extract.ts +++ b/lib/manager/mix/extract.ts @@ -1,12 +1,16 @@ import * as datasourceHex from '../../datasource/hex'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; +import { getSiblingFileName, localPathExists } from '../../util/fs'; import { PackageDependency, PackageFile } from '../common'; const depSectionRegExp = /defp\s+deps.*do/g; const depMatchRegExp = /{:(\w+),\s*([^:"]+)?:?\s*"([^"]+)",?\s*(organization: "(.*)")?.*}/gm; -export function extractPackageFile(content: string): PackageFile { +export async function extractPackageFile( + content: string, + fileName: string +): Promise<PackageFile | null> { logger.trace('mix.extractPackageFile()'); const deps: PackageDependency[] = []; const contentArr = content.split('\n'); @@ -61,5 +65,11 @@ export function extractPackageFile(content: string): PackageFile { } while (depMatch); } } - return { deps }; + const res: PackageFile = { deps }; + const lockFileName = getSiblingFileName(fileName, 'mix.lock'); + // istanbul ignore if + if (await localPathExists(lockFileName)) { + res.lockFiles = [lockFileName]; + } + return res; } diff --git a/lib/manager/npm/extract/__snapshots__/locked-versions.spec.ts.snap b/lib/manager/npm/extract/__snapshots__/locked-versions.spec.ts.snap index 70edc75550..c9f8b48e2b 100644 --- a/lib/manager/npm/extract/__snapshots__/locked-versions.spec.ts.snap +++ b/lib/manager/npm/extract/__snapshots__/locked-versions.spec.ts.snap @@ -18,6 +18,9 @@ Array [ "lockedVersion": "2.0.0", }, ], + "lockFiles": Array [ + "package-lock.json", + ], "npmLock": "package-lock.json", }, ] @@ -36,6 +39,9 @@ Array [ "depName": "b", }, ], + "lockFiles": Array [ + "pnpm-lock.yaml", + ], "pnpmShrinkwrap": "pnpm-lock.yaml", }, ] @@ -59,6 +65,9 @@ Array [ "lockedVersion": "2.0.0", }, ], + "lockFiles": Array [ + "package-lock.json", + ], "npmLock": "package-lock.json", }, ] @@ -80,6 +89,9 @@ Array [ "lockedVersion": "2.0.0", }, ], + "lockFiles": Array [ + "package-lock.json", + ], "npmLock": "package-lock.json", }, ] @@ -101,6 +113,9 @@ Array [ "lockedVersion": "2.0.0", }, ], + "lockFiles": Array [ + "yarn.lock", + ], "npmLock": "package-lock.json", "yarnLock": "yarn.lock", }, @@ -125,6 +140,9 @@ Array [ "lockedVersion": "2.0.0", }, ], + "lockFiles": Array [ + "yarn.lock", + ], "npmLock": "package-lock.json", "yarnLock": "yarn.lock", }, @@ -149,6 +167,9 @@ Array [ "lockedVersion": "2.0.0", }, ], + "lockFiles": Array [ + "yarn.lock", + ], "npmLock": "package-lock.json", "yarnLock": "yarn.lock", }, diff --git a/lib/manager/npm/extract/locked-versions.ts b/lib/manager/npm/extract/locked-versions.ts index 348a826640..8653ade7a6 100644 --- a/lib/manager/npm/extract/locked-versions.ts +++ b/lib/manager/npm/extract/locked-versions.ts @@ -12,8 +12,10 @@ export async function getLockedVersions( logger.debug('Finding locked versions'); for (const packageFile of packageFiles) { const { yarnLock, npmLock, pnpmShrinkwrap } = packageFile; + const lockFiles = []; if (yarnLock) { logger.trace('Found yarnLock'); + lockFiles.push(yarnLock); if (!lockFileCache[yarnLock]) { logger.trace('Retrieving/parsing ' + yarnLock); lockFileCache[yarnLock] = await getYarnLock(yarnLock); @@ -35,6 +37,7 @@ export async function getLockedVersions( } } else if (npmLock) { logger.debug('Found ' + npmLock + ' for ' + packageFile.packageFile); + lockFiles.push(npmLock); if (!lockFileCache[npmLock]) { logger.trace('Retrieving/parsing ' + npmLock); lockFileCache[npmLock] = await getNpmLock(npmLock); @@ -54,6 +57,10 @@ export async function getLockedVersions( } } else if (pnpmShrinkwrap) { logger.debug('TODO: implement pnpm-lock.yaml parsing of lockVersion'); + lockFiles.push(pnpmShrinkwrap); + } + if (lockFiles.length) { + packageFile.lockFiles = lockFiles; } } } diff --git a/lib/manager/nuget/extract.ts b/lib/manager/nuget/extract.ts index aa07afccff..a035085a71 100644 --- a/lib/manager/nuget/extract.ts +++ b/lib/manager/nuget/extract.ts @@ -1,6 +1,7 @@ import { XmlDocument } from 'xmldoc'; import * as datasourceNuget from '../../datasource/nuget'; import { logger } from '../../logger'; +import { getSiblingFileName, localPathExists } from '../../util/fs'; import { ExtractConfig, PackageDependency, PackageFile } from '../common'; import { DotnetToolsManifest } from './types'; import { getConfiguredRegistries } from './util'; @@ -109,9 +110,14 @@ export async function extractPackageFile( ...dep, ...(registryUrls && { registryUrls }), })); - return { deps }; } catch (err) { logger.debug({ err }, `Failed to parse ${packageFile}`); } - return { deps }; + const res: PackageFile = { deps }; + const lockFileName = getSiblingFileName(packageFile, 'packages.lock.json'); + // istanbul ignore if + if (await localPathExists(lockFileName)) { + res.lockFiles = [lockFileName]; + } + return res; } diff --git a/lib/manager/pipenv/__snapshots__/extract.spec.ts.snap b/lib/manager/pipenv/__snapshots__/extract.spec.ts.snap index 5148a4d98e..8646936a9b 100644 --- a/lib/manager/pipenv/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/pipenv/__snapshots__/extract.spec.ts.snap @@ -51,6 +51,9 @@ Object { "managerData": Object {}, }, ], + "lockFiles": Array [ + "Pipfile.lock", + ], "registryUrls": Array [ "https://pypi.org/simple", "http://example.com/private-pypi/", @@ -124,6 +127,9 @@ Object { }, }, ], + "lockFiles": Array [ + "Pipfile.lock", + ], "registryUrls": Array [ "https://pypi.python.org/simple", ], @@ -172,6 +178,9 @@ Object { "managerData": Object {}, }, ], + "lockFiles": Array [ + "Pipfile.lock", + ], "registryUrls": Array [ "https://pypi.org/simple", ], @@ -195,6 +204,9 @@ Object { ], }, ], + "lockFiles": Array [ + "Pipfile.lock", + ], "registryUrls": Array [ "https://pypi.python.org/simple", "https://testpypi.python.org/pypi", diff --git a/lib/manager/pipenv/extract.spec.ts b/lib/manager/pipenv/extract.spec.ts index 86879eb34e..063eaa3e93 100644 --- a/lib/manager/pipenv/extract.spec.ts +++ b/lib/manager/pipenv/extract.spec.ts @@ -1,6 +1,9 @@ import fs from 'fs'; +import { fs as fsutil } from '../../../test/util'; import { extractPackageFile } from './extract'; +jest.mock('../../util/fs'); + const pipfile1 = fs.readFileSync( 'lib/manager/pipenv/__fixtures__/Pipfile1', 'utf8' @@ -24,93 +27,94 @@ const pipfile5 = fs.readFileSync( describe('lib/manager/pipenv/extract', () => { describe('extractPackageFile()', () => { - it('returns null for empty', () => { - expect(extractPackageFile('[packages]\r\n')).toBeNull(); + it('returns null for empty', async () => { + expect(await extractPackageFile('[packages]\r\n', 'Pipfile')).toBeNull(); }); - it('returns null for invalid toml file', () => { - expect(extractPackageFile('nothing here')).toBeNull(); + it('returns null for invalid toml file', async () => { + expect(await extractPackageFile('nothing here', 'Pipfile')).toBeNull(); }); - it('extracts dependencies', () => { - const res = extractPackageFile(pipfile1); + it('extracts dependencies', async () => { + fsutil.localPathExists.mockResolvedValue(true); + const res = await extractPackageFile(pipfile1, 'Pipfile'); expect(res).toMatchSnapshot(); expect(res.deps).toHaveLength(6); expect(res.deps.filter((dep) => !dep.skipReason)).toHaveLength(4); }); - it('marks packages with "extras" as skipReason === any-version', () => { - const res = extractPackageFile(pipfile3); + it('marks packages with "extras" as skipReason === any-version', async () => { + const res = await extractPackageFile(pipfile3, 'Pipfile'); expect(res.deps.filter((r) => !r.skipReason)).toHaveLength(0); expect(res.deps.filter((r) => r.skipReason)).toHaveLength(6); }); - it('extracts multiple dependencies', () => { - const res = extractPackageFile(pipfile2); + it('extracts multiple dependencies', async () => { + const res = await extractPackageFile(pipfile2, 'Pipfile'); expect(res).toMatchSnapshot(); expect(res.deps).toHaveLength(5); }); - it('ignores git dependencies', () => { + it('ignores git dependencies', async () => { const content = '[packages]\r\nflask = {git = "https://github.com/pallets/flask.git"}\r\nwerkzeug = ">=0.14"'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.deps.filter((r) => !r.skipReason)).toHaveLength(1); }); - it('ignores invalid package names', () => { + it('ignores invalid package names', async () => { const content = '[packages]\r\nfoo = "==1.0.0"\r\n_invalid = "==1.0.0"'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.deps).toHaveLength(2); expect(res.deps.filter((dep) => !dep.skipReason)).toHaveLength(1); }); - it('ignores relative path dependencies', () => { + it('ignores relative path dependencies', async () => { const content = '[packages]\r\nfoo = "==1.0.0"\r\ntest = {path = "."}'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.deps.filter((r) => !r.skipReason)).toHaveLength(1); }); - it('ignores invalid versions', () => { + it('ignores invalid versions', async () => { const content = '[packages]\r\nfoo = "==1.0.0"\r\nsome-package = "==0 0"'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.deps).toHaveLength(2); expect(res.deps.filter((dep) => !dep.skipReason)).toHaveLength(1); }); - it('extracts all sources', () => { + it('extracts all sources', async () => { const content = '[[source]]\r\nurl = "source-url"\r\n' + '[[source]]\r\nurl = "other-source-url"\r\n' + '[packages]\r\nfoo = "==1.0.0"\r\n'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.registryUrls).toEqual(['source-url', 'other-source-url']); }); - it('extracts example pipfile', () => { - const res = extractPackageFile(pipfile4); + it('extracts example pipfile', async () => { + const res = await extractPackageFile(pipfile4, 'Pipfile'); expect(res).toMatchSnapshot(); }); - it('supports custom index', () => { - const res = extractPackageFile(pipfile5); + it('supports custom index', async () => { + const res = await extractPackageFile(pipfile5, 'Pipfile'); expect(res).toMatchSnapshot(); expect(res.registryUrls).toBeDefined(); expect(res.registryUrls).toHaveLength(2); expect(res.deps[0].registryUrls).toBeDefined(); expect(res.deps[0].registryUrls).toHaveLength(1); }); - it('gets python constraint from python_version', () => { + it('gets python constraint from python_version', async () => { const content = '[packages]\r\nfoo = "==1.0.0"\r\n' + '[requires]\r\npython_version = "3.8"'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.constraints.python).toEqual('== 3.8.*'); }); - it('gets python constraint from python_full_version', () => { + it('gets python constraint from python_full_version', async () => { const content = '[packages]\r\nfoo = "==1.0.0"\r\n' + '[requires]\r\npython_full_version = "3.8.6"'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.constraints.python).toEqual('== 3.8.6'); }); - it('gets pipenv constraint from packages', () => { + it('gets pipenv constraint from packages', async () => { const content = '[packages]\r\npipenv = "==2020.8.13"'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.constraints.pipenv).toEqual('==2020.8.13'); }); - it('gets pipenv constraint from dev-packages', () => { + it('gets pipenv constraint from dev-packages', async () => { const content = '[dev-packages]\r\npipenv = "==2020.8.13"'; - const res = extractPackageFile(content); + const res = await extractPackageFile(content, 'Pipfile'); expect(res.constraints.pipenv).toEqual('==2020.8.13'); }); }); diff --git a/lib/manager/pipenv/extract.ts b/lib/manager/pipenv/extract.ts index 3754a38889..b08c528dc1 100644 --- a/lib/manager/pipenv/extract.ts +++ b/lib/manager/pipenv/extract.ts @@ -4,6 +4,7 @@ import is from '@sindresorhus/is'; import * as datasourcePypi from '../../datasource/pypi'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; +import { localPathExists } from '../../util/fs'; import { PackageDependency, PackageFile } from '../common'; // based on https://www.python.org/dev/peps/pep-0508/#names @@ -117,7 +118,10 @@ function extractFromSection( return deps; } -export function extractPackageFile(content: string): PackageFile | null { +export async function extractPackageFile( + content: string, + fileName: string +): Promise<PackageFile | null> { logger.debug('pipenv.extractPackageFile()'); let pipfile: PipFile; @@ -155,6 +159,11 @@ export function extractPackageFile(content: string): PackageFile | null { constraints.pipenv = pipfile['dev-packages'].pipenv; } + const lockFileName = fileName + '.lock'; + if (await localPathExists(lockFileName)) { + res.lockFiles = [lockFileName]; + } + res.constraints = constraints; return res; } diff --git a/lib/manager/poetry/extract.ts b/lib/manager/poetry/extract.ts index 5b17c06e04..dbd2efa4e5 100644 --- a/lib/manager/poetry/extract.ts +++ b/lib/manager/poetry/extract.ts @@ -3,7 +3,11 @@ import is from '@sindresorhus/is'; import * as datasourcePypi from '../../datasource/pypi'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; -import { getSiblingFileName, readLocalFile } from '../../util/fs'; +import { + getSiblingFileName, + localPathExists, + readLocalFile, +} from '../../util/fs'; import * as pep440Versioning from '../../versioning/pep440'; import * as poetryVersioning from '../../versioning/poetry'; import { PackageDependency, PackageFile } from '../common'; @@ -156,9 +160,22 @@ export async function extractPackageFile( constraints.python = pyprojectfile.tool?.poetry?.dependencies?.python; } - return { + const res: PackageFile = { deps, registryUrls: extractRegistries(pyprojectfile), constraints, }; + // Try poetry.lock first + let lockFile = getSiblingFileName(fileName, 'poetry.lock'); + // istanbul ignore next + if (await localPathExists(lockFile)) { + res.lockFiles = [lockFile]; + } else { + // Try pyproject.lock next + lockFile = getSiblingFileName(fileName, 'pyproject.lock'); + if (await localPathExists(lockFile)) { + res.lockFiles = [lockFile]; + } + } + return res; } diff --git a/lib/util/cache/repository/index.ts b/lib/util/cache/repository/index.ts index c84643901f..ff342a06a4 100644 --- a/lib/util/cache/repository/index.ts +++ b/lib/util/cache/repository/index.ts @@ -6,7 +6,7 @@ import { PackageFile } from '../../../manager/common'; import { RepoInitConfig } from '../../../workers/repository/init/common'; // Increment this whenever there could be incompatibilities between old and new cache structure -export const CACHE_REVISION = 1; +export const CACHE_REVISION = 2; export interface BaseBranchCache { sha: string; // branch commit sha diff --git a/lib/util/package-rules.spec.ts b/lib/util/package-rules.spec.ts index 87e180806f..258388085f 100644 --- a/lib/util/package-rules.spec.ts +++ b/lib/util/package-rules.spec.ts @@ -627,6 +627,20 @@ describe('applyPackageRules()', () => { }); expect(res2.x).toBeDefined(); }); + it('matches lock files', () => { + const config: TestConfig = { + packageFile: 'examples/foo/package.json', + lockFiles: ['yarn.lock'], + packageRules: [ + { + matchFiles: ['yarn.lock'], + x: 1, + }, + ], + }; + const res = applyPackageRules(config); + expect(res.x).toBeDefined(); + }); it('matches paths', () => { const config: TestConfig = { packageFile: 'examples/foo/package.json', diff --git a/lib/util/package-rules.ts b/lib/util/package-rules.ts index c5e33291f5..2307452978 100644 --- a/lib/util/package-rules.ts +++ b/lib/util/package-rules.ts @@ -28,6 +28,7 @@ function matchesRule(inputConfig: Config, packageRule: PackageRule): boolean { const { versioning, packageFile, + lockFiles, depType, depTypes, depName, @@ -66,7 +67,9 @@ function matchesRule(inputConfig: Config, packageRule: PackageRule): boolean { matchPackagePatterns = ['.*']; } if (matchFiles.length) { - const isMatch = matchFiles.some((fileName) => packageFile === fileName); + const isMatch = matchFiles.some( + (fileName) => packageFile === fileName || lockFiles?.includes(fileName) + ); if (!isMatch) { return false; } -- GitLab