From 5aa2ebfbcb333b7bc5f01fc3de768a78e2abe433 Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Mon, 6 May 2024 14:17:07 -0300
Subject: [PATCH] refactor(gomod): Simplify dependency extraction (#28852)
---
.../gomod/__snapshots__/extract.spec.ts.snap | 12 +-
lib/modules/manager/gomod/extract.spec.ts | 6 +-
lib/modules/manager/gomod/extract.ts | 172 ++---------
lib/modules/manager/gomod/line-parser.spec.ts | 280 ++++++++++++++++++
lib/modules/manager/gomod/line-parser.ts | 154 ++++++++++
5 files changed, 465 insertions(+), 159 deletions(-)
create mode 100644 lib/modules/manager/gomod/line-parser.spec.ts
create mode 100644 lib/modules/manager/gomod/line-parser.ts
diff --git a/lib/modules/manager/gomod/__snapshots__/extract.spec.ts.snap b/lib/modules/manager/gomod/__snapshots__/extract.spec.ts.snap
index f089350ae9..93e19c1eaf 100644
--- a/lib/modules/manager/gomod/__snapshots__/extract.spec.ts.snap
+++ b/lib/modules/manager/gomod/__snapshots__/extract.spec.ts.snap
@@ -721,12 +721,13 @@ exports[`modules/manager/gomod/extract extractPackageFile() extracts single-line
},
{
"currentValue": "abcdef1",
+ "datasource": "go",
"depName": "github.com/rarkins/foo",
"depType": "require",
"managerData": {
"lineNumber": 6,
},
- "skipReason": "unsupported-version",
+ "skipReason": "invalid-version",
},
{
"currentValue": "v1.0.0",
@@ -746,6 +747,15 @@ exports[`modules/manager/gomod/extract extractPackageFile() extracts single-line
"lineNumber": 8,
},
},
+ {
+ "datasource": "go",
+ "depName": "../errors",
+ "depType": "replace",
+ "managerData": {
+ "lineNumber": 10,
+ },
+ "skipReason": "local-dependency",
+ },
{
"currentValue": "v0.0.0",
"datasource": "go",
diff --git a/lib/modules/manager/gomod/extract.spec.ts b/lib/modules/manager/gomod/extract.spec.ts
index 67b7f2c88a..97d093d62b 100644
--- a/lib/modules/manager/gomod/extract.spec.ts
+++ b/lib/modules/manager/gomod/extract.spec.ts
@@ -14,11 +14,11 @@ describe('modules/manager/gomod/extract', () => {
it('extracts single-line requires', () => {
const res = extractPackageFile(gomod1)?.deps;
expect(res).toMatchSnapshot();
- expect(res).toHaveLength(9);
+ expect(res).toHaveLength(10);
expect(res?.filter((e) => e.depType === 'require')).toHaveLength(7);
expect(res?.filter((e) => e.depType === 'indirect')).toHaveLength(1);
- expect(res?.filter((e) => e.skipReason)).toHaveLength(1);
- expect(res?.filter((e) => e.depType === 'replace')).toHaveLength(1);
+ expect(res?.filter((e) => e.skipReason)).toHaveLength(2);
+ expect(res?.filter((e) => e.depType === 'replace')).toHaveLength(2);
});
it('extracts multi-line requires', () => {
diff --git a/lib/modules/manager/gomod/extract.ts b/lib/modules/manager/gomod/extract.ts
index 524158f1d2..731bd2b7c3 100644
--- a/lib/modules/manager/gomod/extract.ts
+++ b/lib/modules/manager/gomod/extract.ts
@@ -1,165 +1,27 @@
-import semver from 'semver';
-import { logger } from '../../../logger';
-import { newlineRegex, regEx } from '../../../util/regex';
-import { GoDatasource } from '../../datasource/go';
-import { GolangVersionDatasource } from '../../datasource/golang-version';
-import { isVersion } from '../../versioning/semver';
+import { newlineRegex } from '../../../util/regex';
import type { PackageDependency, PackageFileContent } from '../types';
-import type { MultiLineParseResult } from './types';
+import { parseLine } from './line-parser';
-function getDep(
- lineNumber: number,
- match: RegExpMatchArray,
- type: string,
-): PackageDependency {
- const [, , currentValue] = match;
- let [, depName] = match;
- depName = depName.replace(regEx(/"/g), '');
- const dep: PackageDependency = {
- managerData: {
- lineNumber,
- },
- depName,
- depType: type,
- currentValue,
- };
- if (isVersion(currentValue)) {
- dep.datasource = GoDatasource.id;
- } else {
- dep.skipReason = 'unsupported-version';
- }
- const digestMatch = regEx(GoDatasource.pversionRegexp).exec(currentValue);
- if (digestMatch?.groups?.digest) {
- dep.currentDigest = digestMatch.groups.digest;
- dep.digestOneAndOnly = true;
- dep.versioning = 'loose';
- }
- return dep;
-}
-
-function getGoDep(
- lineNumber: number,
- goVer: string,
- versioning: string | undefined = undefined,
- depType: string = 'golang',
-): PackageDependency {
- return {
- managerData: {
- lineNumber,
- },
- depName: 'go',
- depType,
- currentValue: goVer,
- datasource: GolangVersionDatasource.id,
- ...(versioning && { versioning }),
- };
-}
-
-export function extractPackageFile(
- content: string,
- packageFile?: string,
-): PackageFileContent | null {
- logger.trace({ content }, 'gomod.extractPackageFile()');
+export function extractPackageFile(content: string): PackageFileContent | null {
const deps: PackageDependency[] = [];
- try {
- const lines = content.split(newlineRegex);
- for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
- const line = lines[lineNumber];
- const goVer = line.startsWith('go ') ? line.replace('go ', '') : null;
- if (goVer && semver.validRange(goVer)) {
- const dep = getGoDep(lineNumber, goVer, 'go-mod-directive');
- deps.push(dep);
- continue;
- }
- const goToolVer = line.startsWith('toolchain go')
- ? line.replace('toolchain go', '')
- : null;
- if (goToolVer && semver.valid(goToolVer)) {
- const dep = getGoDep(lineNumber, goToolVer, undefined, 'toolchain');
- deps.push(dep);
- continue;
- }
- const replaceMatch = regEx(
- /^replace\s+[^\s]+[\s]+[=][>]\s+([^\s]+)\s+([^\s]+)/,
- ).exec(line);
- if (replaceMatch) {
- const dep = getDep(lineNumber, replaceMatch, 'replace');
- deps.push(dep);
- }
- const requireMatch = regEx(/^require\s+([^\s]+)\s+([^\s]+)/).exec(line);
- if (requireMatch) {
- if (line.endsWith('// indirect')) {
- logger.trace({ lineNumber }, `indirect line: "${line}"`);
- const dep = getDep(lineNumber, requireMatch, 'indirect');
- dep.enabled = false;
- deps.push(dep);
- } else {
- logger.trace({ lineNumber }, `require line: "${line}"`);
- const dep = getDep(lineNumber, requireMatch, 'require');
- deps.push(dep);
- }
- }
- if (line.trim() === 'require (') {
- logger.trace(`Matched multi-line require on line ${lineNumber}`);
- const matcher = regEx(/^\s+([^\s]+)\s+([^\s]+)/);
- const { reachedLine, detectedDeps } = parseMultiLine(
- lineNumber,
- lines,
- matcher,
- 'require',
- );
- lineNumber = reachedLine;
- deps.push(...detectedDeps);
- } else if (line.trim() === 'replace (') {
- logger.trace(`Matched multi-line replace on line ${lineNumber}`);
- const matcher = regEx(/^\s+[^\s]+[\s]+[=][>]\s+([^\s]+)\s+([^\s]+)/);
- const { reachedLine, detectedDeps } = parseMultiLine(
- lineNumber,
- lines,
- matcher,
- 'replace',
- );
- lineNumber = reachedLine;
- deps.push(...detectedDeps);
- }
+
+ const lines = content.split(newlineRegex);
+ for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
+ const line = lines[lineNumber];
+ const dep = parseLine(line);
+ if (!dep) {
+ continue;
}
- } catch (err) /* istanbul ignore next */ {
- logger.warn({ err, packageFile }, 'Error extracting go modules');
+
+ dep.managerData ??= {};
+ dep.managerData.lineNumber = lineNumber;
+
+ deps.push(dep);
}
+
if (!deps.length) {
return null;
}
- return { deps };
-}
-function parseMultiLine(
- startingLine: number,
- lines: string[],
- matchRegex: RegExp,
- blockType: 'require' | 'replace',
-): MultiLineParseResult {
- const deps: PackageDependency[] = [];
- let lineNumber = startingLine;
- let line = '';
- do {
- lineNumber += 1;
- line = lines[lineNumber];
- const multiMatch = matchRegex.exec(line);
- logger.trace(`${blockType}: "${line}"`);
- if (multiMatch && !line.endsWith('// indirect')) {
- logger.trace({ lineNumber }, `${blockType} line: "${line}"`);
- const dep = getDep(lineNumber, multiMatch, blockType);
- dep.managerData!.multiLine = true;
- deps.push(dep);
- } else if (multiMatch && line.endsWith('// indirect')) {
- logger.trace({ lineNumber }, `${blockType} indirect line: "${line}"`);
- const dep = getDep(lineNumber, multiMatch, 'indirect');
- dep.managerData!.multiLine = true;
- dep.enabled = false;
- deps.push(dep);
- } else if (line.trim() !== ')') {
- logger.trace(`No multi-line match: ${line}`);
- }
- } while (line.trim() !== ')');
- return { reachedLine: lineNumber, detectedDeps: deps };
+ return { deps };
}
diff --git a/lib/modules/manager/gomod/line-parser.spec.ts b/lib/modules/manager/gomod/line-parser.spec.ts
new file mode 100644
index 0000000000..deef02e43f
--- /dev/null
+++ b/lib/modules/manager/gomod/line-parser.spec.ts
@@ -0,0 +1,280 @@
+import { parseLine } from './line-parser';
+
+describe('modules/manager/gomod/line-parser', () => {
+ it('should return null for invalid input', () => {
+ expect(parseLine('invalid')).toBeNull();
+ });
+
+ it('should parse go version', () => {
+ const line = 'go 1.16';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: '1.16',
+ datasource: 'golang-version',
+ depName: 'go',
+ depType: 'golang',
+ versioning: 'go-mod-directive',
+ });
+ });
+
+ it('should skip invalid go version', () => {
+ const line = 'go invalid';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'invalid',
+ datasource: 'golang-version',
+ depName: 'go',
+ depType: 'golang',
+ skipReason: 'invalid-version',
+ versioning: 'go-mod-directive',
+ });
+ });
+
+ it('should parse toolchain version', () => {
+ const line = 'toolchain go1.16';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: '1.16',
+ datasource: 'golang-version',
+ depName: 'go',
+ depType: 'toolchain',
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should skip invalid toolchain version', () => {
+ const line = 'toolchain go-invalid';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: '-invalid',
+ datasource: 'golang-version',
+ depName: 'go',
+ depType: 'toolchain',
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse require definition', () => {
+ const line = 'require foo/foo v1.2';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'foo/foo',
+ depType: 'require',
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse require definition with pseudo-version', () => {
+ const line = 'require foo/foo v0.0.0-20210101000000-000000000000';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentDigest: '000000000000',
+ currentValue: 'v0.0.0-20210101000000-000000000000',
+ datasource: 'go',
+ depName: 'foo/foo',
+ depType: 'require',
+ digestOneAndOnly: true,
+ versioning: 'loose',
+ });
+ });
+
+ it('should parse require multi-line', () => {
+ const line = ' foo/foo v1.2';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'foo/foo',
+ depType: 'require',
+ managerData: {
+ multiLine: true,
+ },
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse require definition with quotes', () => {
+ const line = 'require "foo/foo" v1.2';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'foo/foo',
+ depType: 'require',
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse require multi-line definition with quotes', () => {
+ const line = ' "foo/foo" v1.2';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'foo/foo',
+ depType: 'require',
+ managerData: {
+ multiLine: true,
+ },
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse require definition with indirect dependency', () => {
+ const line = 'require foo/foo v1.2 // indirect';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'foo/foo',
+ depType: 'indirect',
+ enabled: false,
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse require multi-line definition with indirect dependency', () => {
+ const line = ' foo/foo v1.2 // indirect';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'foo/foo',
+ depType: 'indirect',
+ enabled: false,
+ managerData: {
+ multiLine: true,
+ },
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse replace definition', () => {
+ const line = 'replace foo/foo => bar/bar';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ datasource: 'go',
+ depName: 'bar/bar',
+ depType: 'replace',
+ skipReason: 'unspecified-version',
+ });
+ });
+
+ it('should parse replace multi-line definition', () => {
+ const line = ' foo/foo => bar/bar';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ datasource: 'go',
+ depName: 'bar/bar',
+ depType: 'replace',
+ managerData: {
+ multiLine: true,
+ },
+ skipReason: 'unspecified-version',
+ });
+ });
+
+ it('should parse replace definition with quotes', () => {
+ const line = 'replace "foo/foo" => "bar/bar"';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ datasource: 'go',
+ depName: 'bar/bar',
+ depType: 'replace',
+ skipReason: 'unspecified-version',
+ });
+ });
+
+ it('should parse replace multi-line definition with quotes', () => {
+ const line = ' "foo/foo" => "bar/bar"';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ datasource: 'go',
+ depName: 'bar/bar',
+ depType: 'replace',
+ managerData: {
+ multiLine: true,
+ },
+ skipReason: 'unspecified-version',
+ });
+ });
+
+ it('should parse replace definition with version', () => {
+ const line = 'replace foo/foo => bar/bar v1.2';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'bar/bar',
+ depType: 'replace',
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse replace definition with pseudo-version', () => {
+ const line =
+ 'replace foo/foo => bar/bar v0.0.0-20210101000000-000000000000';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentDigest: '000000000000',
+ currentValue: 'v0.0.0-20210101000000-000000000000',
+ datasource: 'go',
+ depName: 'bar/bar',
+ depType: 'replace',
+ digestOneAndOnly: true,
+ versioning: 'loose',
+ });
+ });
+
+ it('should parse replace indirect definition', () => {
+ const line = 'replace foo/foo => bar/bar v1.2 // indirect';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'bar/bar',
+ depType: 'indirect',
+ enabled: false,
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse replace multi-line definition with version', () => {
+ const line = ' foo/foo => bar/bar v1.2';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ currentValue: 'v1.2',
+ datasource: 'go',
+ depName: 'bar/bar',
+ depType: 'replace',
+ managerData: {
+ multiLine: true,
+ },
+ skipReason: 'invalid-version',
+ });
+ });
+
+ it('should parse replace definition pointing to relative local path', () => {
+ const line = 'replace foo/foo => ../bar';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ datasource: 'go',
+ depName: '../bar',
+ depType: 'replace',
+ skipReason: 'local-dependency',
+ });
+ });
+
+ it('should parse replace definition pointing to absolute local path', () => {
+ const line = 'replace foo/foo => /bar';
+ const res = parseLine(line);
+ expect(res).toStrictEqual({
+ datasource: 'go',
+ depName: '/bar',
+ depType: 'replace',
+ skipReason: 'local-dependency',
+ });
+ });
+});
diff --git a/lib/modules/manager/gomod/line-parser.ts b/lib/modules/manager/gomod/line-parser.ts
new file mode 100644
index 0000000000..58591c7c50
--- /dev/null
+++ b/lib/modules/manager/gomod/line-parser.ts
@@ -0,0 +1,154 @@
+import semver from 'semver';
+import { regEx } from '../../../util/regex';
+import { GoDatasource } from '../../datasource/go';
+import { GolangVersionDatasource } from '../../datasource/golang-version';
+import { isVersion } from '../../versioning/semver';
+import type { PackageDependency } from '../types';
+
+function trimQuotes(str: string): string {
+ return str.replace(regEx(/^"(.*)"$/), '$1');
+}
+
+const requireRegex = regEx(
+ /^(?<keyword>require)?\s+(?<module>[^\s]+\/[^\s]+)\s+(?<version>[^\s]+)(?:\s*\/\/\s*(?<comment>[^\s]+)\s*)?$/,
+);
+
+const replaceRegex = regEx(
+ /^(?<keyword>replace)?\s+(?<module>[^\s]+\/[^\s]+)\s*=>\s*(?<replacement>[^\s]+)(?:\s+(?<version>[^\s]+))?(?:\s*\/\/\s*(?<comment>[^\s]+)\s*)?$/,
+);
+
+const goVersionRegex = regEx(/^\s*go\s+(?<version>[^\s]+)\s*$/);
+
+const toolchainVersionRegex = regEx(/^\s*toolchain\s+go(?<version>[^\s]+)\s*$/);
+
+const pseudoVersionRegex = regEx(GoDatasource.pversionRegexp);
+
+function extractDigest(input: string): string | undefined {
+ const match = pseudoVersionRegex.exec(input);
+ return match?.groups?.digest;
+}
+
+export function parseLine(input: string): PackageDependency | null {
+ const goVersionMatches = goVersionRegex.exec(input)?.groups;
+ if (goVersionMatches) {
+ const { version: currentValue } = goVersionMatches;
+
+ const dep: PackageDependency = {
+ datasource: GolangVersionDatasource.id,
+ versioning: 'go-mod-directive',
+ depType: 'golang',
+ depName: 'go',
+ currentValue,
+ };
+
+ if (!semver.validRange(currentValue)) {
+ dep.skipReason = 'invalid-version';
+ }
+
+ return dep;
+ }
+
+ const toolchainMatches = toolchainVersionRegex.exec(input)?.groups;
+ if (toolchainMatches) {
+ const { version: currentValue } = toolchainMatches;
+
+ const dep: PackageDependency = {
+ datasource: GolangVersionDatasource.id,
+ depType: 'toolchain',
+ depName: 'go',
+ currentValue,
+ };
+
+ if (!semver.valid(currentValue)) {
+ dep.skipReason = 'invalid-version';
+ }
+
+ return dep;
+ }
+
+ const requireMatches = requireRegex.exec(input)?.groups;
+ if (requireMatches) {
+ const { keyword, module, version: currentValue, comment } = requireMatches;
+
+ const depName = trimQuotes(module);
+
+ const dep: PackageDependency = {
+ datasource: GoDatasource.id,
+ depType: 'require',
+ depName,
+ currentValue,
+ };
+
+ if (isVersion(currentValue)) {
+ const digest = extractDigest(currentValue);
+ if (digest) {
+ dep.currentDigest = digest;
+ dep.digestOneAndOnly = true;
+ dep.versioning = 'loose';
+ }
+ } else {
+ dep.skipReason = 'invalid-version';
+ }
+
+ if (comment === 'indirect') {
+ dep.depType = 'indirect';
+ dep.enabled = false;
+ }
+
+ if (!keyword) {
+ dep.managerData = { multiLine: true };
+ }
+
+ return dep;
+ }
+
+ const replaceMatches = replaceRegex.exec(input)?.groups;
+ if (replaceMatches) {
+ const {
+ keyword,
+ replacement,
+ version: currentValue,
+ comment,
+ } = replaceMatches;
+
+ const depName = trimQuotes(replacement);
+
+ const dep: PackageDependency = {
+ datasource: GoDatasource.id,
+ depType: 'replace',
+ depName,
+ currentValue,
+ };
+
+ if (isVersion(currentValue)) {
+ const digest = extractDigest(currentValue);
+ if (digest) {
+ dep.currentDigest = digest;
+ dep.digestOneAndOnly = true;
+ dep.versioning = 'loose';
+ }
+ } else if (currentValue) {
+ dep.skipReason = 'invalid-version';
+ } else {
+ dep.skipReason = 'unspecified-version';
+ delete dep.currentValue;
+ }
+
+ if (comment === 'indirect') {
+ dep.depType = 'indirect';
+ dep.enabled = false;
+ }
+
+ if (!keyword) {
+ dep.managerData = { multiLine: true };
+ }
+
+ if (depName.startsWith('/') || depName.startsWith('.')) {
+ dep.skipReason = 'local-dependency';
+ }
+
+ return dep;
+ }
+
+ return null;
+}
--
GitLab