From 151435acfd1e96d8591935b437e5604e38f75fbf Mon Sep 17 00:00:00 2001
From: Dylan Owen <dylanowen@users.noreply.github.com>
Date: Fri, 25 Aug 2023 05:41:15 -0400
Subject: [PATCH] feat(manager/cargo): support for cargo repository source
replacement (#23956)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
.../cargo/__fixtures__/cargo.6.config.toml | 2 +-
lib/modules/manager/cargo/extract.spec.ts | 255 ++++++++++++++++++
lib/modules/manager/cargo/extract.ts | 80 +++++-
lib/modules/manager/cargo/types.ts | 14 +-
lib/modules/manager/cargo/utils.ts | 4 +
5 files changed, 339 insertions(+), 16 deletions(-)
create mode 100644 lib/modules/manager/cargo/utils.ts
diff --git a/lib/modules/manager/cargo/__fixtures__/cargo.6.config.toml b/lib/modules/manager/cargo/__fixtures__/cargo.6.config.toml
index 16df4b8671..d4e9b88757 100644
--- a/lib/modules/manager/cargo/__fixtures__/cargo.6.config.toml
+++ b/lib/modules/manager/cargo/__fixtures__/cargo.6.config.toml
@@ -2,4 +2,4 @@
private-crates = { index = "https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git" }
[registries.mcorbin]
-index = "https://github.com/mcorbin/testregistry"
\ No newline at end of file
+index = "https://github.com/mcorbin/testregistry"
diff --git a/lib/modules/manager/cargo/extract.spec.ts b/lib/modules/manager/cargo/extract.spec.ts
index 457e92ef87..bbf4be8410 100644
--- a/lib/modules/manager/cargo/extract.spec.ts
+++ b/lib/modules/manager/cargo/extract.spec.ts
@@ -115,6 +115,157 @@ describe('modules/manager/cargo/extract', () => {
expect(res?.deps).toHaveLength(3);
});
+ it('extracts overridden registry indexes from .cargo/config.toml', async () => {
+ await writeLocalFile(
+ '.cargo/config.toml',
+ codeBlock`[registries]
+private-crates = { index = "https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git" }
+
+[registries.mcorbin]
+index = "https://github.com/mcorbin/testregistry"
+
+[source.crates-io]
+replace-with = "mcorbin"
+
+[source.mcorbin]
+replace-with = "private-crates"`
+ );
+ const res = await extractPackageFile(cargo6toml, 'Cargo.toml', {
+ ...config,
+ });
+ expect(res?.deps).toEqual([
+ {
+ currentValue: '0.1.0',
+ datasource: 'crate',
+ depName: 'proprietary-crate',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ registryUrls: [
+ 'https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git',
+ ],
+ },
+ {
+ currentValue: '3.0.0',
+ datasource: 'crate',
+ depName: 'mcorbin-test',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ registryUrls: [
+ 'https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git',
+ ],
+ },
+ {
+ currentValue: '0.2',
+ datasource: 'crate',
+ depName: 'tokio',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: false,
+ },
+ registryUrls: [
+ 'https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git',
+ ],
+ },
+ ]);
+ });
+
+ it('extracts registries overridden to the default', async () => {
+ await writeLocalFile(
+ '.cargo/config.toml',
+ codeBlock`[source.mcorbin]
+replace-with = "crates-io"
+
+[source.private-crates]
+replace-with = "mcorbin"`
+ );
+ const res = await extractPackageFile(cargo6toml, 'Cargo.toml', {
+ ...config,
+ });
+ expect(res?.deps).toEqual([
+ {
+ currentValue: '0.1.0',
+ datasource: 'crate',
+ depName: 'proprietary-crate',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ },
+ {
+ currentValue: '3.0.0',
+ datasource: 'crate',
+ depName: 'mcorbin-test',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ },
+ {
+ currentValue: '0.2',
+ datasource: 'crate',
+ depName: 'tokio',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: false,
+ },
+ },
+ ]);
+ });
+
+ it('extracts registries with an empty config.toml', async () => {
+ await writeLocalFile('.cargo/config.toml', ``);
+ const res = await extractPackageFile(cargo5toml, 'Cargo.toml', {
+ ...config,
+ });
+ expect(res?.deps).toEqual([
+ {
+ currentValue: '0.2.37',
+ datasource: 'crate',
+ depName: 'wasm-bindgen',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: false,
+ },
+ target: 'cfg(target_arch = "wasm32")',
+ },
+ {
+ currentValue: '0.3.14',
+ datasource: 'crate',
+ depName: 'js-sys',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: false,
+ },
+ target: 'cfg(target_arch = "wasm32")',
+ },
+ {
+ currentValue: '',
+ datasource: 'crate',
+ depName: 'js_relative_import',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: false,
+ },
+ skipReason: 'path-dependency',
+ target: 'cfg(target_arch = "wasm32")',
+ },
+ {
+ currentValue: '0.3.14',
+ datasource: 'crate',
+ depName: 'web-sys',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ target: 'cfg(target_arch = "wasm32")',
+ },
+ ]);
+ });
+
it('extracts registry urls from environment', async () => {
process.env.CARGO_REGISTRIES_PRIVATE_CRATES_INDEX =
'https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git';
@@ -249,6 +400,110 @@ tokio = { version = "1.21.1" }`;
expect(res?.deps).toHaveLength(3);
});
+ it('ignore cargo config source replaced registries with missing index', async () => {
+ await writeLocalFile(
+ '.cargo/config',
+ codeBlock`[registries.mine]
+foo = "bar"
+
+[source.crates-io]
+replace-with = "mine"`
+ );
+
+ const res = await extractPackageFile(cargo6toml, 'Cargo.toml', {
+ ...config,
+ });
+ expect(res?.deps).toEqual([
+ {
+ currentValue: '0.1.0',
+ datasource: 'crate',
+ depName: 'proprietary-crate',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ skipReason: 'unknown-registry',
+ },
+ {
+ currentValue: '3.0.0',
+ datasource: 'crate',
+ depName: 'mcorbin-test',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ skipReason: 'unknown-registry',
+ },
+ {
+ currentValue: '0.2',
+ datasource: 'crate',
+ depName: 'tokio',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: false,
+ },
+ skipReason: 'unknown-registry',
+ },
+ ]);
+ });
+
+ it('ignore cargo config with circular registry source replacements', async () => {
+ await writeLocalFile(
+ '.cargo/config',
+ codeBlock`[registries]
+private-crates = { index = "https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git" }
+
+[registries.mcorbin]
+index = "https://github.com/mcorbin/testregistry"
+
+[source.crates-io]
+replace-with = "mcorbin"
+
+[source.mcorbin]
+replace-with = "private-crates"
+
+[source.private-crates]
+replace-with = "mcorbin"
+`
+ );
+
+ const res = await extractPackageFile(cargo6toml, 'Cargo.toml', {
+ ...config,
+ });
+ expect(res?.deps).toEqual([
+ {
+ currentValue: '0.1.0',
+ datasource: 'crate',
+ depName: 'proprietary-crate',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ skipReason: 'unknown-registry',
+ },
+ {
+ currentValue: '3.0.0',
+ datasource: 'crate',
+ depName: 'mcorbin-test',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: true,
+ },
+ skipReason: 'unknown-registry',
+ },
+ {
+ currentValue: '0.2',
+ datasource: 'crate',
+ depName: 'tokio',
+ depType: 'dependencies',
+ managerData: {
+ nestedVersion: false,
+ },
+ skipReason: 'unknown-registry',
+ },
+ ]);
+ });
+
it('extracts original package name of renamed dependencies', async () => {
const cargotoml =
'[dependencies]\nboolector-solver = { package = "boolector", version = "0.4.0" }';
diff --git a/lib/modules/manager/cargo/extract.ts b/lib/modules/manager/cargo/extract.ts
index 8e733cdb11..cf632c2d91 100644
--- a/lib/modules/manager/cargo/extract.ts
+++ b/lib/modules/manager/cargo/extract.ts
@@ -12,8 +12,12 @@ import type {
CargoConfig,
CargoManifest,
CargoRegistries,
+ CargoRegistryUrl,
CargoSection,
} from './types';
+import { DEFAULT_REGISTRY_URL } from './utils';
+
+const DEFAULT_REGISTRY_ID = 'crates-io';
function getCargoIndexEnv(registryName: string): string | null {
const registry = registryName.toUpperCase().replaceAll('-', '_');
@@ -53,10 +57,12 @@ function extractFromSection(
nestedVersion = true;
if (registryName) {
const registryUrl =
- cargoRegistries[registryName] ?? getCargoIndexEnv(registryName);
+ getCargoIndexEnv(registryName) ?? cargoRegistries[registryName];
if (registryUrl) {
- registryUrls = [registryUrl];
+ if (registryUrl !== DEFAULT_REGISTRY_URL) {
+ registryUrls = [registryUrl];
+ }
} else {
skipReason = 'unknown-registry';
}
@@ -90,7 +96,19 @@ function extractFromSection(
};
if (registryUrls) {
dep.registryUrls = registryUrls;
+ } else {
+ // if we don't have an explicit registry URL check if the default registry has a non-standard url
+ if (cargoRegistries[DEFAULT_REGISTRY_ID]) {
+ if (cargoRegistries[DEFAULT_REGISTRY_ID] !== DEFAULT_REGISTRY_URL) {
+ dep.registryUrls = [cargoRegistries[DEFAULT_REGISTRY_ID]];
+ }
+ } else {
+ // we always expect to have DEFAULT_REGISTRY_ID set, if it's not it means the config defines an alternative
+ // registry that we couldn't resolve.
+ skipReason = 'unknown-registry';
+ }
}
+
if (skipReason) {
dep.skipReason = skipReason;
}
@@ -128,24 +146,60 @@ async function readCargoConfig(): Promise<CargoConfig | null> {
}
/** Extracts a map of cargo registries from a CargoConfig */
-function extractCargoRegistries(config: CargoConfig | null): CargoRegistries {
+function extractCargoRegistries(config: CargoConfig): CargoRegistries {
const result: CargoRegistries = {};
- if (!config?.registries) {
- return result;
+ // check if we're overriding our default registry index
+ result[DEFAULT_REGISTRY_ID] = resolveRegistryIndex(
+ DEFAULT_REGISTRY_ID,
+ config
+ );
+
+ const registryNames = new Set([
+ ...Object.keys(config.registries ?? {}),
+ ...Object.keys(config.source ?? {}),
+ ]);
+ for (const registryName of registryNames) {
+ result[registryName] = resolveRegistryIndex(registryName, config);
}
- const { registries } = config;
+ return result;
+}
+
+function resolveRegistryIndex(
+ registryName: string,
+ config: CargoConfig,
+ originalNames: Set<string> = new Set()
+): CargoRegistryUrl {
+ // if we have a source replacement, follow that.
+ // https://doc.rust-lang.org/cargo/reference/source-replacement.html
+ const replacementName = config.source?.[registryName]?.['replace-with'];
+ if (replacementName) {
+ logger.debug(
+ `Replacing index of cargo registry ${registryName} with ${replacementName}`
+ );
+ if (originalNames.has(replacementName)) {
+ logger.warn(`${registryName} cargo registry resolves to itself`);
+ return null;
+ }
+ return resolveRegistryIndex(
+ replacementName,
+ config,
+ originalNames.add(replacementName)
+ );
+ }
- for (const registryName of Object.keys(registries)) {
- const registry = registries[registryName];
- if (registry.index) {
- result[registryName] = registry.index;
+ const registryIndex = config.registries?.[registryName]?.index;
+ if (registryIndex) {
+ return registryIndex;
+ } else {
+ // we don't need an explicit index if we're using the default registry
+ if (registryName === DEFAULT_REGISTRY_ID) {
+ return DEFAULT_REGISTRY_URL;
} else {
logger.debug(`${registryName} cargo registry is missing index`);
+ return null;
}
}
-
- return result;
}
export async function extractPackageFile(
@@ -155,7 +209,7 @@ export async function extractPackageFile(
): Promise<PackageFileContent | null> {
logger.trace(`cargo.extractPackageFile(${packageFile})`);
- const cargoConfig = await readCargoConfig();
+ const cargoConfig = (await readCargoConfig()) ?? {};
const cargoRegistries = extractCargoRegistries(cargoConfig);
let cargoManifest: CargoManifest;
diff --git a/lib/modules/manager/cargo/types.ts b/lib/modules/manager/cargo/types.ts
index fa239f3081..41208ad64e 100644
--- a/lib/modules/manager/cargo/types.ts
+++ b/lib/modules/manager/cargo/types.ts
@@ -1,3 +1,5 @@
+import type { DEFAULT_REGISTRY_URL } from './utils';
+
export interface CargoDep {
/** Path on disk to the crate sources */
path?: string;
@@ -28,13 +30,21 @@ export interface CargoManifest extends CargoSection {
export interface CargoConfig {
registries?: Record<string, CargoRegistry>;
+ source?: Record<string, CargoSource>;
}
export interface CargoRegistry {
index?: string;
}
+export interface CargoSource {
+ 'replace-with'?: string;
+}
+
+/**
+ * null means a registry was defined, but we couldn't find a valid URL
+ */
+export type CargoRegistryUrl = string | typeof DEFAULT_REGISTRY_URL | null;
export interface CargoRegistries {
- // maps registry names to URLs
- [key: string]: string;
+ [key: string]: CargoRegistryUrl;
}
diff --git a/lib/modules/manager/cargo/utils.ts b/lib/modules/manager/cargo/utils.ts
new file mode 100644
index 0000000000..5ca2a1b4eb
--- /dev/null
+++ b/lib/modules/manager/cargo/utils.ts
@@ -0,0 +1,4 @@
+/**
+ * A sentinel value to signal the Default Registry (crates-io)
+ */
+export const DEFAULT_REGISTRY_URL = Symbol('DEFAULT_REGISTRY_URL');
--
GitLab