diff --git a/docs/usage/self-hosted-experimental.md b/docs/usage/self-hosted-experimental.md
index bae03e095f81b4439dfaa9cfcda0cda20ae3cb57..04124a9ba7add2b7bc2dbace934e0c22b4ce4217 100644
--- a/docs/usage/self-hosted-experimental.md
+++ b/docs/usage/self-hosted-experimental.md
@@ -141,6 +141,10 @@ This feature is in private beta.
If set, Renovate will query the merge-confidence JSON API only for datasources that are part of this list.
The expected value for this environment variable is a JSON array of strings.
+## `RENOVATE_X_NUGET_DOWNLOAD_NUPKGS`
+
+If set to any value, Renovate will download `nupkg` files for determining package metadata.
+
## `RENOVATE_X_PLATFORM_VERSION`
Specify this string for Renovate to skip API checks and provide GitLab/Bitbucket server version directly.
diff --git a/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3-no-repo.nupkg b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3-no-repo.nupkg
new file mode 100644
index 0000000000000000000000000000000000000000..77ce3d810f1adeed673f4a98354a14b11797947a
Binary files /dev/null and b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3-no-repo.nupkg differ
diff --git a/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3.nupkg b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3.nupkg
new file mode 100644
index 0000000000000000000000000000000000000000..6ec85f1ec046510454834a7881f0ac9e28af0780
Binary files /dev/null and b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3.nupkg differ
diff --git a/lib/modules/datasource/nuget/common.ts b/lib/modules/datasource/nuget/common.ts
index e3c9afb28fd2d63210f4207c27fd3a5fa92e0b66..14bb38d6942111f0e62197ef9afc7cd3b233f73c 100644
--- a/lib/modules/datasource/nuget/common.ts
+++ b/lib/modules/datasource/nuget/common.ts
@@ -12,10 +12,14 @@ export function removeBuildMeta(version: string): string {
const urlWhitespaceRe = regEx(/\s/g);
-export function massageUrl(url: string): string {
+export function massageUrl(url: string | null | undefined): string | null {
+ if (url === null || url === undefined) {
+ return null;
+ }
+
let resultUrl = url;
- // During `dotnet pack` certain URLs are being URL decoded which may introduce whitespaces
+ // During `dotnet pack` certain URLs are being URL decoded which may introduce whitespace
// and causes Markdown link generation problems.
resultUrl = resultUrl.replace(urlWhitespaceRe, '%20');
diff --git a/lib/modules/datasource/nuget/index.spec.ts b/lib/modules/datasource/nuget/index.spec.ts
index db7bbaa7b52d61dfa8aa041e99bbc183d07e87f8..ce25cf7722b9c04ef57bf7e1898347f436628f0b 100644
--- a/lib/modules/datasource/nuget/index.spec.ts
+++ b/lib/modules/datasource/nuget/index.spec.ts
@@ -1,8 +1,12 @@
+import { Readable } from 'stream';
import { mockDeep } from 'jest-mock-extended';
+import { join } from 'upath';
import { getPkgReleases } from '..';
import { Fixtures } from '../../../../test/fixtures';
import * as httpMock from '../../../../test/http-mock';
-import { logger } from '../../../../test/util';
+import { logger, mocked } from '../../../../test/util';
+import { GlobalConfig } from '../../../config/global';
+import * as _packageCache from '../../../util/cache/package';
import * as _hostRules from '../../../util/host-rules';
import { id as versioning } from '../../versioning/nuget';
import { parseRegistryUrl } from './common';
@@ -14,6 +18,9 @@ const hostRules: any = _hostRules;
jest.mock('../../../util/host-rules', () => mockDeep());
+jest.mock('../../../util/cache/package', () => mockDeep());
+const packageCache = mocked(_packageCache);
+
const pkgInfoV3FromNuget = Fixtures.get('nunit/v3_nuget_org.xml');
const pkgListV3Registration = Fixtures.get('nunit/v3_registration.json');
@@ -105,6 +112,10 @@ const configV3AzureDevOps = {
};
describe('modules/datasource/nuget/index', () => {
+ beforeEach(() => {
+ GlobalConfig.reset();
+ });
+
describe('parseRegistryUrl', () => {
it('extracts feed version from registry URL hash (v3)', () => {
const parsed = parseRegistryUrl('https://my-registry#protocolVersion=3');
@@ -302,6 +313,160 @@ describe('modules/datasource/nuget/index', () => {
);
});
+ describe('determine source URL from nupkg', () => {
+ beforeEach(() => {
+ GlobalConfig.set({
+ cacheDir: join('/tmp/cache'),
+ });
+ process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS = 'true';
+ });
+
+ afterEach(() => {
+ delete process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS;
+ });
+
+ it('can determine source URL from nupkg when PackageBaseAddress is missing', async () => {
+ const nugetIndex = `
+ {
+ "version": "3.0.0",
+ "resources": [
+ {
+ "@id": "https://some-registry/v3/metadata",
+ "@type": "RegistrationsBaseUrl/3.0.0-beta",
+ "comment": "Get package metadata."
+ }
+ ]
+ }
+ `;
+ const nlogRegistration = `
+ {
+ "count": 1,
+ "items": [
+ {
+ "@id": "https://some-registry/v3/metadata/nlog/4.7.3.json",
+ "lower": "4.7.3",
+ "upper": "4.7.3",
+ "count": 1,
+ "items": [
+ {
+ "@id": "foo",
+ "catalogEntry": {
+ "id": "NLog",
+ "version": "4.7.3",
+ "packageContent": "https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ `;
+ httpMock
+ .scope('https://some-registry')
+ .get('/v3/index.json')
+ .twice()
+ .reply(200, nugetIndex)
+ .get('/v3/metadata/nlog/index.json')
+ .reply(200, nlogRegistration)
+ .get('/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg')
+ .reply(200, () => {
+ const readableStream = new Readable();
+ readableStream.push(Fixtures.getBinary('nlog/NLog.4.7.3.nupkg'));
+ readableStream.push(null);
+ return readableStream;
+ });
+ const res = await getPkgReleases({
+ datasource,
+ versioning,
+ packageName: 'NLog',
+ registryUrls: ['https://some-registry/v3/index.json'],
+ });
+ expect(logger.logger.debug).toHaveBeenCalledWith(
+ 'Determined sourceUrl https://github.com/NLog/NLog.git from https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg',
+ );
+ expect(packageCache.set).toHaveBeenCalledWith(
+ 'datasource-nuget',
+ 'cache-decorator:source-url:https://some-registry/v3/index.json:NLog',
+ {
+ cachedAt: expect.any(String),
+ value: 'https://github.com/NLog/NLog.git',
+ },
+ 60 * 24 * 7,
+ );
+ expect(res?.sourceUrl).toBeDefined();
+ });
+
+ it('can handle nupkg without repository metadata', async () => {
+ const nugetIndex = `
+ {
+ "version": "3.0.0",
+ "resources": [
+ {
+ "@id": "https://some-registry/v3/metadata",
+ "@type": "RegistrationsBaseUrl/3.0.0-beta",
+ "comment": "Get package metadata."
+ }
+ ]
+ }
+ `;
+ const nlogRegistration = `
+ {
+ "count": 1,
+ "items": [
+ {
+ "@id": "https://some-registry/v3/metadata/nlog/4.7.3.json",
+ "lower": "4.7.3",
+ "upper": "4.7.3",
+ "count": 1,
+ "items": [
+ {
+ "@id": "foo",
+ "catalogEntry": {
+ "id": "NLog",
+ "version": "4.7.3",
+ "packageContent": "https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ `;
+ httpMock
+ .scope('https://some-registry')
+ .get('/v3/index.json')
+ .twice()
+ .reply(200, nugetIndex)
+ .get('/v3/metadata/nlog/index.json')
+ .reply(200, nlogRegistration)
+ .get('/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg')
+ .reply(200, () => {
+ const readableStream = new Readable();
+ readableStream.push(
+ Fixtures.getBinary('nlog/NLog.4.7.3-no-repo.nupkg'),
+ );
+ readableStream.push(null);
+ return readableStream;
+ });
+ const res = await getPkgReleases({
+ datasource,
+ versioning,
+ packageName: 'NLog',
+ registryUrls: ['https://some-registry/v3/index.json'],
+ });
+ expect(packageCache.set).toHaveBeenCalledWith(
+ 'datasource-nuget',
+ 'cache-decorator:source-url:https://some-registry/v3/index.json:NLog',
+ {
+ cachedAt: expect.any(String),
+ value: null,
+ },
+ 60 * 24 * 7,
+ );
+ expect(res?.sourceUrl).toBeUndefined();
+ });
+ });
+
it('returns null for non 200 (v3v2)', async () => {
httpMock.scope('https://api.nuget.org').get('/v3/index.json').reply(500);
httpMock
diff --git a/lib/modules/datasource/nuget/types.ts b/lib/modules/datasource/nuget/types.ts
index 29aba2a5c6b4a5dbcd27730141844a4e7176d60c..36fa672e83b22399e4dc9dfb34239d689fcf43fc 100644
--- a/lib/modules/datasource/nuget/types.ts
+++ b/lib/modules/datasource/nuget/types.ts
@@ -5,11 +5,13 @@ export interface ServicesIndexRaw {
}[];
}
+// See https://learn.microsoft.com/en-us/nuget/api/registration-base-url-resource#catalog-entry
export interface CatalogEntry {
version: string;
published?: string;
projectUrl?: string;
listed?: boolean;
+ packageContent?: string;
}
export interface CatalogPage {
diff --git a/lib/modules/datasource/nuget/v3.ts b/lib/modules/datasource/nuget/v3.ts
index 2024c175cbee6a28aec5d4ca86fdf62b2cb3cfb4..1d0dc587c01d3a3c7d237bf5542c1e76ced06908 100644
--- a/lib/modules/datasource/nuget/v3.ts
+++ b/lib/modules/datasource/nuget/v3.ts
@@ -1,9 +1,14 @@
import is from '@sindresorhus/is';
+import extract from 'extract-zip';
import semver from 'semver';
+import upath from 'upath';
import { XmlDocument } from 'xmldoc';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
import * as packageCache from '../../../util/cache/package';
+import { cache } from '../../../util/cache/package/decorator';
+import * as fs from '../../../util/fs';
+import { ensureCacheDir } from '../../../util/fs';
import { Http, HttpError } from '../../../util/http';
import * as p from '../../../util/promises';
import { regEx } from '../../../util/regex';
@@ -151,8 +156,15 @@ export class NugetV3Api {
let homepage: string | null = null;
let latestStable: string | null = null;
+ let nupkgUrl: string | null = null;
const releases = catalogEntries.map(
- ({ version, published: releaseTimestamp, projectUrl, listed }) => {
+ ({
+ version,
+ published: releaseTimestamp,
+ projectUrl,
+ listed,
+ packageContent,
+ }) => {
const release: Release = { version: removeBuildMeta(version) };
if (releaseTimestamp) {
release.releaseTimestamp = releaseTimestamp;
@@ -160,6 +172,7 @@ export class NugetV3Api {
if (versioning.isValid(version) && versioning.isStable(version)) {
latestStable = removeBuildMeta(version);
homepage = projectUrl ? massageUrl(projectUrl) : homepage;
+ nupkgUrl = massageUrl(packageContent);
}
if (listed === false) {
release.isDeprecated = true;
@@ -177,6 +190,7 @@ export class NugetV3Api {
const last = catalogEntries.pop()!;
latestStable = removeBuildMeta(last.version);
homepage ??= last.projectUrl ?? null;
+ nupkgUrl ??= massageUrl(last.packageContent);
}
const dep: ReleaseResult = {
@@ -189,7 +203,6 @@ export class NugetV3Api {
registryUrl,
'PackageBaseAddress',
);
- // istanbul ignore else: this is a required v3 api
if (is.nonEmptyString(packageBaseAddress)) {
const nuspecUrl = `${ensureTrailingSlash(
packageBaseAddress,
@@ -203,6 +216,18 @@ export class NugetV3Api {
if (sourceUrl) {
dep.sourceUrl = massageUrl(sourceUrl);
}
+ } else if (nupkgUrl) {
+ const sourceUrl = await this.getSourceUrlFromNupkg(
+ http,
+ registryUrl,
+ pkgName,
+ latestStable,
+ nupkgUrl,
+ );
+ if (sourceUrl) {
+ dep.sourceUrl = massageUrl(sourceUrl);
+ logger.debug(`Determined sourceUrl ${sourceUrl} from ${nupkgUrl}`);
+ }
}
} catch (err) {
// istanbul ignore if: not easy testable with nock
@@ -233,4 +258,52 @@ export class NugetV3Api {
return dep;
}
+
+ @cache({
+ namespace: NugetV3Api.cacheNamespace,
+ key: (
+ _http: Http,
+ registryUrl: string,
+ packageName: string,
+ _packageVersion: string | null,
+ _nupkgUrl: string,
+ ) => `source-url:${registryUrl}:${packageName}`,
+ ttlMinutes: 10080, // 1 week
+ })
+ async getSourceUrlFromNupkg(
+ http: Http,
+ _registryUrl: string,
+ packageName: string,
+ packageVersion: string | null,
+ nupkgUrl: string,
+ ): Promise<string | null> {
+ // istanbul ignore if: experimental feature
+ if (!process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS) {
+ logger.once.debug('RENOVATE_X_NUGET_DOWNLOAD_NUPKGS is not set');
+ return null;
+ }
+ const cacheDir = await ensureCacheDir('nuget');
+ const nupkgFile = upath.join(
+ cacheDir,
+ `${packageName}.${packageVersion}.nupkg`,
+ );
+ const nupkgContentsDir = upath.join(
+ cacheDir,
+ `${packageName}.${packageVersion}`,
+ );
+ const readStream = http.stream(nupkgUrl);
+ try {
+ const writeStream = fs.createCacheWriteStream(nupkgFile);
+ await fs.pipeline(readStream, writeStream);
+ await extract(nupkgFile, { dir: nupkgContentsDir });
+ const nuspecFile = upath.join(nupkgContentsDir, `${packageName}.nuspec`);
+ const nuspec = new XmlDocument(
+ await fs.readCacheFile(nuspecFile, 'utf8'),
+ );
+ return nuspec.valueWithPath('metadata.repository@url') ?? null;
+ } finally {
+ await fs.rmCache(nupkgFile);
+ await fs.rmCache(nupkgContentsDir);
+ }
+ }
}