From d166bc01e64bc8c1ef28aaebe2bea88195c9c801 Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Fri, 28 Apr 2023 20:05:46 +0300
Subject: [PATCH] fix(rubygems): Use memcache instead of module-local const
(#21874)
---
lib/modules/datasource/rubygems/index.spec.ts | 10 +-
lib/modules/datasource/rubygems/index.ts | 12 +-
.../rubygems/versions-datasource.ts | 116 +++++++++---------
3 files changed, 63 insertions(+), 75 deletions(-)
diff --git a/lib/modules/datasource/rubygems/index.spec.ts b/lib/modules/datasource/rubygems/index.spec.ts
index aeb874b5af..2a392166e0 100644
--- a/lib/modules/datasource/rubygems/index.spec.ts
+++ b/lib/modules/datasource/rubygems/index.spec.ts
@@ -1,8 +1,9 @@
import { getPkgReleases } from '..';
import { Fixtures } from '../../../../test/fixtures';
import * as httpMock from '../../../../test/http-mock';
+import * as memCache from '../../../util/cache/memory';
import * as rubyVersioning from '../../versioning/ruby';
-import { VersionsDatasource, resetCache } from './versions-datasource';
+import { VersionsDatasource } from './versions-datasource';
import { RubyGemsDatasource } from '.';
const rubygemsOrgVersions = Fixtures.get('rubygems-org.txt');
@@ -13,8 +14,6 @@ const emptyMarshalArray = Buffer.from([4, 8, 91, 0]);
describe('modules/datasource/rubygems/index', () => {
describe('getReleases', () => {
- const SKIP_CACHE = process.env.RENOVATE_SKIP_CACHE;
-
const params = {
versioning: rubyVersioning.id,
datasource: RubyGemsDatasource.id,
@@ -26,13 +25,12 @@ describe('modules/datasource/rubygems/index', () => {
};
beforeEach(() => {
- resetCache();
- process.env.RENOVATE_SKIP_CACHE = 'true';
+ memCache.init();
jest.resetAllMocks();
});
afterEach(() => {
- process.env.RENOVATE_SKIP_CACHE = SKIP_CACHE;
+ memCache.reset();
});
it('returns null for missing pkg', async () => {
diff --git a/lib/modules/datasource/rubygems/index.ts b/lib/modules/datasource/rubygems/index.ts
index 7807d65ce9..3d141d8678 100644
--- a/lib/modules/datasource/rubygems/index.ts
+++ b/lib/modules/datasource/rubygems/index.ts
@@ -12,6 +12,7 @@ export class RubyGemsDatasource extends Datasource {
constructor() {
super(RubyGemsDatasource.id);
+ this.versionsDatasource = new VersionsDatasource(RubyGemsDatasource.id);
this.internalRubyGemsDatasource = new InternalRubyGemsDatasource(
RubyGemsDatasource.id
);
@@ -23,7 +24,7 @@ export class RubyGemsDatasource extends Datasource {
override readonly registryStrategy = 'hunt';
- private readonly versionsDatasources: Record<string, VersionsDatasource> = {};
+ private readonly versionsDatasource: VersionsDatasource;
private readonly internalRubyGemsDatasource: InternalRubyGemsDatasource;
@@ -43,15 +44,8 @@ export class RubyGemsDatasource extends Datasource {
return null;
}
- if (!this.versionsDatasources[registryUrl]) {
- this.versionsDatasources[registryUrl] = new VersionsDatasource(
- RubyGemsDatasource.id,
- registryUrl
- );
- }
-
try {
- return await this.versionsDatasources[registryUrl].getReleases({
+ return await this.versionsDatasource.getReleases({
packageName,
registryUrl,
});
diff --git a/lib/modules/datasource/rubygems/versions-datasource.ts b/lib/modules/datasource/rubygems/versions-datasource.ts
index 3315185961..07d5a09509 100644
--- a/lib/modules/datasource/rubygems/versions-datasource.ts
+++ b/lib/modules/datasource/rubygems/versions-datasource.ts
@@ -1,6 +1,7 @@
import { PAGE_NOT_FOUND_ERROR } from '../../../constants/error-messages';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
+import * as memCache from '../../../util/cache/memory';
import { getElapsedMinutes } from '../../../util/date';
import { HttpError } from '../../../util/http';
import { newlineRegex } from '../../../util/regex';
@@ -12,57 +13,53 @@ interface RegistryCache {
packageReleases: Record<string, string[]>; // Because we might need a "constructor" key
contentLength: number;
isSupported: boolean;
-}
-
-const registryCaches: Record<string, RegistryCache> = {};
-
-// Note: use only for tests
-export function resetCache(): void {
- Object.keys(registryCaches).forEach((key) => {
- registryCaches[key].lastSync = new Date('2000-01-01');
- registryCaches[key].packageReleases = Object.create(null);
- registryCaches[key].contentLength = 0;
- registryCaches[key].isSupported = false;
- });
+ registryUrl: string;
}
export class VersionsDatasource extends Datasource {
- private registryUrl: string;
- private registryCache: RegistryCache;
-
- constructor(override readonly id: string, registryUrl: string) {
+ constructor(override readonly id: string) {
super(id);
- this.registryUrl = registryUrl;
- if (!registryCaches[registryUrl]) {
- registryCaches[registryUrl] = {
- lastSync: new Date('2000-01-01'),
- packageReleases: Object.create(null),
- contentLength: 0,
- isSupported: false,
- };
- }
- this.registryCache = registryCaches[registryUrl];
+ }
+
+ getRegistryCache(registryUrl: string): RegistryCache {
+ const cacheKey = `rubygems-versions-cache:${registryUrl}`;
+ const regCache = memCache.get<RegistryCache>(cacheKey) ?? {
+ lastSync: new Date('2000-01-01'),
+ packageReleases: {},
+ contentLength: 0,
+ isSupported: false,
+ registryUrl,
+ };
+ memCache.set(cacheKey, regCache);
+ return regCache;
}
async getReleases({
+ registryUrl,
packageName,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
logger.debug(`getRubygemsOrgDependency(${packageName})`);
- await this.syncVersions();
- if (!this.registryCache.isSupported) {
+
+ // istanbul ignore if
+ if (!registryUrl) {
+ return null;
+ }
+ const regCache = this.getRegistryCache(registryUrl);
+
+ await this.syncVersions(regCache);
+
+ if (!regCache.isSupported) {
throw new Error(PAGE_NOT_FOUND_ERROR);
}
- if (!this.registryCache.packageReleases[packageName]) {
+
+ if (!regCache.packageReleases[packageName]) {
return null;
}
- const dep: ReleaseResult = {
- releases: this.registryCache.packageReleases[packageName].map(
- (version) => ({
- version,
- })
- ),
- };
- return dep;
+
+ const releases = regCache.packageReleases[packageName].map((version) => ({
+ version,
+ }));
+ return { releases };
}
/**
@@ -72,12 +69,12 @@ export class VersionsDatasource extends Datasource {
return (' ' + x).slice(1);
}
- async updateRubyGemsVersions(): Promise<void> {
- const url = `${this.registryUrl}/versions`;
+ async updateRubyGemsVersions(regCache: RegistryCache): Promise<void> {
+ const url = `${regCache.registryUrl}/versions`;
const options = {
headers: {
'accept-encoding': 'identity',
- range: `bytes=${this.registryCache.contentLength}-`,
+ range: `bytes=${regCache.contentLength}-`,
},
};
let newLines: string;
@@ -87,30 +84,30 @@ export class VersionsDatasource extends Datasource {
newLines = (await this.http.get(url, options)).body;
const durationMs = Math.round(Date.now() - startTime);
logger.debug(`Rubygems: Fetched rubygems.org versions in ${durationMs}`);
- this.registryCache.isSupported = true;
+ regCache.isSupported = true;
} catch (err) /* istanbul ignore next */ {
if (err instanceof HttpError && err.response?.statusCode === 404) {
- this.registryCache.isSupported = false;
+ regCache.isSupported = false;
return;
}
if (err.statusCode !== 416) {
- this.registryCache.contentLength = 0;
- this.registryCache.packageReleases = Object.create(null); // Because we might need a "constructor" key
+ regCache.contentLength = 0;
+ regCache.packageReleases = {};
logger.debug({ err }, 'Rubygems fetch error');
throw new ExternalHostError(new Error('Rubygems fetch error'));
}
logger.debug('Rubygems: No update');
- this.registryCache.lastSync = new Date();
+ regCache.lastSync = new Date();
return;
}
for (const line of newLines.split(newlineRegex)) {
- this.processLine(line);
+ this.processLine(regCache, line);
}
- this.registryCache.lastSync = new Date();
+ regCache.lastSync = new Date();
}
- private processLine(line: string): void {
+ private processLine(regCache: RegistryCache, line: string): void {
let split: string[] | undefined;
let pkg: string | undefined;
let versions: string | undefined;
@@ -122,19 +119,17 @@ export class VersionsDatasource extends Datasource {
split = l.split(' ');
[pkg, versions] = split;
pkg = VersionsDatasource.copystr(pkg);
- this.registryCache.packageReleases[pkg] =
- this.registryCache.packageReleases[pkg] || [];
+ regCache.packageReleases[pkg] ??= [];
const lineVersions = versions.split(',').map((version) => version.trim());
for (const lineVersion of lineVersions) {
if (lineVersion.startsWith('-')) {
const deletedVersion = lineVersion.slice(1);
logger.trace({ pkg, deletedVersion }, 'Rubygems: Deleting version');
- this.registryCache.packageReleases[pkg] =
- this.registryCache.packageReleases[pkg].filter(
- (version) => version !== deletedVersion
- );
+ regCache.packageReleases[pkg] = regCache.packageReleases[pkg].filter(
+ (version) => version !== deletedVersion
+ );
} else {
- this.registryCache.packageReleases[pkg].push(
+ regCache.packageReleases[pkg].push(
VersionsDatasource.copystr(lineVersion)
);
}
@@ -147,16 +142,17 @@ export class VersionsDatasource extends Datasource {
}
}
- private isDataStale(): boolean {
- return getElapsedMinutes(this.registryCache.lastSync) >= 15;
+ private isDataStale({ lastSync }: RegistryCache): boolean {
+ return getElapsedMinutes(lastSync) >= 15;
}
private updateRubyGemsVersionsPromise: Promise<void> | null = null;
- async syncVersions(): Promise<void> {
- if (this.isDataStale()) {
+ async syncVersions(regCache: RegistryCache): Promise<void> {
+ if (this.isDataStale(regCache)) {
this.updateRubyGemsVersionsPromise =
- this.updateRubyGemsVersionsPromise ?? this.updateRubyGemsVersions();
+ this.updateRubyGemsVersionsPromise ??
+ this.updateRubyGemsVersions(regCache);
await this.updateRubyGemsVersionsPromise;
this.updateRubyGemsVersionsPromise = null;
}
--
GitLab