diff --git a/lib/modules/datasource/rubygems/versions-datasource.ts b/lib/modules/datasource/rubygems/versions-datasource.ts index 6471860f0b65da337fd1c7ccd718973a5f947904..e398bd0a5c769a8b4c5353caada85a39b6e70a98 100644 --- a/lib/modules/datasource/rubygems/versions-datasource.ts +++ b/lib/modules/datasource/rubygems/versions-datasource.ts @@ -1,15 +1,19 @@ +import { z } from 'zod'; import { PAGE_NOT_FOUND_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import { getElapsedMinutes } from '../../../util/date'; import { HttpError } from '../../../util/http'; import { newlineRegex } from '../../../util/regex'; +import { LooseArray } from '../../../util/schema-utils'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; +type PackageReleases = Map<string, string[]>; + interface RegistryCache { lastSync: Date; - packageReleases: Map<string, string[]>; // Because we might need a "constructor" key + packageReleases: PackageReleases; contentLength: number; isSupported: boolean; registryUrl: string; @@ -17,6 +21,35 @@ interface RegistryCache { export const memCache = new Map<string, RegistryCache>(); +const Lines = z + .string() + .transform((x) => x.split(newlineRegex)) + .pipe( + LooseArray( + z + .string() + .transform((line) => line.trim()) + .refine((line) => line.length > 0) + .refine((line) => !line.startsWith('created_at:')) + .refine((line) => line !== '---') + .transform((line) => line.split(' ')) + .pipe(z.tuple([z.string(), z.string()]).rest(z.string())) + .transform(([packageName, versions]) => { + const deletedVersions = new Set<string>(); + const addedVersions: string[] = []; + for (const version of versions.split(',')) { + if (version.startsWith('-')) { + deletedVersions.add(version.slice(1)); + } else { + addedVersions.push(version); + } + } + return { packageName, deletedVersions, addedVersions }; + }) + ) + ); +type Lines = z.infer<typeof Lines>; + export class VersionsDatasource extends Datasource { constructor(override readonly id: string) { super(id); @@ -69,6 +102,34 @@ export class VersionsDatasource extends Datasource { return (' ' + x).slice(1); } + private updatePackageReleases( + packageReleases: PackageReleases, + lines: Lines + ): void { + for (const line of lines) { + const packageName = VersionsDatasource.copystr(line.packageName); + let versions = packageReleases.get(packageName) ?? []; + + const { deletedVersions, addedVersions } = line; + + if (deletedVersions.size > 0) { + versions = versions.filter((v) => !deletedVersions.has(v)); + } + + if (addedVersions.length > 0) { + const existingVersions = new Set(versions); + for (const addedVersion of addedVersions) { + if (!existingVersions.has(addedVersion)) { + const version = VersionsDatasource.copystr(addedVersion); + versions.push(version); + } + } + } + + packageReleases.set(packageName, versions); + } + } + async updateRubyGemsVersions(regCache: RegistryCache): Promise<void> { const url = `${regCache.registryUrl}/versions`; const options = { @@ -84,71 +145,37 @@ export class VersionsDatasource extends Datasource { newLines = (await this.http.get(url, options)).body; const durationMs = Math.round(Date.now() - startTime); logger.debug(`Rubygems: Fetched rubygems.org versions in ${durationMs}`); - regCache.isSupported = true; } catch (err) /* istanbul ignore next */ { if (err instanceof HttpError && err.response?.statusCode === 404) { regCache.isSupported = false; return; } - if (err.statusCode !== 416) { - regCache.contentLength = 0; - regCache.packageReleases = new Map<string, string[]>(); - logger.debug({ err }, 'Rubygems fetch error'); - throw new ExternalHostError(new Error('Rubygems fetch error')); - } - logger.debug('Rubygems: No update'); - regCache.lastSync = new Date(); - return; - } - for (const line of newLines.split(newlineRegex)) { - this.processLine(regCache, line); - } - regCache.lastSync = new Date(); - } - - private processLine(regCache: RegistryCache, line: string): void { - let split: string[] | undefined; - let pkg: string | undefined; - let versions: string | undefined; - try { - const l = line.trim(); - if (!l.length || l.startsWith('created_at:') || l === '---') { + if (err.statusCode === 416) { + logger.debug('Rubygems: No update'); + regCache.lastSync = new Date(); return; } - split = l.split(' '); - [pkg, versions] = split; - pkg = VersionsDatasource.copystr(pkg); - let existingVersions = regCache.packageReleases.get(pkg) ?? []; - const lineVersions = versions.split(',').map((version) => version.trim()); - for (const lineVersion of lineVersions) { - if (lineVersion.startsWith('-')) { - const deletedVersion = lineVersion.slice(1); - logger.trace({ pkg, deletedVersion }, 'Rubygems: Deleting version'); - existingVersions = existingVersions.filter( - (version) => version !== deletedVersion - ); - } else { - existingVersions.push(VersionsDatasource.copystr(lineVersion)); - } - } - regCache.packageReleases.set(pkg, existingVersions); - } catch (err) /* istanbul ignore next */ { - logger.warn( - { err, line, split, pkg, versions }, - 'Rubygems line parsing error' - ); + + regCache.contentLength = 0; + regCache.packageReleases.clear(); + + logger.debug({ err }, 'Rubygems fetch error'); + throw new ExternalHostError(err); } - } - private isDataStale({ lastSync }: RegistryCache): boolean { - return getElapsedMinutes(lastSync) >= 15; + regCache.isSupported = true; + regCache.lastSync = new Date(); + + const lines = Lines.parse(newLines); + this.updatePackageReleases(regCache.packageReleases, lines); } private updateRubyGemsVersionsPromise: Promise<void> | null = null; async syncVersions(regCache: RegistryCache): Promise<void> { - if (this.isDataStale(regCache)) { + const isStale = getElapsedMinutes(regCache.lastSync) >= 15; + if (isStale) { this.updateRubyGemsVersionsPromise = this.updateRubyGemsVersionsPromise ?? this.updateRubyGemsVersions(regCache);