diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts
index 6469279f8c6cdfc6089b7075a427d97620650ce2..d7619f3ac6c2ec700fd3855210e46f50d3cd0cd6 100644
--- a/lib/datasource/api.ts
+++ b/lib/datasource/api.ts
@@ -30,7 +30,7 @@ import * as pod from './pod';
import { PypiDatasource } from './pypi';
import * as repology from './repology';
import { RubyVersionDatasource } from './ruby-version';
-import * as rubygems from './rubygems';
+import { RubyGemsDatasource } from './rubygems';
import * as sbtPackage from './sbt-package';
import * as sbtPlugin from './sbt-plugin';
import { TerraformModuleDatasource } from './terraform-module';
@@ -72,7 +72,7 @@ api.set('pod', pod);
api.set('pypi', new PypiDatasource());
api.set('repology', repology);
api.set('ruby-version', new RubyVersionDatasource());
-api.set('rubygems', rubygems);
+api.set(RubyGemsDatasource.id, new RubyGemsDatasource());
api.set('sbt-package', sbtPackage);
api.set('sbt-plugin', sbtPlugin);
api.set('terraform-module', new TerraformModuleDatasource());
diff --git a/lib/datasource/rubygems/common.ts b/lib/datasource/rubygems/common.ts
deleted file mode 100644
index ce52a088a1e9c6f278fa0ca01eb3690a64ba950a..0000000000000000000000000000000000000000
--- a/lib/datasource/rubygems/common.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import Marshal from 'marshal';
-import urlJoin from 'url-join';
-import { logger } from '../../logger';
-import { Http } from '../../util/http';
-import { getQueryString } from '../../util/url';
-
-export const id = 'rubygems';
-export const http = new Http(id);
-
-export const knownFallbackHosts = ['rubygems.pkg.github.com', 'gitlab.com'];
-
-export async function fetchJson<T>(
- dependency: string,
- registry: string,
- path: string
-): Promise<T> {
- const url = urlJoin(registry, path, `${dependency}.json`);
-
- logger.trace({ registry, dependency, url }, `RubyGems lookup request`);
- const response = (await http.getJson<T>(url)) || {
- body: undefined,
- };
-
- return response.body;
-}
-
-export async function fetchBuffer<T>(
- dependency: string,
- registry: string,
- path: string
-): Promise<T> {
- const url = `${urlJoin(registry, path)}?${getQueryString({
- gems: dependency,
- })}`;
-
- logger.trace({ registry, dependency, url }, `RubyGems lookup request`);
- const response = await http.getBuffer(url);
-
- return new Marshal(response.body).parsed as T;
-}
diff --git a/lib/datasource/rubygems/get-rubygems-org.ts b/lib/datasource/rubygems/get-rubygems-org.ts
index 7ca2bdcda8daf38b807adf07df0122f7298fe1d8..3be067768bda5c8ce004bf3a2adafeb57243fc0b 100644
--- a/lib/datasource/rubygems/get-rubygems-org.ts
+++ b/lib/datasource/rubygems/get-rubygems-org.ts
@@ -1,8 +1,8 @@
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import { getElapsedMinutes } from '../../util/date';
-import type { ReleaseResult } from '../types';
-import { http } from './common';
+import { Datasource } from '../datasource';
+import type { GetReleasesConfig, ReleaseResult } from '../types';
let lastSync = new Date('2000-01-01');
let packageReleases: Record<string, string[]> = Object.create(null); // Because we might need a "constructor" key
@@ -15,38 +15,69 @@ export function resetCache(): void {
contentLength = 0;
}
-/* https://bugs.chromium.org/p/v8/issues/detail?id=2869 */
-const copystr = (x: string): string => (' ' + x).slice(1);
+export class RubyGemsOrgDatasource extends Datasource {
+ constructor(override readonly id: string) {
+ super(id);
+ }
-async function updateRubyGemsVersions(): Promise<void> {
- const url = 'https://rubygems.org/versions';
- const options = {
- headers: {
- 'accept-encoding': 'identity',
- range: `bytes=${contentLength}-`,
- },
- };
- let newLines: string;
- try {
- logger.debug('Rubygems: Fetching rubygems.org versions');
- const startTime = Date.now();
- newLines = (await http.get(url, options)).body;
- const durationMs = Math.round(Date.now() - startTime);
- logger.debug({ durationMs }, 'Rubygems: Fetched rubygems.org versions');
- } catch (err) /* istanbul ignore next */ {
- if (err.statusCode !== 416) {
- contentLength = 0;
- packageReleases = Object.create(null); // Because we might need a "constructor" key
- throw new ExternalHostError(
- new Error('Rubygems fetch error - need to reset cache')
- );
+ async getReleases({
+ lookupName,
+ }: GetReleasesConfig): Promise<ReleaseResult | null> {
+ logger.debug(`getRubygemsOrgDependency(${lookupName})`);
+ await this.syncVersions();
+ if (!packageReleases[lookupName]) {
+ return null;
+ }
+ const dep: ReleaseResult = {
+ releases: packageReleases[lookupName].map((version) => ({
+ version,
+ })),
+ };
+ return dep;
+ }
+
+ /**
+ * https://bugs.chromium.org/p/v8/issues/detail?id=2869
+ */
+ private static copystr(x: string): string {
+ return (' ' + x).slice(1);
+ }
+
+ async updateRubyGemsVersions(): Promise<void> {
+ const url = 'https://rubygems.org/versions';
+ const options = {
+ headers: {
+ 'accept-encoding': 'identity',
+ range: `bytes=${contentLength}-`,
+ },
+ };
+ let newLines: string;
+ try {
+ logger.debug('Rubygems: Fetching rubygems.org versions');
+ const startTime = Date.now();
+ newLines = (await this.http.get(url, options)).body;
+ const durationMs = Math.round(Date.now() - startTime);
+ logger.debug({ durationMs }, 'Rubygems: Fetched rubygems.org versions');
+ } catch (err) /* istanbul ignore next */ {
+ if (err.statusCode !== 416) {
+ contentLength = 0;
+ packageReleases = Object.create(null); // Because we might need a "constructor" key
+ throw new ExternalHostError(
+ new Error('Rubygems fetch error - need to reset cache')
+ );
+ }
+ logger.debug('Rubygems: No update');
+ lastSync = new Date();
+ return;
+ }
+
+ for (const line of newLines.split('\n')) {
+ RubyGemsOrgDatasource.processLine(line);
}
- logger.debug('Rubygems: No update');
lastSync = new Date();
- return;
}
- function processLine(line: string): void {
+ private static processLine(line: string): void {
let split: string[];
let pkg: string;
let versions: string;
@@ -57,7 +88,7 @@ async function updateRubyGemsVersions(): Promise<void> {
}
split = l.split(' ');
[pkg, versions] = split;
- pkg = copystr(pkg);
+ pkg = RubyGemsOrgDatasource.copystr(pkg);
packageReleases[pkg] = packageReleases[pkg] || [];
const lineVersions = versions.split(',').map((version) => version.trim());
for (const lineVersion of lineVersions) {
@@ -68,7 +99,7 @@ async function updateRubyGemsVersions(): Promise<void> {
(version) => version !== deletedVersion
);
} else {
- packageReleases[pkg].push(copystr(lineVersion));
+ packageReleases[pkg].push(RubyGemsOrgDatasource.copystr(lineVersion));
}
}
} catch (err) /* istanbul ignore next */ {
@@ -79,38 +110,19 @@ async function updateRubyGemsVersions(): Promise<void> {
}
}
- for (const line of newLines.split('\n')) {
- processLine(line);
+ private static isDataStale(): boolean {
+ return getElapsedMinutes(lastSync) >= 5;
}
- lastSync = new Date();
-}
-
-function isDataStale(): boolean {
- return getElapsedMinutes(lastSync) >= 5;
-}
-let updateRubyGemsVersionsPromise: Promise<void> | undefined;
+ updateRubyGemsVersionsPromise: Promise<void> | undefined;
-async function syncVersions(): Promise<void> {
- if (isDataStale()) {
- updateRubyGemsVersionsPromise =
- // eslint-disable-next-line @typescript-eslint/no-misused-promises
- updateRubyGemsVersionsPromise || updateRubyGemsVersions();
- await updateRubyGemsVersionsPromise;
- updateRubyGemsVersionsPromise = null;
- }
-}
-
-export async function getRubygemsOrgDependency(
- lookupName: string
-): Promise<ReleaseResult | null> {
- logger.debug(`getRubygemsOrgDependency(${lookupName})`);
- await syncVersions();
- if (!packageReleases[lookupName]) {
- return null;
+ async syncVersions(): Promise<void> {
+ if (RubyGemsOrgDatasource.isDataStale()) {
+ this.updateRubyGemsVersionsPromise =
+ // eslint-disable-next-line @typescript-eslint/no-misused-promises
+ this.updateRubyGemsVersionsPromise || this.updateRubyGemsVersions();
+ await this.updateRubyGemsVersionsPromise;
+ this.updateRubyGemsVersionsPromise = null;
+ }
}
- const dep: ReleaseResult = {
- releases: packageReleases[lookupName].map((version) => ({ version })),
- };
- return dep;
}
diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts
index 6d1d491b198c2aa68469f1b7b8ff594fe301e3b3..a6b45f197ef8a9c3ec358bcc782cf98a6d6e8628 100644
--- a/lib/datasource/rubygems/get.ts
+++ b/lib/datasource/rubygems/get.ts
@@ -1,7 +1,9 @@
+import Marshal from 'marshal';
import { logger } from '../../logger';
import { HttpError } from '../../util/http/types';
-import type { Release, ReleaseResult } from '../types';
-import { fetchBuffer, fetchJson } from './common';
+import { getQueryString, joinUrlParts, parseUrl } from '../../util/url';
+import { Datasource } from '../datasource';
+import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
import type {
JsonGemVersions,
JsonGemsInfo,
@@ -12,114 +14,164 @@ const INFO_PATH = '/api/v1/gems';
const VERSIONS_PATH = '/api/v1/versions';
const DEPENDENCIES_PATH = '/api/v1/dependencies';
-export async function getDependencyFallback(
- dependency: string,
- registry: string
-): Promise<ReleaseResult | null> {
- logger.debug(
- { dependency, api: DEPENDENCIES_PATH },
- 'RubyGems lookup for dependency'
- );
- const info = await fetchBuffer<MarshalledVersionInfo[]>(
- dependency,
- registry,
- DEPENDENCIES_PATH
- );
- if (!info || info.length === 0) {
- return null;
+export class InternalRubyGemsDatasource extends Datasource {
+ constructor(override readonly id: string) {
+ super(id);
}
- const releases = info.map(({ number: version, platform: rubyPlatform }) => ({
- version,
- rubyPlatform,
- }));
- return {
- releases,
- homepage: null,
- sourceUrl: null,
- changelogUrl: null,
- };
-}
-export async function getDependency(
- dependency: string,
- registry: string
-): Promise<ReleaseResult | null> {
- logger.debug(
- { dependency, api: INFO_PATH },
- 'RubyGems lookup for dependency'
- );
- let info: JsonGemsInfo;
-
- try {
- info = await fetchJson(dependency, registry, INFO_PATH);
- } catch (error) {
- // fallback to deps api on 404
- if (error instanceof HttpError && error.response?.statusCode === 404) {
- return await getDependencyFallback(dependency, registry);
+ private knownFallbackHosts = ['rubygems.pkg.github.com', 'gitlab.com'];
+
+ override getReleases({
+ lookupName,
+ registryUrl,
+ }: GetReleasesConfig): Promise<ReleaseResult | null> {
+ if (this.knownFallbackHosts.includes(parseUrl(registryUrl)?.hostname)) {
+ return this.getDependencyFallback(lookupName, registryUrl);
}
- throw error;
+ return this.getDependency(lookupName, registryUrl);
}
- if (!info) {
- logger.debug({ dependency }, 'RubyGems package not found.');
- return null;
+ async getDependencyFallback(
+ dependency: string,
+ registry: string
+ ): Promise<ReleaseResult | null> {
+ logger.debug(
+ { dependency, api: DEPENDENCIES_PATH },
+ 'RubyGems lookup for dependency'
+ );
+ const info = await this.fetchBuffer<MarshalledVersionInfo[]>(
+ dependency,
+ registry,
+ DEPENDENCIES_PATH
+ );
+ if (!info || info.length === 0) {
+ return null;
+ }
+ const releases = info.map(
+ ({ number: version, platform: rubyPlatform }) => ({
+ version,
+ rubyPlatform,
+ })
+ );
+ return {
+ releases,
+ homepage: null,
+ sourceUrl: null,
+ changelogUrl: null,
+ };
}
- if (dependency.toLowerCase() !== info.name.toLowerCase()) {
- logger.warn(
- { lookup: dependency, returned: info.name },
- 'Lookup name does not match with returned.'
+ async getDependency(
+ dependency: string,
+ registry: string
+ ): Promise<ReleaseResult | null> {
+ logger.debug(
+ { dependency, api: INFO_PATH },
+ 'RubyGems lookup for dependency'
);
- return null;
- }
+ let info: JsonGemsInfo;
+
+ try {
+ info = await this.fetchJson(dependency, registry, INFO_PATH);
+ } catch (error) {
+ // fallback to deps api on 404
+ if (error instanceof HttpError && error.response?.statusCode === 404) {
+ return await this.getDependencyFallback(dependency, registry);
+ }
+ throw error;
+ }
- let versions: JsonGemVersions[] = [];
- let releases: Release[] = [];
- try {
- versions = await fetchJson(dependency, registry, VERSIONS_PATH);
- } catch (err) {
- if (err.statusCode === 400 || err.statusCode === 404) {
- logger.debug(
- { registry },
- 'versions endpoint returns error - falling back to info endpoint'
+ if (!info) {
+ logger.debug({ dependency }, 'RubyGems package not found.');
+ return null;
+ }
+
+ if (dependency.toLowerCase() !== info.name.toLowerCase()) {
+ logger.warn(
+ { lookup: dependency, returned: info.name },
+ 'Lookup name does not match with returned.'
);
+ return null;
+ }
+
+ let versions: JsonGemVersions[] = [];
+ let releases: Release[] = [];
+ try {
+ versions = await this.fetchJson(dependency, registry, VERSIONS_PATH);
+ } catch (err) {
+ if (err.statusCode === 400 || err.statusCode === 404) {
+ logger.debug(
+ { registry },
+ 'versions endpoint returns error - falling back to info endpoint'
+ );
+ } else {
+ throw err;
+ }
+ }
+
+ // TODO: invalid properties for `Release` see #11312
+
+ if (versions.length === 0 && info.version) {
+ logger.warn('falling back to the version from the info endpoint');
+ releases = [
+ {
+ version: info.version,
+ rubyPlatform: info.platform,
+ } as Release,
+ ];
} else {
- throw err;
+ releases = versions.map(
+ ({
+ number: version,
+ platform: rubyPlatform,
+ created_at: releaseTimestamp,
+ rubygems_version: rubygemsVersion,
+ ruby_version: rubyVersion,
+ }) => ({
+ version,
+ rubyPlatform,
+ releaseTimestamp,
+ rubygemsVersion,
+ rubyVersion,
+ })
+ );
}
+
+ return {
+ releases,
+ homepage: info.homepage_uri,
+ sourceUrl: info.source_code_uri,
+ changelogUrl: info.changelog_uri,
+ };
}
- // TODO: invalid properties for `Release` see #11312
-
- if (versions.length === 0 && info.version) {
- logger.warn('falling back to the version from the info endpoint');
- releases = [
- {
- version: info.version,
- rubyPlatform: info.platform,
- } as Release,
- ];
- } else {
- releases = versions.map(
- ({
- number: version,
- platform: rubyPlatform,
- created_at: releaseTimestamp,
- rubygems_version: rubygemsVersion,
- ruby_version: rubyVersion,
- }) => ({
- version,
- rubyPlatform,
- releaseTimestamp,
- rubygemsVersion,
- rubyVersion,
- })
- );
+ private async fetchJson<T>(
+ dependency: string,
+ registry: string,
+ path: string
+ ): Promise<T> {
+ const url = joinUrlParts(registry, path, `${dependency}.json`);
+
+ logger.trace({ registry, dependency, url }, `RubyGems lookup request`);
+ const response = (await this.http.getJson<T>(url)) || {
+ body: undefined,
+ };
+
+ return response.body;
}
- return {
- releases,
- homepage: info.homepage_uri,
- sourceUrl: info.source_code_uri,
- changelogUrl: info.changelog_uri,
- };
+ private async fetchBuffer<T>(
+ dependency: string,
+ registry: string,
+ path: string
+ ): Promise<T> {
+ const url = `${joinUrlParts(registry, path)}?${getQueryString({
+ gems: dependency,
+ })}`;
+
+ logger.trace({ registry, dependency, url }, `RubyGems lookup request`);
+ const response = await this.http.getBuffer(url);
+
+ return new Marshal(response.body).parsed as T;
+ }
}
diff --git a/lib/datasource/rubygems/index.spec.ts b/lib/datasource/rubygems/index.spec.ts
index 913204eafb2b5b527147842d8bd75eef649da788..71451f040f6febd2cf92f437a3b4ee52fd48d928 100644
--- a/lib/datasource/rubygems/index.spec.ts
+++ b/lib/datasource/rubygems/index.spec.ts
@@ -7,7 +7,7 @@ import {
} from '../../../test/util';
import * as rubyVersioning from '../../versioning/ruby';
import { resetCache } from './get-rubygems-org';
-import * as rubygems from '.';
+import { RubyGemsDatasource } from '.';
const rubygemsOrgVersions = loadFixture('rubygems-org.txt');
const railsInfo = loadJsonFixture('rails/info.json');
@@ -21,7 +21,7 @@ describe('datasource/rubygems/index', () => {
const params = {
versioning: rubyVersioning.id,
- datasource: rubygems.id,
+ datasource: RubyGemsDatasource.id,
depName: 'rails',
registryUrls: [
'https://thirdparty.com',
diff --git a/lib/datasource/rubygems/index.ts b/lib/datasource/rubygems/index.ts
index 83ca50a1de8f639ae2e3bdd8a3fae9e428667df2..74da9e61e9a88927e44cdd2b81277dfd79d341d5 100644
--- a/lib/datasource/rubygems/index.ts
+++ b/lib/datasource/rubygems/index.ts
@@ -1,8 +1,49 @@
+import { cache } from '../../util/cache/package/decorator';
+import { parseUrl } from '../../util/url';
import * as rubyVersioning from '../../versioning/ruby';
+import { Datasource } from '../datasource';
+import { GetReleasesConfig, ReleaseResult } from '../types';
+import { InternalRubyGemsDatasource } from './get';
+import { RubyGemsOrgDatasource } from './get-rubygems-org';
-export { getReleases } from './releases';
-export { id } from './common';
-export const customRegistrySupport = true;
-export const defaultRegistryUrls = ['https://rubygems.org'];
-export const defaultVersioning = rubyVersioning.id;
-export const registryStrategy = 'hunt';
+export class RubyGemsDatasource extends Datasource {
+ static readonly id = 'rubygems';
+
+ constructor() {
+ super(RubyGemsDatasource.id);
+ this.rubyGemsOrgDatasource = new RubyGemsOrgDatasource(
+ RubyGemsDatasource.id
+ );
+ this.internalRubyGemsDatasource = new InternalRubyGemsDatasource(
+ RubyGemsDatasource.id
+ );
+ }
+
+ override readonly defaultRegistryUrls = ['https://rubygems.org'];
+
+ override readonly defaultVersioning = rubyVersioning.id;
+
+ override readonly registryStrategy = 'hunt';
+
+ private readonly rubyGemsOrgDatasource: RubyGemsOrgDatasource;
+
+ private readonly internalRubyGemsDatasource: InternalRubyGemsDatasource;
+
+ @cache({
+ namespace: `datasource-${RubyGemsDatasource.id}`,
+ key: ({ registryUrl, lookupName }: GetReleasesConfig) =>
+ `${registryUrl}/${lookupName}`,
+ })
+ getReleases({
+ lookupName,
+ registryUrl,
+ }: GetReleasesConfig): Promise<ReleaseResult | null> {
+ if (parseUrl(registryUrl)?.hostname === 'rubygems.org') {
+ return this.rubyGemsOrgDatasource.getReleases({ lookupName });
+ }
+ return this.internalRubyGemsDatasource.getReleases({
+ lookupName,
+ registryUrl,
+ });
+ }
+}
diff --git a/lib/datasource/rubygems/releases.ts b/lib/datasource/rubygems/releases.ts
deleted file mode 100644
index 5a4a2c9e9115bced342276218e64c4ae4c624b77..0000000000000000000000000000000000000000
--- a/lib/datasource/rubygems/releases.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import { parseUrl } from '../../util/url';
-import type { GetReleasesConfig, ReleaseResult } from '../types';
-import { knownFallbackHosts } from './common';
-import { getDependency, getDependencyFallback } from './get';
-import { getRubygemsOrgDependency } from './get-rubygems-org';
-
-export function getReleases({
- lookupName,
- registryUrl,
-}: GetReleasesConfig): Promise<ReleaseResult | null> {
- if (parseUrl(registryUrl)?.hostname === 'rubygems.org') {
- return getRubygemsOrgDependency(lookupName);
- }
- if (knownFallbackHosts.includes(parseUrl(registryUrl)?.hostname)) {
- return getDependencyFallback(lookupName, registryUrl);
- }
- return getDependency(lookupName, registryUrl);
-}
diff --git a/lib/manager/bundler/extract.ts b/lib/manager/bundler/extract.ts
index 228be6f5f26c3f7fce12c15d0154ad123741c31b..76853e34aed402c1ffbe810c8069b1ff76c18b13 100644
--- a/lib/manager/bundler/extract.ts
+++ b/lib/manager/bundler/extract.ts
@@ -1,4 +1,4 @@
-import * as datasourceRubygems from '../../datasource/rubygems';
+import { RubyGemsDatasource } from '../../datasource/rubygems';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import { readLocalFile } from '../../util/fs';
@@ -56,7 +56,7 @@ export async function extractPackageFile(
dep.skipReason = SkipReason.NoVersion;
}
if (!dep.skipReason) {
- dep.datasource = datasourceRubygems.id;
+ dep.datasource = RubyGemsDatasource.id;
}
res.deps.push(dep);
}
diff --git a/lib/workers/repository/init/vulnerability.ts b/lib/workers/repository/init/vulnerability.ts
index 5503daaeead4fb9b8f3dde6eb5279b2138f1d22c..321ddf5bbb5cc90a754599c60a5acdf41d3d7ce8 100644
--- a/lib/workers/repository/init/vulnerability.ts
+++ b/lib/workers/repository/init/vulnerability.ts
@@ -4,7 +4,7 @@ import * as datasourceMaven from '../../../datasource/maven';
import { id as npmId } from '../../../datasource/npm';
import * as datasourceNuget from '../../../datasource/nuget';
import { PypiDatasource } from '../../../datasource/pypi';
-import * as datasourceRubygems from '../../../datasource/rubygems';
+import { RubyGemsDatasource } from '../../../datasource/rubygems';
import { logger } from '../../../logger';
import { platform } from '../../../platform';
import { SecurityAdvisory } from '../../../types';
@@ -91,7 +91,7 @@ export async function detectVulnerabilityAlerts(
NPM: npmId,
NUGET: datasourceNuget.id,
PIP: PypiDatasource.id,
- RUBYGEMS: datasourceRubygems.id,
+ RUBYGEMS: RubyGemsDatasource.id,
};
const datasource =
datasourceMapping[alert.securityVulnerability.package.ecosystem];