From 991f69c36b2c85432debbe790cfef74ba174a312 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Sun, 12 May 2019 15:50:29 +0200
Subject: [PATCH] fix(bundler): refactor rubygems.org fetching to use CDN
(#3669)
Refactors fetching of dependencies from Rubygems.org to use the /versions file instead of the official API. For now this means no metadata from Rubygems, so it will be added in a future PR.
Closes #3373
---
lib/datasource/rubygems/get-rubygems-org.js | 96 +++++++++++++++++++
lib/datasource/rubygems/releases.js | 8 +-
.../rubygems/__snapshots__/index.spec.js.snap | 14 +++
test/datasource/rubygems/index.spec.js | 43 +++++++++
4 files changed, 160 insertions(+), 1 deletion(-)
create mode 100644 lib/datasource/rubygems/get-rubygems-org.js
diff --git a/lib/datasource/rubygems/get-rubygems-org.js b/lib/datasource/rubygems/get-rubygems-org.js
new file mode 100644
index 0000000000..55e1409766
--- /dev/null
+++ b/lib/datasource/rubygems/get-rubygems-org.js
@@ -0,0 +1,96 @@
+const got = require('../../util/got');
+
+module.exports = {
+ getRubygemsOrgDependency,
+};
+
+let lastSync = new Date('2000-01-01');
+let packageReleases = Object.create(null); // Because we might need a "constructor" key
+let contentLength = 0;
+
+async function updateRubyGemsVersions() {
+ const url = 'https://rubygems.org/versions';
+ const options = {
+ headers: {
+ 'accept-encoding': 'identity',
+ range: `bytes=${contentLength}-`,
+ },
+ };
+ let newLines;
+ try {
+ logger.debug('Rubygems: Fetching rubygems.org versions');
+ newLines = (await got(url, options)).body;
+ } catch (err) /* istanbul ignore next */ {
+ if (err.statusCode === 416) {
+ logger.debug('Rubygems: No update');
+ } else {
+ logger.warn({ err }, 'Rubygems error - resetting cache');
+ contentLength = 0;
+ packageReleases = Object.create(null); // Because we might need a "constructor" key
+ }
+ lastSync = new Date();
+ return;
+ }
+
+ function processLine(line) {
+ let split;
+ let pkg;
+ let versions;
+ try {
+ const l = line.trim();
+ if (!l.length || l.startsWith('created_at:') || l === '---') {
+ return;
+ }
+ split = l.split(' ');
+ [pkg, versions] = split;
+ packageReleases[pkg] = packageReleases[pkg] || [];
+ const lineVersions = versions.split(',').map(version => version.trim());
+ for (const lineVersion of lineVersions) {
+ if (lineVersion.startsWith('-')) {
+ const deletedVersion = lineVersion.slice(1);
+ logger.trace({ pkg, deletedVersion }, 'Rubygems: Deleting version');
+ packageReleases[pkg] = packageReleases[pkg].filter(
+ version => version !== deletedVersion
+ );
+ } else {
+ packageReleases[pkg].push(lineVersion);
+ }
+ }
+ } catch (err) /* istanbul ignore next */ {
+ logger.warn(
+ { err, line, split, pkg, versions },
+ 'Rubygems line parsing error'
+ );
+ }
+ }
+
+ for (const line of newLines.split('\n')) {
+ processLine(line);
+ }
+ lastSync = new Date();
+}
+
+function isDataStale() {
+ const minutesElapsed = Math.floor((new Date() - lastSync) / (60 * 1000));
+ return minutesElapsed >= 5;
+}
+
+async function syncVersions() {
+ if (isDataStale()) {
+ global.updateRubyGemsVersions =
+ global.updateRubyGemsVersions || updateRubyGemsVersions();
+ await global.updateRubyGemsVersions;
+ }
+}
+
+async function getRubygemsOrgDependency(lookupName) {
+ await syncVersions();
+ if (!packageReleases[lookupName]) {
+ return null;
+ }
+ const dep = {
+ name: lookupName,
+ releases: packageReleases[lookupName].map(version => ({ version })),
+ };
+ return dep;
+}
diff --git a/lib/datasource/rubygems/releases.js b/lib/datasource/rubygems/releases.js
index cb91619be3..3e1af26328 100644
--- a/lib/datasource/rubygems/releases.js
+++ b/lib/datasource/rubygems/releases.js
@@ -1,11 +1,17 @@
const { nonEmptyArray } = require('@sindresorhus/is');
const { getDependency } = require('./get');
+const { getRubygemsOrgDependency } = require('./get-rubygems-org');
async function getPkgReleases({ lookupName, registryUrls }) {
const registries = nonEmptyArray(registryUrls) ? registryUrls : [];
for (const registry of registries) {
- const pkg = await getDependency({ dependency: lookupName, registry });
+ let pkg;
+ if (registry.endsWith('rubygems.org')) {
+ pkg = await getRubygemsOrgDependency(lookupName);
+ } else {
+ pkg = await getDependency({ dependency: lookupName, registry });
+ }
if (pkg) {
return pkg;
}
diff --git a/test/datasource/rubygems/__snapshots__/index.spec.js.snap b/test/datasource/rubygems/__snapshots__/index.spec.js.snap
index 8e5bc155be..1c1ef2c862 100644
--- a/test/datasource/rubygems/__snapshots__/index.spec.js.snap
+++ b/test/datasource/rubygems/__snapshots__/index.spec.js.snap
@@ -1,5 +1,19 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
+exports[`datasource/rubygems getPkgReleases returns a dep for rubygems.org package hit 1`] = `
+Object {
+ "name": "1pass",
+ "releases": Array [
+ Object {
+ "version": "0.1.0",
+ },
+ Object {
+ "version": "0.1.1",
+ },
+ ],
+}
+`;
+
exports[`datasource/rubygems getPkgReleases uses multiple source urls 1`] = `
Object {
"changelogUrl": null,
diff --git a/test/datasource/rubygems/index.spec.js b/test/datasource/rubygems/index.spec.js
index db8a65e141..cbe9db9f86 100644
--- a/test/datasource/rubygems/index.spec.js
+++ b/test/datasource/rubygems/index.spec.js
@@ -3,6 +3,25 @@ const railsInfo = require('./_fixtures/rails/info.json');
const railsVersions = require('./_fixtures/rails/versions.json');
const rubygems = require('../../../lib/datasource/rubygems/index.js');
+const rubygemsOrgVersions = `created_at: 2017-03-27T04:38:13+00:00
+---
+- 1 05d0116933ba44b0b5d0ee19bfd35ccc
+.cat 0.0.1 631fd60a806eaf5026c86fff3155c289
+0mq 0.1.0,0.1.1,0.1.2,0.2.0,0.2.1,0.3.0,0.4.0,0.4.1,0.5.0,0.5.1,0.5.2,0.5.3 6146193f8f7e944156b0b42ec37bad3e
+0xffffff 0.0.1,0.1.0 0a4a9aeae24152cdb467be02f40482f9
+10to1-crack 0.1.1,0.1.2,0.1.3 e7218e76477e2137355d2e7ded094925
+1234567890_ 1.0,1.1 233e818c2db65d2dad9f9ea9a27b1a30
+12_hour_time 0.0.2,0.0.3,0.0.4 4e58bc03e301f704950410b713c20b69
+16watts-fluently 0.3.0,0.3.1 555088e2b18e97e0293cab1d90dbb0d2
+189seg 0.0.1 c4d329f7d3eb88b6e602358968be0242
+196demo 0.0.0 e00c558565f7b03a438fbd93d854b7de
+1_as_identity_function 1.0.0,1.0.1 bee2f0fbbc3c5c83008c0b8fc64cb168
+1and1 1.1 1853e4495b036ddc5da2035523d48f0d
+1hdoc 0.1.3,0.2.0,0.2.2,0.2.3,0.2.4 7076f29c196df12047a3700c4d6e5915
+1pass 0.1.0,0.1.1,0.1.2 d209547aae4b8f3d67123f18f738ac99
+1pass -0.1.2 abcdef
+21-day-challenge-countdown 0.1.0,0.1.1,0.1.2 57e8873fe713063f4e54e85bbbd709bb`;
+
jest.mock('../../../lib/util/got');
describe('datasource/rubygems', () => {
@@ -24,6 +43,30 @@ describe('datasource/rubygems', () => {
expect(await rubygems.getPkgReleases(params)).toBeNull();
});
+ it('returns null for rubygems.org package miss', async () => {
+ const newparams = { ...params };
+ newparams.registryUrls = ['https://rubygems.org'];
+ got.mockReturnValueOnce({ body: rubygemsOrgVersions });
+ expect(await rubygems.getPkgReleases(newparams)).toBeNull();
+ });
+
+ it('returns a dep for rubygems.org package hit', async () => {
+ const newparams = {
+ lookupName: '1pass',
+ registryUrls: ['https://rubygems.org'],
+ };
+ got.mockReturnValueOnce({ body: rubygemsOrgVersions });
+ const res = await rubygems.getPkgReleases(newparams);
+ expect(res).not.toBeNull();
+ expect(res).toMatchSnapshot();
+ expect(
+ res.releases.find(release => release.version === '0.1.1')
+ ).toBeDefined();
+ expect(
+ res.releases.find(release => release.version === '0.1.2')
+ ).toBeUndefined();
+ });
+
it('works with real data', async () => {
got
.mockReturnValueOnce({ body: railsInfo })
--
GitLab