diff --git a/lib/datasource/crate/index.spec.ts b/lib/datasource/crate/index.spec.ts index e39e2612b82baf1e3a53399440d96a7f0efa805a..5b4c7f29538353141115f4b226eb825310985b09 100644 --- a/lib/datasource/crate/index.spec.ts +++ b/lib/datasource/crate/index.spec.ts @@ -1,3 +1,4 @@ +import delay from 'delay'; import fs from 'fs-extra'; import _simpleGit from 'simple-git'; import { DirectoryResult, dir } from 'tmp-promise'; @@ -28,15 +29,36 @@ const res3 = fs.readFileSync('lib/datasource/crate/__fixtures__/mypkg', 'utf8'); const baseUrl = 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/'; -function setupGitMocks(): { mockClone: jest.Mock<any, any> } { +function setupGitMocks(delayMs?: number): { mockClone: jest.Mock<any, any> } { const mockClone = jest .fn() .mockName('clone') - .mockImplementation((_registryUrl: string, clonePath: string, _opts) => { - const path = `${clonePath}/my/pk/mypkg`; - fs.mkdirSync(dirname(path), { recursive: true }); - fs.writeFileSync(path, res3, { encoding: 'utf8' }); - }); + .mockImplementation( + async (_registryUrl: string, clonePath: string, _opts) => { + if (delayMs > 0) { + await delay(delayMs); + } + + const path = `${clonePath}/my/pk/mypkg`; + fs.mkdirSync(dirname(path), { recursive: true }); + fs.writeFileSync(path, res3, { encoding: 'utf8' }); + } + ); + + simpleGit.mockReturnValue({ + clone: mockClone, + }); + + return { mockClone }; +} + +function setupErrorGitMock(): { mockClone: jest.Mock<any, any> } { + const mockClone = jest + .fn() + .mockName('clone') + .mockImplementation((_registryUrl: string, _clonePath: string, _opts) => + Promise.reject(new Error('mocked error')) + ); simpleGit.mockReturnValue({ clone: mockClone, @@ -259,6 +281,51 @@ describe('datasource/crate', () => { }); expect(mockClone).toHaveBeenCalledTimes(1); }); + it('guards against race conditions while cloning', async () => { + const { mockClone } = setupGitMocks(250); + setAdminConfig({ trustLevel: 'high' }); + const url = 'https://github.com/mcorbin/othertestregistry'; + + await Promise.all([ + getPkgReleases({ + datasource, + depName: 'mypkg', + registryUrls: [url], + }), + getPkgReleases({ + datasource, + depName: 'mypkg-2', + registryUrls: [url], + }), + ]); + + await getPkgReleases({ + datasource, + depName: 'mypkg-3', + registryUrls: [url], + }); + + expect(mockClone).toHaveBeenCalledTimes(1); + }); + it('returns null when git clone fails', async () => { + setupErrorGitMock(); + setAdminConfig({ trustLevel: 'high' }); + const url = 'https://github.com/mcorbin/othertestregistry'; + + const result = await getPkgReleases({ + datasource, + depName: 'mypkg', + registryUrls: [url], + }); + const result2 = await getPkgReleases({ + datasource, + depName: 'mypkg-2', + registryUrls: [url], + }); + + expect(result).toBeNull(); + expect(result2).toBeNull(); + }); }); describe('fetchCrateRecordsPayload', () => { diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index 1655293c9d559f5edc4f7b631bba02723aba80c4..617c326565b8e33b770754995e6197fcacfa81f3 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -170,18 +170,53 @@ async function fetchRegistryInfo( } const cacheKey = `crate-datasource/registry-clone-path/${registryUrl}`; + const cacheKeyForError = `crate-datasource/registry-clone-path/${registryUrl}/error`; - let clonePath: string = memCache.get(cacheKey); - if (!clonePath) { + // We need to ensure we don't run `git clone` in parallel. Therefore we store + // a promise of the running operation in the mem cache, which in the end resolves + // to the file path of the cloned repository. + + const clonePathPromise: Promise<string> | null = memCache.get(cacheKey); + let clonePath: string; + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + if (clonePathPromise) { + clonePath = await clonePathPromise; + } else { clonePath = join(privateCacheDir(), cacheDirFromUrl(url)); logger.info({ clonePath, registryUrl }, `Cloning private cargo registry`); - { - const git = Git(); - await git.clone(registryUrl, clonePath, { - '--depth': 1, - }); + + const git = Git(); + const clonePromise = git.clone(registryUrl, clonePath, { + '--depth': 1, + }); + + memCache.set( + cacheKey, + clonePromise.then(() => clonePath).catch(() => null) + ); + + try { + await clonePromise; + } catch (err) { + logger.warn( + { err, lookupName: config.lookupName, registryUrl }, + 'failed cloning git registry' + ); + memCache.set(cacheKeyForError, err); + + return null; } - memCache.set(cacheKey, clonePath); + } + + if (!clonePath) { + const err = memCache.get(cacheKeyForError); + logger.warn( + { err, lookupName: config.lookupName, registryUrl }, + 'Previous git clone failed, bailing out.' + ); + + return null; } registry.clonePath = clonePath;