import { join } from 'path'; import URL from 'url'; import fs from 'fs-extra'; import Git from 'simple-git/promise'; import { CONFIG_VALIDATION, PLATFORM_FAILURE, REPOSITORY_CHANGED, REPOSITORY_EMPTY, REPOSITORY_TEMPORARY_ERROR, } from '../../constants/error-messages'; import { logger } from '../../logger'; import * as limits from '../../workers/global/limits'; import { CommitFilesConfig } from '../common'; declare module 'fs-extra' { export function exists(pathLike: string): Promise<boolean>; } export type StatusResult = Git.StatusResult; interface StorageConfig { localDir: string; baseBranch?: string; url: string; gitPrivateKey?: string; extraCloneOpts?: Git.Options; } interface LocalConfig extends StorageConfig { baseBranch: string; baseBranchSha: string; branchExists: Record<string, boolean>; branchPrefix: string; fileList: Record<string, Promise<string[]>>; } // istanbul ignore next function checkForPlatformFailure(err: Error): void { if (process.env.NODE_ENV === 'test') { return; } const platformFailureStrings = [ 'remote: Invalid username or password', 'gnutls_handshake() failed', 'The requested URL returned error: 5', 'The remote end hung up unexpectedly', 'access denied or repository not exported', 'Could not write new index file', 'Failed to connect to', 'Connection timed out', ]; for (const errorStr of platformFailureStrings) { if (err.message.includes(errorStr)) { throw new Error(PLATFORM_FAILURE); } } } function localName(branchName: string): string { return branchName.replace(/^origin\//, ''); } function throwBaseBranchValidationError(branchName: string): never { const error = new Error(CONFIG_VALIDATION); error.validationError = 'baseBranch not found'; error.validationMessage = 'The following configured baseBranch could not be found: ' + branchName; throw error; } async function isDirectory(dir: string): Promise<boolean> { try { return (await fs.stat(dir)).isDirectory(); } catch (err) { return false; } } export class Storage { private _config: LocalConfig = {} as any; private _git: Git.SimpleGit | undefined; private _cwd: string | undefined; private async _resetToBranch(branchName: string): Promise<void> { logger.debug(`resetToBranch(${branchName})`); await this._git.raw(['reset', '--hard']); await this._git.checkout(branchName); await this._git.raw(['reset', '--hard', 'origin/' + branchName]); await this._git.raw(['clean', '-fd']); } private async _cleanLocalBranches(): Promise<void> { const existingBranches = (await this._git.raw(['branch'])) .split('\n') .map((branch) => branch.trim()) .filter((branch) => branch.length) .filter((branch) => !branch.startsWith('* ')); logger.debug({ existingBranches }); for (const branchName of existingBranches) { await this._deleteLocalBranch(branchName); } } async initRepo(args: StorageConfig): Promise<void> { this.cleanRepo(); // eslint-disable-next-line no-multi-assign const config: LocalConfig = (this._config = { ...args, fileList: {}, } as any); // eslint-disable-next-line no-multi-assign const cwd = (this._cwd = config.localDir); this._config.branchExists = {}; logger.debug('Initializing git repository into ' + cwd); const gitHead = join(cwd, '.git/HEAD'); let clone = true; // TODO: move to private class scope async function setBaseBranchToDefault(git: Git.SimpleGit): Promise<void> { // see https://stackoverflow.com/a/44750379/1438522 try { config.baseBranch = config.baseBranch || (await git.raw(['symbolic-ref', 'refs/remotes/origin/HEAD'])) .replace('refs/remotes/origin/', '') .trim(); } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); if ( err.message.startsWith( 'fatal: ref refs/remotes/origin/HEAD is not a symbolic ref' ) ) { throw new Error(REPOSITORY_EMPTY); } throw err; } } if (await fs.exists(gitHead)) { try { this._git = Git(cwd).silent(true); await this._git.raw(['remote', 'set-url', 'origin', config.url]); const fetchStart = Date.now(); await this._git.fetch(['--depth=10']); await setBaseBranchToDefault(this._git); await this._resetToBranch(config.baseBranch); await this._cleanLocalBranches(); await this._git.raw(['remote', 'prune', 'origin']); const durationMs = Math.round(Date.now() - fetchStart); logger.debug({ durationMs }, 'git fetch completed'); clone = false; } catch (err) /* istanbul ignore next */ { logger.error({ err }, 'git fetch error'); } } if (clone) { await fs.emptyDir(cwd); this._git = Git(cwd).silent(true); const cloneStart = Date.now(); try { // clone only the default branch let opts = ['--depth=2']; if (config.extraCloneOpts) { opts = opts.concat( Object.entries(config.extraCloneOpts).map((e) => `${e[0]}=${e[1]}`) ); } await this._git.clone(config.url, '.', opts); } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'git clone error'); throw new Error(PLATFORM_FAILURE); } const durationMs = Math.round(Date.now() - cloneStart); logger.debug({ durationMs }, 'git clone completed'); } const submodules = await this.getSubmodules(); for (const submodule of submodules) { try { logger.debug(`Cloning git submodule at ${submodule}`); await this._git.submoduleUpdate(['--init', '--', submodule]); } catch (err) { logger.warn(`Unable to initialise git submodule at ${submodule}`); } } try { const latestCommitDate = (await this._git.log({ n: 1 })).latest.date; logger.debug({ latestCommitDate }, 'latest commit'); } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); if (err.message.includes('does not have any commits yet')) { throw new Error(REPOSITORY_EMPTY); } logger.warn({ err }, 'Cannot retrieve latest commit date'); } // istanbul ignore if if (config.gitPrivateKey) { logger.debug('Git private key configured, but not being set'); } else { logger.debug('No git private key present - commits will be unsigned'); await this._git.raw(['config', 'commit.gpgsign', 'false']); } if (global.gitAuthor) { logger.debug({ gitAuthor: global.gitAuthor }, 'Setting git author'); try { await this._git.raw(['config', 'user.name', global.gitAuthor.name]); await this._git.raw(['config', 'user.email', global.gitAuthor.email]); } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); logger.debug({ err }, 'Error setting git config'); throw new Error(REPOSITORY_TEMPORARY_ERROR); } } await setBaseBranchToDefault(this._git); } // istanbul ignore next getRepoStatus(): Promise<StatusResult> { return this._git.status(); } async createBranch(branchName: string, sha: string): Promise<void> { logger.debug(`createBranch(${branchName})`); await this._git.reset('hard'); await this._git.raw(['clean', '-fd']); await this._git.checkout(['-B', branchName, sha]); await this._git.push('origin', branchName, { '--force': true }); this._config.branchExists[branchName] = true; } // Return the commit SHA for a branch async getBranchCommit(branchName: string): Promise<string> { if (!(await this.branchExists(branchName))) { throw Error( 'Cannot fetch commit for branch that does not exist: ' + branchName ); } const res = await this._git.revparse(['origin/' + branchName]); return res.trim(); } async getCommitMessages(): Promise<string[]> { logger.debug('getCommitMessages'); const res = await this._git.log({ n: 10, format: { message: '%s' }, }); return res.all.map((commit) => commit.message); } async setBaseBranch(branchName: string): Promise<string> { if (branchName) { if (!(await this.branchExists(branchName))) { throwBaseBranchValidationError(branchName); } logger.debug(`Setting baseBranch to ${branchName}`); this._config.baseBranch = branchName; try { if (branchName !== 'master') { this._config.baseBranchSha = ( await this._git.raw(['rev-parse', 'origin/' + branchName]) ).trim(); } await this._git.checkout([branchName, '-f']); await this._git.reset('hard'); const latestCommitDate = (await this._git.log({ n: 1 })).latest.date; logger.debug({ branchName, latestCommitDate }, 'latest commit'); } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); if ( err.message.includes( 'unknown revision or path not in the working tree' ) || err.message.includes('did not match any file(s) known to git') ) { throwBaseBranchValidationError(branchName); } throw err; } } return ( this._config.baseBranchSha || (await this._git.raw(['rev-parse', 'origin/master'])).trim() ); } /* * When we initially clone, we clone only the default branch so how no knowledge of other branches existing. * By calling this function once the repo's branchPrefix is known, we can fetch all of Renovate's branches in one command. */ async setBranchPrefix(branchPrefix: string): Promise<void> { logger.debug('Setting branchPrefix: ' + branchPrefix); this._config.branchPrefix = branchPrefix; const ref = `refs/heads/${branchPrefix}*:refs/remotes/origin/${branchPrefix}*`; try { await this._git.fetch(['origin', ref, '--depth=2', '--force']); } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); throw err; } } async getFileList(branchName?: string): Promise<string[]> { const branch = branchName || this._config.baseBranch; if (this._config.fileList[branch] === undefined) { this._config.fileList[branch] = this.getFileListInner(branchName); } return this._config.fileList[branch]; } async getFileListInner(branch: string): Promise<string[]> { const exists = await this.branchExists(branch); if (!exists) { return []; } const submodules = await this.getSubmodules(); const files: string = await this._git.raw([ 'ls-tree', '-r', '--name-only', 'origin/' + branch, ]); // istanbul ignore if if (!files) { return []; } return files .split('\n') .filter(Boolean) .filter((file: string) => submodules.every((submodule: string) => !file.startsWith(submodule)) ); } async getSubmodules(): Promise<string[]> { return ( (await this._git.raw([ 'config', '--file', '.gitmodules', '--get-regexp', 'path', ])) || '' ) .trim() .split(/[\n\s]/) .filter((_e: string, i: number) => i % 2); } async branchExists(branchName: string): Promise<boolean> { // First check cache if (this._config.branchExists[branchName] !== undefined) { return this._config.branchExists[branchName]; } if (!branchName?.startsWith(this._config.branchPrefix)) { // fetch the branch only if it's not part of the existing branchPrefix try { await this._git.raw([ 'remote', 'set-branches', '--add', 'origin', branchName, ]); await this._git.fetch(['origin', branchName, '--depth=2']); } catch (err) { checkForPlatformFailure(err); } } try { await this._git.raw(['show-branch', 'origin/' + branchName]); this._config.branchExists[branchName] = true; return true; } catch (err) { checkForPlatformFailure(err); this._config.branchExists[branchName] = false; return false; } } async getAllRenovateBranches(branchPrefix: string): Promise<string[]> { const branches = await this._git.branch(['--remotes', '--verbose']); return branches.all .map(localName) .filter((branchName) => branchName.startsWith(branchPrefix)); } async isBranchStale(branchName: string): Promise<boolean> { if (!(await this.branchExists(branchName))) { throw Error( 'Cannot check staleness for branch that does not exist: ' + branchName ); } const branches = await this._git.branch([ '--remotes', '--verbose', '--contains', this._config.baseBranchSha || `origin/${this._config.baseBranch}`, ]); return !branches.all.map(localName).includes(branchName); } private async _deleteLocalBranch(branchName: string): Promise<void> { await this._git.branch(['-D', branchName]); } async deleteBranch(branchName: string): Promise<void> { try { await this._git.raw(['push', '--delete', 'origin', branchName]); logger.debug({ branchName }, 'Deleted remote branch'); } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); logger.debug({ branchName }, 'No remote branch to delete'); } try { await this._deleteLocalBranch(branchName); // istanbul ignore next logger.debug({ branchName }, 'Deleted local branch'); } catch (err) { checkForPlatformFailure(err); logger.debug({ branchName }, 'No local branch to delete'); } this._config.branchExists[branchName] = false; } async mergeBranch(branchName: string): Promise<void> { await this._git.reset('hard'); await this._git.checkout(['-B', branchName, 'origin/' + branchName]); await this._git.checkout(this._config.baseBranch); await this._git.merge(['--ff-only', branchName]); await this._git.push('origin', this._config.baseBranch); limits.incrementLimit('prCommitsPerRunLimit'); } async getBranchLastCommitTime(branchName: string): Promise<Date> { try { const time = await this._git.show([ '-s', '--format=%ai', 'origin/' + branchName, ]); return new Date(Date.parse(time)); } catch (err) { checkForPlatformFailure(err); return new Date(); } } async getFile(filePath: string, branchName?: string): Promise<string | null> { if (branchName) { const exists = await this.branchExists(branchName); if (!exists) { logger.debug({ branchName }, 'branch no longer exists - aborting'); throw new Error(REPOSITORY_CHANGED); } } try { const content = await this._git.show([ 'origin/' + (branchName || this._config.baseBranch) + ':' + filePath, ]); return content; } catch (err) { checkForPlatformFailure(err); return null; } } async hasDiff(branchName: string): Promise<boolean> { try { return (await this._git.diff(['HEAD', branchName])) !== ''; } catch (err) { return true; } } async commitFilesToBranch({ branchName, files, message, parentBranch = this._config.baseBranch, }: CommitFilesConfig): Promise<string | null> { logger.debug(`Committing files to branch ${branchName}`); try { await this._git.reset('hard'); await this._git.raw(['clean', '-fd']); await this._git.checkout(['-B', branchName, 'origin/' + parentBranch]); const fileNames = []; const deleted = []; for (const file of files) { // istanbul ignore if if (file.name === '|delete|') { deleted.push(file.contents); } else if (await isDirectory(join(this._cwd, file.name))) { fileNames.push(file.name); await this._git.add(file.name); } else { fileNames.push(file.name); let contents; // istanbul ignore else if (typeof file.contents === 'string') { contents = Buffer.from(file.contents); } else { contents = file.contents; } await fs.outputFile(join(this._cwd, file.name), contents); } } // istanbul ignore if if (fileNames.length === 1 && fileNames[0] === 'renovate.json') { fileNames.unshift('-f'); } if (fileNames.length) { await this._git.add(fileNames); } if (deleted.length) { for (const f of deleted) { try { await this._git.rm([f]); } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); logger.debug({ err }, 'Cannot delete ' + f); } } } const commitRes = await this._git.commit(message); const commit = commitRes?.commit || 'unknown'; if (!(await this.hasDiff(`origin/${branchName}`))) { logger.debug( { branchName, fileNames }, 'No file changes detected. Skipping commit' ); return null; } await this._git.push('origin', `${branchName}:${branchName}`, { '--force': true, '-u': true, }); // Fetch it after create const ref = `refs/heads/${branchName}:refs/remotes/origin/${branchName}`; await this._git.fetch(['origin', ref, '--depth=2', '--force']); this._config.branchExists[branchName] = true; limits.incrementLimit('prCommitsPerRunLimit'); return commit; } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); logger.debug({ err }, 'Error commiting files'); throw new Error(REPOSITORY_CHANGED); } } // eslint-disable-next-line cleanRepo(): void {} static getUrl({ protocol, auth, hostname, host, repository, }: { protocol?: 'ssh' | 'http' | 'https'; auth?: string; hostname?: string; host?: string; repository: string; }): string { if (protocol === 'ssh') { return `git@${hostname}:${repository}.git`; } return URL.format({ protocol: protocol || 'https', auth, hostname, host, pathname: repository + '.git', }); } } export default Storage;