diff --git a/lib/platform/github/index.js b/lib/platform/github/index.js
index c0be286527d617cf99ab251618d5a11c4470c70e..afcbac876201ef17b7e22dbe3d9191371695349b 100644
--- a/lib/platform/github/index.js
+++ b/lib/platform/github/index.js
@@ -1,13 +1,12 @@
const is = require('@sindresorhus/is');
const addrs = require('email-addresses');
-const moment = require('moment');
-const openpgp = require('openpgp');
const delay = require('delay');
-const path = require('path');
const showdown = require('showdown');
const get = require('./gh-got-wrapper');
+const { expandError } = require('./util');
const endpoints = require('../../util/endpoints');
+const Storage = require('./storage');
const converter = new showdown.Converter();
converter.setFlavor('github');
@@ -80,19 +79,7 @@ async function getRepos(token, endpoint) {
function cleanRepo() {
// In theory most of this isn't necessary. In practice..
get.reset();
- config = null;
config = {};
- delete config.repository;
- delete config.repositoryOwner;
- delete config.repositoryName;
- delete config.owner;
- delete config.defaultBranch;
- delete config.baseBranch;
- delete config.issueList;
- delete config.prList;
- delete config.fileList;
- delete config.branchList;
- delete config.forkToken;
}
// Initialize GitHub by getting base branch and SHA
@@ -151,16 +138,12 @@ async function initRepo({
}
platformConfig.privateRepo = res.body.private === true;
platformConfig.isFork = res.body.fork === true;
- config.owner = res.body.owner.login;
- logger.debug(`${repository} owner = ${config.owner}`);
+ const owner = res.body.owner.login;
+ logger.debug(`${repository} owner = ${owner}`);
// Use default branch as PR target unless later overridden.
config.defaultBranch = res.body.default_branch;
// Base branch may be configured but defaultBranch is always fixed
config.baseBranch = config.defaultBranch;
- // istanbul ignore if
- if (process.env.NODE_ENV !== 'test') {
- getBranchCommit(config.baseBranch); // warm the cache
- }
logger.debug(`${repository} default branch = ${config.baseBranch}`);
// GitHub allows administrators to block certain types of merge, so we need to check it
if (res.body.allow_rebase_merge) {
@@ -196,15 +179,14 @@ async function initRepo({
// This shouldn't be necessary, but occasional strange errors happened until it was added
config.issueList = null;
config.prList = null;
- config.fileList = null;
- config.branchList = null;
+ config.storage = new Storage(config);
logger.debug('Prefetching prList and fileList');
await Promise.all([getPrList(), getFileList()]);
if (forkMode) {
logger.info('Renovate is in forkMode');
config.forkToken = forkToken;
// Save parent SHA then delete
- config.parentSha = await getBaseCommitSHA();
+ const parentSha = await getBaseCommitSHA();
config.baseCommitSHA = null;
// save parent name then delete
config.parentRepo = config.repository;
@@ -224,7 +206,7 @@ async function initRepo({
);
// Need to update base branch
logger.debug(
- { baseBranch: config.baseBranch, parentSha: config.parentSha },
+ { baseBranch: config.baseBranch, parentSha },
'Setting baseBranch ref in fork'
);
// This is a lovely "hack" by GitHub that lets us force update our fork's master
@@ -233,7 +215,7 @@ async function initRepo({
`repos/${config.repository}/git/refs/heads/${config.baseBranch}`,
{
body: {
- sha: config.parentSha,
+ sha: parentSha,
},
token: forkToken || opts.token,
}
@@ -243,6 +225,7 @@ async function initRepo({
// Wait an arbitrary 30s to hopefully give GitHub enough time for forking to complete
await delay(30000);
}
+ config.storage = new Storage(config);
}
// istanbul ignore if
if (mirrorMode) {
@@ -272,7 +255,10 @@ async function initRepo({
);
}
if (!(await branchExists('renovate-config'))) {
- await createBranch('renovate-config', config.baseCommitSHA);
+ await config.storage.createBranch(
+ 'renovate-config',
+ config.baseCommitSHA
+ );
}
}
return platformConfig;
@@ -325,7 +311,9 @@ async function getRepoForceRebase() {
async function getBaseCommitSHA() {
if (!config.baseCommitSHA) {
- config.baseCommitSHA = await getBranchCommit(config.baseBranch);
+ config.baseCommitSHA = await config.storage.getBranchCommit(
+ config.baseBranch
+ );
}
return config.baseCommitSHA;
}
@@ -346,7 +334,7 @@ async function setBaseBranch(branchName) {
logger.debug(`Setting baseBranch to ${branchName}`);
config.baseBranch = branchName;
config.baseCommitSHA = null;
- config.fileList = null;
+ config.storage = new Storage(config);
await getFileList(branchName);
}
}
@@ -354,104 +342,64 @@ async function setBaseBranch(branchName) {
// Search
// Get full file list
-async function getFileList(branchName = config.baseBranch) {
- if (config.fileList) {
- return config.fileList;
- }
- try {
- const res = await get(
- `repos/${config.repository}/git/trees/${branchName}?recursive=true`
- );
- if (res.body.truncated) {
- logger.warn(
- { repository: config.repository },
- 'repository tree is truncated'
- );
- }
- config.fileList = res.body.tree
- .filter(item => item.type === 'blob' && item.mode !== '120000')
- .map(item => item.path)
- .sort();
- logger.debug(`Retrieved fileList with length ${config.fileList.length}`);
- } catch (err) /* istanbul ignore next */ {
- if (err.statusCode === 409) {
- logger.debug('Repository is not initiated');
- throw new Error('uninitiated');
- }
- logger.info(
- { repository: config.repository },
- 'Error retrieving git tree - no files detected'
- );
- config.fileList = [];
- }
-
- return config.fileList;
+function getFileList(branchName = config.baseBranch) {
+ return config.storage.getFileList(branchName);
}
// Branch
// Returns true if branch exists, otherwise false
-async function branchExists(branchName) {
- if (!config.branchList) {
- logger.debug('Retrieving branchList');
- config.branchList = (await get(
- `repos/${config.repository}/branches?per_page=100`,
- {
- paginate: true,
- }
- )).body.map(branch => branch.name);
- logger.debug({ branchList: config.branchList }, 'Retrieved branchList');
- }
- const res = config.branchList.includes(branchName);
- logger.debug(`branchExists(${branchName})=${res}`);
- return res;
+function branchExists(branchName) {
+ return config.storage.branchExists(branchName);
}
-async function getAllRenovateBranches(branchPrefix) {
- logger.trace('getAllRenovateBranches');
- try {
- const allBranches = (await get(
- `repos/${config.repository}/git/refs/heads/${branchPrefix}`,
- {
- paginate: true,
- }
- )).body;
- return allBranches.reduce((arr, branch) => {
- if (branch.ref.startsWith(`refs/heads/${branchPrefix}`)) {
- arr.push(branch.ref.substring('refs/heads/'.length));
- }
- if (
- branchPrefix.endsWith('/') &&
- branch.ref === `refs/heads/${branchPrefix.slice(0, -1)}`
- ) {
- logger.warn(
- `Pruning branch "${branchPrefix.slice(
- 0,
- -1
- )}" so that it does not block PRs`
- );
- arr.push(branch.ref.substring('refs/heads/'.length));
- }
- return arr;
- }, []);
- } catch (err) /* istanbul ignore next */ {
- return [];
+function getAllRenovateBranches(branchPrefix) {
+ return config.storage.getAllRenovateBranches(branchPrefix);
+}
+
+function isBranchStale(branchName) {
+ return config.storage.isBranchStale(branchName);
+}
+
+function getFile(filePath, branchName) {
+ return config.storage.getFile(filePath, branchName);
+}
+
+function deleteBranch(branchName) {
+ return config.storage.deleteBranch(branchName);
+}
+
+function getBranchLastCommitTime(branchName) {
+ return config.storage.getBranchLastCommitTime(branchName);
+}
+
+function mergeBranch(branchName) {
+ // istanbul ignore if
+ if (config.pushProtection) {
+ logger.info(
+ { branchName },
+ 'Branch protection: Attempting to merge branch when push protection is enabled'
+ );
}
+ return config.storage.mergeBranch(branchName);
}
-async function isBranchStale(branchName) {
- // Check if branch's parent SHA = master SHA
- logger.debug(`isBranchStale(${branchName})`);
- const branchCommit = await getBranchCommit(branchName);
- logger.debug(`branchCommit=${branchCommit}`);
- const commitDetails = await getCommitDetails(branchCommit);
- logger.trace({ commitDetails }, `commitDetails`);
- const parentSha = commitDetails.parents[0].sha;
- logger.debug(`parentSha=${parentSha}`);
- const baseCommitSHA = await getBaseCommitSHA();
- logger.debug(`baseCommitSHA=${baseCommitSHA}`);
- // Return true if the SHAs don't match
- return parentSha !== baseCommitSHA;
+function commitFilesToBranch(
+ branchName,
+ files,
+ message,
+ parentBranch = config.baseBranch
+) {
+ return config.storage.commitFilesToBranch(
+ branchName,
+ files,
+ message,
+ parentBranch
+ );
+}
+
+function getCommitMessages() {
+ return config.storage.getCommitMessages();
}
// Returns the Pull Request for a branch. Null if not exists.
@@ -484,7 +432,7 @@ async function getBranchStatus(branchName, requiredStatusChecks) {
}
async function getBranchStatusCheck(branchName, context) {
- const branchCommit = await getBranchCommit(branchName);
+ const branchCommit = await config.storage.getBranchCommit(branchName);
const url = `repos/${config.repository}/commits/${branchCommit}/statuses`;
const res = await get(url);
for (const check of res.body) {
@@ -512,7 +460,7 @@ async function setBranchStatus(
return;
}
logger.info({ branchName, context, state }, 'Setting branch status');
- const branchCommit = await getBranchCommit(branchName);
+ const branchCommit = await config.storage.getBranchCommit(branchName);
const url = `repos/${config.repository}/statuses/${branchCommit}`;
const options = {
state,
@@ -525,67 +473,6 @@ async function setBranchStatus(
await get.post(url, { body: options });
}
-async function deleteBranch(branchName) {
- const options = config.forkToken ? { token: config.forkToken } : undefined;
- try {
- await get.delete(
- `repos/${config.repository}/git/refs/heads/${branchName}`,
- options
- );
- } catch (err) /* istanbul ignore next */ {
- if (err.message.startsWith('Reference does not exist')) {
- logger.info({ branchName }, 'Branch to delete does not exist');
- } else {
- logger.warn(
- { err, body: err.response.body, branchName },
- 'Error deleting branch'
- );
- }
- }
-}
-
-async function mergeBranch(branchName) {
- logger.debug(`mergeBranch(${branchName})`);
- // istanbul ignore if
- if (config.pushProtection) {
- logger.info(
- { branchName },
- 'Branch protection: Attempting to merge branch when push protection is enabled'
- );
- }
- const url = `repos/${config.repository}/git/refs/heads/${config.baseBranch}`;
- const options = {
- body: {
- sha: await getBranchCommit(branchName),
- },
- };
- try {
- await get.patch(url, options);
- } catch (err) {
- logger.info(
- expandError(err),
- `Error pushing branch merge for ${branchName}`
- );
- throw new Error('Branch automerge failed');
- }
- // Update base commit
- config.baseCommitSHA = null;
- // Delete branch
- await deleteBranch(branchName);
-}
-
-async function getBranchLastCommitTime(branchName) {
- try {
- const res = await get(
- `repos/${config.repository}/commits?sha=${branchName}`
- );
- return new Date(res.body[0].commit.committer.date);
- } catch (err) {
- logger.error(expandError(err), `getBranchLastCommitTime error`);
- return new Date();
- }
-}
-
// Issue
async function getIssueList() {
@@ -1136,318 +1023,6 @@ function getPrBody(input) {
);
}
-// Generic File operations
-
-async function getFile(filePath, branchName) {
- logger.trace(`getFile(filePath=${filePath}, branchName=${branchName})`);
- if (!branchName || branchName === config.baseBranch) {
- if (!config.fileList.includes(filePath)) {
- return null;
- }
- }
- let res;
- try {
- res = await get(
- `repos/${config.repository}/contents/${encodeURI(
- filePath
- )}?ref=${branchName || config.baseBranch}`
- );
- } catch (error) {
- if (error.statusCode === 404) {
- // If file not found, then return null JSON
- logger.info({ filePath, branchName }, 'getFile 404');
- return null;
- }
- if (
- error.statusCode === 403 &&
- error.message &&
- error.message.startsWith('This API returns blobs up to 1 MB in size')
- ) {
- logger.info('Large file');
- // istanbul ignore if
- if (branchName && branchName !== config.baseBranch) {
- logger.info('Cannot retrieve large files from non-master branch');
- return null;
- }
- // istanbul ignore if
- if (path.dirname(filePath) !== '.') {
- logger.info('Cannot retrieve large files from non-root directories');
- return null;
- }
- const treeUrl = `repos/${config.repository}/git/trees/${
- config.baseBranch
- }`;
- const baseName = path.basename(filePath);
- let fileSha;
- (await get(treeUrl)).body.tree.forEach(file => {
- if (file.path === baseName) {
- fileSha = file.sha;
- }
- });
- if (!fileSha) {
- logger.warn('Could not locate file blob');
- throw error;
- }
- res = await get(`repos/${config.repository}/git/blobs/${fileSha}`);
- } else {
- // Propagate if it's any other error
- throw error;
- }
- }
- if (res.body.content) {
- return Buffer.from(res.body.content, 'base64').toString();
- }
- return null;
-}
-
-// Add a new commit, create branch if not existing
-async function commitFilesToBranch(
- branchName,
- files,
- message,
- parentBranch = config.baseBranch
-) {
- logger.debug(
- `commitFilesToBranch('${branchName}', files, message, '${parentBranch})'`
- );
- const parentCommit = await getBranchCommit(parentBranch);
- const parentTree = await getCommitTree(parentCommit);
- const fileBlobs = [];
- // Create blobs
- for (const file of files) {
- const blob = await createBlob(file.contents);
- fileBlobs.push({
- name: file.name,
- blob,
- });
- }
- // Create tree
- const tree = await createTree(parentTree, fileBlobs);
- const commit = await createCommit(parentCommit, tree, message);
- const isBranchExisting = await branchExists(branchName);
- try {
- if (isBranchExisting) {
- await updateBranch(branchName, commit);
- } else {
- await createBranch(branchName, commit);
- }
- } catch (err) /* istanbul ignore next */ {
- logger.debug({
- files: files.filter(
- file =>
- !file.name.endsWith('package-lock.json') &&
- !file.name.endsWith('npm-shrinkwrap.json') &&
- !file.name.endsWith('yarn.lock')
- ),
- });
- throw err;
- }
-}
-
-// Internal branch operations
-
-// Creates a new branch with provided commit
-async function createBranch(branchName, sha) {
- logger.debug(`createBranch(${branchName})`);
- const options = {
- body: {
- ref: `refs/heads/${branchName}`,
- sha,
- },
- };
- // istanbul ignore if
- if (config.forkToken) {
- options.token = config.forkToken;
- }
- try {
- // istanbul ignore if
- if (branchName.includes('/')) {
- const [blockingBranch] = branchName.split('/');
- if (await branchExists(blockingBranch)) {
- logger.warn({ blockingBranch }, 'Deleting blocking branch');
- await deleteBranch(blockingBranch);
- }
- }
- logger.debug({ options, branchName }, 'Creating branch');
- await get.post(`repos/${config.repository}/git/refs`, options);
- config.branchList.push(branchName);
- logger.debug('Created branch');
- } catch (err) /* istanbul ignore next */ {
- const headers = err.response.req.getHeaders();
- delete headers.token;
- logger.warn(
- {
- err,
- message: err.message,
- responseBody: err.response.body,
- headers,
- options,
- },
- 'Error creating branch'
- );
- if (err.statusCode === 422) {
- throw new Error('repository-changed');
- }
- throw err;
- }
-}
-
-// Internal: Updates an existing branch to new commit sha
-async function updateBranch(branchName, commit) {
- logger.debug(`Updating branch ${branchName} with commit ${commit}`);
- const options = {
- body: {
- sha: commit,
- force: true,
- },
- };
- // istanbul ignore if
- if (config.forkToken) {
- options.token = config.forkToken;
- }
- try {
- await get.patch(
- `repos/${config.repository}/git/refs/heads/${branchName}`,
- options
- );
- } catch (err) /* istanbul ignore next */ {
- if (err.statusCode === 422) {
- logger.info(expandError(err), 'Branch no longer exists - exiting');
- throw new Error('repository-changed');
- }
- throw err;
- }
-}
-
-// Low-level commit operations
-
-// Create a blob with fileContents and return sha
-async function createBlob(fileContents) {
- logger.debug('Creating blob');
- const options = {
- body: {
- encoding: 'base64',
- content: Buffer.from(fileContents).toString('base64'),
- },
- };
- // istanbul ignore if
- if (config.forkToken) {
- options.token = config.forkToken;
- }
- return (await get.post(`repos/${config.repository}/git/blobs`, options)).body
- .sha;
-}
-
-// Return the commit SHA for a branch
-async function getBranchCommit(branchName) {
- const res = await get(
- `repos/${config.repository}/git/refs/heads/${branchName}`
- );
- return res.body.object.sha;
-}
-
-async function getCommitDetails(commit) {
- logger.debug(`getCommitDetails(${commit})`);
- const results = await get(`repos/${config.repository}/git/commits/${commit}`);
- return results.body;
-}
-
-// Return the tree SHA for a commit
-async function getCommitTree(commit) {
- logger.debug(`getCommitTree(${commit})`);
- return (await get(`repos/${config.repository}/git/commits/${commit}`)).body
- .tree.sha;
-}
-
-// Create a tree and return SHA
-async function createTree(baseTree, files) {
- logger.debug(`createTree(${baseTree}, files)`);
- const body = {
- base_tree: baseTree,
- tree: [],
- };
- files.forEach(file => {
- body.tree.push({
- path: file.name,
- mode: '100644',
- type: 'blob',
- sha: file.blob,
- });
- });
- logger.trace({ body }, 'createTree body');
- const options = { body };
- // istanbul ignore if
- if (config.forkToken) {
- options.token = config.forkToken;
- }
- return (await get.post(`repos/${config.repository}/git/trees`, options)).body
- .sha;
-}
-
-// Create a commit and return commit SHA
-async function createCommit(parent, tree, message) {
- logger.debug(`createCommit(${parent}, ${tree}, ${message})`);
- const { gitAuthor, gitPrivateKey } = config;
- const now = moment();
- let author;
- if (gitAuthor) {
- logger.trace('Setting gitAuthor');
- author = {
- name: gitAuthor.name,
- email: gitAuthor.address,
- date: now.format(),
- };
- }
- const body = {
- message,
- parents: [parent],
- tree,
- };
- if (author) {
- body.author = author;
- // istanbul ignore if
- if (gitPrivateKey) {
- logger.debug('Found gitPrivateKey');
- const privKeyObj = openpgp.key.readArmored(gitPrivateKey).keys[0];
- const commit = `tree ${tree}\nparent ${parent}\nauthor ${author.name} <${
- author.email
- }> ${now.format('X ZZ')}\ncommitter ${author.name} <${
- author.email
- }> ${now.format('X ZZ')}\n\n${message}`;
- const { signature } = await openpgp.sign({
- data: openpgp.util.str2Uint8Array(commit),
- privateKeys: privKeyObj,
- detached: true,
- armor: true,
- });
- body.signature = signature;
- }
- }
- const options = {
- body,
- };
- // istanbul ignore if
- if (config.forkToken) {
- options.token = config.forkToken;
- }
- return (await get.post(`repos/${config.repository}/git/commits`, options))
- .body.sha;
-}
-
-async function getCommitMessages() {
- logger.debug('getCommitMessages');
- const res = await get(`repos/${config.repository}/commits`);
- return res.body.map(commit => commit.commit.message);
-}
-
-function expandError(err) {
- return {
- err,
- message: err.message,
- body: err.response ? err.response.body : undefined,
- };
-}
-
async function getVulnerabilityAlerts() {
const headers = {
accept: 'application/vnd.github.vixen-preview+json',
diff --git a/lib/platform/github/storage.js b/lib/platform/github/storage.js
new file mode 100644
index 0000000000000000000000000000000000000000..ad9bf0baf3810d5a6a6f59af4c2265f6201217ba
--- /dev/null
+++ b/lib/platform/github/storage.js
@@ -0,0 +1,476 @@
+const moment = require('moment');
+const openpgp = require('openpgp');
+const path = require('path');
+const get = require('./gh-got-wrapper');
+const { expandError } = require('./util');
+
+class Storage {
+ constructor(config) {
+ // config
+ this.config = config;
+ this.gitAuthor = config.gitAuthor;
+ this.gitPrivateKey = config.gitPrivateKey;
+ this.forkToken = config.forkToken;
+ this.repository = config.repository;
+ this.baseBranch = config.baseBranch;
+ // cache
+ this.branchFiles = {};
+ this.branchList = null;
+ }
+
+ // Returns true if branch exists, otherwise false
+ async branchExists(branchName) {
+ if (!this.branchList) {
+ logger.debug('Retrieving branchList');
+ this.branchList = (await get(
+ `repos/${this.repository}/branches?per_page=100`,
+ {
+ paginate: true,
+ }
+ )).body.map(branch => branch.name);
+ logger.debug({ branchList: this.branchList }, 'Retrieved branchList');
+ }
+ const res = this.branchList.includes(branchName);
+ logger.debug(`branchExists(${branchName})=${res}`);
+ return res;
+ }
+
+ // Get full file list
+ async getFileList(branchName) {
+ const branch = branchName || this.baseBranch;
+ if (this.branchFiles[branch]) {
+ return this.branchFiles[branch];
+ }
+ try {
+ const res = await get(
+ `repos/${this.repository}/git/trees/${branch}?recursive=true`
+ );
+ if (res.body.truncated) {
+ logger.warn(
+ { repository: this.repository },
+ 'repository tree is truncated'
+ );
+ }
+ const fileList = res.body.tree
+ .filter(item => item.type === 'blob' && item.mode !== '120000')
+ .map(item => item.path)
+ .sort();
+ logger.debug(`Retrieved fileList with length ${fileList.length}`);
+ this.branchFiles[branch] = fileList;
+ return fileList;
+ } catch (err) /* istanbul ignore next */ {
+ if (err.statusCode === 409) {
+ logger.debug('Repository is not initiated');
+ throw new Error('uninitiated');
+ }
+ logger.info(
+ { repository: this.repository },
+ 'Error retrieving git tree - no files detected'
+ );
+ return [];
+ }
+ }
+
+ async getAllRenovateBranches(branchPrefix) {
+ logger.trace('getAllRenovateBranches');
+ try {
+ const allBranches = (await get(
+ `repos/${this.repository}/git/refs/heads/${branchPrefix}`,
+ {
+ paginate: true,
+ }
+ )).body;
+ return allBranches.reduce((arr, branch) => {
+ if (branch.ref.startsWith(`refs/heads/${branchPrefix}`)) {
+ arr.push(branch.ref.substring('refs/heads/'.length));
+ }
+ if (
+ branchPrefix.endsWith('/') &&
+ branch.ref === `refs/heads/${branchPrefix.slice(0, -1)}`
+ ) {
+ logger.warn(
+ `Pruning branch "${branchPrefix.slice(
+ 0,
+ -1
+ )}" so that it does not block PRs`
+ );
+ arr.push(branch.ref.substring('refs/heads/'.length));
+ }
+ return arr;
+ }, []);
+ } catch (err) /* istanbul ignore next */ {
+ return [];
+ }
+ }
+
+ async isBranchStale(branchName) {
+ // Check if branch's parent SHA = master SHA
+ logger.debug(`isBranchStale(${branchName})`);
+ const branchCommit = await this.getBranchCommit(branchName);
+ logger.debug(`branchCommit=${branchCommit}`);
+ const commitDetails = await getCommitDetails(this, branchCommit);
+ logger.trace({ commitDetails }, `commitDetails`);
+ const parentSha = commitDetails.parents[0].sha;
+ logger.debug(`parentSha=${parentSha}`);
+ const baseCommitSHA = await this.getBranchCommit(this.baseBranch);
+ logger.debug(`baseCommitSHA=${baseCommitSHA}`);
+ // Return true if the SHAs don't match
+ return parentSha !== baseCommitSHA;
+ }
+
+ async deleteBranch(branchName) {
+ delete this.branchFiles[branchName];
+ const options = this.forkToken ? { token: this.forkToken } : undefined;
+ try {
+ await get.delete(
+ `repos/${this.repository}/git/refs/heads/${branchName}`,
+ options
+ );
+ } catch (err) /* istanbul ignore next */ {
+ if (err.message.startsWith('Reference does not exist')) {
+ logger.info({ branchName }, 'Branch to delete does not exist');
+ } else {
+ logger.warn(
+ { err, body: err.response.body, branchName },
+ 'Error deleting branch'
+ );
+ }
+ }
+ }
+
+ async mergeBranch(branchName) {
+ logger.debug(`mergeBranch(${branchName})`);
+ const url = `repos/${this.repository}/git/refs/heads/${this.baseBranch}`;
+ const options = {
+ body: {
+ sha: await this.getBranchCommit(branchName),
+ },
+ };
+ try {
+ await get.patch(url, options);
+ } catch (err) {
+ logger.info(
+ expandError(err),
+ `Error pushing branch merge for ${branchName}`
+ );
+ throw new Error('Branch automerge failed');
+ }
+ // Delete branch
+ await this.deleteBranch(branchName);
+ }
+
+ async getBranchLastCommitTime(branchName) {
+ try {
+ const res = await get(
+ `repos/${this.repository}/commits?sha=${branchName}`
+ );
+ return new Date(res.body[0].commit.committer.date);
+ } catch (err) {
+ logger.error(expandError(err), `getBranchLastCommitTime error`);
+ return new Date();
+ }
+ }
+
+ // Generic File operations
+
+ async getFile(filePath, branchName) {
+ logger.trace(`getFile(filePath=${filePath}, branchName=${branchName})`);
+ const branchFiles = await this.getFileList(branchName);
+ if (!branchFiles.includes(filePath)) {
+ return null;
+ }
+ let res;
+ try {
+ res = await get(
+ `repos/${this.repository}/contents/${encodeURI(
+ filePath
+ )}?ref=${branchName || this.baseBranch}`
+ );
+ } catch (error) {
+ if (error.statusCode === 404) {
+ // If file not found, then return null JSON
+ logger.info({ filePath, branchName }, 'getFile 404');
+ return null;
+ }
+ if (
+ error.statusCode === 403 &&
+ error.message &&
+ error.message.startsWith('This API returns blobs up to 1 MB in size')
+ ) {
+ logger.info('Large file');
+ // istanbul ignore if
+ if (branchName && branchName !== this.baseBranch) {
+ logger.info('Cannot retrieve large files from non-master branch');
+ return null;
+ }
+ // istanbul ignore if
+ if (path.dirname(filePath) !== '.') {
+ logger.info('Cannot retrieve large files from non-root directories');
+ return null;
+ }
+ const treeUrl = `repos/${this.repository}/git/trees/${this.baseBranch}`;
+ const baseName = path.basename(filePath);
+ let fileSha;
+ (await get(treeUrl)).body.tree.forEach(file => {
+ if (file.path === baseName) {
+ fileSha = file.sha;
+ }
+ });
+ if (!fileSha) {
+ logger.warn('Could not locate file blob');
+ throw error;
+ }
+ res = await get(`repos/${this.repository}/git/blobs/${fileSha}`);
+ } else {
+ // Propagate if it's any other error
+ throw error;
+ }
+ }
+ if (res && res.body.content) {
+ return Buffer.from(res.body.content, 'base64').toString();
+ }
+ return null;
+ }
+
+ // Add a new commit, create branch if not existing
+ async commitFilesToBranch(
+ branchName,
+ files,
+ message,
+ parentBranch = this.baseBranch
+ ) {
+ logger.debug(
+ `commitFilesToBranch('${branchName}', files, message, '${parentBranch})'`
+ );
+ delete this.branchFiles[branchName];
+ const parentCommit = await this.getBranchCommit(parentBranch);
+ const parentTree = await getCommitTree(this, parentCommit);
+ const fileBlobs = [];
+ // Create blobs
+ for (const file of files) {
+ const blob = await createBlob(this, file.contents);
+ fileBlobs.push({
+ name: file.name,
+ blob,
+ });
+ }
+ // Create tree
+ const tree = await createTree(this, parentTree, fileBlobs);
+ const commit = await createCommit(this, parentCommit, tree, message);
+ const isBranchExisting = await this.branchExists(branchName);
+ try {
+ if (isBranchExisting) {
+ await updateBranch(this, branchName, commit);
+ } else {
+ await this.createBranch(branchName, commit);
+ }
+ } catch (err) /* istanbul ignore next */ {
+ logger.debug({
+ files: files.filter(
+ file =>
+ !file.name.endsWith('package-lock.json') &&
+ !file.name.endsWith('npm-shrinkwrap.json') &&
+ !file.name.endsWith('yarn.lock')
+ ),
+ });
+ throw err;
+ }
+ }
+
+ // Internal branch operations
+
+ // Creates a new branch with provided commit
+ async createBranch(branchName, sha) {
+ logger.debug(`createBranch(${branchName})`);
+ const options = {
+ body: {
+ ref: `refs/heads/${branchName}`,
+ sha,
+ },
+ };
+ // istanbul ignore if
+ if (this.forkToken) {
+ options.token = this.forkToken;
+ }
+ try {
+ // istanbul ignore if
+ if (branchName.includes('/')) {
+ const [blockingBranch] = branchName.split('/');
+ if (await this.branchExists(blockingBranch)) {
+ logger.warn({ blockingBranch }, 'Deleting blocking branch');
+ await this.deleteBranch(blockingBranch);
+ }
+ }
+ logger.debug({ options, branchName }, 'Creating branch');
+ await get.post(`repos/${this.repository}/git/refs`, options);
+ this.branchList.push(branchName);
+ logger.debug('Created branch');
+ } catch (err) /* istanbul ignore next */ {
+ const headers = err.response.req.getHeaders();
+ delete headers.token;
+ logger.warn(
+ {
+ err,
+ message: err.message,
+ responseBody: err.response.body,
+ headers,
+ options,
+ },
+ 'Error creating branch'
+ );
+ if (err.statusCode === 422) {
+ throw new Error('repository-changed');
+ }
+ throw err;
+ }
+ }
+
+ // Return the commit SHA for a branch
+ async getBranchCommit(branchName) {
+ const res = await get(
+ `repos/${this.repository}/git/refs/heads/${branchName}`
+ );
+ return res.body.object.sha;
+ }
+
+ async getCommitMessages() {
+ logger.debug('getCommitMessages');
+ const res = await get(`repos/${this.repository}/commits`);
+ return res.body.map(commit => commit.commit.message);
+ }
+}
+
+// Internal: Updates an existing branch to new commit sha
+async function updateBranch(self, branchName, commit) {
+ logger.debug(`Updating branch ${branchName} with commit ${commit}`);
+ const options = {
+ body: {
+ sha: commit,
+ force: true,
+ },
+ };
+ // istanbul ignore if
+ if (self.forkToken) {
+ options.token = self.forkToken;
+ }
+ try {
+ await get.patch(
+ `repos/${self.repository}/git/refs/heads/${branchName}`,
+ options
+ );
+ } catch (err) /* istanbul ignore next */ {
+ if (err.statusCode === 422) {
+ logger.info(expandError(err), 'Branch no longer exists - exiting');
+ throw new Error('repository-changed');
+ }
+ throw err;
+ }
+}
+// Low-level commit operations
+
+// Create a blob with fileContents and return sha
+async function createBlob(self, fileContents) {
+ logger.debug('Creating blob');
+ const options = {
+ body: {
+ encoding: 'base64',
+ content: Buffer.from(fileContents).toString('base64'),
+ },
+ };
+ // istanbul ignore if
+ if (self.forkToken) {
+ options.token = self.forkToken;
+ }
+ return (await get.post(`repos/${self.repository}/git/blobs`, options)).body
+ .sha;
+}
+
+// Return the tree SHA for a commit
+async function getCommitTree(self, commit) {
+ logger.debug(`getCommitTree(${commit})`);
+ return (await get(`repos/${self.repository}/git/commits/${commit}`)).body.tree
+ .sha;
+}
+
+// Create a tree and return SHA
+async function createTree(self, baseTree, files) {
+ logger.debug(`createTree(${baseTree}, files)`);
+ const body = {
+ base_tree: baseTree,
+ tree: [],
+ };
+ files.forEach(file => {
+ body.tree.push({
+ path: file.name,
+ mode: '100644',
+ type: 'blob',
+ sha: file.blob,
+ });
+ });
+ logger.trace({ body }, 'createTree body');
+ const options = { body };
+ // istanbul ignore if
+ if (self.forkToken) {
+ options.token = self.forkToken;
+ }
+ return (await get.post(`repos/${self.repository}/git/trees`, options)).body
+ .sha;
+}
+
+// Create a commit and return commit SHA
+async function createCommit(self, parent, tree, message) {
+ logger.debug(`createCommit(${parent}, ${tree}, ${message})`);
+ const { gitAuthor, gitPrivateKey } = self;
+ const now = moment();
+ let author;
+ if (gitAuthor) {
+ logger.trace('Setting gitAuthor');
+ author = {
+ name: gitAuthor.name,
+ email: gitAuthor.address,
+ date: now.format(),
+ };
+ }
+ const body = {
+ message,
+ parents: [parent],
+ tree,
+ };
+ if (author) {
+ body.author = author;
+ // istanbul ignore if
+ if (gitPrivateKey) {
+ logger.debug('Found gitPrivateKey');
+ const privKeyObj = openpgp.key.readArmored(gitPrivateKey).keys[0];
+ const commit = `tree ${tree}\nparent ${parent}\nauthor ${author.name} <${
+ author.email
+ }> ${now.format('X ZZ')}\ncommitter ${author.name} <${
+ author.email
+ }> ${now.format('X ZZ')}\n\n${message}`;
+ const { signature } = await openpgp.sign({
+ data: openpgp.util.str2Uint8Array(commit),
+ privateKeys: privKeyObj,
+ detached: true,
+ armor: true,
+ });
+ body.signature = signature;
+ }
+ }
+ const options = {
+ body,
+ };
+ // istanbul ignore if
+ if (self.forkToken) {
+ options.token = self.forkToken;
+ }
+ return (await get.post(`repos/${self.repository}/git/commits`, options)).body
+ .sha;
+}
+
+async function getCommitDetails(self, commit) {
+ logger.debug(`getCommitDetails(${commit})`);
+ const results = await get(`repos/${self.repository}/git/commits/${commit}`);
+ return results.body;
+}
+
+module.exports = Storage;
diff --git a/lib/platform/github/util.js b/lib/platform/github/util.js
new file mode 100644
index 0000000000000000000000000000000000000000..71aa77f5078295b8c9f129ca8a7bd77b6dfe4612
--- /dev/null
+++ b/lib/platform/github/util.js
@@ -0,0 +1,11 @@
+module.exports = {
+ expandError,
+};
+
+function expandError(err) {
+ return {
+ err,
+ message: err.message,
+ body: err.response ? err.response.body : undefined,
+ };
+}
diff --git a/test/platform/github/index.spec.js b/test/platform/github/index.spec.js
index 5cbe68faaa52e697b286b7f58c1d60caa34f6c9c..34893f2556197a564423f65064dbbb9deda2dba2 100644
--- a/test/platform/github/index.spec.js
+++ b/test/platform/github/index.spec.js
@@ -444,11 +444,17 @@ describe('platform/github', () => {
});
});
describe('getFileList', () => {
+ beforeEach(async () => {
+ await initRepo({
+ repository: 'some/repo',
+ token: 'token',
+ });
+ });
it('returns empty array if error', async () => {
get.mockImplementationOnce(() => {
throw new Error('some error');
});
- const files = await github.getFileList();
+ const files = await github.getFileList('error-branch');
expect(files).toEqual([]);
});
it('warns if truncated result', async () => {
@@ -458,7 +464,7 @@ describe('platform/github', () => {
tree: [],
},
}));
- const files = await github.getFileList();
+ const files = await github.getFileList('truncated-branch');
expect(files.length).toBe(0);
});
it('caches the result', async () => {
@@ -468,9 +474,9 @@ describe('platform/github', () => {
tree: [],
},
}));
- let files = await github.getFileList();
+ let files = await github.getFileList('cached-branch');
expect(files.length).toBe(0);
- files = await github.getFileList();
+ files = await github.getFileList('cached-branch');
expect(files.length).toBe(0);
});
it('should return the files matching the fileName', async () => {
@@ -488,7 +494,7 @@ describe('platform/github', () => {
],
},
}));
- const files = await github.getFileList();
+ const files = await github.getFileList('npm-branch');
expect(files).toMatchSnapshot();
});
});
@@ -1835,6 +1841,11 @@ describe('platform/github', () => {
});
describe('getCommitMessages()', () => {
it('returns commits messages', async () => {
+ await initRepo({
+ repository: 'some/repo',
+ token: 'token',
+ gitAuthor: 'Renovate Bot <bot@renovatebot.com>',
+ });
get.mockReturnValueOnce({
body: [
{