From a674f727dd489264cb8bd0bb524326802ab33c39 Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Thu, 24 Feb 2022 17:32:46 +0300
Subject: [PATCH] feat(github): Platform-native REST-based push (#14271)
---
docs/usage/configuration-options.md | 8 +++
lib/config/options/index.ts | 7 +++
lib/platform/github/index.spec.ts | 98 +++++++++++++++++++++++++++++
lib/platform/github/index.ts | 76 ++++++++++++++++++++++
lib/util/git/index.ts | 12 ++++
lib/util/http/github.ts | 10 +--
lib/util/http/types.ts | 13 ++++
7 files changed, 215 insertions(+), 9 deletions(-)
diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
index c9c08f4444..59ef0fc81c 100644
--- a/docs/usage/configuration-options.md
+++ b/docs/usage/configuration-options.md
@@ -1833,6 +1833,14 @@ Normally when you set `rebaseWhen=auto` Renovate rebases any branch that's behin
This behavior is no longer guaranteed when you enable `platformAutomerge` because the platform might automerge a branch which is not up-to-date.
For example, GitHub might automerge a Renovate branch even if it's behind the base branch at the time.
+## platformCommit
+
+Supports only GitHub App mode and not when using Personal Access Tokens.
+
+To avoid errors, `gitAuthor` or `gitIgnoredAuthors` should be manually adjusted accordingly.
+
+The primary reason to use this option is because commits will then be signed automatically if authenticating as an app.
+
## postUpdateOptions
- `gomodTidy`: Run `go mod tidy` after Go module updates. This is implicitly enabled for major module updates when `gomodUpdateImportPaths` is enabled
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index a9ce491962..267b9b7128 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -2244,6 +2244,13 @@ const options: RenovateOptions[] = [
'As this PR has been closed unmerged, Renovate will now ignore this update ({{{newValue}}}). You will still receive a PR once a newer version is released, so if you wish to permanently ignore this dependency, please add it to the `ignoreDeps` array of your renovate config.',
},
},
+ {
+ name: 'platformCommit',
+ description: `Use platform API to perform commits instead of using git directly.`,
+ type: 'boolean',
+ default: false,
+ supportedPlatforms: ['github'],
+ },
];
export function getOptions(): RenovateOptions[] {
diff --git a/lib/platform/github/index.spec.ts b/lib/platform/github/index.spec.ts
index 6173790f6d..ebc3fbb6d4 100644
--- a/lib/platform/github/index.spec.ts
+++ b/lib/platform/github/index.spec.ts
@@ -2579,4 +2579,102 @@ describe('platform/github/index', () => {
expect(httpMock.getTrace()).toMatchSnapshot();
});
});
+
+ describe('pushFiles', () => {
+ beforeEach(() => {
+ git.prepareCommit.mockImplementation(({ files }) =>
+ Promise.resolve({
+ parentCommitSha: '1234567',
+ commitSha: '7654321',
+ files,
+ })
+ );
+ git.fetchCommit.mockImplementation(() => Promise.resolve('0abcdef'));
+ });
+ it('returns null if pre-commit phase has failed', async () => {
+ const scope = httpMock.scope(githubApiHost);
+ initRepoMock(scope, 'some/repo');
+ git.prepareCommit.mockResolvedValueOnce(null);
+
+ await github.initRepo({ repository: 'some/repo', token: 'token' } as any);
+
+ const res = await github.commitFiles({
+ branchName: 'foo/bar',
+ files: [
+ { type: 'addition', path: 'foo.bar', contents: 'foobar' },
+ { type: 'deletion', path: 'baz' },
+ { type: 'deletion', path: 'qux' },
+ ],
+ message: 'Foobar',
+ });
+
+ expect(res).toBeNull();
+ });
+ it('returns null on REST error', async () => {
+ const scope = httpMock.scope(githubApiHost);
+ initRepoMock(scope, 'some/repo');
+ await github.initRepo({ repository: 'some/repo', token: 'token' } as any);
+ scope.post('/repos/some/repo/git/trees').replyWithError('unknown');
+
+ const res = await github.commitFiles({
+ branchName: 'foo/bar',
+ files: [{ type: 'addition', path: 'foo.bar', contents: 'foobar' }],
+ message: 'Foobar',
+ });
+
+ expect(res).toBeNull();
+ });
+ it('commits and returns SHA string', async () => {
+ git.pushCommitToRenovateRef.mockResolvedValueOnce();
+ git.listCommitTree.mockResolvedValueOnce([]);
+ git.branchExists.mockReturnValueOnce(false);
+
+ const scope = httpMock.scope(githubApiHost);
+
+ initRepoMock(scope, 'some/repo');
+ await github.initRepo({ repository: 'some/repo', token: 'token' } as any);
+
+ scope
+ .post('/repos/some/repo/git/trees')
+ .reply(200, { sha: '111' })
+ .post('/repos/some/repo/git/commits')
+ .reply(200, { sha: '222' })
+ .post('/repos/some/repo/git/refs')
+ .reply(200);
+
+ const res = await github.commitFiles({
+ branchName: 'foo/bar',
+ files: [{ type: 'addition', path: 'foo.bar', contents: 'foobar' }],
+ message: 'Foobar',
+ });
+
+ expect(res).toBe('0abcdef');
+ });
+ it('performs rebase', async () => {
+ git.pushCommitToRenovateRef.mockResolvedValueOnce();
+ git.listCommitTree.mockResolvedValueOnce([]);
+ git.branchExists.mockReturnValueOnce(true);
+
+ const scope = httpMock.scope(githubApiHost);
+
+ initRepoMock(scope, 'some/repo');
+ await github.initRepo({ repository: 'some/repo', token: 'token' } as any);
+
+ scope
+ .post('/repos/some/repo/git/trees')
+ .reply(200, { sha: '111' })
+ .post('/repos/some/repo/git/commits')
+ .reply(200, { sha: '222' })
+ .patch('/repos/some/repo/git/refs/heads/foo/bar')
+ .reply(200);
+
+ const res = await github.commitFiles({
+ branchName: 'foo/bar',
+ files: [{ type: 'addition', path: 'foo.bar', contents: 'foobar' }],
+ message: 'Foobar',
+ });
+
+ expect(res).toBe('0abcdef');
+ });
+ });
});
diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts
index f7f854b0f2..013eb31a25 100644
--- a/lib/platform/github/index.ts
+++ b/lib/platform/github/index.ts
@@ -22,6 +22,12 @@ import { logger } from '../../logger';
import { BranchStatus, PrState, VulnerabilityAlert } from '../../types';
import { ExternalHostError } from '../../types/errors/external-host-error';
import * as git from '../../util/git';
+import { listCommitTree, pushCommitToRenovateRef } from '../../util/git';
+import type {
+ CommitFilesConfig,
+ CommitResult,
+ CommitSha,
+} from '../../util/git/types';
import * as hostRules from '../../util/host-rules';
import * as githubHttp from '../../util/http/github';
import { regEx } from '../../util/regex';
@@ -1719,3 +1725,73 @@ export async function getVulnerabilityAlerts(): Promise<VulnerabilityAlert[]> {
}
return alerts;
}
+
+async function pushFiles(
+ { branchName, message }: CommitFilesConfig,
+ { parentCommitSha, commitSha }: CommitResult
+): Promise<CommitSha | null> {
+ try {
+ // Push the commit to GitHub using a custom ref
+ // The associated blobs will be pushed automatically
+ await pushCommitToRenovateRef(commitSha, branchName);
+ // Get all the blobs which the commit/tree points to
+ // The blob SHAs will be the same locally as on GitHub
+ const treeItems = await listCommitTree(commitSha);
+
+ // For reasons unknown, we need to recreate our tree+commit on GitHub
+ // Attempting to reuse the tree or commit SHA we pushed does not work
+ const treeRes = await githubApi.postJson<{ sha: string }>(
+ `/repos/${config.repository}/git/trees`,
+ { body: { tree: treeItems } }
+ );
+ const treeSha = treeRes.body.sha;
+
+ // Now we recreate the commit using the tree we recreated the step before
+ const commitRes = await githubApi.postJson<{ sha: string }>(
+ `/repos/${config.repository}/git/commits`,
+ { body: { message, tree: treeSha, parents: [parentCommitSha] } }
+ );
+ const remoteCommitSha = commitRes.body.sha;
+
+ // Create branch if it didn't already exist, update it otherwise
+ if (git.branchExists(branchName)) {
+ // This is the equivalent of a git force push
+ // We are using this REST API because the GraphQL API doesn't support force push
+ await githubApi.patchJson(
+ `/repos/${config.repository}/git/refs/heads/${branchName}`,
+ { body: { sha: remoteCommitSha, force: true } }
+ );
+ } else {
+ await githubApi.postJson(`/repos/${config.repository}/git/refs`, {
+ body: { ref: `refs/heads/${branchName}`, sha: remoteCommitSha },
+ });
+ }
+
+ return remoteCommitSha;
+ } catch (err) {
+ logger.debug({ branchName, err }, 'Platform-native commit: unknown error');
+ return null;
+ }
+}
+
+export async function commitFiles(
+ config: CommitFilesConfig
+): Promise<CommitSha | null> {
+ const commitResult = await git.prepareCommit(config); // Commit locally and don't push
+ if (!commitResult) {
+ const { branchName, files } = config;
+ logger.debug(
+ { branchName, files: files.map(({ path }) => path) },
+ `Platform-native commit: unable to prepare for commit`
+ );
+ return null;
+ }
+ // Perform the commits using REST API
+ const pushResult = await pushFiles(config, commitResult);
+ if (!pushResult) {
+ return null;
+ }
+ // Because the branch commit was done remotely via REST API, now we git fetch it locally.
+ // We also do this step when committing/pushing using local git tooling.
+ return git.fetchCommit(config);
+}
diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts
index d8a3dc40cc..4396797a4d 100644
--- a/lib/util/git/index.ts
+++ b/lib/util/git/index.ts
@@ -787,6 +787,18 @@ async function handleCommitAuth(localDir: string): Promise<void> {
await writeGitAuthor();
}
+/**
+ *
+ * Prepare local branch with commit
+ *
+ * 0. Hard reset
+ * 1. Creates local branch with `origin/` prefix
+ * 2. Perform `git add` (respecting mode) and `git remove` for each file
+ * 3. Perform commit
+ * 4. Check whether resulting commit is empty or not (due to .gitignore)
+ * 5. If not empty, return commit info for further processing
+ *
+ */
export async function prepareCommit({
branchName,
files,
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
index 9273873442..039763ba38 100644
--- a/lib/util/http/github.ts
+++ b/lib/util/http/github.ts
@@ -17,6 +17,7 @@ import { regEx } from '../regex';
import { parseLinkHeader } from '../url';
import type { GotLegacyError } from './legacy';
import type {
+ GraphqlOptions,
HttpPostOptions,
HttpResponse,
InternalHttpOptions,
@@ -159,15 +160,6 @@ function handleGotError(
return err;
}
-interface GraphqlOptions {
- variables?: Record<string, string | number | null>;
- paginate?: boolean;
- count?: number;
- limit?: number;
- cursor?: string | null;
- acceptHeader?: string;
-}
-
interface GraphqlPaginatedContent<T = unknown> {
nodes: T[];
edges: T[];
diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
index 7b55da4c95..b7f378961a 100644
--- a/lib/util/http/types.ts
+++ b/lib/util/http/types.ts
@@ -33,6 +33,19 @@ export interface RequestStats {
export type OutgoingHttpHeaders = Record<string, string | string[] | undefined>;
+export interface GraphqlVariables {
+ [k: string]: unknown;
+}
+
+export interface GraphqlOptions {
+ variables?: GraphqlVariables;
+ paginate?: boolean;
+ count?: number;
+ limit?: number;
+ cursor?: string | null;
+ acceptHeader?: string;
+}
+
export interface HttpOptions {
body?: any;
username?: string;
--
GitLab