diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..0757c0a8095f64ffda72dcab916d4affeb8a12d5
--- /dev/null
+++ b/lib/util/http/github.spec.ts
@@ -0,0 +1,369 @@
+import delay from 'delay';
+import * as httpMock from '../../../test/httpMock';
+import { getName } from '../../../test/util';
+import {
+ PLATFORM_BAD_CREDENTIALS,
+ PLATFORM_FAILURE,
+ PLATFORM_INTEGRATION_UNAUTHORIZED,
+ PLATFORM_RATE_LIMIT_EXCEEDED,
+ REPOSITORY_CHANGED,
+} from '../../constants/error-messages';
+import { clearRepoCache } from '../cache';
+import { GithubHttp, handleGotError, setBaseUrl } from './github';
+
+const githubApiHost = 'https://api.github.com';
+
+jest.mock('delay');
+
+describe(getName(__filename), () => {
+ let githubApi;
+ beforeEach(() => {
+ githubApi = new GithubHttp();
+ setBaseUrl(githubApiHost);
+ jest.resetAllMocks();
+ delete global.appMode;
+ httpMock.setup();
+ });
+
+ afterEach(() => {
+ httpMock.reset();
+ clearRepoCache();
+ });
+
+ describe('HTTP', () => {
+ function getError(errOrig: any): Error {
+ try {
+ return handleGotError(errOrig, `${githubApiHost}/some-url`, {});
+ } catch (err) {
+ return err;
+ }
+ return null;
+ }
+
+ beforeEach(() => {
+ (delay as any).mockImplementation(() => Promise.resolve());
+ });
+
+ it('supports app mode', async () => {
+ httpMock.scope(githubApiHost).get('/some-url').reply(200);
+ global.appMode = true;
+ await githubApi.get('/some-url', {
+ headers: { accept: 'some-accept' },
+ });
+ const [req] = httpMock.getTrace();
+ expect(req).toBeDefined();
+ expect(req.headers.accept).toBe(
+ 'application/vnd.github.machine-man-preview+json, some-accept'
+ );
+ });
+ it('strips v3 for graphql', async () => {
+ httpMock
+ .scope('https://ghe.mycompany.com')
+ .post('/graphql')
+ .reply(200, {});
+ setBaseUrl('https://ghe.mycompany.com/api/v3/');
+ await githubApi.postJson('/graphql', {
+ body: {},
+ });
+ const [req] = httpMock.getTrace();
+ expect(req).toBeDefined();
+ expect(req.url.includes('/v3')).toBe(false);
+ });
+ it('paginates', async () => {
+ const url = '/some-url';
+ httpMock
+ .scope(githubApiHost)
+ .get(url)
+ .reply(200, ['a'], {
+ link: `<${url}?page=2>; rel="next", <${url}?page=3>; rel="last"`,
+ })
+ .get(`${url}?page=2`)
+ .reply(200, ['b', 'c'], {
+ link: `<${url}?page=3>; rel="next", <${url}?page=3>; rel="last"`,
+ })
+ .get(`${url}?page=3`)
+ .reply(200, ['d']);
+ const res = await githubApi.getJson('some-url', { paginate: true });
+ expect(res.body).toEqual(['a', 'b', 'c', 'd']);
+ const trace = httpMock.getTrace();
+ expect(trace).toHaveLength(3);
+ });
+ it('attempts to paginate', async () => {
+ const url = '/some-url';
+ httpMock
+ .scope(githubApiHost)
+ .get(url)
+ .reply(200, ['a'], {
+ link: `<${url}?page=34>; rel="last"`,
+ });
+ const res = await githubApi.getJson('some-url', { paginate: true });
+ expect(res).toBeDefined();
+ expect(res.body).toEqual(['a']);
+ const trace = httpMock.getTrace();
+ expect(trace).toHaveLength(1);
+ });
+ it('should throw rate limit exceeded', () => {
+ const e = getError({
+ statusCode: 403,
+ message:
+ 'Error updating branch: API rate limit exceeded for installation ID 48411. (403)',
+ });
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_RATE_LIMIT_EXCEEDED);
+ });
+ it('should throw Bad credentials', () => {
+ const e = getError({
+ statusCode: 401,
+ message: 'Bad credentials. (401)',
+ });
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_BAD_CREDENTIALS);
+ });
+ it('should throw platform failure', () => {
+ const e = getError({
+ statusCode: 401,
+ message: 'Bad credentials. (401)',
+ headers: {
+ 'x-ratelimit-limit': '60',
+ },
+ });
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_FAILURE);
+ });
+ it('should throw platform failure for ENOTFOUND, ETIMEDOUT or EAI_AGAIN', () => {
+ const codes = ['ENOTFOUND', 'ETIMEDOUT', 'EAI_AGAIN'];
+ for (let idx = 0; idx < codes.length; idx += 1) {
+ const code = codes[idx];
+ const e = getError({
+ name: 'RequestError',
+ code,
+ });
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_FAILURE);
+ }
+ });
+ it('should throw platform failure for 500', () => {
+ const e = getError({
+ statusCode: 500,
+ message: 'Internal Server Error',
+ });
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_FAILURE);
+ });
+ it('should throw platform failure ParseError', () => {
+ const e = getError({
+ name: 'ParseError',
+ });
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_FAILURE);
+ });
+ it('should throw for unauthorized integration', () => {
+ const e = getError({
+ statusCode: 403,
+ message: 'Resource not accessible by integration (403)',
+ });
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_INTEGRATION_UNAUTHORIZED);
+ });
+ it('should throw for unauthorized integration', () => {
+ const gotErr = {
+ statusCode: 403,
+ body: { message: 'Upgrade to GitHub Pro' },
+ };
+ const e = getError(gotErr);
+ expect(e).toBeDefined();
+ expect(e).toBe(gotErr);
+ });
+ it('should throw on abuse', () => {
+ const gotErr = {
+ statusCode: 403,
+ message: 'You have triggered an abuse detection mechanism',
+ };
+ const e = getError(gotErr);
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_RATE_LIMIT_EXCEEDED);
+ });
+ it('should throw on repository change', () => {
+ const gotErr = {
+ statusCode: 422,
+ body: {
+ message: 'foobar',
+ errors: [{ code: 'invalid' }],
+ },
+ };
+ const e = getError(gotErr);
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(REPOSITORY_CHANGED);
+ });
+ it('should throw platform failure on 422 response', () => {
+ const gotErr = {
+ statusCode: 422,
+ message: 'foobar',
+ };
+ const e = getError(gotErr);
+ expect(e).toBeDefined();
+ expect(e.message).toEqual(PLATFORM_FAILURE);
+ });
+ it('should throw original error when failed to add reviewers', () => {
+ const gotErr = {
+ statusCode: 422,
+ message: 'Review cannot be requested from pull request author.',
+ };
+ const e = getError(gotErr);
+ expect(e).toBeDefined();
+ expect(e).toStrictEqual(gotErr);
+ });
+ it('should throw original error of unknown type', () => {
+ const gotErr = {
+ statusCode: 418,
+ message: 'Sorry, this is a teapot',
+ };
+ const e = getError(gotErr);
+ expect(e).toBe(gotErr);
+ });
+ });
+
+ describe('GraphQL', () => {
+ const query = `
+ query {
+ repository(owner: "testOwner", name: "testName") {
+ testItem (orderBy: {field: UPDATED_AT, direction: DESC}, filterBy: {createdBy: "someone"}) {
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ nodes {
+ number state title body
+ }
+ }
+ }
+ }`;
+
+ it('supports app mode', async () => {
+ httpMock.scope(githubApiHost).post('/graphql').reply(200, {});
+ global.appMode = true;
+ await githubApi.getGraphqlNodes(query, 'testItem', { paginate: false });
+ const [req] = httpMock.getTrace();
+ expect(req).toBeDefined();
+ expect(req.headers.accept).toBe(
+ 'application/vnd.github.machine-man-preview+json, application/vnd.github.merge-info-preview+json'
+ );
+ });
+ it('returns empty array for undefined data', async () => {
+ httpMock
+ .scope(githubApiHost)
+ .post('/graphql')
+ .reply(200, {
+ data: {
+ someprop: 'someval',
+ },
+ });
+ expect(
+ await githubApi.getGraphqlNodes(query, 'testItem', { paginate: false })
+ ).toEqual([]);
+ });
+ it('returns empty array for undefined data.', async () => {
+ httpMock
+ .scope(githubApiHost)
+ .post('/graphql')
+ .reply(200, {
+ data: { repository: { otherField: 'someval' } },
+ });
+ expect(
+ await githubApi.getGraphqlNodes(query, 'testItem', { paginate: false })
+ ).toEqual([]);
+ });
+ it('throws errors for invalid responses', async () => {
+ httpMock.scope(githubApiHost).post('/graphql').reply(418);
+ await expect(
+ githubApi.getGraphqlNodes(query, 'someItem', {
+ paginate: false,
+ })
+ ).rejects.toThrow("Response code 418 (I'm a Teapot)");
+ });
+ it('halves node count and retries request', async () => {
+ httpMock
+ .scope(githubApiHost)
+ .persist()
+ .post('/graphql')
+ .reply(200, {
+ data: {
+ someprop: 'someval',
+ },
+ });
+ await githubApi.getGraphqlNodes(query, 'testItem');
+ expect(httpMock.getTrace()).toHaveLength(7);
+ });
+ it('retrieves all data from all pages', async () => {
+ httpMock
+ .scope(githubApiHost)
+ .post('/graphql')
+ .reply(200, {
+ data: {
+ repository: {
+ testItem: {
+ pageInfo: {
+ endCursor: 'cursor1',
+ hasNextPage: true,
+ },
+ nodes: [
+ {
+ number: 1,
+ state: 'OPEN',
+ title: 'title-1',
+ body: 'the body 1',
+ },
+ ],
+ },
+ },
+ },
+ })
+ .post('/graphql')
+ .reply(200, {
+ data: {
+ repository: {
+ testItem: {
+ pageInfo: {
+ endCursor: 'cursor2',
+ hasNextPage: true,
+ },
+ nodes: [
+ {
+ number: 2,
+ state: 'CLOSED',
+ title: 'title-2',
+ body: 'the body 2',
+ },
+ ],
+ },
+ },
+ },
+ })
+ .post('/graphql')
+ .reply(200, {
+ data: {
+ repository: {
+ testItem: {
+ pageInfo: {
+ endCursor: 'cursor3',
+ hasNextPage: false,
+ },
+ nodes: [
+ {
+ number: 3,
+ state: 'OPEN',
+ title: 'title-3',
+ body: 'the body 3',
+ },
+ ],
+ },
+ },
+ },
+ });
+
+ const items = await githubApi.getGraphqlNodes(query, 'testItem');
+ expect(httpMock.getTrace()).toHaveLength(3);
+ expect(items.length).toEqual(3);
+ });
+ });
+});
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
new file mode 100644
index 0000000000000000000000000000000000000000..b555bb47b3303c4efceb288a847832c606ca08ca
--- /dev/null
+++ b/lib/util/http/github.ts
@@ -0,0 +1,301 @@
+import URL from 'url';
+import { GotError } from 'got';
+import pAll from 'p-all';
+import parseLinkHeader from 'parse-link-header';
+import {
+ PLATFORM_BAD_CREDENTIALS,
+ PLATFORM_FAILURE,
+ PLATFORM_INTEGRATION_UNAUTHORIZED,
+ PLATFORM_RATE_LIMIT_EXCEEDED,
+ REPOSITORY_CHANGED,
+} from '../../constants/error-messages';
+import { PLATFORM_TYPE_GITHUB } from '../../constants/platforms';
+import { logger } from '../../logger';
+import { maskToken } from '../mask';
+import { Http, HttpPostOptions, HttpResponse, InternalHttpOptions } from '.';
+
+let baseUrl = 'https://api.github.com/';
+export const setBaseUrl = (url: string): void => {
+ baseUrl = url;
+};
+
+type GotRequestError<E = unknown, T = unknown> = GotError & {
+ body: {
+ message?: string;
+ errors?: E[];
+ };
+ headers?: Record<string, T>;
+};
+
+interface GithubInternalOptions extends InternalHttpOptions {
+ body?: string;
+}
+
+export interface GithubHttpOptions extends InternalHttpOptions {
+ paginate?: boolean | string;
+ pageLimit?: number;
+ token?: string;
+}
+
+interface GithubGraphqlResponse<T = unknown> {
+ data?: {
+ repository?: T;
+ };
+ errors?: { message: string; locations: unknown }[];
+}
+
+export function handleGotError(
+ err: GotRequestError,
+ url: string | URL,
+ opts: GithubHttpOptions
+): never {
+ const path = url.toString();
+ let message = err.message || '';
+ if (err.body?.message) {
+ message = err.body.message;
+ }
+ if (
+ err.name === 'RequestError' &&
+ (err.code === 'ENOTFOUND' ||
+ err.code === 'ETIMEDOUT' ||
+ err.code === 'EAI_AGAIN')
+ ) {
+ logger.debug({ err }, 'GitHub failure: RequestError');
+ throw new Error(PLATFORM_FAILURE);
+ }
+ if (err.name === 'ParseError') {
+ logger.debug({ err }, '');
+ throw new Error(PLATFORM_FAILURE);
+ }
+ if (err.statusCode >= 500 && err.statusCode < 600) {
+ logger.debug({ err }, 'GitHub failure: 5xx');
+ throw new Error(PLATFORM_FAILURE);
+ }
+ if (
+ err.statusCode === 403 &&
+ message.startsWith('You have triggered an abuse detection mechanism')
+ ) {
+ logger.debug({ err }, 'GitHub failure: abuse detection');
+ throw new Error(PLATFORM_RATE_LIMIT_EXCEEDED);
+ }
+ if (err.statusCode === 403 && message.includes('Upgrade to GitHub Pro')) {
+ logger.debug({ path }, 'Endpoint needs paid GitHub plan');
+ throw err;
+ }
+ if (err.statusCode === 403 && message.includes('rate limit exceeded')) {
+ logger.debug({ err }, 'GitHub failure: rate limit');
+ throw new Error(PLATFORM_RATE_LIMIT_EXCEEDED);
+ }
+ if (
+ err.statusCode === 403 &&
+ message.startsWith('Resource not accessible by integration')
+ ) {
+ logger.debug(
+ { err },
+ 'GitHub failure: Resource not accessible by integration'
+ );
+ throw new Error(PLATFORM_INTEGRATION_UNAUTHORIZED);
+ }
+ if (err.statusCode === 401 && message.includes('Bad credentials')) {
+ const rateLimit = err.headers ? err.headers['x-ratelimit-limit'] : -1;
+ logger.debug(
+ {
+ token: maskToken(opts.token),
+ err,
+ },
+ 'GitHub failure: Bad credentials'
+ );
+ if (rateLimit === '60') {
+ throw new Error(PLATFORM_FAILURE);
+ }
+ throw new Error(PLATFORM_BAD_CREDENTIALS);
+ }
+ if (err.statusCode === 422) {
+ if (
+ message.includes('Review cannot be requested from pull request author')
+ ) {
+ throw err;
+ } else if (
+ err.body &&
+ err.body.errors &&
+ err.body.errors.find((e: any) => e.code === 'invalid')
+ ) {
+ throw new Error(REPOSITORY_CHANGED);
+ }
+ logger.debug({ err }, '422 Error thrown from GitHub');
+ throw new Error(PLATFORM_FAILURE);
+ }
+ throw err;
+}
+
+interface GraphqlOptions {
+ paginate?: boolean;
+ count?: number;
+ acceptHeader?: string;
+ fromEnd?: boolean;
+}
+
+export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
+ constructor(options?: GithubHttpOptions) {
+ super(PLATFORM_TYPE_GITHUB, options);
+ }
+
+ protected async request<T>(
+ url: string | URL,
+ options?: GithubInternalOptions & GithubHttpOptions,
+ okToRetry = true
+ ): Promise<HttpResponse<T> | null> {
+ let result = null;
+
+ const opts = {
+ baseUrl,
+ ...options,
+ throwHttpErrors: true,
+ };
+
+ const method = opts.method || 'get';
+
+ if (method.toLowerCase() === 'post' && url === 'graphql') {
+ // GitHub Enterprise uses unversioned graphql path
+ opts.baseUrl = opts.baseUrl.replace('/v3/', '/');
+ }
+
+ if (global.appMode) {
+ const accept = 'application/vnd.github.machine-man-preview+json';
+ opts.headers = {
+ accept,
+ ...opts.headers,
+ };
+ const optsAccept = opts?.headers?.accept;
+ if (typeof optsAccept === 'string' && !optsAccept.includes(accept)) {
+ opts.headers.accept = `${accept}, ${opts.headers.accept}`;
+ }
+ }
+
+ try {
+ result = await super.request<T>(url, opts);
+
+ if (result !== null) {
+ if (opts.paginate) {
+ // Check if result is paginated
+ const pageLimit = opts.pageLimit || 10;
+ const linkHeader =
+ result?.headers?.link &&
+ parseLinkHeader(result.headers.link as string);
+ if (linkHeader && linkHeader.next && linkHeader.last) {
+ let lastPage = +linkHeader.last.page;
+ if (!process.env.RENOVATE_PAGINATE_ALL && opts.paginate !== 'all') {
+ lastPage = Math.min(pageLimit, lastPage);
+ }
+ const pageNumbers = Array.from(
+ new Array(lastPage),
+ (x, i) => i + 1
+ ).slice(1);
+ const queue = pageNumbers.map((page) => (): Promise<
+ HttpResponse
+ > => {
+ const nextUrl = URL.parse(linkHeader.next.url, true);
+ delete nextUrl.search;
+ nextUrl.query.page = page.toString();
+ return this.request(
+ URL.format(nextUrl),
+ { ...opts, paginate: false },
+ okToRetry
+ );
+ });
+ const pages = await pAll(queue, { concurrency: 5 });
+ result.body = result.body.concat(
+ ...pages.filter(Boolean).map((page) => page.body)
+ );
+ }
+ }
+ }
+ } catch (err) {
+ handleGotError(err, url, opts);
+ }
+
+ return result;
+ }
+
+ private async getGraphql<T = unknown>(
+ query: string,
+ accept = 'application/vnd.github.merge-info-preview+json'
+ ): Promise<GithubGraphqlResponse<T>> {
+ let result = null;
+
+ const path = 'graphql';
+
+ const opts: HttpPostOptions = {
+ body: { query },
+ headers: { accept },
+ };
+
+ logger.trace(`Performing Github GraphQL request`);
+
+ try {
+ const res = await this.postJson('graphql', opts);
+ result = res?.body;
+ } catch (gotErr) {
+ handleGotError(gotErr, path, opts);
+ }
+ return result;
+ }
+
+ async getGraphqlNodes<T = Record<string, unknown>>(
+ queryOrig: string,
+ fieldName: string,
+ options: GraphqlOptions = {}
+ ): Promise<T[]> {
+ const result: T[] = [];
+
+ const regex = new RegExp(`(\\W)${fieldName}(\\s*)\\(`);
+
+ const { paginate = true, acceptHeader } = options;
+ let count = options.count || 100;
+ let cursor = null;
+
+ let isIterating = true;
+ while (isIterating) {
+ let query = queryOrig;
+ if (paginate) {
+ let replacement = `$1${fieldName}$2(first: ${count}`;
+ replacement += cursor ? `, after: "${cursor}", ` : ', ';
+ query = query.replace(regex, replacement);
+ }
+ const gqlRes = await this.getGraphql<T>(query, acceptHeader);
+ if (
+ gqlRes &&
+ gqlRes.data &&
+ gqlRes.data.repository &&
+ gqlRes.data.repository[fieldName]
+ ) {
+ const { nodes = [], edges = [], pageInfo } = gqlRes.data.repository[
+ fieldName
+ ];
+ result.push(...nodes);
+ result.push(...edges);
+
+ if (paginate && pageInfo) {
+ const { hasNextPage, endCursor } = pageInfo;
+ if (hasNextPage && endCursor) {
+ cursor = endCursor;
+ } else {
+ isIterating = false;
+ }
+ }
+ } else {
+ count = Math.floor(count / 2);
+ if (count === 0) {
+ logger.error('Error fetching GraphQL nodes');
+ isIterating = false;
+ }
+ }
+
+ if (!paginate) {
+ isIterating = false;
+ }
+ }
+
+ return result;
+ }
+}
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index e7eb4b97dec15c05f9271036c51f8544468fe4c6..01a1a0899279ef1dfec3faaba57b5171fc8cbbf6 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -18,7 +18,7 @@ export interface HttpPostOptions extends HttpOptions {
body: unknown;
}
-interface InternalHttpOptions extends HttpOptions {
+export interface InternalHttpOptions extends HttpOptions {
json?: boolean;
method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head';
}
@@ -28,13 +28,14 @@ export interface HttpResponse<T = string> {
headers: any;
}
-export class Http {
+export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
constructor(private hostType: string, private options?: HttpOptions) {}
- private async request<T>(
+ protected async request<T>(
url: string | URL,
- options?: InternalHttpOptions
+ httpOpts?: InternalHttpOptions
): Promise<HttpResponse<T> | null> {
+ const options = { ...httpOpts };
let resolvedUrl = url.toString();
if (options?.baseUrl) {
resolvedUrl = URL.resolve(options.baseUrl, resolvedUrl);
@@ -89,46 +90,46 @@ export class Http {
async getJson<T = unknown>(
url: string,
- options: HttpOptions = {}
+ options?: GetOptions
): Promise<HttpResponse<T>> {
- return this.requestJson<T>(url, options);
+ return this.requestJson<T>(url, { ...options });
+ }
+
+ async headJson<T = unknown>(
+ url: string,
+ options?: GetOptions
+ ): Promise<HttpResponse<T>> {
+ return this.requestJson<T>(url, { ...options, method: 'head' });
}
async postJson<T = unknown>(
url: string,
- options: HttpPostOptions
+ options?: PostOptions
): Promise<HttpResponse<T>> {
return this.requestJson<T>(url, { ...options, method: 'post' });
}
async putJson<T = unknown>(
url: string,
- options: HttpPostOptions
+ options?: PostOptions
): Promise<HttpResponse<T>> {
return this.requestJson<T>(url, { ...options, method: 'put' });
}
async patchJson<T = unknown>(
url: string,
- options: HttpPostOptions
+ options?: PostOptions
): Promise<HttpResponse<T>> {
return this.requestJson<T>(url, { ...options, method: 'patch' });
}
async deleteJson<T = unknown>(
url: string,
- options: HttpPostOptions
+ options?: PostOptions
): Promise<HttpResponse<T>> {
return this.requestJson<T>(url, { ...options, method: 'delete' });
}
- async headJson<T = unknown>(
- url: string,
- options: HttpPostOptions
- ): Promise<HttpResponse<T>> {
- return this.requestJson<T>(url, { ...options, method: 'head' });
- }
-
stream(url: string, options?: HttpOptions): NodeJS.ReadableStream {
const combinedOptions: any = {
method: 'get',