From ffef63f47c9a193fb0ab22114d1c6e3d11f1470e Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@keylocation.sg>
Date: Tue, 17 Oct 2017 21:46:49 +0200
Subject: [PATCH] feat: suppor gitlab api pagination (#971)
Adds paginated results capability for gitlab and enables it for getFileList. This should enable all files when using APIv4.
Hopeful this c-l-o-s-e-s #962 & #968
---
lib/api/gitlab.js | 5 +--
lib/api/gl-got-wrapper.js | 25 +++++++++++++++
test/api/gitlab.spec.js | 4 +--
test/api/gl-got-wrapper.spec.js | 57 +++++++++++++++++++++++++++++++++
4 files changed, 87 insertions(+), 4 deletions(-)
create mode 100644 lib/api/gl-got-wrapper.js
create mode 100644 test/api/gl-got-wrapper.spec.js
diff --git a/lib/api/gitlab.js b/lib/api/gitlab.js
index 8da2e4481e..b93d162b18 100644
--- a/lib/api/gitlab.js
+++ b/lib/api/gitlab.js
@@ -1,5 +1,5 @@
let logger = require('../logger');
-const get = require('gl-got');
+const get = require('./gl-got-wrapper');
const config = {};
@@ -129,7 +129,8 @@ async function getFileList(branchName) {
return config.fileList;
}
const res = await get(
- `projects/${config.repoName}/repository/tree?ref=${branchName}&recursive=true`
+ `projects/${config.repoName}/repository/tree?ref=${branchName}&recursive=true`,
+ { paginate: true }
);
config.fileList = res.body
.filter(item => item.type === 'blob')
diff --git a/lib/api/gl-got-wrapper.js b/lib/api/gl-got-wrapper.js
new file mode 100644
index 0000000000..af2f259dce
--- /dev/null
+++ b/lib/api/gl-got-wrapper.js
@@ -0,0 +1,25 @@
+const glGot = require('gl-got');
+const parseLinkHeader = require('parse-link-header');
+
+async function get(path, opts, retries = 5) {
+ const res = await glGot(path, opts);
+ if (opts && opts.paginate) {
+ // Check if result is paginated
+ const linkHeader = parseLinkHeader(res.headers.link);
+ if (linkHeader && linkHeader.next) {
+ res.body = res.body.concat(
+ (await get(linkHeader.next.url, opts, retries)).body
+ );
+ }
+ }
+ return res;
+}
+
+const helpers = ['get', 'post', 'put', 'patch', 'head', 'delete'];
+
+for (const x of helpers) {
+ get[x] = (url, opts) =>
+ get(url, Object.assign({}, opts, { method: x.toUpperCase() }));
+}
+
+module.exports = get;
diff --git a/test/api/gitlab.spec.js b/test/api/gitlab.spec.js
index bd8c631127..e8eb9e2aad 100644
--- a/test/api/gitlab.spec.js
+++ b/test/api/gitlab.spec.js
@@ -10,9 +10,9 @@ describe('api/gitlab', () => {
// reset module
jest.resetModules();
- jest.mock('gl-got');
+ jest.mock('../../lib/api/gl-got-wrapper');
gitlab = require('../../lib/api/gitlab');
- get = require('gl-got');
+ get = require('../../lib/api/gl-got-wrapper');
});
describe('getRepos', () => {
diff --git a/test/api/gl-got-wrapper.spec.js b/test/api/gl-got-wrapper.spec.js
new file mode 100644
index 0000000000..3f2fd4bcdd
--- /dev/null
+++ b/test/api/gl-got-wrapper.spec.js
@@ -0,0 +1,57 @@
+const get = require('../../lib/api/gl-got-wrapper');
+const glGot = require('gl-got');
+
+jest.mock('gl-got');
+
+describe('api/gl-got-wrapper', () => {
+ const body = ['a', 'b'];
+ beforeEach(() => {
+ jest.resetAllMocks();
+ });
+ it('paginates', async () => {
+ glGot.mockReturnValueOnce({
+ headers: {
+ link:
+ '<https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=2>; rel="next", <https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
+ },
+ body: ['a'],
+ });
+ glGot.mockReturnValueOnce({
+ headers: {
+ link:
+ '<https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=3>; rel="next", <https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
+ },
+ body: ['b', 'c'],
+ });
+ glGot.mockReturnValueOnce({
+ headers: {},
+ body: ['d'],
+ });
+ const res = await get('some-url', { paginate: true });
+ expect(res.body).toHaveLength(4);
+ expect(glGot.mock.calls).toHaveLength(3);
+ });
+ it('attempts to paginate', async () => {
+ glGot.mockReturnValueOnce({
+ headers: {
+ link:
+ '<https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
+ },
+ body: ['a'],
+ });
+ glGot.mockReturnValueOnce({
+ headers: {},
+ body: ['b'],
+ });
+ const res = await get('some-url', { paginate: true });
+ expect(res.body).toHaveLength(1);
+ expect(glGot.mock.calls).toHaveLength(1);
+ });
+ it('posts', async () => {
+ glGot.mockImplementationOnce(() => ({
+ body,
+ }));
+ const res = await get.post('some-url');
+ expect(res.body).toEqual(body);
+ });
+});
--
GitLab