From 698a63ad32938f0d7f3688be1c766ed55f452e50 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@keylocation.sg>
Date: Wed, 28 Jun 2017 15:01:05 +0200
Subject: [PATCH] Fetch multiple pages from gitlab projects api endpoint (#377)
Fixes #371
---
lib/api/gitlab.js | 16 ++++++++++++++--
test/api/__snapshots__/gitlab.spec.js.snap | 18 ++++++++++++++++--
test/api/gitlab.spec.js | 19 +++++++++++++++++++
3 files changed, 49 insertions(+), 4 deletions(-)
diff --git a/lib/api/gitlab.js b/lib/api/gitlab.js
index 745eca25f0..aae1e79ef4 100644
--- a/lib/api/gitlab.js
+++ b/lib/api/gitlab.js
@@ -47,8 +47,20 @@ async function getRepos(token, endpoint) {
process.env.GITLAB_ENDPOINT = endpoint;
}
try {
- const res = await glGot('projects');
- return res.body.map(repo => repo.path_with_namespace);
+ let projects = [];
+ const perPage = 100;
+ let i = 1;
+ let res;
+ do {
+ const url = `projects?per_page=100&page=${i}`;
+ res = await glGot(url);
+ projects = projects.concat(
+ res.body.map(repo => repo.path_with_namespace)
+ );
+ i += 1;
+ } while (res.body.length === perPage);
+ logger.info(`Discovered ${projects.length} project(s)`);
+ return projects;
} catch (err) {
logger.error(`GitLab getRepos error: ${JSON.stringify(err)}`);
throw err;
diff --git a/test/api/__snapshots__/gitlab.spec.js.snap b/test/api/__snapshots__/gitlab.spec.js.snap
index 9e82ead5d4..55db2da208 100644
--- a/test/api/__snapshots__/gitlab.spec.js.snap
+++ b/test/api/__snapshots__/gitlab.spec.js.snap
@@ -237,10 +237,24 @@ Object {
}
`;
+exports[`api/gitlab getRepos should fetch multiple pages 1`] = `
+Array [
+ Array [
+ "projects?per_page=100&page=1",
+ ],
+ Array [
+ "projects?per_page=100&page=2",
+ ],
+ Array [
+ "projects?per_page=100&page=3",
+ ],
+]
+`;
+
exports[`api/gitlab getRepos should return an array of repos 1`] = `
Array [
Array [
- "projects",
+ "projects?per_page=100&page=1",
],
]
`;
@@ -255,7 +269,7 @@ Array [
exports[`api/gitlab getRepos should support a custom endpoint 1`] = `
Array [
Array [
- "projects",
+ "projects?per_page=100&page=1",
],
]
`;
diff --git a/test/api/gitlab.spec.js b/test/api/gitlab.spec.js
index 2ae6ff3d97..f8091cfc42 100644
--- a/test/api/gitlab.spec.js
+++ b/test/api/gitlab.spec.js
@@ -61,6 +61,25 @@ describe('api/gitlab', () => {
expect(glGot.mock.calls).toMatchSnapshot();
expect(repos).toMatchSnapshot();
});
+ it('should fetch multiple pages', async () => {
+ const repoCount = 250;
+ const projects = [];
+ for (let i = 0; i < repoCount; i += 1) {
+ projects.push({ path_with_namespace: `project${i}` });
+ }
+ glGot.mockImplementationOnce(() => ({
+ body: projects.slice(0, 100),
+ }));
+ glGot.mockImplementationOnce(() => ({
+ body: projects.slice(100, 200),
+ }));
+ glGot.mockImplementationOnce(() => ({
+ body: projects.slice(200),
+ }));
+ const repos = await gitlab.getRepos('sometoken');
+ expect(glGot.mock.calls).toMatchSnapshot();
+ expect(repos.length).toBe(repoCount);
+ });
});
async function initRepo(...args) {
--
GitLab