From a302b11a266a7895f404c4917763e341a261d447 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Fri, 23 Mar 2018 14:48:36 +0100
Subject: [PATCH] feat: custom docker registries (#1707)
Adds support for custom docker registries. To work (for now), registries must support anonymous public access to their v2 API. Tested against quay.io and gcr.io, including tags pagination for quay. Also needed to add a 10s timeout for registry queries to catch private/firewalled registries that we can't access.
Closes #797
---
lib/datasource/docker.js | 81 ++++++++++++++-----
lib/manager/docker/package.js | 35 ++++----
lib/manager/docker/update.js | 6 +-
test/datasource/docker.spec.js | 20 ++---
.../docker/__snapshots__/package.spec.js.snap | 15 +++-
test/manager/docker/package.spec.js | 10 +++
6 files changed, 118 insertions(+), 49 deletions(-)
diff --git a/lib/datasource/docker.js b/lib/datasource/docker.js
index 35f1b3a1be..23fb2efb17 100644
--- a/lib/datasource/docker.js
+++ b/lib/datasource/docker.js
@@ -1,17 +1,31 @@
const got = require('got');
+const parseLinkHeader = require('parse-link-header');
module.exports = {
getDigest,
getTags,
};
-const registry = 'https://index.docker.io/v2';
+function massageRegistry(input) {
+ let registry = input;
+ if (!registry || registry === 'docker.io') {
+ registry = 'index.docker.io'; // eslint-disable-line no-param-reassign
+ }
+ if (!registry.match('$https?://')) {
+ registry = `https://${registry}`; // eslint-disable-line no-param-reassign
+ }
+ return registry;
+}
function getRepository(pkgName) {
return pkgName.includes('/') ? pkgName : `library/${pkgName}`;
}
-async function getHeaders(repository) {
+async function getAuthHeaders(registry, repository) {
+ // istanbul ignore if
+ if (registry !== 'https://index.docker.io') {
+ return {};
+ }
try {
const authUrl = `https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repository}:pull`;
logger.debug(`Obtaining docker registry token for ${repository}`);
@@ -35,18 +49,19 @@ async function getHeaders(repository) {
}
}
-async function getDigest(name, tag = 'latest') {
+async function getDigest(registry, name, tag = 'latest') {
+ logger.debug(`getDigest(${registry}, ${name}, ${tag})`);
+ const massagedRegistry = massageRegistry(registry);
+ const repository = getRepository(name);
try {
- const repository = getRepository(name);
- const headers = await getHeaders(repository);
- const url = `${registry}/${repository}/manifests/${tag}`;
- const digest = (await got(url, { json: true, headers })).headers[
- 'docker-content-digest'
- ];
+ const url = `${massagedRegistry}/v2/${repository}/manifests/${tag}`;
+ const headers = await getAuthHeaders(massagedRegistry, repository);
+ headers.accept = 'application/vnd.docker.distribution.manifest.v2+json';
+ const digest = (await got(url, { json: true, headers, timeout: 10000 }))
+ .headers['docker-content-digest'];
logger.debug({ digest }, 'Got docker digest');
return digest;
- } catch (err) {
- // istanbul ignore if
+ } catch (err) /* istanbul ignore next */ {
if (err.statusCode === 401) {
logger.info(
{ err, body: err.response ? err.response.body : undefined, name, tag },
@@ -54,7 +69,6 @@ async function getDigest(name, tag = 'latest') {
);
return null;
}
- // istanbul ignore if
if (err.statusCode === 404) {
logger.info(
{ err, body: err.response ? err.response.body : undefined, name, tag },
@@ -62,7 +76,6 @@ async function getDigest(name, tag = 'latest') {
);
return null;
}
- // istanbul ignore if
if (err.statusCode >= 500 && err.statusCode < 600) {
logger.warn(
{ err, body: err.response ? err.response.body : undefined, name, tag },
@@ -70,6 +83,14 @@ async function getDigest(name, tag = 'latest') {
);
throw new Error('registry-failure');
}
+ if (err.code === 'ETIMEDOUT') {
+ logger.info(
+ { massagedRegistry },
+ 'Timeout when attempting to connect to docker registry'
+ );
+ logger.debug({ err });
+ return null;
+ }
logger.info(
{ err, body: err.response ? err.response.body : undefined, name, tag },
'Unknown Error looking up docker image digest'
@@ -78,20 +99,38 @@ async function getDigest(name, tag = 'latest') {
}
}
-async function getTags(name) {
+async function getTags(registry, name) {
+ logger.debug(`getTags(${registry}, ${name})`);
+ const massagedRegistry = massageRegistry(registry);
+ const repository = getRepository(name);
try {
- const repository = getRepository(name);
- const url = `${registry}/${repository}/tags/list?n=10000`;
- const headers = await getHeaders(repository);
- const { tags } = (await got(url, { json: true, headers })).body;
- logger.debug({ tags }, 'Got docker tags');
+ let url = `${massagedRegistry}/v2/${repository}/tags/list?n=10000`;
+ const headers = await getAuthHeaders(massagedRegistry, repository);
+ let tags = [];
+ let page = 1;
+ do {
+ const res = await got(url, { json: true, headers, timeout: 10000 });
+ tags = tags.concat(res.body.tags);
+ const linkHeader = parseLinkHeader(res.headers.link);
+ url = linkHeader && linkHeader.next ? linkHeader.next.url : null;
+ page += 1;
+ } while (url && page < 20);
+ logger.debug({ length: tags.length }, 'Got docker tags');
+ logger.trace({ tags });
return tags;
- } catch (err) {
- // istanbul ignore if
+ } catch (err) /* istanbul ignore next */ {
if (err.statusCode >= 500 && err.statusCode < 600) {
logger.warn({ err }, 'docker registry failure: internal error');
throw new Error('registry-failure');
}
+ if (err.code === 'ETIMEDOUT') {
+ logger.info(
+ { massagedRegistry },
+ 'Timeout when attempting to connect to docker registry'
+ );
+ logger.debug({ err });
+ return null;
+ }
logger.warn({ err, name }, 'Error getting docker image tags');
return null;
}
diff --git a/lib/manager/docker/package.js b/lib/manager/docker/package.js
index b3d04dc7fd..fa70d03b09 100644
--- a/lib/manager/docker/package.js
+++ b/lib/manager/docker/package.js
@@ -11,7 +11,6 @@ module.exports = {
async function getPackageUpdates(config) {
const {
dockerRegistry,
- currentFrom,
depName,
currentDepTag,
currentTag,
@@ -19,17 +18,14 @@ async function getPackageUpdates(config) {
unstablePattern,
ignoreUnstable,
} = config;
- if (dockerRegistry) {
- logger.info(
- { currentFrom, dockerRegistry },
- 'Skipping Dockerfile image with custom host'
- );
- return [];
- }
const upgrades = [];
if (currentDigest || config.pinDigests) {
logger.debug('Checking docker pinDigests');
- const newDigest = await dockerApi.getDigest(depName, currentTag);
+ const newDigest = await dockerApi.getDigest(
+ dockerRegistry,
+ depName,
+ currentTag
+ );
if (!newDigest) {
logger.debug({ content: config.content }, 'Dockerfile no digest');
}
@@ -38,7 +34,13 @@ async function getPackageUpdates(config) {
upgrade.newTag = currentTag || 'latest';
upgrade.newDigest = newDigest;
upgrade.newDigestShort = newDigest.slice(7, 13);
- upgrade.newFrom = `${depName}:${upgrade.newTag}@${newDigest}`;
+ if (dockerRegistry) {
+ upgrade.newFrom = `${dockerRegistry}/`;
+ } else {
+ upgrade.newFrom = '';
+ }
+ upgrade.newFrom += `${depName}:${upgrade.newTag}@${newDigest}`;
+
if (currentDigest) {
upgrade.type = 'digest';
upgrade.isDigest = true;
@@ -62,7 +64,7 @@ async function getPackageUpdates(config) {
const currentMajor = semver.major(padRange(tagVersion));
const currentlyStable = isStable(tagVersion, unstablePattern);
let versionList = [];
- const allTags = await dockerApi.getTags(config.depName);
+ const allTags = await dockerApi.getTags(dockerRegistry, config.depName);
if (allTags) {
versionList = allTags
.filter(tag => getSuffix(tag) === tagSuffix)
@@ -130,15 +132,20 @@ async function getPackageUpdates(config) {
};
upgrade.newVersion = newTag;
upgrade.newDepTag = `${config.depName}:${upgrade.newTag}`;
- let newFrom = upgrade.newDepTag;
+ if (dockerRegistry) {
+ upgrade.newFrom = `${dockerRegistry}/`;
+ } else {
+ upgrade.newFrom = '';
+ }
+ upgrade.newFrom += `${depName}:${upgrade.newTag}`;
if (config.currentDigest || config.pinDigests) {
upgrade.newDigest = await dockerApi.getDigest(
+ dockerRegistry,
config.depName,
upgrade.newTag
);
- newFrom = `${newFrom}@${upgrade.newDigest}`;
+ upgrade.newFrom += `@${upgrade.newDigest}`;
}
- upgrade.newFrom = newFrom;
if (newVersionMajor > currentMajor) {
upgrade.type = 'major';
upgrade.isMajor = true;
diff --git a/lib/manager/docker/update.js b/lib/manager/docker/update.js
index 0a2c868d7f..6d39255587 100644
--- a/lib/manager/docker/update.js
+++ b/lib/manager/docker/update.js
@@ -6,9 +6,9 @@ function setNewValue(currentFileContent, upgrade) {
try {
logger.debug(`setNewValue: ${upgrade.newFrom}`);
const oldLine = new RegExp(
- `(^|\\n)${upgrade.fromPrefix}(\\s+)${upgrade.depName}[^\\s]*(\\s?)${
- upgrade.fromSuffix
- }\\n`
+ `(^|\\n)${upgrade.fromPrefix}(\\s+)${
+ upgrade.dockerRegistry ? upgrade.dockerRegistry + '/' : ''
+ }${upgrade.depName}[^\\s]*(\\s?)${upgrade.fromSuffix}\\n`
);
const newLine = `$1${upgrade.fromPrefix}$2${upgrade.newFrom}$3${
upgrade.fromSuffix
diff --git a/test/datasource/docker.spec.js b/test/datasource/docker.spec.js
index cbd7efaa10..d0b1d58714 100644
--- a/test/datasource/docker.spec.js
+++ b/test/datasource/docker.spec.js
@@ -10,12 +10,12 @@ describe('api/docker', () => {
});
it('returns null if no token', async () => {
got.mockReturnValueOnce({ body: {} });
- const res = await docker.getDigest('some-name', undefined);
+ const res = await docker.getDigest(undefined, 'some-name', undefined);
expect(res).toBe(null);
});
it('returns null if errored', async () => {
got.mockReturnValueOnce({ body: { token: 'some-token' } });
- const res = await docker.getDigest('some-name', undefined);
+ const res = await docker.getDigest(undefined, 'some-name', undefined);
expect(res).toBe(null);
});
it('returns digest', async () => {
@@ -23,7 +23,7 @@ describe('api/docker', () => {
got.mockReturnValueOnce({
headers: { 'docker-content-digest': 'some-digest' },
});
- const res = await docker.getDigest('some-name', undefined);
+ const res = await docker.getDigest(undefined, 'some-name', undefined);
expect(res).toBe('some-digest');
});
it('supports scoped names', async () => {
@@ -31,26 +31,26 @@ describe('api/docker', () => {
got.mockReturnValueOnce({
headers: { 'docker-content-digest': 'some-digest' },
});
- const res = await docker.getDigest('some/name', undefined);
+ const res = await docker.getDigest(undefined, 'some/name', undefined);
expect(res).toBe('some-digest');
});
});
describe('getTags', () => {
it('returns null if no token', async () => {
got.mockReturnValueOnce({ body: {} });
- const res = await docker.getTags('node');
+ const res = await docker.getTags(undefined, 'node');
expect(res).toBe(null);
});
it('returns tags', async () => {
const tags = ['a', 'b'];
- got.mockReturnValueOnce({ body: { token: 'some-token ' } });
- got.mockReturnValueOnce({ body: { tags } });
- const res = await docker.getTags('my/node');
- expect(res).toBe(tags);
+ got.mockReturnValueOnce({ headers: {}, body: { token: 'some-token ' } });
+ got.mockReturnValueOnce({ headers: {}, body: { tags } });
+ const res = await docker.getTags(undefined, 'my/node');
+ expect(res).toEqual(tags);
});
it('returns null on error', async () => {
got.mockReturnValueOnce({});
- const res = await docker.getTags('node');
+ const res = await docker.getTags(undefined, 'node');
expect(res).toBe(null);
});
});
diff --git a/test/manager/docker/__snapshots__/package.spec.js.snap b/test/manager/docker/__snapshots__/package.spec.js.snap
index 6eec3282ad..f891a37523 100644
--- a/test/manager/docker/__snapshots__/package.spec.js.snap
+++ b/test/manager/docker/__snapshots__/package.spec.js.snap
@@ -37,6 +37,19 @@ Array [
]
`;
+exports[`lib/workers/package/docker getPackageUpdates returns a digest when registry is present 1`] = `
+Array [
+ Object {
+ "isDigest": true,
+ "newDigest": "sha256:1234567890",
+ "newDigestShort": "123456",
+ "newFrom": "docker.io/some-dep:1.0.0@sha256:1234567890",
+ "newTag": "1.0.0",
+ "type": "digest",
+ },
+]
+`;
+
exports[`lib/workers/package/docker getPackageUpdates returns major and minor upgrades 1`] = `
Array [
Object {
@@ -118,7 +131,7 @@ Array [
"isMajor": true,
"newDepTag": "some-dep:3.0.0",
"newDigest": "sha256:one",
- "newFrom": "some-dep:3.0.0@sha256:one",
+ "newFrom": "docker.io/some-dep:3.0.0@sha256:one",
"newTag": "3.0.0",
"newVersion": "3.0.0",
"newVersionMajor": "3",
diff --git a/test/manager/docker/package.spec.js b/test/manager/docker/package.spec.js
index 7ee98d2fa5..4105764f5e 100644
--- a/test/manager/docker/package.spec.js
+++ b/test/manager/docker/package.spec.js
@@ -47,6 +47,15 @@ describe('lib/workers/package/docker', () => {
expect(res).toHaveLength(1);
expect(res[0].type).toEqual('digest');
});
+ it('returns a digest when registry is present', async () => {
+ config.dockerRegistry = 'docker.io';
+ config.currentFrom = 'docker.io/some-dep:1.0.0@sha256:abcdefghijklmnop';
+ dockerApi.getDigest.mockReturnValueOnce('sha256:1234567890');
+ const res = await docker.getPackageUpdates(config);
+ expect(res).toMatchSnapshot();
+ expect(res).toHaveLength(1);
+ expect(res[0].type).toEqual('digest');
+ });
it('adds latest tag', async () => {
delete config.currentTag;
dockerApi.getDigest.mockReturnValueOnce('sha256:1234567890');
@@ -67,6 +76,7 @@ describe('lib/workers/package/docker', () => {
expect(await docker.getPackageUpdates(config)).toEqual([]);
});
it('returns only one upgrade if automerging major', async () => {
+ config.dockerRegistry = 'docker.io';
dockerApi.getDigest.mockReturnValueOnce(config.currentDigest);
dockerApi.getDigest.mockReturnValueOnce('sha256:one');
dockerApi.getTags.mockReturnValueOnce([
--
GitLab