diff --git a/lib/datasource/docker/__snapshots__/common.spec.ts.snap b/lib/datasource/docker/__snapshots__/common.spec.ts.snap
new file mode 100644
index 0000000000000000000000000000000000000000..23bcfb457646278737201286dcad995d2f19f91a
--- /dev/null
+++ b/lib/datasource/docker/__snapshots__/common.spec.ts.snap
@@ -0,0 +1,29 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`datasource/docker/common getRegistryRepository handles local registries 1`] = `
+Object {
+ "registry": "https://registry:5000",
+ "repository": "org/package",
+}
+`;
+
+exports[`datasource/docker/common getRegistryRepository supports http registryUrls 1`] = `
+Object {
+ "registry": "http://my.local.registry/prefix",
+ "repository": "image",
+}
+`;
+
+exports[`datasource/docker/common getRegistryRepository supports registryUrls 1`] = `
+Object {
+ "registry": "https://my.local.registry/prefix",
+ "repository": "image",
+}
+`;
+
+exports[`datasource/docker/common getRegistryRepository supports schemeless registryUrls 1`] = `
+Object {
+ "registry": "https://my.local.registry/prefix",
+ "repository": "image",
+}
+`;
diff --git a/lib/datasource/docker/__snapshots__/index.spec.ts.snap b/lib/datasource/docker/__snapshots__/index.spec.ts.snap
index 1cbbf983b7e2fd9904110ab2037382a366b9e0c8..d0768acadcd25d47b7a05f305077865e2159b37f 100644
--- a/lib/datasource/docker/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/docker/__snapshots__/index.spec.ts.snap
@@ -419,34 +419,6 @@ Array [
]
`;
-exports[`datasource/docker/index getRegistryRepository handles local registries 1`] = `
-Object {
- "registry": "https://registry:5000",
- "repository": "org/package",
-}
-`;
-
-exports[`datasource/docker/index getRegistryRepository supports http registryUrls 1`] = `
-Object {
- "registry": "http://my.local.registry/prefix",
- "repository": "image",
-}
-`;
-
-exports[`datasource/docker/index getRegistryRepository supports registryUrls 1`] = `
-Object {
- "registry": "https://my.local.registry/prefix",
- "repository": "image",
-}
-`;
-
-exports[`datasource/docker/index getRegistryRepository supports schemeless registryUrls 1`] = `
-Object {
- "registry": "https://my.local.registry/prefix",
- "repository": "image",
-}
-`;
-
exports[`datasource/docker/index getReleases adds library/ prefix for Docker Hub (explicit) 1`] = `
Array [
Object {
diff --git a/lib/datasource/docker/common.spec.ts b/lib/datasource/docker/common.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..8ddca33389a5b99271cc2ed529eda41efb519104
--- /dev/null
+++ b/lib/datasource/docker/common.spec.ts
@@ -0,0 +1,56 @@
+import * as httpMock from '../../../test/http-mock';
+import { getName, mocked } from '../../../test/util';
+import * as _hostRules from '../../util/host-rules';
+import * as dockerCommon from './common';
+
+const hostRules = mocked(_hostRules);
+
+jest.mock('@aws-sdk/client-ecr');
+jest.mock('../../util/host-rules');
+
+describe(getName(), () => {
+ beforeEach(() => {
+ httpMock.setup();
+ hostRules.find.mockReturnValue({
+ username: 'some-username',
+ password: 'some-password',
+ });
+ hostRules.hosts.mockReturnValue([]);
+ });
+
+ afterEach(() => {
+ jest.resetAllMocks();
+ httpMock.reset();
+ });
+
+ describe('getRegistryRepository', () => {
+ it('handles local registries', () => {
+ const res = dockerCommon.getRegistryRepository(
+ 'registry:5000/org/package',
+ 'https://index.docker.io'
+ );
+ expect(res).toMatchSnapshot();
+ });
+ it('supports registryUrls', () => {
+ const res = dockerCommon.getRegistryRepository(
+ 'my.local.registry/prefix/image',
+ 'https://my.local.registry/prefix'
+ );
+ expect(res).toMatchSnapshot();
+ });
+ it('supports http registryUrls', () => {
+ const res = dockerCommon.getRegistryRepository(
+ 'my.local.registry/prefix/image',
+ 'http://my.local.registry/prefix'
+ );
+ expect(res).toMatchSnapshot();
+ });
+ it('supports schemeless registryUrls', () => {
+ const res = dockerCommon.getRegistryRepository(
+ 'my.local.registry/prefix/image',
+ 'my.local.registry/prefix'
+ );
+ expect(res).toMatchSnapshot();
+ });
+ });
+});
diff --git a/lib/datasource/docker/common.ts b/lib/datasource/docker/common.ts
new file mode 100644
index 0000000000000000000000000000000000000000..f10ed2e8038da05ffd3545d0b2a74235a02fffeb
--- /dev/null
+++ b/lib/datasource/docker/common.ts
@@ -0,0 +1,441 @@
+import { ECR, ECRClientConfig } from '@aws-sdk/client-ecr';
+import hasha from 'hasha';
+import wwwAuthenticate from 'www-authenticate';
+import { HOST_DISABLED } from '../../constants/error-messages';
+import { logger } from '../../logger';
+import { HostRule } from '../../types';
+import { ExternalHostError } from '../../types/errors/external-host-error';
+import * as packageCache from '../../util/cache/package';
+import * as hostRules from '../../util/host-rules';
+import { Http, HttpResponse } from '../../util/http';
+import type { OutgoingHttpHeaders } from '../../util/http/types';
+import { ensureTrailingSlash, trimTrailingSlash } from '../../util/url';
+import { MediaType, RegistryRepository } from './types';
+
+export const id = 'docker';
+export const http = new Http(id);
+
+export const ecrRegex = /\d+\.dkr\.ecr\.([-a-z0-9]+)\.amazonaws\.com/;
+export const defaultRegistryUrls = ['https://index.docker.io'];
+
+async function getECRAuthToken(
+ region: string,
+ opts: HostRule
+): Promise<string | null> {
+ const config: ECRClientConfig = { region };
+ if (opts.username && opts.password) {
+ config.credentials = {
+ accessKeyId: opts.username,
+ secretAccessKey: opts.password,
+ };
+ }
+ const ecr = new ECR(config);
+ try {
+ const data = await ecr.getAuthorizationToken({});
+ const authorizationToken = data?.authorizationData?.[0]?.authorizationToken;
+ if (authorizationToken) {
+ return authorizationToken;
+ }
+ logger.warn(
+ 'Could not extract authorizationToken from ECR getAuthorizationToken response'
+ );
+ } catch (err) {
+ logger.trace({ err }, 'err');
+ logger.debug('ECR getAuthorizationToken error');
+ }
+ return null;
+}
+
+export async function getAuthHeaders(
+ registry: string,
+ dockerRepository: string
+): Promise<OutgoingHttpHeaders | null> {
+ try {
+ const apiCheckUrl = `${registry}/v2/`;
+ const apiCheckResponse = await http.get(apiCheckUrl, {
+ throwHttpErrors: false,
+ });
+ if (apiCheckResponse.headers['www-authenticate'] === undefined) {
+ return {};
+ }
+ const authenticateHeader = new wwwAuthenticate.parsers.WWW_Authenticate(
+ apiCheckResponse.headers['www-authenticate']
+ );
+
+ const opts: HostRule & {
+ headers?: Record<string, string>;
+ } = hostRules.find({ hostType: id, url: apiCheckUrl });
+ if (ecrRegex.test(registry)) {
+ const [, region] = ecrRegex.exec(registry);
+ const auth = await getECRAuthToken(region, opts);
+ if (auth) {
+ opts.headers = { authorization: `Basic ${auth}` };
+ }
+ } else if (opts.username && opts.password) {
+ const auth = Buffer.from(`${opts.username}:${opts.password}`).toString(
+ 'base64'
+ );
+ opts.headers = { authorization: `Basic ${auth}` };
+ }
+ delete opts.username;
+ delete opts.password;
+
+ if (authenticateHeader.scheme.toUpperCase() === 'BASIC') {
+ logger.debug(`Using Basic auth for docker registry ${dockerRepository}`);
+ await http.get(apiCheckUrl, opts);
+ return opts.headers;
+ }
+
+ // prettier-ignore
+ const authUrl = `${String(authenticateHeader.parms.realm)}?service=${String(authenticateHeader.parms.service)}&scope=repository:${dockerRepository}:pull`;
+ logger.trace(
+ `Obtaining docker registry token for ${dockerRepository} using url ${authUrl}`
+ );
+ const authResponse = (
+ await http.getJson<{ token?: string; access_token?: string }>(
+ authUrl,
+ opts
+ )
+ ).body;
+
+ const token = authResponse.token || authResponse.access_token;
+ // istanbul ignore if
+ if (!token) {
+ logger.warn('Failed to obtain docker registry token');
+ return null;
+ }
+ return {
+ authorization: `Bearer ${token}`,
+ };
+ } catch (err) /* istanbul ignore next */ {
+ if (err.host === 'quay.io') {
+ // TODO: debug why quay throws errors (#9604)
+ return null;
+ }
+ if (err.statusCode === 401) {
+ logger.debug(
+ { registry, dockerRepository },
+ 'Unauthorized docker lookup'
+ );
+ logger.debug({ err });
+ return null;
+ }
+ if (err.statusCode === 403) {
+ logger.debug(
+ { registry, dockerRepository },
+ 'Not allowed to access docker registry'
+ );
+ logger.debug({ err });
+ return null;
+ }
+ // prettier-ignore
+ if (err.name === 'RequestError' && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization]
+ throw new ExternalHostError(err);
+ }
+ // prettier-ignore
+ if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization]
+ throw new ExternalHostError(err);
+ }
+ if (err.statusCode >= 500 && err.statusCode < 600) {
+ throw new ExternalHostError(err);
+ }
+ if (err.message === HOST_DISABLED) {
+ logger.trace({ registry, dockerRepository, err }, 'Host disabled');
+ return null;
+ }
+ logger.warn(
+ { registry, dockerRepository, err },
+ 'Error obtaining docker token'
+ );
+ return null;
+ }
+}
+
+export function getRegistryRepository(
+ lookupName: string,
+ registryUrl: string
+): RegistryRepository {
+ if (registryUrl !== defaultRegistryUrls[0]) {
+ const registryEndingWithSlash = ensureTrailingSlash(
+ registryUrl.replace(/^https?:\/\//, '')
+ );
+ if (lookupName.startsWith(registryEndingWithSlash)) {
+ let registry = trimTrailingSlash(registryUrl);
+ if (!/^https?:\/\//.test(registry)) {
+ registry = `https://${registry}`;
+ }
+ return {
+ registry,
+ repository: lookupName.replace(registryEndingWithSlash, ''),
+ };
+ }
+ }
+ let registry: string;
+ const split = lookupName.split('/');
+ if (split.length > 1 && (split[0].includes('.') || split[0].includes(':'))) {
+ [registry] = split;
+ split.shift();
+ }
+ let repository = split.join('/');
+ if (!registry) {
+ registry = registryUrl;
+ }
+ if (registry === 'docker.io') {
+ registry = 'index.docker.io';
+ }
+ if (!/^https?:\/\//.exec(registry)) {
+ registry = `https://${registry}`;
+ }
+ const opts = hostRules.find({ hostType: id, url: registry });
+ if (opts?.insecureRegistry) {
+ registry = registry.replace('https', 'http');
+ }
+ if (registry.endsWith('.docker.io') && !repository.includes('/')) {
+ repository = 'library/' + repository;
+ }
+ return {
+ registry,
+ repository,
+ };
+}
+
+function digestFromManifestStr(str: hasha.HashaInput): string {
+ return 'sha256:' + hasha(str, { algorithm: 'sha256' });
+}
+
+export function extractDigestFromResponse(
+ manifestResponse: HttpResponse
+): string {
+ if (manifestResponse.headers['docker-content-digest'] === undefined) {
+ return digestFromManifestStr(manifestResponse.body);
+ }
+ return manifestResponse.headers['docker-content-digest'] as string;
+}
+
+// TODO: debug why quay throws errors (#9612)
+export async function getManifestResponse(
+ registry: string,
+ dockerRepository: string,
+ tag: string
+): Promise<HttpResponse> {
+ logger.debug(`getManifestResponse(${registry}, ${dockerRepository}, ${tag})`);
+ try {
+ const headers = await getAuthHeaders(registry, dockerRepository);
+ if (!headers) {
+ logger.debug('No docker auth found - returning');
+ return null;
+ }
+ headers.accept =
+ 'application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json';
+ const url = `${registry}/v2/${dockerRepository}/manifests/${tag}`;
+ const manifestResponse = await http.get(url, {
+ headers,
+ });
+ return manifestResponse;
+ } catch (err) /* istanbul ignore next */ {
+ if (err instanceof ExternalHostError) {
+ throw err;
+ }
+ if (err.statusCode === 401) {
+ logger.debug(
+ { registry, dockerRepository },
+ 'Unauthorized docker lookup'
+ );
+ logger.debug({ err });
+ return null;
+ }
+ if (err.statusCode === 404) {
+ logger.debug(
+ {
+ err,
+ registry,
+ dockerRepository,
+ tag,
+ },
+ 'Docker Manifest is unknown'
+ );
+ return null;
+ }
+ // prettier-ignore
+ if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization]
+ throw new ExternalHostError(err);
+ }
+ if (err.statusCode >= 500 && err.statusCode < 600) {
+ throw new ExternalHostError(err);
+ }
+ if (err.code === 'ETIMEDOUT') {
+ logger.debug(
+ { registry },
+ 'Timeout when attempting to connect to docker registry'
+ );
+ logger.debug({ err });
+ return null;
+ }
+ logger.debug(
+ {
+ err,
+ registry,
+ dockerRepository,
+ tag,
+ },
+ 'Unknown Error looking up docker manifest'
+ );
+ return null;
+ }
+}
+
+async function getConfigDigest(
+ registry: string,
+ dockerRepository: string,
+ tag: string
+): Promise<string> {
+ const manifestResponse = await getManifestResponse(
+ registry,
+ dockerRepository,
+ tag
+ );
+ // If getting the manifest fails here, then abort
+ // This means that the latest tag doesn't have a manifest, which shouldn't
+ // be possible
+ // istanbul ignore if
+ if (!manifestResponse) {
+ return null;
+ }
+ const manifest = JSON.parse(manifestResponse.body);
+ if (manifest.schemaVersion !== 2) {
+ logger.debug(
+ { registry, dockerRepository, tag },
+ 'Manifest schema version is not 2'
+ );
+ return null;
+ }
+
+ if (
+ manifest.mediaType === MediaType.manifestListV2 &&
+ manifest.manifests.length
+ ) {
+ logger.trace(
+ { registry, dockerRepository, tag },
+ 'Found manifest list, using first image'
+ );
+ return getConfigDigest(
+ registry,
+ dockerRepository,
+ manifest.manifests[0].digest
+ );
+ }
+
+ if (manifest.mediaType === MediaType.manifestV2) {
+ return manifest.config?.digest || null;
+ }
+
+ logger.debug({ manifest }, 'Invalid manifest - returning');
+ return null;
+}
+
+/*
+ * docker.getLabels
+ *
+ * This function will:
+ * - Return the labels for the requested image
+ */
+
+export async function getLabels(
+ registry: string,
+ dockerRepository: string,
+ tag: string
+): Promise<Record<string, string>> {
+ logger.debug(`getLabels(${registry}, ${dockerRepository}, ${tag})`);
+ const cacheNamespace = 'datasource-docker-labels';
+ const cacheKey = `${registry}:${dockerRepository}:${tag}`;
+ const cachedResult = await packageCache.get<Record<string, string>>(
+ cacheNamespace,
+ cacheKey
+ );
+ // istanbul ignore if
+ if (cachedResult !== undefined) {
+ return cachedResult;
+ }
+ try {
+ let labels: Record<string, string> = {};
+ const configDigest = await getConfigDigest(registry, dockerRepository, tag);
+ if (!configDigest) {
+ return {};
+ }
+
+ const headers = await getAuthHeaders(registry, dockerRepository);
+ // istanbul ignore if: Should never be happen
+ if (!headers) {
+ logger.debug('No docker auth found - returning');
+ return {};
+ }
+ const url = `${registry}/v2/${dockerRepository}/blobs/${configDigest}`;
+ const configResponse = await http.get(url, {
+ headers,
+ });
+ labels = JSON.parse(configResponse.body).config.Labels;
+
+ if (labels) {
+ logger.debug(
+ {
+ labels,
+ },
+ 'found labels in manifest'
+ );
+ }
+ const cacheMinutes = 60;
+ await packageCache.set(cacheNamespace, cacheKey, labels, cacheMinutes);
+ return labels;
+ } catch (err) /* istanbul ignore next: should be tested in future */ {
+ if (err instanceof ExternalHostError) {
+ throw err;
+ }
+ if (err.statusCode === 400 || err.statusCode === 401) {
+ logger.debug(
+ { registry, dockerRepository, err },
+ 'Unauthorized docker lookup'
+ );
+ } else if (err.statusCode === 404) {
+ logger.warn(
+ {
+ err,
+ registry,
+ dockerRepository,
+ tag,
+ },
+ 'Config Manifest is unknown'
+ );
+ } else if (
+ err.statusCode === 429 &&
+ registry.endsWith('docker.io') // lgtm [js/incomplete-url-substring-sanitization]
+ ) {
+ logger.warn({ err }, 'docker registry failure: too many requests');
+ } else if (err.statusCode >= 500 && err.statusCode < 600) {
+ logger.debug(
+ {
+ err,
+ registry,
+ dockerRepository,
+ tag,
+ },
+ 'docker registry failure: internal error'
+ );
+ } else if (
+ err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' ||
+ err.code === 'ETIMEDOUT'
+ ) {
+ logger.debug({ registry, err }, 'Error connecting to docker registry');
+ } else if (registry === 'https://quay.io') {
+ // istanbul ignore next
+ logger.debug(
+ 'Ignoring quay.io errors until they fully support v2 schema'
+ );
+ } else {
+ logger.info(
+ { registry, dockerRepository, tag, err },
+ 'Unknown error getting Docker labels'
+ );
+ }
+ return {};
+ }
+}
diff --git a/lib/datasource/docker/index.spec.ts b/lib/datasource/docker/index.spec.ts
index 02968b500a8b323e44be5d187f264b95d5fb40c4..39918e8257c4d899d63ad220fedbfc15ea1edf5d 100644
--- a/lib/datasource/docker/index.spec.ts
+++ b/lib/datasource/docker/index.spec.ts
@@ -4,8 +4,8 @@ import * as httpMock from '../../../test/http-mock';
import { getName, mocked, partial } from '../../../test/util';
import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
import * as _hostRules from '../../util/host-rules';
+import { id } from './common';
import { MediaType } from './types';
-import * as docker from '.';
const hostRules = mocked(_hostRules);
@@ -57,36 +57,6 @@ describe(getName(), () => {
httpMock.reset();
});
- describe('getRegistryRepository', () => {
- it('handles local registries', () => {
- const res = docker.getRegistryRepository(
- 'registry:5000/org/package',
- 'https://index.docker.io'
- );
- expect(res).toMatchSnapshot();
- });
- it('supports registryUrls', () => {
- const res = docker.getRegistryRepository(
- 'my.local.registry/prefix/image',
- 'https://my.local.registry/prefix'
- );
- expect(res).toMatchSnapshot();
- });
- it('supports http registryUrls', () => {
- const res = docker.getRegistryRepository(
- 'my.local.registry/prefix/image',
- 'http://my.local.registry/prefix'
- );
- expect(res).toMatchSnapshot();
- });
- it('supports schemeless registryUrls', () => {
- const res = docker.getRegistryRepository(
- 'my.local.registry/prefix/image',
- 'my.local.registry/prefix'
- );
- expect(res).toMatchSnapshot();
- });
- });
describe('getDigest', () => {
it('returns null if no token', async () => {
httpMock
@@ -421,7 +391,7 @@ describe(getName(), () => {
.get('/library/node/tags/list?n=10000')
.reply(403);
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'node',
});
expect(res).toBeNull();
@@ -450,7 +420,7 @@ describe(getName(), () => {
.get('/user/9287/repos?page=3&per_page=100')
.reply(200, { tags: ['latest'] }, {});
const config = {
- datasource: docker.id,
+ datasource: id,
depName: 'node',
registryUrls: ['https://registry.company.com'],
};
@@ -471,7 +441,7 @@ describe(getName(), () => {
.get('/node/manifests/1.0.0')
.reply(200, '', {});
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'registry.company.com/node',
});
expect(res.releases).toHaveLength(1);
@@ -491,7 +461,7 @@ describe(getName(), () => {
.get('/node/manifests/undefined')
.reply(200);
await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: '123456789.dkr.ecr.us-east-1.amazonaws.com/node',
});
expect(httpMock.getTrace()).toMatchSnapshot();
@@ -518,7 +488,7 @@ describe(getName(), () => {
)
.reply(200, { token: 'some-token ' });
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'node',
});
expect(res.releases).toHaveLength(1);
@@ -546,7 +516,7 @@ describe(getName(), () => {
)
.reply(200, { token: 'some-token ' });
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'docker.io/node',
});
expect(res.releases).toHaveLength(1);
@@ -572,7 +542,7 @@ describe(getName(), () => {
.get('/kubernetes-dashboard-amd64/manifests/1.0.0')
.reply(200);
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'k8s.gcr.io/kubernetes-dashboard-amd64',
});
expect(res.releases).toHaveLength(1);
@@ -586,7 +556,7 @@ describe(getName(), () => {
.get('/my/node/tags/list?n=10000')
.replyWithError('error');
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'my/node',
});
expect(res).toBeNull();
@@ -604,7 +574,7 @@ describe(getName(), () => {
.get('/')
.reply(403);
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'node',
});
expect(res).toBeNull();
@@ -635,7 +605,7 @@ describe(getName(), () => {
},
});
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'registry.company.com/node',
});
const trace = httpMock.getTrace();
@@ -673,7 +643,7 @@ describe(getName(), () => {
},
});
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'registry.company.com/node',
});
const trace = httpMock.getTrace();
@@ -695,7 +665,7 @@ describe(getName(), () => {
mediaType: MediaType.manifestV1,
});
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'registry.company.com/node',
});
const trace = httpMock.getTrace();
@@ -714,7 +684,7 @@ describe(getName(), () => {
.get('/node/manifests/latest')
.reply(200, {});
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'registry.company.com/node',
});
const trace = httpMock.getTrace();
@@ -749,7 +719,7 @@ describe(getName(), () => {
config: {},
});
const res = await getPkgReleases({
- datasource: docker.id,
+ datasource: id,
depName: 'registry.company.com/node',
});
const trace = httpMock.getTrace();
diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts
index 20869bad554a34c781d3335c6ff0acb4afb2f85d..cd34dbc231ed3784f790e429e9ec8c0eb08301df 100644
--- a/lib/datasource/docker/index.ts
+++ b/lib/datasource/docker/index.ts
@@ -1,27 +1,28 @@
import URL from 'url';
-import { ECR, ECRClientConfig } from '@aws-sdk/client-ecr';
-import hasha from 'hasha';
import parseLinkHeader from 'parse-link-header';
-import wwwAuthenticate from 'www-authenticate';
-import { HOST_DISABLED } from '../../constants/error-messages';
import { logger } from '../../logger';
-import { HostRule } from '../../types';
import { ExternalHostError } from '../../types/errors/external-host-error';
import * as packageCache from '../../util/cache/package';
-import * as hostRules from '../../util/host-rules';
-import { Http, HttpResponse } from '../../util/http';
-import type { OutgoingHttpHeaders } from '../../util/http/types';
-import { ensureTrailingSlash, trimTrailingSlash } from '../../util/url';
import * as dockerVersioning from '../../versioning/docker';
import type { GetReleasesConfig, ReleaseResult } from '../types';
-import { Image, ImageList, MediaType, RegistryRepository } from './types';
+import {
+ defaultRegistryUrls,
+ ecrRegex,
+ extractDigestFromResponse,
+ getAuthHeaders,
+ getLabels,
+ getManifestResponse,
+ getRegistryRepository,
+ http,
+ id,
+} from './common';
// TODO: add got typings when available (#9646)
// TODO: replace www-authenticate with https://www.npmjs.com/package/auth-header (#9645)
-export const id = 'docker';
+export { id };
export const customRegistrySupport = true;
-export const defaultRegistryUrls = ['https://index.docker.io'];
+export { defaultRegistryUrls };
export const defaultVersioning = dockerVersioning.id;
export const registryStrategy = 'first';
@@ -52,381 +53,6 @@ export const defaultConfig = {
},
};
-const http = new Http(id);
-
-const ecrRegex = /\d+\.dkr\.ecr\.([-a-z0-9]+)\.amazonaws\.com/;
-
-export function getRegistryRepository(
- lookupName: string,
- registryUrl: string
-): RegistryRepository {
- if (registryUrl !== defaultRegistryUrls[0]) {
- const registryEndingWithSlash = ensureTrailingSlash(
- registryUrl.replace(/^https?:\/\//, '')
- );
- if (lookupName.startsWith(registryEndingWithSlash)) {
- let registry = trimTrailingSlash(registryUrl);
- if (!/^https?:\/\//.test(registry)) {
- registry = `https://${registry}`;
- }
- return {
- registry,
- repository: lookupName.replace(registryEndingWithSlash, ''),
- };
- }
- }
- let registry: string;
- const split = lookupName.split('/');
- if (split.length > 1 && (split[0].includes('.') || split[0].includes(':'))) {
- [registry] = split;
- split.shift();
- }
- let repository = split.join('/');
- if (!registry) {
- registry = registryUrl;
- }
- if (registry === 'docker.io') {
- registry = 'index.docker.io';
- }
- if (!/^https?:\/\//.exec(registry)) {
- registry = `https://${registry}`;
- }
- const opts = hostRules.find({ hostType: id, url: registry });
- if (opts?.insecureRegistry) {
- registry = registry.replace('https', 'http');
- }
- if (registry.endsWith('.docker.io') && !repository.includes('/')) {
- repository = 'library/' + repository;
- }
- return {
- registry,
- repository,
- };
-}
-
-async function getECRAuthToken(
- region: string,
- opts: HostRule
-): Promise<string | null> {
- const config: ECRClientConfig = { region };
- if (opts.username && opts.password) {
- config.credentials = {
- accessKeyId: opts.username,
- secretAccessKey: opts.password,
- };
- }
- const ecr = new ECR(config);
- try {
- const data = await ecr.getAuthorizationToken({});
- const authorizationToken = data?.authorizationData?.[0]?.authorizationToken;
- if (authorizationToken) {
- return authorizationToken;
- }
- logger.warn(
- 'Could not extract authorizationToken from ECR getAuthorizationToken response'
- );
- } catch (err) {
- logger.trace({ err }, 'err');
- logger.debug('ECR getAuthorizationToken error');
- }
- return null;
-}
-
-async function getAuthHeaders(
- registry: string,
- dockerRepository: string
-): Promise<OutgoingHttpHeaders | null> {
- try {
- const apiCheckUrl = `${registry}/v2/`;
- const apiCheckResponse = await http.get(apiCheckUrl, {
- throwHttpErrors: false,
- });
- if (apiCheckResponse.headers['www-authenticate'] === undefined) {
- return {};
- }
- const authenticateHeader = new wwwAuthenticate.parsers.WWW_Authenticate(
- apiCheckResponse.headers['www-authenticate']
- );
-
- const opts: HostRule & {
- headers?: Record<string, string>;
- } = hostRules.find({ hostType: id, url: apiCheckUrl });
- if (ecrRegex.test(registry)) {
- const [, region] = ecrRegex.exec(registry);
- const auth = await getECRAuthToken(region, opts);
- if (auth) {
- opts.headers = { authorization: `Basic ${auth}` };
- }
- } else if (opts.username && opts.password) {
- const auth = Buffer.from(`${opts.username}:${opts.password}`).toString(
- 'base64'
- );
- opts.headers = { authorization: `Basic ${auth}` };
- }
- delete opts.username;
- delete opts.password;
-
- if (authenticateHeader.scheme.toUpperCase() === 'BASIC') {
- logger.debug(`Using Basic auth for docker registry ${dockerRepository}`);
- await http.get(apiCheckUrl, opts);
- return opts.headers;
- }
-
- // prettier-ignore
- const authUrl = `${String(authenticateHeader.parms.realm)}?service=${String(authenticateHeader.parms.service)}&scope=repository:${dockerRepository}:pull`;
- logger.trace(
- `Obtaining docker registry token for ${dockerRepository} using url ${authUrl}`
- );
- const authResponse = (
- await http.getJson<{ token?: string; access_token?: string }>(
- authUrl,
- opts
- )
- ).body;
-
- const token = authResponse.token || authResponse.access_token;
- // istanbul ignore if
- if (!token) {
- logger.warn('Failed to obtain docker registry token');
- return null;
- }
- return {
- authorization: `Bearer ${token}`,
- };
- } catch (err) /* istanbul ignore next */ {
- if (err.host === 'quay.io') {
- // TODO: debug why quay throws errors (#9604)
- return null;
- }
- if (err.statusCode === 401) {
- logger.debug(
- { registry, dockerRepository },
- 'Unauthorized docker lookup'
- );
- logger.debug({ err });
- return null;
- }
- if (err.statusCode === 403) {
- logger.debug(
- { registry, dockerRepository },
- 'Not allowed to access docker registry'
- );
- logger.debug({ err });
- return null;
- }
- // prettier-ignore
- if (err.name === 'RequestError' && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization]
- throw new ExternalHostError(err);
- }
- // prettier-ignore
- if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization]
- throw new ExternalHostError(err);
- }
- if (err.statusCode >= 500 && err.statusCode < 600) {
- throw new ExternalHostError(err);
- }
- if (err.message === HOST_DISABLED) {
- logger.trace({ registry, dockerRepository, err }, 'Host disabled');
- return null;
- }
- logger.warn(
- { registry, dockerRepository, err },
- 'Error obtaining docker token'
- );
- return null;
- }
-}
-
-function digestFromManifestStr(str: hasha.HashaInput): string {
- return 'sha256:' + hasha(str, { algorithm: 'sha256' });
-}
-
-function extractDigestFromResponse(manifestResponse: HttpResponse): string {
- if (manifestResponse.headers['docker-content-digest'] === undefined) {
- return digestFromManifestStr(manifestResponse.body);
- }
- return manifestResponse.headers['docker-content-digest'] as string;
-}
-
-// TODO: debug why quay throws errors (#9612)
-async function getManifestResponse(
- registry: string,
- dockerRepository: string,
- tag: string
-): Promise<HttpResponse> {
- logger.debug(`getManifestResponse(${registry}, ${dockerRepository}, ${tag})`);
- try {
- const headers = await getAuthHeaders(registry, dockerRepository);
- if (!headers) {
- logger.debug('No docker auth found - returning');
- return null;
- }
- headers.accept =
- 'application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json';
- const url = `${registry}/v2/${dockerRepository}/manifests/${tag}`;
- const manifestResponse = await http.get(url, {
- headers,
- });
- return manifestResponse;
- } catch (err) /* istanbul ignore next */ {
- if (err instanceof ExternalHostError) {
- throw err;
- }
- if (err.statusCode === 401) {
- logger.debug(
- { registry, dockerRepository },
- 'Unauthorized docker lookup'
- );
- logger.debug({ err });
- return null;
- }
- if (err.statusCode === 404) {
- logger.debug(
- {
- err,
- registry,
- dockerRepository,
- tag,
- },
- 'Docker Manifest is unknown'
- );
- return null;
- }
- // prettier-ignore
- if (err.statusCode === 429 && registry.endsWith('docker.io')) { // lgtm [js/incomplete-url-substring-sanitization]
- throw new ExternalHostError(err);
- }
- if (err.statusCode >= 500 && err.statusCode < 600) {
- throw new ExternalHostError(err);
- }
- if (err.code === 'ETIMEDOUT') {
- logger.debug(
- { registry },
- 'Timeout when attempting to connect to docker registry'
- );
- logger.debug({ err });
- return null;
- }
- logger.debug(
- {
- err,
- registry,
- dockerRepository,
- tag,
- },
- 'Unknown Error looking up docker manifest'
- );
- return null;
- }
-}
-
-async function getConfigDigest(
- registry: string,
- dockerRepository: string,
- tag: string
-): Promise<string> {
- const manifestResponse = await getManifestResponse(
- registry,
- dockerRepository,
- tag
- );
- // If getting the manifest fails here, then abort
- // This means that the latest tag doesn't have a manifest, which shouldn't
- // be possible
- // istanbul ignore if
- if (!manifestResponse) {
- return null;
- }
- const manifest = JSON.parse(manifestResponse.body) as ImageList | Image;
- if (manifest.schemaVersion !== 2) {
- logger.debug(
- { registry, dockerRepository, tag },
- 'Manifest schema version is not 2'
- );
- return null;
- }
-
- if (
- manifest.mediaType === MediaType.manifestListV2 &&
- manifest.manifests.length
- ) {
- logger.trace(
- { registry, dockerRepository, tag },
- 'Found manifest list, using first image'
- );
- return getConfigDigest(
- registry,
- dockerRepository,
- manifest.manifests[0].digest
- );
- }
-
- if (manifest.mediaType === MediaType.manifestV2) {
- return manifest.config?.digest || null;
- }
-
- logger.debug({ manifest }, 'Invalid manifest - returning');
- return null;
-}
-
-/**
- * docker.getDigest
- *
- * The `newValue` supplied here should be a valid tag for the docker image.
- *
- * This function will:
- * - Look up a sha256 digest for a tag on its registry
- * - Return the digest as a string
- */
-export async function getDigest(
- { registryUrl, lookupName }: GetReleasesConfig,
- newValue?: string
-): Promise<string | null> {
- const { registry, repository } = getRegistryRepository(
- lookupName,
- registryUrl
- );
- logger.debug(`getDigest(${registry}, ${repository}, ${newValue})`);
- const newTag = newValue || 'latest';
- const cacheNamespace = 'datasource-docker-digest';
- const cacheKey = `${registry}:${repository}:${newTag}`;
- let digest: string = null;
- try {
- const cachedResult = await packageCache.get<string>(
- cacheNamespace,
- cacheKey
- );
- // istanbul ignore if
- if (cachedResult !== undefined) {
- return cachedResult;
- }
- const manifestResponse = await getManifestResponse(
- registry,
- repository,
- newTag
- );
- if (manifestResponse) {
- digest = extractDigestFromResponse(manifestResponse) || null;
- logger.debug({ digest }, 'Got docker digest');
- }
- } catch (err) /* istanbul ignore next */ {
- if (err instanceof ExternalHostError) {
- throw err;
- }
- logger.debug(
- {
- err,
- lookupName,
- newTag,
- },
- 'Unknown Error looking up docker image digest'
- );
- }
- const cacheMinutes = 30;
- await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes);
- return digest;
-}
-
async function getTags(
registry: string,
repository: string
@@ -500,111 +126,62 @@ async function getTags(
}
}
-/*
- * docker.getLabels
+/**
+ * docker.getDigest
+ *
+ * The `newValue` supplied here should be a valid tag for the docker image.
*
* This function will:
- * - Return the labels for the requested image
+ * - Look up a sha256 digest for a tag on its registry
+ * - Return the digest as a string
*/
-
-async function getLabels(
- registry: string,
- dockerRepository: string,
- tag: string
-): Promise<Record<string, string>> {
- logger.debug(`getLabels(${registry}, ${dockerRepository}, ${tag})`);
- const cacheNamespace = 'datasource-docker-labels';
- const cacheKey = `${registry}:${dockerRepository}:${tag}`;
- const cachedResult = await packageCache.get<Record<string, string>>(
- cacheNamespace,
- cacheKey
+export async function getDigest(
+ { registryUrl, lookupName }: GetReleasesConfig,
+ newValue?: string
+): Promise<string | null> {
+ const { registry, repository } = getRegistryRepository(
+ lookupName,
+ registryUrl
);
- // istanbul ignore if
- if (cachedResult !== undefined) {
- return cachedResult;
- }
+ logger.debug(`getDigest(${registry}, ${repository}, ${newValue})`);
+ const newTag = newValue || 'latest';
+ const cacheNamespace = 'datasource-docker-digest';
+ const cacheKey = `${registry}:${repository}:${newTag}`;
+ let digest: string = null;
try {
- let labels: Record<string, string> = {};
- const configDigest = await getConfigDigest(registry, dockerRepository, tag);
- if (!configDigest) {
- return {};
- }
-
- const headers = await getAuthHeaders(registry, dockerRepository);
- // istanbul ignore if: Should never be happen
- if (!headers) {
- logger.debug('No docker auth found - returning');
- return {};
+ const cachedResult = await packageCache.get<string>(
+ cacheNamespace,
+ cacheKey
+ );
+ // istanbul ignore if
+ if (cachedResult !== undefined) {
+ return cachedResult;
}
- const url = `${registry}/v2/${dockerRepository}/blobs/${configDigest}`;
- const configResponse = await http.get(url, {
- headers,
- });
- labels = JSON.parse(configResponse.body).config.Labels;
-
- if (labels) {
- logger.debug(
- {
- labels,
- },
- 'found labels in manifest'
- );
+ const manifestResponse = await getManifestResponse(
+ registry,
+ repository,
+ newTag
+ );
+ if (manifestResponse) {
+ digest = extractDigestFromResponse(manifestResponse) || null;
+ logger.debug({ digest }, 'Got docker digest');
}
- const cacheMinutes = 60;
- await packageCache.set(cacheNamespace, cacheKey, labels, cacheMinutes);
- return labels;
- } catch (err) /* istanbul ignore next: should be tested in future */ {
+ } catch (err) /* istanbul ignore next */ {
if (err instanceof ExternalHostError) {
throw err;
}
- if (err.statusCode === 400 || err.statusCode === 401) {
- logger.debug(
- { registry, dockerRepository, err },
- 'Unauthorized docker lookup'
- );
- } else if (err.statusCode === 404) {
- logger.warn(
- {
- err,
- registry,
- dockerRepository,
- tag,
- },
- 'Config Manifest is unknown'
- );
- } else if (
- err.statusCode === 429 &&
- registry.endsWith('docker.io') // lgtm [js/incomplete-url-substring-sanitization]
- ) {
- logger.warn({ err }, 'docker registry failure: too many requests');
- } else if (err.statusCode >= 500 && err.statusCode < 600) {
- logger.debug(
- {
- err,
- registry,
- dockerRepository,
- tag,
- },
- 'docker registry failure: internal error'
- );
- } else if (
- err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' ||
- err.code === 'ETIMEDOUT'
- ) {
- logger.debug({ registry, err }, 'Error connecting to docker registry');
- } else if (registry === 'https://quay.io') {
- // istanbul ignore next
- logger.debug(
- 'Ignoring quay.io errors until they fully support v2 schema'
- );
- } else {
- logger.info(
- { registry, dockerRepository, tag, err },
- 'Unknown error getting Docker labels'
- );
- }
- return {};
+ logger.debug(
+ {
+ err,
+ lookupName,
+ newTag,
+ },
+ 'Unknown Error looking up docker image digest'
+ );
}
+ const cacheMinutes = 30;
+ await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes);
+ return digest;
}
/**