diff --git a/lib/modules/datasource/docker/dockerhub-cache.spec.ts b/lib/modules/datasource/docker/dockerhub-cache.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..fe8ad65504229e01b317e134a1395b45ce348cf7
--- /dev/null
+++ b/lib/modules/datasource/docker/dockerhub-cache.spec.ts
@@ -0,0 +1,176 @@
+import { mocked } from '../../../../test/util';
+import * as _packageCache from '../../../util/cache/package';
+import { DockerHubCache, DockerHubCacheData } from './dockerhub-cache';
+import type { DockerHubTag } from './schema';
+
+jest.mock('../../../util/cache/package');
+const packageCache = mocked(_packageCache);
+
+function oldCacheData(): DockerHubCacheData {
+ return {
+ items: {
+ 1: {
+ id: 1,
+ last_updated: '2022-01-01',
+ name: '1',
+ tag_last_pushed: '2022-01-01',
+ digest: 'sha256:111',
+ },
+ 2: {
+ id: 2,
+ last_updated: '2022-01-02',
+ name: '2',
+ tag_last_pushed: '2022-01-02',
+ digest: 'sha256:222',
+ },
+ 3: {
+ id: 3,
+ last_updated: '2022-01-03',
+ name: '3',
+ tag_last_pushed: '2022-01-03',
+ digest: 'sha256:333',
+ },
+ },
+ updatedAt: '2022-01-01',
+ };
+}
+
+function newItem(): DockerHubTag {
+ return {
+ id: 4,
+ last_updated: '2022-01-04',
+ name: '4',
+ tag_last_pushed: '2022-01-04',
+ digest: 'sha256:444',
+ };
+}
+
+function newCacheData(): DockerHubCacheData {
+ const { items } = oldCacheData();
+ const item = newItem();
+ return {
+ items: {
+ ...items,
+ [item.id]: item,
+ },
+ updatedAt: '2022-01-04',
+ };
+}
+
+describe('modules/datasource/docker/dockerhub-cache', () => {
+ beforeEach(() => {
+ jest.resetAllMocks();
+ });
+
+ const dockerRepository = 'foo/bar';
+
+ it('initializes empty cache', async () => {
+ packageCache.get.mockResolvedValue(undefined);
+
+ const res = await DockerHubCache.init(dockerRepository);
+
+ expect(res).toEqual({
+ dockerRepository,
+ cache: {
+ items: {},
+ updatedAt: null,
+ },
+ isChanged: false,
+ });
+ });
+
+ it('initializes cache with data', async () => {
+ const oldCache = oldCacheData();
+ packageCache.get.mockResolvedValue(oldCache);
+
+ const res = await DockerHubCache.init(dockerRepository);
+
+ expect(res).toEqual({
+ dockerRepository,
+ cache: oldCache,
+ isChanged: false,
+ });
+ });
+
+ it('reconciles new items', async () => {
+ const oldCache = oldCacheData();
+ const newCache = newCacheData();
+
+ packageCache.get.mockResolvedValue(oldCache);
+ const cache = await DockerHubCache.init(dockerRepository);
+ const newItems: DockerHubTag[] = [newItem()];
+
+ const needNextPage = cache.reconcile(newItems);
+
+ expect(needNextPage).toBe(true);
+ expect(cache).toEqual({
+ cache: newCache,
+ dockerRepository: 'foo/bar',
+ isChanged: true,
+ });
+
+ const res = cache.getItems();
+ expect(res).toEqual(Object.values(newCache.items));
+
+ await cache.save();
+ expect(packageCache.set).toHaveBeenCalledWith(
+ 'datasource-docker-hub-cache',
+ 'foo/bar',
+ newCache,
+ 3 * 60 * 24 * 30,
+ );
+ });
+
+ it('reconciles existing items', async () => {
+ const oldCache = oldCacheData();
+
+ packageCache.get.mockResolvedValue(oldCache);
+ const cache = await DockerHubCache.init(dockerRepository);
+ const items: DockerHubTag[] = Object.values(oldCache.items);
+
+ const needNextPage = cache.reconcile(items);
+
+ expect(needNextPage).toBe(false);
+ expect(cache).toEqual({
+ cache: oldCache,
+ dockerRepository: 'foo/bar',
+ isChanged: false,
+ });
+
+ const res = cache.getItems();
+ expect(res).toEqual(items);
+
+ await cache.save();
+ expect(packageCache.set).not.toHaveBeenCalled();
+ });
+
+ it('reconciles from empty cache', async () => {
+ const item = newItem();
+ const expectedCache = {
+ items: {
+ [item.id]: item,
+ },
+ updatedAt: item.last_updated,
+ };
+ const cache = await DockerHubCache.init(dockerRepository);
+
+ const needNextPage = cache.reconcile([item]);
+ expect(needNextPage).toBe(true);
+ expect(cache).toEqual({
+ cache: expectedCache,
+ dockerRepository: 'foo/bar',
+ isChanged: true,
+ });
+
+ const res = cache.getItems();
+ expect(res).toEqual([item]);
+
+ await cache.save();
+ expect(packageCache.set).toHaveBeenCalledWith(
+ 'datasource-docker-hub-cache',
+ 'foo/bar',
+ expectedCache,
+ 3 * 60 * 24 * 30,
+ );
+ });
+});
diff --git a/lib/modules/datasource/docker/dockerhub-cache.ts b/lib/modules/datasource/docker/dockerhub-cache.ts
new file mode 100644
index 0000000000000000000000000000000000000000..0e97726fc01fb2690fdc9f6d1dea69b028e8a0b9
--- /dev/null
+++ b/lib/modules/datasource/docker/dockerhub-cache.ts
@@ -0,0 +1,78 @@
+import { dequal } from 'dequal';
+import { DateTime } from 'luxon';
+import * as packageCache from '../../../util/cache/package';
+import type { DockerHubTag } from './schema';
+
+export interface DockerHubCacheData {
+ items: Record<number, DockerHubTag>;
+ updatedAt: string | null;
+}
+
+const cacheNamespace = 'datasource-docker-hub-cache';
+
+export class DockerHubCache {
+ private isChanged = false;
+
+ private constructor(
+ private dockerRepository: string,
+ private cache: DockerHubCacheData,
+ ) {}
+
+ static async init(dockerRepository: string): Promise<DockerHubCache> {
+ let repoCache = await packageCache.get<DockerHubCacheData>(
+ cacheNamespace,
+ dockerRepository,
+ );
+
+ repoCache ??= {
+ items: {},
+ updatedAt: null,
+ };
+
+ return new DockerHubCache(dockerRepository, repoCache);
+ }
+
+ reconcile(items: DockerHubTag[]): boolean {
+ let needNextPage = true;
+
+ let { updatedAt } = this.cache;
+ let latestDate = updatedAt ? DateTime.fromISO(updatedAt) : null;
+
+ for (const newItem of items) {
+ const id = newItem.id;
+ const oldItem = this.cache.items[id];
+
+ if (dequal(oldItem, newItem)) {
+ needNextPage = false;
+ continue;
+ }
+
+ this.cache.items[newItem.id] = newItem;
+ const newItemDate = DateTime.fromISO(newItem.last_updated);
+ if (!latestDate || latestDate < newItemDate) {
+ updatedAt = newItem.last_updated;
+ latestDate = newItemDate;
+ }
+
+ this.isChanged = true;
+ }
+
+ this.cache.updatedAt = updatedAt;
+ return needNextPage;
+ }
+
+ async save(): Promise<void> {
+ if (this.isChanged) {
+ await packageCache.set(
+ cacheNamespace,
+ this.dockerRepository,
+ this.cache,
+ 3 * 60 * 24 * 30,
+ );
+ }
+ }
+
+ getItems(): DockerHubTag[] {
+ return Object.values(this.cache.items);
+ }
+}
diff --git a/lib/modules/datasource/docker/index.spec.ts b/lib/modules/datasource/docker/index.spec.ts
index d450cbad1837222f32522261838ee1a24362258a..e16d42ed533b86f8ab90606251c3e0b6bb1696a1 100644
--- a/lib/modules/datasource/docker/index.spec.ts
+++ b/lib/modules/datasource/docker/index.spec.ts
@@ -1796,21 +1796,25 @@ describe('modules/datasource/docker/index', () => {
process.env.RENOVATE_X_DOCKER_HUB_TAGS = 'true';
httpMock
.scope(dockerHubUrl)
- .get('/library/node/tags?page_size=1000')
+ .get('/library/node/tags?page_size=1000&ordering=last_updated')
.reply(200, {
- next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000`,
+ next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`,
results: [
{
+ id: 2,
+ last_updated: '2021-01-01T00:00:00.000Z',
name: '1.0.0',
tag_last_pushed: '2021-01-01T00:00:00.000Z',
digest: 'aaa',
},
],
})
- .get('/library/node/tags?page=2&page_size=1000')
+ .get('/library/node/tags?page=2&page_size=1000&ordering=last_updated')
.reply(200, {
results: [
{
+ id: 1,
+ last_updated: '2020-01-01T00:00:00.000Z',
name: '0.9.0',
tag_last_pushed: '2020-01-01T00:00:00.000Z',
digest: 'bbb',
@@ -1838,7 +1842,7 @@ describe('modules/datasource/docker/index', () => {
const tags = ['1.0.0'];
httpMock
.scope(dockerHubUrl)
- .get('/library/node/tags?page_size=1000')
+ .get('/library/node/tags?page_size=1000&ordering=last_updated')
.reply(404);
httpMock
.scope(baseUrl)
@@ -1866,21 +1870,25 @@ describe('modules/datasource/docker/index', () => {
process.env.RENOVATE_X_DOCKER_HUB_TAGS = 'true';
httpMock
.scope(dockerHubUrl)
- .get('/library/node/tags?page_size=1000')
+ .get('/library/node/tags?page_size=1000&ordering=last_updated')
.reply(200, {
- next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000`,
+ next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`,
results: [
{
+ id: 2,
+ last_updated: '2021-01-01T00:00:00.000Z',
name: '1.0.0',
tag_last_pushed: '2021-01-01T00:00:00.000Z',
digest: 'aaa',
},
],
})
- .get('/library/node/tags?page=2&page_size=1000')
+ .get('/library/node/tags?page=2&page_size=1000&ordering=last_updated')
.reply(200, {
results: [
{
+ id: 1,
+ last_updated: '2020-01-01T00:00:00.000Z',
name: '0.9.0',
tag_last_pushed: '2020-01-01T00:00:00.000Z',
digest: 'bbb',
diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts
index 5d896ed774dee744f2fed4a2d365e83a32856297..0f5563551d38c85245d05171d04421c876fcff53 100644
--- a/lib/modules/datasource/docker/index.ts
+++ b/lib/modules/datasource/docker/index.ts
@@ -37,6 +37,7 @@ import {
sourceLabel,
sourceLabels,
} from './common';
+import { DockerHubCache } from './dockerhub-cache';
import { ecrPublicRegex, ecrRegex, isECRMaxResultsError } from './ecr';
import {
DistributionManifest,
@@ -927,10 +928,11 @@ export class DockerDatasource extends Datasource {
key: (dockerRepository: string) => `${dockerRepository}`,
})
async getDockerHubTags(dockerRepository: string): Promise<Release[] | null> {
- const result: Release[] = [];
- let url: null | string =
- `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000`;
- while (url) {
+ let url = `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000&ordering=last_updated`;
+
+ const cache = await DockerHubCache.init(dockerRepository);
+ let needNextPage: boolean = true;
+ while (needNextPage) {
const { val, err } = await this.http
.getJsonSafe(url, DockerHubTagsPage)
.unwrap();
@@ -940,11 +942,39 @@ export class DockerDatasource extends Datasource {
return null;
}
- result.push(...val.items);
- url = val.nextPage;
+ const { results, next } = val;
+
+ needNextPage = cache.reconcile(results);
+
+ if (!next) {
+ break;
+ }
+
+ url = next;
}
- return result;
+ await cache.save();
+
+ const items = cache.getItems();
+ return items.map(
+ ({
+ name: version,
+ tag_last_pushed: releaseTimestamp,
+ digest: newDigest,
+ }) => {
+ const release: Release = { version };
+
+ if (releaseTimestamp) {
+ release.releaseTimestamp = releaseTimestamp;
+ }
+
+ if (newDigest) {
+ release.newDigest = newDigest;
+ }
+
+ return release;
+ },
+ );
}
/**
diff --git a/lib/modules/datasource/docker/schema.ts b/lib/modules/datasource/docker/schema.ts
index 1af87da24b49d45e8e3f6c259304f88e592a4aa9..6a58e08abc513a4f86f3b0cd1dfd0ac09fd99e96 100644
--- a/lib/modules/datasource/docker/schema.ts
+++ b/lib/modules/datasource/docker/schema.ts
@@ -1,7 +1,6 @@
import { z } from 'zod';
import { logger } from '../../../logger';
import { Json, LooseArray } from '../../../util/schema-utils';
-import type { Release } from '../types';
// OCI manifests
@@ -155,39 +154,23 @@ export const Manifest = ManifestObject.passthrough()
export type Manifest = z.infer<typeof Manifest>;
export const ManifestJson = Json.pipe(Manifest);
-export const DockerHubTag = z
- .object({
- name: z.string(),
- tag_last_pushed: z.string().datetime().nullable().catch(null),
- digest: z.string().nullable().catch(null),
- })
- .transform(({ name, tag_last_pushed, digest }) => {
- const release: Release = { version: name };
-
- if (tag_last_pushed) {
- release.releaseTimestamp = tag_last_pushed;
- }
-
- if (digest) {
- release.newDigest = digest;
- }
-
- return release;
- });
-
-export const DockerHubTagsPage = z
- .object({
- next: z.string().nullable().catch(null),
- results: LooseArray(DockerHubTag, {
- onError: /* istanbul ignore next */ ({ error }) => {
- logger.debug(
- { error },
- 'Docker: Failed to parse some tags from Docker Hub',
- );
- },
- }),
- })
- .transform(({ next, results }) => ({
- nextPage: next,
- items: results,
- }));
+export const DockerHubTag = z.object({
+ id: z.number(),
+ last_updated: z.string().datetime(),
+ name: z.string(),
+ tag_last_pushed: z.string().datetime().nullable().catch(null),
+ digest: z.string().nullable().catch(null),
+});
+export type DockerHubTag = z.infer<typeof DockerHubTag>;
+
+export const DockerHubTagsPage = z.object({
+ next: z.string().nullable().catch(null),
+ results: LooseArray(DockerHubTag, {
+ onError: /* istanbul ignore next */ ({ error }) => {
+ logger.debug(
+ { error },
+ 'Docker: Failed to parse some tags from Docker Hub',
+ );
+ },
+ }),
+});
diff --git a/lib/util/cache/package/types.ts b/lib/util/cache/package/types.ts
index bb1673c399d8ebaf070fa00a662b0e53386e9906..6d600df7e14e7020d383224e36a9e787b6e3539f 100644
--- a/lib/util/cache/package/types.ts
+++ b/lib/util/cache/package/types.ts
@@ -45,6 +45,7 @@ export type PackageCacheNamespace =
| 'datasource-deno-versions'
| 'datasource-deno'
| 'datasource-docker-architecture'
+ | 'datasource-docker-hub-cache'
| 'datasource-docker-digest'
| 'datasource-docker-hub-tags'
| 'datasource-docker-imageconfig'