diff --git a/lib/modules/datasource/docker/dockerhub-cache.spec.ts b/lib/modules/datasource/docker/dockerhub-cache.spec.ts
index fe8ad65504229e01b317e134a1395b45ce348cf7..1a97be3488a0ea2289a1e9af1bc6fb7b72473abb 100644
--- a/lib/modules/datasource/docker/dockerhub-cache.spec.ts
+++ b/lib/modules/datasource/docker/dockerhub-cache.spec.ts
@@ -76,6 +76,7 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
updatedAt: null,
},
isChanged: false,
+ reconciledIds: new Set(),
});
});
@@ -89,6 +90,7 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
dockerRepository,
cache: oldCache,
isChanged: false,
+ reconciledIds: new Set(),
});
});
@@ -100,13 +102,14 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
const cache = await DockerHubCache.init(dockerRepository);
const newItems: DockerHubTag[] = [newItem()];
- const needNextPage = cache.reconcile(newItems);
+ const needNextPage = cache.reconcile(newItems, 4);
expect(needNextPage).toBe(true);
expect(cache).toEqual({
cache: newCache,
dockerRepository: 'foo/bar',
isChanged: true,
+ reconciledIds: new Set([4]),
});
const res = cache.getItems();
@@ -128,13 +131,14 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
const cache = await DockerHubCache.init(dockerRepository);
const items: DockerHubTag[] = Object.values(oldCache.items);
- const needNextPage = cache.reconcile(items);
+ const needNextPage = cache.reconcile(items, 3);
expect(needNextPage).toBe(false);
expect(cache).toEqual({
cache: oldCache,
dockerRepository: 'foo/bar',
isChanged: false,
+ reconciledIds: new Set([1, 2, 3]),
});
const res = cache.getItems();
@@ -144,6 +148,48 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
expect(packageCache.set).not.toHaveBeenCalled();
});
+ it('asks for the next page if the expected count does not match cached items', async () => {
+ const oldCache = oldCacheData();
+
+ packageCache.get.mockResolvedValue(oldCache);
+ const cache = await DockerHubCache.init(dockerRepository);
+ const items: DockerHubTag[] = Object.values(oldCache.items);
+
+ const needNextPage = cache.reconcile(items, 0);
+
+ expect(needNextPage).toBe(true);
+ expect(cache).toEqual({
+ cache: oldCache,
+ dockerRepository: 'foo/bar',
+ isChanged: false,
+ reconciledIds: new Set([1, 2, 3]),
+ });
+
+ const res = cache.getItems();
+ expect(res).toEqual(items);
+
+ await cache.save();
+ expect(packageCache.set).not.toHaveBeenCalled();
+ });
+
+ it('reconciles deleted items', async () => {
+ const oldCache = oldCacheData();
+
+ packageCache.get.mockResolvedValue(oldCache);
+ const cache = await DockerHubCache.init(dockerRepository);
+ const items: DockerHubTag[] = [oldCache.items[1], oldCache.items[3]];
+
+ const needNextPage = cache.reconcile(items, 2);
+
+ expect(needNextPage).toBe(false);
+
+ const res = cache.getItems();
+ expect(res).toEqual(items);
+
+ await cache.save();
+ expect(packageCache.set).toHaveBeenCalled();
+ });
+
it('reconciles from empty cache', async () => {
const item = newItem();
const expectedCache = {
@@ -154,12 +200,13 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
};
const cache = await DockerHubCache.init(dockerRepository);
- const needNextPage = cache.reconcile([item]);
+ const needNextPage = cache.reconcile([item], 1);
expect(needNextPage).toBe(true);
expect(cache).toEqual({
cache: expectedCache,
dockerRepository: 'foo/bar',
isChanged: true,
+ reconciledIds: new Set([4]),
});
const res = cache.getItems();
diff --git a/lib/modules/datasource/docker/dockerhub-cache.ts b/lib/modules/datasource/docker/dockerhub-cache.ts
index 0e97726fc01fb2690fdc9f6d1dea69b028e8a0b9..a90703e97b1b2276c7271b3d50052979801c92fe 100644
--- a/lib/modules/datasource/docker/dockerhub-cache.ts
+++ b/lib/modules/datasource/docker/dockerhub-cache.ts
@@ -12,6 +12,7 @@ const cacheNamespace = 'datasource-docker-hub-cache';
export class DockerHubCache {
private isChanged = false;
+ private reconciledIds = new Set<number>();
private constructor(
private dockerRepository: string,
@@ -32,32 +33,65 @@ export class DockerHubCache {
return new DockerHubCache(dockerRepository, repoCache);
}
- reconcile(items: DockerHubTag[]): boolean {
+ reconcile(items: DockerHubTag[], expectedCount: number): boolean {
let needNextPage = true;
+ let earliestDate = null;
+
let { updatedAt } = this.cache;
let latestDate = updatedAt ? DateTime.fromISO(updatedAt) : null;
for (const newItem of items) {
const id = newItem.id;
+ this.reconciledIds.add(id);
+
const oldItem = this.cache.items[id];
+ const itemDate = DateTime.fromISO(newItem.last_updated);
+
+ if (!earliestDate || earliestDate > itemDate) {
+ earliestDate = itemDate;
+ }
+
+ if (!latestDate || latestDate < itemDate) {
+ latestDate = itemDate;
+ updatedAt = newItem.last_updated;
+ }
+
if (dequal(oldItem, newItem)) {
needNextPage = false;
continue;
}
this.cache.items[newItem.id] = newItem;
- const newItemDate = DateTime.fromISO(newItem.last_updated);
- if (!latestDate || latestDate < newItemDate) {
- updatedAt = newItem.last_updated;
- latestDate = newItemDate;
- }
-
this.isChanged = true;
}
this.cache.updatedAt = updatedAt;
+
+ if (earliestDate && latestDate) {
+ for (const [key, item] of Object.entries(this.cache.items)) {
+ const id = parseInt(key, 10);
+
+ const itemDate = DateTime.fromISO(item.last_updated);
+
+ if (
+ itemDate < earliestDate ||
+ itemDate > latestDate ||
+ this.reconciledIds.has(id)
+ ) {
+ continue;
+ }
+
+ delete this.cache.items[id];
+ this.isChanged = true;
+ }
+
+ if (Object.keys(this.cache.items).length > expectedCount) {
+ return true;
+ }
+ }
+
return needNextPage;
}
diff --git a/lib/modules/datasource/docker/index.spec.ts b/lib/modules/datasource/docker/index.spec.ts
index 3b84b910e43d34f15ebe073856695cd9ed43a960..0380dbffdbf5e7b2ea3f39dedc99faeaafb7b7e0 100644
--- a/lib/modules/datasource/docker/index.spec.ts
+++ b/lib/modules/datasource/docker/index.spec.ts
@@ -1887,6 +1887,7 @@ describe('modules/datasource/docker/index', () => {
.scope(dockerHubUrl)
.get('/library/node/tags?page_size=1000&ordering=last_updated')
.reply(200, {
+ count: 2,
next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`,
results: [
{
@@ -1900,6 +1901,7 @@ describe('modules/datasource/docker/index', () => {
})
.get('/library/node/tags?page=2&page_size=1000&ordering=last_updated')
.reply(200, {
+ count: 2,
results: [
{
id: 1,
@@ -1960,6 +1962,7 @@ describe('modules/datasource/docker/index', () => {
.get('/library/node/tags?page_size=1000&ordering=last_updated')
.reply(200, {
next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`,
+ count: 2,
results: [
{
id: 2,
@@ -1972,6 +1975,7 @@ describe('modules/datasource/docker/index', () => {
})
.get('/library/node/tags?page=2&page_size=1000&ordering=last_updated')
.reply(200, {
+ count: 2,
results: [
{
id: 1,
diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts
index 3786345269875696b575583f51236c7487e61984..f318e3d2c11c807ea46ff1f2cb2fbac276e70b07 100644
--- a/lib/modules/datasource/docker/index.ts
+++ b/lib/modules/datasource/docker/index.ts
@@ -988,9 +988,9 @@ export class DockerDatasource extends Datasource {
return null;
}
- const { results, next } = val;
+ const { results, next, count } = val;
- needNextPage = cache.reconcile(results);
+ needNextPage = cache.reconcile(results, count);
if (!next) {
break;
diff --git a/lib/modules/datasource/docker/schema.ts b/lib/modules/datasource/docker/schema.ts
index a7f494acfa6d036045c17d758db3d7e222e1aec5..1cfc4c0b521ea711e2181f4438b3669cb93ecf4a 100644
--- a/lib/modules/datasource/docker/schema.ts
+++ b/lib/modules/datasource/docker/schema.ts
@@ -165,6 +165,7 @@ export const DockerHubTag = z.object({
export type DockerHubTag = z.infer<typeof DockerHubTag>;
export const DockerHubTagsPage = z.object({
+ count: z.number(),
next: z.string().nullable().catch(null),
results: LooseArray(DockerHubTag, {
onError: /* istanbul ignore next */ ({ error }) => {