From 65a99fb94f8bc4c7f67b2bb1edf9df6ba8c2c80b Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Wed, 6 Apr 2022 19:00:26 +0300
Subject: [PATCH] feat(github): Implement ApiCache data structure (#14943)
---
lib/modules/platform/github/api-cache.spec.ts | 162 ++++++++++++++++++
lib/modules/platform/github/api-cache.ts | 84 +++++++++
lib/modules/platform/github/types.ts | 14 ++
3 files changed, 260 insertions(+)
create mode 100644 lib/modules/platform/github/api-cache.spec.ts
create mode 100644 lib/modules/platform/github/api-cache.ts
diff --git a/lib/modules/platform/github/api-cache.spec.ts b/lib/modules/platform/github/api-cache.spec.ts
new file mode 100644
index 0000000000..d0c554c585
--- /dev/null
+++ b/lib/modules/platform/github/api-cache.spec.ts
@@ -0,0 +1,162 @@
+import { DateTime } from 'luxon';
+import { ApiCache } from './api-cache';
+
+describe('modules/platform/github/api-cache', () => {
+ const now = DateTime.fromISO('2000-01-01T00:00:00.000+00:00');
+ const t1 = now.plus({ years: 1 }).toISO();
+ const t1_http = now.plus({ years: 1 }).toHTTP();
+
+ const t2 = now.plus({ years: 2 }).toISO();
+ const t2_http = now.plus({ years: 2 }).toHTTP();
+
+ const t3 = now.plus({ years: 3 }).toISO();
+ const t3_http = now.plus({ years: 3 }).toHTTP();
+
+ const t4 = now.plus({ years: 4 }).toISO();
+ const t4_http = now.plus({ years: 4 }).toHTTP();
+
+ const t5 = now.plus({ years: 5 }).toISO();
+ const t5_http = now.plus({ years: 5 }).toHTTP();
+
+ it('stores and retrieves items', () => {
+ const item1 = { number: 1, updated_at: t1 };
+ const item2 = { number: 2, updated_at: t2 };
+ const apiCache = new ApiCache({
+ items: { 1: item1 },
+ lastModified: t1,
+ });
+
+ expect(apiCache.getItem(1)).toBe(item1);
+ expect(apiCache.getItem(2)).toBeNull();
+
+ apiCache.updateItem(item2);
+ expect(apiCache.getItem(2)).toBe(item2);
+ expect(apiCache.lastModified).toBe(t1_http); // Not `t2`, see jsdoc for `setItem`
+ expect(apiCache.getItems()).toEqual([item1, item2]);
+ });
+
+ describe('reconcile', () => {
+ it('appends new items', () => {
+ const apiCache = new ApiCache({ items: {} });
+ expect(apiCache.lastModified).toBeNull();
+
+ const res1 = apiCache.reconcile([
+ { number: 2, updated_at: t2 },
+ { number: 1, updated_at: t1 },
+ ]);
+ expect(apiCache.lastModified).toBe(t2_http);
+ expect(res1).toBeTrue();
+
+ const res2 = apiCache.reconcile([
+ { number: 4, updated_at: t4 },
+ { number: 3, updated_at: t3 },
+ ]);
+ expect(apiCache.lastModified).toBe(t4_http);
+ expect(res2).toBeTrue();
+
+ expect(apiCache.getItems()).toEqual([
+ { number: 1, updated_at: t1 },
+ { number: 2, updated_at: t2 },
+ { number: 3, updated_at: t3 },
+ { number: 4, updated_at: t4 },
+ ]);
+ });
+
+ it('handles updated items', () => {
+ const apiCache = new ApiCache({
+ items: {
+ 1: { number: 1, updated_at: t1 },
+ 2: { number: 2, updated_at: t2 },
+ 3: { number: 3, updated_at: t3 },
+ },
+ lastModified: t3,
+ });
+
+ const needNextPage = apiCache.reconcile([
+ { number: 1, updated_at: t5 },
+ { number: 2, updated_at: t4 },
+ { number: 3, updated_at: t3 },
+ ]);
+
+ expect(apiCache.getItems()).toEqual([
+ { number: 1, updated_at: t5 },
+ { number: 2, updated_at: t4 },
+ { number: 3, updated_at: t3 },
+ ]);
+ expect(apiCache.lastModified).toBe(t5_http);
+ expect(needNextPage).toBeFalse();
+ });
+
+ it('ignores page overlap', () => {
+ const apiCache = new ApiCache({
+ items: {},
+ });
+
+ const res1 = apiCache.reconcile([
+ { number: 5, updated_at: t5 },
+ { number: 4, updated_at: t4 },
+ { number: 3, updated_at: t3 },
+ ]);
+ const res2 = apiCache.reconcile([
+ { number: 3, updated_at: t3 },
+ { number: 2, updated_at: t2 },
+ { number: 1, updated_at: t1 },
+ ]);
+
+ expect(apiCache.getItems()).toEqual([
+ { number: 1, updated_at: t1 },
+ { number: 2, updated_at: t2 },
+ { number: 3, updated_at: t3 },
+ { number: 4, updated_at: t4 },
+ { number: 5, updated_at: t5 },
+ ]);
+ expect(apiCache.lastModified).toBe(t5_http);
+ expect(res1).toBeTrue();
+ expect(res2).toBeTrue();
+ });
+
+ it('does not require new page if all items are old', () => {
+ const apiCache = new ApiCache({
+ items: {
+ 1: { number: 1, updated_at: t1 },
+ 2: { number: 2, updated_at: t2 },
+ 3: { number: 3, updated_at: t3 },
+ },
+ lastModified: t3,
+ });
+
+ const needNextPage = apiCache.reconcile([
+ { number: 3, updated_at: t3 },
+ { number: 2, updated_at: t2 },
+ { number: 1, updated_at: t1 },
+ ]);
+
+ expect(apiCache.getItems()).toEqual([
+ { number: 1, updated_at: t1 },
+ { number: 2, updated_at: t2 },
+ { number: 3, updated_at: t3 },
+ ]);
+ expect(apiCache.lastModified).toBe(t3_http);
+ expect(needNextPage).toBeFalse();
+ });
+ });
+
+ describe('etag', () => {
+ it('returns null', () => {
+ const apiCache = new ApiCache({ items: {} });
+ expect(apiCache.etag).toBeNull();
+ });
+
+ it('sets and retrieves non-null value', () => {
+ const apiCache = new ApiCache({ items: {} });
+ apiCache.etag = 'foobar';
+ expect(apiCache.etag).toBe('foobar');
+ });
+
+ it('deletes value for null parameter', () => {
+ const apiCache = new ApiCache({ items: {} });
+ apiCache.etag = null;
+ expect(apiCache.etag).toBeNull();
+ });
+ });
+});
diff --git a/lib/modules/platform/github/api-cache.ts b/lib/modules/platform/github/api-cache.ts
new file mode 100644
index 0000000000..09d4fed91c
--- /dev/null
+++ b/lib/modules/platform/github/api-cache.ts
@@ -0,0 +1,84 @@
+import { DateTime } from 'luxon';
+import type { ApiPageCache, ApiPageItem } from './types';
+
+export class ApiCache<T extends ApiPageItem> {
+ constructor(private cache: ApiPageCache<T>) {}
+
+ get etag(): string | null {
+ return this.cache.etag ?? null;
+ }
+
+ set etag(value: string | null) {
+ if (value === null) {
+ delete this.cache.etag;
+ } else {
+ this.cache.etag = value;
+ }
+ }
+
+ /**
+ * @returns Date formatted to use in HTTP headers
+ */
+ get lastModified(): string | null {
+ const { lastModified } = this.cache;
+ return lastModified ? DateTime.fromISO(lastModified).toHTTP() : null;
+ }
+
+ getItems(): T[] {
+ return Object.values(this.cache.items);
+ }
+
+ getItem(number: number): T | null {
+ return this.cache.items[number] ?? null;
+ }
+
+ /**
+ * It intentionally doesn't alter `lastModified` cache field.
+ *
+ * The point is to allow cache modifications during run, but
+ * force fetching and refreshing of modified items next run.
+ */
+ updateItem(item: T): void {
+ this.cache.items[item.number] = item;
+ }
+
+ /**
+ * Copies items from `page` to `cache`.
+ * Updates internal cache timestamp.
+ *
+ * @param cache Cache object
+ * @param page List of cacheable items, sorted by `updated_at` field
+ * starting from the most recently updated.
+ * @returns `true` when the next page is likely to contain fresh items,
+ * otherwise `false`.
+ */
+ reconcile(page: T[]): boolean {
+ const { items } = this.cache;
+ let { lastModified } = this.cache;
+
+ let needNextPage = true;
+
+ for (const newItem of page) {
+ const number = newItem.number;
+ const oldItem = items[number];
+
+ const itemNewTime = DateTime.fromISO(newItem.updated_at);
+ const itemOldTime = oldItem?.updated_at
+ ? DateTime.fromISO(oldItem.updated_at)
+ : null;
+
+ items[number] = newItem;
+
+ needNextPage = itemOldTime ? itemOldTime < itemNewTime : true;
+
+ const cacheOldTime = lastModified ? DateTime.fromISO(lastModified) : null;
+ if (!cacheOldTime || itemNewTime > cacheOldTime) {
+ lastModified = newItem.updated_at;
+ }
+ }
+
+ this.cache.lastModified = lastModified;
+
+ return needNextPage;
+ }
+}
diff --git a/lib/modules/platform/github/types.ts b/lib/modules/platform/github/types.ts
index 46114f32e2..49e116295c 100644
--- a/lib/modules/platform/github/types.ts
+++ b/lib/modules/platform/github/types.ts
@@ -125,3 +125,17 @@ export interface GhAutomergeResponse {
pullRequest: { number: number };
};
}
+
+export interface ApiPageItem {
+ number: number;
+ updated_at: string;
+}
+
+/**
+ * Mutable object designed to be used in the repository cache
+ */
+export interface ApiPageCache<T extends ApiPageItem = ApiPageItem> {
+ items: Record<number, T>;
+ lastModified?: string;
+ etag?: string;
+}
--
GitLab