feat(github): Implement ApiCache data structure (#14943)

lib/modules/platform/github/api-cache.spec.ts

+import { DateTime } from 'luxon';
+import { ApiCache } from './api-cache';
+
+describe('modules/platform/github/api-cache', () => {
+  const now = DateTime.fromISO('2000-01-01T00:00:00.000+00:00');
+  const t1 = now.plus({ years: 1 }).toISO();
+  const t1_http = now.plus({ years: 1 }).toHTTP();
+
+  const t2 = now.plus({ years: 2 }).toISO();
+  const t2_http = now.plus({ years: 2 }).toHTTP();
+
+  const t3 = now.plus({ years: 3 }).toISO();
+  const t3_http = now.plus({ years: 3 }).toHTTP();
+
+  const t4 = now.plus({ years: 4 }).toISO();
+  const t4_http = now.plus({ years: 4 }).toHTTP();
+
+  const t5 = now.plus({ years: 5 }).toISO();
+  const t5_http = now.plus({ years: 5 }).toHTTP();
+
+  it('stores and retrieves items', () => {
+    const item1 = { number: 1, updated_at: t1 };
+    const item2 = { number: 2, updated_at: t2 };
+    const apiCache = new ApiCache({
+      items: { 1: item1 },
+      lastModified: t1,
+    });
+
+    expect(apiCache.getItem(1)).toBe(item1);
+    expect(apiCache.getItem(2)).toBeNull();
+
+    apiCache.updateItem(item2);
+    expect(apiCache.getItem(2)).toBe(item2);
+    expect(apiCache.lastModified).toBe(t1_http); // Not `t2`, see jsdoc for `setItem`
+    expect(apiCache.getItems()).toEqual([item1, item2]);
+  });
+
+  describe('reconcile', () => {
+    it('appends new items', () => {
+      const apiCache = new ApiCache({ items: {} });
+      expect(apiCache.lastModified).toBeNull();
+
+      const res1 = apiCache.reconcile([
+        { number: 2, updated_at: t2 },
+        { number: 1, updated_at: t1 },
+      ]);
+      expect(apiCache.lastModified).toBe(t2_http);
+      expect(res1).toBeTrue();
+
+      const res2 = apiCache.reconcile([
+        { number: 4, updated_at: t4 },
+        { number: 3, updated_at: t3 },
+      ]);
+      expect(apiCache.lastModified).toBe(t4_http);
+      expect(res2).toBeTrue();
+
+      expect(apiCache.getItems()).toEqual([
+        { number: 1, updated_at: t1 },
+        { number: 2, updated_at: t2 },
+        { number: 3, updated_at: t3 },
+        { number: 4, updated_at: t4 },
+      ]);
+    });
+
+    it('handles updated items', () => {
+      const apiCache = new ApiCache({
+        items: {
+          1: { number: 1, updated_at: t1 },
+          2: { number: 2, updated_at: t2 },
+          3: { number: 3, updated_at: t3 },
+        },
+        lastModified: t3,
+      });
+
+      const needNextPage = apiCache.reconcile([
+        { number: 1, updated_at: t5 },
+        { number: 2, updated_at: t4 },
+        { number: 3, updated_at: t3 },
+      ]);
+
+      expect(apiCache.getItems()).toEqual([
+        { number: 1, updated_at: t5 },
+        { number: 2, updated_at: t4 },
+        { number: 3, updated_at: t3 },
+      ]);
+      expect(apiCache.lastModified).toBe(t5_http);
+      expect(needNextPage).toBeFalse();
+    });
+
+    it('ignores page overlap', () => {
+      const apiCache = new ApiCache({
+        items: {},
+      });
+
+      const res1 = apiCache.reconcile([
+        { number: 5, updated_at: t5 },
+        { number: 4, updated_at: t4 },
+        { number: 3, updated_at: t3 },
+      ]);
+      const res2 = apiCache.reconcile([
+        { number: 3, updated_at: t3 },
+        { number: 2, updated_at: t2 },
+        { number: 1, updated_at: t1 },
+      ]);
+
+      expect(apiCache.getItems()).toEqual([
+        { number: 1, updated_at: t1 },
+        { number: 2, updated_at: t2 },
+        { number: 3, updated_at: t3 },
+        { number: 4, updated_at: t4 },
+        { number: 5, updated_at: t5 },
+      ]);
+      expect(apiCache.lastModified).toBe(t5_http);
+      expect(res1).toBeTrue();
+      expect(res2).toBeTrue();
+    });
+
+    it('does not require new page if all items are old', () => {
+      const apiCache = new ApiCache({
+        items: {
+          1: { number: 1, updated_at: t1 },
+          2: { number: 2, updated_at: t2 },
+          3: { number: 3, updated_at: t3 },
+        },
+        lastModified: t3,
+      });
+
+      const needNextPage = apiCache.reconcile([
+        { number: 3, updated_at: t3 },
+        { number: 2, updated_at: t2 },
+        { number: 1, updated_at: t1 },
+      ]);
+
+      expect(apiCache.getItems()).toEqual([
+        { number: 1, updated_at: t1 },
+        { number: 2, updated_at: t2 },
+        { number: 3, updated_at: t3 },
+      ]);
+      expect(apiCache.lastModified).toBe(t3_http);
+      expect(needNextPage).toBeFalse();
+    });
+  });
+
+  describe('etag', () => {
+    it('returns null', () => {
+      const apiCache = new ApiCache({ items: {} });
+      expect(apiCache.etag).toBeNull();
+    });
+
+    it('sets and retrieves non-null value', () => {
+      const apiCache = new ApiCache({ items: {} });
+      apiCache.etag = 'foobar';
+      expect(apiCache.etag).toBe('foobar');
+    });
+
+    it('deletes value for null parameter', () => {
+      const apiCache = new ApiCache({ items: {} });
+      apiCache.etag = null;
+      expect(apiCache.etag).toBeNull();
+    });
+  });
+});

lib/modules/platform/github/api-cache.ts

+import { DateTime } from 'luxon';
+import type { ApiPageCache, ApiPageItem } from './types';
+
+export class ApiCache<T extends ApiPageItem> {
+  constructor(private cache: ApiPageCache<T>) {}
+
+  get etag(): string | null {
+    return this.cache.etag ?? null;
+  }
+
+  set etag(value: string | null) {
+    if (value === null) {
+      delete this.cache.etag;
+    } else {
+      this.cache.etag = value;
+    }
+  }
+
+  /**
+   * @returns Date formatted to use in HTTP headers
+   */
+  get lastModified(): string | null {
+    const { lastModified } = this.cache;
+    return lastModified ? DateTime.fromISO(lastModified).toHTTP() : null;
+  }
+
+  getItems(): T[] {
+    return Object.values(this.cache.items);
+  }
+
+  getItem(number: number): T | null {
+    return this.cache.items[number] ?? null;
+  }
+
+  /**
+   * It intentionally doesn't alter `lastModified` cache field.
+   *
+   * The point is to allow cache modifications during run, but
+   * force fetching and refreshing of modified items next run.
+   */
+  updateItem(item: T): void {
+    this.cache.items[item.number] = item;
+  }
+
+  /**
+   * Copies items from `page` to `cache`.
+   * Updates internal cache timestamp.
+   *
+   * @param cache Cache object
+   * @param page List of cacheable items, sorted by `updated_at` field
+   * starting from the most recently updated.
+   * @returns `true` when the next page is likely to contain fresh items,
+   * otherwise `false`.
+   */
+  reconcile(page: T[]): boolean {
+    const { items } = this.cache;
+    let { lastModified } = this.cache;
+
+    let needNextPage = true;
+
+    for (const newItem of page) {
+      const number = newItem.number;
+      const oldItem = items[number];
+
+      const itemNewTime = DateTime.fromISO(newItem.updated_at);
+      const itemOldTime = oldItem?.updated_at
+        ? DateTime.fromISO(oldItem.updated_at)
+        : null;
+
+      items[number] = newItem;
+
+      needNextPage = itemOldTime ? itemOldTime < itemNewTime : true;
+
+      const cacheOldTime = lastModified ? DateTime.fromISO(lastModified) : null;
+      if (!cacheOldTime || itemNewTime > cacheOldTime) {
+        lastModified = newItem.updated_at;
+      }
+    }
+
+    this.cache.lastModified = lastModified;
+
+    return needNextPage;
+  }
+}

lib/modules/platform/github/types.ts

@@ -125,3 +125,17 @@ export interface GhAutomergeResponse {
     pullRequest: { number: number };
   };
 }
+
+export interface ApiPageItem {
+  number: number;
+  updated_at: string;
+}
+
+/**
+ * Mutable object designed to be used in the repository cache
+ */
+export interface ApiPageCache<T extends ApiPageItem = ApiPageItem> {
+  items: Record<number, T>;
+  lastModified?: string;
+  etag?: string;
+}