feat(internal): cached datasource lookups (#5870)

lib/datasource/cache.ts

+import { logger } from '../logger';
+
+/**
+ * Cache callback result which has to be returned by the `CacheCallback` function.
+ */
+export interface CacheResult<TResult = unknown> {
+  /**
+   * The data which should be added to the cache
+   */
+  data: TResult;
+  /**
+   * `data` can only be cached if this is not `true`
+   */
+  isPrivate?: boolean;
+}
+
+/**
+ * Simple helper type for defining the `CacheCallback` function return type
+ */
+export type CachePromise<TResult = unknown> = Promise<CacheResult<TResult>>;
+
+/**
+ * The callback function which is called on cache miss.
+ */
+export type CacheCallback<TArg, TResult = unknown> = (
+  lookup: TArg
+) => CachePromise<TResult>;
+
+export type CacheConfig<TArg, TResult> = {
+  /**
+   * Datasource id
+   */
+  id: string;
+  /**
+   * Cache key
+   */
+  lookup: TArg;
+  /**
+   * Callback to use on cache miss to load result
+   */
+  cb: CacheCallback<TArg, TResult>;
+  /**
+   * Time to cache result in minutes
+   */
+  minutes?: number;
+};
+
+/**
+ * Loads result from cache or from passed callback on cache miss.
+ * @param param0 Cache config args
+ */
+export async function cacheAble<TArg, TResult = unknown>({
+  id,
+  lookup,
+  cb,
+  minutes = 60,
+}: CacheConfig<TArg, TResult>): Promise<TResult> {
+  const cacheNamespace = `datasource-${id}`;
+  const cacheKey = JSON.stringify(lookup);
+  const cachedResult = await renovateCache.get<TResult>(
+    cacheNamespace,
+    cacheKey
+  );
+  // istanbul ignore if
+  if (cachedResult) {
+    logger.trace({ id, lookup }, 'datasource cachedResult');
+    return cachedResult;
+  }
+  const { data, isPrivate } = await cb(lookup);
+  // istanbul ignore if
+  if (isPrivate) {
+    logger.trace({ id, lookup }, 'Skipping datasource cache for private data');
+  } else {
+    await renovateCache.set(cacheNamespace, cacheKey, data, minutes);
+  }
+  return data;
+}

lib/datasource/cdnjs/index.spec.ts

@@ -30,17 +30,21 @@ describe('datasource/cdnjs', () => {
         DATASOURCE_FAILURE
       );
     });
-    it('returns null for missing fields', async () => {
+    it('throws for missing fields', async () => {
       got.mockResolvedValueOnce({});
-      expect(await getReleases({ lookupName: 'foo/bar' })).toBeNull();
+      await expect(getReleases({ lookupName: 'foo/bar' })).rejects.toThrowError(
+        DATASOURCE_FAILURE
+      );
     });
     it('returns null for 404', async () => {
       got.mockRejectedValueOnce({ statusCode: 404 });
       expect(await getReleases({ lookupName: 'foo/bar' })).toBeNull();
     });
-    it('returns null for 401', async () => {
+    it('throws for 401', async () => {
       got.mockRejectedValueOnce({ statusCode: 401 });
-      expect(await getReleases({ lookupName: 'foo/bar' })).toBeNull();
+      await expect(getReleases({ lookupName: 'foo/bar' })).rejects.toThrowError(
+        DATASOURCE_FAILURE
+      );
     });
     it('throws for 429', async () => {
       got.mockRejectedValueOnce({ statusCode: 429 });
@@ -62,11 +66,6 @@ describe('datasource/cdnjs', () => {
         DATASOURCE_FAILURE
       );
     });
-    it('returns null with wrong auth token', async () => {
-      got.mockRejectedValueOnce({ statusCode: 401 });
-      const res = await getReleases({ lookupName: 'foo/bar' });
-      expect(res).toBeNull();
-    });
     it('processes real data', async () => {
       got.mockResolvedValueOnce({ body: res1 });
       const res = await getReleases({ lookupName: 'd3-force/d3-force.js' });

lib/datasource/cdnjs/index.ts

 import { logger } from '../../logger';
 import { Http } from '../../util/http';
 import { DatasourceError, ReleaseResult, GetReleasesConfig } from '../common';
+import { cacheAble, CachePromise } from '../cache';
+
+export const id = 'cdnjs';
+
+const http = new Http(id);
 
 export interface CdnjsAsset {
   version: string;
@@ -8,13 +13,6 @@ export interface CdnjsAsset {
   sri?: Record<string, string>;
 }
 
-export const id = 'cdnjs';
-
-const http = new Http(id);
-
-const cacheNamespace = `datasource-${id}`;
-const cacheMinutes = 60;
-
 export interface CdnjsResponse {
   homepage?: string;
   repository?: {
@@ -24,40 +22,22 @@ export interface CdnjsResponse {
   assets?: CdnjsAsset[];
 }
 
-export function depUrl(library: string): string {
-  return `https://api.cdnjs.com/libraries/${library}?fields=homepage,repository,assets`;
+async function downloadLibrary(library: string): CachePromise<CdnjsResponse> {
+  const url = `https://api.cdnjs.com/libraries/${library}?fields=homepage,repository,assets`;
+  return { data: (await http.getJson<CdnjsResponse>(url)).body };
 }
 
 export async function getReleases({
   lookupName,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  const [library, ...assetParts] = lookupName.split('/');
-  const assetName = assetParts.join('/');
-
-  const cacheKey = library;
-  const cachedResult = await renovateCache.get<ReleaseResult>(
-    cacheNamespace,
-    cacheKey
-  );
-  // istanbul ignore if
-  if (cachedResult) {
-    return cachedResult;
-  }
-
-  const url = depUrl(library);
-
+  const library = lookupName.split('/')[0];
   try {
-    const res = await http.getJson(url);
-
-    const cdnjsResp: CdnjsResponse = res.body;
-
-    if (!cdnjsResp || !cdnjsResp.assets) {
-      logger.warn({ library }, `Invalid CDNJS response`);
-      return null;
-    }
-
-    const { assets, homepage, repository } = cdnjsResp;
-
+    const { assets, homepage, repository } = await cacheAble({
+      id,
+      lookup: library,
+      cb: downloadLibrary,
+    });
+    const assetName = lookupName.replace(`${library}/`, '');
     const releases = assets
       .filter(({ files }) => files.includes(assetName))
       .map(({ version, sri }) => ({ version, newDigest: sri[assetName] }));
@@ -67,31 +47,16 @@ export async function getReleases({
     if (homepage) {
       result.homepage = homepage;
     }
-    if (repository && repository.url) {
+    if (repository?.url) {
       result.sourceUrl = repository.url;
     }
-
-    await renovateCache.set(cacheNamespace, cacheKey, result, cacheMinutes);
-
     return result;
   } catch (err) {
-    const errorData = { library, err };
-
-    if (
-      err.statusCode === 429 ||
-      (err.statusCode >= 500 && err.statusCode < 600)
-    ) {
-      throw new DatasourceError(err);
-    }
-    if (err.statusCode === 401) {
-      logger.debug(errorData, 'Authorization error');
-    } else if (err.statusCode === 404) {
-      logger.debug(errorData, 'Package lookup error');
-    } else {
-      logger.debug(errorData, 'CDNJS lookup failure: Unknown error');
-      throw new DatasourceError(err);
+    if (err.statusCode === 404) {
+      logger.debug({ library, err }, 'Package lookup error');
+      return null;
     }
+    // Throw a DatasourceError for all other types of errors
+    throw new DatasourceError(err);
   }
-
-  return null;
 }