From 2df8967ae27851e79d7c665483456825a21a6e6d Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Tue, 6 Sep 2022 13:36:51 +0300
Subject: [PATCH] refactor: Create promises utilities (#17651)
---
.../datasource/galaxy-collection/index.ts | 7 +++--
lib/modules/datasource/go/releases-goproxy.ts | 4 +--
lib/modules/datasource/maven/index.ts | 4 +--
lib/modules/datasource/nuget/v3.ts | 6 ++---
lib/modules/datasource/packagist/index.ts | 4 +--
.../datasource/terraform-provider/index.ts | 4 +--
lib/modules/manager/hermit/artifacts.ts | 17 +++++-------
.../manager/terraform/lockfile/hash.ts | 10 +++----
.../manager/terraform/lockfile/index.ts | 6 ++---
lib/modules/platform/gitlab/index.ts | 7 ++---
lib/util/http/github.ts | 4 +--
lib/util/promises.ts | 27 +++++++++++++++++++
lib/workers/repository/changelog/index.ts | 4 +--
lib/workers/repository/process/fetch.ts | 6 ++---
.../repository/process/vulnerabilities.ts | 8 +++---
15 files changed, 66 insertions(+), 52 deletions(-)
create mode 100644 lib/util/promises.ts
diff --git a/lib/modules/datasource/galaxy-collection/index.ts b/lib/modules/datasource/galaxy-collection/index.ts
index 1839445f1e..0d89d1b1c0 100644
--- a/lib/modules/datasource/galaxy-collection/index.ts
+++ b/lib/modules/datasource/galaxy-collection/index.ts
@@ -1,8 +1,8 @@
import is from '@sindresorhus/is';
-import pMap from 'p-map';
import { logger } from '../../../logger';
import { cache } from '../../../util/cache/package/decorator';
import type { HttpResponse } from '../../../util/http/types';
+import * as p from '../../../util/promises';
import { Datasource } from '../datasource';
import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
import type {
@@ -76,7 +76,7 @@ export class GalaxyCollectionDatasource extends Datasource {
let newestVersionDetails: VersionsDetailResult | undefined;
// asynchronously get release details
- const enrichedReleases: (Release | null)[] = await pMap(
+ const enrichedReleases: (Release | null)[] = await p.map(
releases,
(basicRelease) =>
this.http
@@ -108,8 +108,7 @@ export class GalaxyCollectionDatasource extends Datasource {
);
return null;
}
- }),
- { concurrency: 5 } // allow 5 requests at maximum in parallel
+ })
);
// filter failed versions
const filteredReleases = enrichedReleases.filter(is.truthy);
diff --git a/lib/modules/datasource/go/releases-goproxy.ts b/lib/modules/datasource/go/releases-goproxy.ts
index 0f98bbfbf9..89bcbe95d0 100644
--- a/lib/modules/datasource/go/releases-goproxy.ts
+++ b/lib/modules/datasource/go/releases-goproxy.ts
@@ -1,8 +1,8 @@
import is from '@sindresorhus/is';
import moo from 'moo';
-import pAll from 'p-all';
import { logger } from '../../../logger';
import { cache } from '../../../util/cache/package/decorator';
+import * as p from '../../../util/promises';
import { regEx } from '../../../util/regex';
import { Datasource } from '../datasource';
import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
@@ -60,7 +60,7 @@ export class GoProxyDatasource extends Datasource {
return { version };
}
});
- const releases = await pAll(queue, { concurrency: 5 });
+ const releases = await p.all(queue);
if (releases.length) {
const datasource = await BaseGoDatasource.getDatasource(packageName);
const sourceUrl = getSourceUrl(datasource);
diff --git a/lib/modules/datasource/maven/index.ts b/lib/modules/datasource/maven/index.ts
index aa95a813cb..19e327ec7d 100644
--- a/lib/modules/datasource/maven/index.ts
+++ b/lib/modules/datasource/maven/index.ts
@@ -1,9 +1,9 @@
import is from '@sindresorhus/is';
import { DateTime } from 'luxon';
-import pAll from 'p-all';
import type { XmlDocument } from 'xmldoc';
import { logger } from '../../../logger';
import * as packageCache from '../../../util/cache/package';
+import * as p from '../../../util/promises';
import { newlineRegex, regEx } from '../../../util/regex';
import { ensureTrailingSlash } from '../../../util/url';
import mavenVersion from '../../versioning/maven';
@@ -268,7 +268,7 @@ export class MavenDatasource extends Datasource {
res !== 'not-found' && res !== 'error' ? release : null;
});
- await pAll(queue, { concurrency: 5 });
+ await p.all(queue);
if (!isCacheValid) {
// Store new TTL flag for 24 hours if the previous one is invalidated
diff --git a/lib/modules/datasource/nuget/v3.ts b/lib/modules/datasource/nuget/v3.ts
index d389392391..42b4b8e170 100644
--- a/lib/modules/datasource/nuget/v3.ts
+++ b/lib/modules/datasource/nuget/v3.ts
@@ -1,11 +1,11 @@
import is from '@sindresorhus/is';
-import pAll from 'p-all';
import semver from 'semver';
import { XmlDocument } from 'xmldoc';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
import * as packageCache from '../../../util/cache/package';
import { Http, HttpError } from '../../../util/http';
+import * as p from '../../../util/promises';
import { regEx } from '../../../util/regex';
import { ensureTrailingSlash } from '../../../util/url';
import type { Release, ReleaseResult } from '../types';
@@ -121,9 +121,7 @@ export async function getReleases(
const catalogPagesQueue = catalogPages.map(
(page) => (): Promise<CatalogEntry[]> => getCatalogEntry(http, page)
);
- const catalogEntries = (
- await pAll(catalogPagesQueue, { concurrency: 5 })
- ).flat();
+ const catalogEntries = (await p.all(catalogPagesQueue)).flat();
let homepage: string | null = null;
let latestStable: string | null = null;
diff --git a/lib/modules/datasource/packagist/index.ts b/lib/modules/datasource/packagist/index.ts
index 48a947215b..4bf0110466 100644
--- a/lib/modules/datasource/packagist/index.ts
+++ b/lib/modules/datasource/packagist/index.ts
@@ -1,11 +1,11 @@
import URL from 'url';
-import pAll from 'p-all';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
import * as packageCache from '../../../util/cache/package';
import { cache } from '../../../util/cache/package/decorator';
import * as hostRules from '../../../util/host-rules';
import type { HttpOptions } from '../../../util/http/types';
+import * as p from '../../../util/promises';
import { regEx } from '../../../util/regex';
import { ensureTrailingSlash, joinUrlParts } from '../../../util/url';
import * as composerVersioning from '../../versioning/composer';
@@ -182,7 +182,7 @@ export class PackagistDatasource extends Datasource {
(file) => (): Promise<PackagistFile> =>
this.getPackagistFile(regUrl, file)
);
- const resolvedFiles = await pAll(queue, { concurrency: 5 });
+ const resolvedFiles = await p.all(queue);
for (const res of resolvedFiles) {
for (const [name, val] of Object.entries(res.providers)) {
providerPackages[name] = val.sha256;
diff --git a/lib/modules/datasource/terraform-provider/index.ts b/lib/modules/datasource/terraform-provider/index.ts
index c2f1882dd7..c968a5eecf 100644
--- a/lib/modules/datasource/terraform-provider/index.ts
+++ b/lib/modules/datasource/terraform-provider/index.ts
@@ -1,10 +1,10 @@
// TODO: types (#7154)
/* eslint-disable @typescript-eslint/restrict-template-expressions */
import is from '@sindresorhus/is';
-import pMap from 'p-map';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
import { cache } from '../../../util/cache/package/decorator';
+import * as p from '../../../util/promises';
import { regEx } from '../../../util/regex';
import * as hashicorpVersioning from '../../versioning/hashicorp';
import { TerraformDatasource } from '../terraform-module/base';
@@ -230,7 +230,7 @@ export class TerraformProviderDatasource extends TerraformDatasource {
);
return null;
}
- const result = await pMap(
+ const result = await p.map(
builds.platforms,
async (platform) => {
const buildURL = `${backendURL}/${version}/download/${platform.os}/${platform.arch}`;
diff --git a/lib/modules/manager/hermit/artifacts.ts b/lib/modules/manager/hermit/artifacts.ts
index 40a337a937..d6e1adbc5f 100644
--- a/lib/modules/manager/hermit/artifacts.ts
+++ b/lib/modules/manager/hermit/artifacts.ts
@@ -1,10 +1,10 @@
-import pMap from 'p-map';
import upath from 'upath';
import { logger } from '../../../logger';
import { exec } from '../../../util/exec';
import type { ExecOptions } from '../../../util/exec/types';
import { localPathIsSymbolicLink, readLocalSymlink } from '../../../util/fs';
import { getRepoStatus } from '../../../util/git';
+import * as p from '../../../util/promises';
import type { UpdateArtifact, UpdateArtifactsResult } from '../types';
import type { ReadContentResult } from './types';
@@ -119,19 +119,18 @@ async function getUpdateResult(
);
// handle added files
- const added = await pMap(
+ const added = await p.map(
[...hermitChanges.created, ...hermitChanges.not_added],
async (path: string): Promise<UpdateArtifactsResult> => {
const contents = await getContent(path);
return getAddResult(path, contents);
- },
- { concurrency: 5 }
+ }
);
const deleted = hermitChanges.deleted.map(getDeleteResult);
- const modified = await pMap(
+ const modified = await p.map(
hermitChanges.modified,
async (path: string): Promise<UpdateArtifactsResult[]> => {
const contents = await getContent(path);
@@ -139,11 +138,10 @@ async function getUpdateResult(
getDeleteResult(path), // delete existing link
getAddResult(path, contents), // add a new link
];
- },
- { concurrency: 5 }
+ }
);
- const renamed = await pMap(
+ const renamed = await p.map(
hermitChanges.renamed,
async (renamed): Promise<UpdateArtifactsResult[]> => {
const from = renamed.from;
@@ -151,8 +149,7 @@ async function getUpdateResult(
const toContents = await getContent(to);
return [getDeleteResult(from), getAddResult(to, toContents)];
- },
- { concurrency: 5 }
+ }
);
return [
diff --git a/lib/modules/manager/terraform/lockfile/hash.ts b/lib/modules/manager/terraform/lockfile/hash.ts
index 47556aee44..d3b3757163 100644
--- a/lib/modules/manager/terraform/lockfile/hash.ts
+++ b/lib/modules/manager/terraform/lockfile/hash.ts
@@ -1,12 +1,12 @@
import crypto from 'crypto';
import extract from 'extract-zip';
-import pMap from 'p-map';
import upath from 'upath';
import { logger } from '../../../../logger';
import { cache } from '../../../../util/cache/package/decorator';
import * as fs from '../../../../util/fs';
import { ensureCacheDir } from '../../../../util/fs';
import { Http } from '../../../../util/http';
+import * as p from '../../../../util/promises';
import { regEx } from '../../../../util/regex';
import { TerraformProviderDatasource } from '../../../datasource/terraform-provider';
import type { TerraformBuild } from '../../../datasource/terraform-provider/types';
@@ -94,11 +94,9 @@ export class TerraformProviderHash {
const cacheDir = await ensureCacheDir('./others/terraform');
// for each build download ZIP, extract content and generate hash for all containing files
- return pMap(
- builds,
- (build) => this.calculateSingleHash(build, cacheDir),
- { concurrency: 4 } // allow to look up 4 builds for this version in parallel
- );
+ return p.map(builds, (build) => this.calculateSingleHash(build, cacheDir), {
+ concurrency: 4,
+ });
}
static async createHashes(
diff --git a/lib/modules/manager/terraform/lockfile/index.ts b/lib/modules/manager/terraform/lockfile/index.ts
index f2aee3eab9..69d7968d1a 100644
--- a/lib/modules/manager/terraform/lockfile/index.ts
+++ b/lib/modules/manager/terraform/lockfile/index.ts
@@ -1,6 +1,6 @@
import is from '@sindresorhus/is';
-import pMap from 'p-map';
import { logger } from '../../../../logger';
+import * as p from '../../../../util/promises';
import { GetPkgReleasesConfig, getPkgReleases } from '../../../datasource';
import { TerraformProviderDatasource } from '../../../datasource/terraform-provider';
import { get as getVersioning } from '../../../versioning';
@@ -19,7 +19,7 @@ import {
async function updateAllLocks(
locks: ProviderLock[]
): Promise<ProviderLockUpdate[]> {
- const updates = await pMap(
+ const updates = await p.map(
locks,
async (lock) => {
const updateConfig: GetPkgReleasesConfig = {
@@ -56,7 +56,7 @@ async function updateAllLocks(
};
return update;
},
- { concurrency: 4 } // allow to look up 4 lock in parallel
+ { concurrency: 4 }
);
return updates.filter(is.truthy);
diff --git a/lib/modules/platform/gitlab/index.ts b/lib/modules/platform/gitlab/index.ts
index 2069b99948..805b32f555 100644
--- a/lib/modules/platform/gitlab/index.ts
+++ b/lib/modules/platform/gitlab/index.ts
@@ -2,7 +2,6 @@ import URL from 'url';
import is from '@sindresorhus/is';
import delay from 'delay';
import JSON5 from 'json5';
-import pAll from 'p-all';
import semver from 'semver';
import { PlatformId } from '../../../constants';
import {
@@ -23,6 +22,7 @@ import * as git from '../../../util/git';
import * as hostRules from '../../../util/host-rules';
import { setBaseUrl } from '../../../util/http/gitlab';
import type { HttpResponse } from '../../../util/http/types';
+import * as p from '../../../util/promises';
import { regEx } from '../../../util/regex';
import { sanitize } from '../../../util/sanitize';
import {
@@ -1014,10 +1014,7 @@ export async function addReviewers(
// Gather the IDs for all the reviewers we want to add
let newReviewerIDs: number[];
try {
- newReviewerIDs = await pAll(
- newReviewers.map((r) => () => getUserID(r)),
- { concurrency: 5 }
- );
+ newReviewerIDs = await p.all(newReviewers.map((r) => () => getUserID(r)));
} catch (err) {
logger.warn({ err }, 'Failed to get IDs of the new reviewers');
return;
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
index 1350088968..2f5eca7ebe 100644
--- a/lib/util/http/github.ts
+++ b/lib/util/http/github.ts
@@ -1,6 +1,5 @@
import is from '@sindresorhus/is';
import { DateTime } from 'luxon';
-import pAll from 'p-all';
import { PlatformId } from '../../constants';
import {
PLATFORM_BAD_CREDENTIALS,
@@ -12,6 +11,7 @@ import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import { getCache } from '../cache/repository';
import { maskToken } from '../mask';
+import * as p from '../promises';
import { range } from '../range';
import { regEx } from '../regex';
import { joinUrlParts, parseLinkHeader, resolveBaseUrl } from '../url';
@@ -335,7 +335,7 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
);
}
);
- const pages = await pAll(queue, { concurrency: 5 });
+ const pages = await p.all(queue);
if (opts.paginationField && is.plainObject(result.body)) {
const paginatedResult = result.body[opts.paginationField];
if (is.array<T>(paginatedResult)) {
diff --git a/lib/util/promises.ts b/lib/util/promises.ts
new file mode 100644
index 0000000000..a0145ced65
--- /dev/null
+++ b/lib/util/promises.ts
@@ -0,0 +1,27 @@
+import pAll from 'p-all';
+import pMap from 'p-map';
+
+type PromiseFactory<T> = () => Promise<T>;
+
+export function all<T>(
+ tasks: PromiseFactory<T>[],
+ options?: pAll.Options
+): Promise<T[]> {
+ return pAll(tasks, {
+ concurrency: 5,
+ ...options,
+ stopOnError: true,
+ });
+}
+
+export function map<Element, NewElement>(
+ input: Iterable<Element>,
+ mapper: pMap.Mapper<Element, NewElement>,
+ options?: pMap.Options
+): Promise<NewElement[]> {
+ return pMap(input, mapper, {
+ concurrency: 5,
+ ...options,
+ stopOnError: true,
+ });
+}
diff --git a/lib/workers/repository/changelog/index.ts b/lib/workers/repository/changelog/index.ts
index 9a141dc048..8bc6c45559 100644
--- a/lib/workers/repository/changelog/index.ts
+++ b/lib/workers/repository/changelog/index.ts
@@ -1,4 +1,4 @@
-import pMap from 'p-map';
+import * as p from '../../../util/promises';
import {
containsTemplates,
exposedConfigOptions,
@@ -19,7 +19,7 @@ export async function embedChangelog(
export async function embedChangelogs(
branches: BranchUpgradeConfig[]
): Promise<void> {
- await pMap(branches, embedChangelog, { concurrency: 10 });
+ await p.map(branches, embedChangelog, { concurrency: 10 });
}
export function needsChangelogs(
diff --git a/lib/workers/repository/process/fetch.ts b/lib/workers/repository/process/fetch.ts
index f6b5deea3e..aed4095355 100644
--- a/lib/workers/repository/process/fetch.ts
+++ b/lib/workers/repository/process/fetch.ts
@@ -1,6 +1,5 @@
// TODO #7154
import is from '@sindresorhus/is';
-import pAll from 'p-all';
import { getManagerConfig, mergeChildConfig } from '../../../config';
import type { RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
@@ -15,6 +14,7 @@ import type {
import { ExternalHostError } from '../../../types/errors/external-host-error';
import { clone } from '../../../util/clone';
import { applyPackageRules } from '../../../util/package-rules';
+import * as p from '../../../util/promises';
import { PackageFiles } from '../package-files';
import { lookupUpdates } from './lookup';
import type { LookupUpdateConfig } from './lookup/types';
@@ -96,7 +96,7 @@ async function fetchManagerPackagerFileUpdates(
'fetchManagerPackagerFileUpdates starting with concurrency'
);
- pFile.deps = await pAll(queue, { concurrency: 5 });
+ pFile.deps = await p.all(queue);
logger.trace({ packageFile }, 'fetchManagerPackagerFileUpdates finished');
}
@@ -114,7 +114,7 @@ async function fetchManagerUpdates(
{ manager, queueLength: queue.length },
'fetchManagerUpdates starting'
);
- await pAll(queue, { concurrency: 5 });
+ await p.all(queue);
logger.trace({ manager }, 'fetchManagerUpdates finished');
}
diff --git a/lib/workers/repository/process/vulnerabilities.ts b/lib/workers/repository/process/vulnerabilities.ts
index 9cbe6c9da7..0171dee232 100644
--- a/lib/workers/repository/process/vulnerabilities.ts
+++ b/lib/workers/repository/process/vulnerabilities.ts
@@ -1,6 +1,5 @@
// TODO #7154
import { Ecosystem, Osv, OsvOffline } from '@jamiemagee/osv-offline';
-import pAll from 'p-all';
import { getManagerConfig, mergeChildConfig } from '../../../config';
import type { PackageRule, RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
@@ -8,6 +7,7 @@ import type {
PackageDependency,
PackageFile,
} from '../../../modules/manager/types';
+import * as p from '../../../util/promises';
export class Vulnerabilities {
private osvOffline: OsvOffline | undefined;
@@ -73,7 +73,7 @@ export class Vulnerabilities {
{ manager, queueLength: queue.length },
'fetchManagerUpdates starting'
);
- await pAll(queue, { concurrency: 5 });
+ await p.all(queue);
logger.trace({ manager }, 'fetchManagerUpdates finished');
}
@@ -94,9 +94,7 @@ export class Vulnerabilities {
'fetchManagerPackagerFileUpdates starting with concurrency'
);
- config.packageRules?.push(
- ...(await pAll(queue, { concurrency: 5 })).flat()
- );
+ config.packageRules?.push(...(await p.all(queue)).flat());
logger.trace({ packageFile }, 'fetchManagerPackagerFileUpdates finished');
}
--
GitLab