diff --git a/lib/modules/datasource/galaxy-collection/index.ts b/lib/modules/datasource/galaxy-collection/index.ts index 1839445f1e64392df1a5b5562e3d5fd02120cb0a..0d89d1b1c04612c96f8ff2738d23111afc5b9f2e 100644 --- a/lib/modules/datasource/galaxy-collection/index.ts +++ b/lib/modules/datasource/galaxy-collection/index.ts @@ -1,8 +1,8 @@ import is from '@sindresorhus/is'; -import pMap from 'p-map'; import { logger } from '../../../logger'; import { cache } from '../../../util/cache/package/decorator'; import type { HttpResponse } from '../../../util/http/types'; +import * as p from '../../../util/promises'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; import type { @@ -76,7 +76,7 @@ export class GalaxyCollectionDatasource extends Datasource { let newestVersionDetails: VersionsDetailResult | undefined; // asynchronously get release details - const enrichedReleases: (Release | null)[] = await pMap( + const enrichedReleases: (Release | null)[] = await p.map( releases, (basicRelease) => this.http @@ -108,8 +108,7 @@ export class GalaxyCollectionDatasource extends Datasource { ); return null; } - }), - { concurrency: 5 } // allow 5 requests at maximum in parallel + }) ); // filter failed versions const filteredReleases = enrichedReleases.filter(is.truthy); diff --git a/lib/modules/datasource/go/releases-goproxy.ts b/lib/modules/datasource/go/releases-goproxy.ts index 0f98bbfbf93aef692b6d2fdccaf937af60a34eca..89bcbe95d04fd99d27f29fa96a8b85c3988ab343 100644 --- a/lib/modules/datasource/go/releases-goproxy.ts +++ b/lib/modules/datasource/go/releases-goproxy.ts @@ -1,8 +1,8 @@ import is from '@sindresorhus/is'; import moo from 'moo'; -import pAll from 'p-all'; import { logger } from '../../../logger'; import { cache } from '../../../util/cache/package/decorator'; +import * as p from '../../../util/promises'; import { regEx } from '../../../util/regex'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; @@ -60,7 +60,7 @@ export class GoProxyDatasource extends Datasource { return { version }; } }); - const releases = await pAll(queue, { concurrency: 5 }); + const releases = await p.all(queue); if (releases.length) { const datasource = await BaseGoDatasource.getDatasource(packageName); const sourceUrl = getSourceUrl(datasource); diff --git a/lib/modules/datasource/maven/index.ts b/lib/modules/datasource/maven/index.ts index aa95a813cb39dd33ad62e124983092befc3e16c0..19e327ec7dd36505519cfd12ee95b7334c98f759 100644 --- a/lib/modules/datasource/maven/index.ts +++ b/lib/modules/datasource/maven/index.ts @@ -1,9 +1,9 @@ import is from '@sindresorhus/is'; import { DateTime } from 'luxon'; -import pAll from 'p-all'; import type { XmlDocument } from 'xmldoc'; import { logger } from '../../../logger'; import * as packageCache from '../../../util/cache/package'; +import * as p from '../../../util/promises'; import { newlineRegex, regEx } from '../../../util/regex'; import { ensureTrailingSlash } from '../../../util/url'; import mavenVersion from '../../versioning/maven'; @@ -268,7 +268,7 @@ export class MavenDatasource extends Datasource { res !== 'not-found' && res !== 'error' ? release : null; }); - await pAll(queue, { concurrency: 5 }); + await p.all(queue); if (!isCacheValid) { // Store new TTL flag for 24 hours if the previous one is invalidated diff --git a/lib/modules/datasource/nuget/v3.ts b/lib/modules/datasource/nuget/v3.ts index d3893923912941a60f3457b1fb8e789b815d159f..42b4b8e170f57e6a6f7ec3cc33a50fe45c549037 100644 --- a/lib/modules/datasource/nuget/v3.ts +++ b/lib/modules/datasource/nuget/v3.ts @@ -1,11 +1,11 @@ import is from '@sindresorhus/is'; -import pAll from 'p-all'; import semver from 'semver'; import { XmlDocument } from 'xmldoc'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import * as packageCache from '../../../util/cache/package'; import { Http, HttpError } from '../../../util/http'; +import * as p from '../../../util/promises'; import { regEx } from '../../../util/regex'; import { ensureTrailingSlash } from '../../../util/url'; import type { Release, ReleaseResult } from '../types'; @@ -121,9 +121,7 @@ export async function getReleases( const catalogPagesQueue = catalogPages.map( (page) => (): Promise<CatalogEntry[]> => getCatalogEntry(http, page) ); - const catalogEntries = ( - await pAll(catalogPagesQueue, { concurrency: 5 }) - ).flat(); + const catalogEntries = (await p.all(catalogPagesQueue)).flat(); let homepage: string | null = null; let latestStable: string | null = null; diff --git a/lib/modules/datasource/packagist/index.ts b/lib/modules/datasource/packagist/index.ts index 48a947215bb221ec1cab70dededc8a2ac96b0380..4bf01104664ba59b39a4e79d99a9263e14dd6d2e 100644 --- a/lib/modules/datasource/packagist/index.ts +++ b/lib/modules/datasource/packagist/index.ts @@ -1,11 +1,11 @@ import URL from 'url'; -import pAll from 'p-all'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import * as packageCache from '../../../util/cache/package'; import { cache } from '../../../util/cache/package/decorator'; import * as hostRules from '../../../util/host-rules'; import type { HttpOptions } from '../../../util/http/types'; +import * as p from '../../../util/promises'; import { regEx } from '../../../util/regex'; import { ensureTrailingSlash, joinUrlParts } from '../../../util/url'; import * as composerVersioning from '../../versioning/composer'; @@ -182,7 +182,7 @@ export class PackagistDatasource extends Datasource { (file) => (): Promise<PackagistFile> => this.getPackagistFile(regUrl, file) ); - const resolvedFiles = await pAll(queue, { concurrency: 5 }); + const resolvedFiles = await p.all(queue); for (const res of resolvedFiles) { for (const [name, val] of Object.entries(res.providers)) { providerPackages[name] = val.sha256; diff --git a/lib/modules/datasource/terraform-provider/index.ts b/lib/modules/datasource/terraform-provider/index.ts index c2f1882dd74a6f99ad178fc04ba63eed4b75939f..c968a5eecf7e20288efcee0457cfd4c25bf40b06 100644 --- a/lib/modules/datasource/terraform-provider/index.ts +++ b/lib/modules/datasource/terraform-provider/index.ts @@ -1,10 +1,10 @@ // TODO: types (#7154) /* eslint-disable @typescript-eslint/restrict-template-expressions */ import is from '@sindresorhus/is'; -import pMap from 'p-map'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import { cache } from '../../../util/cache/package/decorator'; +import * as p from '../../../util/promises'; import { regEx } from '../../../util/regex'; import * as hashicorpVersioning from '../../versioning/hashicorp'; import { TerraformDatasource } from '../terraform-module/base'; @@ -230,7 +230,7 @@ export class TerraformProviderDatasource extends TerraformDatasource { ); return null; } - const result = await pMap( + const result = await p.map( builds.platforms, async (platform) => { const buildURL = `${backendURL}/${version}/download/${platform.os}/${platform.arch}`; diff --git a/lib/modules/manager/hermit/artifacts.ts b/lib/modules/manager/hermit/artifacts.ts index 40a337a937cd293e89607d237dc8732292cceea5..d6e1adbc5fbca834214cfa231aa7a82c50e3a095 100644 --- a/lib/modules/manager/hermit/artifacts.ts +++ b/lib/modules/manager/hermit/artifacts.ts @@ -1,10 +1,10 @@ -import pMap from 'p-map'; import upath from 'upath'; import { logger } from '../../../logger'; import { exec } from '../../../util/exec'; import type { ExecOptions } from '../../../util/exec/types'; import { localPathIsSymbolicLink, readLocalSymlink } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; +import * as p from '../../../util/promises'; import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; import type { ReadContentResult } from './types'; @@ -119,19 +119,18 @@ async function getUpdateResult( ); // handle added files - const added = await pMap( + const added = await p.map( [...hermitChanges.created, ...hermitChanges.not_added], async (path: string): Promise<UpdateArtifactsResult> => { const contents = await getContent(path); return getAddResult(path, contents); - }, - { concurrency: 5 } + } ); const deleted = hermitChanges.deleted.map(getDeleteResult); - const modified = await pMap( + const modified = await p.map( hermitChanges.modified, async (path: string): Promise<UpdateArtifactsResult[]> => { const contents = await getContent(path); @@ -139,11 +138,10 @@ async function getUpdateResult( getDeleteResult(path), // delete existing link getAddResult(path, contents), // add a new link ]; - }, - { concurrency: 5 } + } ); - const renamed = await pMap( + const renamed = await p.map( hermitChanges.renamed, async (renamed): Promise<UpdateArtifactsResult[]> => { const from = renamed.from; @@ -151,8 +149,7 @@ async function getUpdateResult( const toContents = await getContent(to); return [getDeleteResult(from), getAddResult(to, toContents)]; - }, - { concurrency: 5 } + } ); return [ diff --git a/lib/modules/manager/terraform/lockfile/hash.ts b/lib/modules/manager/terraform/lockfile/hash.ts index 47556aee4491fa08dae6cdabfa8fa920d1f98889..d3b37571630a2dc862c31127e306c185c555b6a4 100644 --- a/lib/modules/manager/terraform/lockfile/hash.ts +++ b/lib/modules/manager/terraform/lockfile/hash.ts @@ -1,12 +1,12 @@ import crypto from 'crypto'; import extract from 'extract-zip'; -import pMap from 'p-map'; import upath from 'upath'; import { logger } from '../../../../logger'; import { cache } from '../../../../util/cache/package/decorator'; import * as fs from '../../../../util/fs'; import { ensureCacheDir } from '../../../../util/fs'; import { Http } from '../../../../util/http'; +import * as p from '../../../../util/promises'; import { regEx } from '../../../../util/regex'; import { TerraformProviderDatasource } from '../../../datasource/terraform-provider'; import type { TerraformBuild } from '../../../datasource/terraform-provider/types'; @@ -94,11 +94,9 @@ export class TerraformProviderHash { const cacheDir = await ensureCacheDir('./others/terraform'); // for each build download ZIP, extract content and generate hash for all containing files - return pMap( - builds, - (build) => this.calculateSingleHash(build, cacheDir), - { concurrency: 4 } // allow to look up 4 builds for this version in parallel - ); + return p.map(builds, (build) => this.calculateSingleHash(build, cacheDir), { + concurrency: 4, + }); } static async createHashes( diff --git a/lib/modules/manager/terraform/lockfile/index.ts b/lib/modules/manager/terraform/lockfile/index.ts index f2aee3eab9c2c49a0d0ec9564397348b5a57199e..69d7968d1ac5e62de9ea18a6c5fbcebac61e8559 100644 --- a/lib/modules/manager/terraform/lockfile/index.ts +++ b/lib/modules/manager/terraform/lockfile/index.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; -import pMap from 'p-map'; import { logger } from '../../../../logger'; +import * as p from '../../../../util/promises'; import { GetPkgReleasesConfig, getPkgReleases } from '../../../datasource'; import { TerraformProviderDatasource } from '../../../datasource/terraform-provider'; import { get as getVersioning } from '../../../versioning'; @@ -19,7 +19,7 @@ import { async function updateAllLocks( locks: ProviderLock[] ): Promise<ProviderLockUpdate[]> { - const updates = await pMap( + const updates = await p.map( locks, async (lock) => { const updateConfig: GetPkgReleasesConfig = { @@ -56,7 +56,7 @@ async function updateAllLocks( }; return update; }, - { concurrency: 4 } // allow to look up 4 lock in parallel + { concurrency: 4 } ); return updates.filter(is.truthy); diff --git a/lib/modules/platform/gitlab/index.ts b/lib/modules/platform/gitlab/index.ts index 2069b9994873fd9edf9e7fea0bfd2f2ad297dbd3..805b32f55531c9e861f0beef435a888b8c83c673 100644 --- a/lib/modules/platform/gitlab/index.ts +++ b/lib/modules/platform/gitlab/index.ts @@ -2,7 +2,6 @@ import URL from 'url'; import is from '@sindresorhus/is'; import delay from 'delay'; import JSON5 from 'json5'; -import pAll from 'p-all'; import semver from 'semver'; import { PlatformId } from '../../../constants'; import { @@ -23,6 +22,7 @@ import * as git from '../../../util/git'; import * as hostRules from '../../../util/host-rules'; import { setBaseUrl } from '../../../util/http/gitlab'; import type { HttpResponse } from '../../../util/http/types'; +import * as p from '../../../util/promises'; import { regEx } from '../../../util/regex'; import { sanitize } from '../../../util/sanitize'; import { @@ -1014,10 +1014,7 @@ export async function addReviewers( // Gather the IDs for all the reviewers we want to add let newReviewerIDs: number[]; try { - newReviewerIDs = await pAll( - newReviewers.map((r) => () => getUserID(r)), - { concurrency: 5 } - ); + newReviewerIDs = await p.all(newReviewers.map((r) => () => getUserID(r))); } catch (err) { logger.warn({ err }, 'Failed to get IDs of the new reviewers'); return; diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index 1350088968a18aa210b360c902436084a1dbad2b..2f5eca7ebeb3371280049d80c46a995c34a5c07f 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -1,6 +1,5 @@ import is from '@sindresorhus/is'; import { DateTime } from 'luxon'; -import pAll from 'p-all'; import { PlatformId } from '../../constants'; import { PLATFORM_BAD_CREDENTIALS, @@ -12,6 +11,7 @@ import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import { getCache } from '../cache/repository'; import { maskToken } from '../mask'; +import * as p from '../promises'; import { range } from '../range'; import { regEx } from '../regex'; import { joinUrlParts, parseLinkHeader, resolveBaseUrl } from '../url'; @@ -335,7 +335,7 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> { ); } ); - const pages = await pAll(queue, { concurrency: 5 }); + const pages = await p.all(queue); if (opts.paginationField && is.plainObject(result.body)) { const paginatedResult = result.body[opts.paginationField]; if (is.array<T>(paginatedResult)) { diff --git a/lib/util/promises.ts b/lib/util/promises.ts new file mode 100644 index 0000000000000000000000000000000000000000..a0145ced65fe181efbc95b0b2d7fcbb0fea67e43 --- /dev/null +++ b/lib/util/promises.ts @@ -0,0 +1,27 @@ +import pAll from 'p-all'; +import pMap from 'p-map'; + +type PromiseFactory<T> = () => Promise<T>; + +export function all<T>( + tasks: PromiseFactory<T>[], + options?: pAll.Options +): Promise<T[]> { + return pAll(tasks, { + concurrency: 5, + ...options, + stopOnError: true, + }); +} + +export function map<Element, NewElement>( + input: Iterable<Element>, + mapper: pMap.Mapper<Element, NewElement>, + options?: pMap.Options +): Promise<NewElement[]> { + return pMap(input, mapper, { + concurrency: 5, + ...options, + stopOnError: true, + }); +} diff --git a/lib/workers/repository/changelog/index.ts b/lib/workers/repository/changelog/index.ts index 9a141dc04810b269900a48f08c9011e8cbad7eb2..8bc6c4555902230dcc0ae52c9f2b0b337ac0f7e5 100644 --- a/lib/workers/repository/changelog/index.ts +++ b/lib/workers/repository/changelog/index.ts @@ -1,4 +1,4 @@ -import pMap from 'p-map'; +import * as p from '../../../util/promises'; import { containsTemplates, exposedConfigOptions, @@ -19,7 +19,7 @@ export async function embedChangelog( export async function embedChangelogs( branches: BranchUpgradeConfig[] ): Promise<void> { - await pMap(branches, embedChangelog, { concurrency: 10 }); + await p.map(branches, embedChangelog, { concurrency: 10 }); } export function needsChangelogs( diff --git a/lib/workers/repository/process/fetch.ts b/lib/workers/repository/process/fetch.ts index f6b5deea3e19d1caf7727699931f00b14dc0b295..aed40953552f381895ddb46e4b92ef6ba871aaad 100644 --- a/lib/workers/repository/process/fetch.ts +++ b/lib/workers/repository/process/fetch.ts @@ -1,6 +1,5 @@ // TODO #7154 import is from '@sindresorhus/is'; -import pAll from 'p-all'; import { getManagerConfig, mergeChildConfig } from '../../../config'; import type { RenovateConfig } from '../../../config/types'; import { logger } from '../../../logger'; @@ -15,6 +14,7 @@ import type { import { ExternalHostError } from '../../../types/errors/external-host-error'; import { clone } from '../../../util/clone'; import { applyPackageRules } from '../../../util/package-rules'; +import * as p from '../../../util/promises'; import { PackageFiles } from '../package-files'; import { lookupUpdates } from './lookup'; import type { LookupUpdateConfig } from './lookup/types'; @@ -96,7 +96,7 @@ async function fetchManagerPackagerFileUpdates( 'fetchManagerPackagerFileUpdates starting with concurrency' ); - pFile.deps = await pAll(queue, { concurrency: 5 }); + pFile.deps = await p.all(queue); logger.trace({ packageFile }, 'fetchManagerPackagerFileUpdates finished'); } @@ -114,7 +114,7 @@ async function fetchManagerUpdates( { manager, queueLength: queue.length }, 'fetchManagerUpdates starting' ); - await pAll(queue, { concurrency: 5 }); + await p.all(queue); logger.trace({ manager }, 'fetchManagerUpdates finished'); } diff --git a/lib/workers/repository/process/vulnerabilities.ts b/lib/workers/repository/process/vulnerabilities.ts index 9cbe6c9da79945392b14e651b2cb874da2266493..0171dee2325c783b05f0d7677c76da0cbc10284f 100644 --- a/lib/workers/repository/process/vulnerabilities.ts +++ b/lib/workers/repository/process/vulnerabilities.ts @@ -1,6 +1,5 @@ // TODO #7154 import { Ecosystem, Osv, OsvOffline } from '@jamiemagee/osv-offline'; -import pAll from 'p-all'; import { getManagerConfig, mergeChildConfig } from '../../../config'; import type { PackageRule, RenovateConfig } from '../../../config/types'; import { logger } from '../../../logger'; @@ -8,6 +7,7 @@ import type { PackageDependency, PackageFile, } from '../../../modules/manager/types'; +import * as p from '../../../util/promises'; export class Vulnerabilities { private osvOffline: OsvOffline | undefined; @@ -73,7 +73,7 @@ export class Vulnerabilities { { manager, queueLength: queue.length }, 'fetchManagerUpdates starting' ); - await pAll(queue, { concurrency: 5 }); + await p.all(queue); logger.trace({ manager }, 'fetchManagerUpdates finished'); } @@ -94,9 +94,7 @@ export class Vulnerabilities { 'fetchManagerPackagerFileUpdates starting with concurrency' ); - config.packageRules?.push( - ...(await pAll(queue, { concurrency: 5 })).flat() - ); + config.packageRules?.push(...(await p.all(queue)).flat()); logger.trace({ packageFile }, 'fetchManagerPackagerFileUpdates finished'); }