diff --git a/lib/util/promises.ts b/lib/util/promises.ts index 736437bc1db835feb1c93eb3e434e1a4292e6ab8..1b614aa036ab310dbe0b017c0a178da965abc15e 100644 --- a/lib/util/promises.ts +++ b/lib/util/promises.ts @@ -10,7 +10,7 @@ function isExternalHostError(err: any): err is ExternalHostError { return err instanceof ExternalHostError; } -function handleMultipleErrors(errors: Error[]): never { +export function handleMultipleErrors(errors: Error[]): never { const hostError = errors.find(isExternalHostError); if (hostError) { throw hostError; diff --git a/lib/workers/repository/process/fetch.spec.ts b/lib/workers/repository/process/fetch.spec.ts index 0ad23785fe6c00038193dea24728076eaff2b61e..f891c6e81e6fd868e7c181f5e2fe7f70a05322c0 100644 --- a/lib/workers/repository/process/fetch.spec.ts +++ b/lib/workers/repository/process/fetch.spec.ts @@ -4,6 +4,7 @@ import { getConfig } from '../../../config/defaults'; import { MavenDatasource } from '../../../modules/datasource/maven'; import type { PackageFile } from '../../../modules/manager/types'; import { ExternalHostError } from '../../../types/errors/external-host-error'; +import { Result } from '../../../util/result'; import { fetchUpdates } from './fetch'; import * as lookup from './lookup'; @@ -197,7 +198,7 @@ describe('workers/repository/process/fetch', () => { }, ], }; - lookupUpdates.mockRejectedValueOnce(new Error('some error')); + lookupUpdates.mockResolvedValueOnce(Result.err(new Error('some error'))); await expect( fetchUpdates({ ...config, repoIsOnboarded: true }, packageFiles), @@ -214,7 +215,7 @@ describe('workers/repository/process/fetch', () => { }, ], }; - lookupUpdates.mockRejectedValueOnce(new Error('some error')); + lookupUpdates.mockResolvedValueOnce(Result.err(new Error('some error'))); await expect( fetchUpdates({ ...config, repoIsOnboarded: true }, packageFiles), diff --git a/lib/workers/repository/process/fetch.ts b/lib/workers/repository/process/fetch.ts index 6b3fcb3860b0e4e84176c2a2020af851ce2b4244..e9b17a4470f6bc91562c1a52b722af162cf6f711 100644 --- a/lib/workers/repository/process/fetch.ts +++ b/lib/workers/repository/process/fetch.ts @@ -21,6 +21,13 @@ import type { LookupUpdateConfig, UpdateResult } from './lookup/types'; type LookupResult = Result<PackageDependency, Error>; +interface LookupTaskResult { + packageFile: PackageFile; + result: LookupResult; +} + +type LookupTask = Promise<LookupTaskResult>; + async function lookup( packageFileConfig: RenovateConfig & PackageFile, indep: PackageDependency, @@ -106,63 +113,86 @@ async function lookup( }); } -async function fetchManagerPackagerFileUpdates( +function createLookupTasks( config: RenovateConfig, - managerConfig: RenovateConfig, - pFile: PackageFile, -): Promise<void> { - const { packageFile } = pFile; - const packageFileConfig = mergeChildConfig(managerConfig, pFile); - if (pFile.extractedConstraints) { - packageFileConfig.constraints = { - ...pFile.extractedConstraints, - ...config.constraints, - }; + managerPackageFiles: Record<string, PackageFile[]>, +): LookupTask[] { + const lookupTasks: LookupTask[] = []; + + for (const [manager, packageFiles] of Object.entries(managerPackageFiles)) { + const managerConfig = getManagerConfig(config, manager); + + for (const packageFile of packageFiles) { + const packageFileConfig = mergeChildConfig(managerConfig, packageFile); + if (packageFile.extractedConstraints) { + packageFileConfig.constraints = { + ...packageFile.extractedConstraints, + ...config.constraints, + }; + } + + for (const dep of packageFile.deps) { + lookupTasks.push( + lookup(packageFileConfig, dep).then((result) => ({ + packageFile, + result, + })), + ); + } + } } - const { manager } = packageFileConfig; - const queue = pFile.deps.map( - (dep) => async (): Promise<PackageDependency> => { - const updates = await lookup(packageFileConfig, dep); - return updates.unwrapOrThrow(); - }, - ); - logger.trace( - { manager, packageFile, queueLength: queue.length }, - 'fetchManagerPackagerFileUpdates starting with concurrency', - ); - pFile.deps = await p.all(queue); - logger.trace({ packageFile }, 'fetchManagerPackagerFileUpdates finished'); -} - -async function fetchManagerUpdates( - config: RenovateConfig, - packageFiles: Record<string, PackageFile[]>, - manager: string, -): Promise<void> { - const managerConfig = getManagerConfig(config, manager); - const queue = packageFiles[manager].map( - (pFile) => (): Promise<void> => - fetchManagerPackagerFileUpdates(config, managerConfig, pFile), - ); - logger.trace( - { manager, queueLength: queue.length }, - 'fetchManagerUpdates starting', - ); - await p.all(queue); - logger.trace({ manager }, 'fetchManagerUpdates finished'); + return lookupTasks; } export async function fetchUpdates( config: RenovateConfig, - packageFiles: Record<string, PackageFile[]>, + managerPackageFiles: Record<string, PackageFile[]>, ): Promise<void> { - const managers = Object.keys(packageFiles); - const allManagerJobs = managers.map((manager) => - fetchManagerUpdates(config, packageFiles, manager), + logger.debug( + { baseBranch: config.baseBranch }, + 'Starting package releases lookups', ); - await Promise.all(allManagerJobs); - PackageFiles.add(config.baseBranch!, { ...packageFiles }); + + const allTasks = createLookupTasks(config, managerPackageFiles); + + const fetchResults = await Promise.all(allTasks); + + const errors: Error[] = []; + + type PackageDeps = WeakMap<PackageFile, PackageDependency[]>; + const packageDeps: PackageDeps = new WeakMap(); + + // Separate good results from errors + for (const { packageFile, result } of fetchResults) { + const { val: dep, err } = result.unwrap(); + if (dep) { + let deps = packageDeps.get(packageFile); + if (!deps) { + deps = []; + packageDeps.set(packageFile, deps); + } + deps.push(dep); + } else { + errors.push(err); + } + } + + if (errors.length) { + p.handleMultipleErrors(errors); + } + + // Assign fetched deps back to packageFiles + for (const packageFiles of Object.values(managerPackageFiles)) { + for (const packageFile of packageFiles) { + const packageFileDeps = packageDeps.get(packageFile); + if (packageFileDeps) { + packageFile.deps = packageFileDeps; + } + } + } + + PackageFiles.add(config.baseBranch!, { ...managerPackageFiles }); logger.debug( { baseBranch: config.baseBranch }, 'Package releases lookups complete',