refactor(packagist): Improve fetch concurrency (#19981)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

lib/modules/datasource/packagist/index.ts

@@ -118,39 +118,40 @@ export class PackagistDatasource extends Datasource {
       packages,
       providersUrl,
       providersLazyUrl,
-      files,
-      includesFiles,
-      providerPackages,
+      providerIncludes,
+      includes,
+      providers,
     } = registryMeta;
-    if (files) {
-      const queue = files.map(
-        (file) => (): Promise<PackagistFile> =>
-          this.getPackagistFile(regUrl, file)
-      );
-      const resolvedFiles = await p.all(queue);
-      for (const res of resolvedFiles) {
+
+    const includesPackages: Record<string, ReleaseResult> = {};
+
+    const tasks: (() => Promise<void>)[] = [];
+
+    for (const file of providerIncludes) {
+      tasks.push(async () => {
+        const res = await this.getPackagistFile(regUrl, file);
         for (const [name, val] of Object.entries(res.providers)) {
-          providerPackages[name] = val.sha256;
+          providers[name] = val.sha256;
         }
-      }
+      });
     }
-    const includesPackages: Record<string, ReleaseResult> = {};
-    if (includesFiles) {
-      for (const file of includesFiles) {
+
+    for (const file of includes) {
+      tasks.push(async () => {
         const res = await this.getPackagistFile(regUrl, file);
-        if (res.packages) {
-          for (const [key, val] of Object.entries(res.packages)) {
-            const dep = PackagistDatasource.extractDepReleases(val);
-            includesPackages[key] = dep;
-          }
+        for (const [key, val] of Object.entries(res.packages ?? {})) {
+          includesPackages[key] = PackagistDatasource.extractDepReleases(val);
         }
-      }
+      });
     }
+
+    await p.all(tasks);
+
     const allPackages: AllPackages = {
       packages: packages as never, // TODO: fix types (#9610)
       providersUrl,
       providersLazyUrl,
-      providerPackages,
+      providers,
       includesPackages,
     };
     return allPackages;
@@ -196,7 +197,7 @@ export class PackagistDatasource extends Datasource {
         packages,
         providersUrl,
         providersLazyUrl,
-        providerPackages,
+        providers,
         includesPackages,
       } = allPackages;
       if (packages?.[packageName]) {
@@ -209,7 +210,7 @@ export class PackagistDatasource extends Datasource {
         return includesPackages[packageName];
       }
       let pkgUrl: string;
-      const hash = providerPackages[packageName];
+      const hash = providers[packageName];
       if (providersUrl && !is.undefined(hash)) {
         let url = providersUrl.replace('%package%', packageName);
         if (hash) {

lib/modules/datasource/packagist/schema.ts

@@ -182,7 +182,8 @@ const RegistryMetaIncludes = RegistryMetaFiles.transform(
   }
 )
   .nullable()
-  .catch(null);
+  .catch(null)
+  .transform((xs) => xs ?? []);
 
 export const RegistryMeta = z
   .object({
@@ -201,17 +202,17 @@ export const RegistryMeta = z
   })
   .transform(
     ({
-      ['packages']: packages,
-      ['includes']: includesFiles,
-      ['providers']: providerPackages,
-      ['provider-includes']: files,
-      ['providers-url']: providersUrl,
-      ['providers-lazy-url']: providersLazyUrl,
+      packages,
+      includes,
+      providers,
+      'provider-includes': providerIncludes,
+      'providers-url': providersUrl,
+      'providers-lazy-url': providersLazyUrl,
     }) => ({
       packages,
-      includesFiles,
-      providerPackages,
-      files,
+      includes,
+      providers,
+      providerIncludes,
       providersUrl,
       providersLazyUrl,
     })

lib/modules/datasource/packagist/types.ts

@@ -22,7 +22,7 @@ export interface AllPackages {
   packages: Record<string, RegistryFile>;
   providersUrl: string | null;
   providersLazyUrl: string | null;
-  providerPackages: Record<string, string | null>;
+  providers: Record<string, string | null>;
 
   includesPackages: Record<string, ReleaseResult>;
 }