From e97cde57ddec63480bb8e69f3e0887166107e0ee Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Sat, 25 Feb 2023 22:33:47 +0300
Subject: [PATCH] refactor(packagist): Reorganize datasource methods (#20595)
---
lib/modules/datasource/packagist/index.ts | 181 +++++++++---------
.../datasource/packagist/schema.spec.ts | 1 +
lib/modules/datasource/packagist/schema.ts | 89 +++++----
lib/modules/datasource/packagist/types.ts | 9 -
4 files changed, 140 insertions(+), 140 deletions(-)
delete mode 100644 lib/modules/datasource/packagist/types.ts
diff --git a/lib/modules/datasource/packagist/index.ts b/lib/modules/datasource/packagist/index.ts
index b82cde4b59..14e5214001 100644
--- a/lib/modules/datasource/packagist/index.ts
+++ b/lib/modules/datasource/packagist/index.ts
@@ -1,3 +1,4 @@
+import type { z } from 'zod';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
import { cache } from '../../../util/cache/package/decorator';
@@ -8,9 +9,14 @@ import { joinUrlParts, resolveBaseUrl } from '../../../util/url';
import * as composerVersioning from '../../versioning/composer';
import { Datasource } from '../datasource';
import type { GetReleasesConfig, ReleaseResult } from '../types';
-import * as schema from './schema';
-import { extractDepReleases } from './schema';
-import type { PackagistFile, RegistryFile } from './types';
+import {
+ PackagesResponse,
+ PackagistFile,
+ RegistryFile,
+ RegistryMeta,
+ extractDepReleases,
+ parsePackagesResponses,
+} from './schema';
export class PackagistDatasource extends Datasource {
static readonly id = 'packagist';
@@ -34,12 +40,23 @@ export class PackagistDatasource extends Datasource {
return username && password ? { username, password } : {};
}
- private async getRegistryMeta(regUrl: string): Promise<schema.RegistryMeta> {
- const url = resolveBaseUrl(regUrl, 'packages.json');
+ private async getJson<T, U extends z.ZodSchema<T>>(
+ url: string,
+ schema: U
+ ): Promise<z.infer<typeof schema>> {
const opts = PackagistDatasource.getHostOpts(url);
const { body } = await this.http.getJson(url, opts);
- const meta = schema.RegistryMeta.parse(body);
- return meta;
+ return schema.parse(body);
+ }
+
+ @cache({
+ namespace: `datasource-${PackagistDatasource.id}`,
+ key: (regUrl: string) => `getRegistryMeta:${regUrl}`,
+ })
+ async getRegistryMeta(regUrl: string): Promise<RegistryMeta> {
+ const url = resolveBaseUrl(regUrl, 'packages.json');
+ const result = await this.getJson(url, RegistryMeta);
+ return result;
}
private static isPrivatePackage(regUrl: string): boolean {
@@ -70,62 +87,30 @@ export class PackagistDatasource extends Datasource {
regFile: RegistryFile
): Promise<PackagistFile> {
const url = PackagistDatasource.getPackagistFileUrl(regUrl, regFile);
- const opts = PackagistDatasource.getHostOpts(regUrl);
- const { body: packagistFile } = await this.http.getJson<PackagistFile>(
- url,
- opts
- );
+ const packagistFile = await this.getJson(url, PackagistFile);
return packagistFile;
}
- @cache({
- namespace: `datasource-${PackagistDatasource.id}`,
- key: (regUrl: string) => regUrl,
- })
- async getAllPackages(regUrl: string): Promise<schema.AllPackages> {
- const registryMeta = await this.getRegistryMeta(regUrl);
-
- const {
- packages,
- providersUrl,
- providersLazyUrl,
- files,
- includesFiles,
- providerPackages,
- } = registryMeta;
-
- const includesPackages: schema.AllPackages['includesPackages'] = {};
-
- const tasks: (() => Promise<void>)[] = [];
-
- for (const file of files) {
- tasks.push(async () => {
- const res = await this.getPackagistFile(regUrl, file);
- for (const [name, val] of Object.entries(res.providers)) {
- providerPackages[name] = val.sha256;
- }
- });
- }
-
- for (const file of includesFiles) {
- tasks.push(async () => {
- const res = await this.getPackagistFile(regUrl, file);
- for (const [key, val] of Object.entries(res.packages ?? {})) {
- includesPackages[key] = extractDepReleases(val);
- }
- });
- }
-
- await p.all(tasks);
+ async fetchProviderPackages(
+ regUrl: string,
+ meta: RegistryMeta
+ ): Promise<void> {
+ await p.map(meta.files, async (file) => {
+ const res = await this.getPackagistFile(regUrl, file);
+ Object.assign(meta.providerPackages, res.providers);
+ });
+ }
- const allPackages: schema.AllPackages = {
- packages,
- providersUrl,
- providersLazyUrl,
- providerPackages,
- includesPackages,
- };
- return allPackages;
+ async fetchIncludesPackages(
+ regUrl: string,
+ meta: RegistryMeta
+ ): Promise<void> {
+ await p.map(meta.includesFiles, async (file) => {
+ const res = await this.getPackagistFile(regUrl, file);
+ for (const [key, val] of Object.entries(res.packages)) {
+ meta.includesPackages[key] = extractDepReleases(val);
+ }
+ });
}
@cache({
@@ -140,7 +125,34 @@ export class PackagistDatasource extends Datasource {
const results = await p.map([pkgUrl, devUrl], (url) =>
this.http.getJson(url).then(({ body }) => body)
);
- return schema.parsePackagesResponses(name, results);
+ return parsePackagesResponses(name, results);
+ }
+
+ public getPkgUrl(
+ packageName: string,
+ registryUrl: string,
+ registryMeta: RegistryMeta
+ ): string | null {
+ if (
+ registryMeta.providersUrl &&
+ packageName in registryMeta.providerPackages
+ ) {
+ let url = registryMeta.providersUrl.replace('%package%', packageName);
+ const hash = registryMeta.providerPackages[packageName];
+ if (hash) {
+ url = url.replace('%hash%', hash);
+ }
+ return resolveBaseUrl(registryUrl, url);
+ }
+
+ if (registryMeta.providersLazyUrl) {
+ return resolveBaseUrl(
+ registryUrl,
+ registryMeta.providersLazyUrl.replace('%package%', packageName)
+ );
+ }
+
+ return null;
}
public override async getReleases({
@@ -159,42 +171,27 @@ export class PackagistDatasource extends Datasource {
const packagistResult = await this.packagistOrgLookup(packageName);
return packagistResult;
}
- const allPackages = await this.getAllPackages(registryUrl);
- const {
- packages,
- providersUrl,
- providersLazyUrl,
- providerPackages,
- includesPackages,
- } = allPackages;
- if (packages?.[packageName]) {
- const dep = extractDepReleases(packages[packageName]);
- return dep;
+
+ const meta = await this.getRegistryMeta(registryUrl);
+
+ if (meta.packages[packageName]) {
+ const result = extractDepReleases(meta.packages[packageName]);
+ return result;
}
- if (includesPackages?.[packageName]) {
- return includesPackages[packageName];
+
+ await this.fetchIncludesPackages(registryUrl, meta);
+ if (meta.includesPackages[packageName]) {
+ return meta.includesPackages[packageName];
}
- let pkgUrl: string;
- if (providersUrl && packageName in providerPackages) {
- let url = providersUrl.replace('%package%', packageName);
- const hash = providerPackages[packageName];
- if (hash) {
- url = url.replace('%hash%', hash);
- }
- pkgUrl = resolveBaseUrl(registryUrl, url);
- } else if (providersLazyUrl) {
- pkgUrl = resolveBaseUrl(
- registryUrl,
- providersLazyUrl.replace('%package%', packageName)
- );
- } else {
+
+ await this.fetchProviderPackages(registryUrl, meta);
+ const pkgUrl = this.getPkgUrl(packageName, registryUrl, meta);
+ if (!pkgUrl) {
return null;
}
- const opts = PackagistDatasource.getHostOpts(registryUrl);
- // TODO: fix types (#9610)
- const versions = (await this.http.getJson<any>(pkgUrl, opts)).body
- .packages[packageName];
- const dep = extractDepReleases(versions);
+
+ const pkgRes = await this.getJson(pkgUrl, PackagesResponse);
+ const dep = extractDepReleases(pkgRes.packages[packageName]);
logger.trace({ dep }, 'dep');
return dep;
} catch (err) /* istanbul ignore next */ {
diff --git a/lib/modules/datasource/packagist/schema.spec.ts b/lib/modules/datasource/packagist/schema.spec.ts
index 3f3eb45ca7..9e82858be9 100644
--- a/lib/modules/datasource/packagist/schema.spec.ts
+++ b/lib/modules/datasource/packagist/schema.spec.ts
@@ -251,6 +251,7 @@ describe('modules/datasource/packagist/schema', () => {
includesFiles: [],
packages: {},
providerPackages: {},
+ includesPackages: {},
providersLazyUrl: null,
providersUrl: null,
});
diff --git a/lib/modules/datasource/packagist/schema.ts b/lib/modules/datasource/packagist/schema.ts
index 6a6ac2e227..32de3ca10c 100644
--- a/lib/modules/datasource/packagist/schema.ts
+++ b/lib/modules/datasource/packagist/schema.ts
@@ -155,40 +155,58 @@ export function parsePackagesResponses(
return extractReleaseResult(...releaseArrays);
}
+export const RegistryFile = z.object({
+ key: z.string(),
+ sha256: z.string(),
+});
+export type RegistryFile = z.infer<typeof RegistryFile>;
+
+export const PackagesResponse = z.object({
+ packages: looseRecord(ComposerReleases),
+});
+export type PackagesResponse = z.infer<typeof PackagesResponse>;
+
+export const PackagistFile = PackagesResponse.merge(
+ z.object({
+ providers: looseRecord(
+ z.object({
+ sha256: looseValue(z.string()),
+ })
+ ).transform((x) =>
+ Object.fromEntries(
+ Object.entries(x).map(([key, { sha256 }]) => [key, sha256])
+ )
+ ),
+ })
+);
+export type PackagistFile = z.infer<typeof PackagistFile>;
+
export const RegistryMeta = z
.preprocess(
(x) => (is.plainObject(x) ? x : {}),
- z.object({
- ['includes']: looseRecord(
- z.object({
- sha256: z.string(),
- })
- ).transform((x) =>
- Object.entries(x).map(([name, { sha256 }]) => ({
- key: name.replace(sha256, '%hash%'),
- sha256,
- }))
- ),
- ['packages']: looseRecord(ComposerReleases),
- ['provider-includes']: looseRecord(
- z.object({
- sha256: z.string(),
- })
- ).transform((x) =>
- Object.entries(x).map(([key, { sha256 }]) => ({ key, sha256 }))
- ),
- ['providers']: looseRecord(
- z.object({
- sha256: looseValue(z.string()),
- })
- ).transform((x) =>
- Object.fromEntries(
- Object.entries(x).map(([key, { sha256 }]) => [key, sha256])
- )
- ),
- ['providers-lazy-url']: looseValue(z.string()),
- ['providers-url']: looseValue(z.string()),
- })
+ PackagistFile.merge(
+ z.object({
+ ['includes']: looseRecord(
+ z.object({
+ sha256: z.string(),
+ })
+ ).transform((x) =>
+ Object.entries(x).map(([name, { sha256 }]) => ({
+ key: name.replace(sha256, '%hash%'),
+ sha256,
+ }))
+ ),
+ ['provider-includes']: looseRecord(
+ z.object({
+ sha256: z.string(),
+ })
+ ).transform((x) =>
+ Object.entries(x).map(([key, { sha256 }]) => ({ key, sha256 }))
+ ),
+ ['providers-lazy-url']: looseValue(z.string()),
+ ['providers-url']: looseValue(z.string()),
+ })
+ )
)
.transform(
({
@@ -205,14 +223,7 @@ export const RegistryMeta = z
files,
providersUrl,
providersLazyUrl,
+ includesPackages: {} as Record<string, ReleaseResult | null>,
})
);
export type RegistryMeta = z.infer<typeof RegistryMeta>;
-
-export interface AllPackages
- extends Pick<
- RegistryMeta,
- 'packages' | 'providersUrl' | 'providersLazyUrl' | 'providerPackages'
- > {
- includesPackages: Record<string, ReleaseResult | null>;
-}
diff --git a/lib/modules/datasource/packagist/types.ts b/lib/modules/datasource/packagist/types.ts
deleted file mode 100644
index 8687dd3d92..0000000000
--- a/lib/modules/datasource/packagist/types.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-export interface RegistryFile {
- key: string;
- sha256: string;
-}
-
-export interface PackagistFile {
- providers: Record<string, RegistryFile>;
- packages?: Record<string, RegistryFile>;
-}
--
GitLab