Skip to content
Snippets Groups Projects
Select Git revision
  • 9c11e43b8eb4791fa01466bbff22e8bbb8b92007
  • main default protected
  • renovate/main-redis-5.x
  • next
  • revert-31645-feat/rename-gradle-wrapper-validation-action
  • fix/36615b-branch-reuse-no-cache
  • chore/punycode
  • fix/36615-branch-reuse-bug
  • refactor/pin-new-value
  • feat/36219--git-x509-signing
  • feat/structured-logger
  • hotfix/39.264.1
  • feat/skip-dangling
  • gh-readonly-queue/next/pr-36034-7a061c4ca1024a19e2c295d773d9642625d1c2be
  • hotfix/39.238.3
  • refactor/gitlab-auto-approve
  • feat/template-strings
  • gh-readonly-queue/next/pr-35654-137d934242c784e0c45d4b957362214f0eade1d7
  • fix/32307-global-extends-merging
  • fix/32307-global-extends-repositories
  • gh-readonly-queue/next/pr-35009-046ebf7cb84ab859f7fefceb5fa53a54ce9736f8
  • 41.35.2
  • 41.35.1
  • 41.35.0
  • 41.34.1
  • 41.34.0
  • 41.33.0
  • 41.32.3
  • 41.32.2
  • 41.32.1
  • 41.32.0
  • 41.31.1
  • 41.31.0
  • 41.30.5
  • 41.30.4
  • 41.30.3
  • 41.30.2
  • 41.30.1
  • 41.30.0
  • 41.29.1
  • 41.29.0
41 results

extract.ts

Blame
  • user avatar
    george-wilson-rea authored and GitHub committed
    Co-authored-by: default avatarRhys Arkins <rhys@arkins.net>
    9c11e43b
    History
    extract.ts 10.14 KiB
    import { lang, query as q } from 'good-enough-parser';
    import { logger } from '../../../logger';
    import { readLocalFile } from '../../../util/fs';
    import { newlineRegex, regEx } from '../../../util/regex';
    import { parseUrl } from '../../../util/url';
    import { GithubReleasesDatasource } from '../../datasource/github-releases';
    import { MavenDatasource } from '../../datasource/maven';
    import { SbtPackageDatasource } from '../../datasource/sbt-package';
    import {
      SBT_PLUGINS_REPO,
      SbtPluginDatasource,
    } from '../../datasource/sbt-plugin';
    import { get } from '../../versioning';
    import * as mavenVersioning from '../../versioning/maven';
    import * as semverVersioning from '../../versioning/semver';
    import type {
      ExtractConfig,
      PackageDependency,
      PackageFile,
      PackageFileContent,
    } from '../types';
    import { normalizeScalaVersion } from './util';
    
    type Vars = Record<string, string>;
    
    interface Ctx {
      vars: Vars;
      deps: PackageDependency[];
      registryUrls: string[];
    
      scalaVersion?: string;
      packageFileVersion?: string;
    
      groupId?: string;
      artifactId?: string;
      currentValue?: string;
    
      currentVarName?: string;
      depType?: string;
      useScalaVersion?: boolean;
      variableName?: string;
    }
    
    const SBT_MVN_REPO = 'https://repo1.maven.org/maven2';
    
    const scala = lang.createLang('scala');
    
    const sbtVersionRegex = regEx(
      'sbt\\.version *= *(?<version>\\d+\\.\\d+\\.\\d+)',
    );
    
    const sbtProxyUrlRegex = regEx(
      /^\s*(?<repoName>\S+):\s+(?<proxy>https?:\/\/[\w./-]+)/,
    );
    
    const scalaVersionMatch = q
      .sym<Ctx>('scalaVersion')
      .op(':=')
      .alt(
        q.str<Ctx>((ctx, { value: scalaVersion }) => ({ ...ctx, scalaVersion })),
        q.sym<Ctx>((ctx, { value: varName }) => {
          const scalaVersion = ctx.vars[varName];
          if (scalaVersion) {
            ctx.scalaVersion = scalaVersion;
          }
          return ctx;
        }),
      )
      .handler((ctx) => {
        if (ctx.scalaVersion) {
          const version = get(mavenVersioning.id);
    
          let packageName = 'org.scala-lang:scala-library';
          if (version.getMajor(ctx.scalaVersion) === 3) {
            packageName = 'org.scala-lang:scala3-library_3';
          }
    
          const dep: PackageDependency = {
            datasource: MavenDatasource.id,
            depName: 'scala',
            packageName,
            currentValue: ctx.scalaVersion,
            separateMinorPatch: true,
          };
          ctx.scalaVersion = normalizeScalaVersion(ctx.scalaVersion);
          ctx.deps.push(dep);
        }
        return ctx;
      });
    
    const packageFileVersionMatch = q
      .sym<Ctx>('version')
      .op(':=')
      .alt(
        q.str<Ctx>((ctx, { value: packageFileVersion }) => ({
          ...ctx,
          packageFileVersion,
        })),
        q.sym<Ctx>((ctx, { value: varName }) => {
          const packageFileVersion = ctx.vars[varName];
          if (packageFileVersion) {
            ctx.packageFileVersion = packageFileVersion;
          }
          return ctx;
        }),
      );
    
    const variableNameMatch = q
      .sym<Ctx>((ctx, { value: varName }) => ({
        ...ctx,
        currentVarName: varName,
      }))
      .opt(q.op<Ctx>(':').sym('String'));
    
    const variableValueMatch = q.str<Ctx>((ctx, { value }) => {
      ctx.vars[ctx.currentVarName!] = value;
      delete ctx.currentVarName;
      return ctx;
    });
    
    const assignmentMatch = q.sym<Ctx>('val').join(variableNameMatch).op('=');
    
    const variableDefinitionMatch = q
      .alt(
        q.sym<Ctx>('lazy').join(assignmentMatch),
        assignmentMatch,
        variableNameMatch.op(':='),
      )
      .join(variableValueMatch);
    
    const groupIdMatch = q.alt<Ctx>(
      q.sym<Ctx>((ctx, { value: varName }) => {
        const currentGroupId = ctx.vars[varName];
        if (currentGroupId) {
          ctx.groupId = currentGroupId;
        }
        return ctx;
      }),
      q.str<Ctx>((ctx, { value: groupId }) => ({ ...ctx, groupId })),
    );
    
    const artifactIdMatch = q.alt<Ctx>(
      q.sym<Ctx>((ctx, { value: varName }) => {
        const artifactId = ctx.vars[varName];
        if (artifactId) {
          ctx.artifactId = artifactId;
        }
        return ctx;
      }),
      q.str<Ctx>((ctx, { value: artifactId }) => ({ ...ctx, artifactId })),
    );
    
    const versionMatch = q.alt<Ctx>(
      q.sym<Ctx>((ctx, { value: varName }) => {
        const currentValue = ctx.vars[varName];
        if (currentValue) {
          ctx.currentValue = currentValue;
          ctx.variableName = varName;
        }
        return ctx;
      }),
      q.str<Ctx>((ctx, { value: currentValue }) => ({ ...ctx, currentValue })),
    );
    
    const simpleDependencyMatch = groupIdMatch
      .op('%')
      .join(artifactIdMatch)
      .op('%')
      .join(versionMatch);
    
    const versionedDependencyMatch = groupIdMatch
      .op('%%')
      .join(artifactIdMatch)
      .handler((ctx) => ({ ...ctx, useScalaVersion: true }))
      .op('%')
      .join(versionMatch);
    
    const crossDependencyMatch = groupIdMatch
      .op('%%%')
      .join(artifactIdMatch)
      .handler((ctx) => ({ ...ctx, useScalaVersion: true }))
      .op('%')
      .join(versionMatch);
    
    function depHandler(ctx: Ctx): Ctx {
      const {
        scalaVersion,
        groupId,
        artifactId,
        currentValue,
        useScalaVersion,
        depType,
        variableName,
      } = ctx;
    
      delete ctx.groupId;
      delete ctx.artifactId;
      delete ctx.currentValue;
      delete ctx.useScalaVersion;
      delete ctx.depType;
      delete ctx.variableName;
    
      const depName = `${groupId!}:${artifactId!}`;
      const isScala3 = scalaVersion?.[0] === '3';
      const scalaVersionForPackageName = isScala3 ? '3' : scalaVersion;
    
      const dep: PackageDependency = {
        datasource: SbtPackageDatasource.id,
        depName,
        packageName:
          scalaVersionForPackageName && useScalaVersion
            ? `${depName}_${scalaVersionForPackageName}`
            : depName,
        currentValue,
      };
    
      if (depType) {
        dep.depType = depType;
      }
    
      if (depType === 'plugin') {
        dep.datasource = SbtPluginDatasource.id;
      }
    
      if (variableName) {
        dep.groupName = variableName;
        dep.variableName = variableName;
      }
    
      ctx.deps.push(dep);
    
      return ctx;
    }
    
    function depTypeHandler(ctx: Ctx, { value: depType }: { value: string }): Ctx {
      return { ...ctx, depType };
    }
    
    const sbtPackageMatch = q
      .opt<Ctx>(q.opt(q.sym<Ctx>('lazy')).sym('val').sym().op('='))
      .alt(crossDependencyMatch, simpleDependencyMatch, versionedDependencyMatch)
      .opt(
        q.alt<Ctx>(
          q.sym<Ctx>('classifier').str(depTypeHandler),
          q.op<Ctx>('%').sym(depTypeHandler),
          q.op<Ctx>('%').str(depTypeHandler),
        ),
      )
      .handler(depHandler);
    
    const sbtPluginMatch = q
      .sym<Ctx>(regEx(/^(?:addSbtPlugin|addCompilerPlugin)$/))
      .tree({
        type: 'wrapped-tree',
        maxDepth: 1,
        search: q
          .begin<Ctx>()
          .alt(simpleDependencyMatch, versionedDependencyMatch)
          .end(),
      })
      .handler((ctx) => ({ ...ctx, depType: 'plugin' }))
      .handler(depHandler);
    
    const resolverMatch = q
      .str<Ctx>()
      .sym('at')
      .str((ctx, { value }) => {
        if (parseUrl(value)) {
          ctx.registryUrls.push(value);
        }
        return ctx;
      });
    
    const addResolverMatch = q.sym<Ctx>('resolvers').alt(
      q.op<Ctx>('+=').join(resolverMatch),
      q.op<Ctx>('++=').sym('Seq').tree({
        type: 'wrapped-tree',
        maxDepth: 1,
        search: resolverMatch,
      }),
    );
    
    function registryUrlHandler(ctx: Ctx): Ctx {
      for (const dep of ctx.deps) {
        dep.registryUrls = [...ctx.registryUrls];
      }
      return ctx;
    }
    
    const query = q.tree<Ctx>({
      type: 'root-tree',
      maxDepth: 32,
      search: q.alt<Ctx>(
        scalaVersionMatch,
        packageFileVersionMatch,
        sbtPackageMatch,
        sbtPluginMatch,
        addResolverMatch,
        variableDefinitionMatch,
      ),
      postHandler: registryUrlHandler,
    });
    
    export function extractProxyUrls(
      content: string,
      packageFile: string,
    ): string[] {
      const extractedProxyUrls: string[] = [];
      logger.debug(`Parsing proxy repository file ${packageFile}`);
      for (const line of content.split(newlineRegex)) {
        const extraction = sbtProxyUrlRegex.exec(line);
        if (extraction?.groups?.proxy) {
          extractedProxyUrls.push(extraction.groups.proxy);
        } else if (line.trim() === 'maven-central') {
          extractedProxyUrls.push(SBT_MVN_REPO);
        }
      }
      return extractedProxyUrls;
    }
    
    export function extractPackageFile(
      content: string,
      packageFile: string,
    ): PackageFileContent | null {
      if (
        packageFile === 'project/build.properties' ||
        packageFile.endsWith('/project/build.properties')
      ) {
        const regexResult = sbtVersionRegex.exec(content);
        const sbtVersion = regexResult?.groups?.version;
        const matchString = regexResult?.[0];
        if (sbtVersion) {
          const sbtDependency: PackageDependency = {
            datasource: GithubReleasesDatasource.id,
            depName: 'sbt/sbt',
            packageName: 'sbt/sbt',
            versioning: semverVersioning.id,
            currentValue: sbtVersion,
            replaceString: matchString,
            extractVersion: '^v(?<version>\\S+)',
            registryUrls: [],
          };
    
          return {
            deps: [sbtDependency],
          };
        } else {
          return null;
        }
      }
    
      let parsedResult: Ctx | null = null;
    
      try {
        parsedResult = scala.query(content, query, {
          vars: {},
          deps: [],
          registryUrls: [],
        });
      } catch (err) /* istanbul ignore next */ {
        logger.debug({ err, packageFile }, 'Sbt parsing error');
      }
    
      if (!parsedResult) {
        return null;
      }
    
      const { deps, packageFileVersion } = parsedResult;
    
      if (!deps.length) {
        return null;
      }
    
      return { deps, packageFileVersion };
    }
    
    export async function extractAllPackageFiles(
      _config: ExtractConfig,
      packageFiles: string[],
    ): Promise<PackageFile[]> {
      const packages: PackageFile[] = [];
      const proxyUrls: string[] = [];
    
      for (const packageFile of packageFiles) {
        const content = await readLocalFile(packageFile, 'utf8');
        if (!content) {
          logger.debug({ packageFile }, 'packageFile has no content');
          continue;
        }
        if (packageFile === 'repositories') {
          const urls = extractProxyUrls(content, packageFile);
          proxyUrls.push(...urls);
        } else {
          const pkg = extractPackageFile(content, packageFile);
          if (pkg) {
            packages.push({ deps: pkg.deps, packageFile });
          }
        }
      }
      for (const pkg of packages) {
        for (const dep of pkg.deps) {
          if (dep.datasource !== GithubReleasesDatasource.id) {
            if (proxyUrls.length > 0) {
              dep.registryUrls!.unshift(...proxyUrls);
            } else if (dep.depType === 'plugin') {
              dep.registryUrls!.unshift(SBT_PLUGINS_REPO, SBT_MVN_REPO);
            } else {
              dep.registryUrls!.unshift(SBT_MVN_REPO);
            }
          }
        }
      }
      return packages;
    }