feat: Added pre-commit manager (#7662)

lib/manager/helm-values/util.ts

+import { hasKey } from '../../util/object';
+
 export type HelmDockerImageDependency = {
   registry?: string;
   repository: string;
   tag: string;
 };
 
-/**
- * This is a workaround helper to allow the usage of 'unknown' in
- * a type-guard function while checking that keys exist.
- *
- * @see https://github.com/microsoft/TypeScript/issues/21732
- * @see https://stackoverflow.com/a/58630274
- */
-function hasKey<K extends string, T>(k: K, o: T): o is T & Record<K, unknown> {
-  return typeof o === 'object' && k in o;
-}
-
 /**
  * Type guard to determine whether a given partial Helm values.yaml object potentially
  * defines a Helm Docker dependency.

lib/manager/pre-commit/__fixtures__/.pre-commit-config.yaml

+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files

lib/manager/pre-commit/__fixtures__/complex.pre-commit-config.yaml

+fail_fast: true
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.3.0
+    # multiple hooks:
+    hooks:
+      - id: check-ast
+      - id: check-yaml
+      - id: end-of-file-fixer
+        exclude: ^notebooks
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 19.3b0
+    hooks:
+      - id: black
+  - repo: https://gitlab.com/psf/black
+    # should also detect gitlab
+    rev: 19.3b0
+    hooks:
+      - id: black
+  - repo: http://gitlab.com/psf/black
+    # should also detect http
+    rev: 19.3b0
+    hooks:
+      - id: black
+  - repo: https://github.com/prettier/pre-commit
+    # should accept different order of keys
+    hooks:
+      - id: prettier
+        exclude: ^notebooks
+    rev: v2.1.2
+  - repo: git@github.com:prettier/pre-commit
+    # should allow ssh urls
+    hooks:
+      - id: prettier
+        exclude: ^notebooks
+    rev: v2.1.2
+  - repo: some_invalid_url
+    # case with invlalid url.
+    rev: v1.0.0

lib/manager/pre-commit/__fixtures__/empty_repos.pre-commit-config.yaml

+# empty repos element
+repos:

lib/manager/pre-commit/__fixtures__/enterprise.pre-commit-config.yaml

+fail_fast: true
+repos:
+  - repo: https://enterprise.com/pre-commit/pre-commit-hooks
+    # case with non-default url.
+    rev: v1.0.0

lib/manager/pre-commit/__fixtures__/invalid_repo.pre-commit-config.yaml

+# invalid repo item
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    revv: v3.3.0

lib/manager/pre-commit/__fixtures__/no_repos.pre-commit-config.yaml

+# missing repos element
+fail_fast: true

lib/manager/pre-commit/__snapshots__/extract.spec.ts.snap

+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`lib/manager/precommit/extract extractPackageFile() can handle invalid private git repos 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "v1.0.0",
+      "depName": "pre-commit/pre-commit-hooks",
+      "depType": "repository",
+      "lookupName": "pre-commit/pre-commit-hooks",
+      "registryUrls": Array [
+        "enterprise.com",
+      ],
+      "skipReason": "unknown-registry",
+    },
+  ],
+}
+`;
+
+exports[`lib/manager/precommit/extract extractPackageFile() can handle private git repos 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "v1.0.0",
+      "datasource": "gitlab-tags",
+      "depName": "pre-commit/pre-commit-hooks",
+      "depType": "repository",
+      "lookupName": "pre-commit/pre-commit-hooks",
+      "registryUrls": Array [
+        "enterprise.com",
+      ],
+    },
+  ],
+}
+`;
+
+exports[`lib/manager/precommit/extract extractPackageFile() can handle unknown private git repos 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "v1.0.0",
+      "depName": "pre-commit/pre-commit-hooks",
+      "depType": "repository",
+      "lookupName": "pre-commit/pre-commit-hooks",
+      "registryUrls": Array [
+        "enterprise.com",
+      ],
+      "skipReason": "unknown-registry",
+    },
+  ],
+}
+`;
+
+exports[`lib/manager/precommit/extract extractPackageFile() extracts from complex config file correctly 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "v3.3.0",
+      "datasource": "github-tags",
+      "depName": "pre-commit/pre-commit-hooks",
+      "depType": "repository",
+      "lookupName": "pre-commit/pre-commit-hooks",
+    },
+    Object {
+      "currentValue": "19.3b0",
+      "datasource": "github-tags",
+      "depName": "psf/black",
+      "depType": "repository",
+      "lookupName": "psf/black",
+    },
+    Object {
+      "currentValue": "19.3b0",
+      "datasource": "gitlab-tags",
+      "depName": "psf/black",
+      "depType": "repository",
+      "lookupName": "psf/black",
+    },
+    Object {
+      "currentValue": "19.3b0",
+      "datasource": "gitlab-tags",
+      "depName": "psf/black",
+      "depType": "repository",
+      "lookupName": "psf/black",
+    },
+    Object {
+      "currentValue": "v2.1.2",
+      "datasource": "github-tags",
+      "depName": "prettier/pre-commit",
+      "depType": "repository",
+      "lookupName": "prettier/pre-commit",
+    },
+    Object {
+      "currentValue": "v2.1.2",
+      "datasource": "github-tags",
+      "depName": "prettier/pre-commit",
+      "depType": "repository",
+      "lookupName": "prettier/pre-commit",
+    },
+    Object {
+      "currentValue": "v1.0.0",
+      "datasource": undefined,
+      "depName": undefined,
+      "depType": "repository",
+      "lookupName": undefined,
+      "skipReason": "invalid-url",
+    },
+  ],
+}
+`;
+
+exports[`lib/manager/precommit/extract extractPackageFile() extracts from values.yaml correctly with same structure as "pre-commit sample-config" 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "v2.4.0",
+      "datasource": "github-tags",
+      "depName": "pre-commit/pre-commit-hooks",
+      "depType": "repository",
+      "lookupName": "pre-commit/pre-commit-hooks",
+    },
+  ],
+}
+`;

lib/manager/pre-commit/extract.spec.ts

+import { readFileSync } from 'fs';
+import { mocked } from '../../../test/util';
+import * as _hostRules from '../../util/host-rules';
+import { extractPackageFile } from './extract';
+
+jest.mock('../../util/host-rules');
+const hostRules = mocked(_hostRules);
+
+const complexPrecommitConfig = readFileSync(
+  'lib/manager/pre-commit/__fixtures__/complex.pre-commit-config.yaml',
+  'utf8'
+);
+
+const examplePrecommitConfig = readFileSync(
+  'lib/manager/pre-commit/__fixtures__/.pre-commit-config.yaml',
+  'utf8'
+);
+
+const emptyReposPrecommitConfig = readFileSync(
+  'lib/manager/pre-commit/__fixtures__/empty_repos.pre-commit-config.yaml',
+  'utf8'
+);
+
+const noReposPrecommitConfig = readFileSync(
+  'lib/manager/pre-commit/__fixtures__/no_repos.pre-commit-config.yaml',
+  'utf8'
+);
+
+const invalidRepoPrecommitConfig = readFileSync(
+  'lib/manager/pre-commit/__fixtures__/invalid_repo.pre-commit-config.yaml',
+  'utf8'
+);
+
+const enterpriseGitPrecommitConfig = readFileSync(
+  'lib/manager/pre-commit/__fixtures__/enterprise.pre-commit-config.yaml',
+  'utf8'
+);
+
+describe('lib/manager/precommit/extract', () => {
+  describe('extractPackageFile()', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+    });
+    it('returns null for invalid yaml file content', () => {
+      const result = extractPackageFile('nothing here: [');
+      expect(result).toBeNull();
+    });
+    it('returns null for empty yaml file content', () => {
+      const result = extractPackageFile('');
+      expect(result).toBeNull();
+    });
+    it('returns null for no file content', () => {
+      const result = extractPackageFile(null);
+      expect(result).toBeNull();
+    });
+    it('returns null for no repos', () => {
+      const result = extractPackageFile(noReposPrecommitConfig);
+      expect(result).toBeNull();
+    });
+    it('returns null for empty repos', () => {
+      const result = extractPackageFile(emptyReposPrecommitConfig);
+      expect(result).toBeNull();
+    });
+    it('returns null for invalid repo', () => {
+      const result = extractPackageFile(invalidRepoPrecommitConfig);
+      expect(result).toBeNull();
+    });
+    it('extracts from values.yaml correctly with same structure as "pre-commit sample-config"', () => {
+      const result = extractPackageFile(examplePrecommitConfig);
+      expect(result).toMatchSnapshot();
+    });
+    it('extracts from complex config file correctly', () => {
+      const result = extractPackageFile(complexPrecommitConfig);
+      expect(result).toMatchSnapshot();
+    });
+    it('can handle private git repos', () => {
+      hostRules.find.mockReturnValue({ token: 'value' });
+      const result = extractPackageFile(enterpriseGitPrecommitConfig);
+      expect(result).toMatchSnapshot();
+    });
+    it('can handle invalid private git repos', () => {
+      hostRules.find.mockReturnValue({});
+      const result = extractPackageFile(enterpriseGitPrecommitConfig);
+      expect(result).toMatchSnapshot();
+    });
+    it('can handle unknown private git repos', () => {
+      // First attemp returns a result
+      hostRules.find.mockReturnValueOnce({ token: 'value' });
+      // But all subsequent checks (those with hostType), then fail:
+      hostRules.find.mockReturnValue({});
+      const result = extractPackageFile(enterpriseGitPrecommitConfig);
+      expect(result).toMatchSnapshot();
+    });
+  });
+});

lib/manager/pre-commit/extract.ts

+import is from '@sindresorhus/is';
+import yaml from 'js-yaml';
+import {
+  PLATFORM_TYPE_GITEA,
+  PLATFORM_TYPE_GITHUB,
+  PLATFORM_TYPE_GITLAB,
+} from '../../constants/platforms';
+import { id as githubTagsId } from '../../datasource/github-tags';
+import { id as gitlabTagsId } from '../../datasource/gitlab-tags';
+import { logger } from '../../logger';
+import { SkipReason } from '../../types';
+import { find } from '../../util/host-rules';
+import { regEx } from '../../util/regex';
+import { PackageDependency, PackageFile } from '../common';
+
+import {
+  matchesPrecommitConfigHeuristic,
+  matchesPrecommitDependencyHeuristic,
+} from './parsing';
+import { PreCommitConfig } from './types';
+
+function isEmptyObject(obj: any): boolean {
+  return Object.keys(obj).length === 0 && obj.constructor === Object;
+}
+
+/**
+ * Determines the datasource(id) to be used for this dependency
+ * @param repository the full git url, ie git@github.com/user/project.
+ *        Used in debug statements to clearly indicate the related dependency.
+ * @param hostName the hostname (ie github.com)
+ *        Used to determine which renovate datasource should be used.
+ *        Is matched literally against `github.com` and `gitlab.com`.
+ *        If that doesn't match, `hostRules.find()` is used to find related sources.
+ *        In that case, the hostname is passed on as registryUrl to the corresponding datasource.
+ */
+function determineDatasource(
+  repository: string,
+  hostName: string
+): { datasource?: string; registryUrls?: string[]; skipReason?: SkipReason } {
+  if (hostName === 'github.com') {
+    logger.debug({ repository, hostName }, 'Found github dependency');
+    return { datasource: githubTagsId };
+  }
+  if (hostName === 'gitlab.com') {
+    logger.debug({ repository, hostName }, 'Found gitlab dependency');
+    return { datasource: gitlabTagsId };
+  }
+  const hostUrl = 'https://' + hostName;
+  const res = find({ url: hostUrl });
+  if (isEmptyObject(res)) {
+    // 1 check, to possibly prevent 3 failures in combined query of hostType & url.
+    logger.debug(
+      { repository, hostUrl },
+      'Provided hostname does not match any hostRules. Ignoring'
+    );
+    return { skipReason: SkipReason.UnknownRegistry, registryUrls: [hostName] };
+  }
+  for (const [hostType, sourceId] of [
+    [PLATFORM_TYPE_GITEA, gitlabTagsId],
+    [PLATFORM_TYPE_GITHUB, githubTagsId],
+    [PLATFORM_TYPE_GITLAB, gitlabTagsId],
+  ]) {
+    if (!isEmptyObject(find({ hostType, url: hostUrl }))) {
+      logger.debug(
+        { repository, hostUrl, hostType },
+        `Provided hostname matches a ${hostType} hostrule.`
+      );
+      return { datasource: sourceId, registryUrls: [hostName] };
+    }
+  }
+  logger.debug(
+    { repository, registry: hostUrl },
+    'Provided hostname did not match any of the hostRules of hostType gitea,github nor gitlab'
+  );
+  return { skipReason: SkipReason.UnknownRegistry, registryUrls: [hostName] };
+}
+
+function extractDependency(
+  tag: string,
+  repository: string
+): {
+  depName?: string;
+  depType?: string;
+  datasource?: string;
+  lookupName?: string;
+  skipReason?: SkipReason;
+  currentValue?: string;
+} {
+  logger.debug({ tag }, 'Found version');
+
+  const urlMatchers = [
+    // This splits "http://my.github.com/user/repo" -> "my.github.com" "user/repo
+    regEx('^https?:\\/\\/(?<hostName>[^\\/]+)\\/(?<depName>\\S*)'),
+    // This splits "git@private.registry.com:user/repo" -> "private.registry.com" "user/repo
+    regEx('^git@(?<hostName>[^:]+):(?<depName>\\S*)'),
+  ];
+  for (const urlMatcher of urlMatchers) {
+    const match = urlMatcher.exec(repository);
+    if (match) {
+      const { hostName, depName } = match.groups;
+      const sourceDef = determineDatasource(repository, hostName);
+      return {
+        ...sourceDef,
+        depName,
+        depType: 'repository',
+        lookupName: depName,
+        currentValue: tag,
+      };
+    }
+  }
+  logger.info(
+    { repository },
+    'Could not separate hostname from full dependency url.'
+  );
+  return {
+    depName: undefined,
+    depType: 'repository',
+    datasource: undefined,
+    lookupName: undefined,
+    skipReason: SkipReason.InvalidUrl,
+    currentValue: tag,
+  };
+}
+
+/**
+ * Find all supported dependencies in the pre-commit yaml object.
+ *
+ * @param precommitFile the parsed yaml config file
+ */
+function findDependencies(
+  precommitFile: PreCommitConfig
+): Array<PackageDependency> {
+  if (!precommitFile.repos) {
+    logger.debug(`No repos section found, skipping file`);
+    return [];
+  }
+  const packageDependencies = [];
+  precommitFile.repos.forEach((item) => {
+    if (matchesPrecommitDependencyHeuristic(item)) {
+      logger.trace(item, 'Matched pre-commit dependency spec');
+      const repository = String(item.repo);
+      const tag = String(item.rev);
+      const dep = extractDependency(tag, repository);
+
+      packageDependencies.push(dep);
+    } else {
+      logger.trace(item, 'Did not find pre-commit repo spec');
+    }
+  });
+  return packageDependencies;
+}
+
+export function extractPackageFile(content: string): PackageFile | null {
+  let parsedContent: Record<string, unknown> | PreCommitConfig;
+  try {
+    parsedContent = yaml.safeLoad(content, { json: true }) as any;
+  } catch (err) {
+    logger.debug({ err }, 'Failed to parse pre-commit config YAML');
+    return null;
+  }
+  if (!is.plainObject<Record<string, unknown>>(parsedContent)) {
+    logger.warn(`Parsing of pre-commit config YAML returned invalid result`);
+    return null;
+  }
+  if (!matchesPrecommitConfigHeuristic(parsedContent)) {
+    logger.info(`File does not look like a pre-commit config file`);
+    return null;
+  }
+  try {
+    const deps = findDependencies(parsedContent);
+    if (deps.length) {
+      logger.debug({ deps }, 'Found dependencies in pre-commit config');
+      return { deps };
+    }
+  } catch (err) /* istanbul ignore next */ {
+    logger.error({ err }, 'Error scanning parsed pre-commit config');
+  }
+  return null;
+}

lib/manager/pre-commit/index.ts

+export { extractPackageFile } from './extract';
+
+export const defaultConfig = {
+  commitMessageTopic: 'precommit hook {{depName}}',
+  fileMatch: ['(^|/)\\.pre-commit-config\\.yaml$'],
+};

lib/manager/pre-commit/parsing.ts

+import { hasKey } from '../../util/object';
+import { PreCommitConfig, PreCommitDependency } from './types';
+
+/**
+ * Type guard to determine whether the file matches pre-commit configuration format
+ * Example original yaml:
+ *
+ *   repos
+ *   - repo: https://github.com/user/repo
+ *     rev: v1.0.0
+ */
+export function matchesPrecommitConfigHeuristic(
+  data: unknown
+): data is PreCommitConfig {
+  return data && typeof data === 'object' && hasKey('repos', data);
+}
+
+/**
+ * Type guard to determine whether a given repo definition defines a pre-commit Git hook dependency.
+ * Example original yaml portion
+ *
+ *   - repo: https://github.com/user/repo
+ *     rev: v1.0.0
+ */
+export function matchesPrecommitDependencyHeuristic(
+  data: unknown
+): data is PreCommitDependency {
+  return (
+    data &&
+    typeof data === 'object' &&
+    hasKey('repo', data) &&
+    hasKey('rev', data)
+  );
+}

lib/manager/pre-commit/readme.md

+Renovate supports updating of Git dependencies within pre-commit configuration `.pre-commit-config.yaml` files or other YAML files that use the same format (via `fileMatch` configuration).
+Updates are performed if the files follow the conventional format used in typical pre-commit files:
+
+```yaml
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.0.0
+    hooks:
+      - id: some-hook-id
+```

lib/manager/pre-commit/types.ts

+export interface PreCommitConfig {
+  repos: PreCommitDependency[];
+}
+
+export interface PreCommitDependency {
+  repo: string;
+  rev: string;
+}

lib/util/object.spec.ts

+import { hasKey } from './object';
+
+describe('util/object', () => {
+  beforeEach(() => {
+    jest.resetModules();
+  });
+
+  it('finds key in regular object', () => {
+    expect(hasKey('foo', { foo: true })).toBeTrue();
+  });
+  it('detects missing key in regular object', () => {
+    expect(hasKey('foo', { bar: true })).toBeFalse();
+  });
+  it('returns false for wrong instance type', () => {
+    expect(hasKey('foo', 'i-am-not-an-object')).toBeFalse();
+  });
+});

lib/util/object.ts

+/**
+ * This is a workaround helper to allow the usage of 'unknown' in
+ * a type-guard function while checking that keys exist.
+ *
+ * @see https://github.com/microsoft/TypeScript/issues/21732
+ * @see https://stackoverflow.com/a/58630274
+ */
+export function hasKey<K extends string, T>(
+  k: K,
+  o: T
+): o is T & Record<K, unknown> {
+  return typeof o === 'object' && k in o;
+}