diff --git a/lib/manager/helm-values/util.ts b/lib/manager/helm-values/util.ts index 1f6faca1da8076d032ebdc12b82646ad4823c2a6..d886c16d00633526e3a300f9f7967794bfc3dc99 100644 --- a/lib/manager/helm-values/util.ts +++ b/lib/manager/helm-values/util.ts @@ -1,20 +1,11 @@ +import { hasKey } from '../../util/object'; + export type HelmDockerImageDependency = { registry?: string; repository: string; tag: string; }; -/** - * This is a workaround helper to allow the usage of 'unknown' in - * a type-guard function while checking that keys exist. - * - * @see https://github.com/microsoft/TypeScript/issues/21732 - * @see https://stackoverflow.com/a/58630274 - */ -function hasKey<K extends string, T>(k: K, o: T): o is T & Record<K, unknown> { - return typeof o === 'object' && k in o; -} - /** * Type guard to determine whether a given partial Helm values.yaml object potentially * defines a Helm Docker dependency. diff --git a/lib/manager/pre-commit/__fixtures__/.pre-commit-config.yaml b/lib/manager/pre-commit/__fixtures__/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68b1ae9cd81ba92e33175500eda1e96ce72ab97c --- /dev/null +++ b/lib/manager/pre-commit/__fixtures__/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files diff --git a/lib/manager/pre-commit/__fixtures__/complex.pre-commit-config.yaml b/lib/manager/pre-commit/__fixtures__/complex.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..754e3ef7e09b9f7c7c5914ae600f70488461dfac --- /dev/null +++ b/lib/manager/pre-commit/__fixtures__/complex.pre-commit-config.yaml @@ -0,0 +1,40 @@ +fail_fast: true +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.3.0 + # multiple hooks: + hooks: + - id: check-ast + - id: check-yaml + - id: end-of-file-fixer + exclude: ^notebooks + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 19.3b0 + hooks: + - id: black + - repo: https://gitlab.com/psf/black + # should also detect gitlab + rev: 19.3b0 + hooks: + - id: black + - repo: http://gitlab.com/psf/black + # should also detect http + rev: 19.3b0 + hooks: + - id: black + - repo: https://github.com/prettier/pre-commit + # should accept different order of keys + hooks: + - id: prettier + exclude: ^notebooks + rev: v2.1.2 + - repo: git@github.com:prettier/pre-commit + # should allow ssh urls + hooks: + - id: prettier + exclude: ^notebooks + rev: v2.1.2 + - repo: some_invalid_url + # case with invlalid url. + rev: v1.0.0 diff --git a/lib/manager/pre-commit/__fixtures__/empty_repos.pre-commit-config.yaml b/lib/manager/pre-commit/__fixtures__/empty_repos.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..503ab8179c8f5310090a3b18e60038554f0c5194 --- /dev/null +++ b/lib/manager/pre-commit/__fixtures__/empty_repos.pre-commit-config.yaml @@ -0,0 +1,2 @@ +# empty repos element +repos: diff --git a/lib/manager/pre-commit/__fixtures__/enterprise.pre-commit-config.yaml b/lib/manager/pre-commit/__fixtures__/enterprise.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b957be40571b5e41b20b3102460484d52c4ee9fe --- /dev/null +++ b/lib/manager/pre-commit/__fixtures__/enterprise.pre-commit-config.yaml @@ -0,0 +1,5 @@ +fail_fast: true +repos: + - repo: https://enterprise.com/pre-commit/pre-commit-hooks + # case with non-default url. + rev: v1.0.0 diff --git a/lib/manager/pre-commit/__fixtures__/invalid_repo.pre-commit-config.yaml b/lib/manager/pre-commit/__fixtures__/invalid_repo.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7f32a5a25e6028682d13829b1a4746c7847b7bd --- /dev/null +++ b/lib/manager/pre-commit/__fixtures__/invalid_repo.pre-commit-config.yaml @@ -0,0 +1,4 @@ +# invalid repo item +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + revv: v3.3.0 diff --git a/lib/manager/pre-commit/__fixtures__/no_repos.pre-commit-config.yaml b/lib/manager/pre-commit/__fixtures__/no_repos.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..512e6c007479e2555e9ab82566b0ba66cf3a404f --- /dev/null +++ b/lib/manager/pre-commit/__fixtures__/no_repos.pre-commit-config.yaml @@ -0,0 +1,2 @@ +# missing repos element +fail_fast: true diff --git a/lib/manager/pre-commit/__snapshots__/extract.spec.ts.snap b/lib/manager/pre-commit/__snapshots__/extract.spec.ts.snap new file mode 100644 index 0000000000000000000000000000000000000000..cbe52869206ed5e88c2763336b01b1f7b34d4c8b --- /dev/null +++ b/lib/manager/pre-commit/__snapshots__/extract.spec.ts.snap @@ -0,0 +1,123 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`lib/manager/precommit/extract extractPackageFile() can handle invalid private git repos 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "v1.0.0", + "depName": "pre-commit/pre-commit-hooks", + "depType": "repository", + "lookupName": "pre-commit/pre-commit-hooks", + "registryUrls": Array [ + "enterprise.com", + ], + "skipReason": "unknown-registry", + }, + ], +} +`; + +exports[`lib/manager/precommit/extract extractPackageFile() can handle private git repos 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "v1.0.0", + "datasource": "gitlab-tags", + "depName": "pre-commit/pre-commit-hooks", + "depType": "repository", + "lookupName": "pre-commit/pre-commit-hooks", + "registryUrls": Array [ + "enterprise.com", + ], + }, + ], +} +`; + +exports[`lib/manager/precommit/extract extractPackageFile() can handle unknown private git repos 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "v1.0.0", + "depName": "pre-commit/pre-commit-hooks", + "depType": "repository", + "lookupName": "pre-commit/pre-commit-hooks", + "registryUrls": Array [ + "enterprise.com", + ], + "skipReason": "unknown-registry", + }, + ], +} +`; + +exports[`lib/manager/precommit/extract extractPackageFile() extracts from complex config file correctly 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "v3.3.0", + "datasource": "github-tags", + "depName": "pre-commit/pre-commit-hooks", + "depType": "repository", + "lookupName": "pre-commit/pre-commit-hooks", + }, + Object { + "currentValue": "19.3b0", + "datasource": "github-tags", + "depName": "psf/black", + "depType": "repository", + "lookupName": "psf/black", + }, + Object { + "currentValue": "19.3b0", + "datasource": "gitlab-tags", + "depName": "psf/black", + "depType": "repository", + "lookupName": "psf/black", + }, + Object { + "currentValue": "19.3b0", + "datasource": "gitlab-tags", + "depName": "psf/black", + "depType": "repository", + "lookupName": "psf/black", + }, + Object { + "currentValue": "v2.1.2", + "datasource": "github-tags", + "depName": "prettier/pre-commit", + "depType": "repository", + "lookupName": "prettier/pre-commit", + }, + Object { + "currentValue": "v2.1.2", + "datasource": "github-tags", + "depName": "prettier/pre-commit", + "depType": "repository", + "lookupName": "prettier/pre-commit", + }, + Object { + "currentValue": "v1.0.0", + "datasource": undefined, + "depName": undefined, + "depType": "repository", + "lookupName": undefined, + "skipReason": "invalid-url", + }, + ], +} +`; + +exports[`lib/manager/precommit/extract extractPackageFile() extracts from values.yaml correctly with same structure as "pre-commit sample-config" 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "v2.4.0", + "datasource": "github-tags", + "depName": "pre-commit/pre-commit-hooks", + "depType": "repository", + "lookupName": "pre-commit/pre-commit-hooks", + }, + ], +} +`; diff --git a/lib/manager/pre-commit/extract.spec.ts b/lib/manager/pre-commit/extract.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..d2fa9d17ab73fa25d5c6a3c79c9445ea175ae297 --- /dev/null +++ b/lib/manager/pre-commit/extract.spec.ts @@ -0,0 +1,95 @@ +import { readFileSync } from 'fs'; +import { mocked } from '../../../test/util'; +import * as _hostRules from '../../util/host-rules'; +import { extractPackageFile } from './extract'; + +jest.mock('../../util/host-rules'); +const hostRules = mocked(_hostRules); + +const complexPrecommitConfig = readFileSync( + 'lib/manager/pre-commit/__fixtures__/complex.pre-commit-config.yaml', + 'utf8' +); + +const examplePrecommitConfig = readFileSync( + 'lib/manager/pre-commit/__fixtures__/.pre-commit-config.yaml', + 'utf8' +); + +const emptyReposPrecommitConfig = readFileSync( + 'lib/manager/pre-commit/__fixtures__/empty_repos.pre-commit-config.yaml', + 'utf8' +); + +const noReposPrecommitConfig = readFileSync( + 'lib/manager/pre-commit/__fixtures__/no_repos.pre-commit-config.yaml', + 'utf8' +); + +const invalidRepoPrecommitConfig = readFileSync( + 'lib/manager/pre-commit/__fixtures__/invalid_repo.pre-commit-config.yaml', + 'utf8' +); + +const enterpriseGitPrecommitConfig = readFileSync( + 'lib/manager/pre-commit/__fixtures__/enterprise.pre-commit-config.yaml', + 'utf8' +); + +describe('lib/manager/precommit/extract', () => { + describe('extractPackageFile()', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + it('returns null for invalid yaml file content', () => { + const result = extractPackageFile('nothing here: ['); + expect(result).toBeNull(); + }); + it('returns null for empty yaml file content', () => { + const result = extractPackageFile(''); + expect(result).toBeNull(); + }); + it('returns null for no file content', () => { + const result = extractPackageFile(null); + expect(result).toBeNull(); + }); + it('returns null for no repos', () => { + const result = extractPackageFile(noReposPrecommitConfig); + expect(result).toBeNull(); + }); + it('returns null for empty repos', () => { + const result = extractPackageFile(emptyReposPrecommitConfig); + expect(result).toBeNull(); + }); + it('returns null for invalid repo', () => { + const result = extractPackageFile(invalidRepoPrecommitConfig); + expect(result).toBeNull(); + }); + it('extracts from values.yaml correctly with same structure as "pre-commit sample-config"', () => { + const result = extractPackageFile(examplePrecommitConfig); + expect(result).toMatchSnapshot(); + }); + it('extracts from complex config file correctly', () => { + const result = extractPackageFile(complexPrecommitConfig); + expect(result).toMatchSnapshot(); + }); + it('can handle private git repos', () => { + hostRules.find.mockReturnValue({ token: 'value' }); + const result = extractPackageFile(enterpriseGitPrecommitConfig); + expect(result).toMatchSnapshot(); + }); + it('can handle invalid private git repos', () => { + hostRules.find.mockReturnValue({}); + const result = extractPackageFile(enterpriseGitPrecommitConfig); + expect(result).toMatchSnapshot(); + }); + it('can handle unknown private git repos', () => { + // First attemp returns a result + hostRules.find.mockReturnValueOnce({ token: 'value' }); + // But all subsequent checks (those with hostType), then fail: + hostRules.find.mockReturnValue({}); + const result = extractPackageFile(enterpriseGitPrecommitConfig); + expect(result).toMatchSnapshot(); + }); + }); +}); diff --git a/lib/manager/pre-commit/extract.ts b/lib/manager/pre-commit/extract.ts new file mode 100644 index 0000000000000000000000000000000000000000..e49eaa1dadffc9e5825ce45b4a7849407d3daefe --- /dev/null +++ b/lib/manager/pre-commit/extract.ts @@ -0,0 +1,179 @@ +import is from '@sindresorhus/is'; +import yaml from 'js-yaml'; +import { + PLATFORM_TYPE_GITEA, + PLATFORM_TYPE_GITHUB, + PLATFORM_TYPE_GITLAB, +} from '../../constants/platforms'; +import { id as githubTagsId } from '../../datasource/github-tags'; +import { id as gitlabTagsId } from '../../datasource/gitlab-tags'; +import { logger } from '../../logger'; +import { SkipReason } from '../../types'; +import { find } from '../../util/host-rules'; +import { regEx } from '../../util/regex'; +import { PackageDependency, PackageFile } from '../common'; + +import { + matchesPrecommitConfigHeuristic, + matchesPrecommitDependencyHeuristic, +} from './parsing'; +import { PreCommitConfig } from './types'; + +function isEmptyObject(obj: any): boolean { + return Object.keys(obj).length === 0 && obj.constructor === Object; +} + +/** + * Determines the datasource(id) to be used for this dependency + * @param repository the full git url, ie git@github.com/user/project. + * Used in debug statements to clearly indicate the related dependency. + * @param hostName the hostname (ie github.com) + * Used to determine which renovate datasource should be used. + * Is matched literally against `github.com` and `gitlab.com`. + * If that doesn't match, `hostRules.find()` is used to find related sources. + * In that case, the hostname is passed on as registryUrl to the corresponding datasource. + */ +function determineDatasource( + repository: string, + hostName: string +): { datasource?: string; registryUrls?: string[]; skipReason?: SkipReason } { + if (hostName === 'github.com') { + logger.debug({ repository, hostName }, 'Found github dependency'); + return { datasource: githubTagsId }; + } + if (hostName === 'gitlab.com') { + logger.debug({ repository, hostName }, 'Found gitlab dependency'); + return { datasource: gitlabTagsId }; + } + const hostUrl = 'https://' + hostName; + const res = find({ url: hostUrl }); + if (isEmptyObject(res)) { + // 1 check, to possibly prevent 3 failures in combined query of hostType & url. + logger.debug( + { repository, hostUrl }, + 'Provided hostname does not match any hostRules. Ignoring' + ); + return { skipReason: SkipReason.UnknownRegistry, registryUrls: [hostName] }; + } + for (const [hostType, sourceId] of [ + [PLATFORM_TYPE_GITEA, gitlabTagsId], + [PLATFORM_TYPE_GITHUB, githubTagsId], + [PLATFORM_TYPE_GITLAB, gitlabTagsId], + ]) { + if (!isEmptyObject(find({ hostType, url: hostUrl }))) { + logger.debug( + { repository, hostUrl, hostType }, + `Provided hostname matches a ${hostType} hostrule.` + ); + return { datasource: sourceId, registryUrls: [hostName] }; + } + } + logger.debug( + { repository, registry: hostUrl }, + 'Provided hostname did not match any of the hostRules of hostType gitea,github nor gitlab' + ); + return { skipReason: SkipReason.UnknownRegistry, registryUrls: [hostName] }; +} + +function extractDependency( + tag: string, + repository: string +): { + depName?: string; + depType?: string; + datasource?: string; + lookupName?: string; + skipReason?: SkipReason; + currentValue?: string; +} { + logger.debug({ tag }, 'Found version'); + + const urlMatchers = [ + // This splits "http://my.github.com/user/repo" -> "my.github.com" "user/repo + regEx('^https?:\\/\\/(?<hostName>[^\\/]+)\\/(?<depName>\\S*)'), + // This splits "git@private.registry.com:user/repo" -> "private.registry.com" "user/repo + regEx('^git@(?<hostName>[^:]+):(?<depName>\\S*)'), + ]; + for (const urlMatcher of urlMatchers) { + const match = urlMatcher.exec(repository); + if (match) { + const { hostName, depName } = match.groups; + const sourceDef = determineDatasource(repository, hostName); + return { + ...sourceDef, + depName, + depType: 'repository', + lookupName: depName, + currentValue: tag, + }; + } + } + logger.info( + { repository }, + 'Could not separate hostname from full dependency url.' + ); + return { + depName: undefined, + depType: 'repository', + datasource: undefined, + lookupName: undefined, + skipReason: SkipReason.InvalidUrl, + currentValue: tag, + }; +} + +/** + * Find all supported dependencies in the pre-commit yaml object. + * + * @param precommitFile the parsed yaml config file + */ +function findDependencies( + precommitFile: PreCommitConfig +): Array<PackageDependency> { + if (!precommitFile.repos) { + logger.debug(`No repos section found, skipping file`); + return []; + } + const packageDependencies = []; + precommitFile.repos.forEach((item) => { + if (matchesPrecommitDependencyHeuristic(item)) { + logger.trace(item, 'Matched pre-commit dependency spec'); + const repository = String(item.repo); + const tag = String(item.rev); + const dep = extractDependency(tag, repository); + + packageDependencies.push(dep); + } else { + logger.trace(item, 'Did not find pre-commit repo spec'); + } + }); + return packageDependencies; +} + +export function extractPackageFile(content: string): PackageFile | null { + let parsedContent: Record<string, unknown> | PreCommitConfig; + try { + parsedContent = yaml.safeLoad(content, { json: true }) as any; + } catch (err) { + logger.debug({ err }, 'Failed to parse pre-commit config YAML'); + return null; + } + if (!is.plainObject<Record<string, unknown>>(parsedContent)) { + logger.warn(`Parsing of pre-commit config YAML returned invalid result`); + return null; + } + if (!matchesPrecommitConfigHeuristic(parsedContent)) { + logger.info(`File does not look like a pre-commit config file`); + return null; + } + try { + const deps = findDependencies(parsedContent); + if (deps.length) { + logger.debug({ deps }, 'Found dependencies in pre-commit config'); + return { deps }; + } + } catch (err) /* istanbul ignore next */ { + logger.error({ err }, 'Error scanning parsed pre-commit config'); + } + return null; +} diff --git a/lib/manager/pre-commit/index.ts b/lib/manager/pre-commit/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..9f3978c96ec4410bcc5f41ee428e8d05615d171a --- /dev/null +++ b/lib/manager/pre-commit/index.ts @@ -0,0 +1,6 @@ +export { extractPackageFile } from './extract'; + +export const defaultConfig = { + commitMessageTopic: 'precommit hook {{depName}}', + fileMatch: ['(^|/)\\.pre-commit-config\\.yaml$'], +}; diff --git a/lib/manager/pre-commit/parsing.ts b/lib/manager/pre-commit/parsing.ts new file mode 100644 index 0000000000000000000000000000000000000000..642ba64354610c9326afaee32d4ef9e40ae4b912 --- /dev/null +++ b/lib/manager/pre-commit/parsing.ts @@ -0,0 +1,34 @@ +import { hasKey } from '../../util/object'; +import { PreCommitConfig, PreCommitDependency } from './types'; + +/** + * Type guard to determine whether the file matches pre-commit configuration format + * Example original yaml: + * + * repos + * - repo: https://github.com/user/repo + * rev: v1.0.0 + */ +export function matchesPrecommitConfigHeuristic( + data: unknown +): data is PreCommitConfig { + return data && typeof data === 'object' && hasKey('repos', data); +} + +/** + * Type guard to determine whether a given repo definition defines a pre-commit Git hook dependency. + * Example original yaml portion + * + * - repo: https://github.com/user/repo + * rev: v1.0.0 + */ +export function matchesPrecommitDependencyHeuristic( + data: unknown +): data is PreCommitDependency { + return ( + data && + typeof data === 'object' && + hasKey('repo', data) && + hasKey('rev', data) + ); +} diff --git a/lib/manager/pre-commit/readme.md b/lib/manager/pre-commit/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..04146d3e4a1c2e2e297dd4082b7061ee48e334d6 --- /dev/null +++ b/lib/manager/pre-commit/readme.md @@ -0,0 +1,10 @@ +Renovate supports updating of Git dependencies within pre-commit configuration `.pre-commit-config.yaml` files or other YAML files that use the same format (via `fileMatch` configuration). +Updates are performed if the files follow the conventional format used in typical pre-commit files: + +```yaml +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v1.0.0 + hooks: + - id: some-hook-id +``` diff --git a/lib/manager/pre-commit/types.ts b/lib/manager/pre-commit/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..dadcf068e61e9758ed5903e24a57d49c2bfabdbf --- /dev/null +++ b/lib/manager/pre-commit/types.ts @@ -0,0 +1,8 @@ +export interface PreCommitConfig { + repos: PreCommitDependency[]; +} + +export interface PreCommitDependency { + repo: string; + rev: string; +} diff --git a/lib/util/object.spec.ts b/lib/util/object.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..bfc925898efeace5a9f40d5c201f990a620d1404 --- /dev/null +++ b/lib/util/object.spec.ts @@ -0,0 +1,17 @@ +import { hasKey } from './object'; + +describe('util/object', () => { + beforeEach(() => { + jest.resetModules(); + }); + + it('finds key in regular object', () => { + expect(hasKey('foo', { foo: true })).toBeTrue(); + }); + it('detects missing key in regular object', () => { + expect(hasKey('foo', { bar: true })).toBeFalse(); + }); + it('returns false for wrong instance type', () => { + expect(hasKey('foo', 'i-am-not-an-object')).toBeFalse(); + }); +}); diff --git a/lib/util/object.ts b/lib/util/object.ts new file mode 100644 index 0000000000000000000000000000000000000000..e0937d5cd277c6bb644ea8a3271a355d45271d39 --- /dev/null +++ b/lib/util/object.ts @@ -0,0 +1,13 @@ +/** + * This is a workaround helper to allow the usage of 'unknown' in + * a type-guard function while checking that keys exist. + * + * @see https://github.com/microsoft/TypeScript/issues/21732 + * @see https://stackoverflow.com/a/58630274 + */ +export function hasKey<K extends string, T>( + k: K, + o: T +): o is T & Record<K, unknown> { + return typeof o === 'object' && k in o; +}