diff --git a/lib/manager/terragrunt/__fixtures__/2.hcl b/lib/manager/terragrunt/__fixtures__/2.hcl new file mode 100644 index 0000000000000000000000000000000000000000..29d30085460cf2faedb02ceade9910c8cff9567d --- /dev/null +++ b/lib/manager/terragrunt/__fixtures__/2.hcl @@ -0,0 +1,166 @@ +#real +terraform { + extra_arguments "common_vars" { + commands = ["plan", "apply"] + + arguments = [ + "-var-file=../../common.tfvars", + "-var-file=../region.tfvars" + ] + } + + before_hook "before_hook" { + commands = ["apply", "plan"] + execute = ["echo", "Running Terraform"] + } + + source = "github.com/myuser/myrepo//folder/modules/moduleone?ref=v0.0.9" + + after_hook "after_hook" { + commands = ["apply", "plan"] + execute = ["echo", "Finished running Terraform"] + run_on_error = true + } +} + +#foo +terraform { + source = "github.com/hashicorp/example?ref=v1.0.0" +} + +#bar +terraform { + source = "github.com/hashicorp/example?ref=next" +} + +#hostname +terraform { + source = "https://104.196.242.174"example?ref=next" +} + +#local hostname +terraform { + source = "my.host.local/example?ref=v1.2.1" +} + +#local hostname +terraform { + source = "my.host/modules/test" +} + +#local hostname +terraform { + source = "my.host/modules/test?ref=v1.2.1" +} + +#local hostname +terraform { + source = "my.host" +} + +#local hostname +terraform { + source = "my.host.local/sources/example?ref=v1.2.1" +} + +#ip +terraform { + source = "my.host/example?ref=next" +} + +#invalid +terraform { + source = "//terraform/module/test?ref=next" +} + +#repo-with-non-semver-ref +terraform { + source = "github.com/githubuser/myrepo//terraform/modules/moduleone?ref=tfmodule_one-v0.0.9" +} + +#repo-with-dot +terraform { + source = "github.com/hashicorp/example.2.3?ref=v1.0.0" +} + +#repo-with-dot-and-git-suffix +terraform { + source = "github.com/hashicorp/example.2.3.git?ref=v1.0.0" +} + +#source without pinning +terraform { + source = "hashicorp/consul/aws" +} + +# source with double-slash +terraform { + source = "github.com/tieto-cem/terraform-aws-ecs-task-definition//modules/container-definition?ref=v0.1.0" +} + +# regular sources +terraform { + source = "github.com/tieto-cem/terraform-aws-ecs-task-definition?ref=v0.1.0" +} + +terraform { + source = "git@github.com:hashicorp/example.git?ref=v2.0.0" +} + +terraform { + source = "terraform-aws-modules/security-group/aws//modules/http-80" + +} + +terraform { + source = "terraform-aws-modules/security-group/aws" +} + +terraform { + source = "../../terraforms/fe" +} + +# nosource, ignored by test since it does not have source on the next line +terraform { + foo = "bar" +} + +# foobar +terraform { + source = "https://bitbucket.com/hashicorp/example?ref=v1.0.0" +} + +# gittags +terraform { + source = "git::https://bitbucket.com/hashicorp/example?ref=v1.0.0" +} + +# gittags_badversion +terraform { + source = "git::https://bitbucket.com/hashicorp/example?ref=next" +} + +# gittags_subdir +terraform { + source = "git::https://bitbucket.com/hashicorp/example//subdir/test?ref=v1.0.1" +} + +# gittags_http +terraform { + source = "git::http://bitbucket.com/hashicorp/example?ref=v1.0.2" +} + +# gittags_ssh +terraform { + source = "git::ssh://git@bitbucket.com/hashicorp/example?ref=v1.0.3" +} + +# invalid, ignored by test since it does not have source on the next line +terraform { +} + +# unsupported terragrunt, ignored by test since it does not have source on the next line +terraform { + name = "foo" + dummy = "true" +} diff --git a/lib/manager/terragrunt/__snapshots__/extract.spec.ts.snap b/lib/manager/terragrunt/__snapshots__/extract.spec.ts.snap new file mode 100644 index 0000000000000000000000000000000000000000..acc3c3748114b5dd8a6e23b135ab72be514ee995 --- /dev/null +++ b/lib/manager/terragrunt/__snapshots__/extract.spec.ts.snap @@ -0,0 +1,192 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`lib/manager/terragrunt/extract extractPackageFile() extracts terragrunt sources 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "v0.0.9", + "datasource": "github-tags", + "depName": "github.com/myuser/myrepo", + "depNameShort": "myuser/myrepo", + "depType": "github", + "lookupName": "myuser/myrepo", + }, + Object { + "currentValue": "v1.0.0", + "datasource": "github-tags", + "depName": "github.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "github", + "lookupName": "hashicorp/example", + }, + Object { + "currentValue": "next", + "datasource": "github-tags", + "depName": "github.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "github", + "lookupName": "hashicorp/example", + }, + Object { + "skipReason": "no-source", + }, + Object {}, + Object { + "datasource": "terraform-module", + "depName": "my.host/modules/test", + "depNameShort": "my.host/modules/test", + "depType": "terragrunt", + "registryUrls": Array [ + "https://my.host", + ], + }, + Object { + "datasource": "terraform-module", + "depName": "my.host/modules/test?ref=v1.2.1", + "depNameShort": "my.host/modules/test?ref=v1.2.1", + "depType": "terragrunt", + "registryUrls": Array [ + "https://my.host", + ], + }, + Object {}, + Object { + "datasource": "terraform-module", + "depName": "my.host.local/sources/example?ref=v1.2.1", + "depNameShort": "my.host.local/sources/example?ref=v1.2.1", + "depType": "terragrunt", + "registryUrls": Array [ + "https://my.host.local", + ], + }, + Object {}, + Object {}, + Object { + "currentValue": "tfmodule_one-v0.0.9", + "datasource": "github-tags", + "depName": "github.com/githubuser/myrepo", + "depNameShort": "githubuser/myrepo", + "depType": "github", + "lookupName": "githubuser/myrepo", + }, + Object { + "currentValue": "v1.0.0", + "datasource": "github-tags", + "depName": "github.com/hashicorp/example.2.3", + "depNameShort": "hashicorp/example.2.3", + "depType": "github", + "lookupName": "hashicorp/example.2.3", + }, + Object { + "currentValue": "v1.0.0", + "datasource": "github-tags", + "depName": "github.com/hashicorp/example.2.3", + "depNameShort": "hashicorp/example.2.3", + "depType": "github", + "lookupName": "hashicorp/example.2.3", + }, + Object { + "datasource": "terraform-module", + "depName": "hashicorp/consul/aws", + "depNameShort": "hashicorp/consul/aws", + "depType": "terragrunt", + }, + Object { + "currentValue": "v0.1.0", + "datasource": "github-tags", + "depName": "github.com/tieto-cem/terraform-aws-ecs-task-definition", + "depNameShort": "tieto-cem/terraform-aws-ecs-task-definition", + "depType": "github", + "lookupName": "tieto-cem/terraform-aws-ecs-task-definition", + }, + Object { + "currentValue": "v0.1.0", + "datasource": "github-tags", + "depName": "github.com/tieto-cem/terraform-aws-ecs-task-definition", + "depNameShort": "tieto-cem/terraform-aws-ecs-task-definition", + "depType": "github", + "lookupName": "tieto-cem/terraform-aws-ecs-task-definition", + }, + Object { + "currentValue": "v2.0.0", + "datasource": "github-tags", + "depName": "github.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "github", + "lookupName": "hashicorp/example", + }, + Object { + "datasource": "terraform-module", + "depName": "terraform-aws-modules/security-group/aws", + "depNameShort": "terraform-aws-modules/security-group/aws", + "depType": "terragrunt", + }, + Object { + "datasource": "terraform-module", + "depName": "terraform-aws-modules/security-group/aws", + "depNameShort": "terraform-aws-modules/security-group/aws", + "depType": "terragrunt", + }, + Object { + "skipReason": "local", + }, + Object { + "skipReason": "no-source", + }, + Object { + "currentValue": "v1.0.0", + "datasource": "git-tags", + "depName": "bitbucket.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "gitTags", + "lookupName": "https://bitbucket.com/hashicorp/example", + }, + Object { + "currentValue": "v1.0.0", + "datasource": "git-tags", + "depName": "bitbucket.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "gitTags", + "lookupName": "https://bitbucket.com/hashicorp/example", + }, + Object { + "currentValue": "next", + "datasource": "git-tags", + "depName": "bitbucket.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "gitTags", + "lookupName": "https://bitbucket.com/hashicorp/example", + }, + Object { + "currentValue": "v1.0.1", + "datasource": "git-tags", + "depName": "bitbucket.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "gitTags", + "lookupName": "https://bitbucket.com/hashicorp/example", + }, + Object { + "currentValue": "v1.0.2", + "datasource": "git-tags", + "depName": "bitbucket.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "gitTags", + "lookupName": "http://bitbucket.com/hashicorp/example", + }, + Object { + "currentValue": "v1.0.3", + "datasource": "git-tags", + "depName": "bitbucket.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "gitTags", + "lookupName": "ssh://git@bitbucket.com/hashicorp/example", + }, + Object { + "skipReason": "no-source", + }, + Object { + "skipReason": "no-source", + }, + ], +} +`; diff --git a/lib/manager/terragrunt/extract.spec.ts b/lib/manager/terragrunt/extract.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..c0b2bc1f735981dd0559de94cb5db48dd3c2158e --- /dev/null +++ b/lib/manager/terragrunt/extract.spec.ts @@ -0,0 +1,25 @@ +import { readFileSync } from 'fs'; +import { extractPackageFile } from './extract'; + +const tg1 = readFileSync('lib/manager/terragrunt/__fixtures__/2.hcl', 'utf8'); +const tg2 = `terragrunt { + source = "../../modules/fe" +} +`; + +describe('lib/manager/terragrunt/extract', () => { + describe('extractPackageFile()', () => { + it('returns null for empty', () => { + expect(extractPackageFile('nothing here')).toBeNull(); + }); + it('extracts terragrunt sources', () => { + const res = extractPackageFile(tg1); + expect(res).toMatchSnapshot(); + expect(res.deps).toHaveLength(30); + expect(res.deps.filter((dep) => dep.skipReason)).toHaveLength(5); + }); + it('returns null if only local terragrunt deps', () => { + expect(extractPackageFile(tg2)).toBeNull(); + }); + }); +}); diff --git a/lib/manager/terragrunt/extract.ts b/lib/manager/terragrunt/extract.ts new file mode 100644 index 0000000000000000000000000000000000000000..374ad9377c505f87bd0c5f46984b1b2c11268a1e --- /dev/null +++ b/lib/manager/terragrunt/extract.ts @@ -0,0 +1,67 @@ +import { logger } from '../../logger'; +import { PackageDependency, PackageFile } from '../common'; +import { analyseTerragruntModule, extractTerragruntModule } from './modules'; +import { + TerraformManagerData, + TerragruntDependencyTypes, + checkFileContainsDependency, + getTerragruntDependencyType, +} from './util'; + +const dependencyBlockExtractionRegex = /^\s*(?<type>[a-z_]+)\s+{\s*$/; +const contentCheckList = ['terraform {']; + +export function extractPackageFile(content: string): PackageFile | null { + logger.trace({ content }, 'terragrunt.extractPackageFile()'); + if (!checkFileContainsDependency(content, contentCheckList)) { + return null; + } + let deps: PackageDependency<TerraformManagerData>[] = []; + try { + const lines = content.split('\n'); + for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) { + const line = lines[lineNumber]; + const terragruntDependency = dependencyBlockExtractionRegex.exec(line); + if (terragruntDependency) { + logger.trace( + `Matched ${terragruntDependency.groups.type} on line ${lineNumber}` + ); + const tfDepType = getTerragruntDependencyType( + terragruntDependency.groups.type + ); + let result = null; + switch (tfDepType) { + case TerragruntDependencyTypes.terragrunt: { + result = extractTerragruntModule(lineNumber, lines); + break; + } + /* istanbul ignore next */ + default: + logger.trace( + `Could not identify TerragruntDependencyType ${terragruntDependency.groups.type} on line ${lineNumber}.` + ); + break; + } + if (result) { + lineNumber = result.lineNumber; + deps = deps.concat(result.dependencies); + result = null; + } + } + } + } catch (err) /* istanbul ignore next */ { + logger.warn({ err }, 'Error extracting terragrunt plugins'); + } + deps.forEach((dep) => { + switch (dep.managerData.terragruntDependencyType) { + case TerragruntDependencyTypes.terragrunt: + analyseTerragruntModule(dep); + break; + /* istanbul ignore next */ + default: + } + // eslint-disable-next-line no-param-reassign + delete dep.managerData; + }); + return { deps }; +} diff --git a/lib/manager/terragrunt/index.ts b/lib/manager/terragrunt/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d54225041a167fba0b145ae70b0c54f06e570fb --- /dev/null +++ b/lib/manager/terragrunt/index.ts @@ -0,0 +1,9 @@ +import * as hashicorpVersioning from '../../versioning/hashicorp'; + +export { extractPackageFile } from './extract'; + +export const defaultConfig = { + commitMessageTopic: 'Terragrunt dependency {{depNameShort}}', + fileMatch: ['(^|/)terragrunt\\.hcl$'], + versioning: hashicorpVersioning.id, +}; diff --git a/lib/manager/terragrunt/modules.ts b/lib/manager/terragrunt/modules.ts new file mode 100644 index 0000000000000000000000000000000000000000..ccce01e1928878377029fb8049ceada2604e4be3 --- /dev/null +++ b/lib/manager/terragrunt/modules.ts @@ -0,0 +1,74 @@ +import * as datasourceGitTags from '../../datasource/git-tags'; +import * as datasourceGithubTags from '../../datasource/github-tags'; +import * as datasourceTerragruntModule from '../../datasource/terraform-module'; +import { logger } from '../../logger'; +import { SkipReason } from '../../types'; +import { PackageDependency } from '../common'; +import { extractTerragruntProvider } from './providers'; +import { ExtractionResult, TerragruntDependencyTypes } from './util'; + +const githubRefMatchRegex = /github.com([/:])(?<project>[^/]+\/[a-z0-9-.]+).*\?ref=(?<tag>.*)$/; +const gitTagsRefMatchRegex = /(?:git::)?(?<url>(?:http|https|ssh):\/\/(?:.*@)?(?<path>.*.*\/(?<project>.*\/.*)))\?ref=(?<tag>.*)$/; +const hostnameMatchRegex = /^(?<hostname>([\w|\d]+\.)+[\w|\d]+)/; + +export function extractTerragruntModule( + startingLine: number, + lines: string[] +): ExtractionResult { + const moduleName = 'terragrunt'; + const result = extractTerragruntProvider(startingLine, lines, moduleName); + result.dependencies.forEach((dep) => { + // eslint-disable-next-line no-param-reassign + dep.managerData.terragruntDependencyType = + TerragruntDependencyTypes.terragrunt; + }); + return result; +} + +export function analyseTerragruntModule(dep: PackageDependency): void { + const githubRefMatch = githubRefMatchRegex.exec(dep.managerData.source); + const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source); + /* eslint-disable no-param-reassign */ + if (githubRefMatch) { + const depNameShort = githubRefMatch.groups.project.replace(/\.git$/, ''); + dep.depType = 'github'; + dep.depName = 'github.com/' + depNameShort; + dep.depNameShort = depNameShort; + dep.currentValue = githubRefMatch.groups.tag; + dep.datasource = datasourceGithubTags.id; + dep.lookupName = depNameShort; + } else if (gitTagsRefMatch) { + dep.depType = 'gitTags'; + if (gitTagsRefMatch.groups.path.includes('//')) { + logger.debug('Terragrunt module contains subdirectory'); + dep.depName = gitTagsRefMatch.groups.path.split('//')[0]; + dep.depNameShort = dep.depName.split(/\/(.+)/)[1]; + const tempLookupName = gitTagsRefMatch.groups.url.split('//'); + dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1]; + } else { + dep.depName = gitTagsRefMatch.groups.path.replace('.git', ''); + dep.depNameShort = gitTagsRefMatch.groups.project.replace('.git', ''); + dep.lookupName = gitTagsRefMatch.groups.url; + } + dep.currentValue = gitTagsRefMatch.groups.tag; + dep.datasource = datasourceGitTags.id; + } else if (dep.managerData.source) { + const moduleParts = dep.managerData.source.split('//')[0].split('/'); + if (moduleParts[0] === '..') { + dep.skipReason = SkipReason.Local; + } else if (moduleParts.length >= 3) { + const hostnameMatch = hostnameMatchRegex.exec(dep.managerData.source); + if (hostnameMatch) { + dep.registryUrls = [`https://${hostnameMatch.groups.hostname}`]; + } + dep.depType = 'terragrunt'; + dep.depName = moduleParts.join('/'); + dep.depNameShort = dep.depName; + dep.datasource = datasourceTerragruntModule.id; + } + } else { + logger.debug({ dep }, 'terragrunt dep has no source'); + dep.skipReason = SkipReason.NoSource; + } + /* eslint-enable no-param-reassign */ +} diff --git a/lib/manager/terragrunt/providers.ts b/lib/manager/terragrunt/providers.ts new file mode 100644 index 0000000000000000000000000000000000000000..d73fb8145cac92b109e34964762f97d364c15250 --- /dev/null +++ b/lib/manager/terragrunt/providers.ts @@ -0,0 +1,56 @@ +import { PackageDependency } from '../common'; +import { + ExtractionResult, + TerragruntDependencyTypes, + keyValueExtractionRegex, +} from './util'; + +export const sourceExtractionRegex = /^(?:(?<hostname>(?:[a-zA-Z0-9]+\.+)+[a-zA-Z0-9]+)\/)?(?:(?<namespace>[^/]+)\/)?(?<type>[^/]+)/; + +function extractBracesContent(content): number { + const stack = []; + let i = 0; + for (i; i < content.length; i += 1) { + if (content[i] === '{') { + stack.push(content[i]); + } else if (content[i] === '}') { + stack.pop(); + if (stack.length === 0) { + break; + } + } + } + return i; +} + +export function extractTerragruntProvider( + startingLine: number, + lines: string[], + moduleName: string +): ExtractionResult { + const lineNumber = startingLine; + let line: string; + const deps: PackageDependency[] = []; + const dep: PackageDependency = { + managerData: { + moduleName, + terragruntDependencyType: TerragruntDependencyTypes.terragrunt, + }, + }; + const teraformContent = lines + .slice(lineNumber) + .join('\n') + .substring(0, extractBracesContent(lines.slice(lineNumber).join('\n'))) + .split('\n'); + + for (let lineNo = 0; lineNo < teraformContent.length; lineNo += 1) { + line = teraformContent[lineNo]; + const kvMatch = keyValueExtractionRegex.exec(line); + if (kvMatch) { + dep.managerData.source = kvMatch.groups.value; + dep.managerData.sourceLine = lineNumber + lineNo; + } + } + deps.push(dep); + return { lineNumber, dependencies: deps }; +} diff --git a/lib/manager/terragrunt/readme.md b/lib/manager/terragrunt/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..146136a1350c4cc5142f0f7a24f737aeb20bb26c --- /dev/null +++ b/lib/manager/terragrunt/readme.md @@ -0,0 +1,18 @@ +Currently by default, Terragrunt support is limited to Terraform registry sources and GitHub sources that include semver refs, e.g. like `github.com/hashicorp/example?ref=v1.0.0`. + +You can create a custom [versioning config](../../../docs/usage/configuration-options.md#versioning) to support non-semver references. +For example, if you want to reference a tag like `module-v1.2.5`, a block like this would work: + +```json +"terraform": { + "versioning": "regex:^((?<compatibility>.*)-v|v*)(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)$" +} +``` + +Pinned Terragrunt dependencies like the following will receive a PR whenever there is a newer version available: + +```hcl +terraform { + source = "github.com/hashicorp/example?ref=v1.0.0" +} +``` diff --git a/lib/manager/terragrunt/util.spec.ts b/lib/manager/terragrunt/util.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..39e625a3d8ec73dab79402c6f643cfdf7c7a2916 --- /dev/null +++ b/lib/manager/terragrunt/util.spec.ts @@ -0,0 +1,26 @@ +import { TerragruntDependencyTypes, getTerragruntDependencyType } from './util'; + +describe('lib/manager/terragrunt/extract', () => { + describe('getTerragruntDependencyType()', () => { + it('returns TerragruntDependencyTypes.terragrunt', () => { + expect(getTerragruntDependencyType('terraform')).toBe( + TerragruntDependencyTypes.terragrunt + ); + }); + it('returns TerragruntDependencyTypes.unknown', () => { + expect(getTerragruntDependencyType('unknown')).toBe( + TerragruntDependencyTypes.unknown + ); + }); + it('returns TerragruntDependencyTypes.unknown on empty string', () => { + expect(getTerragruntDependencyType('')).toBe( + TerragruntDependencyTypes.unknown + ); + }); + it('returns TerragruntDependencyTypes.unknown on string with random chars', () => { + expect(getTerragruntDependencyType('sdfsgdsfadfhfghfhgdfsdf')).toBe( + TerragruntDependencyTypes.unknown + ); + }); + }); +}); diff --git a/lib/manager/terragrunt/util.ts b/lib/manager/terragrunt/util.ts new file mode 100644 index 0000000000000000000000000000000000000000..97e3cd96b0caabbc68b14db1aa5b75296b9e891c --- /dev/null +++ b/lib/manager/terragrunt/util.ts @@ -0,0 +1,54 @@ +import { PackageDependency } from '../common'; + +export const keyValueExtractionRegex = /^\s*source\s+=\s+"(?<value>[^"]+)"\s*$/; + +export interface ExtractionResult { + lineNumber: number; + dependencies: PackageDependency[]; +} + +export enum TerragruntDependencyTypes { + unknown = 'unknown', + terragrunt = 'terraform', +} + +export interface TerraformManagerData { + terragruntDependencyType: TerragruntDependencyTypes; +} + +export enum TerragruntResourceTypes { + unknown = 'unknown', + /** + * https://www.terraform.io/docs/providers/docker/r/container.html + */ +} + +export interface ResourceManagerData extends TerraformManagerData { + resourceType?: TerragruntResourceTypes; + chart?: string; + image?: string; + name?: string; + repository?: string; +} + +export function getTerragruntDependencyType( + value: string +): TerragruntDependencyTypes { + switch (value) { + case 'terraform': { + return TerragruntDependencyTypes.terragrunt; + } + default: { + return TerragruntDependencyTypes.unknown; + } + } +} + +export function checkFileContainsDependency( + content: string, + checkList: string[] +): boolean { + return checkList.some((check) => { + return content.includes(check); + }); +}