feat: terragrunt support (#7653)

lib/manager/terragrunt/__fixtures__/2.hcl

@@ -0,0 +1,166 @@
+#real
+terraform {
+  extra_arguments "common_vars" {
+    commands = ["plan", "apply"]
+
+    arguments = [
+      "-var-file=../../common.tfvars",
+      "-var-file=../region.tfvars"
+    ]
+  }
+
+  before_hook "before_hook" {
+    commands     = ["apply", "plan"]
+    execute      = ["echo", "Running Terraform"]
+  }
+
+  source = "github.com/myuser/myrepo//folder/modules/moduleone?ref=v0.0.9"
+
+  after_hook "after_hook" {
+    commands     = ["apply", "plan"]
+    execute      = ["echo", "Finished running Terraform"]
+    run_on_error = true
+  }
+}
+
+#foo
+terraform {
+  source = "github.com/hashicorp/example?ref=v1.0.0"
+}
+
+#bar
+terraform {
+  source = "github.com/hashicorp/example?ref=next"
+}
+
+#hostname
+terraform {
+  source = "https://104.196.242.174"example?ref=next"
+}
+
+#local hostname
+terraform {
+  source = "my.host.local/example?ref=v1.2.1"
+}
+
+#local hostname
+terraform {
+  source = "my.host/modules/test"
+}
+
+#local hostname
+terraform {
+  source = "my.host/modules/test?ref=v1.2.1"
+}
+
+#local hostname
+terraform {
+  source = "my.host"
+}
+
+#local hostname
+terraform {
+  source = "my.host.local/sources/example?ref=v1.2.1"
+}
+
+#ip
+terraform {
+  source = "my.host/example?ref=next"
+}
+
+#invalid
+terraform {
+  source = "//terraform/module/test?ref=next"
+}
+
+#repo-with-non-semver-ref
+terraform {
+  source = "github.com/githubuser/myrepo//terraform/modules/moduleone?ref=tfmodule_one-v0.0.9"
+}
+
+#repo-with-dot
+terraform {
+  source = "github.com/hashicorp/example.2.3?ref=v1.0.0"
+}
+
+#repo-with-dot-and-git-suffix
+terraform {
+  source = "github.com/hashicorp/example.2.3.git?ref=v1.0.0"
+}
+
+#source without pinning
+terraform {
+  source  = "hashicorp/consul/aws"
+}
+
+# source with double-slash
+terraform {
+  source         = "github.com/tieto-cem/terraform-aws-ecs-task-definition//modules/container-definition?ref=v0.1.0"
+}
+
+# regular sources
+terraform {
+  source = "github.com/tieto-cem/terraform-aws-ecs-task-definition?ref=v0.1.0"
+}
+
+terraform {
+  source = "git@github.com:hashicorp/example.git?ref=v2.0.0"
+}
+
+terraform {
+  source = "terraform-aws-modules/security-group/aws//modules/http-80"
+
+}
+
+terraform {
+  source  = "terraform-aws-modules/security-group/aws"
+}
+
+terraform {
+  source = "../../terraforms/fe"
+}
+
+# nosource, ignored by test since it does not have source on the next line
+terraform {
+  foo = "bar"
+}
+
+# foobar
+terraform {
+  source = "https://bitbucket.com/hashicorp/example?ref=v1.0.0"
+}
+
+# gittags
+terraform {
+  source = "git::https://bitbucket.com/hashicorp/example?ref=v1.0.0"
+}
+
+# gittags_badversion
+terraform {
+  source = "git::https://bitbucket.com/hashicorp/example?ref=next"
+}
+
+# gittags_subdir
+terraform {
+  source = "git::https://bitbucket.com/hashicorp/example//subdir/test?ref=v1.0.1"
+}
+
+# gittags_http
+terraform {
+  source = "git::http://bitbucket.com/hashicorp/example?ref=v1.0.2"
+}
+
+# gittags_ssh
+terraform {
+  source = "git::ssh://git@bitbucket.com/hashicorp/example?ref=v1.0.3"
+}
+
+# invalid, ignored by test since it does not have source on the next line
+terraform {
+}
+
+# unsupported terragrunt, ignored by test since it does not have source on the next line
+terraform {
+  name  = "foo"
+  dummy = "true"
+}

lib/manager/terragrunt/__snapshots__/extract.spec.ts.snap

@@ -0,0 +1,192 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`lib/manager/terragrunt/extract extractPackageFile() extracts terragrunt sources 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "v0.0.9",
+      "datasource": "github-tags",
+      "depName": "github.com/myuser/myrepo",
+      "depNameShort": "myuser/myrepo",
+      "depType": "github",
+      "lookupName": "myuser/myrepo",
+    },
+    Object {
+      "currentValue": "v1.0.0",
+      "datasource": "github-tags",
+      "depName": "github.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "github",
+      "lookupName": "hashicorp/example",
+    },
+    Object {
+      "currentValue": "next",
+      "datasource": "github-tags",
+      "depName": "github.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "github",
+      "lookupName": "hashicorp/example",
+    },
+    Object {
+      "skipReason": "no-source",
+    },
+    Object {},
+    Object {
+      "datasource": "terraform-module",
+      "depName": "my.host/modules/test",
+      "depNameShort": "my.host/modules/test",
+      "depType": "terragrunt",
+      "registryUrls": Array [
+        "https://my.host",
+      ],
+    },
+    Object {
+      "datasource": "terraform-module",
+      "depName": "my.host/modules/test?ref=v1.2.1",
+      "depNameShort": "my.host/modules/test?ref=v1.2.1",
+      "depType": "terragrunt",
+      "registryUrls": Array [
+        "https://my.host",
+      ],
+    },
+    Object {},
+    Object {
+      "datasource": "terraform-module",
+      "depName": "my.host.local/sources/example?ref=v1.2.1",
+      "depNameShort": "my.host.local/sources/example?ref=v1.2.1",
+      "depType": "terragrunt",
+      "registryUrls": Array [
+        "https://my.host.local",
+      ],
+    },
+    Object {},
+    Object {},
+    Object {
+      "currentValue": "tfmodule_one-v0.0.9",
+      "datasource": "github-tags",
+      "depName": "github.com/githubuser/myrepo",
+      "depNameShort": "githubuser/myrepo",
+      "depType": "github",
+      "lookupName": "githubuser/myrepo",
+    },
+    Object {
+      "currentValue": "v1.0.0",
+      "datasource": "github-tags",
+      "depName": "github.com/hashicorp/example.2.3",
+      "depNameShort": "hashicorp/example.2.3",
+      "depType": "github",
+      "lookupName": "hashicorp/example.2.3",
+    },
+    Object {
+      "currentValue": "v1.0.0",
+      "datasource": "github-tags",
+      "depName": "github.com/hashicorp/example.2.3",
+      "depNameShort": "hashicorp/example.2.3",
+      "depType": "github",
+      "lookupName": "hashicorp/example.2.3",
+    },
+    Object {
+      "datasource": "terraform-module",
+      "depName": "hashicorp/consul/aws",
+      "depNameShort": "hashicorp/consul/aws",
+      "depType": "terragrunt",
+    },
+    Object {
+      "currentValue": "v0.1.0",
+      "datasource": "github-tags",
+      "depName": "github.com/tieto-cem/terraform-aws-ecs-task-definition",
+      "depNameShort": "tieto-cem/terraform-aws-ecs-task-definition",
+      "depType": "github",
+      "lookupName": "tieto-cem/terraform-aws-ecs-task-definition",
+    },
+    Object {
+      "currentValue": "v0.1.0",
+      "datasource": "github-tags",
+      "depName": "github.com/tieto-cem/terraform-aws-ecs-task-definition",
+      "depNameShort": "tieto-cem/terraform-aws-ecs-task-definition",
+      "depType": "github",
+      "lookupName": "tieto-cem/terraform-aws-ecs-task-definition",
+    },
+    Object {
+      "currentValue": "v2.0.0",
+      "datasource": "github-tags",
+      "depName": "github.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "github",
+      "lookupName": "hashicorp/example",
+    },
+    Object {
+      "datasource": "terraform-module",
+      "depName": "terraform-aws-modules/security-group/aws",
+      "depNameShort": "terraform-aws-modules/security-group/aws",
+      "depType": "terragrunt",
+    },
+    Object {
+      "datasource": "terraform-module",
+      "depName": "terraform-aws-modules/security-group/aws",
+      "depNameShort": "terraform-aws-modules/security-group/aws",
+      "depType": "terragrunt",
+    },
+    Object {
+      "skipReason": "local",
+    },
+    Object {
+      "skipReason": "no-source",
+    },
+    Object {
+      "currentValue": "v1.0.0",
+      "datasource": "git-tags",
+      "depName": "bitbucket.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "gitTags",
+      "lookupName": "https://bitbucket.com/hashicorp/example",
+    },
+    Object {
+      "currentValue": "v1.0.0",
+      "datasource": "git-tags",
+      "depName": "bitbucket.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "gitTags",
+      "lookupName": "https://bitbucket.com/hashicorp/example",
+    },
+    Object {
+      "currentValue": "next",
+      "datasource": "git-tags",
+      "depName": "bitbucket.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "gitTags",
+      "lookupName": "https://bitbucket.com/hashicorp/example",
+    },
+    Object {
+      "currentValue": "v1.0.1",
+      "datasource": "git-tags",
+      "depName": "bitbucket.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "gitTags",
+      "lookupName": "https://bitbucket.com/hashicorp/example",
+    },
+    Object {
+      "currentValue": "v1.0.2",
+      "datasource": "git-tags",
+      "depName": "bitbucket.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "gitTags",
+      "lookupName": "http://bitbucket.com/hashicorp/example",
+    },
+    Object {
+      "currentValue": "v1.0.3",
+      "datasource": "git-tags",
+      "depName": "bitbucket.com/hashicorp/example",
+      "depNameShort": "hashicorp/example",
+      "depType": "gitTags",
+      "lookupName": "ssh://git@bitbucket.com/hashicorp/example",
+    },
+    Object {
+      "skipReason": "no-source",
+    },
+    Object {
+      "skipReason": "no-source",
+    },
+  ],
+}
+`;

lib/manager/terragrunt/extract.spec.ts

@@ -0,0 +1,25 @@
+import { readFileSync } from 'fs';
+import { extractPackageFile } from './extract';
+
+const tg1 = readFileSync('lib/manager/terragrunt/__fixtures__/2.hcl', 'utf8');
+const tg2 = `terragrunt {
+  source = "../../modules/fe"
+}
+`;
+
+describe('lib/manager/terragrunt/extract', () => {
+  describe('extractPackageFile()', () => {
+    it('returns null for empty', () => {
+      expect(extractPackageFile('nothing here')).toBeNull();
+    });
+    it('extracts terragrunt sources', () => {
+      const res = extractPackageFile(tg1);
+      expect(res).toMatchSnapshot();
+      expect(res.deps).toHaveLength(30);
+      expect(res.deps.filter((dep) => dep.skipReason)).toHaveLength(5);
+    });
+    it('returns null if only local terragrunt deps', () => {
+      expect(extractPackageFile(tg2)).toBeNull();
+    });
+  });
+});

lib/manager/terragrunt/extract.ts

@@ -0,0 +1,67 @@
+import { logger } from '../../logger';
+import { PackageDependency, PackageFile } from '../common';
+import { analyseTerragruntModule, extractTerragruntModule } from './modules';
+import {
+  TerraformManagerData,
+  TerragruntDependencyTypes,
+  checkFileContainsDependency,
+  getTerragruntDependencyType,
+} from './util';
+
+const dependencyBlockExtractionRegex = /^\s*(?<type>[a-z_]+)\s+{\s*$/;
+const contentCheckList = ['terraform {'];
+
+export function extractPackageFile(content: string): PackageFile | null {
+  logger.trace({ content }, 'terragrunt.extractPackageFile()');
+  if (!checkFileContainsDependency(content, contentCheckList)) {
+    return null;
+  }
+  let deps: PackageDependency<TerraformManagerData>[] = [];
+  try {
+    const lines = content.split('\n');
+    for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
+      const line = lines[lineNumber];
+      const terragruntDependency = dependencyBlockExtractionRegex.exec(line);
+      if (terragruntDependency) {
+        logger.trace(
+          `Matched ${terragruntDependency.groups.type} on line ${lineNumber}`
+        );
+        const tfDepType = getTerragruntDependencyType(
+          terragruntDependency.groups.type
+        );
+        let result = null;
+        switch (tfDepType) {
+          case TerragruntDependencyTypes.terragrunt: {
+            result = extractTerragruntModule(lineNumber, lines);
+            break;
+          }
+          /* istanbul ignore next */
+          default:
+            logger.trace(
+              `Could not identify TerragruntDependencyType ${terragruntDependency.groups.type} on line ${lineNumber}.`
+            );
+            break;
+        }
+        if (result) {
+          lineNumber = result.lineNumber;
+          deps = deps.concat(result.dependencies);
+          result = null;
+        }
+      }
+    }
+  } catch (err) /* istanbul ignore next */ {
+    logger.warn({ err }, 'Error extracting terragrunt plugins');
+  }
+  deps.forEach((dep) => {
+    switch (dep.managerData.terragruntDependencyType) {
+      case TerragruntDependencyTypes.terragrunt:
+        analyseTerragruntModule(dep);
+        break;
+      /* istanbul ignore next */
+      default:
+    }
+    // eslint-disable-next-line no-param-reassign
+    delete dep.managerData;
+  });
+  return { deps };
+}

lib/manager/terragrunt/index.ts

@@ -0,0 +1,9 @@
+import * as hashicorpVersioning from '../../versioning/hashicorp';
+
+export { extractPackageFile } from './extract';
+
+export const defaultConfig = {
+  commitMessageTopic: 'Terragrunt dependency {{depNameShort}}',
+  fileMatch: ['(^|/)terragrunt\\.hcl$'],
+  versioning: hashicorpVersioning.id,
+};

lib/manager/terragrunt/modules.ts

@@ -0,0 +1,74 @@
+import * as datasourceGitTags from '../../datasource/git-tags';
+import * as datasourceGithubTags from '../../datasource/github-tags';
+import * as datasourceTerragruntModule from '../../datasource/terraform-module';
+import { logger } from '../../logger';
+import { SkipReason } from '../../types';
+import { PackageDependency } from '../common';
+import { extractTerragruntProvider } from './providers';
+import { ExtractionResult, TerragruntDependencyTypes } from './util';
+
+const githubRefMatchRegex = /github.com([/:])(?<project>[^/]+\/[a-z0-9-.]+).*\?ref=(?<tag>.*)$/;
+const gitTagsRefMatchRegex = /(?:git::)?(?<url>(?:http|https|ssh):\/\/(?:.*@)?(?<path>.*.*\/(?<project>.*\/.*)))\?ref=(?<tag>.*)$/;
+const hostnameMatchRegex = /^(?<hostname>([\w|\d]+\.)+[\w|\d]+)/;
+
+export function extractTerragruntModule(
+  startingLine: number,
+  lines: string[]
+): ExtractionResult {
+  const moduleName = 'terragrunt';
+  const result = extractTerragruntProvider(startingLine, lines, moduleName);
+  result.dependencies.forEach((dep) => {
+    // eslint-disable-next-line no-param-reassign
+    dep.managerData.terragruntDependencyType =
+      TerragruntDependencyTypes.terragrunt;
+  });
+  return result;
+}
+
+export function analyseTerragruntModule(dep: PackageDependency): void {
+  const githubRefMatch = githubRefMatchRegex.exec(dep.managerData.source);
+  const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source);
+  /* eslint-disable no-param-reassign */
+  if (githubRefMatch) {
+    const depNameShort = githubRefMatch.groups.project.replace(/\.git$/, '');
+    dep.depType = 'github';
+    dep.depName = 'github.com/' + depNameShort;
+    dep.depNameShort = depNameShort;
+    dep.currentValue = githubRefMatch.groups.tag;
+    dep.datasource = datasourceGithubTags.id;
+    dep.lookupName = depNameShort;
+  } else if (gitTagsRefMatch) {
+    dep.depType = 'gitTags';
+    if (gitTagsRefMatch.groups.path.includes('//')) {
+      logger.debug('Terragrunt module contains subdirectory');
+      dep.depName = gitTagsRefMatch.groups.path.split('//')[0];
+      dep.depNameShort = dep.depName.split(/\/(.+)/)[1];
+      const tempLookupName = gitTagsRefMatch.groups.url.split('//');
+      dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1];
+    } else {
+      dep.depName = gitTagsRefMatch.groups.path.replace('.git', '');
+      dep.depNameShort = gitTagsRefMatch.groups.project.replace('.git', '');
+      dep.lookupName = gitTagsRefMatch.groups.url;
+    }
+    dep.currentValue = gitTagsRefMatch.groups.tag;
+    dep.datasource = datasourceGitTags.id;
+  } else if (dep.managerData.source) {
+    const moduleParts = dep.managerData.source.split('//')[0].split('/');
+    if (moduleParts[0] === '..') {
+      dep.skipReason = SkipReason.Local;
+    } else if (moduleParts.length >= 3) {
+      const hostnameMatch = hostnameMatchRegex.exec(dep.managerData.source);
+      if (hostnameMatch) {
+        dep.registryUrls = [`https://${hostnameMatch.groups.hostname}`];
+      }
+      dep.depType = 'terragrunt';
+      dep.depName = moduleParts.join('/');
+      dep.depNameShort = dep.depName;
+      dep.datasource = datasourceTerragruntModule.id;
+    }
+  } else {
+    logger.debug({ dep }, 'terragrunt dep has no source');
+    dep.skipReason = SkipReason.NoSource;
+  }
+  /* eslint-enable no-param-reassign */
+}

lib/manager/terragrunt/providers.ts

@@ -0,0 +1,56 @@
+import { PackageDependency } from '../common';
+import {
+  ExtractionResult,
+  TerragruntDependencyTypes,
+  keyValueExtractionRegex,
+} from './util';
+
+export const sourceExtractionRegex = /^(?:(?<hostname>(?:[a-zA-Z0-9]+\.+)+[a-zA-Z0-9]+)\/)?(?:(?<namespace>[^/]+)\/)?(?<type>[^/]+)/;
+
+function extractBracesContent(content): number {
+  const stack = [];
+  let i = 0;
+  for (i; i < content.length; i += 1) {
+    if (content[i] === '{') {
+      stack.push(content[i]);
+    } else if (content[i] === '}') {
+      stack.pop();
+      if (stack.length === 0) {
+        break;
+      }
+    }
+  }
+  return i;
+}
+
+export function extractTerragruntProvider(
+  startingLine: number,
+  lines: string[],
+  moduleName: string
+): ExtractionResult {
+  const lineNumber = startingLine;
+  let line: string;
+  const deps: PackageDependency[] = [];
+  const dep: PackageDependency = {
+    managerData: {
+      moduleName,
+      terragruntDependencyType: TerragruntDependencyTypes.terragrunt,
+    },
+  };
+  const teraformContent = lines
+    .slice(lineNumber)
+    .join('\n')
+    .substring(0, extractBracesContent(lines.slice(lineNumber).join('\n')))
+    .split('\n');
+
+  for (let lineNo = 0; lineNo < teraformContent.length; lineNo += 1) {
+    line = teraformContent[lineNo];
+    const kvMatch = keyValueExtractionRegex.exec(line);
+    if (kvMatch) {
+      dep.managerData.source = kvMatch.groups.value;
+      dep.managerData.sourceLine = lineNumber + lineNo;
+    }
+  }
+  deps.push(dep);
+  return { lineNumber, dependencies: deps };
+}

lib/manager/terragrunt/readme.md

@@ -0,0 +1,18 @@
+Currently by default, Terragrunt support is limited to Terraform registry sources and GitHub sources that include semver refs, e.g. like `github.com/hashicorp/example?ref=v1.0.0`.
+
+You can create a custom [versioning config](../../../docs/usage/configuration-options.md#versioning) to support non-semver references.
+For example, if you want to reference a tag like `module-v1.2.5`, a block like this would work:
+
+```json
+"terraform": {
+	"versioning": "regex:^((?<compatibility>.*)-v|v*)(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)$"
+}
+```
+
+Pinned Terragrunt dependencies like the following will receive a PR whenever there is a newer version available:
+
+```hcl
+terraform {
+  source = "github.com/hashicorp/example?ref=v1.0.0"
+}
+```

lib/manager/terragrunt/util.spec.ts

@@ -0,0 +1,26 @@
+import { TerragruntDependencyTypes, getTerragruntDependencyType } from './util';
+
+describe('lib/manager/terragrunt/extract', () => {
+  describe('getTerragruntDependencyType()', () => {
+    it('returns TerragruntDependencyTypes.terragrunt', () => {
+      expect(getTerragruntDependencyType('terraform')).toBe(
+        TerragruntDependencyTypes.terragrunt
+      );
+    });
+    it('returns TerragruntDependencyTypes.unknown', () => {
+      expect(getTerragruntDependencyType('unknown')).toBe(
+        TerragruntDependencyTypes.unknown
+      );
+    });
+    it('returns TerragruntDependencyTypes.unknown on empty string', () => {
+      expect(getTerragruntDependencyType('')).toBe(
+        TerragruntDependencyTypes.unknown
+      );
+    });
+    it('returns TerragruntDependencyTypes.unknown on string with random chars', () => {
+      expect(getTerragruntDependencyType('sdfsgdsfadfhfghfhgdfsdf')).toBe(
+        TerragruntDependencyTypes.unknown
+      );
+    });
+  });
+});

lib/manager/terragrunt/util.ts

@@ -0,0 +1,54 @@
+import { PackageDependency } from '../common';
+
+export const keyValueExtractionRegex = /^\s*source\s+=\s+"(?<value>[^"]+)"\s*$/;
+
+export interface ExtractionResult {
+  lineNumber: number;
+  dependencies: PackageDependency[];
+}
+
+export enum TerragruntDependencyTypes {
+  unknown = 'unknown',
+  terragrunt = 'terraform',
+}
+
+export interface TerraformManagerData {
+  terragruntDependencyType: TerragruntDependencyTypes;
+}
+
+export enum TerragruntResourceTypes {
+  unknown = 'unknown',
+  /**
+   * https://www.terraform.io/docs/providers/docker/r/container.html
+   */
+}
+
+export interface ResourceManagerData extends TerraformManagerData {
+  resourceType?: TerragruntResourceTypes;
+  chart?: string;
+  image?: string;
+  name?: string;
+  repository?: string;
+}
+
+export function getTerragruntDependencyType(
+  value: string
+): TerragruntDependencyTypes {
+  switch (value) {
+    case 'terraform': {
+      return TerragruntDependencyTypes.terragrunt;
+    }
+    default: {
+      return TerragruntDependencyTypes.unknown;
+    }
+  }
+}
+
+export function checkFileContainsDependency(
+  content: string,
+  checkList: string[]
+): boolean {
+  return checkList.some((check) => {
+    return content.includes(check);
+  });
+}