Add a "output-module-groups" command (#2130)

* Add "output-module-groups" command

* Rename function

* docs(reference): output-module-groups

* feat(output): improve output with group number as key

* docs(output): update output with index of group

* style(fmt): goimports

* feat(tests): add a test and better description

* fix(build): add config to null for func

---------

Co-authored-by: Benjamin Sanvoisin <benjamin@sanvoisin.io>

cli/cli_app.go

@@ -125,6 +125,7 @@ const (
 	CMD_HCLFMT                        = "hclfmt"
 	CMD_AWS_PROVIDER_PATCH            = "aws-provider-patch"
 	CMD_RENDER_JSON                   = "render-json"
+	CMD_OUTPUT_MODULE_GROUPS          = "output-module-groups"
 )
 
 // START: Constants useful for multimodule command handling
@@ -245,6 +246,7 @@ COMMANDS:
    hclfmt                Recursively find hcl files and rewrite them into a canonical format.
    aws-provider-patch    Overwrite settings on nested AWS providers to work around a Terraform bug (issue #13018)
    render-json           Render the final terragrunt config, with all variables, includes, and functions resolved, as json. This is useful for enforcing policies using static analysis tools like Open Policy Agent, or for debugging your terragrunt config.
+   output-module-groups  Output groups of modules ordered for apply as a list of list in JSON (useful for CI use cases).
    *                     Terragrunt forwards all other commands directly to Terraform
 
 GLOBAL OPTIONS:
@@ -423,6 +425,18 @@ func RunTerragrunt(terragruntOptions *options.TerragruntOptions) error {
 		return runRenderJSON(terragruntOptions, terragruntConfig)
 	}
 
+	if shouldPrintModuleGroups(terragruntOptions) {
+		js, err := runGraphDependenciesGroups(terragruntOptions)
+		if err != nil {
+			return err
+		}
+		_, err = fmt.Fprintf(terragruntOptions.Writer, "%s\n", js)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
 	terragruntOptionsClone := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath)
 	terragruntOptionsClone.TerraformCommand = CMD_TERRAGRUNT_READ_CONFIG
 
@@ -624,6 +638,20 @@ func runGraphDependencies(terragruntOptions *options.TerragruntOptions) error {
 	return nil
 }
 
+// Run graph dependencies returns the dependency graph
+func runGraphDependenciesGroups(terragruntOptions *options.TerragruntOptions) (string, error) {
+	stack, err := configstack.FindStackInSubfolders(terragruntOptions, nil)
+	if err != nil {
+		return "", err
+	}
+
+	js, err := stack.JsonModuleDeployOrder(terragruntOptions.TerraformCommand)
+	if err != nil {
+		return "", err
+	}
+	return js, nil
+}
+
 func shouldPrintTerraformHelp(terragruntOptions *options.TerragruntOptions) bool {
 	for _, tfHelpFlag := range TERRAFORM_HELP_FLAGS {
 		if util.ListContainsElement(terragruntOptions.TerraformCliArgs, tfHelpFlag) {
@@ -668,6 +696,10 @@ func shouldRunRenderJSON(terragruntOptions *options.TerragruntOptions) bool {
 	return util.ListContainsElement(terragruntOptions.TerraformCliArgs, CMD_RENDER_JSON)
 }
 
+func shouldPrintModuleGroups(terragruntOptions *options.TerragruntOptions) bool {
+	return util.ListContainsElement(terragruntOptions.TerraformCliArgs, CMD_OUTPUT_MODULE_GROUPS)
+}
+
 func shouldApplyAwsProviderPatch(terragruntOptions *options.TerragruntOptions) bool {
 	return util.ListContainsElement(terragruntOptions.TerraformCliArgs, CMD_AWS_PROVIDER_PATCH)
 }

configstack/module.go

@@ -1,6 +1,7 @@
 package configstack
 
 import (
+	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"sort"
@@ -41,6 +42,10 @@ func (module *TerraformModule) String() string {
 	)
 }
 
+func (module TerraformModule) MarshalJSON() ([]byte, error) {
+	return json.Marshal(module.Path)
+}
+
 // Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents
 // into a TerraformModule struct. Return the list of these TerraformModule structs.
 func ResolveTerraformModules(terragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions, childTerragruntConfig *config.TerragruntConfig, howThesePathsWereFound string) ([]*TerraformModule, error) {

configstack/stack.go

@@ -2,6 +2,7 @@ package configstack
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
@@ -51,6 +52,30 @@ func (stack *Stack) LogModuleDeployOrder(logger *logrus.Entry, terraformCommand
 	return nil
 }
 
+// JsonModuleDeployOrder will return the modules that will be deployed by a plan/apply operation, in the order
+// that the operations happen.
+func (stack *Stack) JsonModuleDeployOrder(terraformCommand string) (string, error) {
+	runGraph, err := stack.getModuleRunGraph(terraformCommand)
+	if err != nil {
+		return "", err
+	}
+	// Convert the module paths to a string array for JSON marshalling
+	// The index should be the group number, and the value should be an array of module paths
+	jsonGraph := make(map[string][]string)
+	for i, group := range runGraph {
+		groupNum := "Group " + fmt.Sprintf("%d", i+1)
+		jsonGraph[groupNum] = make([]string, len(group))
+		for j, module := range group {
+			jsonGraph[groupNum][j] = module.Path
+		}
+	}
+	j, _ := json.MarshalIndent(jsonGraph, "", "  ")
+	if err != nil {
+		return "", err
+	}
+	return string(j), nil
+}
+
 // Graph creates a graphviz representation of the modules
 func (stack *Stack) Graph(terragruntOptions *options.TerragruntOptions) {
 	WriteDot(terragruntOptions.Writer, terragruntOptions, stack.Modules)

docs/_docs/04_reference/cli-options.md

@@ -36,6 +36,7 @@ Terragrunt supports the following CLI commands:
   - [hclfmt](#hclfmt)
   - [aws-provider-patch](#aws-provider-patch)
   - [render-json](#render-json)
+  - [output-module-groups](#output-module-groups)
 
 ### All Terraform built-in commands
 
@@ -464,6 +465,46 @@ Example:
 }
 ```
 
+### output-module-groups
+
+Output groups of modules ordered for apply as a list of list in JSON (useful for CI use cases).
+
+Example:
+
+```bash
+terragrunt output-module-groups
+```
+
+This will recursively search the current working directory for any folders that contain Terragrunt modules and build
+the dependency graph based on [`dependency`](/docs/reference/config-blocks-and-attributes/#dependency) and
+[`dependencies`](/docs/reference/config-blocks-and-attributes/#dependencies) blocks. This may produce output such as:
+
+```
+{
+  "Group 1": [
+    "stage/frontend-app"
+  ],
+  "Group 2": [
+    "stage/backend-app"
+  ],
+  "Group 3": [
+    "mgmt/bastion-host",
+    "stage/search-app"
+  ],
+  "Group 4": [
+    "mgmt/kms-master-key",
+    "stage/mysql",
+    "stage/redis"
+  ],
+  "Group 5": [
+    "stage/vpc"
+  ],
+  "Group 6": [
+    "mgmt/vpc"
+  ]
+}
+```
+
 ## CLI options
 
 Terragrunt forwards all options to Terraform. The only exceptions are `--version` and arguments that start with the

test/fixture-output-module-groups/root/backend-app/main.tf

@@ -0,0 +1,3 @@
+terraform {
+  backend "s3" {}
+}

test/fixture-output-module-groups/root/backend-app/terragrunt.hcl

@@ -0,0 +1,8 @@
+include {
+  path = find_in_parent_folders()
+}
+
+dependencies {
+  paths = ["../mysql", "../redis", "../vpc"]
+}
+

test/fixture-output-module-groups/root/frontend-app/main.tf

@@ -0,0 +1,3 @@
+terraform {
+  backend "s3" {}
+}

test/fixture-output-module-groups/root/frontend-app/terragrunt.hcl

@@ -0,0 +1,7 @@
+include {
+  path = find_in_parent_folders()
+}
+
+dependencies {
+  paths = ["../backend-app", "../vpc"]
+}

test/fixture-output-module-groups/root/mysql/main.tf

@@ -0,0 +1,3 @@
+terraform {
+  backend "s3" {}
+}

test/fixture-output-module-groups/root/mysql/terragrunt.hcl

@@ -0,0 +1,8 @@
+include {
+  path = find_in_parent_folders()
+}
+
+dependencies {
+  paths = ["../vpc"]
+}
+

test/fixture-output-module-groups/root/redis/main.tf

@@ -0,0 +1,3 @@
+terraform {
+  backend "s3" {}
+}

test/fixture-output-module-groups/root/redis/terragrunt.hcl

@@ -0,0 +1,8 @@
+include {
+  path = find_in_parent_folders()
+}
+
+dependencies {
+  paths = ["../vpc"]
+}
+

test/fixture-output-module-groups/root/vpc/main.tf

@@ -0,0 +1,3 @@
+terraform {
+  backend "s3" {}
+}

test/fixture-output-module-groups/root/vpc/terragrunt.hcl

@@ -0,0 +1,5 @@
+include {
+  path = find_in_parent_folders()
+}
+
+

test/fixture-output-module-groups/terragrunt.hcl

@@ -0,0 +1,14 @@
+# Configure Terragrunt to automatically store tfstate files in an S3 bucket
+remote_state {
+  backend = "s3"
+  config = {
+    encrypt = true
+    bucket = "__FILL_IN_BUCKET_NAME__"
+    key = "${path_relative_to_include()}/terraform.tfstate"
+    region = "us-west-2"
+  }
+}
+
+inputs = {
+  terraform_remote_state_s3_bucket = "__FILL_IN_BUCKET_NAME__"
+}

test/integration_test.go

@@ -135,6 +135,7 @@ const (
 	TEST_FIXTURE_RENDER_JSON_METADATA                                        = "fixture-render-json-metadata"
 	TEST_FIXTURE_RENDER_JSON_MOCK_OUTPUTS                                    = "fixture-render-json-mock-outputs"
 	TEST_FIXTURE_RENDER_JSON_INPUTS                                          = "fixture-render-json-inputs"
+	TEST_FIXTURE_OUTPUT_MODULE_GROUPS                                        = "fixture-output-module-groups"
 	TEST_FIXTURE_STARTSWITH                                                  = "fixture-startswith"
 	TEST_FIXTURE_TIMECMP                                                     = "fixture-timecmp"
 	TEST_FIXTURE_TIMECMP_INVALID_TIMESTAMP                                   = "fixture-timecmp-errors/invalid-timestamp"
@@ -4882,6 +4883,37 @@ func TestRenderJsonAttributesMetadata(t *testing.T) {
 	assert.True(t, reflect.DeepEqual(expectedTerraformVersionConstraint, terraformVersionConstraint))
 }
 
+func TestOutputModuleGroups(t *testing.T) {
+	t.Parallel()
+
+	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_OUTPUT_MODULE_GROUPS)
+	cleanupTerraformFolder(t, tmpEnvPath)
+	environmentPath := fmt.Sprintf("%s/%s", tmpEnvPath, TEST_FIXTURE_OUTPUT_MODULE_GROUPS)
+	var (
+		stdout bytes.Buffer
+		stderr bytes.Buffer
+	)
+	runTerragruntRedirectOutput(t, fmt.Sprintf("terragrunt output-module-groups --terragrunt-working-dir %s", environmentPath), &stdout, &stderr)
+	output := stdout.String()
+	expectedOutput := fmt.Sprintf(`
+{
+  "Group 1": [
+    "%[1]s/root/vpc"
+  ],
+  "Group 2": [
+    "%[1]s/root/mysql",
+    "%[1]s/root/redis"
+  ],
+  "Group 3": [
+    "%[1]s/root/backend-app"
+  ],
+  "Group 4": [
+    "%[1]s/root/frontend-app"
+  ]
+}`, environmentPath)
+	assert.True(t, strings.Contains(output, strings.TrimSpace(expectedOutput)))
+}
+
 func TestRenderJsonMetadataDependencies(t *testing.T) {
 	t.Parallel()