Optimize dependency output retrieval

config/config.go

@@ -90,6 +90,35 @@ func (remoteState *remoteStateConfigFile) String() string {
 	return fmt.Sprintf("remoteStateConfigFile{Backend = %v, Config = %v}", remoteState.Backend, remoteState.Config)
 }
 
+// Convert the parsed config file remote state struct to the internal representation struct of remote state
+// configurations.
+func (remoteState *remoteStateConfigFile) toConfig() (*remote.RemoteState, error) {
+	remoteStateConfig, err := parseCtyValueToMap(remoteState.Config)
+	if err != nil {
+		return nil, err
+	}
+
+	config := &remote.RemoteState{}
+	config.Backend = remoteState.Backend
+	if remoteState.Generate != nil {
+		config.Generate = &remote.RemoteStateGenerate{
+			Path:     remoteState.Generate.Path,
+			IfExists: remoteState.Generate.IfExists,
+		}
+	}
+	config.Config = remoteStateConfig
+
+	if remoteState.DisableInit != nil {
+		config.DisableInit = *remoteState.DisableInit
+	}
+
+	config.FillDefaults()
+	if err := config.Validate(); err != nil {
+		return nil, err
+	}
+	return config, err
+}
+
 type remoteStateConfigGenerate struct {
 	// We use cty instead of hcl, since we are using this type to convert an attr and not a block.
 	Path     string `cty:"path"`
@@ -679,30 +708,10 @@ func convertToTerragruntConfig(
 	}
 
 	if terragruntConfigFromFile.RemoteState != nil {
-		remoteStateConfig, err := parseCtyValueToMap(terragruntConfigFromFile.RemoteState.Config)
+		remoteState, err := terragruntConfigFromFile.RemoteState.toConfig()
 		if err != nil {
 			return nil, err
 		}
-
-		remoteState := &remote.RemoteState{}
-		remoteState.Backend = terragruntConfigFromFile.RemoteState.Backend
-		if terragruntConfigFromFile.RemoteState.Generate != nil {
-			remoteState.Generate = &remote.RemoteStateGenerate{
-				Path:     terragruntConfigFromFile.RemoteState.Generate.Path,
-				IfExists: terragruntConfigFromFile.RemoteState.Generate.IfExists,
-			}
-		}
-		remoteState.Config = remoteStateConfig
-
-		if terragruntConfigFromFile.RemoteState.DisableInit != nil {
-			remoteState.DisableInit = *terragruntConfigFromFile.RemoteState.DisableInit
-		}
-
-		remoteState.FillDefaults()
-		if err := remoteState.Validate(); err != nil {
-			return nil, err
-		}
-
 		terragruntConfig.RemoteState = remoteState
 	}
 

config/config_partial.go

@@ -24,6 +24,7 @@ const (
 	TerraformSource
 	TerragruntFlags
 	TerragruntVersionConstraints
+	RemoteStateBlock
 )
 
 // terragruntInclude is a struct that can be used to only decode the include block.
@@ -79,6 +80,12 @@ type terragruntDependency struct {
 	Remain       hcl.Body     `hcl:",remain"`
 }
 
+// terragruntRemoteState is a struct that can be used to only decode the remote_state blocks in the terragrunt config
+type terragruntRemoteState struct {
+	RemoteState *remoteStateConfigFile `hcl:"remote_state,block"`
+	Remain      hcl.Body               `hcl:",remain"`
+}
+
 // DecodeBaseBlocks takes in a parsed HCL2 file and decodes the base blocks. Base blocks are blocks that should always
 // be decoded even in partial decoding, because they provide bindings that are necessary for parsing any block in the
 // file. Currently base blocks are:
@@ -148,6 +155,7 @@ func PartialParseConfigFile(
 // - TerragruntFlags: Parses the boolean flags `prevent_destroy` and `skip` in the config
 // - TerragruntVersionConstraints: Parses the attributes related to constraining terragrunt and terraform versions in
 //                                 the config.
+// - RemoteStateBlock: Parses the `remote_state` block in the config
 // Note that the following blocks are always decoded:
 // - locals
 // - include
@@ -263,6 +271,20 @@ func PartialParseConfigString(
 				output.TerraformBinary = *decoded.TerraformBinary
 			}
 
+		case RemoteStateBlock:
+			decoded := terragruntRemoteState{}
+			err := decodeHcl(file, filename, &decoded, terragruntOptions, contextExtensions)
+			if err != nil {
+				return nil, err
+			}
+			if decoded.RemoteState != nil {
+				remoteState, err := decoded.RemoteState.toConfig()
+				if err != nil {
+					return nil, err
+				}
+				output.RemoteState = remoteState
+			}
+
 		default:
 			return nil, InvalidPartialBlockName{decode}
 		}

config/dependency.go

@@ -6,6 +6,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -19,6 +21,8 @@ import (
 
 	"github.com/gruntwork-io/terragrunt/errors"
 	"github.com/gruntwork-io/terragrunt/options"
+	"github.com/gruntwork-io/terragrunt/remote"
+	"github.com/gruntwork-io/terragrunt/shell"
 	"github.com/gruntwork-io/terragrunt/util"
 )
 
@@ -326,7 +330,7 @@ func getOutputJsonWithCaching(targetConfig string, terragruntOptions *options.Te
 	}
 
 	// Cache miss, so look up the output and store in cache
-	newJsonBytes, err := runTerragruntOutputJson(terragruntOptions, targetConfig)
+	newJsonBytes, err := getTerragruntOutputJson(terragruntOptions, targetConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -382,6 +386,72 @@ func cloneTerragruntOptionsForDependencyOutput(terragruntOptions *options.Terrag
 	return targetOptions, nil
 }
 
+// Retrieve the outputs from the terraform state in the target configuration. This attempts to optimize the output
+// retrieval if the following conditions are true:
+// - State backends are managed with a `remote_state` block.
+// - The `remote_state` block is using the generator pattern (the `generate` attribute is set).
+// - The `remote_state` block does not depend on any `dependency` outputs.
+// If these conditions are met, terragrunt can optimize the retrieval to avoid recursively retrieving dependency outputs
+// by directly pulling down the state file. Otherwise, terragrunt will fallback to running `terragrunt output` on the
+// target module.
+func getTerragruntOutputJson(terragruntOptions *options.TerragruntOptions, targetConfig string) ([]byte, error) {
+	// First, attempt to parse the `remote_state` blocks without parsing/getting dependency outputs. If this is
+	// possible, proceed to routine that fetches remote state directly. Otherwise, fallback to calling
+	// `terragrunt output` directly.
+	remoteStateTGConfig, err := PartialParseConfigFile(targetConfig, terragruntOptions, nil, []PartialDecodeSectionType{RemoteStateBlock})
+	if err != nil || !canGetRemoteState(remoteStateTGConfig.RemoteState) {
+		terragruntOptions.Logger.Printf("WARNING: Could not parse remote_state block from target config %s", targetConfig)
+		terragruntOptions.Logger.Printf("WARNING: Falling back to terragrunt output.")
+		return runTerragruntOutputJson(terragruntOptions, targetConfig)
+	}
+	return getTerragruntOutputJsonFromRemoteState(terragruntOptions, targetConfig, remoteStateTGConfig.RemoteState)
+}
+
+// canGetRemoteState returns true if the remote state block is not nil and has the generate block configured.
+func canGetRemoteState(remoteState *remote.RemoteState) bool {
+	return remoteState != nil && remoteState.Generate != nil
+}
+
+// getTerragruntOutputJsonFromRemoteState will retrieve the outputs directly by using just the remote state block. This
+// uses terraform's feature where `output` and `init` can work without the real source, as long as you have the
+// `backend` configured.
+// To do this, this function will:
+// - Create a temporary folder
+// - Generate the backend.tf file with the backend configuration from the remote_state block
+// - Run terraform init and terraform output
+// - Clean up folder once json file is generated
+func getTerragruntOutputJsonFromRemoteState(terragruntOptions *options.TerragruntOptions, targetConfig string, remoteState *remote.RemoteState) ([]byte, error) {
+	// Create working directory where we will run terraform in. Make sure it is cleaned up before the function returns.
+	tempWorkDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tempWorkDir)
+
+	// Set the terraform working dir to the tempdir
+	tmpTGOptions := terragruntOptions.Clone(tempWorkDir)
+	tmpTGOptions.WorkingDir = tempWorkDir
+
+	// Generate the backend configuration in the working dir
+	if err := remoteState.GenerateTerraformCode(tmpTGOptions); err != nil {
+		return nil, err
+	}
+
+	// The working directory is now set up to interact with the state, so pull it down to get the json output.
+	// First run init, then output.
+	if err := shell.RunTerraformCommand(tmpTGOptions, "init"); err != nil {
+		return nil, err
+	}
+	out, err := shell.RunTerraformCommandWithOutput(tmpTGOptions, "output", "-json")
+	if err != nil {
+		return nil, err
+	}
+	jsonString := out.Stdout
+	jsonBytes := []byte(strings.TrimSpace(jsonString))
+	util.Debugf(terragruntOptions.Logger, "Retrieved output from %s as json: %s", targetConfig, jsonString)
+	return jsonBytes, nil
+}
+
 // runTerragruntOutputJson uses terragrunt running functions to extract the json output from the target config. Make a
 // copy of the terragruntOptions so that we can reuse the same execution environment.
 func runTerragruntOutputJson(terragruntOptions *options.TerragruntOptions, targetConfig string) ([]byte, error) {

test/fixture-get-output/nested-optimization/deepdep/main.tf

@@ -0,0 +1,3 @@
+output "output" {
+  value = "The answer is 42"
+}

test/fixture-get-output/nested-optimization/deepdep/terragrunt.hcl

@@ -0,0 +1 @@
+# Intentionally empty.

test/fixture-get-output/nested-optimization/dep/main.tf

@@ -0,0 +1,4 @@
+variable "input" {}
+output "output" {
+  value = "No, ${var.input}"
+}

test/fixture-get-output/nested-optimization/dep/terragrunt.hcl

@@ -0,0 +1,12 @@
+include {
+  path = find_in_parent_folders()
+}
+
+# Retrieve a dependency. In the test, we will destroy this state and verify we can still get the output.
+dependency "deepdep" {
+  config_path = "../deepdep"
+}
+
+inputs = {
+  input = dependency.deepdep.outputs.output
+}

test/fixture-get-output/nested-optimization/live/main.tf

@@ -0,0 +1,4 @@
+variable "input" {}
+output "output" {
+  value = "They said, \"${var.input}\""
+}

test/fixture-get-output/nested-optimization/live/terragrunt.hcl

@@ -0,0 +1,8 @@
+# Retrieve a dependency. In the test, we will destroy this state and verify we can still get the output.
+dependency "dep" {
+  config_path = "../dep"
+}
+
+inputs = {
+  input = dependency.dep.outputs.output
+}

test/fixture-get-output/nested-optimization/terragrunt.hcl

@@ -0,0 +1,16 @@
+# Configure Terragrunt to automatically store tfstate files in an S3 bucket
+remote_state {
+  backend = "s3"
+  config = {
+    encrypt = true
+    bucket = "__FILL_IN_BUCKET_NAME__"
+    key = "${path_relative_to_include()}/terraform.tfstate"
+    region = "us-west-2"
+    dynamodb_table = "__FILL_IN_LOCK_TABLE_NAME__"
+    enable_lock_table_ssencryption = true
+  }
+  generate = {
+    path      = "backend.tf"
+    if_exists = "overwrite"
+  }
+}

test/integration_test.go

@@ -2030,6 +2030,49 @@ func TestYamlDecodeRegressions(t *testing.T) {
 	assert.Equal(t, outputs["test2"].Value, "1.00")
 }
 
+// We test the path with remote_state blocks by:
+// - Applying all modules initially
+// - Deleting the local state of the nested deep dependency
+// - Running apply on the root module
+// If output optimization is working, we should still get the same correct output even though the state of the upmost
+// module has been destroyed.
+func TestDependencyOutputOptimization(t *testing.T) {
+	t.Parallel()
+
+	expectedOutput := `They said, "No, The answer is 42"`
+
+	cleanupTerraformFolder(t, TEST_FIXTURE_GET_OUTPUT)
+	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_GET_OUTPUT)
+	rootPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_GET_OUTPUT, "nested-optimization")
+	rootTerragruntConfigPath := filepath.Join(rootPath, config.DefaultTerragruntConfigPath)
+	livePath := filepath.Join(rootPath, "live")
+	deepDepPath := filepath.Join(rootPath, "deepdep")
+
+	s3BucketName := fmt.Sprintf("terragrunt-test-bucket-%s", strings.ToLower(uniqueId()))
+	lockTableName := fmt.Sprintf("terragrunt-test-locks-%s", strings.ToLower(uniqueId()))
+	defer deleteS3Bucket(t, TERRAFORM_REMOTE_STATE_S3_REGION, s3BucketName)
+	defer cleanupTableForTest(t, lockTableName, TERRAFORM_REMOTE_STATE_S3_REGION)
+	copyTerragruntConfigAndFillPlaceholders(t, rootTerragruntConfigPath, rootTerragruntConfigPath, s3BucketName, lockTableName, TERRAFORM_REMOTE_STATE_S3_REGION)
+
+	runTerragrunt(t, fmt.Sprintf("terragrunt apply-all --terragrunt-non-interactive --terragrunt-working-dir %s", rootPath))
+
+	// verify expected output
+	stdout, _, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt output -no-color -json --terragrunt-non-interactive --terragrunt-working-dir %s", livePath))
+	require.NoError(t, err)
+
+	outputs := map[string]TerraformOutput{}
+	require.NoError(t, json.Unmarshal([]byte(stdout), &outputs))
+	assert.Equal(t, expectedOutput, outputs["output"].Value)
+
+	// Now delete the deepdep state and verify still works
+	require.NoError(t, os.Remove(filepath.Join(deepDepPath, "terraform.tfstate")))
+	reout, _, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt output -no-color -json --terragrunt-non-interactive --terragrunt-working-dir %s", livePath))
+	require.NoError(t, err)
+
+	require.NoError(t, json.Unmarshal([]byte(reout), &outputs))
+	assert.Equal(t, expectedOutput, outputs["output"].Value)
+}
+
 func TestDependencyOutput(t *testing.T) {
 	t.Parallel()
 
@@ -3108,6 +3151,16 @@ func runTerragrunt(t *testing.T, command string) {
 	runTerragruntRedirectOutput(t, command, os.Stdout, os.Stderr)
 }
 
+func runTerragruntCommandWithOutput(t *testing.T, command string) (string, string, error) {
+	stdout := bytes.Buffer{}
+	stderr := bytes.Buffer{}
+	err := runTerragruntCommand(t, command, &stdout, &stderr)
+	if err != nil {
+		return "", "", err
+	}
+	return stdout.String(), stderr.String(), nil
+}
+
 func runTerragruntRedirectOutput(t *testing.T, command string, writer io.Writer, errwriter io.Writer) {
 	if err := runTerragruntCommand(t, command, writer, errwriter); err != nil {
 		stdout := "(see log output above)"