Merge pull request #898 from gruntwork-io/yori-change-order

Change order of parsing so that include is parsed first

README.md

@@ -2380,8 +2380,8 @@ from `dependency` into `locals`. On the other hand, for the same reason, you can
 
 Currently terragrunt parses the config in the following order:
 
-1. `locals` block
 1. `include` block
+1. `locals` block
 1. `dependencies` block
 1. `dependency` blocks, including calling `terragrunt output` on the dependent modules to retrieve the outputs
 1. Everything else
@@ -2395,8 +2395,8 @@ you cannot use blocks that are parsed later earlier in the process (e.g you can'
 Note that the parsing order is slightly different when using the `-all` flavors of the command. In the `-all` flavors of
 the command, Terragrunt parses the configuration twice. In the first pass, it follows the following parsing order:
 
-1. `locals` block of all configurations in the tree
 1. `include` block of all configurations in the tree
+1. `locals` block of all configurations in the tree
 1. `dependency` blocks of all configurations in the tree, but does NOT retrieve the outputs
 1. `terraform` block of all configurations in the tree
 1. `dependencies` block of all configurations in the tree

config/config.go

@@ -270,12 +270,13 @@ func ParseConfigFile(filename string, terragruntOptions *options.TerragruntOptio
 // Parse the Terragrunt config contained in the given string and merge it with the given include config (if any). Note
 // that the config parsing consists of multiple stages so as to allow referencing of data resulting from parsing
 // previous config. The parsing order is:
-// 1. Parse locals. Since locals are parsed first, you can only reference other locals in the locals block and it is not
-//    merged from a config imported with an include block.
-//    Allowed References:
-//      - locals
-// 2. Parse include. Include is parsed next and is used to import another config. All the config in the include block is
-//    then merged into the current TerragruntConfig.
+// 1. Parse include. Include is parsed first and is used to import another config. All the config in the include block is
+//    then merged into the current TerragruntConfig, except for locals (by design). Note that since the include block is
+//    parsed first, you cannot reference locals in the include block config.
+// 2. Parse locals. Since locals are parsed next, you can only reference other locals in the locals block. Although it
+//    is possible to merge locals from a config imported with an include block, we do not do that here to avoid
+//    complicated referencing issues. Please refer to the globals proposal for an alternative that allows merging from
+//    included config: https://github.com/gruntwork-io/terragrunt/issues/814
 //    Allowed References:
 //      - locals
 // 3. Parse dependency blocks. This includes running `terragrunt output` to fetch the output data from another

config/config_partial.go

@@ -67,23 +67,12 @@ func DecodeBaseBlocks(
 	filename string,
 	includeFromChild *IncludeConfig,
 ) (*cty.Value, *terragruntInclude, *IncludeConfig, error) {
-	// Evaluate all the expressions in the locals block separately and generate the variables list to use in the
-	// evaluation context.
-	locals, err := evaluateLocalsBlock(terragruntOptions, parser, hclFile, filename)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-	localsAsCty, err := convertLocalsMapToCtyVal(locals)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
 	// Decode just the `include` block, and verify that it's allowed here
 	terragruntInclude, err := decodeAsTerragruntInclude(
 		hclFile,
 		filename,
 		terragruntOptions,
-		EvalContextExtensions{Locals: localsAsCty},
+		EvalContextExtensions{},
 	)
 	if err != nil {
 		return nil, nil, nil, err
@@ -93,6 +82,17 @@ func DecodeBaseBlocks(
 		return nil, nil, nil, err
 	}
 
+	// Evaluate all the expressions in the locals block separately and generate the variables list to use in the
+	// evaluation context.
+	locals, err := evaluateLocalsBlock(terragruntOptions, parser, hclFile, filename, includeForDecode)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	localsAsCty, err := convertLocalsMapToCtyVal(locals)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
 	return localsAsCty, terragruntInclude, includeForDecode, nil
 }
 

config/locals.go

@@ -47,6 +47,7 @@ func evaluateLocalsBlock(
 	parser *hclparse.Parser,
 	hclFile *hcl.File,
 	filename string,
+	included *IncludeConfig,
 ) (map[string]cty.Value, error) {
 	diagsWriter := util.GetDiagnosticsWriter(parser)
 
@@ -86,6 +87,7 @@ func evaluateLocalsBlock(
 			terragruntOptions,
 			filename,
 			locals,
+			included,
 			evaluatedLocals,
 			diagsWriter,
 		)
@@ -116,6 +118,7 @@ func attemptEvaluateLocals(
 	terragruntOptions *options.TerragruntOptions,
 	filename string,
 	locals []*Local,
+	included *IncludeConfig,
 	evaluatedLocals map[string]cty.Value,
 	diagsWriter hcl.DiagnosticWriter,
 ) (unevaluatedLocals []*Local, newEvaluatedLocals map[string]cty.Value, evaluated bool, err error) {
@@ -137,7 +140,11 @@ func attemptEvaluateLocals(
 		terragruntOptions.Logger.Printf("Could not convert evaluated locals to the execution context to evaluate additional locals")
 		return nil, evaluatedLocals, false, err
 	}
-	evalCtx := CreateTerragruntEvalContext(filename, terragruntOptions, EvalContextExtensions{Locals: evaluatedLocalsAsCty})
+	evalCtx := CreateTerragruntEvalContext(
+		filename,
+		terragruntOptions,
+		EvalContextExtensions{Include: included, Locals: evaluatedLocalsAsCty},
+	)
 
 	// Track the locals that were evaluated for logging purposes
 	newlyEvaluatedLocalNames := []string{}

config/locals_test.go

@@ -22,7 +22,7 @@ func TestEvaluateLocalsBlock(t *testing.T) {
 	file, err := parseHcl(parser, LocalsTestConfig, mockFilename)
 	require.NoError(t, err)
 
-	evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename)
+	evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil)
 	require.NoError(t, err)
 
 	var actualRegion string
@@ -67,7 +67,7 @@ func TestEvaluateLocalsBlockMultiDeepReference(t *testing.T) {
 	file, err := parseHcl(parser, LocalsTestMultiDeepReferenceConfig, mockFilename)
 	require.NoError(t, err)
 
-	evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename)
+	evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil)
 	require.NoError(t, err)
 
 	expected := "a"
@@ -106,7 +106,7 @@ func TestEvaluateLocalsBlockImpossibleWillFail(t *testing.T) {
 	file, err := parseHcl(parser, LocalsTestImpossibleConfig, mockFilename)
 	require.NoError(t, err)
 
-	_, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename)
+	_, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil)
 	require.Error(t, err)
 
 	switch errors.Unwrap(err).(type) {
@@ -126,7 +126,7 @@ func TestEvaluateLocalsBlockMultipleLocalsBlocksWillFail(t *testing.T) {
 	file, err := parseHcl(parser, MultipleLocalsBlockConfig, mockFilename)
 	require.NoError(t, err)
 
-	_, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename)
+	_, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil)
 	require.Error(t, err)
 }
 

test/fixture-locals/local-in-include/qa/my-app/main.tf

@@ -1,8 +1,10 @@
-terraform {
-  backend "s3" {}
+variable "parent_terragrunt_dir" {}
+variable "terragrunt_dir" {}
+
+output "parent_terragrunt_dir" {
+  value = var.parent_terragrunt_dir
 }
 
-# Create an arbitrary local resource
-data "template_file" "test" {
-  template = "Hello, I am a template."
+output "terragrunt_dir" {
+  value = var.terragrunt_dir
 }

test/fixture-locals/local-in-include/qa/my-app/terragrunt.hcl

@@ -1,7 +1,3 @@
-locals {
-  parent_path = find_in_parent_folders()
-}
-
 include {
-  path = local.parent_path
+  path = find_in_parent_folders()
 }

test/fixture-locals/local-in-include/terragrunt.hcl

@@ -1,10 +1,9 @@
-# Configure Terragrunt to automatically store tfstate files in an S3 bucket
-remote_state {
-  backend = "s3"
-  config = {
-    encrypt = true
-    bucket = "__FILL_IN_BUCKET_NAME__"
-    key = "${path_relative_to_include()}/terraform.tfstate"
-    region = "us-west-2"
-  }
+locals {
+  parent_terragrunt_dir = get_parent_terragrunt_dir()
+  terragrunt_dir = get_terragrunt_dir()
+}
+
+inputs = {
+  parent_terragrunt_dir = local.parent_terragrunt_dir
+  terragrunt_dir = local.terragrunt_dir
 }

test/integration_test.go

@@ -1139,14 +1139,33 @@ func TestLocalsParsing(t *testing.T) {
 func TestLocalsInInclude(t *testing.T) {
 	t.Parallel()
 
-	childPath := util.JoinPath(TEST_FIXTURE_LOCALS_IN_INCLUDE, TEST_FIXTURE_LOCALS_IN_INCLUDE_CHILD_REL_PATH)
 	cleanupTerraformFolder(t, TEST_FIXTURE_LOCALS_IN_INCLUDE)
+	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_LOCALS_IN_INCLUDE)
+	childPath := filepath.Join(tmpEnvPath, TEST_FIXTURE_LOCALS_IN_INCLUDE, TEST_FIXTURE_LOCALS_IN_INCLUDE_CHILD_REL_PATH)
+	runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-working-dir %s", childPath))
 
-	s3BucketName := fmt.Sprintf("terragrunt-%s-%s", strings.ToLower(t.Name()), strings.ToLower(uniqueId()))
-	defer deleteS3Bucket(t, TERRAFORM_REMOTE_STATE_S3_REGION, s3BucketName)
+	// Check the outputs of the dir functions referenced in locals to make sure they return what is expected
+	stdout := bytes.Buffer{}
+	stderr := bytes.Buffer{}
 
-	tmpTerragruntConfigPath := createTmpTerragruntConfigWithParentAndChild(t, TEST_FIXTURE_LOCALS_IN_INCLUDE, TEST_FIXTURE_LOCALS_IN_INCLUDE_CHILD_REL_PATH, s3BucketName, config.DefaultTerragruntConfigPath, config.DefaultTerragruntConfigPath)
-	runTerragrunt(t, fmt.Sprintf("terragrunt apply --terragrunt-non-interactive --terragrunt-config %s --terragrunt-working-dir %s", tmpTerragruntConfigPath, childPath))
+	require.NoError(
+		t,
+		runTerragruntCommand(t, fmt.Sprintf("terragrunt output -no-color -json --terragrunt-non-interactive --terragrunt-working-dir %s", childPath), &stdout, &stderr),
+	)
+
+	outputs := map[string]TerraformOutput{}
+	require.NoError(t, json.Unmarshal([]byte(stdout.String()), &outputs))
+
+	assert.Equal(
+		t,
+		outputs["parent_terragrunt_dir"].Value,
+		filepath.Join(tmpEnvPath, TEST_FIXTURE_LOCALS_IN_INCLUDE),
+	)
+	assert.Equal(
+		t,
+		outputs["terragrunt_dir"].Value,
+		childPath,
+	)
 }
 
 func TestUndefinedLocalsReferenceBreaks(t *testing.T) {