terraform: ugly huge change to weave in new HCL2-oriented types

Due to how deeply the configuration types go into Terraform Core, there
isn't a great way to switch out to HCL2 gradually. As a consequence, this
huge commit gets us from the old state to a _compilable_ new state, but
does not yet attempt to fix any tests and has a number of known missing
parts and bugs. We will continue to iterate on this in forthcoming
commits, heading back towards passing tests and making Terraform
fully-functional again.

The three main goals here are:
- Use the configuration models from the "configs" package instead of the
  older models in the "config" package, which is now deprecated and
  preserved only to help us write our migration tool.
- Do expression inspection and evaluation using the functionality of the
  new "lang" package, instead of the Interpolator type and related
  functionality in the main "terraform" package.
- Represent addresses of various objects using types in the addrs package,
  rather than hand-constructed strings. This is not critical to support
  the above, but was a big help during the implementation of these other
  points since it made it much more explicit what kind of address is
  expected in each context.

Since our new packages are built to accommodate some future planned
features that are not yet implemented (e.g. the "for_each" argument on
resources, "count"/"for_each" on modules), and since there's still a fair
amount of functionality still using old-style APIs, there is a moderate
amount of shimming here to connect new assumptions with old, hopefully in
a way that makes it easier to find and eliminate these shims later.

I apologize in advance to the person who inevitably just found this huge
commit while spelunking through the commit history.

backend/backend.go

@@ -9,12 +9,12 @@ import (
 	"errors"
 	"time"
 
-	"github.com/hashicorp/terraform/configs"
-
 	"github.com/zclconf/go-cty/cty"
 
+	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/command/clistate"
 	"github.com/hashicorp/terraform/config/configschema"
+	"github.com/hashicorp/terraform/configs"
 	"github.com/hashicorp/terraform/configs/configload"
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/terraform"
@@ -184,11 +184,10 @@ type Operation struct {
 	// behavior of the operation.
 	AutoApprove  bool
 	Destroy      bool
+	Targets      []addrs.Targetable
+	Variables    map[string]UnparsedVariableValue
+	AutoApprove  bool
 	DestroyForce bool
-	ModuleDepth  int
-	Parallelism  int
-	Targets      []string
-	Variables    map[string]interface{}
 
 	// Input/output/control options.
 	UIIn  terraform.UIInput

backend/local/backend_apply.go

@@ -137,11 +137,11 @@ func (b *Local) opApply(
 
 	// Start the apply in a goroutine so that we can be interrupted.
 	var applyState *terraform.State
-	var applyErr error
+	var applyDiags tfdiags.Diagnostics
 	doneCh := make(chan struct{})
 	go func() {
 		defer close(doneCh)
-		_, applyErr = tfCtx.Apply()
+		_, applyDiags = tfCtx.Apply()
 		// we always want the state, even if apply failed
 		applyState = tfCtx.State()
 	}()
@@ -165,12 +165,8 @@ func (b *Local) opApply(
 		return
 	}
 
-	if applyErr != nil {
-		diags = diags.Append(tfdiags.Sourceless(
-			tfdiags.Error,
-			applyErr.Error(),
-			"Terraform does not automatically rollback in the face of errors. Instead, your Terraform state file has been partially updated with any resources that successfully completed. Please address the error above and apply again to incrementally change your infrastructure.",
-		))
+	diags = diags.Append(applyDiags)
+	if applyDiags.HasErrors() {
 		b.ReportResult(runningOp, diags)
 		return
 	}

backend/local/backend_local.go

@@ -2,7 +2,6 @@ package local
 
 import (
 	"context"
-	"errors"
 
 	"github.com/hashicorp/errwrap"
 
@@ -58,15 +57,23 @@ func (b *Local) context(op *backend.Operation) (*terraform.Context, state.State,
 	opts.Destroy = op.Destroy
 	opts.Targets = op.Targets
 	opts.UIInput = op.UIIn
-	if op.Variables != nil {
-		opts.Variables = op.Variables
+
+	// Load the configuration using the caller-provided configuration loader.
+	config, configDiags := op.ConfigLoader.LoadConfig(op.ConfigDir)
+	diags = diags.Append(configDiags)
+	if configDiags.HasErrors() {
+		return nil, nil, diags
 	}
+	opts.Config = config
 
-	// FIXME: Configuration is temporarily stubbed out here to artificially
-	// create a stopping point in our work to switch to the new config loader.
-	// This means no backend-provided Terraform operations will actually work.
-	// This will be addressed in a subsequent commit.
-	opts.Module = nil
+	variables, varDiags := backend.ParseVariableValues(op.Variables, config.Module.Variables)
+	diags = diags.Append(varDiags)
+	if diags.HasErrors() {
+		return nil, nil, diags
+	}
+	if op.Variables != nil {
+		opts.Variables = variables
+	}
 
 	// Load our state
 	// By the time we get here, the backend creation code in "command" took
@@ -77,23 +84,14 @@ func (b *Local) context(op *backend.Operation) (*terraform.Context, state.State,
 
 	// Build the context
 	var tfCtx *terraform.Context
+	var ctxDiags tfdiags.Diagnostics
 	if op.Plan != nil {
-		tfCtx, err = op.Plan.Context(&opts)
+		tfCtx, ctxDiags = op.Plan.Context(&opts)
 	} else {
-		tfCtx, err = terraform.NewContext(&opts)
-	}
-
-	// any errors resolving plugins returns this
-	if rpe, ok := err.(*terraform.ResourceProviderError); ok {
-		b.pluginInitRequired(rpe)
-		// we wrote the full UI error here, so return a generic error for flow
-		// control in the command.
-		diags = diags.Append(errors.New("Can't satisfy plugin requirements"))
-		return nil, nil, diags
+		tfCtx, ctxDiags = terraform.NewContext(&opts)
 	}
-
-	if err != nil {
-		diags = diags.Append(err)
+	diags = diags.Append(ctxDiags)
+	if ctxDiags.HasErrors() {
 		return nil, nil, diags
 	}
 
@@ -106,8 +104,9 @@ func (b *Local) context(op *backend.Operation) (*terraform.Context, state.State,
 			mode |= terraform.InputModeVar
 			mode |= terraform.InputModeVarUnset
 
-			if err := tfCtx.Input(mode); err != nil {
-				diags = diags.Append(errwrap.Wrapf("Error asking for user input: {{err}}", err))
+			inputDiags := tfCtx.Input(mode)
+			diags = diags.Append(inputDiags)
+			if inputDiags.HasErrors() {
 				return nil, nil, diags
 			}
 		}

backend/local/backend_plan.go

@@ -87,20 +87,20 @@ func (b *Local) opPlan(
 
 	// Perform the plan in a goroutine so we can be interrupted
 	var plan *terraform.Plan
-	var planErr error
+	var planDiags tfdiags.Diagnostics
 	doneCh := make(chan struct{})
 	go func() {
 		defer close(doneCh)
 		log.Printf("[INFO] backend/local: plan calling Plan")
-		plan, planErr = tfCtx.Plan()
+		plan, planDiags = tfCtx.Plan()
 	}()
 
 	if b.opWait(doneCh, stopCtx, cancelCtx, tfCtx, opState) {
 		return
 	}
 
-	if planErr != nil {
-		diags = diags.Append(planErr)
+	diags = diags.Append(planDiags)
+	if planDiags.HasErrors() {
 		b.ReportResult(runningOp, diags)
 		return
 	}

backend/local/backend_refresh.go

@@ -60,11 +60,11 @@ func (b *Local) opRefresh(
 
 	// Perform the refresh in a goroutine so we can be interrupted
 	var newState *terraform.State
-	var refreshErr error
+	var refreshDiags tfdiags.Diagnostics
 	doneCh := make(chan struct{})
 	go func() {
 		defer close(doneCh)
-		newState, refreshErr = tfCtx.Refresh()
+		newState, refreshDiags = tfCtx.Refresh()
 		log.Printf("[INFO] backend/local: refresh calling Refresh")
 	}()
 
@@ -74,8 +74,8 @@ func (b *Local) opRefresh(
 
 	// write the resulting state to the running op
 	runningOp.State = newState
-	if refreshErr != nil {
-		diags = diags.Append(refreshErr)
+	diags = diags.Append(refreshDiags)
+	if refreshDiags.HasErrors() {
 		b.ReportResult(runningOp, diags)
 		return
 	}

backend/unparsed_value.go

@@ -0,0 +1,84 @@
+package backend
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/terraform"
+	"github.com/hashicorp/terraform/tfdiags"
+)
+
+// UnparsedVariableValue represents a variable value provided by the caller
+// whose parsing must be deferred until configuration is available.
+//
+// This exists to allow processing of variable-setting arguments (e.g. in the
+// command package) to be separated from parsing (in the backend package).
+type UnparsedVariableValue interface {
+	// ParseVariableValue information in the provided variable configuration
+	// to parse (if necessary) and return the variable value encapsulated in
+	// the receiver.
+	//
+	// If error diagnostics are returned, the resulting value may be invalid
+	// or incomplete.
+	ParseVariableValue(mode configs.VariableParsingMode) (*terraform.InputValue, tfdiags.Diagnostics)
+}
+
+func ParseVariableValues(vv map[string]UnparsedVariableValue, decls map[string]*configs.Variable) (terraform.InputValues, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+	ret := make(terraform.InputValues, len(vv))
+
+	for name, rv := range vv {
+		var mode configs.VariableParsingMode
+		config, declared := decls[name]
+		if declared {
+			mode = config.ParsingMode
+		} else {
+			mode = configs.VariableParseLiteral
+		}
+
+		val, valDiags := rv.ParseVariableValue(mode)
+		diags = diags.Append(valDiags)
+		if valDiags.HasErrors() {
+			continue
+		}
+
+		if !declared {
+			switch val.SourceType {
+			case terraform.ValueFromConfig, terraform.ValueFromFile:
+				// These source types have source ranges, so we can produce
+				// a nice error message with good context.
+				diags = diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Value for undeclared variable",
+					Detail:   fmt.Sprintf("The root module does not declare a variable named %q. To use this value, add a \"variable\" block to the configuration.", name),
+					Subject:  val.SourceRange.ToHCL().Ptr(),
+				})
+			case terraform.ValueFromEnvVar:
+				// We allow and ignore undeclared names for environment
+				// variables, because users will often set these globally
+				// when they are used across many (but not necessarily all)
+				// configurations.
+			case terraform.ValueFromCLIArg:
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Value for undeclared variable",
+					fmt.Sprintf("A variable named %q was assigned on the command line, but the root module does not declare a variable of that name. To use this value, add a \"variable\" block to the configuration.", name),
+				))
+			default:
+				// For all other source types we are more vague, but other situations
+				// don't generally crop up at this layer in practice.
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Value for undeclared variable",
+					fmt.Sprintf("A variable named %q was assigned a value, but the root module does not declare a variable of that name. To use this value, add a \"variable\" block to the configuration.", name),
+				))
+			}
+			continue
+		}
+
+		ret[name] = val
+	}
+
+	return ret, diags
+}

command/apply.go

@@ -7,14 +7,14 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/hashicorp/terraform/configs"
-
-	"github.com/hashicorp/terraform/tfdiags"
-
 	"github.com/hashicorp/go-getter"
+
+	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/backend"
 	"github.com/hashicorp/terraform/config"
+	"github.com/hashicorp/terraform/configs"
 	"github.com/hashicorp/terraform/terraform"
+	"github.com/hashicorp/terraform/tfdiags"
 )
 
 // ApplyCommand is a Command implementation that applies a Terraform
@@ -314,7 +314,7 @@ Options:
 	return strings.TrimSpace(helpText)
 }
 
-func outputsAsString(state *terraform.State, modPath []string, schema []*config.Output, includeHeader bool) string {
+func outputsAsString(state *terraform.State, modPath addrs.ModuleInstance, schema []*config.Output, includeHeader bool) string {
 	if state == nil {
 		return ""
 	}

command/flag_kv.go

@@ -3,6 +3,11 @@ package command
 import (
 	"fmt"
 	"strings"
+
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/hcl2/hcl/hclsyntax"
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/tfdiags"
 )
 
 // FlagStringKV is a flag.Value implementation for parsing user variables
@@ -41,3 +46,34 @@ func (v *FlagStringSlice) Set(raw string) error {
 
 	return nil
 }
+
+// FlagTargetSlice is a flag.Value implementation for parsing target addresses
+// from the command line, such as -target=aws_instance.foo -target=aws_vpc.bar .
+type FlagTargetSlice []addrs.Targetable
+
+func (v *FlagTargetSlice) String() string {
+	return ""
+}
+
+func (v *FlagTargetSlice) Set(raw string) error {
+	// FIXME: This is not an ideal way to deal with this because it requires
+	// us to do parsing in a context where we can't nicely return errors
+	// to the user.
+
+	var diags tfdiags.Diagnostics
+	synthFilename := fmt.Sprintf("-target=%q", raw)
+	traversal, syntaxDiags := hclsyntax.ParseTraversalAbs([]byte(raw), synthFilename, hcl.Pos{Line: 1, Column: 1})
+	diags = diags.Append(syntaxDiags)
+	if syntaxDiags.HasErrors() {
+		return diags.Err()
+	}
+
+	target, targetDiags := addrs.ParseTarget(traversal)
+	diags = diags.Append(targetDiags)
+	if targetDiags.HasErrors() {
+		return diags.Err()
+	}
+
+	*v = append(*v, target.Subject)
+	return nil
+}

command/graph.go

@@ -127,12 +127,13 @@ func (c *GraphCommand) Run(args []string) int {
 
 	// Skip validation during graph generation - we want to see the graph even if
 	// it is invalid for some reason.
-	g, err := ctx.Graph(graphType, &terraform.ContextGraphOpts{
+	g, graphDiags := ctx.Graph(graphType, &terraform.ContextGraphOpts{
 		Verbose:  verbose,
 		Validate: false,
 	})
-	if err != nil {
-		c.Ui.Error(fmt.Sprintf("Error creating graph: %s", err))
+	diags = diags.Append(graphDiags)
+	if graphDiags.HasErrors() {
+		c.showDiagnostics(diags)
 		return 1
 	}