Backport reattach testing to v1.

acctest/helper.go

@@ -1,15 +1,17 @@
 package acctest
 
 import (
+	"log"
 	"os"
 
 	"github.com/hashicorp/terraform-plugin-sdk/plugin"
-	tftest "github.com/hashicorp/terraform-plugin-test"
+	tftest "github.com/hashicorp/terraform-plugin-test/v2"
 )
 
 var TestHelper *tftest.Helper
 
 func UseBinaryDriver(name string, providerFunc plugin.ProviderFunc) {
+	log.Println("[DEBUG] not using binary driver name, it's no longer needed")
 	sourceDir, err := os.Getwd()
 	if err != nil {
 		panic(err)
@@ -21,6 +23,6 @@ func UseBinaryDriver(name string, providerFunc plugin.ProviderFunc) {
 		})
 		os.Exit(0)
 	} else {
-		TestHelper = tftest.AutoInitProviderHelper(name, sourceDir)
+		TestHelper = tftest.AutoInitProviderHelper(sourceDir)
 	}
 }

go.mod

@@ -3,6 +3,7 @@ module github.com/hashicorp/terraform-plugin-sdk
 go 1.12
 
 require (
+	cloud.google.com/go v0.61.0 // indirect
 	github.com/agext/levenshtein v1.2.2
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
 	github.com/apparentlymart/go-cidr v1.0.1
@@ -11,24 +12,24 @@ require (
 	github.com/aws/aws-sdk-go v1.25.3 // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-test/deep v1.0.3
-	github.com/golang/mock v1.3.1
-	github.com/golang/protobuf v1.3.4
+	github.com/golang/mock v1.4.3
+	github.com/golang/protobuf v1.4.2
 	github.com/golang/snappy v0.0.1
-	github.com/google/go-cmp v0.3.1
+	github.com/google/go-cmp v0.5.0
 	github.com/google/uuid v1.1.1
 	github.com/hashicorp/errwrap v1.0.0
 	github.com/hashicorp/go-cleanhttp v0.5.1
-	github.com/hashicorp/go-getter v1.4.0
+	github.com/hashicorp/go-getter v1.4.2-0.20200106182914-9813cbd4eb02
 	github.com/hashicorp/go-hclog v0.9.2
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/hashicorp/go-plugin v1.2.0
+	github.com/hashicorp/go-plugin v1.3.0
 	github.com/hashicorp/go-uuid v1.0.1
 	github.com/hashicorp/go-version v1.2.0
-	github.com/hashicorp/hcl/v2 v2.0.0
+	github.com/hashicorp/hcl/v2 v2.3.0
 	github.com/hashicorp/logutils v1.0.0
 	github.com/hashicorp/terraform-config-inspect v0.0.0-20191115094559-17f92b0546e8
 	github.com/hashicorp/terraform-json v0.5.0
-	github.com/hashicorp/terraform-plugin-test v1.4.3
+	github.com/hashicorp/terraform-plugin-test/v2 v2.0.0-20200808000650-7f258174604f
 	github.com/hashicorp/terraform-svchost v0.0.0-20191011084731-65d371908596
 	github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect
 	github.com/keybase/go-crypto v0.0.0-20161004153544-93f5b35093ba
@@ -37,19 +38,19 @@ require (
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
 	github.com/mitchellh/copystructure v1.0.0
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/mitchellh/go-testing-interface v1.0.4
 	github.com/mitchellh/go-wordwrap v1.0.0
 	github.com/mitchellh/mapstructure v1.1.2
 	github.com/mitchellh/reflectwalk v1.0.1
 	github.com/pierrec/lz4 v2.0.5+incompatible
 	github.com/posener/complete v1.2.1 // indirect
 	github.com/spf13/afero v1.2.2
+	github.com/ulikunitz/xz v0.5.7 // indirect
 	github.com/vmihailenco/msgpack v4.0.1+incompatible // indirect
 	github.com/zclconf/go-cty v1.2.1
 	github.com/zclconf/go-cty-yaml v1.0.1
-	golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
-	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
-	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
-	golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0
-	google.golang.org/genproto v0.0.0-20200310143817-43be25429f5a // indirect
-	google.golang.org/grpc v1.27.1
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
+	golang.org/x/tools v0.0.0-20200713011307-fd294ab11aed
+	google.golang.org/grpc v1.30.0
 )

helper/logging/logging.go

@@ -9,6 +9,7 @@ import (
 	"syscall"
 
 	"github.com/hashicorp/logutils"
+	testing "github.com/mitchellh/go-testing-interface"
 )
 
 // These are the environmental variables that determine if we log, and if
@@ -48,10 +49,23 @@ func LogOutput() (logOutput io.Writer, err error) {
 	return
 }
 
+// SetTestOutput is equivalent to SetOutput, but declares itself a test
+// helper so it doesn't show up as the source of errors when testing.
+func SetTestOutput(t testing.T) {
+	setOutput(t)
+}
+
 // SetOutput checks for a log destination with LogOutput, and calls
 // log.SetOutput with the result. If LogOutput returns nil, SetOutput uses
 // ioutil.Discard. Any error from LogOutout is fatal.
 func SetOutput() {
+	setOutput(nil)
+}
+
+func setOutput(t testing.T) {
+	if t != nil {
+		t.Helper()
+	}
 	out, err := LogOutput()
 	if err != nil {
 		log.Fatal(err)

helper/resource/plugin.go

@@ -0,0 +1,168 @@
+package resource
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/terraform-plugin-sdk/acctest"
+	"github.com/hashicorp/terraform-plugin-sdk/helper/logging"
+	grpcplugin "github.com/hashicorp/terraform-plugin-sdk/internal/helper/plugin"
+	proto "github.com/hashicorp/terraform-plugin-sdk/internal/tfplugin5"
+	"github.com/hashicorp/terraform-plugin-sdk/plugin"
+	"github.com/hashicorp/terraform-plugin-sdk/terraform"
+	tftest "github.com/hashicorp/terraform-plugin-test/v2"
+	testing "github.com/mitchellh/go-testing-interface"
+)
+
+func runProviderCommand(t testing.T, f func() error, wd *tftest.WorkingDir, factories map[string]terraform.ResourceProviderFactory) error {
+	// don't point to this as a test failure location
+	// point to whatever called it
+	t.Helper()
+
+	// for backwards compatibility, make this opt-in
+	if os.Getenv("TF_ACCTEST_REATTACH") != "1" {
+		log.Println("[DEBUG] TF_ACCTEST_REATTACH not set to 1, not using reattach-based testing")
+		return f()
+	}
+	if acctest.TestHelper == nil {
+		log.Println("[DEBUG] acctest.TestHelper is nil, assuming we're not using binary acceptance testing")
+		return f()
+	}
+	log.Println("[DEBUG] TF_ACCTEST_REATTACH set to 1 and acctest.TestHelper is not nil, using reattach-based testing")
+
+	// Run the providers in the same process as the test runner using the
+	// reattach behavior in Terraform. This ensures we get test coverage
+	// and enables the use of delve as a debugger.
+	//
+	// This behavior is only available in Terraform 0.12.26 and later.
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// this is needed so Terraform doesn't default to expecting protocol 4;
+	// we're skipping the handshake because Terraform didn't launch the
+	// plugins.
+	os.Setenv("PLUGIN_PROTOCOL_VERSIONS", "5")
+
+	// Terraform 0.12.X and 0.13.X+ treat namespaceless providers
+	// differently in terms of what namespace they default to. So we're
+	// going to set both variations, as we don't know which version of
+	// Terraform we're talking to. We're also going to allow overriding
+	// the host or namespace using environment variables.
+	var namespaces []string
+	host := "registry.terraform.io"
+	if v := os.Getenv("TF_ACC_PROVIDER_NAMESPACE"); v != "" {
+		namespaces = append(namespaces, v)
+	} else {
+		namespaces = append(namespaces, "-", "hashicorp")
+	}
+	if v := os.Getenv("TF_ACC_PROVIDER_HOST"); v != "" {
+		host = v
+	}
+
+	// Spin up gRPC servers for every provider factory, start a
+	// WaitGroup to listen for all of the close channels.
+	var wg sync.WaitGroup
+	reattachInfo := map[string]plugin.ReattachConfig{}
+	for providerName, factory := range factories {
+		// providerName may be returned as terraform-provider-foo, and
+		// we need just foo. So let's fix that.
+		providerName = strings.TrimPrefix(providerName, "terraform-provider-")
+
+		provider, err := factory()
+		if err != nil {
+			return fmt.Errorf("unable to create provider %q from factory: %v", providerName, err)
+		}
+
+		// keep track of the running factory, so we can make sure it's
+		// shut down.
+		wg.Add(1)
+
+		// configure the settings our plugin will be served with
+		// the GRPCProviderFunc wraps a non-gRPC provider server
+		// into a gRPC interface, and the logger just discards logs
+		// from go-plugin.
+		opts := &plugin.ServeOpts{
+			GRPCProviderFunc: func() proto.ProviderServer {
+				return grpcplugin.NewGRPCProviderServerShim(provider)
+			},
+			Logger: hclog.New(&hclog.LoggerOptions{
+				Name:   "plugintest",
+				Level:  hclog.Trace,
+				Output: ioutil.Discard,
+			}),
+		}
+
+		// let's actually start the provider server
+		config, closeCh, err := plugin.DebugServe(ctx, opts)
+		if err != nil {
+			return fmt.Errorf("unable to serve provider %q: %v", providerName, err)
+		}
+
+		// plugin.DebugServe hijacks our log output location, so let's
+		// reset it
+		logging.SetTestOutput(t)
+
+		// when the provider exits, remove one from the waitgroup
+		// so we can track when everything is done
+		go func(c <-chan struct{}) {
+			<-c
+			wg.Done()
+		}(closeCh)
+
+		// set our provider's reattachinfo in our map, once
+		// for every namespace that different Terraform versions
+		// may expect.
+		for _, ns := range namespaces {
+			reattachInfo[strings.TrimSuffix(host, "/")+"/"+
+				strings.TrimSuffix(ns, "/")+"/"+
+				providerName] = config
+		}
+	}
+
+	// set the environment variable that will tell Terraform how to
+	// connect to our various running servers.
+	reattachStr, err := json.Marshal(reattachInfo)
+	if err != nil {
+		return err
+	}
+	wd.Setenv("TF_REATTACH_PROVIDERS", string(reattachStr))
+
+	// ok, let's call whatever Terraform command the test was trying to
+	// call, now that we know it'll attach back to those servers we just
+	// started.
+	err = f()
+	if err != nil {
+		log.Printf("[WARN] Got error running Terraform: %s", err)
+	}
+
+	// cancel the servers so they'll return. Otherwise, this closeCh won't
+	// get closed, and we'll hang here.
+	cancel()
+
+	// wait for the servers to actually shut down; it may take a moment for
+	// them to clean up, or whatever.
+	// TODO: add a timeout here?
+	// PC: do we need one? The test will time out automatically...
+	wg.Wait()
+
+	// once we've run the Terraform command, let's remove the reattach
+	// information from the WorkingDir's environment. The WorkingDir will
+	// persist until the next call, but the server in the reattach info
+	// doesn't exist anymore at this point, so the reattach info is no
+	// longer valid. In theory it should be overwritten in the next call,
+	// but just to avoid any confusing bug reports, let's just unset the
+	// environment variable altogether.
+	wd.Unsetenv("TF_REATTACH_PROVIDERS")
+
+	// return any error returned from the orchestration code running
+	// Terraform commands
+	return err
+}

helper/resource/testing.go

@@ -312,6 +312,17 @@ type TestCase struct {
 	Providers         map[string]terraform.ResourceProvider
 	ProviderFactories map[string]terraform.ResourceProviderFactory
 
+	// ExternalProviders are providers the TestCase relies on that should
+	// be downloaded from the registry during init. This is only really
+	// necessary to set if you're using import, as providers in your config
+	// will be automatically retrieved during init. Import doesn't always
+	// use a config, however, so we allow manually specifying them here to
+	// be downloaded for import tests.
+	//
+	// ExternalProviders will only be used when using binary acceptance
+	// testing in reattach mode.
+	ExternalProviders map[string]ExternalProvider
+
 	// PreventPostDestroyRefresh can be set to true for cases where data sources
 	// are tested alongside real resources
 	PreventPostDestroyRefresh bool
@@ -343,6 +354,13 @@ type TestCase struct {
 	DisableBinaryDriver bool
 }
 
+// ExternalProvider holds information about third-party providers that should
+// be downloaded by Terraform as part of running the test step.
+type ExternalProvider struct {
+	VersionConstraint string // the version constraint for the provider
+	Source            string // the provider source
+}
+
 // TestStep is a single apply sequence of a test, done within the
 // context of a state.
 //
@@ -648,7 +666,10 @@ func Test(t TestT, c TestCase) {
 		} else {
 			if step.ImportState {
 				if step.Config == "" {
-					step.Config = testProviderConfig(c)
+					step.Config, err = testProviderConfig(c)
+					if err != nil {
+						t.Fatal("Error setting config for providers: " + err.Error())
+					}
 				}
 
 				// Can optionally set step.Config in addition to
@@ -757,13 +778,38 @@ func Test(t TestT, c TestCase) {
 // testProviderConfig takes the list of Providers in a TestCase and returns a
 // config with only empty provider blocks. This is useful for Import, where no
 // config is provided, but the providers must be defined.
-func testProviderConfig(c TestCase) string {
+func testProviderConfig(c TestCase) (string, error) {
 	var lines []string
+	var requiredProviders []string
 	for p := range c.Providers {
 		lines = append(lines, fmt.Sprintf("provider %q {}\n", p))
 	}
+	for p, v := range c.ExternalProviders {
+		if _, ok := c.Providers[p]; ok {
+			return "", fmt.Errorf("Provider %q set in both Providers and ExternalProviders for TestCase. Must be set in only one.", p)
+		}
+		if _, ok := c.ProviderFactories[p]; ok {
+			return "", fmt.Errorf("Provider %q set in both ProviderFactories and ExternalProviders for TestCase. Must be set in only one.", p)
+		}
+		lines = append(lines, fmt.Sprintf("provider %q {}\n", p))
+		var providerBlock string
+		if v.VersionConstraint != "" {
+			providerBlock = fmt.Sprintf("%s\nversion = %q", providerBlock, v.VersionConstraint)
+		}
+		if v.Source != "" {
+			providerBlock = fmt.Sprintf("%s\nsource = %q", providerBlock, v.Source)
+		}
+		if providerBlock != "" {
+			providerBlock = fmt.Sprintf("%s = {%s\n}\n", p, providerBlock)
+		}
+		requiredProviders = append(requiredProviders, providerBlock)
+	}
+
+	if len(requiredProviders) > 0 {
+		lines = append([]string{fmt.Sprintf("terraform {\nrequired_providers {\n%s}\n}\n\n", strings.Join(requiredProviders, ""))}, lines...)
+	}
 
-	return strings.Join(lines, "")
+	return strings.Join(lines, ""), nil
 }
 
 // testProviderFactories combines the fixed Providers and