Adapt installer to use Terraform 0.10.*

Makefile

@@ -1,50 +1,54 @@
 CLUSTER ?= demo
 PLATFORM ?= aws
 TMPDIR ?= /tmp
+GOOS=$(shell uname -s | tr '[:upper:]' '[:lower:]')
+GOARCH=amd64
 TOP_DIR := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 BUILD_DIR = $(TOP_DIR)/build/$(CLUSTER)
-INSTALLER_BIN = $(TOP_DIR)/installer/bin/$(shell uname | tr '[:upper:]' '[:lower:]')/installer
+PLUGIN_DIR = $(BUILD_DIR)/terraform.d/plugins/$(GOOS)_$(GOARCH)
+INSTALLER_PATH = $(TOP_DIR)/installer/bin/$(shell uname | tr '[:upper:]' '[:lower:]')
+INSTALLER_BIN = $(INSTALLER_PATH)/installer
 TF_DOCS := $(shell which terraform-docs 2> /dev/null)
 TF_EXAMPLES := $(shell which terraform-examples 2> /dev/null)
-TF_RC := $(TOP_DIR)/.terraformrc
-TF_CMD = TERRAFORM_CONFIG=$(TF_RC) terraform
+TF_CMD = terraform
+
+include ./makelib/*.mk
 
 $(info Using build directory [${BUILD_DIR}])
 
 .PHONY: all
-all: apply
+all: $(INSTALLER_BIN) custom-providers
 
 $(INSTALLER_BIN):
 	$(MAKE) build -C $(TOP_DIR)/installer
 
-installer-env: $(INSTALLER_BIN) terraformrc.example
-	sed "s|<PATH_TO_INSTALLER>|$(INSTALLER_BIN)|g" terraformrc.example > $(TF_RC)
-
 .PHONY: localconfig
 localconfig:
 	mkdir -p $(BUILD_DIR)
 	cp examples/*$(subst /,-,$(PLATFORM)) $(BUILD_DIR)/terraform.tfvars
 
+$(PLUGIN_DIR):
+	mkdir -p $(PLUGIN_DIR)
+	ln -s $(INSTALLER_PATH)/terraform-provider-* $(PLUGIN_DIR)
+
 .PHONY: terraform-init
-terraform-init: installer-env
+terraform-init: custom-providers $(PLUGIN_DIR)
 ifneq ($(shell $(TF_CMD) version | grep -E "Terraform v0\.1[0-9]\.[0-9]+"), )
 	cd $(BUILD_DIR) && $(TF_CMD) init $(TF_INIT_OPTIONS) $(TOP_DIR)/platforms/$(PLATFORM)
-endif
-
-.PHONY: terraform-get
-terraform-get: terraform-init
+else
 	cd $(BUILD_DIR) && $(TF_CMD) get $(TF_GET_OPTIONS) $(TOP_DIR)/platforms/$(PLATFORM)
+endif
 
 .PHONY: plan
-plan: installer-env terraform-get
+plan: terraform-init
 	cd $(BUILD_DIR) && $(TF_CMD) plan $(TF_PLAN_OPTIONS) $(TOP_DIR)/platforms/$(PLATFORM)
 
 .PHONY: apply
-apply: installer-env terraform-get
+apply: terraform-init
 	cd $(BUILD_DIR) && $(TF_CMD) apply $(TF_APPLY_OPTIONS) $(TOP_DIR)/platforms/$(PLATFORM)
 
 .PHONY: destroy
-destroy: installer-env terraform-get
+destroy: terraform-init
 	cd $(BUILD_DIR) && $(TF_CMD) destroy $(TF_DESTROY_OPTIONS) -force $(TOP_DIR)/platforms/$(PLATFORM)
 
 .PHONY: payload
@@ -118,7 +122,7 @@ examples:
 			platforms/vmware/variables.tf)
 
 .PHONY: clean
-clean: destroy
+clean:
 	rm -rf $(BUILD_DIR)
 	$(MAKE) clean -C $(TOP_DIR)/installer
 	rm -f $(TF_RC)

images/tectonic-builder/Dockerfile

@@ -37,7 +37,7 @@ RUN chmod 777 -R /go
 
 ### Install Shellcheck, Terraform, NodeJS and Yarn
 ENV SHELLCHECK_VERSION="v0.4.6"
-ENV TERRAFORM_VERSION="v0.9.11-coreos.2"
+ENV TERRAFORM_VERSION="v0.10.4-coreos.1"
 ENV NODE_VERSION="v8.1.2"
 ENV YARN_VERSION="v0.24.6"
 ENV MATCHBOXVERSION="v0.6.1"

images/tectonic-installer/Dockerfile

@@ -1,6 +1,6 @@
 FROM golang:1.8.3-stretch
 
-ENV TERRAFORM_VERSION="0.9.6"
+ENV TERRAFORM_VERSION="0.10.4"
 
 RUN apt-get update \
     && apt-get install --no-install-recommends -y -q \
@@ -16,5 +16,4 @@ ADD config.tf $PROJECT_DIR/
 ADD installer/bin/linux/installer $PROJECT_DIR/installer/bin/linux/
 ADD platforms $PROJECT_DIR/platforms
 ADD modules $PROJECT_DIR/modules
-ADD terraformrc.example $PROJECT_DIR/
 

images/tectonic-smoke-test-env/Dockerfile

@@ -1,6 +1,6 @@
 FROM ruby:2.4-slim
 
-ENV TERRAFORM_VERSION="v0.9.11-coreos.2"
+ENV TERRAFORM_VERSION="v0.10.4-coreos.1"
 
 COPY ./tests/rspec/Gemfile* /tmp/app/
 

installer/api/handlers_terraform.go

@@ -4,7 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"regexp"
 	"time"
@@ -17,7 +19,8 @@ import (
 )
 
 const (
-	bcryptCost = 12
+	bcryptCost        = 12
+	pluginsFolderName = "terraform.d/plugins"
 )
 
 // TerraformApplyHandlerInput describes the input expected by the
@@ -175,12 +178,27 @@ func newExecutorFromApplyHandlerInput(input *TerraformApplyHandlerInput) (*terra
 	}
 	exPath := filepath.Join(binaryPath, "clusters", clusterName+time.Now().Format("_2006-01-02_15-04-05"))
 
+	// Publish custom providers to execution environment
+	clusterPluginDir := filepath.Join(exPath, pluginsFolderName)
+	err = os.MkdirAll(clusterPluginDir, os.ModeDir|0755)
+	if err != nil {
+		return nil, newInternalServerError("Could not create custom provider plugins location: %s", err)
+	}
+	customPlugins := []string{}
+	customPlugins, err = filepath.Glob(path.Join(binaryPath, "terraform-provider-*"))
+	if err != nil {
+		return nil, newInternalServerError("Could not locate custom provider plugins: %s", err)
+	}
+	for _, pluginBinPath := range customPlugins {
+		pluginBin := filepath.Base(pluginBinPath)
+		os.Symlink(pluginBinPath, filepath.Join(clusterPluginDir, pluginBin))
+	}
+
 	// Create a new Executor.
 	ex, err := terraform.NewExecutor(exPath)
 	if err != nil {
 		return nil, newInternalServerError("Could not create Terraform executor: %s", err)
 	}
-
 	// Write the License and Pull Secret to disk, and wire these files in the
 	// variables.
 	if input.License == "" {

installer/cmd/installer/main.go

@@ -13,15 +13,9 @@ import (
 	"github.com/toqueteos/webbrowser"
 
 	"github.com/coreos/tectonic-installer/installer/api"
-	"github.com/coreos/tectonic-installer/installer/pkg/terraform"
 )
 
 func main() {
-	// TerraForm entrypoint.
-	if os.Getenv("TF_PLUGIN_MAGIC_COOKIE") != "" {
-		terraform.ServePlugin(os.Args[1])
-		return
-	}
 
 	flags := struct {
 		address             string

installer/pkg/terraform/executor.go

@@ -22,7 +22,6 @@ import (
 )
 
 const (
-	configFileName = ".terraformrc"
 	stateFileName  = "terraform.tfstate"
 	tfVarsFileName = "terraform.tfvars"
 	logsFolderName = "logs"
@@ -75,7 +74,6 @@ const (
 // expose the live state to a file (or else).
 type Executor struct {
 	executionPath string
-	configPath    string
 	binaryPath    string
 	envVariables  map[string]string
 }
@@ -89,23 +87,12 @@ func NewExecutor(executionPath string) (*Executor, error) {
 	// if not existing.
 	os.MkdirAll(filepath.Join(ex.executionPath, logsFolderName), 0770)
 
-	// Create a Executor CLI configuration file, that contains the list of
-	// vendored providers/provisioners.
-	config, err := BuildPluginsConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	ex.configPath = filepath.Join(ex.WorkingDirectory(), configFileName)
-	if err = ioutil.WriteFile(ex.configPath, []byte(config), 0660); err != nil {
-		return nil, err
-	}
-
 	// Find the TerraForm binary.
-	ex.binaryPath, err = tfBinaryPath()
+	out, err := tfBinaryPath()
 	if err != nil {
 		return nil, err
 	}
+	ex.binaryPath = out
 
 	return ex, nil
 }
@@ -151,6 +138,7 @@ func (ex *Executor) AddEnvironmentVariables(envVars map[string]string) {
 	for k, v := range envVars {
 		ex.envVariables[k] = v
 	}
+	ex.envVariables["HOME"] = os.Getenv("HOME")
 }
 
 // AddCredentials is a convenience function that converts the given Credentials
@@ -187,7 +175,6 @@ func (ex *Executor) Execute(args ...string) (int, chan struct{}, error) {
 	// working directory (so the files such as terraform.tfstate are stored at
 	// the right place), extra environment variables and outputs.
 	cmd := exec.Command(ex.binaryPath, args...)
-	cmd.Env = append(cmd.Env, fmt.Sprintf("TERRAFORM_CONFIG=%s", ex.configPath))
 	// ssh changes its behavior based on these. pass them through so ssh-agent & stuff works
 	cmd.Env = append(cmd.Env, fmt.Sprintf("DISPLAY=%s", os.Getenv("DISPLAY")))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("PATH=%s", os.Getenv("PATH")))

installer/pkg/terraform/executor_test.go

@@ -8,8 +8,6 @@ import (
 	"io/ioutil"
 	"path/filepath"
 
-	"os"
-
 	"github.com/stretchr/testify/assert"
 )
 
@@ -36,18 +34,6 @@ output "foobar" {
 }
 `
 
-func TestMain(m *testing.M) {
-	// We need to hijack the testing execution when TerraForm calls back the
-	// binary to execute the plugins. Otherwise TerraForm calls back into the
-	// test suite and the RPC plugin handshake can't happen.
-	if os.Getenv("TF_PLUGIN_MAGIC_COOKIE") != "" {
-		ServePlugin(os.Args[1])
-		return
-	}
-
-	os.Exit(m.Run())
-}
-
 // TestExecutorSimple executes TerraForm apply with a custom plugin, verifies it
 // worked (State/Status), and then create a new executor at the path of the
 // existing one and verify the state is shared.
@@ -78,8 +64,25 @@ func TestExecutorSimple(t *testing.T) {
 	mainTFPath := filepath.Join(ex.WorkingDirectory(), "main.tf")
 	assert.Nil(t, ioutil.WriteFile(mainTFPath, []byte(tfTemplate), 0666))
 
+	// Execute TerraForm init.
+	id, done, err := ex.Execute("init")
+	assert.Nil(t, err)
+	assert.NotZero(t, id)
+
+	// Wait for its termination.
+	select {
+	case <-done:
+	case <-time.After(30 * time.Second):
+		assert.FailNow(t, "TerraForm init timed out")
+	}
+
+	// Verify status, state and output.
+	status, err := ex.Status(id)
+	assert.Nil(t, err)
+	assert.Equal(t, ExecutionStatusSuccess, status)
+
 	// Execute TerraForm apply.
-	id, done, err := ex.Execute("apply")
+	id, done, err = ex.Execute("apply")
 	assert.Nil(t, err)
 	assert.NotZero(t, id)
 
@@ -91,7 +94,7 @@ func TestExecutorSimple(t *testing.T) {
 	}
 
 	// Verify status, state and output.
-	status, err := ex.Status(id)
+	status, err = ex.Status(id)
 	assert.Nil(t, err)
 	assert.Equal(t, ExecutionStatusSuccess, status)