Avoid unseal failure if plugin backends fail to setup during postUnseal

vault/auth.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/helper/jsonutil"
@@ -460,10 +459,11 @@ func (c *Core) setupCredentials() error {
 		backend, err = c.newCredentialBackend(entry.Type, sysView, view, conf)
 		if err != nil {
 			c.logger.Error("core: failed to create credential entry", "path", entry.Path, "error", err)
-			if errwrap.Contains(err, ErrPluginNotFound.Error()) && entry.Type == "plugin" {
-				// If we encounter an error instantiating the backend due to it being missing from the catalog,
-				// skip backend initialization but register the entry to the mount table to preserve storage
-				// and path.
+			if entry.Type == "plugin" {
+				// If we encounter an error instantiating the backend due to an error,
+				// skip backend initialization but register the entry to the mount table
+				// to preserve storage and path.
+				c.logger.Warn("core: skipping plugin-based credential entry", "path", entry.Path)
 				goto ROUTER_MOUNT
 			}
 			return errLoadAuthFailed

vault/logical_system_integ_test.go

@@ -2,7 +2,9 @@ package vault_test
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -178,21 +180,22 @@ func testPlugin_CatalogRemoved(t *testing.T, btype logical.BackendType, testMoun
 	}
 
 	if testMount {
-		// Add plugin back to the catalog
-		vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical")
-
 		// Mount the plugin at the same path after plugin is re-added to the catalog
 		// and expect an error due to existing path.
 		var err error
 		switch btype {
 		case logical.TypeLogical:
+			// Add plugin back to the catalog
+			vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical")
 			_, err = core.Client.Logical().Write("sys/mounts/mock-0", map[string]interface{}{
 				"type": "plugin",
 				"config": map[string]interface{}{
 					"plugin_name": "mock-plugin",
 				},
 			})
 		case logical.TypeCredential:
+			// Add plugin back to the catalog
+			vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials")
 			_, err = core.Client.Logical().Write("sys/auth/mock-0", map[string]interface{}{
 				"type":        "plugin",
 				"plugin_name": "mock-plugin",
@@ -204,6 +207,129 @@ func testPlugin_CatalogRemoved(t *testing.T, btype logical.BackendType, testMoun
 	}
 }
 
+func TestSystemBackend_Plugin_continueOnError(t *testing.T) {
+	t.Run("secret", func(t *testing.T) {
+		t.Run("sha256_mismatch", func(t *testing.T) {
+			testPlugin_continueOnError(t, logical.TypeLogical, true)
+		})
+
+		t.Run("missing_plugin", func(t *testing.T) {
+			testPlugin_continueOnError(t, logical.TypeLogical, false)
+		})
+	})
+
+	t.Run("auth", func(t *testing.T) {
+		t.Run("sha256_mismatch", func(t *testing.T) {
+			testPlugin_continueOnError(t, logical.TypeCredential, true)
+		})
+
+		t.Run("missing_plugin", func(t *testing.T) {
+			testPlugin_continueOnError(t, logical.TypeCredential, false)
+		})
+	})
+}
+
+func testPlugin_continueOnError(t *testing.T, btype logical.BackendType, mismatch bool) {
+	cluster := testSystemBackendMock(t, 1, 1, btype)
+	defer cluster.Cleanup()
+
+	core := cluster.Cores[0]
+
+	// Get the registered plugin
+	req := logical.TestRequest(t, logical.ReadOperation, "sys/plugins/catalog/mock-plugin")
+	req.ClientToken = core.Client.Token()
+	resp, err := core.HandleRequest(req)
+	if err != nil || resp == nil || (resp != nil && resp.IsError()) {
+		t.Fatalf("err:%v resp:%#v", err, resp)
+	}
+
+	command, ok := resp.Data["command"].(string)
+	if !ok || command == "" {
+		t.Fatal("invalid command")
+	}
+
+	// Trigger a sha256 mistmatch or missing plugin error
+	if mismatch {
+		req = logical.TestRequest(t, logical.UpdateOperation, "sys/plugins/catalog/mock-plugin")
+		req.Data = map[string]interface{}{
+			"sha256":  "d17bd7334758e53e6fbab15745d2520765c06e296f2ce8e25b7919effa0ac216",
+			"command": filepath.Base(command),
+		}
+		req.ClientToken = core.Client.Token()
+		resp, err = core.HandleRequest(req)
+		if err != nil || (resp != nil && resp.IsError()) {
+			t.Fatalf("err:%v resp:%#v", err, resp)
+		}
+	} else {
+		err := os.Remove(filepath.Join(cluster.TempDir, filepath.Base(command)))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Seal the cluster
+	cluster.EnsureCoresSealed(t)
+
+	// Unseal the cluster
+	barrierKeys := cluster.BarrierKeys
+	for _, core := range cluster.Cores {
+		for _, key := range barrierKeys {
+			_, err := core.Unseal(vault.TestKeyCopy(key))
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+		sealed, err := core.Sealed()
+		if err != nil {
+			t.Fatalf("err checking seal status: %s", err)
+		}
+		if sealed {
+			t.Fatal("should not be sealed")
+		}
+		// Wait for active so post-unseal takes place
+		// If it fails, it means unseal process failed
+		vault.TestWaitActive(t, core.Core)
+	}
+
+	// Re-add the plugin to the catalog
+	switch btype {
+	case logical.TypeLogical:
+		vault.TestAddTestPluginTempDir(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical", cluster.TempDir)
+	case logical.TypeCredential:
+		vault.TestAddTestPluginTempDir(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials", cluster.TempDir)
+	}
+
+	// Reload the plugin
+	req = logical.TestRequest(t, logical.UpdateOperation, "sys/plugins/reload/backend")
+	req.Data = map[string]interface{}{
+		"plugin": "mock-plugin",
+	}
+	req.ClientToken = core.Client.Token()
+	resp, err = core.HandleRequest(req)
+	if err != nil || (resp != nil && resp.IsError()) {
+		t.Fatalf("err:%v resp:%#v", err, resp)
+	}
+
+	// Make a request to lazy load the plugin
+	var reqPath string
+	switch btype {
+	case logical.TypeLogical:
+		reqPath = "mock-0/internal"
+	case logical.TypeCredential:
+		reqPath = "auth/mock-0/internal"
+	}
+
+	req = logical.TestRequest(t, logical.ReadOperation, reqPath)
+	req.ClientToken = core.Client.Token()
+	resp, err = core.HandleRequest(req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if resp == nil {
+		t.Fatalf("bad: response should not be nil")
+	}
+}
+
 func TestSystemBackend_Plugin_autoReload(t *testing.T) {
 	cluster := testSystemBackendMock(t, 1, 1, logical.TypeLogical)
 	defer cluster.Cleanup()
@@ -332,7 +458,10 @@ func testSystemBackend_PluginReload(t *testing.T, reqData map[string]interface{}
 }
 
 // testSystemBackendMock returns a systemBackend with the desired number
-// of mounted mock plugin backends
+// of mounted mock plugin backends. numMounts alternates between different
+// ways of providing the plugin_name.
+//
+// The mounts are mounted at sys/mounts/mock-[numMounts] or sys/auth/mock-[numMounts]
 func testSystemBackendMock(t *testing.T, numCores, numMounts int, backendType logical.BackendType) *vault.TestCluster {
 	coreConfig := &vault.CoreConfig{
 		LogicalBackends: map[string]logical.Factory{
@@ -343,10 +472,17 @@ func testSystemBackendMock(t *testing.T, numCores, numMounts int, backendType lo
 		},
 	}
 
+	// Create a tempdir, cluster.Cleanup will clean up this directory
+	tempDir, err := ioutil.TempDir("", "vault-test-cluster")
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
 		HandlerFunc:        vaulthttp.Handler,
 		KeepStandbysSealed: true,
 		NumCores:           numCores,
+		TempDir:            tempDir,
 	})
 	cluster.Start()
 
@@ -358,7 +494,7 @@ func testSystemBackendMock(t *testing.T, numCores, numMounts int, backendType lo
 
 	switch backendType {
 	case logical.TypeLogical:
-		vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical")
+		vault.TestAddTestPluginTempDir(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical", tempDir)
 		for i := 0; i < numMounts; i++ {
 			// Alternate input styles for plugin_name on every other mount
 			options := map[string]interface{}{
@@ -380,7 +516,7 @@ func testSystemBackendMock(t *testing.T, numCores, numMounts int, backendType lo
 			}
 		}
 	case logical.TypeCredential:
-		vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials")
+		vault.TestAddTestPluginTempDir(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials", tempDir)
 		for i := 0; i < numMounts; i++ {
 			// Alternate input styles for plugin_name on every other mount
 			options := map[string]interface{}{

vault/mount.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/helper/jsonutil"
@@ -753,10 +752,11 @@ func (c *Core) setupMounts() error {
 		backend, err = c.newLogicalBackend(entry.Type, sysView, view, conf)
 		if err != nil {
 			c.logger.Error("core: failed to create mount entry", "path", entry.Path, "error", err)
-			if errwrap.Contains(err, ErrPluginNotFound.Error()) && entry.Type == "plugin" {
-				// If we encounter an error instantiating the backend due to it being missing from the catalog,
-				// skip backend initialization but register the entry to the mount table to preserve storage
-				// and path.
+			if entry.Type == "plugin" {
+				// If we encounter an error instantiating the backend due to an error,
+				// skip backend initialization but register the entry to the mount table
+				// to preserve storage and path.
+				c.logger.Warn("core: skipping plugin-based mount entry", "path", entry.Path)
 				goto ROUTER_MOUNT
 			}
 			return errLoadMountsFailed

vault/plugin_reload.go

@@ -79,15 +79,23 @@ func (c *Core) reloadMatchingPlugin(pluginName string) error {
 func (c *Core) reloadPluginCommon(entry *MountEntry, isAuth bool) error {
 	path := entry.Path
 
+	if isAuth {
+		path = credentialRoutePrefix + path
+	}
+
 	// Fast-path out if the backend doesn't exist
 	raw, ok := c.router.root.Get(path)
 	if !ok {
 		return nil
 	}
 
-	// Call backend's Cleanup routine
 	re := raw.(*routeEntry)
-	re.backend.Cleanup()
+
+	// Only call Cleanup if backend is initialized
+	if re.backend != nil {
+		// Call backend's Cleanup routine
+		re.backend.Cleanup()
+	}
 
 	view := re.storageView
 

vault/testing.go

@@ -379,6 +379,8 @@ func TestDynamicSystemView(c *Core) *dynamicSystemView {
 	return &dynamicSystemView{c, me}
 }
 
+// TestAddTestPlugin registers the testFunc as part of the plugin command to the
+// plugin catalog.
 func TestAddTestPlugin(t testing.T, c *Core, name, testFunc string) {
 	file, err := os.Open(os.Args[0])
 	if err != nil {
@@ -413,6 +415,65 @@ func TestAddTestPlugin(t testing.T, c *Core, name, testFunc string) {
 	}
 }
 
+func TestAddTestPluginTempDir(t testing.T, c *Core, name, testFunc, tempDir string) {
+	file, err := os.Open(os.Args[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Copy over the file to the temp dir
+	dst := filepath.Join(tempDir, filepath.Base(os.Args[0]))
+	out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fi.Mode())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer out.Close()
+
+	if _, err = io.Copy(out, file); err != nil {
+		t.Fatal(err)
+	}
+	err = out.Sync()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fullPath, err := filepath.EvalSymlinks(tempDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := os.Open(filepath.Join(fullPath, filepath.Base(os.Args[0])))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Find out the sha256
+	hash := sha256.New()
+
+	_, err = io.Copy(hash, reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sum := hash.Sum(nil)
+
+	// Set core's plugin directory and plugin catalog directory
+	c.pluginDirectory = fullPath
+	c.pluginCatalog.directory = fullPath
+
+	command := fmt.Sprintf("%s --test.run=%s", filepath.Base(os.Args[0]), testFunc)
+	err = c.pluginCatalog.Set(name, command, sum)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 var testLogicalBackends = map[string]logical.Factory{}
 var testCredentialBackends = map[string]logical.Factory{}