Pluggable secret backend

This commit extends SwarmKit secret management with pluggable secret
backends support. The solution uses the existing docker plugin
framework for loading plugins and the existing SwarmKit data backend for
storing them.

The approach is to add a new `driver` parameter to existing secrets,
which defines whether the values are taken as is or fetched from one of
the secret plugins. The loading of secrets is done using the standard
docker plugin infrastructure, which is already accessible in SwarmKit
and used in other flows (e.g., networking).
The fetched values are evaluated before assigning them to worker nodes,
so the payload is not stored in the raft store.

Remarks:
* I've added support for mocking the plugin subsystem when settings up
the controlapi server.
I preferred this approach over loading the full plugin subsystem in UT.

Work still needed in this CR:
- [ ] More unit tests (pending initial iteration)
- [ ] Customized error handling (e.g., customize error string for Not
Found)

Work still needed to complete this feature:
- [ ] Inject secrets as part of plugin initialization
- [ ] CLI support in docker
- [ ] Docs
- [ ] Support scheduling plugins in swarm
moby/moby#33575

Signed-off-by: liron <liron@twistlock.com>

api/specs.proto

@@ -395,7 +395,7 @@ message SecretSpec {
 	Driver templating = 3;
 
 	// Driver is the the secret driver that is used to store the specified secret
-	Driver driver = 4 [(gogoproto.nullable) = false];
+	Driver driver = 4;
 }
 
 // ConfigSpec specifies user-provided configuration files.

cmd/swarmctl/secret/create.go

@@ -55,7 +55,7 @@ var createCmd = &cobra.Command{
 		spec := &api.SecretSpec{
 			Annotations: api.Annotations{Name: args[0]},
 			Data:        secretData,
-			Driver:      api.Driver{Name: driver},
+			Driver:      &api.Driver{Name: driver},
 		}
 
 		resp, err := client.CreateSecret(common.Context(cmd), &api.CreateSecretRequest{Spec: spec})

manager/controlapi/secret.go

@@ -242,7 +242,12 @@ func validateSecretSpec(spec *api.SecretSpec) error {
 	if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil {
 		return err
 	}
-	if spec.Driver.Name != "" {
+	// Check if secret driver is defined
+	if spec.Driver != nil {
+		// Ensure secret driver has a name
+		if spec.Driver.Name == "" {
+			return grpc.Errorf(codes.InvalidArgument, "secret driver must have a name")
+		}
 		return nil
 	}
 	if err := validation.ValidateSecretPayload(spec.Data); err != nil {

manager/controlapi/secret_test.go

@@ -89,6 +89,16 @@ func TestValidateSecretSpec(t *testing.T) {
 		err := validateSecretSpec(good)
 		assert.NoError(t, err)
 	}
+
+	// Ensure secret driver has a name
+	spec := createSecretSpec("secret-driver", make([]byte, 1), nil)
+	spec.Driver = &api.Driver{}
+	err := validateSecretSpec(spec)
+	assert.Error(t, err)
+	assert.Equal(t, codes.InvalidArgument, grpc.Code(err), grpc.ErrorDesc(err))
+	spec.Driver.Name = "secret-driver"
+	err = validateSecretSpec(spec)
+	assert.NoError(t, err)
 }
 
 func TestCreateSecret(t *testing.T) {

manager/dispatcher/assignments.go

@@ -1,6 +1,7 @@
 package dispatcher
 
 import (
+	"fmt"
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/api/equality"
@@ -56,23 +57,14 @@ func (a *assignmentSet) addTaskDependencies(readTx store.ReadTx, t *api.Task) {
 		if len(a.tasksUsingDependency[mapKey]) == 0 {
 			a.tasksUsingDependency[mapKey] = make(map[string]struct{})
 
-			secret := store.GetSecret(readTx, secretID)
-			if secret == nil {
-				a.log.WithFields(logrus.Fields{
-					"secret.id":   secretID,
-					"secret.name": secretRef.SecretName,
-				}).Debug("secret not found")
-				continue
-			}
-
-			// populate secrets from secret driver
-			if err := a.populateSecretFromDriver(&secret.Spec, readTx); err != nil {
+			secret, err := a.secret(secretID, readTx)
+			if err != nil {
 				if err != nil {
 					a.log.WithFields(logrus.Fields{
 						"secret.id":   secretID,
 						"secret.name": secretRef.SecretName,
 						"error":       err,
-					}).Error("failed to populate driver secret")
+					}).Error("failed to fetcj secret")
 					continue
 				}
 			}
@@ -261,23 +253,28 @@ func (a *assignmentSet) message() api.AssignmentsMessage {
 	return message
 }
 
-// populateSecretFromPlugin populates the secret value for the given specification using the secret plugin subsystem.
-func (a *assignmentSet) populateSecretFromDriver(spec *api.SecretSpec, readTx store.ReadTx) error {
-	if spec == nil || spec.Driver.Name == "" {
-		return nil
+// secret populates the secret value from raft store. For external secrets, the value is populated
+// from the secret driver.
+func (a *assignmentSet) secret(secretID string, readTx store.ReadTx) (*api.Secret, error) {
+	secret := store.GetSecret(readTx, secretID)
+	if secret == nil {
+		return nil, fmt.Errorf("secret not found")
+	}
+	if secret.Spec.Driver == nil {
+		return secret, nil
 	}
-	d, err := a.dp.NewSecretDriver(&spec.Driver)
+	d, err := a.dp.NewSecretDriver(secret.Spec.Driver)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	value, err := d.Get(spec)
+	value, err := d.Get(&secret.Spec)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if err := validation.ValidateSecretPayload(value); err != nil {
-		return err
+		return nil, err
 	}
 	// Assign the secret
-	spec.Data = value
-	return nil
+	secret.Spec.Data = value
+	return secret, nil
 }

manager/dispatcher/dispatcher_test.go

@@ -455,7 +455,7 @@ func TestAssignmentsSecretDriver(t *testing.T) {
 		ID: "driverSecret",
 		Spec: api.SecretSpec{
 			Annotations: api.Annotations{Name: existingSecretName},
-			Driver:      api.Driver{Name: secretDriver},
+			Driver:      &api.Driver{Name: secretDriver},
 		},
 	}
 	config := &api.Config{