Add Secrets Database

cmd/bootstrap/utils/key_generation.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"crypto/rand"
 	"crypto/sha256"
 	"fmt"
 	gohash "hash"
@@ -29,6 +30,21 @@ func GenerateMachineAccountKey(seed []byte) (crypto.PrivateKey, error) {
 	return keys[0], nil
 }
 
+// GenerateSecretsDBEncryptionKey generates an encryption key for encrypting a
+// Badger database.
+func GenerateSecretsDBEncryptionKey() ([]byte, error) {
+	// 32-byte key to use AES-256
+	// https://pkg.go.dev/github.com/dgraph-io/badger/v2#Options.WithEncryptionKey
+	const keyLen = 32
+
+	key := make([]byte, keyLen)
+	_, err := rand.Read(key)
+	if err != nil {
+		return nil, fmt.Errorf("could not generate key: %w", err)
+	}
+	return key, nil
+}
+
 // The unstaked nodes have special networking keys, in two aspects:
 // - they use crypto.ECDSASecp256k1 keys, not crypto.ECDSAP256 keys,
 // - they use only positive keys (in the sense that the elliptic curve point of their public key is positive)
@@ -133,6 +149,10 @@ func GenerateKeys(algo crypto.SigningAlgorithm, n int, seeds [][]byte) ([]crypto
 // the result to the given path.
 type WriteJSONFileFunc func(relativePath string, value interface{}) error
 
+// WriteFileFunc is the same as WriteJSONFileFunc, but it writes the bytes directly
+// rather than marshalling a structure to json.
+type WriteFileFunc func(relativePath string, data []byte) error
+
 // WriteMachineAccountFiles writes machine account key files for a set of nodeInfos.
 // Assumes that machine accounts have been created using the default execution state
 // bootstrapping. Further assumes that the order of nodeInfos is the same order that
@@ -208,6 +228,27 @@ func WriteMachineAccountFiles(chainID flow.ChainID, nodeInfos []bootstrap.NodeIn
 	return nil
 }
 
+// WriteSecretsDBEncryptionKeyFiles writes secret db encryption keys to private
+// node info directory.
+func WriteSecretsDBEncryptionKeyFiles(nodeInfos []bootstrap.NodeInfo, write WriteFileFunc) error {
+
+	for _, nodeInfo := range nodeInfos {
+
+		// generate an encryption key for the node
+		encryptionKey, err := GenerateSecretsDBEncryptionKey()
+		if err != nil {
+			return err
+		}
+
+		path := fmt.Sprintf(bootstrap.PathSecretsEncryptionKey, nodeInfo.NodeID)
+		err = write(path, encryptionKey)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // WriteStakingNetworkingKeyFiles writes staking and networking keys to private
 // node info files.
 func WriteStakingNetworkingKeyFiles(nodeInfos []bootstrap.NodeInfo, write WriteJSONFileFunc) error {

cmd/consensus/main.go

@@ -63,7 +63,6 @@ import (
 	"github.com/onflow/flow-go/state/protocol/blocktimer"
 	"github.com/onflow/flow-go/state/protocol/events/gadgets"
 	"github.com/onflow/flow-go/storage"
-	"github.com/onflow/flow-go/storage/badger"
 	bstorage "github.com/onflow/flow-go/storage/badger"
 	"github.com/onflow/flow-go/utils/io"
 )
@@ -117,6 +116,7 @@ func main() {
 		dkgBrokerTunnel         *dkgmodule.BrokerTunnel
 		blockTimer              protocol.BlockTimer
 		finalizedHeader         *synceng.FinalizedHeaderCache
+		dkgKeyStore             *bstorage.DKGKeys
 	)
 
 	nodeBuilder := cmd.FlowNode(flow.RoleConsensus.String())
@@ -156,6 +156,10 @@ func main() {
 			conMetrics = metrics.NewConsensusCollector(node.Tracer, node.MetricsRegisterer)
 			return nil
 		}).
+		Module("dkg key storage", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) error {
+			dkgKeyStore, err = bstorage.NewDKGKeys(node.Metrics.Cache, node.SecretsDB)
+			return err
+		}).
 		Module("mutable follower state", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) error {
 			// For now, we only support state implementations from package badger.
 			// If we ever support different implementations, the following can be replaced by a type-aware factory
@@ -252,7 +256,7 @@ func main() {
 			if err != nil {
 				return err
 			}
-			err = node.Storage.DKGKeys.InsertMyDKGPrivateInfo(epochCounter, privateDKGData)
+			err = dkgKeyStore.InsertMyDKGPrivateInfo(epochCounter, privateDKGData)
 			if err != nil && !errors.Is(err, storage.ErrAlreadyExists) {
 				return err
 			}
@@ -270,7 +274,7 @@ func main() {
 					if err != nil {
 						return err
 					}
-					_, err = node.Storage.DKGKeys.RetrieveMyDKGPrivateInfo(counter)
+					_, err = dkgKeyStore.RetrieveMyDKGPrivateInfo(counter)
 					if err != nil {
 						return err
 					}
@@ -594,7 +598,7 @@ func main() {
 
 			epochLookup := epochs.NewEpochLookup(node.State)
 
-			thresholdSignerStore := signature.NewEpochAwareSignerStore(epochLookup, node.Storage.DKGKeys)
+			thresholdSignerStore := signature.NewEpochAwareSignerStore(epochLookup, dkgKeyStore)
 
 			// initialize the combined signer for hotstuff
 			var signer hotstuff.SignerVerifier
@@ -714,9 +718,37 @@ func main() {
 			viewsObserver := gadgets.NewViews()
 			node.ProtocolEvents.AddConsumer(viewsObserver)
 
-			// keyDB is used to store the private key resulting from the node's
-			// participation in the DKG run
-			keyDB := badger.NewDKGKeys(node.Metrics.Cache, node.DB)
+			// create flow client with correct GRPC configuration for QC contract client
+			var flowClient *client.Client
+			if insecureAccessAPI {
+				flowClient, err = common.InsecureFlowClient(accessAddress)
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				if secureAccessNodeID == "" {
+					return nil, fmt.Errorf("invalid flag --secure-access-node-id required")
+				}
+
+				nodeID, err := flow.HexStringToIdentifier(secureAccessNodeID)
+				if err != nil {
+					return nil, fmt.Errorf("could not get flow identifer from secured access node id: %s", secureAccessNodeID)
+				}
+
+				identities, err := node.State.Sealed().Identities(filter.HasNodeID(nodeID))
+				if err != nil {
+					return nil, fmt.Errorf("could not get identity of secure access node: %s", secureAccessNodeID)
+				}
+
+				if len(identities) < 1 {
+					return nil, fmt.Errorf("could not find identity of secure access node: %s", secureAccessNodeID)
+				}
+
+				flowClient, err = common.SecureFlowClient(accessAddress, identities[0].NetworkPubKey.String()[2:])
+				if err != nil {
+					return nil, err
+				}
+			}
 
 			// construct DKG contract client
 			dkgContractClient, err := createDKGContractClient(node, machineAccountInfo, flowClient)
@@ -730,7 +762,7 @@ func main() {
 				node.Logger,
 				node.Me,
 				node.State,
-				keyDB,
+				dkgKeyStore,
 				dkgmodule.NewControllerFactory(
 					node.Logger,
 					node.Me,

cmd/node_builder.go

@@ -112,6 +112,8 @@ type BaseConfig struct {
 	NodeRole              string
 	timeout               time.Duration
 	datadir               string
+	secretsdir            string
+	secretsDBEnabled      bool
 	level                 string
 	metricsPort           uint
 	BootstrapDir          string
@@ -143,6 +145,7 @@ type NodeConfig struct {
 	MetricsRegisterer prometheus.Registerer
 	Metrics           Metrics
 	DB                *badger.DB
+	SecretsDB         *badger.DB
 	Storage           Storage
 	ProtocolEvents    *events.Distributor
 	State             protocol.State
@@ -171,6 +174,7 @@ type NodeConfig struct {
 func DefaultBaseConfig() *BaseConfig {
 	homedir, _ := os.UserHomeDir()
 	datadir := filepath.Join(homedir, ".flow", "database")
+
 	return &BaseConfig{
 		nodeIDHex:             NotSet,
 		adminAddr:             NotSet,
@@ -181,6 +185,8 @@ func DefaultBaseConfig() *BaseConfig {
 		BootstrapDir:          "bootstrap",
 		timeout:               1 * time.Minute,
 		datadir:               datadir,
+		secretsdir:            NotSet,
+		secretsDBEnabled:      true,
 		level:                 "info",
 		PeerUpdateInterval:    p2p.DefaultPeerUpdateInterval,
 		UnicastMessageTimeout: p2p.DefaultUnicastTimeout,

cmd/scaffold.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
@@ -73,7 +74,6 @@ type Storage struct {
 	Setups       storage.EpochSetups
 	Commits      storage.EpochCommits
 	Statuses     storage.EpochStatuses
-	DKGKeys      storage.DKGKeys
 }
 
 type namedModuleFunc struct {
@@ -122,7 +122,8 @@ func (fnb *FlowNodeBuilder) BaseFlags() {
 	fnb.flags.StringVar(&fnb.BaseConfig.BindAddr, "bind", defaultConfig.BindAddr, "address to bind on")
 	fnb.flags.StringVarP(&fnb.BaseConfig.BootstrapDir, "bootstrapdir", "b", defaultConfig.BootstrapDir, "path to the bootstrap directory")
 	fnb.flags.DurationVarP(&fnb.BaseConfig.timeout, "timeout", "t", defaultConfig.timeout, "node startup / shutdown timeout")
-	fnb.flags.StringVarP(&fnb.BaseConfig.datadir, "datadir", "d", defaultConfig.datadir, "directory to store the protocol state")
+	fnb.flags.StringVarP(&fnb.BaseConfig.datadir, "datadir", "d", defaultConfig.datadir, "directory to store the public database (protocol state)")
+	fnb.flags.StringVar(&fnb.BaseConfig.secretsdir, "secretsdir", defaultConfig.secretsdir, "directory to store private database (secrets)")
 	fnb.flags.StringVarP(&fnb.BaseConfig.level, "loglevel", "l", defaultConfig.level, "level for logging output")
 	fnb.flags.DurationVar(&fnb.BaseConfig.PeerUpdateInterval, "peerupdate-interval", defaultConfig.PeerUpdateInterval, "how often to refresh the peer connections for the node")
 	fnb.flags.DurationVar(&fnb.BaseConfig.UnicastMessageTimeout, "unicast-timeout", defaultConfig.UnicastMessageTimeout, "how long a unicast transmission can take to complete")
@@ -456,9 +457,42 @@ func (fnb *FlowNodeBuilder) initDB() {
 		WithValueLogFileSize(128 << 23).
 		WithValueLogMaxEntries(100000) // Default is 1000000
 
-	db, err := badger.Open(opts)
-	fnb.MustNot(err).Msg("could not open key-value store")
-	fnb.DB = db
+	publicDB, err := bstorage.InitPublic(opts)
+	fnb.MustNot(err).Msg("could not open public db")
+	fnb.DB = publicDB
+}
+
+func (fnb *FlowNodeBuilder) initSecretsDB() {
+
+	// if the secrets DB is disabled (only applicable for Consensus Follower,
+	// which makes use of this same logic), skip this initialization
+	if !fnb.BaseConfig.secretsDBEnabled {
+		return
+	}
+
+	if fnb.BaseConfig.secretsdir == NotSet {
+		fnb.Logger.Fatal().Msgf("missing required flag '--secretsdir'")
+	}
+
+	err := os.MkdirAll(fnb.BaseConfig.secretsdir, 0700)
+	fnb.MustNot(err).Str("dir", fnb.BaseConfig.secretsdir).Msg("could not create secrets db dir")
+
+	log := sutil.NewLogger(fnb.Logger)
+
+	opts := badger.DefaultOptions(fnb.BaseConfig.secretsdir).WithLogger(log)
+	// attempt to read an encryption key for the secrets DB from the canonical path
+	encryptionKey, err := loadSecretsEncryptionKey(fnb.BootstrapDir, fnb.NodeID)
+	if errors.Is(err, os.ErrNotExist) {
+		fnb.Logger.Warn().Msg("starting with secrets database encryption disabled")
+	} else if err != nil {
+		fnb.Logger.Fatal().Err(err).Msg("failed to read secrets db encryption key")
+	} else {
+		opts = opts.WithEncryptionKey(encryptionKey)
+	}
+
+	secretsDB, err := bstorage.InitSecret(opts)
+	fnb.MustNot(err).Msg("could not open secrets db")
+	fnb.SecretsDB = secretsDB
 }
 
 func (fnb *FlowNodeBuilder) initStorage() {
@@ -484,7 +518,6 @@ func (fnb *FlowNodeBuilder) initStorage() {
 	setups := bstorage.NewEpochSetups(fnb.Metrics.Cache, fnb.DB)
 	commits := bstorage.NewEpochCommits(fnb.Metrics.Cache, fnb.DB)
 	statuses := bstorage.NewEpochStatuses(fnb.Metrics.Cache, fnb.DB)
-	dkgKeys := bstorage.NewDKGKeys(fnb.Metrics.Cache, fnb.DB)
 
 	fnb.Storage = Storage{
 		Headers:      headers,
@@ -500,7 +533,6 @@ func (fnb *FlowNodeBuilder) initStorage() {
 		Setups:       setups,
 		Commits:      commits,
 		Statuses:     statuses,
-		DKGKeys:      dkgKeys,
 	}
 }
 
@@ -805,6 +837,12 @@ func WithDataDir(dataDir string) Option {
 	}
 }
 
+func WithSecretsDBEnabled(enabled bool) Option {
+	return func(config *BaseConfig) {
+		config.secretsDBEnabled = enabled
+	}
+}
+
 func WithMetricsEnabled(enabled bool) Option {
 	return func(config *BaseConfig) {
 		config.metricsEnabled = enabled
@@ -939,6 +977,7 @@ func (fnb *FlowNodeBuilder) Ready() <-chan struct{} {
 		fnb.initProfiler()
 
 		fnb.initDB()
+		fnb.initSecretsDB()
 
 		fnb.initMetrics()
 
@@ -1040,3 +1079,13 @@ func loadPrivateNodeInfo(dir string, myID flow.Identifier) (*bootstrap.NodeInfoP
 	err = json.Unmarshal(data, &info)
 	return &info, err
 }
+
+// loadSecretsEncryptionKey loads the encryption key for the secrets database.
+// If the file does not exist, returns os.ErrNotExist.
+func loadSecretsEncryptionKey(dir string, myID flow.Identifier) ([]byte, error) {
+	data, err := io.ReadFile(filepath.Join(dir, fmt.Sprintf(bootstrap.PathSecretsEncryptionKey, myID)))
+	if err != nil {
+		return nil, fmt.Errorf("could not read key: %w", err)
+	}
+	return data, nil
+}

cmd/scaffold_test.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/onflow/flow-go/cmd/bootstrap/utils"
+	"github.com/onflow/flow-go/fvm/errors"
+	"github.com/onflow/flow-go/model/bootstrap"
+	"github.com/onflow/flow-go/utils/unittest"
+)
+
+// TestLoadSecretsEncryptionKey checks that the key file is read correctly if it exists
+// and returns the expected sentinel error if it does not exist.
+func TestLoadSecretsEncryptionKey(t *testing.T) {
+	myID := unittest.IdentifierFixture()
+
+	unittest.RunWithTempDir(t, func(dir string) {
+		path := filepath.Join(dir, fmt.Sprintf(bootstrap.PathSecretsEncryptionKey, myID))
+
+		t.Run("should return ErrNotExist if file doesn't exist", func(t *testing.T) {
+			require.NoFileExists(t, path)
+			_, err := loadSecretsEncryptionKey(dir, myID)
+			assert.Error(t, err)
+			assert.True(t, errors.Is(err, os.ErrNotExist))
+		})
+
+		t.Run("should return key and no error if file exists", func(t *testing.T) {
+			err := os.MkdirAll(filepath.Join(dir, bootstrap.DirPrivateRoot, fmt.Sprintf("private-node-info_%v", myID)), 0700)
+			require.NoError(t, err)
+			key, err := utils.GenerateSecretsDBEncryptionKey()
+			require.NoError(t, err)
+			err = ioutil.WriteFile(path, key, 0700)
+			require.NoError(t, err)
+
+			data, err := loadSecretsEncryptionKey(dir, myID)
+			assert.NoError(t, err)
+			assert.Equal(t, key, data)
+		})
+	})
+}

deploy/systemd-docker/flow-access.service

@@ -30,6 +30,7 @@ ExecStart=docker run --rm \
 	--nodeid ${FLOW_GO_NODE_ID} \
 	--bootstrapdir /bootstrap \
 	--datadir /data/protocol \
+	--secretsdir /data/secrets \
 	--rpc-addr 0.0.0.0:9000 \
 	--secure-rpc-addr 0.0.0.0:9001 \
 	--http-addr 0.0.0.0:8000 \

deploy/systemd-docker/flow-collection.service

@@ -29,6 +29,7 @@ ExecStart=docker run --rm \
 	--nodeid ${FLOW_GO_NODE_ID} \
 	--bootstrapdir /bootstrap \
 	--datadir /data/protocol \
+	--secretsdir /data/secrets \
 	--ingress-addr 0.0.0.0:9000 \
 	--bind 0.0.0.0:3569 \
 	--loglevel ERROR