secret key should not be derived at runtime

CHANGELOG.md

@@ -1,4 +1,12 @@
 # [Monax Hoard](https://github.com/monax/hoard) Changelog
+## [7.1.0] - 2019-12-12
+### Changed
+- [CMD] Secret keys can now be loaded from the environment
+
+### Fixed
+- [ENCRYPTION] Secret keys are no longer derived at runtime due to high scrypt memory overhead
+
+
 ## [7.0.0] - 2019-12-02
 This release makes some changes to the Hoard protobuf and service that are backwards compatible for clients - Hoard v6 clients should work with Hoard v7 but hoard-js v7 will not work entirely correctly with Hoard v6 due to removal of oneof.
 
@@ -145,6 +153,7 @@ This is the first Hoard open source release and includes:
 	- Hoar-Daemon hoard
 	- Hoar-Control hoarctl CLI
 
+[7.1.0]: https://github.com/monax/hoard/compare/v7.0.0...v7.1.0
 [7.0.0]: https://github.com/monax/hoard/compare/v6.0.0...v7.0.0
 [6.0.0]: https://github.com/monax/hoard/compare/v5.1.0...v6.0.0
 [5.1.0]: https://github.com/monax/hoard/compare/v5.0.1...v5.1.0

NOTES.md

@@ -1,10 +1,6 @@
-This release makes some changes to the Hoard protobuf and service that are backwards compatible for clients - Hoard v6 clients should work with Hoard v7 but hoard-js v7 will not work entirely correctly with Hoard v6 due to removal of oneof.
-
 ### Changed
-- [API] Drop use of oneof in protobuf files - allow singleton fields to be sent with streamable fields
-- [API] Enforce that we only receive exactly one salt and grant spec in streams and that they come first
-- [NODEJS] Expose streaming promise client-side API to take advantage of streaming rather than loading entire file into buffer
+- [CMD] Secret keys can now be loaded from the environment
 
 ### Fixed
-- Ignoring Spec if Salt present in single message
+- [ENCRYPTION] Secret keys are no longer derived at runtime due to high scrypt memory overhead
 

cmd/hoard/config.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	b64 "encoding/base64"
+	"github.com/cep21/xdgbasedir"
+	cli "github.com/jawher/mow.cli"
+	"github.com/monax/hoard/v7/config"
+	"github.com/monax/hoard/v7/encryption"
+)
+
+func Config(cmd *cli.Cmd) {
+	conf := config.DefaultHoardConfig
+
+	outputOpt := cmd.StringOpt("o output", "",
+		"Instead of writing to STDOUT, write to the specified file")
+
+	overwriteOpt := cmd.BoolOpt("f force", false,
+		"Overwrite config file if it exists.")
+
+	jsonOpt := cmd.BoolOpt("j json", false,
+		"Print config to STDOUT as a single line of JSON (suitable for the --env config source)")
+
+	yamlOpt := cmd.BoolOpt("y yaml", false,
+		"Print config to STDOUT as a YAML specification")
+
+	initOpt := cmd.BoolOpt("i init", false, "Write file to "+
+		"XDG standard location")
+
+	chunkSizeOpt := cmd.IntOpt("c chunk-size", config.DefaultChunkSize,
+		"number of bytes on which to split plaintext/ciphertext across message boundaries when streaming")
+
+	secretsOpt := cmd.StringsOpt("s secret", nil, "Pairs of PublicID and Passphrase to use as symmetric secrets in config")
+
+	arg := cmd.StringArg("CONFIG", "", fmt.Sprintf("Storage type to generate, one of: %s",
+		strings.Join(configTypes(), ", ")))
+
+	cmd.Spec = "[--json | --yaml] | (([--output=<output file>] |  [--init]) [--force]) CONFIG " +
+		"[--secret=<PublicID:Passphrase>...] [--chunk-size=<message chunk size in bytes>]"
+
+	cmd.Action = func() {
+		store, err := config.GetDefaultStorage(config.StorageType(*arg))
+		if err != nil {
+			fatalf("Error fetching default config for %v: %v", arg, err)
+		}
+		conf.Storage = store
+		conf.ChunkSize = *chunkSizeOpt
+		if len(*secretsOpt) > 0 {
+			conf.Secrets = &config.Secrets{
+				Symmetric: make([]config.SymmetricSecret, len(*secretsOpt)),
+			}
+			for i, ss := range *secretsOpt {
+				pair := strings.Split(ss, ":")
+				if len(pair) != 2 {
+					fatalf("got symmetric secret specification '%s' but must be specified as <PublicID:Passphrase>", ss)
+				}
+				salt, err := encryption.NewNonce(encryption.NonceSize)
+				if err != nil {
+					fatalf("failed to generate salt: %v", err)
+				}
+
+				data, err := encryption.DeriveSecretKey([]byte(pair[1]), salt)
+				if err != nil {
+					fatalf("could not derive secret key for config: %v", err)
+				}
+
+				conf.Secrets.Symmetric[i].PublicID = pair[0]
+				conf.Secrets.Symmetric[i].SecretKey = b64.StdEncoding.EncodeToString(data)
+			}
+		}
+	}
+
+	cmd.After = func() {
+		configString := conf.TOMLString()
+		if *jsonOpt {
+			configString = conf.JSONString()
+		} else if *yamlOpt {
+			configString = conf.YAMLString()
+		}
+		if *initOpt {
+			configFileName, err := xdgbasedir.GetConfigFileLocation(
+				config.DefaultHoardConfigFileName)
+			if err != nil {
+				fatalf("Error getting config file location: %s", err)
+			}
+			outputOpt = &configFileName
+		}
+		if *outputOpt != "" {
+			printf("Writing to config file '%s'", *outputOpt)
+			err := writeFile(*outputOpt, ([]byte)(configString), *overwriteOpt)
+			if err != nil {
+				fatalf("Error writing config file: %s", err)
+			}
+		} else {
+			fmt.Print(configString)
+		}
+	}
+}

cmd/hoard/main.go

@@ -5,10 +5,8 @@ import (
 	"io/ioutil"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 
-	"github.com/cep21/xdgbasedir"
 	"github.com/go-kit/kit/log"
 	cli "github.com/jawher/mow.cli"
 	"github.com/monax/hoard/v7/cmd"
@@ -31,13 +29,16 @@ func main() {
 	configFileOpt := hoardApp.StringOpt("c config", "", "Path to "+
 		"config file. If omitted default config is used. Use '-' to read config from STDIN.")
 
-	environmentOpt := hoardApp.BoolOpt("e env", false,
+	environmentOpt := hoardApp.BoolOpt("env-config", false,
 		fmt.Sprintf("Parse the contents of the environment variable %s as a complete JSON config",
 			config.DefaultJSONConfigEnvironmentVariable))
 
+	secretsFromEnv := hoardApp.BoolOpt("env-secrets", false,
+		fmt.Sprintf("Decode the environment variables pointed to by the symmetric public IDs."))
+
 	// This string spec is parsed by mow.cli and has actual semantic significance
 	// around optionality and ordering of options and arguments
-	hoardApp.Spec = "[--config=<path to config file> | --env] [--address=<address to listen on>] [--logging]"
+	hoardApp.Spec = "[--config=<path to config file> | --env-config] [--address=<address to listen on>] [--logging] [--env-secrets]"
 
 	cmd.AddVersionCommand(hoardApp)
 
@@ -47,8 +48,12 @@ func main() {
 			fatalf("Could not get Hoard config: %s", err)
 		}
 
-		var logger log.Logger
+		// I can't think of a good reason to allow this...
+		if conf.ChunkSize == 0 {
+			conf.ChunkSize = config.DefaultChunkSize
+		}
 
+		var logger log.Logger
 		if *loggingOpt {
 			logger, err = config.Logger(conf.Logging, os.Stderr)
 			if err != nil {
@@ -60,12 +65,18 @@ func main() {
 		if err != nil {
 			fatalf("Could not configure store from storage config: %s", err)
 		}
+
 		if *listenAddressOpt != "" {
 			conf.ListenAddress = *listenAddressOpt
 		}
-		symmetricProvider := config.NewSymmetricProvider(conf.Secrets)
+
+		symmetricProvider, err := config.NewSymmetricProvider(conf.Secrets, *secretsFromEnv)
+		if err != nil {
+			fatalf("Could not load symmetric keys: %s", err)
+		}
 		openPGPConf := config.NewOpenPGPSecret(conf.Secrets)
 		secretsManager := config.SecretsManager{Provider: symmetricProvider, OpenPGP: openPGPConf}
+
 		serv := server.New(conf.ListenAddress, store, secretsManager, conf.ChunkSize, logger)
 		// Catch interrupt etc
 		signalCh := make(chan os.Signal, 1)
@@ -88,84 +99,7 @@ func main() {
 
 	hoardApp.Command("config", "Initialise Hoard configuration by "+
 		"printing an example configuration file to STDOUT. Most config files emitted are "+
-		"examples demonstrating some features and need to be edited.",
-		func(configCmd *cli.Cmd) {
-			conf := config.DefaultHoardConfig
-
-			outputOpt := configCmd.StringOpt("o output", "",
-				"Instead of writing to STDOUT, write to the specified file")
-
-			overwriteOpt := configCmd.BoolOpt("f force", false,
-				"Overwrite config file if it exists.")
-
-			jsonOpt := configCmd.BoolOpt("j json", false,
-				"Print config to STDOUT as a single line of JSON (suitable for the --env config source)")
-
-			yamlOpt := configCmd.BoolOpt("y yaml", false,
-				"Print config to STDOUT as a YAML specification")
-
-			initOpt := configCmd.BoolOpt("i init", false, "Write file to "+
-				"XDG standard location")
-
-			chunkSizeOpt := configCmd.IntOpt("c chunk-size", config.DefaultChunkSize,
-				"number of bytes on which to split plaintext/ciphertext across message boundaries when streaming")
-
-			secretsOpt := configCmd.StringsOpt("s secret", nil, "Pairs of PublicID and Passphrase to use as symmetric secrets in config")
-
-			arg := configCmd.StringArg("CONFIG", "", fmt.Sprintf("Storage type to generate, one of: %s",
-				strings.Join(configTypes(), ", ")))
-
-			configCmd.Spec = "[--json | --yaml] | (([--output=<output file>] |  [--init]) [--force]) CONFIG " +
-				"[--secret=<PublicID:Passphrase>...] [--chunk-size=<message chunk size in bytes>]"
-
-			configCmd.Action = func() {
-				store, err := config.GetDefaultStorage(config.StorageType(*arg))
-				if err != nil {
-					fatalf("Error fetching default config for %v: %v", arg, err)
-				}
-				conf.Storage = store
-				conf.ChunkSize = *chunkSizeOpt
-				if len(*secretsOpt) > 0 {
-					conf.Secrets = &config.Secrets{
-						Symmetric: make([]config.SymmetricSecret, len(*secretsOpt)),
-					}
-					for i, ss := range *secretsOpt {
-						pair := strings.Split(ss, ":")
-						if len(pair) != 2 {
-							fatalf("got symmetric secret specification '%s' but must be specified as <PublicID:Passphrase>", ss)
-						}
-						conf.Secrets.Symmetric[i].PublicID = pair[0]
-						conf.Secrets.Symmetric[i].Passphrase = pair[1]
-					}
-				}
-			}
-
-			configCmd.After = func() {
-				configString := conf.TOMLString()
-				if *jsonOpt {
-					configString = conf.JSONString()
-				} else if *yamlOpt {
-					configString = conf.YAMLString()
-				}
-				if *initOpt {
-					configFileName, err := xdgbasedir.GetConfigFileLocation(
-						config.DefaultHoardConfigFileName)
-					if err != nil {
-						fatalf("Error getting config file location: %s", err)
-					}
-					outputOpt = &configFileName
-				}
-				if *outputOpt != "" {
-					printf("Writing to config file '%s'", *outputOpt)
-					err := writeFile(*outputOpt, ([]byte)(configString), *overwriteOpt)
-					if err != nil {
-						fatalf("Error writing config file: %s", err)
-					}
-				} else {
-					fmt.Print(configString)
-				}
-			}
-		})
+		"examples demonstrating some features and need to be edited.", Config)
 
 	hoardApp.Run(os.Args)
 }

config/secrets.go

@@ -1,8 +1,10 @@
 package config
 
 import (
+	b64 "encoding/base64"
 	"fmt"
 	"io/ioutil"
+	"os"
 )
 
 // Secrets lists the configured secrets,
@@ -15,8 +17,8 @@ type Secrets struct {
 
 type SymmetricSecret struct {
 	// An identifier for this secret that will be stored in the clear with the grant
-	PublicID   string
-	Passphrase string
+	PublicID  string
+	SecretKey string
 }
 
 type OpenPGPSecret struct {
@@ -46,13 +48,21 @@ func NoopSymmetricProvider(_ string) ([]byte, error) {
 }
 
 // ProviderFromConfig creates a secret reader from a set of symmetric secrets
-func NewSymmetricProvider(conf *Secrets) SymmetricProvider {
+func NewSymmetricProvider(conf *Secrets, fromEnv bool) (SymmetricProvider, error) {
 	if conf == nil || len(conf.Symmetric) == 0 {
-		return NoopSymmetricProvider
+		return NoopSymmetricProvider, nil
 	}
 	secs := make(map[string][]byte, len(conf.Symmetric))
 	for _, s := range conf.Symmetric {
-		secs[s.PublicID] = []byte(s.Passphrase)
+		if fromEnv {
+			// sometimes we don't want to specify these in the config
+			s.SecretKey = os.Getenv(s.PublicID)
+		}
+		secret, err := b64.StdEncoding.DecodeString(s.SecretKey)
+		if err != nil {
+			return nil, err
+		}
+		secs[s.PublicID] = secret
 	}
 	return func(id string) ([]byte, error) {
 		if id == "" {
@@ -62,7 +72,7 @@ func NewSymmetricProvider(conf *Secrets) SymmetricProvider {
 			return val, nil
 		}
 		return nil, fmt.Errorf("could not find symmetric secret with ID '%s'", id)
-	}
+	}, nil
 }
 
 // OpenPGPFromConfig reads a given PGP keyring

encryption/encryption.go

@@ -3,9 +3,12 @@ package encryption
 import (
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+
+	"golang.org/x/crypto/scrypt"
 )
 
 // There are issues with the proof of security with other nonce sizes so we only use 0 (for convergent) or 12
@@ -176,3 +179,16 @@ func additionalDataForSalt(salt []byte) []byte {
 	}
 	return jsonBytes
 }
+
+// We bump it a little from the 100ms for interactive logins rule: https://blog.filippo.io/the-scrypt-parameters/
+const scryptSecurityWorkExponent = 16
+
+func DeriveSecretKey(secret, salt []byte) ([]byte, error) {
+	return scrypt.Key(secret, salt, 1<<scryptSecurityWorkExponent, 8, 1, KeySize)
+}
+
+func NewNonce(n int) ([]byte, error) {
+	salt := make([]byte, n)
+	_, err := rand.Read(salt)
+	return salt, err
+}