Read config values from env on init command

api/ipfsproxy/config.go

@@ -82,7 +82,7 @@ type Config struct {
 type jsonConfig struct {
 	ListenMultiaddress string `json:"listen_multiaddress"`
 	NodeMultiaddress   string `json:"node_multiaddress"`
-	NodeHTTPS          string `json:"node_https,omitempty"`
+	NodeHTTPS          bool   `json:"node_https,omitempty"`
 
 	ReadTimeout       string `json:"read_timeout"`
 	ReadHeaderTimeout string `json:"read_header_timeout"`
@@ -154,6 +154,22 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return err
+	}
+
+	err = envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have sensible values,
 // at least in appearance.
 func (cfg *Config) Validate() error {
@@ -210,12 +226,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 		return fmt.Errorf("error setting config to default values: %s", err)
 	}
 
-	// override json config with env var
-	err = envconfig.Process("cluster_ipfsproxy", jcfg)
-	if err != nil {
-		return err
-	}
+	return cfg.applyJSONConfig(jcfg)
+}
 
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	proxyAddr, err := ma.NewMultiaddr(jcfg.ListenMultiaddress)
 	if err != nil {
 		return fmt.Errorf("error parsing proxy listen_multiaddress: %s", err)
@@ -251,14 +265,24 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *Config) ToJSON() (raw []byte, err error) {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return
+	}
+
+	raw, err = config.DefaultJSONMarshal(jcfg)
+	return
+}
+
+func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
 	// Multiaddress String() may panic
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("%s", r)
 		}
 	}()
 
-	jcfg := &jsonConfig{}
+	jcfg = &jsonConfig{}
 
 	// Set all configuration fields
 	jcfg.ListenMultiaddress = cfg.ListenAddr.String()
@@ -267,11 +291,11 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 	jcfg.ReadHeaderTimeout = cfg.ReadHeaderTimeout.String()
 	jcfg.WriteTimeout = cfg.WriteTimeout.String()
 	jcfg.IdleTimeout = cfg.IdleTimeout.String()
+	jcfg.NodeHTTPS = cfg.NodeHTTPS
 
 	jcfg.ExtractHeadersExtra = cfg.ExtractHeadersExtra
 	jcfg.ExtractHeadersPath = cfg.ExtractHeadersPath
 	jcfg.ExtractHeadersTTL = cfg.ExtractHeadersTTL.String()
 
-	raw, err = config.DefaultJSONMarshal(jcfg)
 	return
 }

api/ipfsproxy/config_test.go

@@ -2,7 +2,9 @@ package ipfsproxy
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
+	"time"
 )
 
 var cfgJSON = []byte(`
@@ -125,3 +127,14 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_IPFSPROXY_IDLETIMEOUT", "22s")
+	cfg := &Config{}
+	cfg.Default()
+	cfg.ApplyEnvVars()
+
+	if cfg.IdleTimeout != 22*time.Second {
+		t.Error("failed to override idle_timeout with env var")
+	}
+}

api/rest/config.go

@@ -181,6 +181,22 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return err
+	}
+
+	err = envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate makes sure that all fields in this Config have
 // working values, at least in appearance.
 func (cfg *Config) Validate() error {
@@ -230,13 +246,11 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 	cfg.Default()
 
-	// override json config with env var
-	err = envconfig.Process(envConfigKey, jcfg)
-	if err != nil {
-		return err
-	}
+	return cfg.applyJSONConfig(jcfg)
+}
 
-	err = cfg.loadHTTPOptions(jcfg)
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
+	err := cfg.loadHTTPOptions(jcfg)
 	if err != nil {
 		return err
 	}
@@ -361,14 +375,24 @@ func (cfg *Config) loadLibp2pOptions(jcfg *jsonConfig) error {
 // ToJSON produce a human-friendly JSON representation of the Config
 // object.
 func (cfg *Config) ToJSON() (raw []byte, err error) {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return
+	}
+
+	raw, err = config.DefaultJSONMarshal(jcfg)
+	return
+}
+
+func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
 	// Multiaddress String() may panic
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("%s", r)
 		}
 	}()
 
-	jcfg := &jsonConfig{
+	jcfg = &jsonConfig{
 		HTTPListenMultiaddress: cfg.HTTPListenAddr.String(),
 		SSLCertFile:            cfg.pathSSLCertFile,
 		SSLKeyFile:             cfg.pathSSLKeyFile,
@@ -400,7 +424,6 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 		jcfg.Libp2pListenMultiaddress = cfg.Libp2pListenAddr.String()
 	}
 
-	raw, err = config.DefaultJSONMarshal(jcfg)
 	return
 }
 

api/rest/config_test.go

@@ -110,14 +110,15 @@ func TestLoadJSON(t *testing.T) {
 	}
 }
 
-func TestLoadJSONEnvConfig(t *testing.T) {
+func TestApplyEnvVars(t *testing.T) {
 	username := "admin"
 	password := "thisaintmypassword"
 	user1 := "user1"
 	user1pass := "user1passwd"
 	os.Setenv("CLUSTER_RESTAPI_BASICAUTHCREDS", username+":"+password+","+user1+":"+user1pass)
 	cfg := &Config{}
-	err := cfg.LoadJSON(cfgJSON)
+	cfg.Default()
+	err := cfg.ApplyEnvVars()
 	if err != nil {
 		t.Fatal(err)
 	}

cluster_config.go

@@ -186,6 +186,22 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg, err := cfg.toConfigJSON()
+	if err != nil {
+		return err
+	}
+
+	err = envconfig.Process(cfg.ConfigKey(), jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyConfigJSON(jcfg)
+}
+
 // Validate will check that the values of this config
 // seem to be working ones.
 func (cfg *Config) Validate() error {
@@ -305,20 +321,10 @@ for more information.`)
 		return errors.New("cluster.Peers and cluster.Bootstrap keys have been deprecated")
 	}
 
-	// override json config with env var
-	err = envconfig.Process(cfg.ConfigKey(), jcfg)
-	if err != nil {
-		return err
-	}
-
-	parseDuration := func(txt string) time.Duration {
-		d, _ := time.ParseDuration(txt)
-		if txt != "" && d == 0 {
-			logger.Warningf("%s is not a valid duration. Default will be used", txt)
-		}
-		return d
-	}
+	return cfg.applyConfigJSON(jcfg)
+}
 
+func (cfg *Config) applyConfigJSON(jcfg *configJSON) error {
 	config.SetIfNotDefault(jcfg.PeerstoreFile, &cfg.PeerstoreFile)
 
 	id, err := peer.IDB58Decode(jcfg.ID)
@@ -365,15 +371,15 @@ for more information.`)
 	config.SetIfNotDefault(rplMin, &cfg.ReplicationFactorMin)
 	config.SetIfNotDefault(rplMax, &cfg.ReplicationFactorMax)
 
-	stateSyncInterval := parseDuration(jcfg.StateSyncInterval)
-	ipfsSyncInterval := parseDuration(jcfg.IPFSSyncInterval)
-	monitorPingInterval := parseDuration(jcfg.MonitorPingInterval)
-	peerWatchInterval := parseDuration(jcfg.PeerWatchInterval)
-
-	config.SetIfNotDefault(stateSyncInterval, &cfg.StateSyncInterval)
-	config.SetIfNotDefault(ipfsSyncInterval, &cfg.IPFSSyncInterval)
-	config.SetIfNotDefault(monitorPingInterval, &cfg.MonitorPingInterval)
-	config.SetIfNotDefault(peerWatchInterval, &cfg.PeerWatchInterval)
+	err = config.ParseDurations("cluster",
+		&config.DurationOpt{Duration: jcfg.StateSyncInterval, Dst: &cfg.StateSyncInterval, Name: "state_sync_interval"},
+		&config.DurationOpt{Duration: jcfg.IPFSSyncInterval, Dst: &cfg.IPFSSyncInterval, Name: "ipfs_sync_interval"},
+		&config.DurationOpt{Duration: jcfg.MonitorPingInterval, Dst: &cfg.MonitorPingInterval, Name: "monitor_ping_interval"},
+		&config.DurationOpt{Duration: jcfg.PeerWatchInterval, Dst: &cfg.PeerWatchInterval, Name: "peer_watch_interval"},
+	)
+	if err != nil {
+		return err
+	}
 
 	cfg.LeaveOnShutdown = jcfg.LeaveOnShutdown
 	cfg.DisableRepinning = jcfg.DisableRepinning
@@ -383,14 +389,24 @@ for more information.`)
 
 // ToJSON generates a human-friendly version of Config.
 func (cfg *Config) ToJSON() (raw []byte, err error) {
+	jcfg, err := cfg.toConfigJSON()
+	if err != nil {
+		return
+	}
+
+	raw, err = json.MarshalIndent(jcfg, "", "    ")
+	return
+}
+
+func (cfg *Config) toConfigJSON() (jcfg *configJSON, err error) {
 	// Multiaddress String() may panic
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("%s", r)
 		}
 	}()
 
-	jcfg := &configJSON{}
+	jcfg = &configJSON{}
 
 	// Private Key
 	pkeyBytes, err := cfg.PrivateKey.Bytes()
@@ -415,7 +431,6 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 	jcfg.DisableRepinning = cfg.DisableRepinning
 	jcfg.PeerstoreFile = cfg.PeerstoreFile
 
-	raw, err = json.MarshalIndent(jcfg, "", "    ")
 	return
 }
 

cluster_config_test.go

@@ -187,15 +187,6 @@ func TestLoadJSON(t *testing.T) {
 			t.Error("expected default replication factors")
 		}
 	})
-
-	t.Run("env var override", func(t *testing.T) {
-		os.Setenv("CLUSTER_PEERNAME", "envsetpeername")
-		cfg := &Config{}
-		cfg.LoadJSON(ccfgTestJSON)
-		if cfg.Peername != "envsetpeername" {
-			t.Fatal("failed to override peername with env var")
-		}
-	})
 }
 
 func TestToJSON(t *testing.T) {
@@ -220,6 +211,16 @@ func TestDefault(t *testing.T) {
 	}
 }
 
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_PEERNAME", "envsetpeername")
+	cfg := &Config{}
+	cfg.Default()
+	cfg.ApplyEnvVars()
+	if cfg.Peername != "envsetpeername" {
+		t.Fatal("failed to override peername with env var")
+	}
+}
+
 func TestValidate(t *testing.T) {
 	cfg := &Config{}
 	cfg.Default()

cmd/ipfs-cluster-service/daemon.go

@@ -69,7 +69,7 @@ func daemon(c *cli.Context) error {
 	// always wait for configuration to be saved
 	defer cfgMgr.Shutdown()
 
-	err = cfgMgr.LoadJSONFromFile(configPath)
+	err = cfgMgr.LoadJSONFileAndEnv(configPath)
 	checkErr("loading configuration", err)
 
 	if c.Bool("stats") {

cmd/ipfs-cluster-service/main.go

@@ -240,6 +240,9 @@ configuration.
 				err := cfgMgr.Default()
 				checkErr("generating default configuration", err)
 
+				err = cfgMgr.ApplyEnvVars()
+				checkErr("applying environment variables to configuration", err)
+
 				// Set user secret
 				if userSecretDefined {
 					cfgs.clusterCfg.Secret = userSecret
@@ -439,7 +442,7 @@ the mth data folder (m currently defaults to 5)
 						}
 
 						cfgMgr, cfgs := makeConfigs()
-						err = cfgMgr.LoadJSONFromFile(configPath)
+						err = cfgMgr.LoadJSONFileAndEnv(configPath)
 						checkErr("reading configuration", err)
 
 						err = cleanupState(cfgs.consensusCfg)
@@ -501,18 +504,14 @@ func setupDebug() {
 }
 
 func userProvidedSecret(enterSecret bool) ([]byte, bool) {
-	var secret string
 	if enterSecret {
-		secret = promptUser("Enter cluster secret (32-byte hex string): ")
-	} else if envSecret, envSecretDefined := os.LookupEnv("CLUSTER_SECRET"); envSecretDefined {
-		secret = envSecret
-	} else {
-		return nil, false
+		secret := promptUser("Enter cluster secret (32-byte hex string): ")
+		decodedSecret, err := ipfscluster.DecodeClusterSecret(secret)
+		checkErr("parsing user-provided secret", err)
+		return decodedSecret, true
 	}
 
-	decodedSecret, err := ipfscluster.DecodeClusterSecret(secret)
-	checkErr("parsing user-provided secret", err)
-	return decodedSecret, true
+	return nil, false
 }
 
 func promptUser(msg string) string {