Add orderer config mechanism

This changeset adds an orderer.yaml config file which
specifies the defaults which were previously hardcoded.

It also correspondingly removes the hardcoded defaults and
utilizes the configuration in those places.

It resolves https://jira.hyperledger.org/browse/FAB-386

This configuration is being done using a variant of Viper's
Unmarshal.  In newer versions of Viper there is a new
function UnmarshalExact which throws errors when Unmarshaling
encounters config fields which are unexpected.

However, there are two outstanding bugs around this Viper
feature which affect us and do not make pulling the newer code
worthwhile at this point.

1. Unmarshaling does not appropriately support time durations

spf13/viper#105
spf13/viper#205

2. Unmarshaling does not correctly handle env overrides of
nested config parameters

spf13/viper#160

This changeset includes a stand-in implementation of UnmarshalExact
which does not suffer from these Viper bugs.  These workarounds
should be removed once fixes are pushed upstream.

Committing to reclaim this commit after a little Gerrit accident

Change-Id: I931b955b0d8fdaacb240a1b480eb695109774e38
Signed-off-by: Jason Yellick <jyellick@us.ibm.com>

bddtests/docker-compose-orderer-solo.yml

@@ -1,12 +1,14 @@
 orderer0:
   image: hyperledger/fabric-orderer
   environment:
-    - ORDERER_LISTEN_ADDRESS=0.0.0.0
-    - ORDERER_LISTEN_PORT=5005
-    #- ORDERER_WINDOW_SIZE_MAX=1000 # TODO (implement)
-    #- ORDERER_BATCH_TIMEOUT=10s # TODO (implement)
-    #- ORDERER_BATCH_SIZE=10 # TODO (implement)
-    #- ORDERER_BLOCK_HISTORY_SIZE=100 # TODO (implement)
+    - ORDERER_GENERAL_ORDERERTYPE=solo
+    - ORDERER_GENERAL_LEDGERTYPE=ram
+    - ORDERER_GENERAL_BATCHTIMEOUT=10s
+    - ORDERER_GENERAL_BATCHSIZE=10
+    - ORDERER_GENERAL_MAXWINDOWSIZE=1000
+    - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
+    - ORDERER_GENERAL_LISTENPORT=5005
+    - ORDERER_RAMLEDGER_HISTORY_SIZE=100
 
   working_dir: /opt/gopath/src/github.com/hyperledger/fabric/orderer
   command: orderer

orderer/config/config.go

@@ -0,0 +1,167 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/op/go-logging"
+	"github.com/spf13/viper"
+)
+
+var logger = logging.MustGetLogger("orderer/config")
+
+func init() {
+	logging.SetLevel(logging.DEBUG, "")
+}
+
+// Prefix is the default config prefix for the orderer
+const Prefix string = "ORDERER"
+
+// General contains config which should be common among all orderer types
+type General struct {
+	OrdererType   string
+	LedgerType    string
+	BatchTimeout  time.Duration
+	BatchSize     uint
+	QueueSize     uint
+	MaxWindowSize uint
+	ListenAddress string
+	ListenPort    uint16
+}
+
+// RAMLedger contains config for the RAM ledger
+type RAMLedger struct {
+	HistorySize uint
+}
+
+// FileLedger contains config for the File ledger
+type FileLedger struct {
+	Location string
+	Prefix   string
+}
+
+// TopLevel directly corresponds to the orderer config yaml
+// Note, for non 1-1 mappings, you may append
+// something like `mapstructure:"weirdFoRMat"` to
+// modify the default mapping, see the "Unmarshal"
+// section of https://github.com/spf13/viper for more info
+type TopLevel struct {
+	General    General
+	RAMLedger  RAMLedger
+	FileLedger FileLedger
+}
+
+var defaults = TopLevel{
+	General: General{
+		OrdererType:   "solo",
+		LedgerType:    "ram",
+		BatchTimeout:  10 * time.Second,
+		BatchSize:     10,
+		QueueSize:     1000,
+		MaxWindowSize: 1000,
+		ListenAddress: "127.0.0.1",
+		ListenPort:    5151,
+	},
+	RAMLedger: RAMLedger{
+		HistorySize: 10000,
+	},
+	FileLedger: FileLedger{
+		Location: "",
+		Prefix:   "hyperledger-fabric-rawledger",
+	},
+}
+
+func (c *TopLevel) completeInitialization() {
+	defer logger.Infof("Validated configuration to: %+v", c)
+
+	for {
+		switch {
+		case c.General.OrdererType == "":
+			logger.Infof("General.OrdererType unset, setting to %s", defaults.General.OrdererType)
+			c.General.OrdererType = defaults.General.OrdererType
+		case c.General.LedgerType == "":
+			logger.Infof("General.LedgerType unset, setting to %s", defaults.General.LedgerType)
+			c.General.LedgerType = defaults.General.LedgerType
+		case c.General.BatchTimeout == 0:
+			logger.Infof("General.BatchTimeout unset, setting to %s", defaults.General.BatchTimeout)
+			c.General.BatchTimeout = defaults.General.BatchTimeout
+		case c.General.BatchSize == 0:
+			logger.Infof("General.BatchSize unset, setting to %s", defaults.General.BatchSize)
+			c.General.BatchSize = defaults.General.BatchSize
+		case c.General.QueueSize == 0:
+			logger.Infof("General.QueueSize unset, setting to %s", defaults.General.QueueSize)
+			c.General.QueueSize = defaults.General.QueueSize
+		case c.General.MaxWindowSize == 0:
+			logger.Infof("General.MaxWindowSize unset, setting to %s", defaults.General.MaxWindowSize)
+			c.General.MaxWindowSize = defaults.General.MaxWindowSize
+		case c.General.ListenAddress == "":
+			logger.Infof("General.ListenAddress unset, setting to %s", defaults.General.ListenAddress)
+			c.General.ListenAddress = defaults.General.ListenAddress
+		case c.General.ListenPort == 0:
+			logger.Infof("General.ListenPort unset, setting to %s", defaults.General.ListenPort)
+			c.General.ListenPort = defaults.General.ListenPort
+		case c.FileLedger.Prefix == "":
+			logger.Infof("FileLedger.Prefix unset, setting to %s", defaults.FileLedger.Prefix)
+			c.FileLedger.Prefix = defaults.FileLedger.Prefix
+		default:
+			return
+		}
+	}
+}
+
+// Load parses the orderer.yaml file and environment, producing a struct suitable for config use
+func Load() *TopLevel {
+	config := viper.New()
+
+	// for environment variables
+	config.SetEnvPrefix(Prefix)
+	config.AutomaticEnv()
+	replacer := strings.NewReplacer(".", "_")
+	config.SetEnvKeyReplacer(replacer)
+
+	config.SetConfigName("orderer")
+	config.AddConfigPath("./")
+	config.AddConfigPath("../orderer/")
+	config.AddConfigPath("../../orderer/")
+	// Path to look for the config file in based on GOPATH
+	gopath := os.Getenv("GOPATH")
+	for _, p := range filepath.SplitList(gopath) {
+		ordererPath := filepath.Join(p, "src/github.com/hyperledger/fabric/orderer/")
+		config.AddConfigPath(ordererPath)
+	}
+
+	err := config.ReadInConfig()
+	if err != nil {
+		panic(fmt.Errorf("Error reading %s plugin config: %s", Prefix, err))
+	}
+
+	var uconf TopLevel
+
+	err = ExactWithDateUnmarshal(config, &uconf)
+	if err != nil {
+		panic(fmt.Errorf("Error unmarshaling into structure: %s", err))
+	}
+
+	uconf.completeInitialization()
+
+	return &uconf
+}

orderer/config/config_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/spf13/viper"
+)
+
+func TestGoodConfig(t *testing.T) {
+	config := Load()
+	if config == nil {
+		t.Fatalf("Could not load config")
+	}
+	t.Logf("%+v", config)
+}
+
+func TestBadConfig(t *testing.T) {
+	config := viper.New()
+	config.SetConfigName("orderer")
+	config.AddConfigPath("../")
+
+	err := config.ReadInConfig()
+	if err != nil {
+		t.Fatalf("Error reading %s plugin config: %s", Prefix, err)
+	}
+
+	var uconf struct{}
+
+	err = ExactWithDateUnmarshal(config, &uconf)
+	if err == nil {
+		t.Fatalf("Should have failed to unmarshal")
+	}
+}
+
+// TestEnvInnerVar verifies that with the Unmarshal function that
+// the environmental overrides still work on internal vars.  This was
+// a bug in the original viper implementation that is worked around in
+// the Load codepath for now
+func TestEnvInnerVar(t *testing.T) {
+	envVar := "ORDERER_GENERAL_LISTENPORT"
+	envVal := uint16(80)
+	os.Setenv(envVar, fmt.Sprintf("%d", envVal))
+	defer os.Unsetenv(envVar)
+	config := Load()
+
+	if config == nil {
+		t.Fatalf("Could not load config")
+	}
+
+	if config.General.ListenPort != envVal {
+		t.Fatalf("Environmental override of inner config did not work")
+	}
+}

orderer/config/config_util.go

@@ -0,0 +1,69 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+)
+
+func getKeysRecursively(base string, v *viper.Viper, nodeKeys map[string]interface{}) map[string]interface{} {
+	result := make(map[string]interface{})
+	for key := range nodeKeys {
+		fqKey := base + key
+		val := v.Get(fqKey)
+		if m, ok := val.(map[interface{}]interface{}); ok {
+			logger.Debugf("Found map value for %s", fqKey)
+			tmp := make(map[string]interface{})
+			for ik, iv := range m {
+				cik, ok := ik.(string)
+				if !ok {
+					panic("Non string key-entry")
+				}
+				tmp[cik] = iv
+			}
+			result[key] = getKeysRecursively(fqKey+".", v, tmp)
+		} else {
+			logger.Debugf("Found real value for %s setting to %T %v", fqKey, val, val)
+			result[key] = val
+		}
+	}
+	return result
+}
+
+// ExactWithDateUnmarshal is intended to unmarshal a config file into a structure
+// producing error when extraneous variables are introduced and supporting
+// the time.Duration type
+func ExactWithDateUnmarshal(v *viper.Viper, output interface{}) error {
+	baseKeys := v.AllSettings() // AllKeys doesn't actually return all keys, it only returns the base ones
+	leafKeys := getKeysRecursively("", v, baseKeys)
+
+	logger.Infof("%+v", leafKeys)
+	config := &mapstructure.DecoderConfig{
+		ErrorUnused:      true,
+		Metadata:         nil,
+		Result:           output,
+		WeaklyTypedInput: true,
+		DecodeHook:       mapstructure.StringToTimeDurationHookFunc(),
+	}
+
+	decoder, err := mapstructure.NewDecoder(config)
+	if err != nil {
+		return err
+	}
+	return decoder.Decode(leafKeys)
+}