Initial prototype of CouchDB state DB in ledgernext

https://jira.hyperledger.org/browse/FAB-728

This commit adds a transaction manager (state database) based on CouchDB,
and a sample client to demonstrate/test it.

KVLedger will continue to use file based block storage
for the blockchain, but will use CouchDB as the state database for
simulation and commit.

This experimental feature is enabled via a feature toggle switch in
the code (useCouchDB).

CouchDB must be already installed separately.
There is a script to start CouchDB in dev env and download a docker
image of CouchDB if not already downloaded.  Run this command anywhere
inside the dev env /fabric:
"couchdb start"

To switch ledger to use CouchDB,
update kv_ledger_config.go variable useCouchDB to true.
In kv_ledger.go NewKVLedger(), you will also need to set the CouchDB connection
host, port, db name, id, password if using a non-local secured CouchDB.

This initial commit is only a stand alone ledger prototype and not meant
for end-to-end chaincode processing. That will come in a subsequent commit.

To run the sample:
In CouchDB http://localhost:5984/_utils create a db named marbles_app.
You can do this from your host which has port 5984 mapped to guest 5984
(assuming you have done vagrant up after
https://gerrit.hyperledger.org/r/#/c/1935/ went in Oct 25th, as that
changeset open up vagrant port 5984).
Then run the sample as follows:
/core/ledgernext/kvledger/marble_example/main$ go run marble_example.go

After running the sample, you can view the marble1 document in CouchDB.
Be sure to delete it in CouchDB if you'd like to re-run the sample.

Change-Id: Iea4f6ad498dc0e637f0254b6f749060e0298622c
Signed-off-by: denyeart <enyeart@us.ibm.com>

core/ledger/kvledger/example/committer.go

@@ -36,9 +36,11 @@ func (c *Committer) CommitBlock(rawBlock *protos.Block2) (*protos.Block2, []*pro
 	var validBlock *protos.Block2
 	var invalidTxs []*protos.InvalidTransaction
 	var err error
+	logger.Debugf("Committer validating the block...")
 	if validBlock, invalidTxs, err = c.ledger.RemoveInvalidTransactionsAndPrepare(rawBlock); err != nil {
 		return nil, nil, err
 	}
+	logger.Debugf("Committer committing the block...")
 	if err = c.ledger.Commit(); err != nil {
 		return nil, nil, err
 	}

core/ledger/kvledger/example/consenter.go

@@ -32,6 +32,7 @@ func ConstructConsenter() *Consenter {
 
 // ConstructBlock constructs a block from a list of transactions
 func (c *Consenter) ConstructBlock(transactions ...*protos.Transaction2) *protos.Block2 {
+	logger.Debugf("Construct a block based on the transactions")
 	block := &protos.Block2{}
 	for _, tx := range transactions {
 		txBytes, _ := proto.Marshal(tx)

core/ledger/kvledger/example/marble_app.go

@@ -0,0 +1,178 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package example
+
+import (
+	"encoding/json"
+	"errors"
+	"strconv"
+	"strings"
+
+	ledger "github.com/hyperledger/fabric/core/ledger"
+	"github.com/hyperledger/fabric/protos"
+	logging "github.com/op/go-logging"
+)
+
+var logger = logging.MustGetLogger("example")
+
+// App - a sample fund transfer app
+type MarbleApp struct {
+	name   string
+	ledger ledger.ValidatedLedger
+}
+
+// ConstructAppInstance constructs an instance of an app
+func ConstructMarbleAppInstance(ledger ledger.ValidatedLedger) *MarbleApp {
+	return &MarbleApp{"marbles_app", ledger}
+}
+
+var marbleIndexStr = "_marbleindex" //name for the key/value that will store a list of all known marbles
+var openTradesStr = "_opentrades"   //name for the key/value that will store all open trades
+
+type Marble struct {
+	Name  string `json:"asset_name"` //the fieldtags are needed to keep case from bouncing around
+	Color string `json:"color"`
+	Size  int    `json:"size"`
+	User  string `json:"owner"`
+	Rev   string `json:"_rev"`
+	Txid  string `json:"txid"`
+}
+
+// CreateMarble simulates init transaction
+func (marbleApp *MarbleApp) CreateMarble(args []string) (*protos.Transaction2, error) {
+	//   0       1       2     3
+	// "asdf", "blue", "35", "bob"
+	logger.Debugf("===COUCHDB=== Entering ----------CreateMarble()----------")
+	marbleName := args[0]
+	marbleJsonBytes, err := init_marble(args)
+	if err != nil {
+		return nil, err
+	}
+
+	var txSimulator ledger.TxSimulator
+	if txSimulator, err = marbleApp.ledger.NewTxSimulator(); err != nil {
+		return nil, err
+	}
+	defer txSimulator.Done()
+
+	txSimulator.SetState(marbleApp.name, marbleName, marbleJsonBytes)
+
+	var txSimulationResults []byte
+	if txSimulationResults, err = txSimulator.GetTxSimulationResults(); err != nil {
+		return nil, err
+	}
+	logger.Debugf("===COUCHDB=== CreateMarble() simulation done, packaging into a transaction...")
+	tx := constructTransaction(txSimulationResults)
+	logger.Debugf("===COUCHDB=== Exiting CreateMarble()")
+	return tx, nil
+}
+
+// ============================================================================================================================
+// Init Marble - create a new marble, store into chaincode state
+// ============================================================================================================================
+func init_marble(args []string) ([]byte, error) {
+	var err error
+
+	//   0       1       2     3
+	// "asdf", "blue", "35", "bob"
+	if len(args) != 4 {
+		return nil, errors.New("Incorrect number of arguments. Expecting 4")
+	}
+
+	logger.Debugf("===COUCHDB=== Entering init marble")
+	if len(args[0]) <= 0 {
+		return nil, errors.New("1st argument must be a non-empty string")
+	}
+	if len(args[1]) <= 0 {
+		return nil, errors.New("2nd argument must be a non-empty string")
+	}
+	if len(args[2]) <= 0 {
+		return nil, errors.New("3rd argument must be a non-empty string")
+	}
+	if len(args[3]) <= 0 {
+		return nil, errors.New("4th argument must be a non-empty string")
+	}
+
+	size, err := strconv.Atoi(args[2])
+	if err != nil {
+		return nil, errors.New("3rd argument must be a numeric string")
+	}
+
+	color := strings.ToLower(args[1])
+	user := strings.ToLower(args[3])
+
+	tx := "tx000000000000001" // COUCHDB hardcode a txid for now for demo purpose
+	marbleJson := `{"txid": "` + tx + `",  "asset_name": "` + args[0] + `", "color": "` + color + `", "size": ` + strconv.Itoa(size) + `, "owner": "` + user + `"}`
+	marbleBytes := []byte(marbleJson)
+
+	logger.Debugf("===COUCHDB=== Exiting init marble")
+	return marbleBytes, nil
+}
+
+// TransferMarble simulates transfer transaction
+func (marbleApp *MarbleApp) TransferMarble(args []string) (*protos.Transaction2, error) {
+
+	//   0       1
+	// "name", "bob"
+	if len(args) < 2 {
+		return nil, errors.New("Incorrect number of arguments. Expecting 2")
+	}
+	marbleName := args[0]
+	marbleNewOwner := args[1]
+
+	logger.Debugf("===COUCHDB=== Entering ----------TransferMarble----------")
+	var txSimulator ledger.TxSimulator
+	var err error
+	if txSimulator, err = marbleApp.ledger.NewTxSimulator(); err != nil {
+		return nil, err
+	}
+	defer txSimulator.Done()
+
+	marbleBytes, err := txSimulator.GetState(marbleApp.name, marbleName)
+	logger.Debugf("===COUCHDB=== marbleBytes is: %v", marbleBytes)
+	if marbleBytes != nil {
+		jsonString := string(marbleBytes[:])
+		logger.Debugf("===COUCHDB=== TransferMarble() Retrieved jsonString: \n   %s", jsonString)
+	}
+
+	theMarble := Marble{}
+	json.Unmarshal(marbleBytes, &theMarble) //Unmarshal JSON bytes into a Marble struct
+
+	logger.Debugf("===COUCHDB===  theMarble after unmarshal: %v", theMarble)
+
+	logger.Debugf("===COUCHDB===  Setting the owner to: %s", marbleNewOwner)
+	theMarble.User = marbleNewOwner      //change the user
+	theMarble.Txid = "tx000000000000002" // COUCHDB hardcode a txid for now for demo purpose
+
+	updatedMarbleBytes, _ := json.Marshal(theMarble)
+	if updatedMarbleBytes != nil {
+		updatedJsonString := string(updatedMarbleBytes[:])
+		logger.Debugf("===COUCHDB=== updatedJsonString:\n   %s", updatedJsonString)
+	}
+	err = txSimulator.SetState(marbleApp.name, marbleName, updatedMarbleBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	var txSimulationResults []byte
+	if txSimulationResults, err = txSimulator.GetTxSimulationResults(); err != nil {
+		return nil, err
+	}
+	logger.Debugf("===COUCHDB=== TransferMarble() simulation done, packaging into a transaction...")
+	tx := constructTransaction(txSimulationResults)
+	return tx, nil
+}

core/ledger/kvledger/kv_ledger.go

@@ -24,11 +24,16 @@ import (
 	"github.com/hyperledger/fabric/core/ledger"
 	"github.com/hyperledger/fabric/core/ledger/blkstorage"
 	"github.com/hyperledger/fabric/core/ledger/blkstorage/fsblkstorage"
+	"github.com/hyperledger/fabric/core/ledger/kvledger/kvledgerconfig"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt"
+	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/couchdbtxmgmt"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/lockbasedtxmgmt"
 	"github.com/hyperledger/fabric/protos"
+	logging "github.com/op/go-logging"
 )
 
+var logger = logging.MustGetLogger("kvledger")
+
 // Conf captures `KVLedger` configurations
 type Conf struct {
 	blockStorageDir  string
@@ -58,8 +63,23 @@ type KVLedger struct {
 // NewKVLedger constructs new `KVLedger`
 func NewKVLedger(conf *Conf) (*KVLedger, error) {
 	blockStore := fsblkstorage.NewFsBlockStore(fsblkstorage.NewConf(conf.blockStorageDir, conf.maxBlockfileSize))
+
+	if kvledgerconfig.IsCouchDBEnabled() == true {
+		//By default we can talk to CouchDB with empty id and pw (""), or you can add your own id and password to talk to a secured CouchDB
+		logger.Debugf("===COUCHDB=== NewKVLedger() Using CouchDB instead of RocksDB...hardcoding and passing connection config for now")
+		txmgmt := couchdbtxmgmt.NewCouchDBTxMgr(&couchdbtxmgmt.Conf{DBPath: conf.txMgrDBPath},
+			"127.0.0.1",   //couchDB host
+			5984,          //couchDB port
+			"marbles_app", //couchDB db name
+			"",            //enter couchDB id here
+			"")            //enter couchDB pw here
+		return &KVLedger{blockStore, txmgmt, nil}, nil
+	}
+
+	// Fall back to using RocksDB lockbased transaction manager
 	txmgmt := lockbasedtxmgmt.NewLockBasedTxMgr(&lockbasedtxmgmt.Conf{DBPath: conf.txMgrDBPath})
 	return &KVLedger{blockStore, txmgmt, nil}, nil
+
 }
 
 // GetTransactionByID retrieves a transaction by id

core/ledger/kvledger/kvledgerconfig/kv_ledger_config.go

@@ -0,0 +1,25 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kvledgerconfig
+
+// Change this feature toggle to true to use CouchDB for state database
+var useCouchDB = false
+
+//IsCouchDBEnabled exposes the useCouchDB variable
+func IsCouchDBEnabled() bool {
+	return useCouchDB
+}

core/ledger/kvledger/marble_example/main/marble_example.go

@@ -0,0 +1,107 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/hyperledger/fabric/core/ledger"
+	"github.com/hyperledger/fabric/core/ledger/kvledger"
+	"github.com/hyperledger/fabric/core/ledger/kvledger/example"
+	"github.com/hyperledger/fabric/protos"
+	logging "github.com/op/go-logging"
+)
+
+var logger = logging.MustGetLogger("main")
+
+const (
+	ledgerPath = "/tmp/test/ledgernext/kvledger/example"
+)
+
+var finalLedger ledger.ValidatedLedger
+var marbleApp *example.MarbleApp
+var committer *example.Committer
+var consenter *example.Consenter
+
+func init() {
+
+	// Initialization will get a handle to the ledger at the specified path
+	// Note, if subledgers are supported in the future,
+	// the various ledgers could be created/managed at this level
+	logger.Debugf("===COUCHDB=== Marble Example main init()")
+
+	os.RemoveAll(ledgerPath)
+	ledgerConf := kvledger.NewConf(ledgerPath, 0)
+	var err error
+	finalLedger, err = kvledger.NewKVLedger(ledgerConf)
+	if err != nil {
+		panic(fmt.Errorf("Error in NewKVLedger(): %s", err))
+	}
+	marbleApp = example.ConstructMarbleAppInstance(finalLedger)
+	committer = example.ConstructCommitter(finalLedger)
+	consenter = example.ConstructConsenter()
+}
+
+func main() {
+	defer finalLedger.Close()
+
+	// Each of the functions here will emulate endorser, orderer,
+	// and committer by calling ledger APIs to similate the proposal,
+	// get simulation results, create a transaction, add it to a block,
+	// and then commit the block.
+
+	initApp()
+	transferMarble()
+
+}
+
+func initApp() {
+	logger.Debugf("===COUCHDB=== Marble Example initApp() to create a marble")
+	marble := []string{"marble1", "blue", "35", "tom"}
+	tx, err := marbleApp.CreateMarble(marble)
+	handleError(err, true)
+	rawBlock := consenter.ConstructBlock(tx)
+	finalBlock, invalidTx, err := committer.CommitBlock(rawBlock)
+	handleError(err, true)
+	printBlocksInfo(rawBlock, finalBlock, invalidTx)
+}
+
+func transferMarble() {
+	logger.Debugf("===COUCHDB=== Marble Example transferMarble()")
+	tx1, err := marbleApp.TransferMarble([]string{"marble1", "jerry"})
+	handleError(err, true)
+	rawBlock := consenter.ConstructBlock(tx1)
+	finalBlock, invalidTx, err := committer.CommitBlock(rawBlock)
+	handleError(err, true)
+	printBlocksInfo(rawBlock, finalBlock, invalidTx)
+}
+
+func printBlocksInfo(rawBlock *protos.Block2, finalBlock *protos.Block2, invalidTxs []*protos.InvalidTransaction) {
+	fmt.Printf("Num txs in rawBlock = [%d], num txs in final block = [%d], num invalidTxs = [%d]\n",
+		len(rawBlock.Transactions), len(finalBlock.Transactions), len(invalidTxs))
+}
+
+func handleError(err error, quit bool) {
+	if err != nil {
+		if quit {
+			panic(fmt.Errorf("Error: %s\n", err))
+		} else {
+			fmt.Printf("Error: %s\n", err)
+		}
+	}
+}