[FAB-5353]: Qualify sys. failure vs validation error

Currently as stated in [FAB-5353], there is no clear separation during
transaction validation during block commmit, between invalid transaction
and some system failure which migh lead to inability to validate the
transaction. For example db is down or file system is unavailable. This
might lead to inconsistency of the state accross peers, therefore this
commit takes care to distinguish between real case of invalid
transaction versus system failure, later the error propagated down to
the committer and forces peer to stop with panic, so admin will be able
to take manual control and fix the problem therefore preventing peer
state to diverge.

Change-Id: I384e16d37e2f2b0fe144d504f566e0b744a5095c
Signed-off-by: Artem Barger <bartem@il.ibm.com>

core/committer/txvalidator/validator.go

@@ -86,6 +86,40 @@ type txValidator struct {
 	vscc    vsccValidator
 }
 
+// VSCCInfoLookupFailureError error to indicate inability
+// to obtain VSCC information from LCCC
+type VSCCInfoLookupFailureError struct {
+	reason string
+}
+
+// Error returns reasons which lead to the failure
+func (e VSCCInfoLookupFailureError) Error() string {
+	return e.reason
+}
+
+// VSCCEndorsementPolicyError error to mark transaction
+// failed endrosement policy check
+type VSCCEndorsementPolicyError struct {
+	reason string
+}
+
+// Error returns reasons which lead to the failure
+func (e VSCCEndorsementPolicyError) Error() string {
+	return e.reason
+}
+
+// VSCCExecutionFailureError error to indicate
+// failure during attempt of executing VSCC
+// endorsement policy check
+type VSCCExecutionFailureError struct {
+	reason string
+}
+
+// Error returns reasons which lead to the failure
+func (e VSCCExecutionFailureError) Error() string {
+	return e.reason
+}
+
 var logger *logging.Logger // package-level logger
 
 func init() {
@@ -170,8 +204,15 @@ func (v *txValidator) Validate(block *common.Block) error {
 					if err != nil {
 						txID := txID
 						logger.Errorf("VSCCValidateTx for transaction txId = %s returned error %s", txID, err)
-						txsfltr.SetFlag(tIdx, cde)
-						continue
+						switch err.(type) {
+						case *VSCCExecutionFailureError:
+							return err
+						case *VSCCInfoLookupFailureError:
+							return err
+						default:
+							txsfltr.SetFlag(tIdx, cde)
+							continue
+						}
 					}
 
 					invokeCC, upgradeCC, err := v.getTxCCInstance(payload)
@@ -370,7 +411,8 @@ func (v *vsccValidatorImpl) GetInfoForValidate(txid, chID, ccID string) (*sysccp
 		// obtain name of the VSCC and the policy from LSCC
 		cd, err := v.getCDataForCC(ccID)
 		if err != nil {
-			logger.Errorf("Unable to get chaincode data from ledger for txid %s, due to %s", txid, err)
+			msg := fmt.Sprintf("Unable to get chaincode data from ledger for txid %s, due to %s", txid, err)
+			logger.Errorf(msg)
 			return nil, nil, nil, err
 		}
 		cc.ChaincodeName = cd.Name
@@ -514,8 +556,12 @@ func (v *vsccValidatorImpl) VSCCValidateTx(payload *common.Payload, envBytes []b
 
 			// do VSCC validation
 			if err = v.VSCCValidateTxForCC(envBytes, chdr.TxId, chdr.ChannelId, vscc.ChaincodeName, vscc.ChaincodeVersion, policy); err != nil {
-				return fmt.Errorf("VSCCValidateTxForCC failed for cc %s, error %s", ccID, err),
-					peer.TxValidationCode_ENDORSEMENT_POLICY_FAILURE
+				switch err.(type) {
+				case VSCCEndorsementPolicyError:
+					return err, peer.TxValidationCode_ENDORSEMENT_POLICY_FAILURE
+				default:
+					return err, peer.TxValidationCode_INVALID_OTHER_REASON
+				}
 			}
 		}
 	} else {
@@ -541,8 +587,12 @@ func (v *vsccValidatorImpl) VSCCValidateTx(payload *common.Payload, envBytes []b
 		// user creates a new system chaincode which is invokable from the outside
 		// they have to modify VSCC to provide appropriate validation
 		if err = v.VSCCValidateTxForCC(envBytes, chdr.TxId, vscc.ChainID, vscc.ChaincodeName, vscc.ChaincodeVersion, policy); err != nil {
-			return fmt.Errorf("VSCCValidateTxForCC failed for cc %s, error %s", ccID, err),
-				peer.TxValidationCode_ENDORSEMENT_POLICY_FAILURE
+			switch err.(type) {
+			case VSCCEndorsementPolicyError:
+				return err, peer.TxValidationCode_ENDORSEMENT_POLICY_FAILURE
+			default:
+				return err, peer.TxValidationCode_INVALID_OTHER_REASON
+			}
 		}
 	}
 
@@ -552,8 +602,9 @@ func (v *vsccValidatorImpl) VSCCValidateTx(payload *common.Payload, envBytes []b
 func (v *vsccValidatorImpl) VSCCValidateTxForCC(envBytes []byte, txid, chid, vsccName, vsccVer string, policy []byte) error {
 	ctxt, err := v.ccprovider.GetContext(v.support.Ledger())
 	if err != nil {
-		logger.Errorf("Cannot obtain context for txid=%s, err %s", txid, err)
-		return err
+		msg := fmt.Sprintf("Cannot obtain context for txid=%s, err %s", txid, err)
+		logger.Errorf(msg)
+		return &VSCCExecutionFailureError{msg}
 	}
 	defer v.ccprovider.ReleaseContext()
 
@@ -571,12 +622,13 @@ func (v *vsccValidatorImpl) VSCCValidateTxForCC(envBytes []byte, txid, chid, vsc
 	logger.Debug("Invoking VSCC txid", txid, "chaindID", chid)
 	res, _, err := v.ccprovider.ExecuteChaincode(ctxt, cccid, args)
 	if err != nil {
-		logger.Errorf("Invoke VSCC failed for transaction txid=%s, error %s", txid, err)
-		return err
+		msg := fmt.Sprintf("Invoke VSCC failed for transaction txid=%s, error %s", txid, err)
+		logger.Errorf(msg)
+		return &VSCCExecutionFailureError{msg}
 	}
 	if res.Status != shim.OK {
 		logger.Errorf("VSCC check failed for transaction txid=%s, error %s", txid, res.Message)
-		return fmt.Errorf("%s", res.Message)
+		return &VSCCEndorsementPolicyError{fmt.Sprintf("%s", res.Message)}
 	}
 
 	return nil
@@ -596,7 +648,7 @@ func (v *vsccValidatorImpl) getCDataForCC(ccid string) (*ccprovider.ChaincodeDat
 
 	bytes, err := qe.GetState("lscc", ccid)
 	if err != nil {
-		return nil, fmt.Errorf("Could not retrieve state for chaincode %s, error %s", ccid, err)
+		return nil, &VSCCInfoLookupFailureError{fmt.Sprintf("Could not retrieve state for chaincode %s, error %s", ccid, err)}
 	}
 
 	if bytes == nil {

core/committer/txvalidator/validator_test.go

@@ -17,12 +17,14 @@ limitations under the License.
 package txvalidator
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"testing"
 
 	"github.com/hyperledger/fabric/common/cauthdsl"
 	ctxt "github.com/hyperledger/fabric/common/configtx/test"
+	ledger2 "github.com/hyperledger/fabric/common/ledger"
 	"github.com/hyperledger/fabric/common/ledger/testutil"
 	"github.com/hyperledger/fabric/common/mocks/scc"
 	"github.com/hyperledger/fabric/common/util"
@@ -41,6 +43,7 @@ import (
 	"github.com/hyperledger/fabric/protos/utils"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 )
 
 func signedByAnyMember(ids []string) []byte {
@@ -385,6 +388,149 @@ func TestInvokeNoBlock(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+// mockLedger structure used to test ledger
+// failure, therefore leveraging mocking
+// library as need to simulate ledger which not
+// able to get access to state db
+type mockLedger struct {
+	mock.Mock
+}
+
+// GetTransactionByID returns transaction by ud
+func (m *mockLedger) GetTransactionByID(txID string) (*peer.ProcessedTransaction, error) {
+	args := m.Called(txID)
+	return args.Get(0).(*peer.ProcessedTransaction), args.Error(1)
+}
+
+// GetBlockByHash returns block using its hash value
+func (m *mockLedger) GetBlockByHash(blockHash []byte) (*common.Block, error) {
+	args := m.Called(blockHash)
+	return args.Get(0).(*common.Block), nil
+}
+
+// GetBlockByTxID given transaction id return block transaction was committed with
+func (m *mockLedger) GetBlockByTxID(txID string) (*common.Block, error) {
+	args := m.Called(txID)
+	return args.Get(0).(*common.Block), nil
+}
+
+// GetTxValidationCodeByTxID returns validation code of give tx
+func (m *mockLedger) GetTxValidationCodeByTxID(txID string) (peer.TxValidationCode, error) {
+	args := m.Called(txID)
+	return args.Get(0).(peer.TxValidationCode), nil
+}
+
+// NewTxSimulator creates new transaction simulator
+func (m *mockLedger) NewTxSimulator() (ledger.TxSimulator, error) {
+	args := m.Called()
+	return args.Get(0).(ledger.TxSimulator), nil
+}
+
+// NewQueryExecutor creates query executor
+func (m *mockLedger) NewQueryExecutor() (ledger.QueryExecutor, error) {
+	args := m.Called()
+	return args.Get(0).(ledger.QueryExecutor), nil
+}
+
+// NewHistoryQueryExecutor history query executor
+func (m *mockLedger) NewHistoryQueryExecutor() (ledger.HistoryQueryExecutor, error) {
+	args := m.Called()
+	return args.Get(0).(ledger.HistoryQueryExecutor), nil
+}
+
+// Prune prune using policy
+func (m *mockLedger) Prune(policy ledger2.PrunePolicy) error {
+	return nil
+}
+
+func (m *mockLedger) GetBlockchainInfo() (*common.BlockchainInfo, error) {
+	args := m.Called()
+	return args.Get(0).(*common.BlockchainInfo), nil
+}
+
+func (m *mockLedger) GetBlockByNumber(blockNumber uint64) (*common.Block, error) {
+	args := m.Called(blockNumber)
+	return args.Get(0).(*common.Block), nil
+}
+
+func (m *mockLedger) GetBlocksIterator(startBlockNumber uint64) (ledger2.ResultsIterator, error) {
+	args := m.Called(startBlockNumber)
+	return args.Get(0).(ledger2.ResultsIterator), nil
+}
+
+func (m *mockLedger) Close() {
+
+}
+
+func (m *mockLedger) Commit(block *common.Block) error {
+	return nil
+}
+
+// mockQueryExecutor mock of the query executor,
+// needed to simulate inability to access state db, e.g.
+// the case where due to db failure it's not possible to
+// query for state, for example if we would like to query
+// the lccc for VSCC info and db is not avaible we expect
+// to stop validating block and fail commit procedure with
+// an error.
+type mockQueryExecutor struct {
+	mock.Mock
+}
+
+func (exec *mockQueryExecutor) GetState(namespace string, key string) ([]byte, error) {
+	args := exec.Called(namespace, key)
+	return args.Get(0).([]byte), args.Error(1)
+}
+
+func (exec *mockQueryExecutor) GetStateMultipleKeys(namespace string, keys []string) ([][]byte, error) {
+	args := exec.Called(namespace, keys)
+	return args.Get(0).([][]byte), args.Error(1)
+}
+
+func (exec *mockQueryExecutor) GetStateRangeScanIterator(namespace string, startKey string, endKey string) (ledger2.ResultsIterator, error) {
+	args := exec.Called(namespace, startKey, endKey)
+	return args.Get(0).(ledger2.ResultsIterator), args.Error(1)
+}
+
+func (exec *mockQueryExecutor) ExecuteQuery(namespace, query string) (ledger2.ResultsIterator, error) {
+	args := exec.Called(namespace)
+	return args.Get(0).(ledger2.ResultsIterator), args.Error(1)
+}
+
+func (exec *mockQueryExecutor) Done() {
+}
+
+// TestLedgerIsNoAvailable simulates and provides a test for following scenario,
+// which is based on FAB-535. Test checks the validation path which expects that
+// DB won't available while trying to lookup for VSCC from LCCC and therefore
+// transaction validation will have to fail. In such case the outcome should be
+// the error return from validate block method and proccessing of transactions
+// has to stop. There is suppose to be clear indication of the failure with error
+// returned from the function call.
+func TestLedgerIsNoAvailable(t *testing.T) {
+	theLedger := new(mockLedger)
+	validator := NewTxValidator(&mockSupport{l: theLedger})
+
+	ccID := "mycc"
+	tx := getEnv(ccID, createRWset(t, ccID), t)
+
+	theLedger.On("GetTransactionByID", mock.Anything).Return(&peer.ProcessedTransaction{}, errors.New("Cannot find the transaction"))
+
+	queryExecutor := new(mockQueryExecutor)
+	queryExecutor.On("GetState", mock.Anything, mock.Anything).Return([]byte{}, errors.New("Unable to connect to DB"))
+	theLedger.On("NewQueryExecutor", mock.Anything).Return(queryExecutor, nil)
+
+	b := &common.Block{Data: &common.BlockData{Data: [][]byte{utils.MarshalOrPanic(tx)}}}
+
+	err := validator.Validate(b)
+
+	assertion := assert.New(t)
+	// We suppose to get the error which indicates we cannot commit the block
+	assertion.Error(err)
+	// The error exptected to be of type VSCCInfoLookupFailureError
+	assertion.NotNil(err.(*VSCCInfoLookupFailureError))
+}
+
 var signer msp.SigningIdentity
 var signerSerialized []byte
 

gossip/state/state.go

@@ -428,7 +428,9 @@ func (s *GossipStateProviderImpl) deliverPayloads() {
 					continue
 				}
 				logger.Debug("New block with claimed sequence number ", payload.SeqNum, " transactions num ", len(rawBlock.Data.Data))
-				s.commitBlock(rawBlock)
+				if err := s.commitBlock(rawBlock); err != nil {
+					logger.Panicf("Cannot commit block to the ledger due to %s", err)
+				}
 			}
 		case <-s.stopCh:
 			s.stopCh <- struct{}{}