Add rebuild test with real ledger data

- Add an IT test to generate ledger data - Test rebuild command with the generated ledger data and verify statedb, blocks, pvtdata, historydb, configHistorydb Change-Id: I9f469d95f7120b402545b027cc5fed9b188afb95 Signed-off-by: Wenjian Qiao <wenjianq@gmail.com>
hyperledger · Nov 20, 2019 · 3d8825e · 3d8825e
1 parent bbdfa7f
commit 3d8825e
Show file tree

Hide file tree

Showing 6 changed files with 354 additions and 1 deletion.
diff --git a/core/ledger/kvledger/tests/env.go b/core/ledger/kvledger/tests/env.go
@@ -18,6 +18,7 @@ import (
 	"github.com/hyperledger/fabric/common/ledger/blkstorage/fsblkstorage"
 	"github.com/hyperledger/fabric/common/ledger/util"
 	"github.com/hyperledger/fabric/common/metrics/disabled"
+	"github.com/hyperledger/fabric/core/chaincode/lifecycle"
 	"github.com/hyperledger/fabric/core/common/privdata"
 	"github.com/hyperledger/fabric/core/container/externalbuilder"
 	"github.com/hyperledger/fabric/core/ledger"
@@ -272,3 +273,59 @@ func createSelfSignedData(cryptoProvider bccsp.BCCSP) protoutil.SignedData {
 		Identity:  peerIdentity,
 	}
 }
+
+// deployedCCInfoProviderWrapper is a wrapper type that overrides ChaincodeImplicitCollections
+type deployedCCInfoProviderWrapper struct {
+	*lifecycle.ValidatorCommitter
+	orgMSPIDs []string
+}
+
+// AllCollectionsConfigPkg overrides the same method in lifecycle.AllCollectionsConfigPkg.
+// It is basically a copy of lifecycle.AllCollectionsConfigPkg and invokes ImplicitCollections in the wrapper.
+// This method is called when the unit test code gets private data code path.
+func (dc *deployedCCInfoProviderWrapper) AllCollectionsConfigPkg(channelName, chaincodeName string, qe ledger.SimpleQueryExecutor) (*common.CollectionConfigPackage, error) {
+	chaincodeInfo, err := dc.ChaincodeInfo(channelName, chaincodeName, qe)
+	if err != nil {
+		return nil, err
+	}
+	explicitCollectionConfigPkg := chaincodeInfo.ExplicitCollectionConfigPkg
+
+	implicitCollections, _ := dc.ImplicitCollections(channelName, "", nil)
+
+	var combinedColls []*common.CollectionConfig
+	if explicitCollectionConfigPkg != nil {
+		combinedColls = append(combinedColls, explicitCollectionConfigPkg.Config...)
+	}
+	for _, implicitColl := range implicitCollections {
+		c := &common.CollectionConfig{}
+		c.Payload = &common.CollectionConfig_StaticCollectionConfig{StaticCollectionConfig: implicitColl}
+		combinedColls = append(combinedColls, c)
+	}
+	return &common.CollectionConfigPackage{
+		Config: combinedColls,
+	}, nil
+}
+
+// ImplicitCollections overrides the same method in lifecycle.ValidatorCommitter.
+// It constructs static collection config using known mspids from the sample ledger.
+// This method is called when the unit test code gets collection configuration.
+func (dc *deployedCCInfoProviderWrapper) ImplicitCollections(channelName, chaincodeName string, qe ledger.SimpleQueryExecutor) ([]*common.StaticCollectionConfig, error) {
+	collConfigs := make([]*common.StaticCollectionConfig, 0, len(dc.orgMSPIDs))
+	for _, mspID := range dc.orgMSPIDs {
+		collConfigs = append(collConfigs, lifecycle.GenerateImplicitCollectionForOrg(mspID))
+	}
+	return collConfigs, nil
+}
+
+func createDeployedCCInfoProvider(orgMSPIDs []string) ledger.DeployedChaincodeInfoProvider {
+	deployedCCInfoProvider := &lifecycle.ValidatorCommitter{
+		Resources: &lifecycle.Resources{
+			Serializer: &lifecycle.Serializer{},
+		},
+		LegacyDeployedCCInfoProvider: &lscc.DeployedCCInfoProvider{},
+	}
+	return &deployedCCInfoProviderWrapper{
+		ValidatorCommitter: deployedCCInfoProvider,
+		orgMSPIDs:          orgMSPIDs,
+	}
+}
diff --git a/core/ledger/kvledger/tests/testdata/v20/sample_ledgers/ledgersData.zip b/core/ledger/kvledger/tests/testdata/v20/sample_ledgers/ledgersData.zip
diff --git a/core/ledger/kvledger/tests/util.go b/core/ledger/kvledger/tests/util.go
@@ -13,6 +13,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric-protos-go/common"
 	"github.com/hyperledger/fabric-protos-go/ledger/rwset"
+	"github.com/hyperledger/fabric-protos-go/msp"
 	protopeer "github.com/hyperledger/fabric-protos-go/peer"
 	"github.com/hyperledger/fabric/common/cauthdsl"
 	configtxtest "github.com/hyperledger/fabric/common/configtx/test"
@@ -88,7 +89,17 @@ func convertFromMemberOrgsPolicy(policy *common.CollectionPolicyConfig) []string
 	ids := policy.GetSignaturePolicy().Identities
 	var members []string
 	for _, id := range ids {
-		members = append(members, string(id.Principal))
+		role := &msp.MSPRole{}
+		err := proto.Unmarshal(id.Principal, role)
+		if err == nil {
+			// This is for sample ledger generated by fabric (e.g., integration test),
+			// where id.Principal was properly marshalled during sample ledger generation.
+			members = append(members, role.MspIdentifier)
+		} else {
+			// This is for sample ledger generated by sampleDataHelper.populateLedger,
+			// where id.Principal was a []byte cast from a string (not a marshalled msp.MSPRole)
+			members = append(members, string(id.Principal))
+		}
 	}
 	return members
 }

diff --git a/core/ledger/kvledger/tests/v20_test.go b/core/ledger/kvledger/tests/v20_test.go
@@ -0,0 +1,131 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package tests
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hyperledger/fabric/common/ledger/testutil"
+	"github.com/hyperledger/fabric/core/chaincode/lifecycle"
+	"github.com/hyperledger/fabric/core/ledger/kvledger"
+	"github.com/stretchr/testify/require"
+)
+
+// TestV20SampleLedger tests rebuild function with sample v2.0 ledger data generated by integration/ledger/ledger_generate_test.go
+func TestV20SampleLedger(t *testing.T) {
+	env := newEnv(t)
+	defer env.cleanup()
+
+	dataHelper := &v20SampleDataHelper{sampleDataVersion: "v2.0", t: t}
+	env.initializer.DeployedChaincodeInfoProvider = createDeployedCCInfoProvider(dataHelper.mspIDsInChannelConfig())
+	ledgerFSRoot := env.initializer.Config.RootFSPath
+	require.NoError(t, testutil.Unzip("testdata/v20/sample_ledgers/ledgersData.zip", ledgerFSRoot, false))
+
+	env.initLedgerMgmt()
+
+	h1 := env.newTestHelperOpenLgr("testchannel", t)
+	dataHelper.verify(h1)
+
+	// rebuild and verify again
+	env.closeLedgerMgmt()
+	kvledger.RebuildDBs(env.getLedgerRootPath())
+	env.initLedgerMgmt()
+	h1 = env.newTestHelperOpenLgr("testchannel", t)
+	dataHelper.verify(h1)
+}
+
+// The generated ledger has the following blocks:
+// block 0: genesis
+// block 1 to 4: network setup
+// block 5 to 8: marblesp chaincode instantiation
+// block 9 to 12: marbles chancode instantiation
+// block 13: marblesp chaincode invocation (new marble1)
+// block 14 to 17: upgrade marblesp chaincode with a new collection config
+// block 18 to 19: marbles chaincode invocation (new marble100 and transfer)
+type v20SampleDataHelper struct {
+	sampleDataVersion string
+	t                 *testing.T
+}
+
+func (d *v20SampleDataHelper) verify(h *testhelper) {
+	d.verifyState(h)
+	d.verifyBlockAndPvtdata(h)
+	d.verifyConfigHistory(h)
+	d.verifyHistory(h)
+}
+
+func (d *v20SampleDataHelper) verifyState(h *testhelper) {
+	h.verifyPubState("marbles", "marble100", d.marbleValue("marble100", "blue", "jerry", 35))
+	h.verifyPvtState("marblesp", "collectionMarbles", "marble1", d.marbleValue("marble1", "blue", "tom", 35))
+	h.verifyPvtState("marblesp", "collectionMarblePrivateDetails", "marble1", d.marbleDetail("marble1", 99))
+}
+
+func (d *v20SampleDataHelper) verifyHistory(h *testhelper) {
+	expectedHistoryValue1 := []string{
+		d.marbleValue("marble100", "blue", "jerry", 35),
+		d.marbleValue("marble100", "blue", "tom", 35),
+	}
+	h.verifyHistory("marbles", "marble100", expectedHistoryValue1)
+}
+
+func (d *v20SampleDataHelper) verifyConfigHistory(h *testhelper) {
+	// below block 10 should match integration/ledger/testdata/collection_configs/collections_config1.json
+	h.verifyMostRecentCollectionConfigBelow(10, "marblesp",
+		&expectedCollConfInfo{8, d.marbleCollConf1("marbelsp")})
+
+	// below block 18 should match integration/ledger/testdata/collection_configs/collections_config2.json
+	h.verifyMostRecentCollectionConfigBelow(18, "marblesp",
+		&expectedCollConfInfo{17, d.marbleCollConf2("marbelsp")})
+}
+
+func (d *v20SampleDataHelper) verifyBlockAndPvtdata(h *testhelper) {
+	h.verifyBlockAndPvtData(8, nil, func(r *retrievedBlockAndPvtdata) {
+		r.hasNumTx(1)
+		r.hasNoPvtdata()
+	})
+
+	h.verifyBlockAndPvtData(13, nil, func(r *retrievedBlockAndPvtdata) {
+		r.hasNumTx(1)
+		r.pvtdataShouldContain(0, "marblesp", "collectionMarbles", "marble1", d.marbleValue("marble1", "blue", "tom", 35))
+		r.pvtdataShouldContain(0, "marblesp", "collectionMarblePrivateDetails", "marble1", d.marbleDetail("marble1", 99))
+	})
+}
+
+func (d *v20SampleDataHelper) marbleValue(name, color, owner string, size int) string {
+	return fmt.Sprintf(`{"docType":"marble","name":"%s","color":"%s","size":%d,"owner":"%s"}`, name, color, size, owner)
+}
+
+func (d *v20SampleDataHelper) marbleDetail(name string, price int) string {
+	return fmt.Sprintf(`{"docType":"marblePrivateDetails","name":"%s","price":%d}`, name, price)
+}
+
+func (d *v20SampleDataHelper) mspIDsInChannelConfig() []string {
+	return []string{"Org1MSP", "Org2MSP", "Org2MSP"}
+}
+
+// match integration/ledger/testdata/collection_configs/collections_config1.json
+func (d *v20SampleDataHelper) marbleCollConf1(ccName string) []*collConf {
+	collConfigs := make([]*collConf, 0)
+	collConfigs = append(collConfigs, &collConf{name: "collectionMarbles", btl: 1000000, members: []string{"Org1MSP", "Org2MSP"}})
+	collConfigs = append(collConfigs, &collConf{name: "collectionMarblePrivateDetails", btl: 1000000, members: []string{"Org2MSP", "Org3MSP"}})
+	for _, mspID := range d.mspIDsInChannelConfig() {
+		collConfigs = append(collConfigs, &collConf{name: lifecycle.ImplicitCollectionNameForOrg(mspID), btl: 0, members: []string{mspID}})
+	}
+	return collConfigs
+}
+
+// match integration/ledger/testdata/collection_configs/collections_config2.json
+func (d *v20SampleDataHelper) marbleCollConf2(ccName string) []*collConf {
+	collConfigs := make([]*collConf, 0)
+	collConfigs = append(collConfigs, &collConf{name: "collectionMarbles", btl: 1000000, members: []string{"Org2MSP", "Org3MSP"}})
+	collConfigs = append(collConfigs, &collConf{name: "collectionMarblePrivateDetails", btl: 1000000, members: []string{"Org1MSP", "Org2MSP", "Org3MSP"}})
+	for _, mspID := range d.mspIDsInChannelConfig() {
+		collConfigs = append(collConfigs, &collConf{name: lifecycle.ImplicitCollectionNameForOrg(mspID), btl: 0, members: []string{mspID}})
+	}
+	return collConfigs
+}
diff --git a/integration/ledger/ledger_generate_test.go b/integration/ledger/ledger_generate_test.go
@@ -0,0 +1,136 @@
+/*
+Copyright IBM Corp All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package ledger
+
+import (
+	"fmt"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo"
+
+	"github.com/hyperledger/fabric/integration/nwo"
+)
+
+// This test generate sample ledger data that can be used to verify rebuild ledger function and upgrade function (in a future release).
+// It is skipped in general. To generate sample ledger data, comment out the line `xxx` and run this test in isolation.
+// It does not delete the network directory so that you can copy the generated data to a different directory for unit tests.
+// At the end of test, it prints `setup.testDir is <directory>`. Copy the network data under <directory> to
+// the unit test directory for rebuild tests as needed.
+// It generates the following blocks:
+// block 0: genesis
+// block 1 to 4: network setup
+// block 5 to 8: marblesp chaincode instantiation
+// block 9 to 12: marbles chancode instantiation
+// block 13: marblesp chaincode invocation
+// block 14 to 17: upgrade marblesp chaincode with a new collection config
+// block 18: marbles chaincode invocation
+var _ = Describe("sample ledger generation", func() {
+	var (
+		setup       *setup
+		helper      *testHelper
+		chaincodemp nwo.Chaincode
+		chaincodem  nwo.Chaincode
+	)
+
+	BeforeEach(func() {
+		setup = initThreeOrgsSetup()
+		nwo.EnableCapabilities(setup.network, setup.channelID, "Application", "V2_0", setup.orderer, setup.peers...)
+		helper = &testHelper{
+			networkHelper: &networkHelper{
+				Network:   setup.network,
+				orderer:   setup.orderer,
+				peers:     setup.peers,
+				testDir:   setup.testDir,
+				channelID: setup.channelID,
+			},
+		}
+
+		chaincodemp = nwo.Chaincode{
+			Name:              "marblesp",
+			Version:           "1.0",
+			Path:              components.Build("github.com/hyperledger/fabric/integration/chaincode/marbles_private/cmd"),
+			Lang:              "binary",
+			PackageFile:       filepath.Join(setup.testDir, "marbles-pvtdata.tar.gz"),
+			Label:             "marbles-private-20",
+			SignaturePolicy:   `OR ('Org1MSP.member','Org2MSP.member', 'Org3MSP.member')`,
+			CollectionsConfig: filepath.Join("testdata", "collection_configs", "collections_config1.json"),
+			Sequence:          "1",
+		}
+
+		chaincodem = nwo.Chaincode{
+			Name:            "marbles",
+			Version:         "0.0",
+			Path:            "github.com/hyperledger/fabric/integration/chaincode/marbles/cmd",
+			Lang:            "golang",
+			PackageFile:     filepath.Join(setup.testDir, "marbles.tar.gz"),
+			Label:           "marbles",
+			SignaturePolicy: `OR ('Org1MSP.member','Org2MSP.member', 'Org3MSP.member')`,
+			Sequence:        "1",
+		}
+	})
+
+	AfterEach(func() {
+		setup.terminateAllProcess()
+		setup.network.Cleanup()
+		// do not delete testDir and log it so that we can copy the test data to unit tests for verification purpose
+		fmt.Printf("The test dir is %s. Use peers/org2.peer0/filesystem/ledgersData as the sample ledger for kvledger rebuild tests\n", setup.testDir)
+	})
+
+	It("creates marbles", func() {
+		Skip("Uncomment to generate sample ledger in v2.0 with new lifecycle chaincode deployment")
+
+		org2peer0 := setup.network.Peer("org2", "peer0")
+		height := helper.getLedgerHeight(org2peer0)
+
+		By(fmt.Sprintf("deploying marblesp chaincode at block height %d", height))
+		helper.deployChaincode(chaincodemp)
+
+		height = helper.getLedgerHeight(org2peer0)
+		By(fmt.Sprintf("deploying marbles chaincode at block height %d", height))
+		helper.deployChaincode(chaincodem)
+
+		height = helper.getLedgerHeight(org2peer0)
+		By(fmt.Sprintf("creating marbles1 with marblesp chaincode at block height %d", height))
+		helper.addMarble("marblesp", `{"name":"marble1", "color":"blue", "size":35, "owner":"tom", "price":99}`, org2peer0)
+		helper.waitUntilEqualLedgerHeight(height + 1)
+
+		By("verifying marble1 exist in collectionMarbles & collectionMarblePrivateDetails in peer0.org2")
+		helper.assertPresentInCollectionM("marblesp", "marble1", org2peer0)
+		helper.assertPresentInCollectionMPD("marblesp", "marble1", org2peer0)
+
+		By(fmt.Sprintf("upgrading marblesp chaincode at block height %d", helper.getLedgerHeight(org2peer0)))
+		chaincodemp.Version = "1.1"
+		chaincodemp.CollectionsConfig = filepath.Join("testdata", "collection_configs", "collections_config2.json")
+		chaincodemp.Sequence = "2"
+		nwo.DeployChaincode(setup.network, setup.channelID, setup.orderer, chaincodemp)
+
+		mhelper := &marblesTestHelper{
+			networkHelper: &networkHelper{
+				Network:   setup.network,
+				orderer:   setup.orderer,
+				peers:     setup.peers,
+				testDir:   setup.testDir,
+				channelID: setup.channelID,
+			},
+		}
+		By(fmt.Sprintf("creating marble100 with marbles chaincode at block height %d", helper.getLedgerHeight(org2peer0)))
+		mhelper.invokeMarblesChaincode("marbles", org2peer0, "initMarble", "marble100", "blue", "35", "tom")
+		By("transferring marble100 owner")
+		mhelper.invokeMarblesChaincode("marbles", org2peer0, "transferMarble", "marble100", "jerry")
+
+		By("verifying marble100 new owner after transfer by color")
+		expectedResult := newMarble("marble100", "blue", 35, "jerry")
+		mhelper.assertMarbleExists("marbles", org2peer0, expectedResult, "marble100")
+
+		By("getting history for marble100")
+		expectedHistoryResult := []*marbleHistoryResult{
+			{IsDelete: "false", Value: newMarble("marble100", "blue", 35, "jerry")},
+			{IsDelete: "false", Value: newMarble("marble100", "blue", 35, "tom")},
+		}
+		mhelper.assertGetHistoryForMarble("marbles", org2peer0, expectedHistoryResult, "marble100")
+	})
+})
diff --git a/integration/ledger/testdata/collection_configs/collections_config2.json b/integration/ledger/testdata/collection_configs/collections_config2.json
@@ -0,0 +1,18 @@
+[
+ 	{
+	 	"name": "collectionMarbles",
+	 	"policy": "OR('Org2MSP.member', 'Org3MSP.member')",
+	 	"requiredPeerCount": 1,
+	 	"maxPeerCount": 2,
+	 	"blockToLive":1000000,
+		"memberOnlyRead": false
+	},
+ 	{
+	 	"name": "collectionMarblePrivateDetails",
+	 	"policy": "OR('Org1MSP.member', 'Org2MSP.member', 'Org3MSP.member')",
+	 	"requiredPeerCount": 1,
+	 	"maxPeerCount": 2,
+	 	"blockToLive":1000000,
+		"memberOnlyRead": false
+ 	}
+]