Skip to content

Commit 3d8825e

Browse files
wenjianqiaowenjianqiao
committed
Add rebuild test with real ledger data
- Add an IT test to generate ledger data - Test rebuild command with the generated ledger data and verify statedb, blocks, pvtdata, historydb, configHistorydb Change-Id: I9f469d95f7120b402545b027cc5fed9b188afb95 Signed-off-by: Wenjian Qiao <wenjianq@gmail.com>
1 parent bbdfa7f commit 3d8825e

File tree

6 files changed

+354
-1
lines changed

6 files changed

+354
-1
lines changed

‎core/ledger/kvledger/tests/env.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/hyperledger/fabric/common/ledger/blkstorage/fsblkstorage"
1919
"github.com/hyperledger/fabric/common/ledger/util"
2020
"github.com/hyperledger/fabric/common/metrics/disabled"
21+
"github.com/hyperledger/fabric/core/chaincode/lifecycle"
2122
"github.com/hyperledger/fabric/core/common/privdata"
2223
"github.com/hyperledger/fabric/core/container/externalbuilder"
2324
"github.com/hyperledger/fabric/core/ledger"
@@ -272,3 +273,59 @@ func createSelfSignedData(cryptoProvider bccsp.BCCSP) protoutil.SignedData {
272273
Identity: peerIdentity,
273274
}
274275
}
276+
277+
// deployedCCInfoProviderWrapper is a wrapper type that overrides ChaincodeImplicitCollections
278+
type deployedCCInfoProviderWrapper struct {
279+
*lifecycle.ValidatorCommitter
280+
orgMSPIDs []string
281+
}
282+
283+
// AllCollectionsConfigPkg overrides the same method in lifecycle.AllCollectionsConfigPkg.
284+
// It is basically a copy of lifecycle.AllCollectionsConfigPkg and invokes ImplicitCollections in the wrapper.
285+
// This method is called when the unit test code gets private data code path.
286+
func (dc *deployedCCInfoProviderWrapper) AllCollectionsConfigPkg(channelName, chaincodeName string, qe ledger.SimpleQueryExecutor) (*common.CollectionConfigPackage, error) {
287+
chaincodeInfo, err := dc.ChaincodeInfo(channelName, chaincodeName, qe)
288+
if err != nil {
289+
return nil, err
290+
}
291+
explicitCollectionConfigPkg := chaincodeInfo.ExplicitCollectionConfigPkg
292+
293+
implicitCollections, _ := dc.ImplicitCollections(channelName, "", nil)
294+
295+
var combinedColls []*common.CollectionConfig
296+
if explicitCollectionConfigPkg != nil {
297+
combinedColls = append(combinedColls, explicitCollectionConfigPkg.Config...)
298+
}
299+
for _, implicitColl := range implicitCollections {
300+
c := &common.CollectionConfig{}
301+
c.Payload = &common.CollectionConfig_StaticCollectionConfig{StaticCollectionConfig: implicitColl}
302+
combinedColls = append(combinedColls, c)
303+
}
304+
return &common.CollectionConfigPackage{
305+
Config: combinedColls,
306+
}, nil
307+
}
308+
309+
// ImplicitCollections overrides the same method in lifecycle.ValidatorCommitter.
310+
// It constructs static collection config using known mspids from the sample ledger.
311+
// This method is called when the unit test code gets collection configuration.
312+
func (dc *deployedCCInfoProviderWrapper) ImplicitCollections(channelName, chaincodeName string, qe ledger.SimpleQueryExecutor) ([]*common.StaticCollectionConfig, error) {
313+
collConfigs := make([]*common.StaticCollectionConfig, 0, len(dc.orgMSPIDs))
314+
for _, mspID := range dc.orgMSPIDs {
315+
collConfigs = append(collConfigs, lifecycle.GenerateImplicitCollectionForOrg(mspID))
316+
}
317+
return collConfigs, nil
318+
}
319+
320+
func createDeployedCCInfoProvider(orgMSPIDs []string) ledger.DeployedChaincodeInfoProvider {
321+
deployedCCInfoProvider := &lifecycle.ValidatorCommitter{
322+
Resources: &lifecycle.Resources{
323+
Serializer: &lifecycle.Serializer{},
324+
},
325+
LegacyDeployedCCInfoProvider: &lscc.DeployedCCInfoProvider{},
326+
}
327+
return &deployedCCInfoProviderWrapper{
328+
ValidatorCommitter: deployedCCInfoProvider,
329+
orgMSPIDs: orgMSPIDs,
330+
}
331+
}

‎core/ledger/kvledger/tests/testdata/v20/sample_ledgers/ledgersData.zip

56.6 KB
Binary file not shown.

‎core/ledger/kvledger/tests/util.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/golang/protobuf/proto"
1414
"github.com/hyperledger/fabric-protos-go/common"
1515
"github.com/hyperledger/fabric-protos-go/ledger/rwset"
16+
"github.com/hyperledger/fabric-protos-go/msp"
1617
protopeer "github.com/hyperledger/fabric-protos-go/peer"
1718
"github.com/hyperledger/fabric/common/cauthdsl"
1819
configtxtest "github.com/hyperledger/fabric/common/configtx/test"
@@ -88,7 +89,17 @@ func convertFromMemberOrgsPolicy(policy *common.CollectionPolicyConfig) []string
8889
ids := policy.GetSignaturePolicy().Identities
8990
var members []string
9091
for _, id := range ids {
91-
members = append(members, string(id.Principal))
92+
role := &msp.MSPRole{}
93+
err := proto.Unmarshal(id.Principal, role)
94+
if err == nil {
95+
// This is for sample ledger generated by fabric (e.g., integration test),
96+
// where id.Principal was properly marshalled during sample ledger generation.
97+
members = append(members, role.MspIdentifier)
98+
} else {
99+
// This is for sample ledger generated by sampleDataHelper.populateLedger,
100+
// where id.Principal was a []byte cast from a string (not a marshalled msp.MSPRole)
101+
members = append(members, string(id.Principal))
102+
}
92103
}
93104
return members
94105
}

‎core/ledger/kvledger/tests/v20_test.go

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/*
2+
Copyright IBM Corp. All Rights Reserved.
3+
4+
SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
package tests
8+
9+
import (
10+
"fmt"
11+
"testing"
12+
13+
"github.com/hyperledger/fabric/common/ledger/testutil"
14+
"github.com/hyperledger/fabric/core/chaincode/lifecycle"
15+
"github.com/hyperledger/fabric/core/ledger/kvledger"
16+
"github.com/stretchr/testify/require"
17+
)
18+
19+
// TestV20SampleLedger tests rebuild function with sample v2.0 ledger data generated by integration/ledger/ledger_generate_test.go
20+
func TestV20SampleLedger(t *testing.T) {
21+
env := newEnv(t)
22+
defer env.cleanup()
23+
24+
dataHelper := &v20SampleDataHelper{sampleDataVersion: "v2.0", t: t}
25+
env.initializer.DeployedChaincodeInfoProvider = createDeployedCCInfoProvider(dataHelper.mspIDsInChannelConfig())
26+
ledgerFSRoot := env.initializer.Config.RootFSPath
27+
require.NoError(t, testutil.Unzip("testdata/v20/sample_ledgers/ledgersData.zip", ledgerFSRoot, false))
28+
29+
env.initLedgerMgmt()
30+
31+
h1 := env.newTestHelperOpenLgr("testchannel", t)
32+
dataHelper.verify(h1)
33+
34+
// rebuild and verify again
35+
env.closeLedgerMgmt()
36+
kvledger.RebuildDBs(env.getLedgerRootPath())
37+
env.initLedgerMgmt()
38+
h1 = env.newTestHelperOpenLgr("testchannel", t)
39+
dataHelper.verify(h1)
40+
}
41+
42+
// The generated ledger has the following blocks:
43+
// block 0: genesis
44+
// block 1 to 4: network setup
45+
// block 5 to 8: marblesp chaincode instantiation
46+
// block 9 to 12: marbles chancode instantiation
47+
// block 13: marblesp chaincode invocation (new marble1)
48+
// block 14 to 17: upgrade marblesp chaincode with a new collection config
49+
// block 18 to 19: marbles chaincode invocation (new marble100 and transfer)
50+
type v20SampleDataHelper struct {
51+
sampleDataVersion string
52+
t *testing.T
53+
}
54+
55+
func (d *v20SampleDataHelper) verify(h *testhelper) {
56+
d.verifyState(h)
57+
d.verifyBlockAndPvtdata(h)
58+
d.verifyConfigHistory(h)
59+
d.verifyHistory(h)
60+
}
61+
62+
func (d *v20SampleDataHelper) verifyState(h *testhelper) {
63+
h.verifyPubState("marbles", "marble100", d.marbleValue("marble100", "blue", "jerry", 35))
64+
h.verifyPvtState("marblesp", "collectionMarbles", "marble1", d.marbleValue("marble1", "blue", "tom", 35))
65+
h.verifyPvtState("marblesp", "collectionMarblePrivateDetails", "marble1", d.marbleDetail("marble1", 99))
66+
}
67+
68+
func (d *v20SampleDataHelper) verifyHistory(h *testhelper) {
69+
expectedHistoryValue1 := []string{
70+
d.marbleValue("marble100", "blue", "jerry", 35),
71+
d.marbleValue("marble100", "blue", "tom", 35),
72+
}
73+
h.verifyHistory("marbles", "marble100", expectedHistoryValue1)
74+
}
75+
76+
func (d *v20SampleDataHelper) verifyConfigHistory(h *testhelper) {
77+
// below block 10 should match integration/ledger/testdata/collection_configs/collections_config1.json
78+
h.verifyMostRecentCollectionConfigBelow(10, "marblesp",
79+
&expectedCollConfInfo{8, d.marbleCollConf1("marbelsp")})
80+
81+
// below block 18 should match integration/ledger/testdata/collection_configs/collections_config2.json
82+
h.verifyMostRecentCollectionConfigBelow(18, "marblesp",
83+
&expectedCollConfInfo{17, d.marbleCollConf2("marbelsp")})
84+
}
85+
86+
func (d *v20SampleDataHelper) verifyBlockAndPvtdata(h *testhelper) {
87+
h.verifyBlockAndPvtData(8, nil, func(r *retrievedBlockAndPvtdata) {
88+
r.hasNumTx(1)
89+
r.hasNoPvtdata()
90+
})
91+
92+
h.verifyBlockAndPvtData(13, nil, func(r *retrievedBlockAndPvtdata) {
93+
r.hasNumTx(1)
94+
r.pvtdataShouldContain(0, "marblesp", "collectionMarbles", "marble1", d.marbleValue("marble1", "blue", "tom", 35))
95+
r.pvtdataShouldContain(0, "marblesp", "collectionMarblePrivateDetails", "marble1", d.marbleDetail("marble1", 99))
96+
})
97+
}
98+
99+
func (d *v20SampleDataHelper) marbleValue(name, color, owner string, size int) string {
100+
return fmt.Sprintf(`{"docType":"marble","name":"%s","color":"%s","size":%d,"owner":"%s"}`, name, color, size, owner)
101+
}
102+
103+
func (d *v20SampleDataHelper) marbleDetail(name string, price int) string {
104+
return fmt.Sprintf(`{"docType":"marblePrivateDetails","name":"%s","price":%d}`, name, price)
105+
}
106+
107+
func (d *v20SampleDataHelper) mspIDsInChannelConfig() []string {
108+
return []string{"Org1MSP", "Org2MSP", "Org2MSP"}
109+
}
110+
111+
// match integration/ledger/testdata/collection_configs/collections_config1.json
112+
func (d *v20SampleDataHelper) marbleCollConf1(ccName string) []*collConf {
113+
collConfigs := make([]*collConf, 0)
114+
collConfigs = append(collConfigs, &collConf{name: "collectionMarbles", btl: 1000000, members: []string{"Org1MSP", "Org2MSP"}})
115+
collConfigs = append(collConfigs, &collConf{name: "collectionMarblePrivateDetails", btl: 1000000, members: []string{"Org2MSP", "Org3MSP"}})
116+
for _, mspID := range d.mspIDsInChannelConfig() {
117+
collConfigs = append(collConfigs, &collConf{name: lifecycle.ImplicitCollectionNameForOrg(mspID), btl: 0, members: []string{mspID}})
118+
}
119+
return collConfigs
120+
}
121+
122+
// match integration/ledger/testdata/collection_configs/collections_config2.json
123+
func (d *v20SampleDataHelper) marbleCollConf2(ccName string) []*collConf {
124+
collConfigs := make([]*collConf, 0)
125+
collConfigs = append(collConfigs, &collConf{name: "collectionMarbles", btl: 1000000, members: []string{"Org2MSP", "Org3MSP"}})
126+
collConfigs = append(collConfigs, &collConf{name: "collectionMarblePrivateDetails", btl: 1000000, members: []string{"Org1MSP", "Org2MSP", "Org3MSP"}})
127+
for _, mspID := range d.mspIDsInChannelConfig() {
128+
collConfigs = append(collConfigs, &collConf{name: lifecycle.ImplicitCollectionNameForOrg(mspID), btl: 0, members: []string{mspID}})
129+
}
130+
return collConfigs
131+
}

‎integration/ledger/ledger_generate_test.go

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*
2+
Copyright IBM Corp All Rights Reserved.
3+
4+
SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
package ledger
8+
9+
import (
10+
"fmt"
11+
"path/filepath"
12+
13+
. "github.com/onsi/ginkgo"
14+
15+
"github.com/hyperledger/fabric/integration/nwo"
16+
)
17+
18+
// This test generate sample ledger data that can be used to verify rebuild ledger function and upgrade function (in a future release).
19+
// It is skipped in general. To generate sample ledger data, comment out the line `xxx` and run this test in isolation.
20+
// It does not delete the network directory so that you can copy the generated data to a different directory for unit tests.
21+
// At the end of test, it prints `setup.testDir is <directory>`. Copy the network data under <directory> to
22+
// the unit test directory for rebuild tests as needed.
23+
// It generates the following blocks:
24+
// block 0: genesis
25+
// block 1 to 4: network setup
26+
// block 5 to 8: marblesp chaincode instantiation
27+
// block 9 to 12: marbles chancode instantiation
28+
// block 13: marblesp chaincode invocation
29+
// block 14 to 17: upgrade marblesp chaincode with a new collection config
30+
// block 18: marbles chaincode invocation
31+
var _ = Describe("sample ledger generation", func() {
32+
var (
33+
setup *setup
34+
helper *testHelper
35+
chaincodemp nwo.Chaincode
36+
chaincodem nwo.Chaincode
37+
)
38+
39+
BeforeEach(func() {
40+
setup = initThreeOrgsSetup()
41+
nwo.EnableCapabilities(setup.network, setup.channelID, "Application", "V2_0", setup.orderer, setup.peers...)
42+
helper = &testHelper{
43+
networkHelper: &networkHelper{
44+
Network: setup.network,
45+
orderer: setup.orderer,
46+
peers: setup.peers,
47+
testDir: setup.testDir,
48+
channelID: setup.channelID,
49+
},
50+
}
51+
52+
chaincodemp = nwo.Chaincode{
53+
Name: "marblesp",
54+
Version: "1.0",
55+
Path: components.Build("github.com/hyperledger/fabric/integration/chaincode/marbles_private/cmd"),
56+
Lang: "binary",
57+
PackageFile: filepath.Join(setup.testDir, "marbles-pvtdata.tar.gz"),
58+
Label: "marbles-private-20",
59+
SignaturePolicy: `OR ('Org1MSP.member','Org2MSP.member', 'Org3MSP.member')`,
60+
CollectionsConfig: filepath.Join("testdata", "collection_configs", "collections_config1.json"),
61+
Sequence: "1",
62+
}
63+
64+
chaincodem = nwo.Chaincode{
65+
Name: "marbles",
66+
Version: "0.0",
67+
Path: "github.com/hyperledger/fabric/integration/chaincode/marbles/cmd",
68+
Lang: "golang",
69+
PackageFile: filepath.Join(setup.testDir, "marbles.tar.gz"),
70+
Label: "marbles",
71+
SignaturePolicy: `OR ('Org1MSP.member','Org2MSP.member', 'Org3MSP.member')`,
72+
Sequence: "1",
73+
}
74+
})
75+
76+
AfterEach(func() {
77+
setup.terminateAllProcess()
78+
setup.network.Cleanup()
79+
// do not delete testDir and log it so that we can copy the test data to unit tests for verification purpose
80+
fmt.Printf("The test dir is %s. Use peers/org2.peer0/filesystem/ledgersData as the sample ledger for kvledger rebuild tests\n", setup.testDir)
81+
})
82+
83+
It("creates marbles", func() {
84+
Skip("Uncomment to generate sample ledger in v2.0 with new lifecycle chaincode deployment")
85+
86+
org2peer0 := setup.network.Peer("org2", "peer0")
87+
height := helper.getLedgerHeight(org2peer0)
88+
89+
By(fmt.Sprintf("deploying marblesp chaincode at block height %d", height))
90+
helper.deployChaincode(chaincodemp)
91+
92+
height = helper.getLedgerHeight(org2peer0)
93+
By(fmt.Sprintf("deploying marbles chaincode at block height %d", height))
94+
helper.deployChaincode(chaincodem)
95+
96+
height = helper.getLedgerHeight(org2peer0)
97+
By(fmt.Sprintf("creating marbles1 with marblesp chaincode at block height %d", height))
98+
helper.addMarble("marblesp", `{"name":"marble1", "color":"blue", "size":35, "owner":"tom", "price":99}`, org2peer0)
99+
helper.waitUntilEqualLedgerHeight(height + 1)
100+
101+
By("verifying marble1 exist in collectionMarbles & collectionMarblePrivateDetails in peer0.org2")
102+
helper.assertPresentInCollectionM("marblesp", "marble1", org2peer0)
103+
helper.assertPresentInCollectionMPD("marblesp", "marble1", org2peer0)
104+
105+
By(fmt.Sprintf("upgrading marblesp chaincode at block height %d", helper.getLedgerHeight(org2peer0)))
106+
chaincodemp.Version = "1.1"
107+
chaincodemp.CollectionsConfig = filepath.Join("testdata", "collection_configs", "collections_config2.json")
108+
chaincodemp.Sequence = "2"
109+
nwo.DeployChaincode(setup.network, setup.channelID, setup.orderer, chaincodemp)
110+
111+
mhelper := &marblesTestHelper{
112+
networkHelper: &networkHelper{
113+
Network: setup.network,
114+
orderer: setup.orderer,
115+
peers: setup.peers,
116+
testDir: setup.testDir,
117+
channelID: setup.channelID,
118+
},
119+
}
120+
By(fmt.Sprintf("creating marble100 with marbles chaincode at block height %d", helper.getLedgerHeight(org2peer0)))
121+
mhelper.invokeMarblesChaincode("marbles", org2peer0, "initMarble", "marble100", "blue", "35", "tom")
122+
By("transferring marble100 owner")
123+
mhelper.invokeMarblesChaincode("marbles", org2peer0, "transferMarble", "marble100", "jerry")
124+
125+
By("verifying marble100 new owner after transfer by color")
126+
expectedResult := newMarble("marble100", "blue", 35, "jerry")
127+
mhelper.assertMarbleExists("marbles", org2peer0, expectedResult, "marble100")
128+
129+
By("getting history for marble100")
130+
expectedHistoryResult := []*marbleHistoryResult{
131+
{IsDelete: "false", Value: newMarble("marble100", "blue", 35, "jerry")},
132+
{IsDelete: "false", Value: newMarble("marble100", "blue", 35, "tom")},
133+
}
134+
mhelper.assertGetHistoryForMarble("marbles", org2peer0, expectedHistoryResult, "marble100")
135+
})
136+
})

‎integration/ledger/testdata/collection_configs/collections_config2.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[
2+
{
3+
"name": "collectionMarbles",
4+
"policy": "OR('Org2MSP.member', 'Org3MSP.member')",
5+
"requiredPeerCount": 1,
6+
"maxPeerCount": 2,
7+
"blockToLive":1000000,
8+
"memberOnlyRead": false
9+
},
10+
{
11+
"name": "collectionMarblePrivateDetails",
12+
"policy": "OR('Org1MSP.member', 'Org2MSP.member', 'Org3MSP.member')",
13+
"requiredPeerCount": 1,
14+
"maxPeerCount": 2,
15+
"blockToLive":1000000,
16+
"memberOnlyRead": false
17+
}
18+
]

0 commit comments

Comments
 (0)