Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix perf issue by key-endorsement to regular chaincode
Key-level endorsement needs to access the metadata for each of the keys in the write-set. Even the chaincodes that do not use this feature will end up paying this additional penality. Especially, in an environment that uses couchdb, the performance regression will be visible for the existing chaincodes. The major reason is that the metadata is accessed one at a time during validation and commit path whereas, couchdb is better used if the data can be loaded in bulk. Ideally, the data required by key-level endorsement for the entire block should be loaded in a single bulk load call. Which would benefit even the chaincodes that leverage this feature. However, this would need some refactoring that can be taken up in a future release. This CR introduces a fix which would save the regular chaincodes (that do not use key-level endorsement feature) from paying the avobe mentioned penality. In this fix, we track the chaincodes that do not use metadata and for such chaincodes, the GetStateMetadata call simply returns nil without going to the underlying db. Even when we start using the bulk-load for key-level endorsement, this fix will still be relevant in the sense that the bulkload call will be avoided for regular chaincodes FAB-11700 #done Change-Id: Icfc654b7d27f727f1811a5a300400e92b6e8be9d Signed-off-by: manish <manish.sethi@gmail.com>
- Loading branch information
1 parent
861254d
commit 945138e
Showing
17 changed files
with
1,238 additions
and
73 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
core/ledger/kvledger/txmgmt/privacyenabledstate/optimization.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| Copyright IBM Corp. All Rights Reserved. | ||
| SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package privacyenabledstate | ||
|
|
||
| import ( | ||
| "github.com/hyperledger/fabric/common/ledger/util/leveldbhelper" | ||
| ) | ||
|
|
||
| type metadataHint struct { | ||
| cache map[string]bool | ||
| bookkeeper *leveldbhelper.DBHandle | ||
| } | ||
|
|
||
| func newMetadataHint(bookkeeper *leveldbhelper.DBHandle) *metadataHint { | ||
| cache := map[string]bool{} | ||
| itr := bookkeeper.GetIterator(nil, nil) | ||
| defer itr.Release() | ||
| for itr.Next() { | ||
| namespace := string(itr.Key()) | ||
| cache[namespace] = true | ||
| } | ||
| return &metadataHint{cache, bookkeeper} | ||
| } | ||
|
|
||
| func (h *metadataHint) metadataEverUsedFor(namespace string) bool { | ||
| return h.cache[namespace] | ||
| } | ||
|
|
||
| func (h *metadataHint) setMetadataUsedFlag(updates *UpdateBatch) { | ||
| batch := leveldbhelper.NewUpdateBatch() | ||
| for ns := range filterNamespacesThatHasMetadata(updates) { | ||
| if h.cache[ns] { | ||
| continue | ||
| } | ||
| h.cache[ns] = true | ||
| batch.Put([]byte(ns), []byte{}) | ||
| } | ||
| h.bookkeeper.WriteBatch(batch, true) | ||
| } | ||
|
|
||
| func filterNamespacesThatHasMetadata(updates *UpdateBatch) map[string]bool { | ||
| namespaces := map[string]bool{} | ||
| pubUpdates, hashUpdates := updates.PubUpdates, updates.HashUpdates | ||
| // add ns for public data | ||
| for _, ns := range pubUpdates.GetUpdatedNamespaces() { | ||
| for _, vv := range updates.PubUpdates.GetUpdates(ns) { | ||
| if vv.Metadata == nil { | ||
| continue | ||
| } | ||
| namespaces[ns] = true | ||
| } | ||
| } | ||
| // add ns for private hashes | ||
| for ns, nsBatch := range hashUpdates.UpdateMap { | ||
| for _, coll := range nsBatch.GetCollectionNames() { | ||
| for _, vv := range nsBatch.GetUpdates(coll) { | ||
| if vv.Metadata == nil { | ||
| continue | ||
| } | ||
| namespaces[ns] = true | ||
| } | ||
| } | ||
| } | ||
| return namespaces | ||
| } |
80 changes: 80 additions & 0 deletions
80
core/ledger/kvledger/txmgmt/privacyenabledstate/optimization_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| /* | ||
| Copyright IBM Corp. All Rights Reserved. | ||
| SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package privacyenabledstate | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/hyperledger/fabric/core/ledger/kvledger/bookkeeping" | ||
| "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/mock" | ||
| "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/version" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestMetadataHintCorrectness(t *testing.T) { | ||
| bookkeepingTestEnv := bookkeeping.NewTestEnv(t) | ||
| defer bookkeepingTestEnv.Cleanup() | ||
| bookkeeper := bookkeepingTestEnv.TestProvider.GetDBHandle("ledger1", bookkeeping.MetadataPresenceIndicator) | ||
|
|
||
| metadataHint := newMetadataHint(bookkeeper) | ||
| assert.False(t, metadataHint.metadataEverUsedFor("ns1")) | ||
|
|
||
| updates := NewUpdateBatch() | ||
| updates.PubUpdates.PutValAndMetadata("ns1", "key", []byte("value"), []byte("metadata"), version.NewHeight(1, 1)) | ||
| updates.PubUpdates.PutValAndMetadata("ns2", "key", []byte("value"), []byte("metadata"), version.NewHeight(1, 2)) | ||
| updates.PubUpdates.PutValAndMetadata("ns3", "key", []byte("value"), nil, version.NewHeight(1, 3)) | ||
| updates.HashUpdates.PutValAndMetadata("ns1_pvt", "key", "coll", []byte("value"), []byte("metadata"), version.NewHeight(1, 1)) | ||
| updates.HashUpdates.PutValAndMetadata("ns2_pvt", "key", "coll", []byte("value"), []byte("metadata"), version.NewHeight(1, 3)) | ||
| updates.HashUpdates.PutValAndMetadata("ns3_pvt", "key", "coll", []byte("value"), nil, version.NewHeight(1, 3)) | ||
| metadataHint.setMetadataUsedFlag(updates) | ||
|
|
||
| t.Run("MetadataAddedInCurrentSession", func(t *testing.T) { | ||
| assert.True(t, metadataHint.metadataEverUsedFor("ns1")) | ||
| assert.True(t, metadataHint.metadataEverUsedFor("ns2")) | ||
| assert.True(t, metadataHint.metadataEverUsedFor("ns1_pvt")) | ||
| assert.True(t, metadataHint.metadataEverUsedFor("ns2_pvt")) | ||
| assert.False(t, metadataHint.metadataEverUsedFor("ns3")) | ||
| assert.False(t, metadataHint.metadataEverUsedFor("ns4")) | ||
| }) | ||
|
|
||
| t.Run("MetadataFromPersistence", func(t *testing.T) { | ||
| metadataHintFromPersistence := newMetadataHint(bookkeeper) | ||
| assert.True(t, metadataHintFromPersistence.metadataEverUsedFor("ns1")) | ||
| assert.True(t, metadataHintFromPersistence.metadataEverUsedFor("ns2")) | ||
| assert.True(t, metadataHintFromPersistence.metadataEverUsedFor("ns1_pvt")) | ||
| assert.True(t, metadataHintFromPersistence.metadataEverUsedFor("ns2_pvt")) | ||
| assert.False(t, metadataHintFromPersistence.metadataEverUsedFor("ns3")) | ||
| assert.False(t, metadataHintFromPersistence.metadataEverUsedFor("ns4")) | ||
| }) | ||
| } | ||
|
|
||
| func TestMetadataHintOptimizationSkippingGoingToDB(t *testing.T) { | ||
| bookkeepingTestEnv := bookkeeping.NewTestEnv(t) | ||
| defer bookkeepingTestEnv.Cleanup() | ||
| bookkeeper := bookkeepingTestEnv.TestProvider.GetDBHandle("ledger1", bookkeeping.MetadataPresenceIndicator) | ||
|
|
||
| mockVersionedDB := &mock.VersionedDB{} | ||
| db, err := NewCommonStorageDB(mockVersionedDB, "testledger", newMetadataHint(bookkeeper)) | ||
| assert.NoError(t, err) | ||
| updates := NewUpdateBatch() | ||
| updates.PubUpdates.PutValAndMetadata("ns1", "key", []byte("value"), []byte("metadata"), version.NewHeight(1, 1)) | ||
| updates.PubUpdates.PutValAndMetadata("ns2", "key", []byte("value"), nil, version.NewHeight(1, 2)) | ||
| db.ApplyPrivacyAwareUpdates(updates, version.NewHeight(1, 3)) | ||
|
|
||
| db.GetStateMetadata("ns1", "randomkey") | ||
| assert.Equal(t, 1, mockVersionedDB.GetStateCallCount()) | ||
| db.GetPrivateDataMetadataByHash("ns1", "randomColl", []byte("randomKeyhash")) | ||
| assert.Equal(t, 2, mockVersionedDB.GetStateCallCount()) | ||
|
|
||
| db.GetStateMetadata("ns2", "randomkey") | ||
| db.GetPrivateDataMetadataByHash("ns2", "randomColl", []byte("randomKeyhash")) | ||
| assert.Equal(t, 2, mockVersionedDB.GetStateCallCount()) | ||
|
|
||
| db.GetStateMetadata("randomeNs", "randomkey") | ||
| db.GetPrivateDataMetadataByHash("randomeNs", "randomColl", []byte("randomKeyhash")) | ||
| assert.Equal(t, 2, mockVersionedDB.GetStateCallCount()) | ||
| } |
Oops, something went wrong.