Skip to content

Commit

Permalink
FAB-2462: Re-enable paging results for queries
Browse files Browse the repository at this point in the history 
Paging result sets is required in order to avoid failures
related to large result sets.

The Next() call on StateQueryIterator from Chaincode was failing
in peer due to a corner case. This fix addresses the corner case (where
the number of results is an exact multiple of shim batch size).

This is done by keeping the HasMore flag in iterator accurate by
pre-fetching the next batch of results when the last cached item
from shim is accessed.

This fix also adds the Namespace field to commonledger.KV type. The data
returned to peer has namespace information. All state iterators return
 QueryResult with commonledger.KV type elements.

Change-Id: Id872130db8a20ec5a59593b1fa004dce43ee36ac
Signed-off-by: Balaji Viswanathan <balaji.viswanathan@gmail.com>
Signed-off-by: denyeart <enyeart@us.ibm.com>
  • Loading branch information
@bviswana101
bviswana101 committed Apr 25, 2017
1 parent 7f114bb commit 868a3e9
Show file tree
Hide file tree
Showing 6 changed files with 189 additions and 259 deletions.
297 changes: 113 additions & 184 deletions core/chaincode/exectransaction_test.go
View file
Expand Up @@ -1055,160 +1055,41 @@ func TestQueries(t *testing.T) {
return
}

// Add 12 marbles for testing range queries and rich queries (for capable ledgers)
// Add 101 marbles for testing range queries and rich queries (for capable ledgers)
// The tests will test both range and rich queries and queries with query limits
f = "put"
args = util.ToChaincodeArgs(f, "marble01", "{\"docType\":\"marble\",\"name\":\"marble01\",\"color\":\"blue\",\"size\":35,\"owner\":\"tom\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++

if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble02", "{\"docType\":\"marble\",\"name\":\"marble02\",\"color\":\"red\",\"size\":25,\"owner\":\"tom\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++

if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble03", "{\"docType\":\"marble\",\"name\":\"marble03\",\"color\":\"green\",\"size\":15,\"owner\":\"tom\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble04", "{\"docType\":\"marble\",\"name\":\"marble04\",\"color\":\"green\",\"size\":20,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble05", "{\"docType\":\"marble\",\"name\":\"marble05\",\"color\":\"red\",\"size\":25,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble06", "{\"docType\":\"marble\",\"name\":\"marble06\",\"color\":\"blue\",\"size\":35,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble07", "{\"docType\":\"marble\",\"name\":\"marble07\",\"color\":\"yellow\",\"size\":20,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble08", "{\"docType\":\"marble\",\"name\":\"marble08\",\"color\":\"green\",\"size\":40,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}
for i := 1; i <= 101; i++ {
f = "put"

f = "put"
args = util.ToChaincodeArgs(f, "marble09", "{\"docType\":\"marble\",\"name\":\"marble09\",\"color\":\"yellow\",\"size\":10,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}
// 51 owned by tom, 50 by jerry
owner := "tom"
if i%2 == 0 {
owner = "jerry"
}

f = "put"
args = util.ToChaincodeArgs(f, "marble10", "{\"docType\":\"marble\",\"name\":\"marble10\",\"color\":\"red\",\"size\":20,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}
// one marble color is red, 100 are blue
color := "blue"
if i == 12 {
color = "red"
}

f = "put"
args = util.ToChaincodeArgs(f, "marble11", "{\"docType\":\"marble\",\"name\":\"marble11\",\"color\":\"green\",\"size\":40,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}
key := fmt.Sprintf("marble%03d", i)
argsString := fmt.Sprintf("{\"docType\":\"marble\",\"name\":\"%s\",\"color\":\"%s\",\"size\":35,\"owner\":\"%s\"}", key, color, owner)
args = util.ToChaincodeArgs(f, key, argsString)
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++

f = "put"
args = util.ToChaincodeArgs(f, "marble12", "{\"docType\":\"marble\",\"name\":\"marble12\",\"color\":\"red\",\"size\":30,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}
}

//TODO - the following query tests for queryLimits may change due to future designs
// for batch "paging"

//The following range query for "marble01" to "marble11" should return 10 marbles
//The following range query for "marble001" to "marble011" should return 10 marbles
f = "keys"
args = util.ToChaincodeArgs(f, "marble01", "marble11")
args = util.ToChaincodeArgs(f, "marble001", "marble011")

spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, retval, err := invoke(ctxt, chainID, spec, nextBlockNumber, nil)
Expand All @@ -1222,25 +1103,21 @@ func TestQueries(t *testing.T) {

var keys []interface{}
err = json.Unmarshal(retval, &keys)

//default query limit of 10000 is used, query should return all records that meet the criteria
if len(keys) != 10 {
t.Fail()
t.Logf("Error detected with the range query, should have returned 10 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//Reset the query limit to 5
viper.Set("ledger.state.queryLimit", 5)

//The following range query for "marble01" to "marble11" should return 5 marbles due to the queryLimit
// querying for all marbles will return 101 marbles
// this query should return exactly 101 results (one call to Next())
//The following range query for "marble001" to "marble102" should return 101 marbles
f = "keys"
args = util.ToChaincodeArgs(f, "marble01", "marble11")
args = util.ToChaincodeArgs(f, "marble001", "marble102")

spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)

nextBlockNumber++
if err != nil {
t.Fail()
Expand All @@ -1252,46 +1129,52 @@ func TestQueries(t *testing.T) {
//unmarshal the results
err = json.Unmarshal(retval, &keys)

//check to see if there are 5 values
if len(keys) != 5 {
//check to see if there are 101 values
//default query limit of 10000 is used, this query is effectively unlimited
if len(keys) != 101 {
t.Fail()
t.Logf("Error detected with the range query, should have returned 5 but returned %v", len(keys))
t.Logf("Error detected with the range query, should have returned 101 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//Reset the query limit to default
viper.Set("ledger.state.queryLimit", 10000)
// ExecuteQuery supported only for CouchDB and
// query limits apply for CouchDB range and rich queries only
if ledgerconfig.IsCouchDBEnabled() == true {

if ledgerconfig.IsHistoryDBEnabled() == true {
// corner cases for shim batching. currnt shim batch size is 100
// this query should return exactly 100 results (no call to Next())
f = "query"
args = util.ToChaincodeArgs(f, "{\"selector\":{\"color\":\"blue\"}}")

f = "put"
args = util.ToChaincodeArgs(f, "marble12", "{\"docType\":\"marble\",\"name\":\"marble12\",\"color\":\"red\",\"size\":30,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++

if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble12", "{\"docType\":\"marble\",\"name\":\"marble12\",\"color\":\"red\",\"size\":30,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
//unmarshal the results
err = json.Unmarshal(retval, &keys)

//check to see if there are 100 values
if len(keys) != 100 {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
t.Logf("Error detected with the rich query, should have returned 100 but returned %v %s", len(keys), keys)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//The following history query for "marble12" should return 3 records
f = "history"
args = util.ToChaincodeArgs(f, "marble12")
//Reset the query limit to 5
viper.Set("ledger.state.queryLimit", 5)

//The following range query for "marble01" to "marble11" should return 5 marbles due to the queryLimit
f = "keys"
args = util.ToChaincodeArgs(f, "marble001", "marble011")

spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, retval, err := invoke(ctxt, chainID, spec, nextBlockNumber, nil)
Expand All @@ -1303,22 +1186,21 @@ func TestQueries(t *testing.T) {
return
}

var history []interface{}
err = json.Unmarshal(retval, &history)
//unmarshal the results
err = json.Unmarshal(retval, &keys)

//default query limit of 10000 is used, query should return all records that meet the criteria
if len(history) != 3 {
//check to see if there are 5 values
if len(keys) != 5 {
t.Fail()
t.Logf("Error detected with the history query, should have returned 3 but returned %v", len(keys))
t.Logf("Error detected with the range query, should have returned 5 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

}

if ledgerconfig.IsCouchDBEnabled() == true {
//Reset the query limit to 10000
viper.Set("ledger.state.queryLimit", 10000)

//The following rich query for should return 9 marbles
//The following rich query for should return 50 marbles
f = "query"
args = util.ToChaincodeArgs(f, "{\"selector\":{\"owner\":\"jerry\"}}")

Expand All @@ -1336,9 +1218,9 @@ func TestQueries(t *testing.T) {
//unmarshal the results
err = json.Unmarshal(retval, &keys)

//check to see if there are 9 values
//check to see if there are 50 values
//default query limit of 10000 is used, this query is effectively unlimited
if len(keys) != 9 {
if len(keys) != 50 {
t.Fail()
t.Logf("Error detected with the rich query, should have returned 9 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
Expand All @@ -1354,7 +1236,7 @@ func TestQueries(t *testing.T) {

spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)

nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
Expand All @@ -1375,6 +1257,53 @@ func TestQueries(t *testing.T) {

}

// modifications for history query
f = "put"
args = util.ToChaincodeArgs(f, "marble012", "{\"docType\":\"marble\",\"name\":\"marble012\",\"color\":\"red\",\"size\":30,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble012", "{\"docType\":\"marble\",\"name\":\"marble012\",\"color\":\"red\",\"size\":30,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//The following history query for "marble12" should return 3 records
f = "history"
args = util.ToChaincodeArgs(f, "marble012")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber, nil)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

var history []interface{}
err = json.Unmarshal(retval, &history)
if len(history) != 3 {
t.Fail()
t.Logf("Error detected with the history query, should have returned 3 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
}

Expand Down

0 comments on commit 868a3e9

Please sign in to comment.