Skip to content

Commit 1dc96ee

Browse files
author
Chris Elder
committed
[FAB-6175] Add index management to couchdb layer
Add the ability to create/update, delete and list indexes to the couchdb layer. Change-Id: Ia79243a6aff4665cca0f3d144ea24cdd8f312fc4 Signed-off-by: Chris Elder <chris.elder@us.ibm.com>
1 parent 3e9b686 commit 1dc96ee

File tree

2 files changed

+324
-0
lines changed

2 files changed

+324
-0
lines changed

‎core/ledger/util/couchdb/couchdb.go

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,27 @@ type Base64Attachment struct {
202202
AttachmentData string `json:"data"`
203203
}
204204

205+
//ListIndexResponse contains the definition for listing couchdb indexes
206+
type ListIndexResponse struct {
207+
TotalRows int `json:"total_rows"`
208+
Indexes []IndexDefinition `json:"indexes"`
209+
}
210+
211+
//IndexDefinition contains the definition for a couchdb index
212+
type IndexDefinition struct {
213+
DesignDocument string `json:"ddoc"`
214+
Name string `json:"name"`
215+
Type string `json:"type"`
216+
Definition json.RawMessage `json:"def"`
217+
}
218+
219+
//IndexResult contains the definition for a couchdb index
220+
type IndexResult struct {
221+
DesignDocument string `json:"designdoc"`
222+
Name string `json:"name"`
223+
Definition string `json:"definition"`
224+
}
225+
205226
// closeResponseBody discards the body and then closes it to enable returning it to
206227
// connection pool
207228
func closeResponseBody(resp *http.Response) {
@@ -1006,6 +1027,122 @@ func (dbclient *CouchDatabase) QueryDocuments(query string) (*[]QueryResult, err
10061027

10071028
}
10081029

1030+
// ListIndex method lists the defined indexes for a database
1031+
func (dbclient *CouchDatabase) ListIndex() (*[]IndexResult, error) {
1032+
1033+
logger.Debugf("Entering ListIndex()")
1034+
1035+
indexURL, err := url.Parse(dbclient.CouchInstance.conf.URL)
1036+
if err != nil {
1037+
logger.Errorf("URL parse error: %s", err.Error())
1038+
return nil, err
1039+
}
1040+
1041+
indexURL.Path = dbclient.DBName + "/_index/"
1042+
1043+
//get the number of retries
1044+
maxRetries := dbclient.CouchInstance.conf.MaxRetries
1045+
1046+
resp, _, err := dbclient.CouchInstance.handleRequest(http.MethodGet, indexURL.String(), nil, "", "", maxRetries, true)
1047+
if err != nil {
1048+
return nil, err
1049+
}
1050+
defer closeResponseBody(resp)
1051+
1052+
//handle as JSON document
1053+
jsonResponseRaw, err := ioutil.ReadAll(resp.Body)
1054+
if err != nil {
1055+
return nil, err
1056+
}
1057+
1058+
var jsonResponse = &ListIndexResponse{}
1059+
1060+
err2 := json.Unmarshal(jsonResponseRaw, &jsonResponse)
1061+
if err2 != nil {
1062+
return nil, err2
1063+
}
1064+
1065+
var results []IndexResult
1066+
1067+
for _, row := range jsonResponse.Indexes {
1068+
1069+
//if the DesignDocument does not begin with "_design/", then this is a system
1070+
//level index and is not meaningful and cannot be edited or deleted
1071+
designDoc := row.DesignDocument
1072+
s := strings.SplitAfterN(designDoc, "_design/", 2)
1073+
if len(s) > 1 {
1074+
designDoc = s[1]
1075+
1076+
//Add the index definition to the results
1077+
var addIndexResult = &IndexResult{DesignDocument: designDoc, Name: row.Name, Definition: fmt.Sprintf("%s", row.Definition)}
1078+
results = append(results, *addIndexResult)
1079+
}
1080+
1081+
}
1082+
1083+
logger.Debugf("Exiting ListIndex()")
1084+
1085+
return &results, nil
1086+
1087+
}
1088+
1089+
// CreateIndex method provides a function creating an index
1090+
func (dbclient *CouchDatabase) CreateIndex(indexdefinition string) error {
1091+
1092+
logger.Debugf("Entering CreateIndex() indexdefinition=%s", indexdefinition)
1093+
1094+
//Test to see if this is a valid JSON
1095+
if IsJSON(indexdefinition) != true {
1096+
return fmt.Errorf("JSON format is not valid")
1097+
}
1098+
1099+
indexURL, err := url.Parse(dbclient.CouchInstance.conf.URL)
1100+
if err != nil {
1101+
logger.Errorf("URL parse error: %s", err.Error())
1102+
return err
1103+
}
1104+
1105+
indexURL.Path = dbclient.DBName + "/_index"
1106+
1107+
//get the number of retries
1108+
maxRetries := dbclient.CouchInstance.conf.MaxRetries
1109+
1110+
resp, _, err := dbclient.CouchInstance.handleRequest(http.MethodPost, indexURL.String(), []byte(indexdefinition), "", "", maxRetries, true)
1111+
if err != nil {
1112+
return err
1113+
}
1114+
defer closeResponseBody(resp)
1115+
1116+
return nil
1117+
1118+
}
1119+
1120+
// DeleteIndex method provides a function deleting an index
1121+
func (dbclient *CouchDatabase) DeleteIndex(designdoc, indexname string) error {
1122+
1123+
logger.Debugf("Entering DeleteIndex() designdoc=%s indexname=%s", designdoc, indexname)
1124+
1125+
indexURL, err := url.Parse(dbclient.CouchInstance.conf.URL)
1126+
if err != nil {
1127+
logger.Errorf("URL parse error: %s", err.Error())
1128+
return err
1129+
}
1130+
1131+
indexURL.Path = dbclient.DBName + "/_index/" + designdoc + "/json/" + indexname
1132+
1133+
//get the number of retries
1134+
maxRetries := dbclient.CouchInstance.conf.MaxRetries
1135+
1136+
resp, _, err := dbclient.CouchInstance.handleRequest(http.MethodDelete, indexURL.String(), nil, "", "", maxRetries, true)
1137+
if err != nil {
1138+
return err
1139+
}
1140+
defer closeResponseBody(resp)
1141+
1142+
return nil
1143+
1144+
}
1145+
10091146
//BatchRetrieveDocumentMetadata - batch method to retrieve document metadata for a set of keys,
10101147
// including ID, couchdb revision number, and ledger version
10111148
func (dbclient *CouchDatabase) BatchRetrieveDocumentMetadata(keys []string) ([]*DocMetadata, error) {

‎core/ledger/util/couchdb/couchdb_test.go

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/hyperledger/fabric/common/ledger/testutil"
3030
"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
3131
ledgertestutil "github.com/hyperledger/fabric/core/ledger/testutil"
32+
logging "github.com/op/go-logging"
3233
"github.com/spf13/viper"
3334
)
3435

@@ -82,6 +83,11 @@ func TestMain(m *testing.M) {
8283
viper.Set("ledger.state.couchDBConfig.maxRetriesOnStartup", 10)
8384
viper.Set("ledger.state.couchDBConfig.requestTimeout", time.Second*35)
8485

86+
//set the logging level to DEBUG to test debug only code
87+
logging.SetLevel(logging.DEBUG, "couchdb")
88+
89+
viper.Set("logging.peer", "debug")
90+
8591
// Create CouchDB definition from config parameters
8692
couchDBDef = GetCouchDBDefinition()
8793

@@ -205,6 +211,18 @@ func TestBadCouchDBInstance(t *testing.T) {
205211
_, err = badDB.BatchUpdateDocuments(nil)
206212
testutil.AssertError(t, err, "Error should have been thrown with BatchUpdateDocuments and invalid connection")
207213

214+
//Test ListIndex with bad connection
215+
_, err = badDB.ListIndex()
216+
testutil.AssertError(t, err, "Error should have been thrown with ListIndex and invalid connection")
217+
218+
//Test CreateIndex with bad connection
219+
err = badDB.CreateIndex("")
220+
testutil.AssertError(t, err, "Error should have been thrown with CreateIndex and invalid connection")
221+
222+
//Test DeleteIndex with bad connection
223+
err = badDB.DeleteIndex("", "")
224+
testutil.AssertError(t, err, "Error should have been thrown with DeleteIndex and invalid connection")
225+
208226
}
209227

210228
func TestDBCreateSaveWithoutRevision(t *testing.T) {
@@ -316,6 +334,27 @@ func TestDBBadConnection(t *testing.T) {
316334
}
317335
}
318336

337+
func TestBadDBCredentials(t *testing.T) {
338+
339+
if ledgerconfig.IsCouchDBEnabled() {
340+
341+
database := "testdbbadcredentials"
342+
err := cleanup(database)
343+
testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to cleanup Error: %s", err))
344+
defer cleanup(database)
345+
346+
if err == nil {
347+
//create a new instance and database object
348+
_, err := CreateCouchInstance(couchDBDef.URL, "fred", "fred",
349+
couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout)
350+
testutil.AssertError(t, err, fmt.Sprintf("Error should have been thrown for bad credentials"))
351+
352+
}
353+
354+
}
355+
356+
}
357+
319358
func TestDBCreateDatabaseAndPersist(t *testing.T) {
320359

321360
if ledgerconfig.IsCouchDBEnabled() {
@@ -862,6 +901,154 @@ func TestCouchDBVersion(t *testing.T) {
862901

863902
}
864903

904+
func TestIndexOperations(t *testing.T) {
905+
906+
if ledgerconfig.IsCouchDBEnabled() {
907+
908+
database := "testindexoperations"
909+
err := cleanup(database)
910+
testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to cleanup Error: %s", err))
911+
defer cleanup(database)
912+
913+
byteJSON1 := []byte(`{"_id":"1", "asset_name":"marble1","color":"blue","size":1,"owner":"jerry"}`)
914+
byteJSON2 := []byte(`{"_id":"2", "asset_name":"marble2","color":"red","size":2,"owner":"tom"}`)
915+
byteJSON3 := []byte(`{"_id":"3", "asset_name":"marble3","color":"green","size":3,"owner":"jerry"}`)
916+
byteJSON4 := []byte(`{"_id":"4", "asset_name":"marble4","color":"purple","size":4,"owner":"tom"}`)
917+
byteJSON5 := []byte(`{"_id":"5", "asset_name":"marble5","color":"blue","size":5,"owner":"jerry"}`)
918+
byteJSON6 := []byte(`{"_id":"6", "asset_name":"marble6","color":"white","size":6,"owner":"tom"}`)
919+
byteJSON7 := []byte(`{"_id":"7", "asset_name":"marble7","color":"white","size":7,"owner":"tom"}`)
920+
byteJSON8 := []byte(`{"_id":"8", "asset_name":"marble8","color":"white","size":8,"owner":"tom"}`)
921+
byteJSON9 := []byte(`{"_id":"9", "asset_name":"marble9","color":"white","size":9,"owner":"tom"}`)
922+
byteJSON10 := []byte(`{"_id":"10", "asset_name":"marble10","color":"white","size":10,"owner":"tom"}`)
923+
924+
//create a new instance and database object --------------------------------------------------------
925+
couchInstance, err := CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password,
926+
couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout)
927+
testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance"))
928+
db := CouchDatabase{CouchInstance: *couchInstance, DBName: database}
929+
930+
//create a new database
931+
_, errdb := db.CreateDatabaseIfNotExist()
932+
testutil.AssertNoError(t, errdb, fmt.Sprintf("Error when trying to create database"))
933+
934+
batchUpdateDocs := []*CouchDoc{}
935+
936+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON1, Attachments: nil})
937+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON2, Attachments: nil})
938+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON3, Attachments: nil})
939+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON4, Attachments: nil})
940+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON5, Attachments: nil})
941+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON6, Attachments: nil})
942+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON7, Attachments: nil})
943+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON8, Attachments: nil})
944+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON9, Attachments: nil})
945+
batchUpdateDocs = append(batchUpdateDocs, &CouchDoc{JSONValue: byteJSON10, Attachments: nil})
946+
947+
_, err = db.BatchUpdateDocuments(batchUpdateDocs)
948+
testutil.AssertNoError(t, err, fmt.Sprintf("Error adding batch of documents"))
949+
950+
//Create an index definition
951+
indexDefSize := "{\"index\":{\"fields\":[{\"size\":\"desc\"}]},\"ddoc\":\"indexSizeSortDoc\", \"name\":\"indexSizeSortName\",\"type\":\"json\"}"
952+
953+
//Create the index
954+
err = db.CreateIndex(indexDefSize)
955+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))
956+
957+
//Retrieve the index
958+
listResult, err := db.ListIndex()
959+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))
960+
961+
//There should only be one item returned
962+
testutil.AssertEquals(t, len(*listResult), 1)
963+
for _, elem := range *listResult {
964+
testutil.AssertEquals(t, elem.DesignDocument, "indexSizeSortDoc")
965+
testutil.AssertEquals(t, elem.Name, "indexSizeSortName")
966+
testutil.AssertEquals(t, elem.Definition, "{\"fields\":[{\"size\":\"desc\"}]}")
967+
}
968+
969+
//Create an index definition with no DesignDocument or name
970+
indexDefColor := "{\"index\":{\"fields\":[{\"color\":\"desc\"}]}}"
971+
972+
//Create the index
973+
err = db.CreateIndex(indexDefColor)
974+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))
975+
976+
//Retrieve the list of indexes
977+
listResult, err = db.ListIndex()
978+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))
979+
980+
//There should be two indexes returned
981+
testutil.AssertEquals(t, len(*listResult), 2)
982+
983+
//Delete the named index
984+
err = db.DeleteIndex("indexSizeSortDoc", "indexSizeSortName")
985+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while deleting an index"))
986+
987+
//Retrieve the list of indexes
988+
listResult, err = db.ListIndex()
989+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))
990+
991+
//There should be one index returned
992+
testutil.AssertEquals(t, len(*listResult), 1)
993+
994+
//Delete the unnamed index
995+
for _, elem := range *listResult {
996+
err = db.DeleteIndex(elem.DesignDocument, string(elem.Name))
997+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while deleting an index"))
998+
}
999+
1000+
//Retrieve the list of indexes, should be zero
1001+
listResult, err = db.ListIndex()
1002+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))
1003+
testutil.AssertEquals(t, len(*listResult), 0)
1004+
1005+
//Create a query string with a descending sort, this will require an index
1006+
queryString := "{\"selector\":{\"size\": {\"$gt\": 0}},\"fields\": [\"_id\", \"_rev\", \"owner\", \"asset_name\", \"color\", \"size\"], \"sort\":[{\"size\":\"desc\"}], \"limit\": 10,\"skip\": 0}"
1007+
1008+
//Execute a query with a sort, this should throw the exception
1009+
_, err = db.QueryDocuments(queryString)
1010+
testutil.AssertError(t, err, fmt.Sprintf("Error thrown while querying without a valid index"))
1011+
1012+
//Create the index
1013+
err = db.CreateIndex(indexDefSize)
1014+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))
1015+
1016+
//Execute a query with an index, this should succeed
1017+
_, err = db.QueryDocuments(queryString)
1018+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while querying with an index"))
1019+
1020+
//Create another index definition
1021+
indexDefSize = "{\"index\":{\"fields\":[{\"data.size\":\"desc\"},{\"data.owner\":\"desc\"}]},\"ddoc\":\"indexSizeOwnerSortDoc\", \"name\":\"indexSizeOwnerSortName\",\"type\":\"json\"}"
1022+
1023+
//Create the index
1024+
err = db.CreateIndex(indexDefSize)
1025+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while creating an index"))
1026+
1027+
//Retrieve the indexes
1028+
listResult, err = db.ListIndex()
1029+
testutil.AssertNoError(t, err, fmt.Sprintf("Error thrown while retrieving indexes"))
1030+
1031+
//There should only be two definitions
1032+
testutil.AssertEquals(t, len(*listResult), 2)
1033+
1034+
//Create an invalid index definition with an invalid JSON
1035+
indexDefSize = "{\"index\"{\"fields\":[{\"data.size\":\"desc\"},{\"data.owner\":\"desc\"}]},\"ddoc\":\"indexSizeOwnerSortDoc\", \"name\":\"indexSizeOwnerSortName\",\"type\":\"json\"}"
1036+
1037+
//Create the index
1038+
err = db.CreateIndex(indexDefSize)
1039+
testutil.AssertError(t, err, fmt.Sprintf("Error should have been thrown for an invalid index JSON"))
1040+
1041+
//Create an invalid index definition with a valid JSON and an invalid index definition
1042+
indexDefSize = "{\"index\":{\"fields2\":[{\"data.size\":\"desc\"},{\"data.owner\":\"desc\"}]},\"ddoc\":\"indexSizeOwnerSortDoc\", \"name\":\"indexSizeOwnerSortName\",\"type\":\"json\"}"
1043+
1044+
//Create the index
1045+
err = db.CreateIndex(indexDefSize)
1046+
testutil.AssertError(t, err, fmt.Sprintf("Error should have been thrown for an invalid index definition"))
1047+
1048+
}
1049+
1050+
}
1051+
8651052
func TestRichQuery(t *testing.T) {
8661053

8671054
if ledgerconfig.IsCouchDBEnabled() {

0 commit comments

Comments
 (0)