Skip to content

Commit

Permalink
automatically index objects in JSON array
Browse files Browse the repository at this point in the history
  • Loading branch information
@HouzuoGuo
HouzuoGuo committed Nov 11, 2013
1 parent 78a60b5 commit f88a0b8
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 14 deletions.
8 changes: 6 additions & 2 deletions db/col.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,16 +124,20 @@ func GetIn(doc interface{}, path []string) (ret []interface{}) {
log.Printf("%v cannot be indexed because type conversation to map[string]interface{} failed", doc)
}
var thing interface{} = docMap
for _, seg := range path {
for i, seg := range path {
if aMap, ok := thing.(map[string]interface{}); ok {
thing = aMap[seg]
} else if anArray, ok := thing.([]interface{}); ok {
for _, element := range anArray {
ret = append(ret, GetIn(element, path[i:])...)
}
} else {
return nil
}
}
switch thing.(type) {
case []interface{}:
return thing.([]interface{})
return append(ret, thing.([]interface{})...)
default:
return append(ret, thing)
}
Expand Down
55 changes: 52 additions & 3 deletions db/col_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,39 @@ import (

const COL_BENCH_SIZE = 200000 // Number of documents made available for collection benchmark

func TestGetIn(t *testing.T) {
var obj interface{}
// Get inside a JSON object
json.Unmarshal([]byte(`{"a": {"b": {"c": 1}}}`), &obj)
if val, ok := GetIn(obj, []string{"a", "b", "c"})[0].(float64); !ok || val != 1 {
t.Fatal()
}

// Get inside a JSON array
json.Unmarshal([]byte(`{"a": {"b": {"c": [1, 2, 3]}}}`), &obj)
if val, ok := GetIn(obj, []string{"a", "b", "c"})[0].(float64); !ok || val != 1 {
t.Fatal()
}
if val, ok := GetIn(obj, []string{"a", "b", "c"})[1].(float64); !ok || val != 2 {
t.Fatal()
}
if val, ok := GetIn(obj, []string{"a", "b", "c"})[2].(float64); !ok || val != 3 {
t.Fatal()
}

// Get inside JSON objects contained in JSON array
json.Unmarshal([]byte(`{"a": [{"b": {"c": [1]}}, {"b": {"c": [2, 3]}}]}`), &obj)
if val, ok := GetIn(obj, []string{"a", "b", "c"})[0].(float64); !ok || val != 1 {
t.Fatal()
}
if val, ok := GetIn(obj, []string{"a", "b", "c"})[1].(float64); !ok || val != 2 {
t.Fatal()
}
if val, ok := GetIn(obj, []string{"a", "b", "c"})[2].(float64); !ok || val != 3 {
t.Fatal()
}
}

func TestInsertRead(t *testing.T) {
tmp := "/tmp/tiedot_col_test"
os.RemoveAll(tmp)
Expand Down Expand Up @@ -240,16 +273,17 @@ func TestIndex(t *testing.T) {
defer col.Close()
docs := []string{
`{"a": {"b": {"c": 1}}, "d": 0}`,
`{"a": {"b": {"c": 2}}, "d": 0}`,
`{"a": {"b": {"c": 3}}, "d": 0}`,
`{"a": {"b": {"c": 4}}, "d": [0, 9]}`,
`{"a": {"b": [{"c": 2}]}, "d": 0}`,
`{"a": [{"b": {"c": 3}}], "d": 0}`,
`{"a": [{"b": {"c": [4]}}, {"b": {"c": [5, 6]}}], "d": [0, 9]}`,
`{"a": {"b": {"c": null}}, "d": null}`}
var jsonDoc [4]interface{}
json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
json.Unmarshal([]byte(docs[1]), &jsonDoc[1])
json.Unmarshal([]byte(docs[2]), &jsonDoc[2])
json.Unmarshal([]byte(docs[3]), &jsonDoc[3])
ids := [3]uint64{}
// Insert first document
ids[0], _ = col.Insert(jsonDoc[0])
if err = col.Index([]string{"a", "b", "c"}); err != nil {
t.Fatal(err)
Expand All @@ -259,13 +293,16 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
return
}
// There should be one document on index - the first doc
keys, vals := col.StrHT["a,b,c"].GetAll(0)
if !(len(keys) == 1 && len(vals) == 1 && vals[0] == ids[0]) {
t.Fatalf("Did not index existing document, got %v, %v", keys, vals)
}
// Insert second and third document, replace third document by fouth document
ids[1], _ = col.Insert(jsonDoc[1])
ids[2], _ = col.Insert(jsonDoc[2])
ids[2], _ = col.Update(ids[2], jsonDoc[3])
// Then remove second document
col.Delete(ids[1])
// jsonDoc[0,3], ids[0, 2] are the ones left
index1 := col.StrHT["a,b,c"]
Expand All @@ -290,12 +327,24 @@ func TestIndex(t *testing.T) {
k4, v4 := index1.Get(StrHash(4), 0, func(k, v uint64) bool {
return true
})
k5, v5 := index1.Get(StrHash(5), 0, func(k, v uint64) bool {
return true
})
k6, v6 := index1.Get(StrHash(6), 0, func(k, v uint64) bool {
return true
})
if !(len(k1) == 1 && len(v1) == 1 && k1[0] == StrHash(1) && v1[0] == ids[0]) {
t.Fatalf("Index fault, %v, %v", k1, v1)
}
if !(len(k4) == 1 && len(v4) == 1 && k4[0] == StrHash(4) && v4[0] == ids[2]) {
t.Fatalf("Index fault, %v, %v", k4, v4)
}
if !(len(k5) == 1 && len(v5) == 1 && k5[0] == StrHash(5) && v5[0] == ids[2]) {
t.Fatalf("Index fault, %v, %v", k5, v5)
}
if !(len(k6) == 1 && len(v6) == 1 && k6[0] == StrHash(6) && v6[0] == ids[2]) {
t.Fatalf("Index fault, %v, %v", k6, v6)
}
// now remove a,b,c index
if err = col.Unindex([]string{"a", "b", "c"}); err != nil {
t.Fatal(err)
Expand Down
32 changes: 23 additions & 9 deletions db/queryv2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ func TestQueryV2(t *testing.T) {
docs := []string{
`{"a": {"b": [1]}, "c": 1, "d": 1, "f": 1, "g": 1, "special": {"thing": null} }`,
`{"a": {"b": 1}, "c": [1], "d": 2, "f": 2, "g": 2}`,
`{"a": {"b": [2]}, "c": 2, "d": 1, "f": 3, "g": 3}`,
`{"a": [{"b": [2]}], "c": 2, "d": 1, "f": 3, "g": 3}`,
`{"a": {"b": 3}, "c": [3], "d": 2, "f": 4, "g": 4}`,
`{"a": {"b": [4]}, "c": 4, "d": 1, "f": 5, "g": 5}`}
ids := [5]uint64{}
`{"a": {"b": [4]}, "c": 4, "d": 1, "f": 5, "g": 5}`,
`{"a": [{"b": 5}, {"b": 6}], "c": 4, "d": 1, "f": 5, "g": 5}`}
ids := [6]uint64{}
for i, doc := range docs {
var jsonDoc interface{}
json.Unmarshal([]byte(doc), &jsonDoc)
Expand All @@ -60,6 +61,20 @@ func TestQueryV2(t *testing.T) {
if !ensureMapHasKeys(q, ids[0], ids[1]) {
t.Fatal(q)
}
q, err = runQueryV2(`{"eq": 5, "in": ["a", "b"]}`, col)
if err != nil {
t.Fatal(err)
}
if !ensureMapHasKeys(q, ids[5]) {
t.Fatal(q)
}
q, err = runQueryV2(`{"eq": 6, "in": ["a", "b"]}`, col)
if err != nil {
t.Fatal(err)
}
if !ensureMapHasKeys(q, ids[5]) {
t.Fatal(q)
}
q, err = runQueryV2(`{"eq": 1, "limit": 1, "in": ["a", "b"]}`, col)
if err != nil {
fmt.Println(err)
Expand Down Expand Up @@ -87,7 +102,7 @@ func TestQueryV2(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !ensureMapHasKeys(q, ids[0], ids[1], ids[2], ids[3], ids[4]) {
if !ensureMapHasKeys(q, ids[0], ids[1], ids[2], ids[3], ids[4], ids[5]) {
t.Fatal(q)
}
// union
Expand Down Expand Up @@ -196,30 +211,29 @@ func TestQueryV2(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !ensureMapHasKeys(q, ids[4], ids[3], ids[2], ids[1], ids[0]) {
if !ensureMapHasKeys(q, ids[5], ids[4], ids[3], ids[2], ids[1], ids[0]) {
t.Fatal(q)
}
q, err = runQueryV2(`{"int-from": 10, "int-to": 0, "in": ["f"], "limit": 2}`, col)
if err != nil {
t.Fatal(err)
}
if !ensureMapHasKeys(q, ids[4], ids[3]) {
if !ensureMapHasKeys(q, ids[5], ids[4]) {
t.Fatal(q)
}
// regexes
q, err = runQueryV2(`{"re": "^[0-9]*$", "in": ["f"]}`, col)
if err != nil {
t.Fatal(err)
}
if !ensureMapHasKeys(q, ids[0], ids[1], ids[2], ids[3], ids[4]) {
fmt.Printf("%+v\n", q)
if !ensureMapHasKeys(q, ids[0], ids[1], ids[2], ids[3], ids[4], ids[5]) {
t.Fatal(q)
}
q, err = runQueryV2(`{"re": ".*", "in": ["a"]}`, col)
if err != nil {
t.Fatal(err)
}
if !ensureMapHasKeys(q, ids[0], ids[1], ids[2], ids[3], ids[4]) {
if !ensureMapHasKeys(q, ids[0], ids[1], ids[2], ids[3], ids[4], ids[5]) {
fmt.Printf("%+v\n", q)
t.Fatal(q)
}
Expand Down

0 comments on commit f88a0b8

Please sign in to comment.