Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

expression: remove duplicate element in json_extract function result #37118

Merged
merged 6 commits into from Aug 16, 2022
Merged

expression: remove duplicate element in json_extract function result #37118

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4a41d93
fix dup element
xiongjiwei Aug 16, 2022
c3f96b3
nogo
xiongjiwei Aug 16, 2022
488aed3
Merge branch 'master' into json-dup
ti-chi-bot Aug 16, 2022
dac832b
Merge branch 'master' into json-dup
hawkingrei Aug 16, 2022
3f9b8b8
Merge branch 'master' into json-dup
hawkingrei Aug 16, 2022
363ed8c
Merge branch 'master' into json-dup
ti-chi-bot Aug 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions build/nogo_config.json
View file
Open in desktop
Expand Up @@ -154,6 +154,8 @@
".*_generated\\.go$": "ignore generated code"
},
"only_files": {
"types/json/binary_functions.go": "types/json/binary_functions.go",
"types/json/binary_test.go": "types/json/binary_test.go",
"ddl/backfilling.go": "ddl/backfilling.go",
"ddl/column.go": "ddl/column.go",
"ddl/index.go": "ddl/index.go",
Expand Down Expand Up @@ -286,6 +288,8 @@
".*_generated\\.go$": "ignore generated code"
},
"only_files": {
"types/json/binary_functions.go": "types/json/binary_functions.go",
"types/json/binary_test.go": "types/json/binary_test.go",
"ddl/backfilling.go": "ddl/backfilling.go",
"ddl/column.go": "ddl/column.go",
"ddl/index.go": "ddl/index.go",
Expand Down Expand Up @@ -644,6 +648,8 @@
".*_generated\\.go$": "ignore generated code"
},
"only_files": {
"types/json/binary_functions.go": "types/json/binary_functions.go",
"types/json/binary_test.go": "types/json/binary_test.go",
"ddl/backfilling.go": "ddl/backfilling.go",
"ddl/column.go": "ddl/column.go",
"ddl/index.go": "ddl/index.go",
Expand Down
62 changes: 36 additions & 26 deletions types/json/binary_functions.go
View file
Open in desktop
Expand Up @@ -214,8 +214,9 @@ func quoteString(s string) string {
func (bj BinaryJSON) Extract(pathExprList []PathExpression) (ret BinaryJSON, found bool) {
buf := make([]BinaryJSON, 0, 1)
for _, pathExpr := range pathExprList {
buf = bj.extractTo(buf, pathExpr)
buf = bj.extractTo(buf, pathExpr, make(map[*byte]struct{}), false)
}

if len(buf) == 0 {
found = false
} else if len(pathExprList) == 1 && len(buf) == 1 {
Expand All @@ -234,55 +235,70 @@ func (bj BinaryJSON) Extract(pathExprList []PathExpression) (ret BinaryJSON, fou
return
}

func (bj BinaryJSON) extractTo(buf []BinaryJSON, pathExpr PathExpression) []BinaryJSON {
func (bj BinaryJSON) extractOne(pathExpr PathExpression) []BinaryJSON {
result := make([]BinaryJSON, 0, 1)
return bj.extractTo(result, pathExpr, nil, true)
}

func (bj BinaryJSON) extractTo(buf []BinaryJSON, pathExpr PathExpression, dup map[*byte]struct{}, one bool) []BinaryJSON {
if len(pathExpr.legs) == 0 {
if dup != nil {
if _, exists := dup[&bj.Value[0]]; exists {
return buf
}
dup[&bj.Value[0]] = struct{}{}
}
return append(buf, bj)
}
currentLeg, subPathExpr := pathExpr.popOneLeg()
if currentLeg.typ == pathLegIndex {
if bj.TypeCode != TypeCodeArray {
if currentLeg.arrayIndex <= 0 && currentLeg.arrayIndex != arrayIndexAsterisk {
buf = bj.extractTo(buf, subPathExpr)
buf = bj.extractTo(buf, subPathExpr, dup, one)
}
return buf
}
elemCount := bj.GetElemCount()
if currentLeg.arrayIndex == arrayIndexAsterisk {
for i := 0; i < elemCount; i++ {
buf = bj.arrayGetElem(i).extractTo(buf, subPathExpr)
for i := 0; i < elemCount && !finished(buf, one); i++ {
buf = bj.arrayGetElem(i).extractTo(buf, subPathExpr, dup, one)
}
} else if currentLeg.arrayIndex < elemCount {
buf = bj.arrayGetElem(currentLeg.arrayIndex).extractTo(buf, subPathExpr)
buf = bj.arrayGetElem(currentLeg.arrayIndex).extractTo(buf, subPathExpr, dup, one)
}
} else if currentLeg.typ == pathLegKey && bj.TypeCode == TypeCodeObject {
elemCount := bj.GetElemCount()
if currentLeg.dotKey == "*" {
for i := 0; i < elemCount; i++ {
buf = bj.objectGetVal(i).extractTo(buf, subPathExpr)
for i := 0; i < elemCount && !finished(buf, one); i++ {
buf = bj.objectGetVal(i).extractTo(buf, subPathExpr, dup, one)
}
} else {
child, ok := bj.objectSearchKey(hack.Slice(currentLeg.dotKey))
if ok {
buf = child.extractTo(buf, subPathExpr)
buf = child.extractTo(buf, subPathExpr, dup, one)
}
}
} else if currentLeg.typ == pathLegDoubleAsterisk {
buf = bj.extractTo(buf, subPathExpr)
buf = bj.extractTo(buf, subPathExpr, dup, one)
if bj.TypeCode == TypeCodeArray {
elemCount := bj.GetElemCount()
for i := 0; i < elemCount; i++ {
buf = bj.arrayGetElem(i).extractTo(buf, pathExpr)
for i := 0; i < elemCount && !finished(buf, one); i++ {
buf = bj.arrayGetElem(i).extractTo(buf, pathExpr, dup, one)
}
} else if bj.TypeCode == TypeCodeObject {
elemCount := bj.GetElemCount()
for i := 0; i < elemCount; i++ {
buf = bj.objectGetVal(i).extractTo(buf, pathExpr)
for i := 0; i < elemCount && !finished(buf, one); i++ {
buf = bj.objectGetVal(i).extractTo(buf, pathExpr, dup, one)
}
}
}
return buf
}

func finished(buf []BinaryJSON, one bool) bool {
return one && len(buf) > 0
}

func (bj BinaryJSON) objectSearchKey(key []byte) (BinaryJSON, bool) {
elemCount := bj.GetElemCount()
idx := sort.Search(elemCount, func(i int) bool {
Expand Down Expand Up @@ -448,8 +464,7 @@ type binaryModifier struct {
}

func (bm *binaryModifier) set(path PathExpression, newBj BinaryJSON) BinaryJSON {
result := make([]BinaryJSON, 0, 1)
result = bm.bj.extractTo(result, path)
result := bm.bj.extractOne(path)
if len(result) > 0 {
bm.modifyPtr = &result[0].Value[0]
bm.modifyValue = newBj
Expand All @@ -463,8 +478,7 @@ func (bm *binaryModifier) set(path PathExpression, newBj BinaryJSON) BinaryJSON
}

func (bm *binaryModifier) replace(path PathExpression, newBj BinaryJSON) BinaryJSON {
result := make([]BinaryJSON, 0, 1)
result = bm.bj.extractTo(result, path)
result := bm.bj.extractOne(path)
if len(result) == 0 {
return bm.bj
}
Expand All @@ -474,8 +488,7 @@ func (bm *binaryModifier) replace(path PathExpression, newBj BinaryJSON) BinaryJ
}

func (bm *binaryModifier) insert(path PathExpression, newBj BinaryJSON) BinaryJSON {
result := make([]BinaryJSON, 0, 1)
result = bm.bj.extractTo(result, path)
result := bm.bj.extractOne(path)
if len(result) > 0 {
return bm.bj
}
Expand All @@ -489,8 +502,7 @@ func (bm *binaryModifier) insert(path PathExpression, newBj BinaryJSON) BinaryJS
// doInsert inserts the newBj to its parent, and builds the new parent.
func (bm *binaryModifier) doInsert(path PathExpression, newBj BinaryJSON) {
parentPath, lastLeg := path.popOneLastLeg()
result := make([]BinaryJSON, 0, 1)
result = bm.bj.extractTo(result, parentPath)
result := bm.bj.extractOne(parentPath)
if len(result) == 0 {
return
}
Expand Down Expand Up @@ -537,8 +549,7 @@ func (bm *binaryModifier) doInsert(path PathExpression, newBj BinaryJSON) {
}

func (bm *binaryModifier) remove(path PathExpression) BinaryJSON {
result := make([]BinaryJSON, 0, 1)
result = bm.bj.extractTo(result, path)
result := bm.bj.extractOne(path)
if len(result) == 0 {
return bm.bj
}
Expand All @@ -551,8 +562,7 @@ func (bm *binaryModifier) remove(path PathExpression) BinaryJSON {

func (bm *binaryModifier) doRemove(path PathExpression) {
parentPath, lastLeg := path.popOneLastLeg()
result := make([]BinaryJSON, 0, 1)
result = bm.bj.extractTo(result, parentPath)
result := bm.bj.extractOne(parentPath)
if len(result) == 0 {
return
}
Expand Down
8 changes: 8 additions & 0 deletions types/json/binary_test.go
View file
Open in desktop
Expand Up @@ -44,6 +44,10 @@ func TestBinaryJSONExtract(t *testing.T) {
bj4 := mustParseBinaryFromString(t, `{"properties": {"$type$type": {"$a$a" : "TiDB"}}}`)
bj5 := mustParseBinaryFromString(t, `{"properties": {"$type": {"$a" : {"$b" : "TiDB"}}}}`)
bj6 := mustParseBinaryFromString(t, `{"properties": {"$type": {"$a$a" : "TiDB"}},"hello": {"$b$b": "world","$c": "amazing"}}`)
bj7 := mustParseBinaryFromString(t, `{ "a": { "x" : { "b": { "y": { "b": { "z": { "c": 100 } } } } } } }`)
bj8 := mustParseBinaryFromString(t, `{ "a": { "b" : [ 1, 2, 3 ] } }`)
bj9 := mustParseBinaryFromString(t, `[[0,1],[2,3],[4,[5,6]]]`)
bj10 := mustParseBinaryFromString(t, `[1]`)

var tests = []struct {
bj BinaryJSON
Expand Down Expand Up @@ -74,6 +78,10 @@ func TestBinaryJSONExtract(t *testing.T) {
{bj2, []string{"$.a", "$[0]"}, mustParseBinaryFromString(t, `[{"a": 1, "b": true}]`), true, nil},
{bj6, []string{"$.properties", "$[1]"}, mustParseBinaryFromString(t, `[{"$type": {"$a$a" : "TiDB"}}]`), true, nil},
{bj6, []string{"$.hello", "$[2]"}, mustParseBinaryFromString(t, `[{"$b$b": "world","$c": "amazing"}]`), true, nil},
{bj7, []string{"$.a**.b**.c"}, mustParseBinaryFromString(t, `[100]`), true, nil},
{bj8, []string{"$**[0]"}, mustParseBinaryFromString(t, `[{"a": {"b": [1, 2, 3]}}, {"b": [1, 2, 3]}, 1, 2, 3]`), true, nil},
{bj9, []string{"$**[0]"}, mustParseBinaryFromString(t, `[[0, 1], 0, 1, 2, 3, 4, 5, 6] `), true, nil},
{bj10, []string{"$**[0]"}, mustParseBinaryFromString(t, `[1]`), true, nil},
}

for _, test := range tests {
Expand Down