From 9e99f960ab325f1677fbdcf5171bec712870b60a Mon Sep 17 00:00:00 2001 From: ThreadDao Date: Thu, 13 Jun 2024 14:43:57 +0800 Subject: [PATCH] test: add cases for go milvusclient delete (#33777) issue: #33419 Signed-off-by: ThreadDao --- tests/go_client/common/utils.go | 32 + tests/go_client/go.mod | 2 +- tests/go_client/go.sum | 23 + tests/go_client/testcases/collection_test.go | 4 +- tests/go_client/testcases/delete_test.go | 558 ++++++++++++++++++ .../go_client/testcases/helper/data_helper.go | 16 +- .../testcases/helper/field_helper.go | 70 ++- tests/go_client/testcases/helper/helper.go | 15 +- .../testcases/helper/index_helper.go | 7 +- .../go_client/testcases/helper/read_helper.go | 4 +- tests/go_client/testcases/insert_test.go | 2 +- 11 files changed, 712 insertions(+), 21 deletions(-) create mode 100644 tests/go_client/testcases/delete_test.go diff --git a/tests/go_client/common/utils.go b/tests/go_client/common/utils.go index 9de777374453..97949bb00262 100644 --- a/tests/go_client/common/utils.go +++ b/tests/go_client/common/utils.go @@ -121,3 +121,35 @@ func GenSparseVector(maxLen int) entity.SparseEmbedding { } return vector } + +// InvalidExprStruct invalid expr +type InvalidExprStruct struct { + Expr string + ErrNil bool + ErrMsg string +} + +var InvalidExpressions = []InvalidExprStruct{ + {Expr: "id in [0]", ErrNil: true, ErrMsg: "fieldName(id) not found"}, // not exist field but no error + {Expr: "int64 in not [0]", ErrNil: false, ErrMsg: "cannot parse expression"}, // wrong term expr keyword + {Expr: "int64 > 10 AND int64 < 100", ErrNil: false, ErrMsg: "cannot parse expression"}, // AND isn't supported + {Expr: "int64 < 10 OR int64 > 100", ErrNil: false, ErrMsg: "cannot parse expression"}, // OR isn't supported + {Expr: "int64 < floatVec", ErrNil: false, ErrMsg: "not supported"}, // unsupported compare field + {Expr: "floatVec in [0]", ErrNil: false, ErrMsg: "cannot be casted to FloatVector"}, // value and field type mismatch + {Expr: fmt.Sprintf("%s == 1", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // hist empty + {Expr: fmt.Sprintf("%s like 'a%%' ", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // hist empty + {Expr: fmt.Sprintf("%s like `a%%` ", DefaultJSONFieldName), ErrNil: false, ErrMsg: "cannot parse expression"}, // `` + {Expr: fmt.Sprintf("%s > 1", DefaultDynamicFieldName), ErrNil: true, ErrMsg: ""}, // hits empty + {Expr: fmt.Sprintf("%s[\"dynamicList\"] == [2, 3]", DefaultDynamicFieldName), ErrNil: true, ErrMsg: ""}, + {Expr: fmt.Sprintf("%s['a'] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // json field not exist + {Expr: fmt.Sprintf("%s['number'] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // field exist but type not match + {Expr: fmt.Sprintf("%s[0] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // field exist but type not match + {Expr: fmt.Sprintf("json_contains (%s['number'], 2)", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, + {Expr: fmt.Sprintf("json_contains (%s['list'], [2])", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, + {Expr: fmt.Sprintf("json_contains_all (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_all operation element must be an array"}, + {Expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_any operation element must be an array"}, + {Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"}, + {Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"}, + {Expr: fmt.Sprintf("%s[-1] > %d", DefaultInt8ArrayField, TestCapacity), ErrNil: false, ErrMsg: "cannot parse expression"}, // array[-1] > + {Expr: fmt.Sprintf(fmt.Sprintf("%s[-1] > 1", DefaultJSONFieldName)), ErrNil: false, ErrMsg: "invalid expression"}, // json[-1] > +} \ No newline at end of file diff --git a/tests/go_client/go.mod b/tests/go_client/go.mod index cd63d3198eab..2bd796fbaa08 100644 --- a/tests/go_client/go.mod +++ b/tests/go_client/go.mod @@ -5,7 +5,7 @@ go 1.21 toolchain go1.21.10 require ( - github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de + github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 github.com/stretchr/testify v1.9.0 github.com/x448/float16 v0.8.4 diff --git a/tests/go_client/go.sum b/tests/go_client/go.sum index b461c4e1c3ee..818480c239f1 100644 --- a/tests/go_client/go.sum +++ b/tests/go_client/go.sum @@ -26,7 +26,9 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= @@ -43,6 +45,7 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= @@ -95,6 +98,7 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= @@ -146,6 +150,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -153,6 +158,7 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -204,6 +210,7 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -256,9 +263,11 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -279,6 +288,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -393,6 +403,10 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 h1:8WV4maXLeGEyJCCYIc1DmZ18H+VFAjMrwXJg5iI2nX4= github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= +github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de h1:sZ1K+F+1RMlWzuAFlhF7m68e9+G6UlRDoL7Q4Bmsc08= +github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de/go.mod h1:FU4rii9fMaMy1CGWgjROyKo5SCaJqcxZeIhdhuuEhrQ= +github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a h1:iyXy8ee4pDutzO40VA9xem1O1h/o5gI5x86ZVqVzto4= +github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a/go.mod h1:thfuEkUztRRmQ+qu4hCoO/6uxDJoUVNNx4vHqx9yh5I= github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 h1:ZBpRWhBa7FTFxW4YYVv9AUESoW1Xyb3KNXTzTqfkZmw= github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3/go.mod h1:jQ2BUZny1COsgv1Qbcv8dmbppW+V9J/c4YQZNb3EOm8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -434,6 +448,7 @@ github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5d github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -446,6 +461,7 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -502,6 +518,7 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= +github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -533,6 +550,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -547,6 +565,7 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -651,6 +670,7 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= @@ -773,6 +793,7 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -968,6 +989,7 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1066,6 +1088,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= diff --git a/tests/go_client/testcases/collection_test.go b/tests/go_client/testcases/collection_test.go index d7a2aff1d879..70211dbacdc1 100644 --- a/tests/go_client/testcases/collection_test.go +++ b/tests/go_client/testcases/collection_test.go @@ -281,7 +281,7 @@ func TestCreateCollectionDynamicSchema(t *testing.T) { columnOption := *hp.TNewDataOption() varcharColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeVarChar, columnOption) vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, columnOption) - dynamicData := hp.GenDynamicFieldData(0, common.DefaultNb) + dynamicData := hp.GenDynamicColumnData(0, common.DefaultNb) _, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, varcharColumn, vecColumn).WithColumns(dynamicData...)) common.CheckErr(t, err, true) } @@ -313,7 +313,7 @@ func TestCreateCollectionDynamic(t *testing.T) { columnOption := *hp.TNewDataOption() varcharColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeVarChar, columnOption) vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, columnOption) - dynamicData := hp.GenDynamicFieldData(0, common.DefaultNb) + dynamicData := hp.GenDynamicColumnData(0, common.DefaultNb) _, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, varcharColumn, vecColumn).WithColumns(dynamicData...)) common.CheckErr(t, err, false, "field dynamicNumber does not exist") } diff --git a/tests/go_client/testcases/delete_test.go b/tests/go_client/testcases/delete_test.go new file mode 100644 index 000000000000..ab2bb0f61801 --- /dev/null +++ b/tests/go_client/testcases/delete_test.go @@ -0,0 +1,558 @@ +package testcases + +import ( + "fmt" + "testing" + "time" + + "github.com/milvus-io/milvus/pkg/log" + "go.uber.org/zap" + + clientv2 "github.com/milvus-io/milvus/client/v2" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/tests/go_client/common" + hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper" + "github.com/stretchr/testify/require" +) + +func TestDelete(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + cp := hp.NewCreateCollectionParams(hp.Int64Vec) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + // insert + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load collection + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // delete with expr + expr := fmt.Sprintf("%s < 10", common.DefaultInt64FieldName) + ids := []int64{10, 11, 12, 13, 14} + delRes, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(expr)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(10), delRes.DeleteCount) + + // delete with int64 pk + delRes, errDelete = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithInt64IDs(common.DefaultInt64FieldName, ids)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(5), delRes.DeleteCount) + + // query, verify delete success + exprQuery := fmt.Sprintf("%s < 15", common.DefaultInt64FieldName) + queryRes, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(exprQuery).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errQuery, true) + require.Zero(t, queryRes.ResultCount) +} + +// test delete with string pks +func TestDeleteVarcharPks(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + cp := hp.NewCreateCollectionParams(hp.VarcharBinary) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + // insert + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load collection + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // delete varchar with pk + ids := []string{"0", "1", "2", "3", "4"} + expr := "varchar like '1%' " + delRes, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithStringIDs(common.DefaultVarcharFieldName, ids)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(5), delRes.DeleteCount) + + delRes, errDelete = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(expr)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(1110), delRes.DeleteCount) + + // query, verify delete success + exprQuery := "varchar like '1%' and varchar not in ['0', '1', '2', '3', '4'] " + queryRes, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(exprQuery).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errQuery, true) + require.Zero(t, queryRes.ResultCount) +} + +// test delete from empty collection +func TestDeleteEmptyCollection(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + cp := hp.NewCreateCollectionParams(hp.Int64Vec) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + // delete expr-in from empty collection + delExpr := fmt.Sprintf("%s in [0]", common.DefaultInt64FieldName) + delRes, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(delExpr)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(1), delRes.DeleteCount) + + // delete complex expr from empty collection + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + comExpr := fmt.Sprintf("%s < 10", common.DefaultInt64FieldName) + delRes, errDelete = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(comExpr)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(0), delRes.DeleteCount) +} + +// test delete from an not exist collection or partition +func TestDeleteNotExistName(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // delete from not existed collection + _, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption("aaa").WithExpr("")) + common.CheckErr(t, errDelete, false, "collection not found") + + // delete from not existed partition + cp := hp.NewCreateCollectionParams(hp.Int64Vec) + _, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + _, errDelete = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithPartition("aaa")) + common.CheckErr(t, errDelete, false, "partition not found[partition=aaa]") +} + +// test delete with complex expr without loading +// delete without loading support: pk ids +func TestDeleteComplexExprWithoutLoad(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + cp := hp.NewCreateCollectionParams(hp.Int64VecAllScalar) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + // insert + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + idsPk := []int64{0, 1, 2, 3, 4} + _, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithInt64IDs(common.DefaultInt64FieldName, idsPk)) + common.CheckErr(t, errDelete, true) + + _, errDelete = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithStringIDs(common.DefaultVarcharFieldName, []string{"0", "1"})) + common.CheckErr(t, errDelete, false, "collection not loaded") + + // delete varchar with pk + expr := fmt.Sprintf("%s < 100", common.DefaultInt64FieldName) + _, errDelete2 := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(expr)) + common.CheckErr(t, errDelete2, false, "collection not loaded") + + // index and load collection + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + res, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(fmt.Sprintf("%s >= 0 ", common.DefaultInt64FieldName)). + WithOutputFields([]string{common.QueryCountFieldName}).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + count, _ := res.Fields[0].GetAsInt64(0) + require.Equal(t, int64(common.DefaultNb-5), count) +} + +// test delete with nil ids +func TestDeleteEmptyIds(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + cp := hp.NewCreateCollectionParams(hp.VarcharBinary) + _, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + // delete + _, err := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithInt64IDs(common.DefaultInt64FieldName, nil)) + common.CheckErr(t, err, false, "failed to create delete plan: cannot parse expression: int64 in []") + + _, err = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithInt64IDs(common.DefaultInt64FieldName, []int64{})) + common.CheckErr(t, err, false, "failed to create delete plan: cannot parse expression: int64 in []") + + _, err = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithStringIDs(common.DefaultInt64FieldName, []string{""})) + common.CheckErr(t, err, false, "failed to create delete plan: cannot parse expression: int64 in [\"\"]") + + t.Log("https://github.com/milvus-io/milvus/issues/33761") + _, err = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr("")) + common.CheckErr(t, err, false, "delete plan can't be empty or always true") + + _, err = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName)) + common.CheckErr(t, err, false, "delete plan can't be empty or always true") +} + +// test delete with string pks +func TestDeleteVarcharEmptyIds(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + cp := hp.NewCreateCollectionParams(hp.VarcharBinary) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + // insert + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load collection + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + exprQuery := "varchar != '' " + + // delete varchar with empty ids + delRes, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithStringIDs(common.DefaultVarcharFieldName, []string{})) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(0), delRes.DeleteCount) + queryRes, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(exprQuery).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errQuery, true) + require.Equal(t, common.DefaultNb, queryRes.ResultCount) + + // delete with default string ids + delRes, errDelete = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithStringIDs(common.DefaultVarcharFieldName, []string{""})) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(1), delRes.DeleteCount) + queryRes, errQuery = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(exprQuery).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errQuery, true) + require.Equal(t, common.DefaultNb, queryRes.ResultCount) +} + +// test delete with invalid ids +func TestDeleteInvalidIds(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + cp := hp.NewCreateCollectionParams(hp.VarcharBinary) + _, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + _, err := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithInt64IDs(common.DefaultVarcharFieldName, []int64{0})) + common.CheckErr(t, err, false, "failed to create delete plan: cannot parse expression: varchar in [0]") + + _, err = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithInt64IDs(common.DefaultInt64FieldName, []int64{0})) + common.CheckErr(t, err, false, "failed to create delete plan: cannot parse expression: int64 in [0]") + + _, err = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithStringIDs(common.DefaultInt64FieldName, []string{"0"})) + common.CheckErr(t, err, false, "failed to create delete plan: cannot parse expression: int64 in [\"0\"]") +} + +// test delete with non-pk ids +func TestDeleteWithIds(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and a partition + pkName := "pk" + vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim) + pkField := entity.NewField().WithName(pkName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true) + int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64) + varcharField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithMaxLength(common.MaxLength) + collName := common.GenRandomString(prefix, 6) + schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithField(int64Field).WithField(varcharField) + err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema)) + common.CheckErr(t, err, true) + + // insert + insertOpt := clientv2.NewColumnBasedInsertOption(collName) + for _, field := range schema.Fields { + if field.Name == pkName { + insertOpt.WithColumns(hp.GenColumnData(common.DefaultNb, field.DataType, *hp.TNewDataOption().TWithFieldName(pkName))) + } else { + insertOpt.WithColumns(hp.GenColumnData(common.DefaultNb, field.DataType, *hp.TNewDataOption())) + } + } + _, err = mc.Insert(ctx, insertOpt) + common.CheckErr(t, err, true) + // index and load + hp.CollPrepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + hp.CollPrepare.Load(ctx, t, mc, hp.NewLoadParams(collName)) + + // delete with non-pk fields ids + resDe1, err := mc.Delete(ctx, clientv2.NewDeleteOption(collName).WithInt64IDs(common.DefaultInt64FieldName, []int64{0, 1})) + common.CheckErr(t, err, true) + require.Equal(t, int64(2), resDe1.DeleteCount) + + resDe2, err2 := mc.Delete(ctx, clientv2.NewDeleteOption(collName).WithStringIDs(common.DefaultVarcharFieldName, []string{"2", "3", "4"})) + common.CheckErr(t, err2, true) + require.Equal(t, int64(3), resDe2.DeleteCount) + + // query and verify + resQuery, err := mc.Query(ctx, clientv2.NewQueryOption(collName).WithFilter("pk < 5").WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + require.Zero(t, resQuery.ResultCount) +} + +// test delete with default partition name params +func TestDeleteDefaultPartitionName(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and a partition + cp := hp.NewCreateCollectionParams(hp.Int64Vec) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + parName := "p1" + err := mc.CreatePartition(ctx, clientv2.NewCreatePartitionOption(schema.CollectionName, parName)) + common.CheckErr(t, err, true) + + // insert [0, 3000) into default, insert [3000, 6000) into p1 + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb).TWithPartitionName(parName), hp.TNewDataOption().TWithStart(common.DefaultNb)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // delete with default params, actually delete from all partitions + expr := fmt.Sprintf("%s >= 0", common.DefaultInt64FieldName) + resDel, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(expr)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(common.DefaultNb*2), resDel.DeleteCount) + + // query, verify delete all partitions + queryRes, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errQuery, true) + require.Zero(t, queryRes.ResultCount) + + queryRes, errQuery = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithPartitions([]string{common.DefaultPartition, parName}). + WithConsistencyLevel(entity.ClStrong).WithFilter(expr)) + common.CheckErr(t, errQuery, true) + require.Zero(t, queryRes.ResultCount) +} + +// test delete with empty partition "": actually delete from all partitions +func TestDeleteEmptyPartitionName(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and a partition + cp := hp.NewCreateCollectionParams(hp.Int64Vec) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + parName := "p1" + err := mc.CreatePartition(ctx, clientv2.NewCreatePartitionOption(schema.CollectionName, parName)) + common.CheckErr(t, err, true) + + // insert [0, 3000) into default, insert [3000, 6000) into p1 + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb).TWithPartitionName(parName), hp.TNewDataOption().TWithStart(common.DefaultNb)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // delete with default params, actually delete from all partitions + expr := fmt.Sprintf("%s >= 0", common.DefaultInt64FieldName) + resDel, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(expr).WithPartition("")) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(common.DefaultNb*2), resDel.DeleteCount) + + // query, verify delete all partitions + queryRes, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errQuery, true) + require.Zero(t, queryRes.ResultCount) + + queryRes, errQuery = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithPartitions([]string{common.DefaultPartition, parName}). + WithConsistencyLevel(entity.ClStrong).WithFilter(expr)) + common.CheckErr(t, errQuery, true) + require.Zero(t, queryRes.ResultCount) +} + +// test delete with partition name +func TestDeletePartitionName(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and a partition + cp := hp.NewCreateCollectionParams(hp.Int64Vec) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) + parName := "p1" + err := mc.CreatePartition(ctx, clientv2.NewCreatePartitionOption(schema.CollectionName, parName)) + common.CheckErr(t, err, true) + + // insert [0, 3000) into default, insert [3000, 6000) into parName + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb).TWithPartitionName(parName), hp.TNewDataOption().TWithStart(common.DefaultNb)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // delete with default params, actually delete from all partitions + exprDefault := fmt.Sprintf("%s < 200", common.DefaultInt64FieldName) + exprP1 := fmt.Sprintf("%s >= 4500", common.DefaultInt64FieldName) + exprQuery := fmt.Sprintf("%s >= 0", common.DefaultInt64FieldName) + + // delete ids that not existed in partition + // delete [0, 200) from p1 + del1, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(exprDefault).WithPartition(parName)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(0), del1.DeleteCount) + + // delete [4800, 6000) from _default + del2, errDelete := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(exprP1).WithPartition(common.DefaultPartition)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(0), del2.DeleteCount) + + // query and verify + resQuery, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(exprQuery).WithOutputFields([]string{common.QueryCountFieldName}). + WithConsistencyLevel(entity.ClStrong)) + count, _ := resQuery.Fields[0].GetAsInt64(0) + require.Equal(t, int64(common.DefaultNb*2), count) + + // delete from partition + del1, errDelete = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(exprDefault).WithPartition(common.DefaultPartition)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(200), del1.DeleteCount) + + del2, errDelete = mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(exprP1).WithPartition(parName)) + common.CheckErr(t, errDelete, true) + require.Equal(t, int64(1500), del2.DeleteCount) + + // query, verify delete all partitions + queryRes, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(exprQuery).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errQuery, true) + require.Equal(t, common.DefaultNb*2-200-1500, queryRes.ResultCount) + + queryRes, errQuery = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(exprQuery).WithConsistencyLevel(entity.ClStrong). + WithPartitions([]string{common.DefaultPartition, parName})) + common.CheckErr(t, errQuery, true) + require.Equal(t, common.DefaultNb*2-200-1500, queryRes.ResultCount) +} + +// test delete ids field not pk int64 +func TestDeleteComplexExpr(t *testing.T) { + t.Parallel() + + type exprCount struct { + expr string + count int + } + capacity := common.TestCapacity + exprLimits := []exprCount{ + {expr: fmt.Sprintf("%s >= 1000 || %s > 2000", common.DefaultInt64FieldName, common.DefaultInt64FieldName), count: 2000}, + + //json and dynamic field filter expr: == < in bool/ list/ int + {expr: fmt.Sprintf("%s['number'] < 100 and %s['number'] != 0", common.DefaultJSONFieldName, common.DefaultJSONFieldName), count: 50}, + {expr: fmt.Sprintf("%s < 100", common.DefaultDynamicNumberField), count: 100}, + {expr: fmt.Sprintf("%s == false", common.DefaultDynamicBoolField), count: 2000}, + {expr: fmt.Sprintf("%s['string'] in ['1', '2', '5'] ", common.DefaultJSONFieldName), count: 3}, + {expr: fmt.Sprintf("%s['list'][0] < 10 ", common.DefaultJSONFieldName), count: 5}, + {expr: fmt.Sprintf("%s[\"dynamicList\"] != [2, 3]", common.DefaultDynamicFieldName), count: 0}, + + // json contains + {expr: fmt.Sprintf("json_contains (%s['list'], 2)", common.DefaultJSONFieldName), count: 1}, + {expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], [1, 3])", common.DefaultJSONFieldName), count: 2}, + // string like + {expr: "dynamicString like '1%' ", count: 1111}, + + // key exist + {expr: fmt.Sprintf("exists %s['list']", common.DefaultJSONFieldName), count: common.DefaultNb / 2}, + + // data type not match and no error + {expr: fmt.Sprintf("%s['number'] == '0' ", common.DefaultJSONFieldName), count: 0}, + + // json field + {expr: fmt.Sprintf("%s > 1499.5", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500.0 + {expr: fmt.Sprintf("%s like '21%%'", common.DefaultJSONFieldName), count: 100 / 4}, // json like '21%' + {expr: fmt.Sprintf("%s == [1503, 1504]", common.DefaultJSONFieldName), count: 1}, // json == [1,2] + {expr: fmt.Sprintf("%s[0][0] > 1", common.DefaultJSONFieldName), count: 0}, // json == [1,2] + {expr: fmt.Sprintf("%s[0] == false", common.DefaultBoolArrayField), count: common.DefaultNb / 2}, // array[0] == + {expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt8ArrayField), count: 1524}, // array[0] > int8 range: [-128, 127] + {expr: fmt.Sprintf("json_contains (%s, 1)", common.DefaultInt32ArrayField), count: 2}, // json_contains(array, 1) + {expr: fmt.Sprintf("json_contains_any (%s, [0, 100, 10])", common.DefaultFloatArrayField), count: 101}, // json_contains_any (array, [x]) + {expr: fmt.Sprintf("%s == [0, 1]", common.DefaultDoubleArrayField), count: 0}, // array == + {expr: fmt.Sprintf("array_length(%s) == %d", common.DefaultDoubleArrayField, capacity), count: common.DefaultNb}, // array_length + } + for _, exprLimit := range exprLimits { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and a partition + cp := hp.NewCreateCollectionParams(hp.Int64VecAllScalar) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + + // insert [0, 3000) into default + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + log.Debug("TestDeleteComplexExpr", zap.Any("expr", exprLimit.expr)) + + resDe, err := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(exprLimit.expr)) + common.CheckErr(t, err, true) + log.Debug("delete count", zap.Bool("equal", int64(exprLimit.count) == resDe.DeleteCount)) + //require.Equal(t, int64(exprLimit.count), resDe.DeleteCount) + + resQuery, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(exprLimit.expr).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + require.Zero(t, resQuery.ResultCount) + } +} + +func TestDeleteInvalidExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and a partition + cp := hp.NewCreateCollectionParams(hp.Int64VecAllScalar) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + + // insert [0, 3000) into default + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + for _, _invalidExpr := range common.InvalidExpressions { + _, err := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithExpr(_invalidExpr.Expr)) + common.CheckErr(t, err, _invalidExpr.ErrNil, _invalidExpr.ErrMsg) + } +} + +// test delete with duplicated data ids +func TestDeleteDuplicatedPks(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and a partition + cp := hp.NewCreateCollectionParams(hp.Int64Vec) + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption().TWithIsDynamic(true), hp.TNewSchemaOption()) + + // insert [0, 3000) into default + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // index and load + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // delete + deleteIds := []int64{0, 0, 0, 0, 0} + delRes, err := mc.Delete(ctx, clientv2.NewDeleteOption(schema.CollectionName).WithInt64IDs(common.DefaultInt64FieldName, deleteIds)) + common.CheckErr(t, err, true) + require.Equal(t, 5, int(delRes.DeleteCount)) + + // query, verify delete success + expr := fmt.Sprintf("%s >= 0 ", common.DefaultInt64FieldName) + resQuery, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errQuery, true) + require.Equal(t, common.DefaultNb-1, resQuery.ResultCount) +} + diff --git a/tests/go_client/testcases/helper/data_helper.go b/tests/go_client/testcases/helper/data_helper.go index 18956cf3d3c7..160bba153b0c 100644 --- a/tests/go_client/testcases/helper/data_helper.go +++ b/tests/go_client/testcases/helper/data_helper.go @@ -16,7 +16,6 @@ import ( type InsertParams struct { Schema *entity.Schema PartitionName string - Start int Nb int IsRows bool } @@ -33,11 +32,6 @@ func (opt *InsertParams) TWithPartitionName(partitionName string) *InsertParams return opt } -func (opt *InsertParams) TWithStart(start int) *InsertParams { - opt.Start = start - return opt -} - func (opt *InsertParams) TWithIsRows(isRows bool) *InsertParams { opt.IsRows = isRows return opt @@ -256,7 +250,7 @@ func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) col start := option.start fieldName := option.fieldName if option.fieldName == "" { - fieldName = GetFieldNameByFieldType(fieldType, option.elementType) + fieldName = GetFieldNameByFieldType(fieldType, TWithElementType(option.elementType)) } switch fieldType { case entity.FieldTypeInt64: @@ -317,9 +311,11 @@ func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) col case entity.FieldTypeArray: return GenArrayColumnData(nb, option.elementType, option) + case entity.FieldTypeJSON: jsonValues := GenDefaultJSONData(nb, option) return column.NewColumnJSONBytes(fieldName, jsonValues) + case entity.FieldTypeFloatVector: vecFloatValues := make([][]float32, 0, nb) for i := start; i < start+nb; i++ { @@ -327,6 +323,7 @@ func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) col vecFloatValues = append(vecFloatValues, vec) } return column.NewColumnFloatVector(fieldName, option.dim, vecFloatValues) + case entity.FieldTypeBinaryVector: binaryVectors := make([][]byte, 0, nb) for i := 0; i < nb; i++ { @@ -341,6 +338,7 @@ func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) col fp16Vectors = append(fp16Vectors, vec) } return column.NewColumnFloat16Vector(fieldName, dim, fp16Vectors) + case entity.FieldTypeBFloat16Vector: bf16Vectors := make([][]byte, 0, nb) for i := start; i < start+nb; i++ { @@ -348,6 +346,7 @@ func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) col bf16Vectors = append(bf16Vectors, vec) } return column.NewColumnBFloat16Vector(fieldName, dim, bf16Vectors) + case entity.FieldTypeSparseVector: vectors := make([]entity.SparseEmbedding, 0, nb) for i := start; i < start+nb; i++ { @@ -355,13 +354,14 @@ func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) col vectors = append(vectors, vec) } return column.NewColumnSparseVectors(fieldName, vectors) + default: log.Fatal("GenColumnData failed", zap.Any("FieldType", fieldType)) return nil } } -func GenDynamicFieldData(start int, nb int) []column.Column { +func GenDynamicColumnData(start int, nb int) []column.Column { type ListStruct struct { List []int64 `json:"list" milvus:"name:list"` } diff --git a/tests/go_client/testcases/helper/field_helper.go b/tests/go_client/testcases/helper/field_helper.go index 657ca57280aa..926c8fc00849 100644 --- a/tests/go_client/testcases/helper/field_helper.go +++ b/tests/go_client/testcases/helper/field_helper.go @@ -7,6 +7,25 @@ import ( "go.uber.org/zap" ) +type GetFieldNameOpt func(opt *getFieldNameOpt) + +type getFieldNameOpt struct { + elementType entity.FieldType + isDynamic bool +} + +func TWithElementType(eleType entity.FieldType) GetFieldNameOpt { + return func(opt *getFieldNameOpt) { + opt.elementType = eleType + } +} + +func TWithIsDynamic(isDynamic bool) GetFieldNameOpt { + return func(opt *getFieldNameOpt) { + opt.isDynamic = isDynamic + } +} + func GetFieldNameByElementType(t entity.FieldType) string { switch t { case entity.FieldTypeBool: @@ -26,11 +45,16 @@ func GetFieldNameByElementType(t entity.FieldType) string { case entity.FieldTypeVarChar: return common.DefaultVarcharArrayField default: + log.Warn("GetFieldNameByElementType", zap.Any("ElementType", t)) return common.DefaultArrayFieldName } } -func GetFieldNameByFieldType(t entity.FieldType, eleType ...entity.FieldType) string { +func GetFieldNameByFieldType(t entity.FieldType, opts ...GetFieldNameOpt) string { + opt := &getFieldNameOpt{} + for _, o := range opts { + o(opt) + } switch t { case entity.FieldTypeBool: return common.DefaultBoolFieldName @@ -49,9 +73,12 @@ func GetFieldNameByFieldType(t entity.FieldType, eleType ...entity.FieldType) st case entity.FieldTypeVarChar: return common.DefaultVarcharFieldName case entity.FieldTypeJSON: + if opt.isDynamic{ + return common.DefaultDynamicFieldName + } return common.DefaultJSONFieldName case entity.FieldTypeArray: - return GetFieldNameByElementType(eleType[0]) + return GetFieldNameByElementType(opt.elementType) case entity.FieldTypeBinaryVector: return common.DefaultBinaryVecFieldName case entity.FieldTypeFloatVector: @@ -78,6 +105,7 @@ const ( Int64VarcharSparseVec CollectionFieldsType = 5 // int64 + varchar + sparse vector Int64MultiVec CollectionFieldsType = 6 // int64 + floatVec + binaryVec + fp16Vec + bf16vec AllFields CollectionFieldsType = 7 // all fields excepted sparse + Int64VecAllScalar CollectionFieldsType = 8 // int64 + floatVec + all scalar fields ) type GenFieldsOption struct { @@ -276,6 +304,42 @@ func (cf FieldsAllFields) GenFields(option GenFieldsOption) []*entity.Field { return fields } +type FieldsInt64VecAllScalar struct{} // except sparse vector field +func (cf FieldsInt64VecAllScalar) GenFields(option GenFieldsOption) []*entity.Field { + pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true) + fields := []*entity.Field{ + pkField, + } + // scalar fields and array fields + for _, fieldType := range GetAllScalarFieldType() { + if fieldType == entity.FieldTypeInt64 { + continue + } else if fieldType == entity.FieldTypeArray { + for _, eleType := range GetAllArrayElementType() { + arrayField := entity.NewField().WithName(GetFieldNameByElementType(eleType)).WithDataType(entity.FieldTypeArray).WithElementType(eleType).WithMaxCapacity(option.MaxCapacity) + if eleType == entity.FieldTypeVarChar { + arrayField.WithMaxLength(option.MaxLength) + } + fields = append(fields, arrayField) + } + } else if fieldType == entity.FieldTypeVarChar { + varcharField := entity.NewField().WithName(GetFieldNameByFieldType(fieldType)).WithDataType(fieldType).WithMaxLength(option.MaxLength) + fields = append(fields, varcharField) + } else { + scalarField := entity.NewField().WithName(GetFieldNameByFieldType(fieldType)).WithDataType(fieldType) + fields = append(fields, scalarField) + } + + } + vecField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeFloatVector)).WithDataType(entity.FieldTypeFloatVector).WithDim(option.Dim) + fields = append(fields, vecField) + + if option.AutoID { + pkField.WithIsAutoID(option.AutoID) + } + return fields +} + func (ff FieldsFactory) GenFieldsForCollection(collectionFieldsType CollectionFieldsType, option *GenFieldsOption) []*entity.Field { log.Info("GenFieldsForCollection", zap.Any("GenFieldsOption", option)) switch collectionFieldsType { @@ -293,6 +357,8 @@ func (ff FieldsFactory) GenFieldsForCollection(collectionFieldsType CollectionFi return FieldsInt64MultiVec{}.GenFields(*option) case AllFields: return FieldsAllFields{}.GenFields(*option) + case Int64VecAllScalar: + return FieldsInt64VecAllScalar{}.GenFields(*option) default: return FieldsInt64Vec{}.GenFields(*option) } diff --git a/tests/go_client/testcases/helper/helper.go b/tests/go_client/testcases/helper/helper.go index 92df5f839f9b..35fa8e387fb7 100644 --- a/tests/go_client/testcases/helper/helper.go +++ b/tests/go_client/testcases/helper/helper.go @@ -135,10 +135,19 @@ func (chainTask *CollectionPrepare) InsertData(ctx context.Context, t *testing.T fields := ip.Schema.Fields insertOpt := clientv2.NewColumnBasedInsertOption(ip.Schema.CollectionName) for _, field := range fields { - column := GenColumnData(ip.Nb, field.DataType, *option) - insertOpt.WithColumns(column) + if field.IsDynamic { + insertOpt.WithColumns(GenDynamicColumnData(option.start, ip.Nb)...) + } else { + if field.DataType == entity.FieldTypeArray{ + option.TWithElementType(field.ElementType) + } + column := GenColumnData(ip.Nb, field.DataType, *option) + insertOpt.WithColumns(column) + } + } + if ip.PartitionName != "" { + insertOpt.WithPartition(ip.PartitionName) } - insertRes, err := mc.Insert(ctx, insertOpt) common.CheckErr(t, err, true) return chainTask, insertRes diff --git a/tests/go_client/testcases/helper/index_helper.go b/tests/go_client/testcases/helper/index_helper.go index 554fcbb3789e..e69d4e5931a2 100644 --- a/tests/go_client/testcases/helper/index_helper.go +++ b/tests/go_client/testcases/helper/index_helper.go @@ -3,15 +3,18 @@ package helper import ( "github.com/milvus-io/milvus/client/v2/entity" "github.com/milvus-io/milvus/client/v2/index" + "github.com/milvus-io/milvus/tests/go_client/common" ) func GetDefaultVectorIndex(fieldType entity.FieldType) index.Index { switch fieldType { - case entity.FieldTypeFloatVector, entity.FieldTypeFloat16Vector, entity.FieldTypeBFloat16Vector, entity.FieldTypeSparseVector: + case entity.FieldTypeFloatVector, entity.FieldTypeFloat16Vector, entity.FieldTypeBFloat16Vector: return index.NewHNSWIndex(entity.COSINE, 8, 200) case entity.FieldTypeBinaryVector: - return nil + return index.NewGenericIndex(common.DefaultBinaryVecFieldName, map[string]string{"nlist": "64", index.MetricTypeKey: "JACCARD", index.IndexTypeKey: "BIN_IVF_FLAT"}) // return binary index + case entity.FieldTypeSparseVector: + return index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.1", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_INVERTED_INDEX"}) default: return nil // return auto index diff --git a/tests/go_client/testcases/helper/read_helper.go b/tests/go_client/testcases/helper/read_helper.go index 085dfec31d25..2f0f14c78d59 100644 --- a/tests/go_client/testcases/helper/read_helper.go +++ b/tests/go_client/testcases/helper/read_helper.go @@ -10,9 +10,9 @@ type LoadParams struct { Replica int } -func NewLoadParams(name string) *LoadParams { +func NewLoadParams(collectionName string) *LoadParams { return &LoadParams{ - CollectionName: name, + CollectionName: collectionName, } } diff --git a/tests/go_client/testcases/insert_test.go b/tests/go_client/testcases/insert_test.go index 280d09828945..be44548a2c8d 100644 --- a/tests/go_client/testcases/insert_test.go +++ b/tests/go_client/testcases/insert_test.go @@ -116,7 +116,7 @@ func TestInsertAllFieldsData(t *testing.T) { insertOpt.WithColumns(_column) } if dynamic { - insertOpt.WithColumns(hp.GenDynamicFieldData(0, common.DefaultNb)...) + insertOpt.WithColumns(hp.GenDynamicColumnData(0, common.DefaultNb)...) } insertRes, errInsert := mc.Insert(ctx, insertOpt) common.CheckErr(t, errInsert, true)