diff --git a/client/entity/sparse.go b/client/entity/sparse.go index 062a6195cd08..c2d736b830ee 100644 --- a/client/entity/sparse.go +++ b/client/entity/sparse.go @@ -68,7 +68,7 @@ func (e sliceSparseEmbedding) Serialize() []byte { for idx := 0; idx < e.Len(); idx++ { pos, value, _ := e.Get(idx) binary.LittleEndian.PutUint32(row[idx*8:], pos) - binary.LittleEndian.PutUint32(row[pos*8+4:], math.Float32bits(value)) + binary.LittleEndian.PutUint32(row[idx*8+4:], math.Float32bits(value)) } return row } diff --git a/client/read.go b/client/read.go index f46e89b374d4..d13f5e2601cf 100644 --- a/client/read.go +++ b/client/read.go @@ -43,6 +43,16 @@ type ResultSet struct { // DataSet is an alias type for column slice. type DataSet []column.Column +// GetColumn returns column with provided field name. +func (rs ResultSet) GetColumn(fieldName string) column.Column { + for _, column := range rs.Fields { + if column.Name() == fieldName { + return column + } + } + return nil +} + func (c *Client) Search(ctx context.Context, option SearchOption, callOptions ...grpc.CallOption) ([]ResultSet, error) { req := option.Request() collection, err := c.getCollection(ctx, req.GetCollectionName()) diff --git a/tests/go_client/common/consts.go b/tests/go_client/common/consts.go index 46e964f1c8ea..d72dec2971be 100644 --- a/tests/go_client/common/consts.go +++ b/tests/go_client/common/consts.go @@ -46,7 +46,7 @@ const ( // const default value from milvus config const ( - MaxPartitionNum = 4096 + MaxPartitionNum = 1024 DefaultDynamicFieldName = "$meta" QueryCountFieldName = "count(*)" DefaultPartition = "_default" diff --git a/tests/go_client/common/response_checker.go b/tests/go_client/common/response_checker.go index 08849b729468..cffb67ca39c6 100644 --- a/tests/go_client/common/response_checker.go +++ b/tests/go_client/common/response_checker.go @@ -154,6 +154,8 @@ func CheckSearchResult(t *testing.T, actualSearchResults []clientv2.ResultSet, e require.Len(t, actualSearchResults, expNq) for _, actualSearchResult := range actualSearchResults { require.Equal(t, actualSearchResult.ResultCount, expTopK) + require.Equal(t, actualSearchResult.IDs.Len(), expTopK) + require.Equal(t, len(actualSearchResult.Scores), expTopK) } } diff --git a/tests/go_client/go.mod b/tests/go_client/go.mod index 8885a3d0d966..838f6dd833ec 100644 --- a/tests/go_client/go.mod +++ b/tests/go_client/go.mod @@ -5,7 +5,7 @@ go 1.21 toolchain go1.21.10 require ( - github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5 + github.com/milvus-io/milvus/client/v2 v2.0.0-20240625063004-b12c34a8baf2 github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/stretchr/testify v1.9.0 @@ -14,7 +14,7 @@ require ( google.golang.org/grpc v1.64.0 ) -//replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5 => ../../../milvus/client + // replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240625063004-b12c34a8baf2 => ../../../milvus/client require ( github.com/beorn7/perks v1.0.1 // indirect diff --git a/tests/go_client/go.sum b/tests/go_client/go.sum index 2426e346bc3c..9014386cc18e 100644 --- a/tests/go_client/go.sum +++ b/tests/go_client/go.sum @@ -403,8 +403,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 h1:KUSaWVePVlHMIluAXf2qmNffI1CMlGFLLiP+4iy9014= github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= -github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5 h1:KNE4Smy6HxIpoJHLpds5BI2ZyvhrE5FcLhUcsjxAxAk= -github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5/go.mod h1:thfuEkUztRRmQ+qu4hCoO/6uxDJoUVNNx4vHqx9yh5I= +github.com/milvus-io/milvus/client/v2 v2.0.0-20240625063004-b12c34a8baf2 h1:Eb3E5TQwNAImS2M1yRNc1/IzlfD8iQJ9HZt8Lf41xVc= +github.com/milvus-io/milvus/client/v2 v2.0.0-20240625063004-b12c34a8baf2/go.mod h1:thfuEkUztRRmQ+qu4hCoO/6uxDJoUVNNx4vHqx9yh5I= github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 h1:ZBpRWhBa7FTFxW4YYVv9AUESoW1Xyb3KNXTzTqfkZmw= github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3/go.mod h1:jQ2BUZny1COsgv1Qbcv8dmbppW+V9J/c4YQZNb3EOm8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= diff --git a/tests/go_client/testcases/collection_test.go b/tests/go_client/testcases/collection_test.go index 645aebbd2a15..a5a6a7c067db 100644 --- a/tests/go_client/testcases/collection_test.go +++ b/tests/go_client/testcases/collection_test.go @@ -40,7 +40,6 @@ func TestCreateCollection(t *testing.T) { } } -// func TestCreateCollection(t *testing.T) {} func TestCreateAutoIdCollectionField(t *testing.T) { ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) mc := createDefaultMilvusClient(ctx, t) @@ -626,7 +625,7 @@ func TestPartitionKeyInvalidNumPartition(t *testing.T) { numPartitions int64 errMsg string }{ - {common.MaxPartitionNum + 1, "exceeds max configuration (4096)"}, + {common.MaxPartitionNum + 1, "exceeds max configuration (1024)"}, {-1, "the specified partitions should be greater than 0 if partition key is used"}, } for _, npStruct := range invalidNumPartitionStruct { diff --git a/tests/go_client/testcases/database_test.go b/tests/go_client/testcases/database_test.go new file mode 100644 index 000000000000..cf4d25da5594 --- /dev/null +++ b/tests/go_client/testcases/database_test.go @@ -0,0 +1,281 @@ +package testcases + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + clientv2 "github.com/milvus-io/milvus/client/v2" + "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/tests/go_client/base" + "github.com/milvus-io/milvus/tests/go_client/common" + hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper" +) + +// teardownTest +func teardownTest(t *testing.T) func(t *testing.T) { + log.Info("setup test func") + return func(t *testing.T) { + log.Info("teardown func drop all non-default db") + // drop all db + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + dbs, _ := mc.ListDatabases(ctx, clientv2.NewListDatabaseOption()) + for _, db := range dbs { + if db != common.DefaultDb { + _ = mc.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(db)) + collections, _ := mc.ListCollections(ctx, clientv2.NewListCollectionOption()) + for _, coll := range collections { + _ = mc.DropCollection(ctx, clientv2.NewDropCollectionOption(coll)) + } + _ = mc.DropDatabase(ctx, clientv2.NewDropDatabaseOption(db)) + } + } + } +} + +func TestDatabase(t *testing.T) { + teardownSuite := teardownTest(t) + defer teardownSuite(t) + + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + clientDefault := createMilvusClient(ctx, t, &defaultCfg) + + // create db1 + dbName1 := common.GenRandomString("db1", 4) + err := clientDefault.CreateDatabase(ctx, clientv2.NewCreateDatabaseOption(dbName1)) + common.CheckErr(t, err, true) + + // list db and verify db1 in dbs + dbs, errList := clientDefault.ListDatabases(ctx, clientv2.NewListDatabaseOption()) + common.CheckErr(t, errList, true) + require.Containsf(t, dbs, dbName1, fmt.Sprintf("%s db not in dbs: %v", dbName1, dbs)) + + // new client with db1 -> using db + clientDB1 := createMilvusClient(ctx, t, &clientv2.ClientConfig{Address: *addr, DBName: dbName1}) + t.Log("https://github.com/milvus-io/milvus/issues/34137") + err = clientDB1.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(dbName1)) + common.CheckErr(t, err, true) + + // create collections -> verify collections contains + _, db1Col1 := hp.CollPrepare.CreateCollection(ctx, t, clientDB1, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + _, db1Col2 := hp.CollPrepare.CreateCollection(ctx, t, clientDB1, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + collections, errListCollections := clientDB1.ListCollections(ctx, clientv2.NewListCollectionOption()) + common.CheckErr(t, errListCollections, true) + require.Containsf(t, collections, db1Col1.CollectionName, fmt.Sprintf("The collection %s not in: %v", db1Col1.CollectionName, collections)) + require.Containsf(t, collections, db1Col2.CollectionName, fmt.Sprintf("The collection %s not in: %v", db1Col2.CollectionName, collections)) + + // create db2 + dbName2 := common.GenRandomString("db2", 4) + err = clientDefault.CreateDatabase(ctx, clientv2.NewCreateDatabaseOption(dbName2)) + common.CheckErr(t, err, true) + dbs, err = clientDefault.ListDatabases(ctx, clientv2.NewListDatabaseOption()) + common.CheckErr(t, err, true) + require.Containsf(t, dbs, dbName2, fmt.Sprintf("%s db not in dbs: %v", dbName2, dbs)) + + // using db2 -> create collection -> drop collection + err = clientDefault.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(dbName2)) + common.CheckErr(t, err, true) + _, db2Col1 := hp.CollPrepare.CreateCollection(ctx, t, clientDefault, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + err = clientDefault.DropCollection(ctx, clientv2.NewDropCollectionOption(db2Col1.CollectionName)) + common.CheckErr(t, err, true) + + // using empty db -> drop db2 + clientDefault.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption("")) + err = clientDefault.DropDatabase(ctx, clientv2.NewDropDatabaseOption(dbName2)) + common.CheckErr(t, err, true) + + // list db and verify db drop success + dbs, err = clientDefault.ListDatabases(ctx, clientv2.NewListDatabaseOption()) + common.CheckErr(t, err, true) + require.NotContains(t, dbs, dbName2) + + // drop db1 which has some collections + err = clientDB1.DropDatabase(ctx, clientv2.NewDropDatabaseOption(dbName1)) + common.CheckErr(t, err, false, "must drop all collections before drop database") + + // drop all db1's collections -> drop db1 + clientDB1.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(dbName1)) + err = clientDB1.DropCollection(ctx, clientv2.NewDropCollectionOption(db1Col1.CollectionName)) + common.CheckErr(t, err, true) + + err = clientDB1.DropCollection(ctx, clientv2.NewDropCollectionOption(db1Col2.CollectionName)) + common.CheckErr(t, err, true) + + err = clientDB1.DropDatabase(ctx, clientv2.NewDropDatabaseOption(dbName1)) + common.CheckErr(t, err, true) + + // drop default db + err = clientDefault.DropDatabase(ctx, clientv2.NewDropDatabaseOption(common.DefaultDb)) + common.CheckErr(t, err, false, "can not drop default database") + + dbs, err = clientDefault.ListDatabases(ctx, clientv2.NewListDatabaseOption()) + common.CheckErr(t, err, true) + require.Containsf(t, dbs, common.DefaultDb, fmt.Sprintf("The db %s not in: %v", common.DefaultDb, dbs)) +} + +// test create with invalid db name +func TestCreateDb(t *testing.T) { + teardownSuite := teardownTest(t) + defer teardownSuite(t) + + // create db + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + dbName := common.GenRandomString("db", 4) + err := mc.CreateDatabase(ctx, clientv2.NewCreateDatabaseOption(dbName)) + common.CheckErr(t, err, true) + + // create existed db + err = mc.CreateDatabase(ctx, clientv2.NewCreateDatabaseOption(dbName)) + common.CheckErr(t, err, false, fmt.Sprintf("database already exist: %s", dbName)) + + // create default db + err = mc.CreateDatabase(ctx, clientv2.NewCreateDatabaseOption(common.DefaultDb)) + common.CheckErr(t, err, false, fmt.Sprintf("database already exist: %s", common.DefaultDb)) + + emptyErr := mc.CreateDatabase(ctx, clientv2.NewCreateDatabaseOption("")) + common.CheckErr(t, emptyErr, false, "database name couldn't be empty") +} + +// test drop db +func TestDropDb(t *testing.T) { + teardownSuite := teardownTest(t) + defer teardownSuite(t) + + // create collection in default db + listCollOpt := clientv2.NewListCollectionOption() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + _, defCol := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + collections, _ := mc.ListCollections(ctx, listCollOpt) + require.Contains(t, collections, defCol.CollectionName) + + // create db + dbName := common.GenRandomString("db", 4) + err := mc.CreateDatabase(ctx, clientv2.NewCreateDatabaseOption(dbName)) + common.CheckErr(t, err, true) + + // using db and drop the db + err = mc.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(dbName)) + common.CheckErr(t, err, true) + err = mc.DropDatabase(ctx, clientv2.NewDropDatabaseOption(dbName)) + common.CheckErr(t, err, true) + + // verify current db + _, err = mc.ListCollections(ctx, listCollOpt) + common.CheckErr(t, err, false, fmt.Sprintf("database not found[database=%s]", dbName)) + + // using default db and verify collections + err = mc.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(common.DefaultDb)) + common.CheckErr(t, err, true) + collections, _ = mc.ListCollections(ctx, listCollOpt) + require.Contains(t, collections, defCol.CollectionName) + + // drop not existed db + err = mc.DropDatabase(ctx, clientv2.NewDropDatabaseOption(common.GenRandomString("db", 4))) + common.CheckErr(t, err, true) + + // drop empty db + err = mc.DropDatabase(ctx, clientv2.NewDropDatabaseOption("")) + common.CheckErr(t, err, false, "database name couldn't be empty") + + // drop default db + err = mc.DropDatabase(ctx, clientv2.NewDropDatabaseOption(common.DefaultDb)) + common.CheckErr(t, err, false, "can not drop default database") +} + +// test using db +func TestUsingDb(t *testing.T) { + teardownSuite := teardownTest(t) + defer teardownSuite(t) + + // create collection in default db + listCollOpt := clientv2.NewListCollectionOption() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + _, col := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + // collName := createDefaultCollection(ctx, t, mc, true, common.DefaultShards) + collections, _ := mc.ListCollections(ctx, listCollOpt) + require.Contains(t, collections, col.CollectionName) + + // using not existed db + dbName := common.GenRandomString("db", 4) + err := mc.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(dbName)) + common.CheckErr(t, err, false, fmt.Sprintf("database not found[database=%s]", dbName)) + + // using empty db + err = mc.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption("")) + common.CheckErr(t, err, true) + collections, _ = mc.ListCollections(ctx, listCollOpt) + require.Contains(t, collections, col.CollectionName) + + // using current db + err = mc.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(common.DefaultDb)) + common.CheckErr(t, err, true) + collections, _ = mc.ListCollections(ctx, listCollOpt) + require.Contains(t, collections, col.CollectionName) +} + +func TestClientWithDb(t *testing.T) { + t.Skip("https://github.com/milvus-io/milvus/issues/34137") + teardownSuite := teardownTest(t) + defer teardownSuite(t) + + listCollOpt := clientv2.NewListCollectionOption() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + + // connect with not existed db + _, err := base.NewMilvusClient(ctx, &clientv2.ClientConfig{Address: *addr, DBName: "dbName"}) + common.CheckErr(t, err, false, "database not found") + + // connect default db -> create a collection in default db + mcDefault, errDefault := base.NewMilvusClient(ctx, &clientv2.ClientConfig{ + Address: *addr, + // DBName: common.DefaultDb, + }) + common.CheckErr(t, errDefault, true) + _, defCol1 := hp.CollPrepare.CreateCollection(ctx, t, mcDefault, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + defCollections, _ := mcDefault.ListCollections(ctx, listCollOpt) + require.Contains(t, defCollections, defCol1.CollectionName) + log.Debug("default db collections:", zap.Any("default collections", defCollections)) + + // create a db and create collection in db + dbName := common.GenRandomString("db", 5) + err = mcDefault.CreateDatabase(ctx, clientv2.NewCreateDatabaseOption(dbName)) + common.CheckErr(t, err, true) + + // and connect with db + mcDb, err := base.NewMilvusClient(ctx, &clientv2.ClientConfig{ + Address: *addr, + DBName: dbName, + }) + common.CheckErr(t, err, true) + _, dbCol1 := hp.CollPrepare.CreateCollection(ctx, t, mcDb, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + dbCollections, _ := mcDb.ListCollections(ctx, listCollOpt) + log.Debug("db collections:", zap.Any("db collections", dbCollections)) + require.Containsf(t, dbCollections, dbCol1.CollectionName, fmt.Sprintf("The collection %s not in: %v", dbCol1.CollectionName, dbCollections)) + + // using default db and collection not in + _ = mcDb.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(common.DefaultDb)) + defCollections, _ = mcDb.ListCollections(ctx, listCollOpt) + require.NotContains(t, defCollections, dbCol1.CollectionName) + + // connect empty db (actually default db) + mcEmpty, err := base.NewMilvusClient(ctx, &clientv2.ClientConfig{ + Address: *addr, + DBName: "", + }) + common.CheckErr(t, err, true) + defCollections, _ = mcEmpty.ListCollections(ctx, listCollOpt) + require.Contains(t, defCollections, defCol1.CollectionName) +} + +func TestAlterDatabase(t *testing.T) { + t.Skip("waiting for AlterDatabase and DescribeDatabase") +} diff --git a/tests/go_client/testcases/main_test.go b/tests/go_client/testcases/main_test.go index c3d2eb1b9752..58c590f996b8 100644 --- a/tests/go_client/testcases/main_test.go +++ b/tests/go_client/testcases/main_test.go @@ -63,6 +63,24 @@ func createDefaultMilvusClient(ctx context.Context, t *testing.T) *base.MilvusCl return mc } +// create connect +func createMilvusClient(ctx context.Context, t *testing.T, cfg *clientv2.ClientConfig) *base.MilvusClient { + t.Helper() + + var ( + mc *base.MilvusClient + err error + ) + mc, err = base.NewMilvusClient(ctx, cfg) + common.CheckErr(t, err, true) + + t.Cleanup(func() { + mc.Close(ctx) + }) + + return mc +} + func TestMain(m *testing.M) { flag.Parse() log.Info("Parser Milvus address", zap.String("address", *addr)) diff --git a/tests/go_client/testcases/query_test.go b/tests/go_client/testcases/query_test.go index da7c4a29b7d9..0c9115d5a33a 100644 --- a/tests/go_client/testcases/query_test.go +++ b/tests/go_client/testcases/query_test.go @@ -159,9 +159,13 @@ func TestQueryWithoutExpr(t *testing.T) { common.CheckErr(t, err, false, "empty expression should be used with limit") } -// test query empty output fields: []string{}, []string{""} -// test query with not existed field -func TestQueryEmptyOutputFields(t *testing.T) { +// test query empty output fields: []string{} -> default pk +// test query empty output fields: []string{""} -> error +// test query with not existed field ["aa"]: error or as dynamic field +// test query with part not existed field ["aa", "$meat"]: error or as dynamic field +// test query with repeated field: ["*", "$meat"], ["floatVec", floatVec"] unique field +func TestQueryOutputFields(t *testing.T) { + t.Skip("verify TODO") t.Parallel() ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) mc := createDefaultMilvusClient(ctx, t) @@ -189,29 +193,52 @@ func TestQueryEmptyOutputFields(t *testing.T) { common.CheckErr(t, err1, false, "not exist") } - // query with empty not existed field -> output field as dynamic or error + // query with not existed field -> output field as dynamic or error fakeName := "aaa" - res, err2 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{fakeName})) + res2, err2 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{fakeName})) if enableDynamic { common.CheckErr(t, err2, true) - for _, c := range res.Fields { + for _, c := range res2.Fields { log.Debug("data", zap.String("name", c.Name()), zap.Any("type", c.Type()), zap.Any("data", c.FieldData())) } - common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, fakeName}, res.Fields) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, fakeName}, res2.Fields) dynamicColumn := hp.MergeColumnsToDynamic(10, hp.GenDynamicColumnData(0, 10), common.DefaultDynamicFieldName) expColumns := []column.Column{ hp.GenColumnData(10, entity.FieldTypeInt64, *hp.TNewDataOption()), column.NewColumnDynamic(dynamicColumn, fakeName), } - common.CheckQueryResult(t, expColumns, res.Fields) + common.CheckQueryResult(t, expColumns, res2.Fields) } else { common.CheckErr(t, err2, false, fmt.Sprintf("%s not exist", fakeName)) } + + // query with part not existed field ["aa", "$meat"]: error or as dynamic field + res3, err3 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{fakeName, common.DefaultDynamicFieldName})) + if enableDynamic { + common.CheckErr(t, err3, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, fakeName, common.DefaultDynamicFieldName}, res3.Fields) + } else { + common.CheckErr(t, err3, false, "not exist") + } + + // query with repeated field: ["*", "$meat"], ["floatVec", floatVec"] unique field + res4, err4 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{"*", common.DefaultDynamicFieldName})) + if enableDynamic { + common.CheckErr(t, err4, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName}, res4.Fields) + } else { + common.CheckErr(t, err4, false, "$meta not exist") + } + + res5, err5 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields( + []string{common.DefaultFloatVecFieldName, common.DefaultFloatVecFieldName, common.DefaultInt64FieldName})) + common.CheckErr(t, err5, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultFloatVecFieldName}, res5.Fields) } } // test query output all fields and verify data -func TestOutputAllFieldsColumn(t *testing.T) { +func TestQueryOutputAllFieldsColumn(t *testing.T) { t.Skip("https://github.com/milvus-io/milvus/issues/33848") ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) mc := createDefaultMilvusClient(ctx, t) @@ -267,7 +294,7 @@ func TestOutputAllFieldsColumn(t *testing.T) { } // test query output all fields -func TestOutputAllFieldsRows(t *testing.T) { +func TestQueryOutputAllFieldsRows(t *testing.T) { t.Skip("https://github.com/milvus-io/milvus/issues/33459") ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) mc := createDefaultMilvusClient(ctx, t) diff --git a/tests/go_client/testcases/search_test.go b/tests/go_client/testcases/search_test.go index a190f35a0033..8d429cfa2b1b 100644 --- a/tests/go_client/testcases/search_test.go +++ b/tests/go_client/testcases/search_test.go @@ -1,43 +1,1009 @@ package testcases import ( + "fmt" + "math/rand" "testing" "time" + "github.com/stretchr/testify/require" "go.uber.org/zap" clientv2 "github.com/milvus-io/milvus/client/v2" + "github.com/milvus-io/milvus/client/v2/column" "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/index" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/tests/go_client/common" hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper" ) -func TestSearch(t *testing.T) { +func TestSearchDefault(t *testing.T) { ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) mc := createDefaultMilvusClient(ctx, t) - cp := hp.NewCreateCollectionParams(hp.Int64Vec) - _, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption()) - log.Info("schema", zap.Any("schema", schema)) + // create -> insert -> flush -> index -> load + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) - insertParams := hp.NewInsertParams(schema, common.DefaultNb) - hp.CollPrepare.InsertData(ctx, t, mc, insertParams, hp.TNewDataOption()) + // search + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + resSearch, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit) +} - // flush -> index -> load - hp.CollPrepare.FlushData(ctx, t, mc, schema.CollectionName) - hp.CollPrepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) - hp.CollPrepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) +func TestSearchDefaultGrowing(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create -> index -> load -> insert + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.VarcharBinary), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) // search - vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBinaryVector) resSearch, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong)) common.CheckErr(t, err, true) common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit) +} + +// test search collection and partition name not exist +func TestSearchInvalidCollectionPartitionName(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // search with not exist collection + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + _, err := mc.Search(ctx, clientv2.NewSearchOption("aaa", common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, false, "can't find collection") + + // search with empty collections name + _, err = mc.Search(ctx, clientv2.NewSearchOption("", common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, false, "collection name should not be empty") + + // search with not exist partition + _, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.VarcharBinary), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + _, err1 := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithPartitions([]string{"aaa"})) + common.CheckErr(t, err1, false, "partition name aaa not found") + + // search with empty partition name []string{""} -> error + _, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors). + WithConsistencyLevel(entity.ClStrong).WithANNSField(common.DefaultFloatVecFieldName).WithPartitions([]string{""})) + common.CheckErr(t, errSearch, false, "Partition name should not be empty") +} + +// test search empty collection -> return empty +func TestSearchEmptyCollection(t *testing.T) { + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + for _, enableDynamicField := range []bool{true, false} { + // create -> index -> load + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), + hp.TNewSchemaOption().TWithEnableDynamicField(enableDynamicField)) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + type mNameVec struct { + fieldName string + queryVec []entity.Vector + } + for _, _mNameVec := range []mNameVec{ + {fieldName: common.DefaultFloatVecFieldName, queryVec: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)}, + {fieldName: common.DefaultFloat16VecFieldName, queryVec: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloat16Vector)}, + {fieldName: common.DefaultBFloat16VecFieldName, queryVec: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBFloat16Vector)}, + {fieldName: common.DefaultBinaryVecFieldName, queryVec: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBinaryVector)}, + } { + resSearch, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, _mNameVec.queryVec). + WithConsistencyLevel(entity.ClStrong).WithANNSField(_mNameVec.fieldName)) + common.CheckErr(t, errSearch, true) + t.Log("https://github.com/milvus-io/milvus/issues/33952") + common.CheckSearchResult(t, resSearch, 0, 0) + } + } +} + +func TestSearchEmptySparseCollection(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), + hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeSparseVector) + resSearch, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors). + WithConsistencyLevel(entity.ClStrong).WithANNSField(common.DefaultSparseVecFieldName)) + common.CheckErr(t, errSearch, true) + t.Log("https://github.com/milvus-io/milvus/issues/33952") + common.CheckSearchResult(t, resSearch, 0, 0) +} + +// test search with partition names []string{}, specify partitions +func TestSearchPartitions(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + parName := common.GenRandomString("p", 4) + // create collection and partition + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption().TWithAutoID(true), + hp.TNewSchemaOption().TWithEnableDynamicField(true)) + err := mc.CreatePartition(ctx, clientv2.NewCreatePartitionOption(schema.CollectionName, parName)) + common.CheckErr(t, err, true) + + // insert autoID data into parName and _default partitions + _defVec := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, *hp.TNewDataOption()) + _defDynamic := hp.GenDynamicColumnData(0, common.DefaultNb) + insertRes1, err1 := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName).WithColumns(_defVec).WithColumns(_defDynamic...)) + common.CheckErr(t, err1, true) + + _parVec := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, *hp.TNewDataOption()) + insertRes2, err2 := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName).WithColumns(_parVec)) + common.CheckErr(t, err2, true) + + // flush -> FLAT index -> load + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultFloatVecFieldName: index.NewFlatIndex(entity.COSINE)})) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search with empty partition name []string{""} -> error + vectors := make([]entity.Vector, 0, 2) + // query first ID of _default and parName partition + _defId0, _ := insertRes1.IDs.GetAsInt64(0) + _parId0, _ := insertRes2.IDs.GetAsInt64(0) + queryRes, _ := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(fmt.Sprintf("int64 in [%d, %d]", _defId0, _parId0)).WithOutputFields([]string{"*"})) + require.ElementsMatch(t, []int64{_defId0, _parId0}, queryRes.GetColumn(common.DefaultInt64FieldName).(*column.ColumnInt64).Data()) + for _, vec := range queryRes.GetColumn(common.DefaultFloatVecFieldName).(*column.ColumnFloatVector).Data() { + vectors = append(vectors, entity.FloatVector(vec)) + } + + for _, partitions := range [][]string{{}, {common.DefaultPartition, parName}} { + // search with empty partition names slice []string{} -> all partitions + searchResult, errSearch1 := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, 5, vectors). + WithConsistencyLevel(entity.ClStrong).WithANNSField(common.DefaultFloatVecFieldName).WithPartitions(partitions).WithOutputFields([]string{"*"})) + + // check search result contains search vector, which from all partitions + common.CheckErr(t, errSearch1, true) + common.CheckSearchResult(t, searchResult, len(vectors), 5) + require.Contains(t, searchResult[0].IDs.(*column.ColumnInt64).Data(), _defId0) + require.Contains(t, searchResult[1].IDs.(*column.ColumnInt64).Data(), _parId0) + require.EqualValues(t, entity.FloatVector(searchResult[0].GetColumn(common.DefaultFloatVecFieldName).(*column.ColumnFloatVector).Data()[0]), vectors[0]) + require.EqualValues(t, entity.FloatVector(searchResult[1].GetColumn(common.DefaultFloatVecFieldName).(*column.ColumnFloatVector).Data()[0]), vectors[1]) + } +} + +// test query empty output fields: []string{} -> []string{} +// test query empty output fields: []string{""} -> error +func TestSearchEmptyOutputFields(t *testing.T) { + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + for _, dynamic := range []bool{true, false} { + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(dynamic)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 100), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + resSearch, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong).WithOutputFields([]string{})) + common.CheckErr(t, err, true) + common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit) + common.CheckOutputFields(t, []string{}, resSearch[0].Fields) + + _, err = mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong).WithOutputFields([]string{""})) + if dynamic { + common.CheckErr(t, err, false, "parse output field name failed") + } else { + common.CheckErr(t, err, false, "field not exist") + } + } +} + +// test query with not existed field ["aa"]: error or as dynamic field +// test query with part not existed field ["aa", "$meat"]: error or as dynamic field +// test query with repeated field: ["*", "$meat"], ["floatVec", floatVec"] unique field +func TestSearchNotExistOutputFields(t *testing.T) { + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + for _, enableDynamic := range []bool{false, true} { + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(enableDynamic)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search vector output fields not exist, part exist + type dynamicOutputFields struct { + outputFields []string + expOutputFields []string + } + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + dof := []dynamicOutputFields{ + {outputFields: []string{"aaa"}, expOutputFields: []string{"aaa"}}, + {outputFields: []string{"aaa", common.DefaultDynamicFieldName}, expOutputFields: []string{"aaa", common.DefaultDynamicFieldName}}, + {outputFields: []string{"*", common.DefaultDynamicFieldName}, expOutputFields: []string{common.DefaultInt64FieldName, common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName}}, + } + + for _, _dof := range dof { + resSearch, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong).WithOutputFields(_dof.outputFields)) + if enableDynamic { + common.CheckErr(t, err, true) + common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit) + common.CheckOutputFields(t, _dof.expOutputFields, resSearch[0].Fields) + } else { + common.CheckErr(t, err, false, "not exist") + } + } + existedRepeatedFields := []string{common.DefaultInt64FieldName, common.DefaultFloatVecFieldName, common.DefaultInt64FieldName, common.DefaultFloatVecFieldName} + resSearch2, err2 := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong).WithOutputFields(existedRepeatedFields)) + common.CheckErr(t, err2, true) + common.CheckSearchResult(t, resSearch2, common.DefaultNq, common.DefaultLimit) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultFloatVecFieldName}, resSearch2[0].Fields) + } +} + +// test search output all * fields when enable dynamic and insert dynamic column data +func TestSearchOutputAllFields(t *testing.T) { + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // + allFieldsName := []string{common.DefaultDynamicFieldName} + for _, field := range schema.Fields { + allFieldsName = append(allFieldsName, field.Name) + } + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + + searchRes, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithANNSField(common.DefaultFloatVecFieldName).WithOutputFields([]string{"*"})) + common.CheckErr(t, err, true) + common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit) + for _, res := range searchRes { + common.CheckOutputFields(t, allFieldsName, res.Fields) + } +} + +// test search output all * fields when enable dynamic and insert dynamic column data +func TestSearchOutputBinaryPk(t *testing.T) { + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.VarcharBinary), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // + allFieldsName := []string{common.DefaultDynamicFieldName} + for _, field := range schema.Fields { + allFieldsName = append(allFieldsName, field.Name) + } + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBinaryVector) + searchRes, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong).WithOutputFields([]string{"*"})) + common.CheckErr(t, err, true) + common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit) + for _, res := range searchRes { + common.CheckOutputFields(t, allFieldsName, res.Fields) + } +} + +// test search output all * fields when enable dynamic and insert dynamic column data +func TestSearchOutputSparse(t *testing.T) { + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // + allFieldsName := []string{common.DefaultDynamicFieldName} + for _, field := range schema.Fields { + allFieldsName = append(allFieldsName, field.Name) + } + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeSparseVector) + searchRes, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithANNSField(common.DefaultSparseVecFieldName).WithOutputFields([]string{"*"})) + common.CheckErr(t, err, true) + common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit) + for _, res := range searchRes { + common.CheckOutputFields(t, allFieldsName, res.Fields) + } +} + +// test search with invalid vector field name: not exist; non-vector field, empty fiend name, json and dynamic field -> error +func TestSearchInvalidVectorField(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 500), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + type invalidVectorFieldStruct struct { + vectorField string + errNil bool + errMsg string + } + + invalidVectorFields := []invalidVectorFieldStruct{ + // not exist field + {vectorField: common.DefaultBinaryVecFieldName, errNil: false, errMsg: fmt.Sprintf("failed to get field schema by name: fieldName(%s) not found", common.DefaultBinaryVecFieldName)}, + + // non-vector field + {vectorField: common.DefaultInt64FieldName, errNil: false, errMsg: fmt.Sprintf("failed to create query plan: field (%s) to search is not of vector data type", common.DefaultInt64FieldName)}, + + // json field + {vectorField: common.DefaultJSONFieldName, errNil: false, errMsg: fmt.Sprintf("failed to get field schema by name: fieldName(%s) not found", common.DefaultJSONFieldName)}, + + // dynamic field + {vectorField: common.DefaultDynamicFieldName, errNil: false, errMsg: fmt.Sprintf("failed to get field schema by name: fieldName(%s) not found", common.DefaultDynamicFieldName)}, + + // allows empty vector field name + {vectorField: "", errNil: true, errMsg: ""}, + } + + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeSparseVector) + for _, invalidVectorField := range invalidVectorFields { + _, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithANNSField(invalidVectorField.vectorField)) + common.CheckErr(t, err, invalidVectorField.errNil, invalidVectorField.errMsg) + } +} + +// test search with invalid vectors +func TestSearchInvalidVectors(t *testing.T) { + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 500), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + type invalidVectorsStruct struct { + fieldName string + vectors []entity.Vector + errMsg string + } + + invalidVectors := []invalidVectorsStruct{ + // dim not match + {fieldName: common.DefaultFloatVecFieldName, vectors: hp.GenSearchVectors(common.DefaultNq, 64, entity.FieldTypeFloatVector), errMsg: "vector dimension mismatch"}, + {fieldName: common.DefaultFloat16VecFieldName, vectors: hp.GenSearchVectors(common.DefaultNq, 64, entity.FieldTypeFloat16Vector), errMsg: "vector dimension mismatch"}, + + // vector type not match + {fieldName: common.DefaultFloatVecFieldName, vectors: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBinaryVector), errMsg: "vector type must be the same"}, + {fieldName: common.DefaultBFloat16VecFieldName, vectors: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloat16Vector), errMsg: "vector type must be the same"}, + + // empty vectors + {fieldName: common.DefaultBinaryVecFieldName, vectors: []entity.Vector{}, errMsg: "nq [0] is invalid"}, + {fieldName: common.DefaultFloatVecFieldName, vectors: []entity.Vector{entity.FloatVector{}}, errMsg: "vector dimension mismatch"}, + {vectors: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector), errMsg: "multiple anns_fields exist, please specify a anns_field in search_params"}, + {fieldName: "", vectors: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector), errMsg: "multiple anns_fields exist, please specify a anns_field in search_params"}, + } + + for _, invalidVector := range invalidVectors { + _, errSearchEmpty := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, invalidVector.vectors).WithANNSField(invalidVector.fieldName)) + common.CheckErr(t, errSearchEmpty, false, invalidVector.errMsg) + } +} + +// test search with invalid vectors +func TestSearchEmptyInvalidVectors(t *testing.T) { + t.Log("https://github.com/milvus-io/milvus/issues/33639") + t.Log("https://github.com/milvus-io/milvus/issues/33637") + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + type invalidVectorsStruct struct { + vectors []entity.Vector + errNil bool + errMsg string + } + + invalidVectors := []invalidVectorsStruct{ + // dim not match + {vectors: hp.GenSearchVectors(common.DefaultNq, 64, entity.FieldTypeFloatVector), errNil: true, errMsg: "vector dimension mismatch"}, + + // vector type not match + {vectors: hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBinaryVector), errNil: true, errMsg: "vector type must be the same"}, + + // empty vectors + {vectors: []entity.Vector{}, errNil: false, errMsg: "nq [0] is invalid"}, + {vectors: []entity.Vector{entity.FloatVector{}}, errNil: true, errMsg: "vector dimension mismatch"}, + } + + for _, invalidVector := range invalidVectors { + _, errSearchEmpty := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, invalidVector.vectors).WithANNSField(common.DefaultFloatVecFieldName)) + common.CheckErr(t, errSearchEmpty, invalidVector.errNil, invalidVector.errMsg) + } +} + +// test search metric type isn't the same with index metric type +func TestSearchNotMatchMetricType(t *testing.T) { + t.Skip("Waiting for support for specifying search parameters") + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 500), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema). + TWithFieldIndex(map[string]index.Index{common.DefaultFloatVecFieldName: index.NewHNSWIndex(entity.COSINE, 8, 200)})) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + vectors := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector) + _, errSearchEmpty := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors)) + common.CheckErr(t, errSearchEmpty, false, "metric type not match: invalid parameter") +} + +// test search with invalid topK -> error +func TestSearchInvalidTopK(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 500), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + vectors := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector) + for _, invalidTopK := range []int{-1, 0, 16385} { + _, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, invalidTopK, vectors)) + common.CheckErr(t, errSearch, false, "should be in range [1, 16384]") + } +} + +// test search with invalid topK -> error +func TestSearchInvalidOffset(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 500), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + vectors := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector) + for _, invalidOffset := range []int{-1, common.MaxTopK + 1} { + _, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithOffset(invalidOffset)) + common.CheckErr(t, errSearch, false, "should be in range [1, 16384]") + } +} + +// test search with invalid search params +func TestSearchInvalidSearchParams(t *testing.T) { + t.Skip("Waiting for support for specifying search parameters") +} + +// search with index hnsw search param ef < topK -> error +func TestSearchEfHnsw(t *testing.T) { + t.Skip("Waiting for support for specifying search parameters") + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 500), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema). + TWithFieldIndex(map[string]index.Index{common.DefaultFloatVecFieldName: index.NewHNSWIndex(entity.COSINE, 8, 200)})) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + vectors := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector) + _, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors)) + common.CheckErr(t, err, false, "ef(7) should be larger than k(10)") +} + +// test search params mismatch index type, hnsw index and ivf sq8 search param -> search with default hnsw params, ef=topK +func TestSearchSearchParamsMismatchIndex(t *testing.T) { + t.Skip("Waiting for support for specifying search parameters") +} + +// search with index scann search param ef < topK -> error +func TestSearchInvalidScannReorderK(t *testing.T) { + t.Skip("Waiting for support for specifying search parameters") + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 500), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema).TWithFieldIndex(map[string]index.Index{ + common.DefaultFloatVecFieldName: index.NewSCANNIndex(entity.COSINE, 16), + })) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search with invalid reorder_k < topK + + // valid scann index search reorder_k +} + +// test search with scann index params: with_raw_data and metrics_type [L2, IP, COSINE] +func TestSearchScannAllMetricsWithRawData(t *testing.T) { + t.Skip("Waiting for support scann index params withRawData") + t.Parallel() + /*for _, withRawData := range []bool{true, false} { + for _, metricType := range []entity.MetricType{entity.L2, entity.IP, entity.COSINE} { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 500), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema).TWithFieldIndex(map[string]index.Index{ + common.DefaultFloatVecFieldName: index.NewSCANNIndex(entity.COSINE, 16), + })) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search and output all fields + vectors := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector) + resSearch, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong).WithOutputFields([]string{"*"})) + common.CheckErr(t, errSearch, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultFloatFieldName, + common.DefaultJSONFieldName, common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName}, resSearch[0].Fields) + common.CheckSearchResult(t, resSearch, 1, common.DefaultLimit) + } + }*/ +} + +// test search with valid expression +func TestSearchExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + type mExprExpected struct { + expr string + ids []int64 + } + + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + for _, _mExpr := range []mExprExpected{ + {expr: fmt.Sprintf("%s < 10", common.DefaultInt64FieldName), ids: []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}}, + {expr: fmt.Sprintf("%s in [10, 100]", common.DefaultInt64FieldName), ids: []int64{10, 100}}, + } { + resSearch, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithFilter(_mExpr.expr)) + common.CheckErr(t, errSearch, true) + for _, res := range resSearch { + require.ElementsMatch(t, _mExpr.ids, res.IDs.(*column.ColumnInt64).Data()) + } + } +} + +// test search with invalid expression +func TestSearchInvalidExpr(t *testing.T) { + t.Parallel() + + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search with invalid expr + vectors := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector) + for _, exprStruct := range common.InvalidExpressions { + log.Debug("TestSearchInvalidExpr", zap.String("expr", exprStruct.Expr)) + _, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithFilter(exprStruct.Expr).WithANNSField(common.DefaultFloatVecFieldName)) + common.CheckErr(t, errSearch, exprStruct.ErrNil, exprStruct.ErrMsg) + } +} + +func TestSearchJsonFieldExpr(t *testing.T) { + t.Parallel() + + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) + mc := createDefaultMilvusClient(ctx, t) + + exprs := []string{ + "", + fmt.Sprintf("exists %s['number'] ", common.DefaultJSONFieldName), // exists + "json[\"number\"] > 1 and json[\"number\"] < 1000", // > and + fmt.Sprintf("%s[\"number\"] > 10", common.DefaultJSONFieldName), // number > + fmt.Sprintf("%s != 10 ", common.DefaultJSONFieldName), // json != 10 + fmt.Sprintf("%s[\"number\"] < 2000", common.DefaultJSONFieldName), // number < + fmt.Sprintf("%s[\"bool\"] != true", common.DefaultJSONFieldName), // bool != + fmt.Sprintf("%s[\"bool\"] == False", common.DefaultJSONFieldName), // bool == + fmt.Sprintf("%s[\"bool\"] in [true]", common.DefaultJSONFieldName), // bool in + fmt.Sprintf("%s[\"string\"] >= '1' ", common.DefaultJSONFieldName), // string >= + fmt.Sprintf("%s['list'][0] > 200", common.DefaultJSONFieldName), // list filter + fmt.Sprintf("%s['list'] != [2, 3]", common.DefaultJSONFieldName), // json[list] != + fmt.Sprintf("%s > 2000", common.DefaultJSONFieldName), // json > 2000 + fmt.Sprintf("%s like '2%%' ", common.DefaultJSONFieldName), // json like '2%' + fmt.Sprintf("%s[0] > 2000 ", common.DefaultJSONFieldName), // json[0] > 2000 + fmt.Sprintf("%s > 2000.5 ", common.DefaultJSONFieldName), // json > 2000.5 + } + + for _, dynamicField := range []bool{false, true} { + // create collection + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(dynamicField)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search with jsonField expr key datatype and json data type mismatch + for _, expr := range exprs { + log.Debug("TestSearchJsonFieldExpr", zap.String("expr", expr)) + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + searchRes, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithFilter(expr).WithANNSField(common.DefaultFloatVecFieldName).WithOutputFields([]string{common.DefaultInt64FieldName, common.DefaultJSONFieldName})) + common.CheckErr(t, errSearch, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultJSONFieldName}, searchRes[0].Fields) + common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit) + } + } +} + +func TestSearchDynamicFieldExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + // create collection + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + exprs := []string{ + "", + "exists dynamicNumber", // exist without dynamic fieldName + fmt.Sprintf("exists %s[\"dynamicNumber\"]", common.DefaultDynamicFieldName), // exist with fieldName + fmt.Sprintf("%s[\"dynamicNumber\"] > 10", common.DefaultDynamicFieldName), // int expr with fieldName + fmt.Sprintf("%s[\"dynamicBool\"] == true", common.DefaultDynamicFieldName), // bool with fieldName + "dynamicBool == False", // bool without fieldName + fmt.Sprintf("%s['dynamicString'] == '1'", common.DefaultDynamicFieldName), // string with fieldName + "dynamicString != \"2\" ", // string without fieldName + } + + // search with jsonField expr key datatype and json data type mismatch + for _, expr := range exprs { + log.Debug("TestSearchDynamicFieldExpr", zap.String("expr", expr)) + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + searchRes, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithFilter(expr).WithANNSField(common.DefaultFloatVecFieldName).WithOutputFields([]string{common.DefaultInt64FieldName, "dynamicNumber", "number"})) + common.CheckErr(t, errSearch, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, "dynamicNumber", "number"}, searchRes[0].Fields) + if expr == "$meta['dynamicString'] == '1'" { + common.CheckSearchResult(t, searchRes, common.DefaultNq, 1) + } else { + common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit) + } + } + + // search with expr filter number and, &&, or, || + exprs2 := []string{ + "dynamicNumber > 1 and dynamicNumber <= 999", // int expr without fieldName + fmt.Sprintf("%s['dynamicNumber'] > 1 && %s['dynamicNumber'] < 1000", common.DefaultDynamicFieldName, common.DefaultDynamicFieldName), + "dynamicNumber < 888 || dynamicNumber < 1000", + fmt.Sprintf("%s['dynamicNumber'] < 888 or %s['dynamicNumber'] < 1000", common.DefaultDynamicFieldName, common.DefaultDynamicFieldName), + fmt.Sprintf("%s[\"dynamicNumber\"] < 1000", common.DefaultDynamicFieldName), // int expr with fieldName + } + + for _, expr := range exprs2 { + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + searchRes, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithFilter(expr).WithANNSField(common.DefaultFloatVecFieldName). + WithOutputFields([]string{common.DefaultInt64FieldName, common.DefaultJSONFieldName, common.DefaultDynamicFieldName, "dynamicNumber", "number"})) + common.CheckErr(t, errSearch, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultJSONFieldName, common.DefaultDynamicFieldName, "dynamicNumber", "number"}, searchRes[0].Fields) + for _, res := range searchRes { + for _, id := range res.IDs.(*column.ColumnInt64).Data() { + require.Less(t, id, int64(1000)) + } + } + } +} + +func TestSearchArrayFieldExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecArray), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + var capacity int64 = common.TestCapacity + exprs := []string{ + fmt.Sprintf("%s[0] == false", common.DefaultBoolArrayField), // array[0] == + fmt.Sprintf("%s[0] > 0", common.DefaultInt64ArrayField), // array[0] > + fmt.Sprintf("json_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), // json_contains + fmt.Sprintf("array_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), // array_contains + fmt.Sprintf("json_contains_all (%s, [90, 91])", common.DefaultInt64ArrayField), // json_contains_all + fmt.Sprintf("array_contains_all (%s, [90, 91])", common.DefaultInt64ArrayField), // array_contains_all + fmt.Sprintf("array_contains_any (%s, [0, 100, 10000])", common.DefaultFloatArrayField), // array_contains_any + fmt.Sprintf("json_contains_any (%s, [0, 100, 10])", common.DefaultFloatArrayField), // json_contains_any + fmt.Sprintf("array_length(%s) == %d", common.DefaultDoubleArrayField, capacity), // array_length + } + + // search with jsonField expr key datatype and json data type mismatch + allArrayFields := make([]string, 0, len(schema.Fields)) + for _, field := range schema.Fields { + if field.DataType == entity.FieldTypeArray { + allArrayFields = append(allArrayFields, field.Name) + } + } + vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) + for _, expr := range exprs { + searchRes, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithFilter(expr).WithOutputFields(allArrayFields)) + common.CheckErr(t, errSearch, true) + common.CheckOutputFields(t, allArrayFields, searchRes[0].Fields) + common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit) + } + + // search hits empty + searchRes, errSearchEmpty := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithFilter(fmt.Sprintf("array_contains (%s, 1000000)", common.DefaultInt32ArrayField)).WithOutputFields(allArrayFields)) + common.CheckErr(t, errSearchEmpty, true) + common.CheckSearchResult(t, searchRes, common.DefaultNq, 0) +} + +// test search with field not existed expr: if dynamic +func TestSearchNotExistedExpr(t *testing.T) { + t.Parallel() + + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + for _, isDynamic := range [2]bool{true, false} { + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(isDynamic)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search with invalid expr + vectors := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector) + expr := "id in [0]" + res, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong). + WithFilter(expr).WithANNSField(common.DefaultFloatVecFieldName)) + if isDynamic { + common.CheckErr(t, errSearch, true) + common.CheckSearchResult(t, res, 1, 0) + } else { + common.CheckErr(t, errSearch, false, "not exist") + } + } +} + +// test search with fp16/ bf16 /binary vector +func TestSearchMultiVectors(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb*2), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + flatIndex := index.NewFlatIndex(entity.L2) + binIndex := index.NewGenericIndex(common.DefaultBinaryVecFieldName, map[string]string{"nlist": "64", index.MetricTypeKey: "JACCARD", index.IndexTypeKey: "BIN_IVF_FLAT"}) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema).TWithFieldIndex(map[string]index.Index{ + common.DefaultFloatVecFieldName: flatIndex, + common.DefaultFloat16VecFieldName: flatIndex, + common.DefaultBFloat16VecFieldName: flatIndex, + common.DefaultBinaryVecFieldName: binIndex, + })) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search with all kinds of vectors + type mFieldNameType struct { + fieldName string + fieldType entity.FieldType + metricType entity.MetricType + } + fnts := []mFieldNameType{ + {fieldName: common.DefaultFloatVecFieldName, fieldType: entity.FieldTypeFloatVector, metricType: entity.L2}, + {fieldName: common.DefaultBinaryVecFieldName, fieldType: entity.FieldTypeBinaryVector, metricType: entity.JACCARD}, + {fieldName: common.DefaultFloat16VecFieldName, fieldType: entity.FieldTypeFloat16Vector, metricType: entity.L2}, + {fieldName: common.DefaultBFloat16VecFieldName, fieldType: entity.FieldTypeBFloat16Vector, metricType: entity.L2}, + } + + for _, fnt := range fnts { + queryVec := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, fnt.fieldType) + expr := fmt.Sprintf("%s > 10", common.DefaultInt64FieldName) + + resSearch, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit*2, queryVec).WithConsistencyLevel(entity.ClStrong). + WithFilter(expr).WithANNSField(fnt.fieldName).WithOutputFields([]string{"*"})) + common.CheckErr(t, errSearch, true) + common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit*2) + common.CheckOutputFields(t, []string{ + common.DefaultInt64FieldName, common.DefaultFloatVecFieldName, + common.DefaultBinaryVecFieldName, common.DefaultFloat16VecFieldName, common.DefaultBFloat16VecFieldName, common.DefaultDynamicFieldName, + }, resSearch[0].Fields) + + // pagination search + resPage, errPage := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, queryVec).WithConsistencyLevel(entity.ClStrong). + WithFilter(expr).WithANNSField(fnt.fieldName).WithOutputFields([]string{"*"}).WithOffset(10)) + + common.CheckErr(t, errPage, true) + common.CheckSearchResult(t, resPage, common.DefaultNq, common.DefaultLimit) + for i := 0; i < common.DefaultNq; i++ { + require.Equal(t, resSearch[i].IDs.(*column.ColumnInt64).Data()[10:], resPage[i].IDs.(*column.ColumnInt64).Data()) + } + common.CheckOutputFields(t, []string{ + common.DefaultInt64FieldName, common.DefaultFloatVecFieldName, + common.DefaultBinaryVecFieldName, common.DefaultFloat16VecFieldName, common.DefaultBFloat16VecFieldName, common.DefaultDynamicFieldName, + }, resPage[0].Fields) + + // TODO range search + // TODO iterator search + } +} + +func TestSearchSparseVector(t *testing.T) { + t.Parallel() + idxInverted := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.2", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_INVERTED_INDEX"}) + idxWand := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.3", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_WAND"}) + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) + mc := createDefaultMilvusClient(ctx, t) + + for _, idx := range []index.Index{idxInverted, idxWand} { + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb*2), hp.TNewDataOption().TWithSparseMaxLen(128)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultSparseVecFieldName: idx})) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search + queryVec := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeSparseVector) + resSearch, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, queryVec).WithConsistencyLevel(entity.ClStrong). + WithOutputFields([]string{"*"})) + + common.CheckErr(t, errSearch, true) + require.Len(t, resSearch, common.DefaultNq) + outputFields := []string{common.DefaultInt64FieldName, common.DefaultVarcharFieldName, common.DefaultSparseVecFieldName, common.DefaultDynamicFieldName} + for _, res := range resSearch { + require.LessOrEqual(t, res.ResultCount, common.DefaultLimit) + if res.ResultCount == common.DefaultLimit { + common.CheckOutputFields(t, outputFields, resSearch[0].Fields) + } + } + } +} + +// test search with invalid sparse vector +func TestSearchInvalidSparseVector(t *testing.T) { + t.Parallel() + + idxInverted := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.2", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_INVERTED_INDEX"}) + idxWand := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.3", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_WAND"}) + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) + mc := createDefaultMilvusClient(ctx, t) + + for _, idx := range []index.Index{idxInverted, idxWand} { + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption().TWithSparseMaxLen(128)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultSparseVecFieldName: idx})) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + _, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, []entity.Vector{}).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errSearch, false, "nq (number of search vector per search request) should be in range [1, 16384]") + + vector1, err := entity.NewSliceSparseEmbedding([]uint32{}, []float32{}) + common.CheckErr(t, err, true) + _, errSearch1 := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, []entity.Vector{vector1}).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errSearch1, false, "Sparse row data should not be empty") + + positions := make([]uint32, 100) + values := make([]float32, 100) + for i := 0; i < 100; i++ { + positions[i] = uint32(1) + values[i] = rand.Float32() + } + vector, _ := entity.NewSliceSparseEmbedding(positions, values) + _, errSearch2 := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, []entity.Vector{vector}).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, errSearch2, false, "Invalid sparse row: id should be strict ascending") + } +} + +func TestSearchSparseVectorPagination(t *testing.T) { + t.Parallel() + idxInverted := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.2", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_INVERTED_INDEX"}) + idxWand := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.3", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_WAND"}) + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) + mc := createDefaultMilvusClient(ctx, t) + + for _, idx := range []index.Index{idxInverted, idxWand} { + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption().TWithSparseMaxLen(128)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultSparseVecFieldName: idx})) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // search + queryVec := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeSparseVector) + resSearch, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, queryVec).WithConsistencyLevel(entity.ClStrong). + WithOutputFields([]string{"*"})) + common.CheckErr(t, errSearch, true) + require.Len(t, resSearch, common.DefaultNq) + + pageSearch, errSearch := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, queryVec).WithConsistencyLevel(entity.ClStrong). + WithOutputFields([]string{"*"}).WithOffset(5)) + common.CheckErr(t, errSearch, true) + require.Len(t, pageSearch, common.DefaultNq) + for i := 0; i < len(resSearch); i++ { + if resSearch[i].ResultCount == common.DefaultLimit && pageSearch[i].ResultCount == 5 { + require.Equal(t, resSearch[i].IDs.(*column.ColumnInt64).Data()[5:], pageSearch[i].IDs.(*column.ColumnInt64).Data()) + } + } + } +} + +// test sparse vector unsupported search: TODO iterator search +func TestSearchSparseVectorNotSupported(t *testing.T) { + t.Skip("Go-sdk support iterator search in progress") +} + +func TestRangeSearchSparseVector(t *testing.T) { + t.Skip("Waiting for support range search") + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption(). + TWithEnableDynamicField(true)) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption().TWithSparseMaxLen(128)) + prepare.FlushData(ctx, t, mc, schema.CollectionName) - log.Info("search", zap.Any("resSearch", resSearch)) - log.Info("search", zap.Any("ids", resSearch[0].IDs)) - log.Info("search", zap.Any("scores", resSearch[0].Scores)) - id, _ := resSearch[0].IDs.GetAsInt64(0) - log.Info("search", zap.Int64("ids", id)) + // TODO range search }