test: add test cases for gosdk v2 index (#34431)

issue: #33419 
- Add test cases for gosdk v2 index
- Add sparse index for go client

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

client/index/common.go

@@ -51,6 +51,10 @@ const (
 	DISKANN    IndexType = "DISKANN"
 	SCANN      IndexType = "SCANN"
 
+	// Sparse
+	SparseInverted IndexType = "SPARSE_INVERTED_INDEX"
+	SparseWAND     IndexType = "SPARSE_WAND"
+
 	GPUIvfFlat IndexType = "GPU_IVF_FLAT"
 	GPUIvfPQ   IndexType = "GPU_IVF_PQ"
 

client/index/ivf.go

@@ -101,7 +101,7 @@ func (idx ivfSQ8Index) Params() map[string]string {
 }
 
 func NewIvfSQ8Index(metricType MetricType, nlist int) Index {
-	return ivfPQIndex{
+	return ivfSQ8Index{
 		baseIndex: baseIndex{
 			metricType: metricType,
 			indexType:  IvfSQ8,
@@ -122,7 +122,7 @@ type binIvfFlat struct {
 func (idx binIvfFlat) Params() map[string]string {
 	return map[string]string{
 		MetricTypeKey: string(idx.metricType),
-		IndexTypeKey:  string(IvfSQ8),
+		IndexTypeKey:  string(BinIvfFlat),
 		ivfNlistKey:   strconv.Itoa(idx.nlist),
 	}
 }

client/index/scann.go

@@ -40,12 +40,12 @@ func (idx scannIndex) Params() map[string]string {
 }
 
 func NewSCANNIndex(metricType MetricType, nlist int, withRawData bool) Index {
-	return ivfFlatIndex{
+	return scannIndex{
 		baseIndex: baseIndex{
 			metricType: metricType,
 			indexType:  SCANN,
 		},
-
 		nlist: nlist,
+		withRawData: withRawData,
 	}
 }

client/index/sparse.go

@@ -0,0 +1,63 @@
+package index
+
+import (
+	"fmt"
+)
+
+const (
+	dropRatio           = `drop_ratio_build`
+)
+
+var _ Index = sparseInvertedIndex{}
+
+// IndexSparseInverted index type for SPARSE_INVERTED_INDEX
+type sparseInvertedIndex struct {
+	baseIndex
+	dropRatio  float64
+}
+
+func (idx sparseInvertedIndex) Params() map[string]string {
+	return map[string]string{
+		MetricTypeKey:      string(idx.metricType),
+		IndexTypeKey:       string(SparseInverted),
+		dropRatio:           fmt.Sprintf("%v", idx.dropRatio),
+	}
+}
+
+func NewSparseInvertedIndex(metricType MetricType, dropRatio float64) Index {
+	return sparseInvertedIndex {
+		baseIndex: baseIndex{
+			metricType: metricType,
+			indexType:  SparseInverted,
+		},
+
+		dropRatio: dropRatio,
+	}
+}
+
+var _ Index = sparseWANDIndex{}
+type sparseWANDIndex struct {
+	baseIndex
+	dropRatio  float64
+}
+
+func (idx sparseWANDIndex) Params() map[string]string {
+	return map[string]string{
+		MetricTypeKey:      string(idx.metricType),
+		IndexTypeKey:       string(SparseWAND),
+		dropRatio:           fmt.Sprintf("%v", idx.dropRatio),
+	}
+}
+
+// IndexSparseWAND index type for SPARSE_WAND, weak-and
+func NewSparseWANDIndex(metricType MetricType, dropRatio float64) Index {
+	return sparseWANDIndex {
+		baseIndex: baseIndex{
+			metricType: metricType,
+			indexType:  SparseWAND,
+		},
+
+		dropRatio: dropRatio,
+	}
+}
+

tests/go_client/base/milvus_client.go

@@ -180,8 +180,8 @@ func (mc *MilvusClient) ListIndexes(ctx context.Context, option clientv2.ListInd
 
 // DescribeIndex Describe Index
 func (mc *MilvusClient) DescribeIndex(ctx context.Context, option clientv2.DescribeIndexOption, callOptions ...grpc.CallOption) (index.Index, error) {
-	index, err := mc.mClient.DescribeIndex(ctx, option, callOptions...)
-	return index, err
+	idx, err := mc.mClient.DescribeIndex(ctx, option, callOptions...)
+	return idx, err
 }
 
 // DropIndex Drop Index

tests/go_client/common/response_checker.go

@@ -33,7 +33,7 @@ func CheckErr(t *testing.T, actualErr error, expErrNil bool, expErrorMsg ...stri
 				}
 			}
 			if !contains {
-				t.Fatalf("CheckErr failed, actualErr doesn contains any expErrorMsg, please check test cases!")
+				t.Fatalf("CheckErr failed, actualErr doesn't contains any expErrorMsg, actual msg:%s", actualErr)
 			}
 		}
 	}
@@ -175,4 +175,4 @@ func CheckQueryResult(t *testing.T, expColumns []column.Column, actualColumns []
 			log.Error("CheckQueryResult actualColumns no column", zap.String("name", expColumn.Name()))
 		}
 	}
-}
+}

tests/go_client/go.mod

@@ -5,7 +5,7 @@ go 1.21
 toolchain go1.21.10
 
 require (
-	github.com/milvus-io/milvus/client/v2 v2.0.0-20240703023208-fb61344dc9b5
+	github.com/milvus-io/milvus/client/v2 v2.0.0-20240704083609-fcafdb6d5f68
 	github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
 	github.com/stretchr/testify v1.9.0
@@ -14,7 +14,7 @@ require (
 	google.golang.org/grpc v1.64.0
 )
 
-// replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240703023208-fb61344dc9b5 => ../../../milvus/client
+replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240704083609-fcafdb6d5f68 => ../../../milvus/client
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect

tests/go_client/go.sum

@@ -403,8 +403,6 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 h1:KUSaWVePVlHMIluAXf2qmNffI1CMlGFLLiP+4iy9014=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
-github.com/milvus-io/milvus/client/v2 v2.0.0-20240703023208-fb61344dc9b5 h1:jsMriUhlv82KS34VV6y/SDpeL+MEWcO6nR4Ur1diEf8=
-github.com/milvus-io/milvus/client/v2 v2.0.0-20240703023208-fb61344dc9b5/go.mod h1:13uL9ukc9KRK5ZtWqWwaORWlRccZLIysZzT6KUlOx+A=
 github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 h1:ZBpRWhBa7FTFxW4YYVv9AUESoW1Xyb3KNXTzTqfkZmw=
 github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3/go.mod h1:jQ2BUZny1COsgv1Qbcv8dmbppW+V9J/c4YQZNb3EOm8=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=

tests/go_client/testcases/delete_test.go

@@ -28,7 +28,7 @@ func TestDelete(t *testing.T) {
 	prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 	// index and load collection
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	// delete with expr
@@ -64,7 +64,7 @@ func TestDeleteVarcharPks(t *testing.T) {
 	prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 	// index and load collection
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	// delete varchar with pk
@@ -101,7 +101,7 @@ func TestDeleteEmptyCollection(t *testing.T) {
 	require.Equal(t, int64(1), delRes.DeleteCount)
 
 	// delete complex expr from empty collection
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	comExpr := fmt.Sprintf("%s < 10", common.DefaultInt64FieldName)
@@ -154,7 +154,7 @@ func TestDeleteComplexExprWithoutLoad(t *testing.T) {
 	common.CheckErr(t, errDelete2, false, "collection not loaded")
 
 	// index and load collection
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	res, err := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter(fmt.Sprintf("%s >= 0 ", common.DefaultInt64FieldName)).
@@ -205,7 +205,7 @@ func TestDeleteVarcharEmptyIds(t *testing.T) {
 	prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 	// index and load collection
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	exprQuery := "varchar != '' "
@@ -274,7 +274,7 @@ func TestDeleteWithIds(t *testing.T) {
 	_, err = mc.Insert(ctx, insertOpt)
 	common.CheckErr(t, err, true)
 	// index and load
-	hp.CollPrepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	hp.CollPrepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	hp.CollPrepare.Load(ctx, t, mc, hp.NewLoadParams(collName))
 
 	// delete with non-pk fields ids
@@ -310,7 +310,7 @@ func TestDeleteDefaultPartitionName(t *testing.T) {
 	prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 	// index and load
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	// delete with default params, actually delete from all partitions
@@ -348,7 +348,7 @@ func TestDeleteEmptyPartitionName(t *testing.T) {
 	prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 	// index and load
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	// delete with default params, actually delete from all partitions
@@ -386,7 +386,7 @@ func TestDeletePartitionName(t *testing.T) {
 	prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 	// index and load
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	// delete with default params, actually delete from all partitions
@@ -489,7 +489,7 @@ func TestDeleteComplexExpr(t *testing.T) {
 		prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 		// index and load
-		prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+		prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 		prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 		log.Debug("TestDeleteComplexExpr", zap.Any("expr", exprLimit.expr))
@@ -518,7 +518,7 @@ func TestDeleteInvalidExpr(t *testing.T) {
 	prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 	// index and load
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	for _, _invalidExpr := range common.InvalidExpressions {
@@ -541,7 +541,7 @@ func TestDeleteDuplicatedPks(t *testing.T) {
 	prepare.FlushData(ctx, t, mc, schema.CollectionName)
 
 	// index and load
-	prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
+	prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
 	prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
 
 	// delete

tests/go_client/testcases/helper/data_helper.go

@@ -249,7 +249,7 @@ func GenDefaultJSONData(nb int, option GenDataOption) [][]byte {
 	return jsonValues
 }
 
-// GenColumnData GenColumnDataOption
+// GenColumnData GenColumnDataOption except dynamic column
 func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) column.Column {
 	dim := option.dim
 	sparseMaxLen := option.sparseMaxLen

tests/go_client/testcases/helper/helper.go

@@ -2,6 +2,7 @@ package helper
 
 import (
 	"context"
+	"github.com/stretchr/testify/require"
 	"testing"
 	"time"
 
@@ -141,6 +142,7 @@ func (chainTask *CollectionPrepare) InsertData(ctx context.Context, t *testing.T
 	}
 	insertRes, err := mc.Insert(ctx, insertOpt)
 	common.CheckErr(t, err, true)
+	require.Equal(t, option.nb, insertRes.IDs.Len())
 	return chainTask, insertRes
 }
 

tests/go_client/testcases/helper/index_helper.go

@@ -3,21 +3,18 @@ package helper
 import (
 	"github.com/milvus-io/milvus/client/v2/entity"
 	"github.com/milvus-io/milvus/client/v2/index"
-	"github.com/milvus-io/milvus/tests/go_client/common"
 )
 
 func GetDefaultVectorIndex(fieldType entity.FieldType) index.Index {
 	switch fieldType {
 	case entity.FieldTypeFloatVector, entity.FieldTypeFloat16Vector, entity.FieldTypeBFloat16Vector:
 		return index.NewHNSWIndex(entity.COSINE, 8, 200)
 	case entity.FieldTypeBinaryVector:
-		return index.NewGenericIndex(common.DefaultBinaryVecFieldName, map[string]string{"nlist": "64", index.MetricTypeKey: "JACCARD", index.IndexTypeKey: "BIN_IVF_FLAT"})
-	//	return binary index
+		return index.NewBinIvfFlatIndex(entity.JACCARD, 64)
 	case entity.FieldTypeSparseVector:
-		return index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.1", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_INVERTED_INDEX"})
+		return index.NewSparseInvertedIndex(entity.IP, 0.1)
 	default:
-		return nil
-		//	return auto index
+		return index.NewAutoIndex(entity.COSINE)
 	}
 }
 
@@ -26,7 +23,7 @@ type IndexParams struct {
 	FieldIndexMap map[string]index.Index
 }
 
-func NewIndexParams(schema *entity.Schema) *IndexParams {
+func TNewIndexParams(schema *entity.Schema) *IndexParams {
 	return &IndexParams{
 		Schema: schema,
 	}
@@ -36,3 +33,65 @@ func (opt *IndexParams) TWithFieldIndex(mFieldIndex map[string]index.Index) *Ind
 	opt.FieldIndexMap = mFieldIndex
 	return opt
 }
+
+/*
+utils func
+*/
+var SupportFloatMetricType = []entity.MetricType{
+	entity.L2,
+	entity.IP,
+	entity.COSINE,
+}
+
+var SupportBinFlatMetricType = []entity.MetricType{
+	entity.JACCARD,
+	entity.HAMMING,
+	entity.SUBSTRUCTURE,
+	entity.SUPERSTRUCTURE,
+}
+
+var SupportBinIvfFlatMetricType = []entity.MetricType{
+	entity.JACCARD,
+	entity.HAMMING,
+}
+
+var UnsupportedSparseVecMetricsType = []entity.MetricType{
+	entity.L2,
+	entity.COSINE,
+	entity.JACCARD,
+	entity.HAMMING,
+	entity.SUBSTRUCTURE,
+	entity.SUPERSTRUCTURE,
+}
+
+
+// GenAllFloatIndex gen all float vector index
+func GenAllFloatIndex(metricType entity.MetricType) []index.Index {
+	nlist := 128
+	var allFloatIndex []index.Index
+
+	idxFlat := index.NewFlatIndex(metricType)
+	idxIvfFlat := index.NewIvfFlatIndex(metricType, nlist)
+	idxIvfSq8 := index.NewIvfSQ8Index(metricType, nlist)
+	idxIvfPq := index.NewIvfPQIndex(metricType, nlist, 16, 8)
+	idxHnsw := index.NewHNSWIndex(metricType, 8, 96)
+	idxScann := index.NewSCANNIndex(metricType, 16, true)
+	idxDiskAnn := index.NewDiskANNIndex(metricType)
+	allFloatIndex = append(allFloatIndex, idxFlat, idxIvfFlat, idxIvfSq8, idxIvfPq, idxHnsw, idxScann, idxDiskAnn)
+
+	return allFloatIndex
+}
+
+func SupportScalarIndexFieldType(field entity.FieldType) bool {
+	vectorFieldTypes := []entity.FieldType{
+		entity.FieldTypeBinaryVector, entity.FieldTypeFloatVector,
+		entity.FieldTypeFloat16Vector, entity.FieldTypeBFloat16Vector,
+		entity.FieldTypeSparseVector, entity.FieldTypeJSON,
+	}
+	for _, vectorFieldType := range vectorFieldTypes {
+		if field == vectorFieldType {
+			return false
+		}
+	}
+	return true
+}