feat: add test case for go client insert (#33603)

issue: #33419

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

client/column/sparse.go

@@ -82,7 +82,7 @@ func (c *ColumnSparseFloatVector) FieldData() *schemapb.FieldData {
 		for idx := 0; idx < vector.Len(); idx++ {
 			pos, value, _ := vector.Get(idx)
 			binary.LittleEndian.PutUint32(row[idx*8:], pos)
-			binary.LittleEndian.PutUint32(row[pos*8+4:], math.Float32bits(value))
+			binary.LittleEndian.PutUint32(row[idx*8+4:], math.Float32bits(value))
 		}
 		data = append(data, row)
 		if vector.Dim() > dim {

client/write_options.go

@@ -87,7 +87,7 @@ func (opt *columnBasedDataOption) processInsertColumns(colSchema *entity.Schema,
 
 		mNameColumn[col.Name()] = col
 		if col.Type() != field.DataType {
-			return nil, 0, fmt.Errorf("param column %s has type %v but collection field definition is %v", col.Name(), col.FieldData(), field.DataType)
+			return nil, 0, fmt.Errorf("param column %s has type %v but collection field definition is %v", col.Name(), col.Type(), field.DataType)
 		}
 		if field.DataType == entity.FieldTypeFloatVector || field.DataType == entity.FieldTypeBinaryVector {
 			dim := 0

tests/go_client/base/milvus_client.go

@@ -197,7 +197,9 @@ func (mc *MilvusClient) DropIndex(ctx context.Context, option clientv2.DropIndex
 // Insert insert data
 func (mc *MilvusClient) Insert(ctx context.Context, option clientv2.InsertOption, callOptions ...grpc.CallOption) (clientv2.InsertResult, error) {
 	insertRes, err := mc.mClient.Insert(ctx, option, callOptions...)
-	log.Info("Insert", zap.Any("result", insertRes))
+	if err == nil{
+		log.Info("Insert", zap.Any("result", insertRes))
+	}
 	return insertRes, err
 }
 

tests/go_client/common/response_checker.go

@@ -1,9 +1,14 @@
 package common
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
+	"github.com/milvus-io/milvus/client/v2/column"
+	"github.com/milvus-io/milvus/client/v2/entity"
+	"go.uber.org/zap"
+
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/stretchr/testify/require"
 
@@ -34,6 +39,54 @@ func CheckErr(t *testing.T, actualErr error, expErrNil bool, expErrorMsg ...stri
 	}
 }
 
+// EqualColumn assert field data is equal of two columns
+func EqualColumn(t *testing.T, columnA column.Column, columnB column.Column) {
+	require.Equal(t, columnA.Name(), columnB.Name())
+	require.Equal(t, columnA.Type(), columnB.Type())
+	switch columnA.Type() {
+	case entity.FieldTypeBool:
+		require.ElementsMatch(t, columnA.(*column.ColumnBool).Data(), columnB.(*column.ColumnBool).Data())
+	case entity.FieldTypeInt8:
+		require.ElementsMatch(t, columnA.(*column.ColumnInt8).Data(), columnB.(*column.ColumnInt8).Data())
+	case entity.FieldTypeInt16:
+		require.ElementsMatch(t, columnA.(*column.ColumnInt16).Data(), columnB.(*column.ColumnInt16).Data())
+	case entity.FieldTypeInt32:
+		require.ElementsMatch(t, columnA.(*column.ColumnInt32).Data(), columnB.(*column.ColumnInt32).Data())
+	case entity.FieldTypeInt64:
+		require.ElementsMatch(t, columnA.(*column.ColumnInt64).Data(), columnB.(*column.ColumnInt64).Data())
+	case entity.FieldTypeFloat:
+		require.ElementsMatch(t, columnA.(*column.ColumnFloat).Data(), columnB.(*column.ColumnFloat).Data())
+	case entity.FieldTypeDouble:
+		require.ElementsMatch(t, columnA.(*column.ColumnDouble).Data(), columnB.(*column.ColumnDouble).Data())
+	case entity.FieldTypeVarChar:
+		require.ElementsMatch(t, columnA.(*column.ColumnVarChar).Data(), columnB.(*column.ColumnVarChar).Data())
+	case entity.FieldTypeJSON:
+		log.Debug("columnA", zap.Any("data", columnA.(*column.ColumnJSONBytes).Data()))
+		log.Debug("columnB", zap.Any("data", columnB.(*column.ColumnJSONBytes).Data()))
+		require.ElementsMatch(t, columnA.(*column.ColumnJSONBytes).Data(), columnB.(*column.ColumnJSONBytes).Data())
+	case entity.FieldTypeFloatVector:
+		require.ElementsMatch(t, columnA.(*column.ColumnFloatVector).Data(), columnB.(*column.ColumnFloatVector).Data())
+	case entity.FieldTypeBinaryVector:
+		require.ElementsMatch(t, columnA.(*column.ColumnBinaryVector).Data(), columnB.(*column.ColumnBinaryVector).Data())
+	case entity.FieldTypeArray:
+		log.Info("TODO support column element type")
+	default:
+		log.Info("Support column type is:", zap.Any("FieldType", []entity.FieldType{entity.FieldTypeBool,
+			entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32, entity.FieldTypeInt64,
+			entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString, entity.FieldTypeVarChar,
+			entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector}))
+	}
+}
+
+// CheckOutputFields check query output fields
+func CheckOutputFields(t *testing.T, expFields []string, actualColumns []column.Column) {
+	actualFields := make([]string, 0)
+	for _, actualColumn := range actualColumns {
+		actualFields = append(actualFields, actualColumn.Name())
+	}
+	require.ElementsMatchf(t, expFields, actualFields, fmt.Sprintf("Expected search output fields: %v, actual: %v", expFields, actualFields))
+}
+
 // CheckSearchResult check search result, check nq, topk, ids, score
 func CheckSearchResult(t *testing.T, actualSearchResults []clientv2.ResultSet, expNq int, expTopK int) {
 	require.Equal(t, len(actualSearchResults), expNq)
@@ -42,3 +95,19 @@ func CheckSearchResult(t *testing.T, actualSearchResults []clientv2.ResultSet, e
 		require.Equal(t, actualSearchResult.ResultCount, expTopK)
 	}
 }
+
+// CheckInsertResult check insert result, ids len (insert count), ids data (pks, but no auto ids)
+func CheckInsertResult(t *testing.T, expIds column.Column, insertRes clientv2.InsertResult) {
+	require.Equal(t, expIds.Len(), insertRes.IDs.Len())
+	require.Equal(t, expIds.Len(), int(insertRes.InsertCount))
+	actualIds := insertRes.IDs
+	switch expIds.Type() {
+	// pk field support int64 and varchar type
+	case entity.FieldTypeInt64:
+		require.ElementsMatch(t, actualIds.(*column.ColumnInt64).Data(), expIds.(*column.ColumnInt64).Data())
+	case entity.FieldTypeVarChar:
+		require.ElementsMatch(t, actualIds.(*column.ColumnVarChar).Data(), expIds.(*column.ColumnVarChar).Data())
+	default:
+		log.Info("The primary field only support ", zap.Any("type", []entity.FieldType{entity.FieldTypeInt64, entity.FieldTypeVarChar}))
+	}
+}

tests/go_client/common/utils.go

@@ -3,14 +3,15 @@ package common
 import (
 	"encoding/binary"
 	"fmt"
-	"log"
 	"math"
 	"math/rand"
 	"strings"
 	"time"
 
 	"github.com/milvus-io/milvus/client/v2/entity"
+	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/x448/float16"
+	"go.uber.org/zap"
 )
 
 var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
@@ -116,7 +117,7 @@ func GenSparseVector(maxLen int) entity.SparseEmbedding {
 	}
 	vector, err := entity.NewSliceSparseEmbedding(positions, values)
 	if err != nil {
-		log.Fatalf("Generate vector failed %s", err)
+		log.Fatal("Generate vector failed %s", zap.Error(err))
 	}
 	return vector
 }

tests/go_client/go.mod

@@ -13,7 +13,7 @@ require (
 	google.golang.org/grpc v1.64.0
 )
 
-replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de => ../../../milvus/client
+//replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de => ../../../milvus/client
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect

tests/go_client/testcases/collection_test.go

@@ -64,7 +64,7 @@ func TestCreateAutoIdCollectionField(t *testing.T) {
 		require.True(t, coll.Schema.Fields[0].AutoID)
 
 		// insert
-		vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewColumnOption())
+		vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewDataOption())
 		_, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, vecColumn))
 		common.CheckErr(t, err, true)
 	}
@@ -117,7 +117,7 @@ func TestCreateAutoIdCollectionSchema(t *testing.T) {
 		log.Info("field autoID", zap.Bool("fieldAuto", coll.Schema.Fields[0].AutoID))
 
 		// insert
-		vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewColumnOption())
+		vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewDataOption())
 		_, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, vecColumn))
 		common.CheckErr(t, err, false, "field pk not passed")
 	}
@@ -146,7 +146,7 @@ func TestCreateAutoIdCollection(t *testing.T) {
 		log.Info("field autoID", zap.Bool("fieldAuto", coll.Schema.Fields[0].AutoID))
 
 		// insert
-		vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewColumnOption())
+		vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewDataOption())
 		_, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, vecColumn))
 		common.CheckErr(t, err, false, "field pk not passed")
 	}
@@ -278,7 +278,7 @@ func TestCreateCollectionDynamicSchema(t *testing.T) {
 	require.True(t, coll.Schema.EnableDynamicField)
 
 	// insert dynamic
-	columnOption := *hp.TNewColumnOption()
+	columnOption := *hp.TNewDataOption()
 	varcharColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeVarChar, columnOption)
 	vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, columnOption)
 	dynamicData := hp.GenDynamicFieldData(0, common.DefaultNb)
@@ -310,7 +310,7 @@ func TestCreateCollectionDynamic(t *testing.T) {
 	//require.True(t, coll.Schema.Fields[0].IsDynamic)
 
 	// insert dynamic
-	columnOption := *hp.TNewColumnOption()
+	columnOption := *hp.TNewDataOption()
 	varcharColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeVarChar, columnOption)
 	vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, columnOption)
 	dynamicData := hp.GenDynamicFieldData(0, common.DefaultNb)
@@ -426,7 +426,8 @@ func TestCreateCollectionWithInvalidFieldName(t *testing.T) {
 	for _, invalidName := range common.GenInvalidNames() {
 		log.Debug("TestCreateCollectionWithInvalidFieldName", zap.String("fieldName", invalidName))
 		pkField := entity.NewField().WithName(invalidName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
-		schema := entity.NewSchema().WithName("aaa").WithField(pkField)
+		vecField := entity.NewField().WithName("vec").WithDataType(entity.FieldTypeFloatVector).WithDim(128)
+		schema := entity.NewSchema().WithName("aaa").WithField(pkField).WithField(vecField)
 		collOpt := clientv2.NewCreateCollectionOption("aaa", schema)
 
 		err := mc.CreateCollection(ctx, collOpt)
@@ -490,8 +491,7 @@ func TestCreateCollectionInvalidFields(t *testing.T) {
 	vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
 	noneField := entity.NewField().WithName("none").WithDataType(entity.FieldTypeNone)
 	invalidFields := []invalidFieldsStruct{
-		// TODO https://github.com/milvus-io/milvus/issues/33199
-		//{fields: []*entity.Field{pkField}, errMsg: "vector field not set"},
+		{fields: []*entity.Field{pkField}, errMsg: "schema does not contain vector field"},
 		{fields: []*entity.Field{vecField}, errMsg: "primary key is not specified"},
 		{fields: []*entity.Field{pkField, pkField2, vecField}, errMsg: "there are more than one primary key"},
 		{fields: []*entity.Field{pkField, vecField, noneField}, errMsg: "data type None is not valid"},
@@ -935,13 +935,14 @@ func TestCreateCollectionInvalid(t *testing.T) {
 		schema *entity.Schema
 		errMsg string
 	}
+	vecField := entity.NewField().WithName("vec").WithDataType(entity.FieldTypeFloatVector).WithDim(8)
 	mSchemaErrs := []mSchemaErr{
 		{schema: nil, errMsg: "duplicated field name"},
-		{schema: entity.NewSchema(), errMsg: "collection name should not be empty"},
-		{schema: entity.NewSchema().WithName("aaa"), errMsg: "primary key is not specified"},
-		{schema: entity.NewSchema().WithName("aaa").WithField(entity.NewField()), errMsg: "primary key is not specified"},
-		{schema: entity.NewSchema().WithName("aaa").WithField(entity.NewField().WithIsPrimaryKey(true)), errMsg: "the data type of primary key should be Int64 or VarChar"},
-		{schema: entity.NewSchema().WithName("aaa").WithField(entity.NewField().WithIsPrimaryKey(true).WithDataType(entity.FieldTypeVarChar)), errMsg: "field name should not be empty"},
+		{schema: entity.NewSchema().WithField(vecField), errMsg: "collection name should not be empty"}, // no collection name
+		{schema: entity.NewSchema().WithName("aaa").WithField(vecField), errMsg: "primary key is not specified"}, // no pk field
+		{schema: entity.NewSchema().WithName("aaa").WithField(vecField).WithField(entity.NewField()), errMsg: "primary key is not specified"},
+		{schema: entity.NewSchema().WithName("aaa").WithField(vecField).WithField(entity.NewField().WithIsPrimaryKey(true)), errMsg: "the data type of primary key should be Int64 or VarChar"},
+		{schema: entity.NewSchema().WithName("aaa").WithField(vecField).WithField(entity.NewField().WithIsPrimaryKey(true).WithDataType(entity.FieldTypeVarChar)), errMsg: "field name should not be empty"},
 	}
 	for _, mSchema := range mSchemaErrs {
 		err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, mSchema.schema))