add benchmark

Signed-off-by: shaoting-huang <shaoting.huang@zilliz.com>
milvus-io · Jun 28, 2024 · bc88066 · bc88066
1 parent bd6950d
commit bc88066
Showing 1 changed file with 120 additions and 0 deletions.
diff --git a/internal/storage/serde_events_test.go b/internal/storage/serde_events_test.go
@@ -440,3 +440,123 @@ func TestDeltalogPkTsSeparateFormat(t *testing.T) {
 		})
 	}
 }
+
+func BenchmarkDeltalogFormat(b *testing.B) {
+	size := 10000000
+	blob, err := generateTestDeltalogData(size)
+	assert.NoError(b, err)
+
+	// Benchmark DeserializeReader
+	b.Run("one string format reader", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			reader, err := NewDeltalogDeserializeReader([]*Blob{blob})
+			assert.NoError(b, err)
+			defer reader.Close()
+			for j := 0; j < size; j++ {
+				err = reader.Next()
+				_ = reader.Value()
+				assert.NoError(b, err)
+			}
+			err = reader.Next()
+			assert.Equal(b, io.EOF, err)
+		}
+		b.ReportAllocs()
+	})
+
+	blob, err = generateTestDeltalogNewFormatData(size)
+	assert.NoError(b, err)
+
+	// Benchmark DeltalogPkTsSeparateFormat
+	b.Run("pk ts separate format reader", func(b *testing.B) {
+		// Deserialize the data
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			reader, err := NewDeltalogDeserializeReader([]*Blob{blob})
+			assert.NoError(b, err)
+			defer reader.Close()
+			for j := 0; j < size; j++ {
+				err = reader.Next()
+				_ = reader.Value()
+				assert.NoError(b, err)
+			}
+			err = reader.Next()
+			assert.Equal(b, io.EOF, err)
+		}
+		b.ReportAllocs()
+	})
+
+	b.Run("one string format writer", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			eventWriter := NewDeltalogStreamWriter(0, 0, 0)
+			writer, _ := NewDeltalogSerializeWriter(0, 0, eventWriter, size)
+
+			for j := 0; j < size; j++ {
+				value := NewDeleteLog(&Int64PrimaryKey{int64(j)}, uint64(j+1))
+				writer.Write(value)
+			}
+			writer.Close()
+			eventWriter.Finalize()
+		}
+		b.ReportAllocs()
+	})
+
+	b.Run("pk and ts separate format writer", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			eventWriter := NewMultiFieldDeltalogStreamWriter(0, 0, 0, []*schemapb.FieldSchema{
+				{FieldID: common.RowIDField, Name: "pk", DataType: schemapb.DataType_Int64},
+				{FieldID: common.TimeStampField, Name: "ts", DataType: schemapb.DataType_Int64},
+			})
+			writer, _ := NewDeltalogMultiFieldWriter(0, 0, eventWriter, size)
+
+			for j := 0; j < size; j++ {
+				value := NewDeleteLog(&Int64PrimaryKey{int64(j)}, uint64(j+1))
+				writer.Write(value)
+			}
+			writer.Close()
+			eventWriter.Finalize()
+		}
+		b.ReportAllocs()
+	})
+}
+
+func generateTestDeltalogNewFormatData(size int) (*Blob, error) {
+	eventWriter := NewMultiFieldDeltalogStreamWriter(0, 0, 0, []*schemapb.FieldSchema{
+		{FieldID: common.RowIDField, Name: "pk", DataType: schemapb.DataType_Int64},
+		{FieldID: common.TimeStampField, Name: "ts", DataType: schemapb.DataType_Int64},
+	})
+	writer, err := NewDeltalogMultiFieldWriter(0, 0, eventWriter, size)
+	if err != nil {
+		return nil, err
+	}
+	pks := make([]interface{}, size)
+	tss := make([]uint64, size)
+	for i := 0; i < size; i++ {
+		pks[i] = NewInt64PrimaryKey(int64(i))
+		tss[i] = uint64(i + 1)
+	}
+	data := make([]*DeleteLog, size)
+	for i := range pks {
+		pk, ok := pks[i].(PrimaryKey)
+		if !ok {
+			return nil, err
+		}
+		data[i] = NewDeleteLog(pk, tss[i])
+	}
+
+	// Serialize the data
+	for i := 0; i < size; i++ {
+		writer.Write(data[i])
+	}
+	err = writer.Close()
+	if err != nil {
+		return nil, err
+	}
+	blob, err := eventWriter.Finalize()
+	if err != nil {
+		return nil, err
+	}
+	return blob, nil
+}