-
Notifications
You must be signed in to change notification settings - Fork 0
/
sizeDataPacker.go
95 lines (78 loc) · 2.59 KB
/
sizeDataPacker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
package partitioning
import (
"github.com/bhagyaraj1208117/andes-abc-1/core"
"github.com/bhagyaraj1208117/andes-abc-1/data/batch"
"github.com/bhagyaraj1208117/andes-abc-1/marshal"
)
const minimumMaxPacketSizeInBytes = 1
// SizeDataPacker can split a large slice of byte slices in chunks <= maxPacketSize
// If one element still exceeds maxPacketSize, it will be returned alone
// It does the marshaling of the resulted (smaller) slice of byte slices
type SizeDataPacker struct {
marshalizer marshal.Marshalizer
}
// NewSizeDataPacker creates a new SizeDataPacker instance
func NewSizeDataPacker(marshalizer marshal.Marshalizer) (*SizeDataPacker, error) {
if marshalizer == nil || marshalizer.IsInterfaceNil() {
return nil, core.ErrNilMarshalizer
}
return &SizeDataPacker{
marshalizer: marshalizer,
}, nil
}
// PackDataInChunks packs the provided data into smaller chunks
// limit is expressed in bytes
func (sdp *SizeDataPacker) PackDataInChunks(data [][]byte, limit int) ([][]byte, error) {
if limit < minimumMaxPacketSizeInBytes {
return nil, core.ErrInvalidValue
}
if data == nil {
return nil, core.ErrNilInputData
}
returningBuff := make([][]byte, 0)
elements := make([][]byte, 0)
lastMarshalized := make([]byte, 0)
for _, element := range data {
elements = append(elements, element)
marshaledElements, err := sdp.marshalizer.Marshal(&batch.Batch{Data: elements})
if err != nil {
return nil, err
}
isSingleElement := len(elements) == 1
isMarshaledBuffTooLarge := len(marshaledElements) >= limit
if isMarshaledBuffTooLarge {
if isSingleElement {
returningBuff = append(returningBuff, marshaledElements)
elements = make([][]byte, 0)
} else {
returningBuff = append(returningBuff, lastMarshalized)
elements = make([][]byte, 0)
elements = append(elements, element)
marshaledElements, err = sdp.marshalizer.Marshal(&batch.Batch{Data: elements})
if err != nil {
return nil, err
}
isMarshaledBuffTooLarge = len(marshaledElements) >= limit
if isMarshaledBuffTooLarge {
returningBuff = append(returningBuff, marshaledElements)
elements = make([][]byte, 0)
}
}
lastMarshalized = make([]byte, 0)
continue
}
lastMarshalized = marshaledElements
}
if len(elements) > 0 {
marshaledElements, err := sdp.marshalizer.Marshal(&batch.Batch{Data: elements})
if err != nil {
return nil, err
}
returningBuff = append(returningBuff, marshaledElements)
}
return returningBuff, nil
}
// IsInterfaceNil returns true if there is no value under the interface
func (sdp *SizeDataPacker) IsInterfaceNil() bool {
return sdp == nil
}