-
Notifications
You must be signed in to change notification settings - Fork 0
/
simpleDataPacker.go
72 lines (61 loc) · 2.24 KB
/
simpleDataPacker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
package partitioning
import (
"github.com/subrahamanyam341/andes-core-go/core"
"github.com/subrahamanyam341/andes-core-go/data/batch"
"github.com/subrahamanyam341/andes-core-go/marshal"
)
// SimpleDataPacker can split a large slice of byte slices in chunks <= maxPacketSize
// If one element still exceeds maxPacketSize, it will be returned alone
// It does the marshaling of the resulted (smaller) slice of byte slices
// This is a simpler version of a data packer that does not marshall in a repetitive manner currentChunk slice
// as the SizeDataPacker does. This limitation is lighter in terms of CPU cycles and memory used but is not as precise
// as SizeDataPacker.
type SimpleDataPacker struct {
marshalizer marshal.Marshalizer
}
// NewSimpleDataPacker creates a new SizeDataPacker instance
func NewSimpleDataPacker(marshalizer marshal.Marshalizer) (*SimpleDataPacker, error) {
if marshalizer == nil || marshalizer.IsInterfaceNil() {
return nil, core.ErrNilMarshalizer
}
return &SimpleDataPacker{
marshalizer: marshalizer,
}, nil
}
// PackDataInChunks packs the provided data into smaller chunks
// limit is expressed in bytes
func (sdp *SimpleDataPacker) PackDataInChunks(data [][]byte, limit int) ([][]byte, error) {
if limit < minimumMaxPacketSizeInBytes {
return nil, core.ErrInvalidValue
}
if data == nil {
return nil, core.ErrNilInputData
}
returningBuff := make([][]byte, 0)
currentChunk := make([][]byte, 0)
lenChunk := 0
for _, element := range data {
isBuffToLarge := lenChunk+len(element) >= limit
chunkNotEmpty := len(currentChunk) > 0
if isBuffToLarge && chunkNotEmpty {
marshaledChunk, _ := sdp.marshalizer.Marshal(&batch.Batch{Data: currentChunk})
returningBuff = append(returningBuff, marshaledChunk)
currentChunk = make([][]byte, 0)
lenChunk = 0
}
currentChunk = append(currentChunk, element)
lenChunk += len(element)
}
if len(currentChunk) > 0 {
marshaledElements, err := sdp.marshalizer.Marshal(&batch.Batch{Data: currentChunk})
if err != nil {
return nil, err
}
returningBuff = append(returningBuff, marshaledElements)
}
return returningBuff, nil
}
// IsInterfaceNil returns true if there is no value under the interface
func (sdp *SimpleDataPacker) IsInterfaceNil() bool {
return sdp == nil
}