-
Notifications
You must be signed in to change notification settings - Fork 31
/
merge.go
81 lines (69 loc) · 1.8 KB
/
merge.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
// Copyright 2019-2020 Grabtaxi Holdings PTE LTE (GRAB), All rights reserved.
// Use of this source code is governed by an MIT-style license that can be found in the LICENSE file
package merge
import (
"bytes"
"encoding/json"
"strings"
"sync"
)
// Func represents merge function
type Func func(interface{}) ([]byte, error)
// New creates a new merge function
func New(mergeFunc string) (map[string]Func, error) {
encoder := make(map[string]Func)
blockEncoder, err := newBlockEncoder(mergeFunc)
if err != nil {
return nil, err
}
rowEncoder, err := newRowEncoder(mergeFunc)
if err != nil {
return nil, err
}
encoder["block"] = blockEncoder
encoder["row"] = rowEncoder
return encoder, nil
}
func newBlockEncoder(mergeFunc string) (Func, error) {
switch strings.ToLower(mergeFunc) {
case "orc":
return ToOrc, nil
case "parquet":
return ToParquet, nil
case "block":
return ToBlock, nil
case "": // Default to "orc" so we don't break existing configs
return ToOrc, nil
}
return nil, nil
}
func newRowEncoder(mergeFunc string) (Func, error) {
switch mergeFunc {
case "json":
return Func(json.Marshal), nil
default:
return Func(json.Marshal), nil
}
}
// ----------------------------------------------------------------------------
// Clone clones the buffer into one which can be returned
func clone(b *bytes.Buffer) []byte {
output := make([]byte, len(b.Bytes()))
copy(output, b.Bytes())
return output
}
// A memory pool for reusable temporary buffers
var buffers = sync.Pool{
New: func() interface{} {
return bytes.NewBuffer(make([]byte, 0, 16*1<<20))
},
}
// Acquire gets a buffer from the pool
func acquire() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
// Release releases the buffer back to the pool
func release(buffer *bytes.Buffer) {
buffer.Reset()
buffers.Put(buffer)
}