forked from influxdata/influxdb
-
Notifications
You must be signed in to change notification settings - Fork 0
/
shard_writer.go
96 lines (78 loc) · 1.61 KB
/
shard_writer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
package importer
import (
"fmt"
"os"
"path/filepath"
"strconv"
"github.com/influxdata/influxdb/cmd/influx-tools/internal/errlist"
"github.com/influxdata/influxdb/tsdb/engine/tsm1"
)
const (
maxTSMFileSize = uint32(2048 * 1024 * 1024) // 2GB
)
type shardWriter struct {
w tsm1.TSMWriter
id uint64
path string
gen, seq int
err error
}
func newShardWriter(id uint64, path string) *shardWriter {
t := &shardWriter{id: id, path: path, gen: 1, seq: 1}
return t
}
func (t *shardWriter) Write(key []byte, values tsm1.Values) {
if t.err != nil {
return
}
if t.w == nil {
t.nextTSM()
}
if t.w.Size() > maxTSMFileSize {
t.closeTSM()
t.nextTSM()
}
if err := t.w.Write(key, values); err != nil {
if err == tsm1.ErrMaxBlocksExceeded {
t.closeTSM()
t.nextTSM()
} else {
t.err = err
}
}
}
func (t *shardWriter) Close() {
if t.w != nil {
t.closeTSM()
}
}
func (t *shardWriter) Err() error { return t.err }
func (t *shardWriter) nextTSM() {
fileName := filepath.Join(t.path, strconv.Itoa(int(t.id)), fmt.Sprintf("%09d-%09d.%s", t.gen, t.seq, tsm1.TSMFileExtension))
t.seq++
fd, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
t.err = err
return
}
// Create the writer for the new TSM file.
t.w, err = tsm1.NewTSMWriter(fd)
if err != nil {
t.err = err
return
}
}
func (t *shardWriter) closeTSM() {
el := errlist.NewErrorList()
if err := t.w.WriteIndex(); err != nil && err != tsm1.ErrNoValues {
el.Add(err)
}
if err := t.w.Close(); err != nil {
el.Add(err)
}
err := el.Err()
if err != nil {
t.err = err
}
t.w = nil
}