-
Notifications
You must be signed in to change notification settings - Fork 178
/
compactor.go
136 lines (113 loc) · 3 KB
/
compactor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
package wal
import (
"fmt"
"io"
"sync"
"time"
)
type Compactor struct {
checkpointer *Checkpointer
done chan struct{}
stopc chan struct{}
wg sync.WaitGroup
sync.Mutex
interval time.Duration
checkpointDistance uint
checkpointsToKeep uint
}
func NewCompactor(checkpointer *Checkpointer, interval time.Duration, checkpointDistance uint, checkpointsToKeep uint) *Compactor {
if checkpointDistance < 1 {
checkpointDistance = 1
}
return &Compactor{
checkpointer: checkpointer,
done: make(chan struct{}),
stopc: make(chan struct{}),
interval: interval,
checkpointDistance: checkpointDistance,
checkpointsToKeep: checkpointsToKeep,
}
}
// Ready periodically fires Run function, every `interval`
// If called more then once, behaviour is undefined.
func (c *Compactor) Ready() <-chan struct{} {
ch := make(chan struct{})
c.wg.Add(1)
go c.start()
defer close(ch)
return ch
}
func (c *Compactor) Done() <-chan struct{} {
c.stopc <- struct{}{}
ch := make(chan struct{})
go func() {
c.wg.Wait()
close(ch)
}()
return ch
}
func (c *Compactor) start() {
for {
//TODO Log error
_ = c.Run()
select {
case <-c.stopc:
c.wg.Done()
return
case <-time.After(c.interval):
}
}
}
func (c *Compactor) Run() error {
c.Lock()
defer c.Unlock()
err := c.createCheckpoints()
if err != nil {
return fmt.Errorf("cannot create checkpoints: %w", err)
}
err = c.cleanupCheckpoints()
if err != nil {
return fmt.Errorf("cannot cleanup checkpoints: %w", err)
}
return nil
}
func (c *Compactor) createCheckpoints() error {
from, to, err := c.checkpointer.NotCheckpointedSegments()
if err != nil {
return fmt.Errorf("cannot get latest checkpoint: %w", err)
}
fmt.Printf("creating a checkpoint from segment %d to segment %d\n", from, to)
// more then one segment means we can checkpoint safely up to `to`-1
// presumably last segment is being written to
if to-from > int(c.checkpointDistance) {
checkpointNumber := to - 1
fmt.Printf("checkpointing to %d\n", checkpointNumber)
err = c.checkpointer.Checkpoint(checkpointNumber, func() (io.WriteCloser, error) {
return c.checkpointer.CheckpointWriter(checkpointNumber)
})
if err != nil {
return fmt.Errorf("error creating checkpoint (%d): %w", checkpointNumber, err)
}
}
return nil
}
func (c *Compactor) cleanupCheckpoints() error {
// don't bother listing checkpoints if we keep them all
if c.checkpointsToKeep == 0 {
return nil
}
checkpoints, err := c.checkpointer.Checkpoints()
if err != nil {
return fmt.Errorf("cannot list checkpoints: %w", err)
}
if len(checkpoints) > int(c.checkpointsToKeep) {
checkpointsToRemove := checkpoints[:len(checkpoints)-int(c.checkpointsToKeep)] // if condition guarantees this never fails
for _, checkpoint := range checkpointsToRemove {
err := c.checkpointer.RemoveCheckpoint(checkpoint)
if err != nil {
return fmt.Errorf("cannot remove checkpoint %d: %w", checkpoint, err)
}
}
}
return nil
}