-
Notifications
You must be signed in to change notification settings - Fork 166
/
compactor.go
152 lines (132 loc) · 3.81 KB
/
compactor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
package wal
import (
"fmt"
"io"
"sync"
"time"
"github.com/rs/zerolog"
"github.com/onflow/flow-go/module/lifecycle"
"github.com/onflow/flow-go/module/observable"
)
type Compactor struct {
checkpointer *Checkpointer
logger zerolog.Logger
stopc chan struct{}
lm *lifecycle.LifecycleManager
sync.Mutex
observers map[observable.Observer]struct{}
interval time.Duration
checkpointDistance uint
checkpointsToKeep uint
}
func NewCompactor(checkpointer *Checkpointer, interval time.Duration, checkpointDistance uint, checkpointsToKeep uint, logger zerolog.Logger) *Compactor {
if checkpointDistance < 1 {
checkpointDistance = 1
}
return &Compactor{
checkpointer: checkpointer,
logger: logger,
stopc: make(chan struct{}),
observers: make(map[observable.Observer]struct{}),
lm: lifecycle.NewLifecycleManager(),
interval: interval,
checkpointDistance: checkpointDistance,
checkpointsToKeep: checkpointsToKeep,
}
}
func (c *Compactor) Subscribe(observer observable.Observer) {
var void struct{}
c.observers[observer] = void
}
func (c *Compactor) Unsubscribe(observer observable.Observer) {
delete(c.observers, observer)
}
// Ready periodically fires Run function, every `interval`
func (c *Compactor) Ready() <-chan struct{} {
c.lm.OnStart(func() {
go c.start()
})
return c.lm.Started()
}
func (c *Compactor) Done() <-chan struct{} {
c.lm.OnStop(func() {
for observer := range c.observers {
observer.OnComplete()
}
c.stopc <- struct{}{}
})
return c.lm.Stopped()
}
func (c *Compactor) start() {
for {
err := c.Run()
if err != nil {
c.logger.Error().Err(err).Msg("error running compactor")
}
select {
case <-c.stopc:
return
case <-time.After(c.interval):
}
}
}
func (c *Compactor) Run() error {
c.Lock()
defer c.Unlock()
newLatestCheckpoint, err := c.createCheckpoints()
if err != nil {
return fmt.Errorf("cannot create checkpoints: %w", err)
}
err = c.cleanupCheckpoints()
if err != nil {
return fmt.Errorf("cannot cleanup checkpoints: %w", err)
}
if newLatestCheckpoint > 0 {
for observer := range c.observers {
observer.OnNext(newLatestCheckpoint)
}
}
return nil
}
func (c *Compactor) createCheckpoints() (int, error) {
from, to, err := c.checkpointer.NotCheckpointedSegments()
if err != nil {
return -1, fmt.Errorf("cannot get latest checkpoint: %w", err)
}
// we only return a positive value if the latest checkpoint index has changed
newLatestCheckpoint := -1
// more then one segment means we can checkpoint safely up to `to`-1
// presumably last segment is being written to
if to-from > int(c.checkpointDistance) {
checkpointNumber := to - 1
c.logger.Info().Msgf("creating a checkpoint from segment %d to segment %d\n", from, checkpointNumber)
err = c.checkpointer.Checkpoint(checkpointNumber, func() (io.WriteCloser, error) {
return c.checkpointer.CheckpointWriter(checkpointNumber)
})
if err != nil {
return -1, fmt.Errorf("error creating checkpoint (%d): %w", checkpointNumber, err)
}
newLatestCheckpoint = checkpointNumber
}
return newLatestCheckpoint, nil
}
func (c *Compactor) cleanupCheckpoints() error {
// don't bother listing checkpoints if we keep them all
if c.checkpointsToKeep == 0 {
return nil
}
checkpoints, err := c.checkpointer.Checkpoints()
if err != nil {
return fmt.Errorf("cannot list checkpoints: %w", err)
}
if len(checkpoints) > int(c.checkpointsToKeep) {
checkpointsToRemove := checkpoints[:len(checkpoints)-int(c.checkpointsToKeep)] // if condition guarantees this never fails
for _, checkpoint := range checkpointsToRemove {
err := c.checkpointer.RemoveCheckpoint(checkpoint)
if err != nil {
return fmt.Errorf("cannot remove checkpoint %d: %w", checkpoint, err)
}
}
}
return nil
}