Skip to content

Commit

Permalink
Fix compactions sometimes getting stuck
Browse files Browse the repository at this point in the history
I ran into an issue where the cache snapshotting seemed to stop
completely causing the cache to fill up and never recover.  I believe
this is due to the the Timer being reused incorrectly.  Instead,
use a Ticker that will fire more regularly and not require the resetting
logic (which was wrong).
  • Loading branch information
jwilder committed Feb 27, 2017
1 parent 9bf54d9 commit bea91f7
Showing 1 changed file with 3 additions and 6 deletions.
9 changes: 3 additions & 6 deletions tsdb/engine/tsm1/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -930,7 +930,7 @@ func (e *Engine) writeSnapshotAndCommit(closedFiles []string, snapshot *Cache) (

// compactCache continually checks if the WAL cache should be written to disk
func (e *Engine) compactCache(quit <-chan struct{}) {
t := time.NewTimer(time.Second)
t := time.NewTicker(time.Second)
defer t.Stop()
for {
select {
Expand All @@ -952,7 +952,6 @@ func (e *Engine) compactCache(quit <-chan struct{}) {
atomic.AddInt64(&e.stats.CacheCompactionDuration, time.Since(start).Nanoseconds())
}
}
t.Reset(time.Second)
}
}

Expand All @@ -970,7 +969,7 @@ func (e *Engine) ShouldCompactCache(lastWriteTime time.Time) bool {
}

func (e *Engine) compactTSMLevel(fast bool, level int, quit <-chan struct{}) {
t := time.NewTimer(time.Second)
t := time.NewTicker(time.Second)
defer t.Stop()

for {
Expand All @@ -984,12 +983,11 @@ func (e *Engine) compactTSMLevel(fast bool, level int, quit <-chan struct{}) {
s.Apply()
}
}
t.Reset(time.Second)
}
}

func (e *Engine) compactTSMFull(quit <-chan struct{}) {
t := time.NewTimer(time.Second)
t := time.NewTicker(time.Second)
defer t.Stop()

for {
Expand All @@ -1004,7 +1002,6 @@ func (e *Engine) compactTSMFull(quit <-chan struct{}) {
}

}
t.Reset(time.Second)
}
}

Expand Down

0 comments on commit bea91f7

Please sign in to comment.