deadlock-detector.go

/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"os"
	"sync"
	"time"

	"github.com/golang/glog"
)

type rwMutexToLockableAdapter struct {
	rw *sync.RWMutex
}

func (r *rwMutexToLockableAdapter) Lock() {
	r.rw.RLock()
}

func (r *rwMutexToLockableAdapter) Unlock() {
	r.rw.RUnlock()
}

type deadlockDetector struct {
	name          string
	lock          sync.Locker
	maxLockPeriod time.Duration
	exiter        exiter
	exitChannelFn func() <-chan time.Time
	// Really only useful for testing
	stopChannel <-chan bool
}

// DeadlockWatchdogReadLock creates a watchdog on read/write mutex.  If the mutex can not be acquired
// for read access within 'maxLockPeriod', the program exits via glog.Exitf() or os.Exit() if that fails
// 'name' is a semantic name that is useful for the user and is printed on exit.
func DeadlockWatchdogReadLock(lock *sync.RWMutex, name string, maxLockPeriod time.Duration) {
	DeadlockWatchdog(&rwMutexToLockableAdapter{lock}, name, maxLockPeriod)
}

func DeadlockWatchdog(lock sync.Locker, name string, maxLockPeriod time.Duration) {
	if maxLockPeriod <= 0 {
		panic("maxLockPeriod is <= 0, that can't be what you wanted")
	}
	detector := &deadlockDetector{
		lock:          lock,
		name:          name,
		maxLockPeriod: maxLockPeriod,
		exitChannelFn: func() <-chan time.Time { return time.After(maxLockPeriod) },
		stopChannel:   make(chan bool),
	}
	go detector.run()
}

// Useful for injecting tests
type exiter interface {
	Exitf(format string, args ...interface{})
}

type realExiter struct{}

func (realExiter) Exitf(format string, args ...interface{}) {
	func() {
		defer func() {
			// Let's just be extra sure we die, even if Exitf panics
			if r := recover(); r != nil {
				glog.Errorf(format, args...)
				os.Exit(2)
			}
		}()
		glog.Exitf(format, args...)
	}()
}

func (d *deadlockDetector) run() {
	for {
		if !d.runOnce() {
			return
		}
		time.Sleep(d.maxLockPeriod / 2)
	}
}

func (d *deadlockDetector) runOnce() bool {
	ch := make(chan bool, 1)
	go func() {
		d.lock.Lock()
		d.lock.Unlock()

		ch <- true
	}()
	exitCh := d.exitChannelFn()
	select {
	case <-exitCh:
		d.exiter.Exitf("Deadlock on %s, exiting", d.name)
		// return is here for when we use a fake exiter in testing
		return false
	case <-ch:
		glog.V(6).Infof("%s is not deadlocked", d.name)
	case <-d.stopChannel:
		glog.V(4).Infof("Stopping deadlock detector for %s", d.name)
		return false
	}
	return true
}