enhance: add retry helper method for Trylock

The ctrd maybe hold the lock for a while. In the concurrent request, it
might fail to serve request frequently. For this case, pouchcontainer
should retry in the specific duration to reduce the number of failed
request.

Signed-off-by: Wei Fu <fuweid89@gmail.com>

ctrd/container.go

@@ -29,7 +29,8 @@ import (
 )
 
 var (
-	runtimeRoot = "/run"
+	runtimeRoot           = "/run"
+	defaultTrylockTimeout = 5 * time.Second
 )
 
 type containerPack struct {
@@ -55,7 +56,7 @@ func (c *Client) ContainerStats(ctx context.Context, id string) (*containerdtype
 
 // containerStats returns stats of the container.
 func (c *Client) containerStats(ctx context.Context, id string) (*containerdtypes.Metric, error) {
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return nil, errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)
@@ -196,7 +197,7 @@ func (c *Client) ContainerPIDs(ctx context.Context, id string) ([]int, error) {
 
 // containerPIDs returns the all processes's ids inside the container.
 func (c *Client) containerPIDs(ctx context.Context, id string) ([]int, error) {
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return nil, errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)
@@ -255,7 +256,7 @@ func (c *Client) recoverContainer(ctx context.Context, id string, io *containeri
 		return fmt.Errorf("failed to get a containerd grpc client: %v", err)
 	}
 
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)
@@ -317,7 +318,7 @@ func (c *Client) destroyContainer(ctx context.Context, id string, timeout int64)
 
 	ctx = leases.WithLease(ctx, wrapperCli.lease.ID())
 
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return nil, errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)
@@ -385,7 +386,7 @@ func (c *Client) PauseContainer(ctx context.Context, id string) error {
 
 // pauseContainer pause container.
 func (c *Client) pauseContainer(ctx context.Context, id string) error {
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)
@@ -416,7 +417,7 @@ func (c *Client) UnpauseContainer(ctx context.Context, id string) error {
 
 // unpauseContainer unpauses a container.
 func (c *Client) unpauseContainer(ctx context.Context, id string) error {
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)
@@ -444,7 +445,7 @@ func (c *Client) CreateContainer(ctx context.Context, container *Container, chec
 		id  = container.ID
 	)
 
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)
@@ -596,7 +597,7 @@ func (c *Client) UpdateResources(ctx context.Context, id string, resources types
 
 // updateResources updates the configurations of a container.
 func (c *Client) updateResources(ctx context.Context, id string, resources types.Resources) error {
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)
@@ -626,7 +627,7 @@ func (c *Client) ResizeContainer(ctx context.Context, id string, opts types.Resi
 // resizeContainer changes the size of the TTY of the init process running
 // in the container to the given height and width.
 func (c *Client) resizeContainer(ctx context.Context, id string, opts types.ResizeOptions) error {
-	if !c.lock.Trylock(id) {
+	if !c.lock.TrylockWithRetry(ctx, id) {
 		return errtypes.ErrLockfailed
 	}
 	defer c.lock.Unlock(id)

ctrd/container_lock.go

@@ -1,7 +1,10 @@
 package ctrd
 
 import (
+	"context"
+	"math/rand"
 	"sync"
+	"time"
 )
 
 // containerLock use to make sure that only one operates the container at the same time.
@@ -27,3 +30,25 @@ func (l *containerLock) Unlock(id string) {
 	defer l.mutex.Unlock()
 	delete(l.ids, id)
 }
+
+func (l *containerLock) TrylockWithRetry(ctx context.Context, id string) bool {
+	var retry = 32
+
+	for {
+		ok := l.Trylock(id)
+		if ok {
+			return true
+		}
+
+		// sleep random duration by retry
+		select {
+		case <-time.After(time.Millisecond * time.Duration(rand.Intn(retry))):
+			if retry < 2048 {
+				retry = retry << 1
+			}
+			continue
+		case <-ctx.Done():
+			return false
+		}
+	}
+}

ctrd/container_lock_test.go

@@ -1,11 +1,54 @@
 package ctrd
 
 import (
+	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
 
+func Test_containerLock_TrylockWithRetry(t *testing.T) {
+	l := &containerLock{
+		ids: make(map[string]struct{}),
+	}
+
+	// basically, if the releaseTimeout < the tryLockTimeout,
+	// TrylockWithTimeout will lock successfully. If not, it will fail.
+	runTrylockWithT := func(tryLockTimeout, releaseTimeout time.Duration) bool {
+		id := "c"
+		assert.Equal(t, l.Trylock(id), true)
+		defer l.Unlock(id)
+
+		var (
+			releaseCh = make(chan struct{})
+			waitCh    = make(chan bool)
+			res       bool
+		)
+
+		go func() {
+			close(releaseCh)
+			ctx, cancel := context.WithTimeout(context.TODO(), tryLockTimeout)
+			defer cancel()
+			waitCh <- l.TrylockWithRetry(ctx, id)
+		}()
+
+		<-releaseCh
+		time.Sleep(releaseTimeout)
+		l.Unlock(id)
+
+		select {
+		case <-time.After(3 * time.Second):
+			t.Fatalf("timeout to get the Trylock result")
+		case res = <-waitCh:
+		}
+		return res
+	}
+
+	assert.Equal(t, true, runTrylockWithT(5*time.Second, 200*time.Millisecond))
+	assert.Equal(t, false, runTrylockWithT(200*time.Millisecond, 500*time.Millisecond))
+}
+
 func Test_containerLock_Trylock(t *testing.T) {
 	l := &containerLock{
 		ids: make(map[string]struct{}),