use mocked time for testing request heding

Signed-off-by: Jake Moshenko <jacob.moshenko@gmail.com>

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/authzed/authzed-go v0.1.1-0.20210923172306-b4b512e4d359
 	github.com/authzed/grpcutil v0.0.0-20210914195113-c0d8369e7e1f
 	github.com/aws/aws-sdk-go v1.40.53
+	github.com/benbjohnson/clock v1.1.0
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cespare/xxhash v1.1.0
 	github.com/containerd/continuity v0.0.0-20210315143101-93e15499afd5 // indirect

go.sum

@@ -73,6 +73,8 @@ github.com/authzed/grpcutil v0.0.0-20210914195113-c0d8369e7e1f/go.mod h1:HwO/KbR
 github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.40.53 h1:wi4UAslOQ1HfF2NjnIwI6st8n7sQg7shUUNLkaCgIpc=
 github.com/aws/aws-sdk-go v1.40.53/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

internal/datastore/proxy/hedging.go

@@ -8,6 +8,7 @@ import (
 
 	v0 "github.com/authzed/authzed-go/proto/authzed/api/v0"
 	v1 "github.com/authzed/authzed-go/proto/authzed/api/v1"
+	"github.com/benbjohnson/clock"
 	"github.com/influxdata/tdigest"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -40,6 +41,7 @@ type subrequest func(ctx context.Context, responseReady chan<- struct{})
 type hedger func(ctx context.Context, req subrequest)
 
 func newHedger(
+	timeSource clock.Clock,
 	initialSlowRequestThreshold time.Duration,
 	maxSampleCount uint64,
 	quantile float64,
@@ -65,8 +67,8 @@ func newHedger(
 		digestLock.Unlock()
 		slowRequestThreshold := time.Duration(slowRequestThresholdSeconds * float64(time.Second))
 
-		timer := time.NewTimer(slowRequestThreshold)
-		start := time.Now()
+		timer := timeSource.Timer(slowRequestThreshold)
+		start := timeSource.Now()
 
 		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
@@ -83,7 +85,7 @@ func newHedger(
 		}
 
 		// Compute how long it took for us to get any answer
-		duration := time.Since(start)
+		duration := timeSource.Since(start)
 		digestLock.Lock()
 		defer digestLock.Unlock()
 
@@ -121,6 +123,22 @@ func NewHedgingProxy(
 	initialSlowRequestThreshold time.Duration,
 	maxSampleCount uint64,
 	hedgingQuantile float64,
+) datastore.Datastore {
+	return newHedgingProxyWithTimeSource(
+		delegate,
+		initialSlowRequestThreshold,
+		maxSampleCount,
+		hedgingQuantile,
+		clock.New(),
+	)
+}
+
+func newHedgingProxyWithTimeSource(
+	delegate datastore.Datastore,
+	initialSlowRequestThreshold time.Duration,
+	maxSampleCount uint64,
+	hedgingQuantile float64,
+	timeSource clock.Clock,
 ) datastore.Datastore {
 	if initialSlowRequestThreshold < 0 {
 		panic("initial slow request threshold negative")
@@ -136,10 +154,10 @@ func NewHedgingProxy(
 
 	return hedgingProxy{
 		delegate,
-		newHedger(initialSlowRequestThreshold, maxSampleCount, hedgingQuantile),
-		newHedger(initialSlowRequestThreshold, maxSampleCount, hedgingQuantile),
-		newHedger(initialSlowRequestThreshold, maxSampleCount, hedgingQuantile),
-		newHedger(initialSlowRequestThreshold, maxSampleCount, hedgingQuantile),
+		newHedger(timeSource, initialSlowRequestThreshold, maxSampleCount, hedgingQuantile),
+		newHedger(timeSource, initialSlowRequestThreshold, maxSampleCount, hedgingQuantile),
+		newHedger(timeSource, initialSlowRequestThreshold, maxSampleCount, hedgingQuantile),
+		newHedger(timeSource, initialSlowRequestThreshold, maxSampleCount, hedgingQuantile),
 	}
 }