-
Notifications
You must be signed in to change notification settings - Fork 279
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* core/grpc: fix graceful stop * core/http: add graceful stop serve
- Loading branch information
1 parent
33b4662
commit 2c91ea6
Showing
7 changed files
with
311 additions
and
48 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
package grpcutil | ||
|
||
import ( | ||
"context" | ||
"net" | ||
"time" | ||
|
||
"google.golang.org/grpc" | ||
) | ||
|
||
// ServeWithGracefulStop serves the gRPC listener until ctx.Done(), and then gracefully stops and waits for gracefulTimeout | ||
// before definitively stopping. | ||
func ServeWithGracefulStop(ctx context.Context, srv *grpc.Server, li net.Listener, gracefulTimeout time.Duration) error { | ||
go func() { | ||
// wait for the context to complete | ||
<-ctx.Done() | ||
|
||
sctx, stopped := context.WithCancel(context.Background()) | ||
go func() { | ||
srv.GracefulStop() | ||
stopped() | ||
}() | ||
|
||
wait := time.NewTimer(gracefulTimeout) | ||
defer wait.Stop() | ||
|
||
select { | ||
case <-wait.C: | ||
case <-sctx.Done(): | ||
return | ||
} | ||
|
||
// finally stop it completely | ||
srv.Stop() | ||
}() | ||
|
||
return srv.Serve(li) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,100 @@ | ||
package grpcutil_test | ||
|
||
import ( | ||
"context" | ||
"net" | ||
"testing" | ||
"time" | ||
|
||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
"golang.org/x/sync/errgroup" | ||
"google.golang.org/grpc" | ||
"google.golang.org/grpc/credentials/insecure" | ||
"google.golang.org/grpc/health" | ||
"google.golang.org/grpc/health/grpc_health_v1" | ||
|
||
"github.com/pomerium/pomerium/pkg/grpcutil" | ||
) | ||
|
||
func TestServeWithGracefulStop(t *testing.T) { | ||
t.Parallel() | ||
|
||
t.Run("immediate", func(t *testing.T) { | ||
t.Parallel() | ||
|
||
li, err := net.Listen("tcp4", "127.0.0.1:0") | ||
require.NoError(t, err) | ||
|
||
srv := grpc.NewServer() | ||
|
||
ctx, cancel := context.WithCancel(context.Background()) | ||
cancel() | ||
|
||
now := time.Now() | ||
err = grpcutil.ServeWithGracefulStop(ctx, srv, li, time.Millisecond*100) | ||
elapsed := time.Since(now) | ||
assert.Nil(t, err) | ||
assert.Less(t, elapsed, time.Millisecond*100, "should complete immediately") | ||
}) | ||
t.Run("graceful", func(t *testing.T) { | ||
t.Parallel() | ||
|
||
li, err := net.Listen("tcp4", "127.0.0.1:0") | ||
require.NoError(t, err) | ||
|
||
srv := grpc.NewServer() | ||
hsrv := health.NewServer() | ||
grpc_health_v1.RegisterHealthServer(srv, hsrv) | ||
hsrv.SetServingStatus("test", grpc_health_v1.HealthCheckResponse_SERVING) | ||
|
||
now := time.Now() | ||
ctx, cancel := context.WithCancel(context.Background()) | ||
eg, ctx := errgroup.WithContext(ctx) | ||
eg.Go(func() error { | ||
return grpcutil.ServeWithGracefulStop(ctx, srv, li, time.Millisecond*100) | ||
}) | ||
eg.Go(func() error { | ||
var cc *grpc.ClientConn | ||
for { | ||
var err error | ||
cc, err = grpc.Dial(li.Addr().String(), | ||
grpc.WithTransportCredentials(insecure.NewCredentials())) | ||
if err != nil { | ||
continue | ||
} | ||
|
||
break | ||
} | ||
|
||
c := grpc_health_v1.NewHealthClient(cc) | ||
|
||
// wait till the server is ready | ||
for { | ||
_, err := c.Check(ctx, &grpc_health_v1.HealthCheckRequest{ | ||
Service: "test", | ||
}) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
break | ||
} | ||
|
||
// start streaming to hold open the server during graceful stop | ||
_, err = c.Watch(context.Background(), &grpc_health_v1.HealthCheckRequest{ | ||
Service: "test", | ||
}) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
cancel() | ||
|
||
return nil | ||
}) | ||
eg.Wait() | ||
elapsed := time.Since(now) | ||
assert.Greater(t, elapsed, time.Millisecond*100, "should complete after 100ms") | ||
}) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
// Package httputil contains additional functionality for working with http. | ||
package httputil | ||
|
||
import ( | ||
"context" | ||
"errors" | ||
"net" | ||
"net/http" | ||
"time" | ||
|
||
"github.com/pomerium/pomerium/pkg/contextutil" | ||
) | ||
|
||
// ServeWithGracefulStop serves the HTTP listener until ctx.Done(), and then gracefully stops and waits for gracefulTimeout | ||
// before definitively stopping. | ||
func ServeWithGracefulStop(ctx context.Context, handler http.Handler, li net.Listener, gracefulTimeout time.Duration) error { | ||
// create a context that will be used for the http requests | ||
// it will only be cancelled when baseCancel is called but will | ||
// preserve the values from ctx | ||
baseCtx, baseCancel := context.WithCancelCause(contextutil.OnlyValues(ctx)) | ||
|
||
srv := http.Server{ | ||
Handler: handler, | ||
BaseContext: func(l net.Listener) context.Context { | ||
return baseCtx | ||
}, | ||
} | ||
|
||
go func() { | ||
<-ctx.Done() | ||
|
||
// create a context that will cancel after the graceful timeout | ||
timeoutCtx, clearTimeout := context.WithTimeout(context.Background(), gracefulTimeout) | ||
defer clearTimeout() | ||
|
||
// shut the http server down | ||
_ = srv.Shutdown(timeoutCtx) | ||
|
||
// cancel the base context used for http requests | ||
baseCancel(ctx.Err()) | ||
}() | ||
|
||
err := srv.Serve(li) | ||
if errors.Is(err, http.ErrServerClosed) { | ||
err = nil | ||
} | ||
return err | ||
} |
Oops, something went wrong.