Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(run): Add sigterm handler sample #1902

Merged
merged 25 commits into from
Apr 5, 2021
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions run/sigterm-handler/.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# The .dockerignore file excludes files from the container build process.
#
# https://docs.docker.com/engine/reference/builder/#dockerignore-file

# Exclude locally vendored dependencies.
vendor/

# Exclude "build-time" ignore files.
.dockerignore
.gcloudignore

# Exclude git history and configuration.
.gitignore
12 changes: 12 additions & 0 deletions run/sigterm-handler/.gcloudignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# The .gcloudignore file excludes file from upload to Cloud Build.
# If this file is deleted, gcloud will default to .gitignore.
#
# https://cloud.google.com/cloud-build/docs/speeding-up-builds#gcloudignore
# https://cloud.google.com/sdk/gcloud/reference/topic/gcloudignore

# Exclude locally vendored dependencies.
vendor/

# Exclude git history and configuration.
.git/
.gitignore
47 changes: 47 additions & 0 deletions run/sigterm-handler/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Copyright 2021 Google LLC
averikitsch marked this conversation as resolved.
Show resolved Hide resolved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Use the offical golang image to create a binary.
# This is based on Debian and sets the GOPATH to /go.
# https://hub.docker.com/_/golang
FROM golang:1.15-buster as builder

# Create and change to the app directory.
WORKDIR /app

# Retrieve application dependencies.
# This allows the container build to reuse cached dependencies.
# Expecting to copy go.mod and if present go.sum.
COPY go.* ./
RUN go mod download

# Copy local code to the container image.
COPY . ./

# Build the binary.
RUN go build -mod=readonly -v -o server

# Use the official Debian slim image for a lean production container.
# https://hub.docker.com/_/debian
# https://docs.docker.com/develop/develop-images/multistage-build/#use-multi-stage-builds
FROM debian:buster-slim
RUN set -x && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
ca-certificates && \
rm -rf /var/lib/apt/lists/*

# Copy the binary to the production image from the builder stage.
COPY --from=builder /app/server /app/server

# Run the web service on container startup.
CMD ["/app/server"]
3 changes: 3 additions & 0 deletions run/sigterm-handler/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
module github.com/GoogleCloudPlatform/golang-samples/run/sigterm-handler
averikitsch marked this conversation as resolved.
Show resolved Hide resolved

go 1.13
81 changes: 81 additions & 0 deletions run/sigterm-handler/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Sample sigterm-handler shows how to gracefully shut down in response to a SIGTERM signal.
package main

// [START cloudrun_sigterm_handler]
averikitsch marked this conversation as resolved.
Show resolved Hide resolved
import (
"context"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"syscall"
"time"
)

func main() {
// Determine port for HTTP service.
port := os.Getenv("PORT")
if port == "" {
port = "8080"
log.Printf("defaulting to port %s", port)
}

srv := &http.Server{
Addr: ":" + port,
Handler: http.HandlerFunc(handler),
}

// Create channel to listen for signals.
signalChan := make(chan os.Signal, 1)
// SIGINT handles Ctrl+C locally.
// SIGTERM handles Cloud Run termination signal.
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

// Start HTTP server.
go func() {
log.Printf("listening on port %s", port)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatal(err)
averikitsch marked this conversation as resolved.
Show resolved Hide resolved
}
}()

// Receive output from signalChan.
sig := <-signalChan
log.Printf("%s signal caught", sig)
log.Print("server stopped")
averikitsch marked this conversation as resolved.
Show resolved Hide resolved

// Timeout if waiting for connections to return idle.
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

defer func() {
// Add extra handling here to close any DB connections, redis, or flush logs.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure where the comment went. But, I'd expect any DB connections, etc. to handled outside this defer call. I might be missing something.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@grayside please advise

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure the idiomatic approach here, did a little searching to see how folks talk about this, and in the first 5 results, 2 of them do cleanup inside the defer and three just do them after the shutdown.

cancel() // Release resources.
}()

// Gracefully shutdown the server by waiting on existing requests (except websockets).
if err := srv.Shutdown(ctx); err != nil {
log.Printf("server shutdown failed:%+v", err)
}
log.Print("server exited")
}

// [END cloudrun_sigterm_handler]

func handler(w http.ResponseWriter, r *http.Request) {
averikitsch marked this conversation as resolved.
Show resolved Hide resolved
fmt.Fprint(w, "Hello World!\n")
}
30 changes: 30 additions & 0 deletions run/sigterm-handler/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"net/http/httptest"
"testing"
)

func TestHandler(t *testing.T) {
req := httptest.NewRequest("GET", "/", nil)
rr := httptest.NewRecorder()
handler(rr, req)
averikitsch marked this conversation as resolved.
Show resolved Hide resolved

if got := rr.Body.String(); got != "Hello World!\n" {
t.Errorf("got %q, want %q", got, "Hello World!\n")
}
}
8 changes: 5 additions & 3 deletions run/testing/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@ module github.com/GoogleCloudPlatform/golang-samples/run/testing
go 1.15

require (
github.com/GoogleCloudPlatform/golang-samples v0.0.0-20210319194002-1e35cc23a955
github.com/GoogleCloudPlatform/golang-samples/run/grpc-ping v0.0.0-20210319194002-1e35cc23a955
github.com/GoogleCloudPlatform/golang-samples/run/grpc-server-streaming v0.0.0-20210319194002-1e35cc23a955
cloud.google.com/go/logging v1.0.0
github.com/GoogleCloudPlatform/golang-samples v0.0.0-20201216233243-555da975282a
github.com/GoogleCloudPlatform/golang-samples/run/grpc-ping v0.0.0-20201216233243-555da975282a
github.com/GoogleCloudPlatform/golang-samples/run/grpc-server-streaming v0.0.0-20201216233243-555da975282a
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4
google.golang.org/api v0.42.0
google.golang.org/grpc v1.36.0
)

Expand Down
6 changes: 4 additions & 2 deletions run/testing/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/datastore v1.2.0/go.mod h1:FKd9dFEjRui5757lkOJ7z/eKtL74o5hsbY0o6Z0ozz8=
cloud.google.com/go/firestore v1.3.0/go.mod h1:Qt0gS9Qz9tROrmgFavo36+hdST1FXvmtnGnO0Dr03pU=
cloud.google.com/go/logging v1.0.0 h1:kaunpnoEh9L4hu6JUsBa8Y20LBfKnCuDhKUgdZp7oK8=
cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
Expand All @@ -55,8 +56,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/GoogleCloudPlatform/golang-samples/run/grpc-server-streaming v0.0.0-20210319194002-1e35cc23a955 h1:9+6yna6sfK5eYmgtMr342euvumMpDdwFy/emQPwpETk=
github.com/GoogleCloudPlatform/golang-samples/run/grpc-server-streaming v0.0.0-20210319194002-1e35cc23a955/go.mod h1:BcTipDbvpsgadsC1tMPJOKyWXIfnrZuQrMdrT9QQHjY=
github.com/GoogleCloudPlatform/golang-samples/run/grpc-server-streaming v0.0.0-20201216233243-555da975282a h1:3QoJzt0CuSC6UO+0mUN8LTR5NtE5AFzvSYr7dXJLdSM=
github.com/GoogleCloudPlatform/golang-samples/run/grpc-server-streaming v0.0.0-20201216233243-555da975282a/go.mod h1:YorXqDMZdELAYt365rOGQ5b9ieVdfkoJBEETNOboQzA=
github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.38.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
Expand Down Expand Up @@ -311,6 +312,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down
103 changes: 103 additions & 0 deletions run/testing/sigterm_handler.e2e_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cloudruntests

import (
"context"
"fmt"
"net/http"
"strings"
"testing"
"time"

"cloud.google.com/go/logging/logadmin"
"google.golang.org/api/iterator"

"github.com/GoogleCloudPlatform/golang-samples/internal/cloudrunci"
"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
)

func TestSigtermHandlerService(t *testing.T) {
tc := testutil.EndToEndTest(t)

runID := time.Now().Format("20060102-150405")
service := cloudrunci.NewService("sigterm-handler", tc.ProjectID)
service.Dir = "../sigterm-handler"
if err := service.Deploy(); err != nil {
t.Fatalf("service.Deploy %q: %v", service.Name, err)
}
defer GetLogEntries(service, runID, tc.ProjectID, t)
defer service.Clean()

requestPath := "/"
req, err := service.NewRequest("GET", requestPath)
if err != nil {
t.Fatalf("service.NewRequest: %v", err)
}

client := http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
t.Fatalf("client.Do: %v", err)
}
defer resp.Body.Close()
fmt.Printf("client.Do: %s %s\n", req.Method, req.URL)

if got := resp.StatusCode; got != http.StatusOK {
t.Errorf("response status: got %d, want %d", got, http.StatusOK)
}
}

func GetLogEntries(service *cloudrunci.Service, runID string, projectID string, t *testing.T) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Future: Nice function! Seems like a good candidate to move into cloudrunci.

ctx := context.Background()
client, err := logadmin.NewClient(ctx, projectID)
if err != nil {
t.Fatalf("Failed to create client: %v", err)
}
defer client.Close()

// Create service and timestamp filters
minsAgo := time.Now().Add(-5 * time.Minute)
timeFormat := minsAgo.Format(time.RFC3339)
filter := fmt.Sprintf(`resource.labels.service_name="%s" timestamp>="%s"`, fmt.Sprintf("%s-%s", service.Name, runID), timeFormat)
preparedFilter := fmt.Sprintf(`resource.type="cloud_run_revision" severity="default" %s NOT protoPayload.serviceName="run.googleapis.com"`, filter)

fmt.Println("Waiting for logs...")
averikitsch marked this conversation as resolved.
Show resolved Hide resolved
time.Sleep(3 * time.Minute)
MAX := 5
averikitsch marked this conversation as resolved.
Show resolved Hide resolved
fmt.Printf("Using log filter: %s\n", preparedFilter)
for i := 1; i < MAX; i++ {
fmt.Printf("Attempt #%d\n", i)
it := client.Entries(ctx, logadmin.Filter(preparedFilter))
for {
entry, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
t.Errorf("error fetching logs: %s", err)
}
if len(fmt.Sprintf("%v", entry.Payload)) > 0 {
fmt.Printf("Found log: %v\n", entry.Payload)
}
if strings.Contains(fmt.Sprintf("%v", entry.Payload), "terminated signal caught") {
fmt.Println("SIGTERM log entry: found.")
return
}
}
time.Sleep(30 * time.Second)
}
t.Error("SIGTERM log entry: not found.")
}
3 changes: 2 additions & 1 deletion testing/kokoro/configure_gcloud.bash
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ fi

gcloud -q components update
gcloud -q components install app-engine-go
gcloud -q components install beta # Install beta component needed for Cloud Run E2E tests
gcloud -q components install beta # Needed for Cloud Run E2E tests until --pack goes to GA
gcloud -q components install alpha # Needed for Cloud Run E2E tests until --use-http2 goes GA

# Set config.
gcloud config set disable_prompts True
Expand Down