Skip to content

Commit

Permalink
Extract GCP Monitoring (#192)
Browse files Browse the repository at this point in the history
* Extract GCP Monitoring

* remove s

* add pkg doc

* return exporter

* return flush func

* try another approach

* bypass if check

* gcputil -> observe

* fix doc

* pr fixes

* fix gae metrics

* pr fixes

* pr fixes

* doc fixes

* better err msg

* log fix

* remove prom from observe

* sd brand

* fix build

* observe README
  • Loading branch information
marwan-at-work authored and jprobinson committed Feb 12, 2019
1 parent dc1269d commit 73cf43d
Show file tree
Hide file tree
Showing 7 changed files with 135 additions and 91 deletions.
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ The `server/kit` package embodies Gizmo's goals to combine with go-kit.
* Services using this package are expected to deploy to GCP.


#### [`observe`](https://godoc.org/github.com/NYTimes/gizmo/observe)

The `observe` package provides observability helpers for metrics and tracing through OpenCensus

* `server/kit` (and soon SimpleServer) utilizes this packge to create a StackDriver exporter with sane defaults
* `GoogleProjectID` and `IsGAE` can help you make decisions about the underlying platform

#### [`auth`](https://godoc.org/github.com/NYTimes/gizmo/auth)

The `auth` package provides primitives for verifying inbound authentication tokens:
Expand Down
104 changes: 104 additions & 0 deletions observe/observe.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
// Package observe provides functions
// that help with setting tracing/metrics
// in cloud providers, mainly GCP.
package observe

import (
"context"
"os"

traceapi "cloud.google.com/go/trace/apiv2"
"contrib.go.opencensus.io/exporter/stackdriver"
"contrib.go.opencensus.io/exporter/stackdriver/monitoredresource"
"golang.org/x/oauth2/google"
"google.golang.org/api/option"
)

// NewStackdriverExporter will return the tracing and metrics through
// the stack driver exporter, if exists in the underlying platform.
// If exporter is registered, it returns the exporter so you can register
// it and ensure to call Flush on termination.
func NewStackdriverExporter(projectID string, onErr func(error)) (*stackdriver.Exporter, error) {
_, svcName, svcVersion := GetServiceInfo()
opts := getSDOpts(projectID, svcName, svcVersion, onErr)
if opts == nil {
return nil, nil
}
return stackdriver.NewExporter(*opts)
}

// GoogleProjectID returns the GCP Project ID
// that can be used to instantiate various
// GCP clients such as Stack Driver.
func GoogleProjectID() string {
return os.Getenv("GOOGLE_CLOUD_PROJECT")
}

// IsGAE tells you whether your program is running
// within the App Engine platform.
func IsGAE() bool {
return os.Getenv("GAE_DEPLOYMENT_ID") != ""
}

// GetGAEInfo returns the GCP Project ID,
// the service, and the version of the application.
func GetGAEInfo() (projectID, service, version string) {
return GoogleProjectID(),
os.Getenv("GAE_SERVICE"),
os.Getenv("GAE_VERSION")
}

// GetServiceInfo returns the GCP Project ID,
// the service name and version (GAE or through
// SERVICE_NAME/SERVICE_VERSION env vars). Note
// that SERVICE_NAME/SERVICE_VERSION are not standard but
// your application can pass them in as variables
// to be included in your trace attributes
func GetServiceInfo() (projectID, service, version string) {
if IsGAE() {
return GetGAEInfo()
}
return GoogleProjectID(), os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION")
}

// getSDOpts returns Stack Driver Options that you can pass directly
// to the OpenCensus exporter or other libraries.
func getSDOpts(projectID, service, version string, onErr func(err error)) *stackdriver.Options {
var mr monitoredresource.Interface

// this is so that you can export views from your local server up to SD if you wish
creds, err := google.FindDefaultCredentials(context.Background(), traceapi.DefaultAuthScopes()...)
if err != nil {
return nil
}
canExport := IsGAE()
if m := monitoredresource.Autodetect(); m != nil {
mr = m
canExport = true
}
if !canExport {
return nil
}

return &stackdriver.Options{
ProjectID: projectID,
MonitoredResource: mr,
MonitoringClientOptions: []option.ClientOption{
option.WithCredentials(creds),
},
TraceClientOptions: []option.ClientOption{
option.WithCredentials(creds),
},
OnError: onErr,
DefaultTraceAttributes: map[string]interface{}{
"service": service,
"version": version,
},
}
}

// IsGCPEnabled returns whether the running application
// is inside GCP or has access to its products.
func IsGCPEnabled() bool {
return monitoredresource.Autodetect() != nil || IsGAE()
}
1 change: 1 addition & 0 deletions server/kit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Since NYT uses Google Cloud, deploying this server to that environment provides
* Automatically catch any panics and send them to Stackdriver Error reporting
* Automatically use Stackdriver logging and, if `kit.LogXXX` functions are used, logs will be trace enabled and will be combined with their parent access log in the Stackdriver logging console.
* Automatically register Stackdriver exporters for Open Census trace and monitoring. Most Google Cloud clients (like [Cloud Spanner](https://godoc.org/cloud.google.com/go/spanner)) will detect this and emit the traces. Users can also add their own trace and monitoring spans via [the Open Census clients](https://godoc.org/go.opencensus.io/trace#example-StartSpan).
* Monitoring, traces and metrics are automatically registered if running within App Engine, Kubernetes Engine, Compute Engine or AWS EC2 Instances. To change the name and version for Error reporting and Traces use `SERVICE_NAME` and `SERVICE_VERSION` environment variables.


For an example of how to build a server that utilizes this package, see the [Reading List example](https://github.com/NYTimes/gizmo/tree/master/examples/servers/reading-list#the-reading-list-example).
30 changes: 19 additions & 11 deletions server/kit/kitserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,19 @@ import (
"net"
"net/http"
"net/http/pprof"
"os"
"strings"

"cloud.google.com/go/errorreporting"
sdpropagation "contrib.go.opencensus.io/exporter/stackdriver/propagation"
"github.com/NYTimes/gizmo/observe"
"github.com/go-kit/kit/log"
httptransport "github.com/go-kit/kit/transport/http"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
"github.com/pkg/errors"
"go.opencensus.io/plugin/ocgrpc"
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/stats/view"
"go.opencensus.io/trace"
"go.opencensus.io/trace/propagation"
ocontext "golang.org/x/net/context"
"google.golang.org/grpc"
Expand All @@ -28,6 +30,7 @@ import (
type Server struct {
logger log.Logger
logClose func() error
ocFlush func()

errs *errorreporting.Client

Expand Down Expand Up @@ -89,27 +92,26 @@ func NewServer(svc Service) *Server {
propr propagation.HTTPFormat
)

projectID := googleProjectID()
var svcName, svcVersion string
if isGAE() {
_, svcName, svcVersion = getGAEInfo()
} else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" {
svcName, svcVersion = n, v
projectID, svcName, svcVersion := observe.GetServiceInfo()
onErr := func(err error) {
lg.Log("error", err, "message", "exporter client encountered an error")
}

if opt := sdExporterOptions(projectID, svcName, svcVersion, lg); opt != nil {
err = initSDExporter(*opt)
ocFlush := func() {}
if observe.IsGCPEnabled() {
exp, err := observe.NewStackdriverExporter(projectID, onErr)
if err != nil {
lg.Log("error", err,
"message", "unable to initiate error tracing exporter")
}
ocFlush = exp.Flush
trace.RegisterExporter(exp)
view.RegisterExporter(exp)

propr = &sdpropagation.HTTPFormat{}

errs, err = errorreporting.NewClient(ctx, projectID, errorreporting.Config{
ServiceName: svcName,
ServiceVersion: svcVersion,

OnError: func(err error) {
lg.Log("error", err,
"message", "error reporting client encountered an error")
Expand All @@ -127,6 +129,7 @@ func NewServer(svc Service) *Server {
exit: make(chan chan error),
logger: lg,
logClose: logClose,
ocFlush: ocFlush,
errs: errs,
}
s.svr = &http.Server{
Expand Down Expand Up @@ -337,6 +340,11 @@ func (s *Server) start() error {
s.logClose()
}

// flush the stack driver exporter
if s.ocFlush != nil {
s.ocFlush()
}

if s.errs != nil {
s.errs.Close()
}
Expand Down
3 changes: 2 additions & 1 deletion server/kit/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"os"

"github.com/NYTimes/gizmo/observe"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/transport/http"
"google.golang.org/grpc/metadata"
Expand All @@ -23,7 +24,7 @@ import (
// If an empty string is provided, "gae_log" will be used in App Engine and "stdout" elsewhere.
// For more information about to use of logID see the documentation here: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#FIELDS.log_name
func NewLogger(ctx context.Context, logID string) (log.Logger, func() error, error) {
projectID, serviceID, svcVersion := getGAEInfo()
projectID, serviceID, svcVersion := observe.GetServiceInfo()
lg, cl, err := newStackdriverLogger(ctx, logID, projectID, serviceID, svcVersion)
// if Stackdriver logger was not able to find information about monitored resource it returns nil.
if err != nil {
Expand Down
15 changes: 2 additions & 13 deletions server/kit/sd_log.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,18 @@ import (
"encoding"
"encoding/json"
"fmt"
"os"
"reflect"
"strings"

"cloud.google.com/go/logging"
"contrib.go.opencensus.io/exporter/stackdriver/monitoredresource"
"github.com/NYTimes/gizmo/observe"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/pkg/errors"
"google.golang.org/genproto/googleapis/api/monitoredres"
)

// project, service, version
func getGAEInfo() (string, string, string) {
return googleProjectID(),
os.Getenv("GAE_SERVICE"),
os.Getenv("GAE_VERSION")
}

func isGAE() bool {
return os.Getenv("GAE_DEPLOYMENT_ID") != ""
}

type sdLogger struct {
project string
monRes *monitoredres.MonitoredResource
Expand All @@ -43,7 +32,7 @@ func newStackdriverLogger(ctx context.Context, logID, projectID, service, versio
"version_id": version,
},
}
if isGAE() {
if observe.IsGAE() {
resource.Type = "gae_app"
if logID == "" {
logID = "app_logs"
Expand Down
66 changes: 0 additions & 66 deletions server/kit/stackdriver.go

This file was deleted.

0 comments on commit 73cf43d

Please sign in to comment.