diff --git a/README.md b/README.md index bcf73587d..2b5b25bd8 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,13 @@ The `server/kit` package embodies Gizmo's goals to combine with go-kit. * Services using this package are expected to deploy to GCP. +#### [`observe`](https://godoc.org/github.com/NYTimes/gizmo/observe) + +The `observe` package provides observability helpers for metrics and tracing through OpenCensus + +* `server/kit` (and soon SimpleServer) utilizes this packge to create a StackDriver exporter with sane defaults +* `GoogleProjectID` and `IsGAE` can help you make decisions about the underlying platform + #### [`auth`](https://godoc.org/github.com/NYTimes/gizmo/auth) The `auth` package provides primitives for verifying inbound authentication tokens: diff --git a/observe/observe.go b/observe/observe.go new file mode 100644 index 000000000..e2ab334ef --- /dev/null +++ b/observe/observe.go @@ -0,0 +1,104 @@ +// Package observe provides functions +// that help with setting tracing/metrics +// in cloud providers, mainly GCP. +package observe + +import ( + "context" + "os" + + traceapi "cloud.google.com/go/trace/apiv2" + "contrib.go.opencensus.io/exporter/stackdriver" + "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + "golang.org/x/oauth2/google" + "google.golang.org/api/option" +) + +// NewStackdriverExporter will return the tracing and metrics through +// the stack driver exporter, if exists in the underlying platform. +// If exporter is registered, it returns the exporter so you can register +// it and ensure to call Flush on termination. +func NewStackdriverExporter(projectID string, onErr func(error)) (*stackdriver.Exporter, error) { + _, svcName, svcVersion := GetServiceInfo() + opts := getSDOpts(projectID, svcName, svcVersion, onErr) + if opts == nil { + return nil, nil + } + return stackdriver.NewExporter(*opts) +} + +// GoogleProjectID returns the GCP Project ID +// that can be used to instantiate various +// GCP clients such as Stack Driver. +func GoogleProjectID() string { + return os.Getenv("GOOGLE_CLOUD_PROJECT") +} + +// IsGAE tells you whether your program is running +// within the App Engine platform. +func IsGAE() bool { + return os.Getenv("GAE_DEPLOYMENT_ID") != "" +} + +// GetGAEInfo returns the GCP Project ID, +// the service, and the version of the application. +func GetGAEInfo() (projectID, service, version string) { + return GoogleProjectID(), + os.Getenv("GAE_SERVICE"), + os.Getenv("GAE_VERSION") +} + +// GetServiceInfo returns the GCP Project ID, +// the service name and version (GAE or through +// SERVICE_NAME/SERVICE_VERSION env vars). Note +// that SERVICE_NAME/SERVICE_VERSION are not standard but +// your application can pass them in as variables +// to be included in your trace attributes +func GetServiceInfo() (projectID, service, version string) { + if IsGAE() { + return GetGAEInfo() + } + return GoogleProjectID(), os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION") +} + +// getSDOpts returns Stack Driver Options that you can pass directly +// to the OpenCensus exporter or other libraries. +func getSDOpts(projectID, service, version string, onErr func(err error)) *stackdriver.Options { + var mr monitoredresource.Interface + + // this is so that you can export views from your local server up to SD if you wish + creds, err := google.FindDefaultCredentials(context.Background(), traceapi.DefaultAuthScopes()...) + if err != nil { + return nil + } + canExport := IsGAE() + if m := monitoredresource.Autodetect(); m != nil { + mr = m + canExport = true + } + if !canExport { + return nil + } + + return &stackdriver.Options{ + ProjectID: projectID, + MonitoredResource: mr, + MonitoringClientOptions: []option.ClientOption{ + option.WithCredentials(creds), + }, + TraceClientOptions: []option.ClientOption{ + option.WithCredentials(creds), + }, + OnError: onErr, + DefaultTraceAttributes: map[string]interface{}{ + "service": service, + "version": version, + }, + } +} + +// IsGCPEnabled returns whether the running application +// is inside GCP or has access to its products. +func IsGCPEnabled() bool { + return monitoredresource.Autodetect() != nil || IsGAE() +} diff --git a/server/kit/README.md b/server/kit/README.md index 602a655f8..0f324c35a 100644 --- a/server/kit/README.md +++ b/server/kit/README.md @@ -12,6 +12,7 @@ Since NYT uses Google Cloud, deploying this server to that environment provides * Automatically catch any panics and send them to Stackdriver Error reporting * Automatically use Stackdriver logging and, if `kit.LogXXX` functions are used, logs will be trace enabled and will be combined with their parent access log in the Stackdriver logging console. * Automatically register Stackdriver exporters for Open Census trace and monitoring. Most Google Cloud clients (like [Cloud Spanner](https://godoc.org/cloud.google.com/go/spanner)) will detect this and emit the traces. Users can also add their own trace and monitoring spans via [the Open Census clients](https://godoc.org/go.opencensus.io/trace#example-StartSpan). + * Monitoring, traces and metrics are automatically registered if running within App Engine, Kubernetes Engine, Compute Engine or AWS EC2 Instances. To change the name and version for Error reporting and Traces use `SERVICE_NAME` and `SERVICE_VERSION` environment variables. For an example of how to build a server that utilizes this package, see the [Reading List example](https://github.com/NYTimes/gizmo/tree/master/examples/servers/reading-list#the-reading-list-example). diff --git a/server/kit/kitserver.go b/server/kit/kitserver.go index cc40da300..5360926a6 100644 --- a/server/kit/kitserver.go +++ b/server/kit/kitserver.go @@ -8,17 +8,19 @@ import ( "net" "net/http" "net/http/pprof" - "os" "strings" "cloud.google.com/go/errorreporting" sdpropagation "contrib.go.opencensus.io/exporter/stackdriver/propagation" + "github.com/NYTimes/gizmo/observe" "github.com/go-kit/kit/log" httptransport "github.com/go-kit/kit/transport/http" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/pkg/errors" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/plugin/ochttp" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" "go.opencensus.io/trace/propagation" ocontext "golang.org/x/net/context" "google.golang.org/grpc" @@ -28,6 +30,7 @@ import ( type Server struct { logger log.Logger logClose func() error + ocFlush func() errs *errorreporting.Client @@ -89,27 +92,26 @@ func NewServer(svc Service) *Server { propr propagation.HTTPFormat ) - projectID := googleProjectID() - var svcName, svcVersion string - if isGAE() { - _, svcName, svcVersion = getGAEInfo() - } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { - svcName, svcVersion = n, v + projectID, svcName, svcVersion := observe.GetServiceInfo() + onErr := func(err error) { + lg.Log("error", err, "message", "exporter client encountered an error") } - - if opt := sdExporterOptions(projectID, svcName, svcVersion, lg); opt != nil { - err = initSDExporter(*opt) + ocFlush := func() {} + if observe.IsGCPEnabled() { + exp, err := observe.NewStackdriverExporter(projectID, onErr) if err != nil { lg.Log("error", err, "message", "unable to initiate error tracing exporter") } + ocFlush = exp.Flush + trace.RegisterExporter(exp) + view.RegisterExporter(exp) propr = &sdpropagation.HTTPFormat{} errs, err = errorreporting.NewClient(ctx, projectID, errorreporting.Config{ ServiceName: svcName, ServiceVersion: svcVersion, - OnError: func(err error) { lg.Log("error", err, "message", "error reporting client encountered an error") @@ -127,6 +129,7 @@ func NewServer(svc Service) *Server { exit: make(chan chan error), logger: lg, logClose: logClose, + ocFlush: ocFlush, errs: errs, } s.svr = &http.Server{ @@ -337,6 +340,11 @@ func (s *Server) start() error { s.logClose() } + // flush the stack driver exporter + if s.ocFlush != nil { + s.ocFlush() + } + if s.errs != nil { s.errs.Close() } diff --git a/server/kit/log.go b/server/kit/log.go index da9fd2642..bc8753506 100644 --- a/server/kit/log.go +++ b/server/kit/log.go @@ -4,6 +4,7 @@ import ( "context" "os" + "github.com/NYTimes/gizmo/observe" "github.com/go-kit/kit/log" "github.com/go-kit/kit/transport/http" "google.golang.org/grpc/metadata" @@ -23,7 +24,7 @@ import ( // If an empty string is provided, "gae_log" will be used in App Engine and "stdout" elsewhere. // For more information about to use of logID see the documentation here: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#FIELDS.log_name func NewLogger(ctx context.Context, logID string) (log.Logger, func() error, error) { - projectID, serviceID, svcVersion := getGAEInfo() + projectID, serviceID, svcVersion := observe.GetServiceInfo() lg, cl, err := newStackdriverLogger(ctx, logID, projectID, serviceID, svcVersion) // if Stackdriver logger was not able to find information about monitored resource it returns nil. if err != nil { diff --git a/server/kit/sd_log.go b/server/kit/sd_log.go index 4c15870e3..d3f1e278a 100644 --- a/server/kit/sd_log.go +++ b/server/kit/sd_log.go @@ -5,29 +5,18 @@ import ( "encoding" "encoding/json" "fmt" - "os" "reflect" "strings" "cloud.google.com/go/logging" "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + "github.com/NYTimes/gizmo/observe" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "google.golang.org/genproto/googleapis/api/monitoredres" ) -// project, service, version -func getGAEInfo() (string, string, string) { - return googleProjectID(), - os.Getenv("GAE_SERVICE"), - os.Getenv("GAE_VERSION") -} - -func isGAE() bool { - return os.Getenv("GAE_DEPLOYMENT_ID") != "" -} - type sdLogger struct { project string monRes *monitoredres.MonitoredResource @@ -43,7 +32,7 @@ func newStackdriverLogger(ctx context.Context, logID, projectID, service, versio "version_id": version, }, } - if isGAE() { + if observe.IsGAE() { resource.Type = "gae_app" if logID == "" { logID = "app_logs" diff --git a/server/kit/stackdriver.go b/server/kit/stackdriver.go deleted file mode 100644 index 88161657b..000000000 --- a/server/kit/stackdriver.go +++ /dev/null @@ -1,66 +0,0 @@ -package kit - -import ( - "os" - - "contrib.go.opencensus.io/exporter/stackdriver" - "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" - "github.com/go-kit/kit/log" - "go.opencensus.io/stats/view" - "go.opencensus.io/trace" -) - -func sdExporterOptions(projectID, service, version string, lg log.Logger) *stackdriver.Options { - var mr monitoredresource.Interface - if m := monitoredresource.Autodetect(); m != nil { - mr = m - } else if isGAE() { - mr = gaeInterface{ - typ: "gae_app", - labels: map[string]string{ - "project_id": projectID, - }, - } - } - if mr == nil { - return nil - } - - return &stackdriver.Options{ - ProjectID: projectID, - MonitoredResource: mr, - OnError: func(err error) { - lg.Log("error", err, - "message", "tracing client encountered an error") - }, - DefaultMonitoringLabels: &stackdriver.Labels{}, - DefaultTraceAttributes: map[string]interface{}{ - "service": service, - "version": version, - }, - } -} - -func googleProjectID() string { - return os.Getenv("GOOGLE_CLOUD_PROJECT") -} - -func initSDExporter(opt stackdriver.Options) error { - exporter, err := stackdriver.NewExporter(opt) - if err != nil { - return err - } - trace.RegisterExporter(exporter) - view.RegisterExporter(exporter) - return nil -} - -// implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface -type gaeInterface struct { - typ string - labels map[string]string -} - -func (g gaeInterface) MonitoredResource() (string, map[string]string) { - return g.typ, g.labels -}