From 4eeeff5218f13c94a40a878cfad6a97d5af00a17 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Sat, 9 Feb 2019 13:53:46 -0500 Subject: [PATCH 01/13] Extract GCP Monitoring --- gcputils/gcputils.go | 99 +++++++++++++++++++++++++++++++++++++++ server/kit/kitserver.go | 15 ++++-- server/kit/log.go | 3 +- server/kit/sd_log.go | 15 +----- server/kit/stackdriver.go | 66 -------------------------- 5 files changed, 113 insertions(+), 85 deletions(-) create mode 100644 gcputils/gcputils.go delete mode 100644 server/kit/stackdriver.go diff --git a/gcputils/gcputils.go b/gcputils/gcputils.go new file mode 100644 index 000000000..bb45602fd --- /dev/null +++ b/gcputils/gcputils.go @@ -0,0 +1,99 @@ +package gcputils + +import ( + "os" + + "contrib.go.opencensus.io/exporter/stackdriver" + "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" +) + +// RegisterOpenCensus will register +// the tracing and metrics through +// the stack driver exporter, if exists in the underlying platform. +func RegisterOpenCensus(projectID string, onErr func(error)) error { + svcName, svcVersion := "", "" + if IsGAE() { + _, svcName, svcVersion = GetGAEInfo() + } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { + svcName, svcVersion = n, v + } + opts := SDExporterOptions(projectID, svcName, svcVersion, onErr) + if opts == nil { + return nil + } + return InitSDExporter(*opts) +} + +// GoogleProjectID returns the GCP Project ID +// that can be used to instantiate various +// GCP clients such as Stack Driver. +func GoogleProjectID() string { + return os.Getenv("GOOGLE_CLOUD_PROJECT") +} + +// IsGAE tells you whether your program is running +// within the App Engine platform. +func IsGAE() bool { + return os.Getenv("GAE_DEPLOYMENT_ID") != "" +} + +// GetGAEInfo returns the GCP Project ID, +// the service, and the version of the application. +func GetGAEInfo() (projectID, service, version string) { + return GoogleProjectID(), + os.Getenv("GAE_SERVICE"), + os.Getenv("GAE_VERSION") +} + +// SDExporterOptions returns Stack Driver Options that you can pass directly +// to the OpenCensus exporter or other libraries. +func SDExporterOptions(projectID, service, version string, onErr func(err error)) *stackdriver.Options { + var mr monitoredresource.Interface + if m := monitoredresource.Autodetect(); m != nil { + mr = m + } else if IsGAE() { + mr = gaeInterface{ + typ: "gae_app", + labels: map[string]string{ + "project_id": projectID, + }, + } + } + if mr == nil { + return nil + } + + return &stackdriver.Options{ + ProjectID: projectID, + MonitoredResource: mr, + OnError: onErr, + DefaultMonitoringLabels: &stackdriver.Labels{}, + DefaultTraceAttributes: map[string]interface{}{ + "service": service, + "version": version, + }, + } +} + +// InitSDExporter will initialize the OpenCensus tracing/metrics exporter +func InitSDExporter(opts stackdriver.Options) error { + exporter, err := stackdriver.NewExporter(opts) + if err != nil { + return err + } + trace.RegisterExporter(exporter) + view.RegisterExporter(exporter) + return nil +} + +// implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface +type gaeInterface struct { + typ string + labels map[string]string +} + +func (g gaeInterface) MonitoredResource() (string, map[string]string) { + return g.typ, g.labels +} diff --git a/server/kit/kitserver.go b/server/kit/kitserver.go index cc40da300..8649c2369 100644 --- a/server/kit/kitserver.go +++ b/server/kit/kitserver.go @@ -13,6 +13,7 @@ import ( "cloud.google.com/go/errorreporting" sdpropagation "contrib.go.opencensus.io/exporter/stackdriver/propagation" + "github.com/NYTimes/gizmo/gcputils" "github.com/go-kit/kit/log" httptransport "github.com/go-kit/kit/transport/http" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" @@ -89,16 +90,20 @@ func NewServer(svc Service) *Server { propr propagation.HTTPFormat ) - projectID := googleProjectID() + projectID := gcputils.GoogleProjectID() var svcName, svcVersion string - if isGAE() { - _, svcName, svcVersion = getGAEInfo() + if gcputils.IsGAE() { + _, svcName, svcVersion = gcputils.GetGAEInfo() } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { svcName, svcVersion = n, v } - if opt := sdExporterOptions(projectID, svcName, svcVersion, lg); opt != nil { - err = initSDExporter(*opt) + onErr := func(err error) { + lg.Log("error", err, "message", "tracing client encountered an error") + } + + if opt := gcputils.SDExporterOptions(projectID, svcName, svcVersion, onErr); opt != nil { + err = gcputils.InitSDExporter(*opt) if err != nil { lg.Log("error", err, "message", "unable to initiate error tracing exporter") diff --git a/server/kit/log.go b/server/kit/log.go index da9fd2642..8d883b085 100644 --- a/server/kit/log.go +++ b/server/kit/log.go @@ -4,6 +4,7 @@ import ( "context" "os" + "github.com/NYTimes/gizmo/gcputils" "github.com/go-kit/kit/log" "github.com/go-kit/kit/transport/http" "google.golang.org/grpc/metadata" @@ -23,7 +24,7 @@ import ( // If an empty string is provided, "gae_log" will be used in App Engine and "stdout" elsewhere. // For more information about to use of logID see the documentation here: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#FIELDS.log_name func NewLogger(ctx context.Context, logID string) (log.Logger, func() error, error) { - projectID, serviceID, svcVersion := getGAEInfo() + projectID, serviceID, svcVersion := gcputils.GetGAEInfo() lg, cl, err := newStackdriverLogger(ctx, logID, projectID, serviceID, svcVersion) // if Stackdriver logger was not able to find information about monitored resource it returns nil. if err != nil { diff --git a/server/kit/sd_log.go b/server/kit/sd_log.go index 4c15870e3..a3654f0cd 100644 --- a/server/kit/sd_log.go +++ b/server/kit/sd_log.go @@ -5,29 +5,18 @@ import ( "encoding" "encoding/json" "fmt" - "os" "reflect" "strings" "cloud.google.com/go/logging" "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + "github.com/NYTimes/gizmo/gcputils" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "google.golang.org/genproto/googleapis/api/monitoredres" ) -// project, service, version -func getGAEInfo() (string, string, string) { - return googleProjectID(), - os.Getenv("GAE_SERVICE"), - os.Getenv("GAE_VERSION") -} - -func isGAE() bool { - return os.Getenv("GAE_DEPLOYMENT_ID") != "" -} - type sdLogger struct { project string monRes *monitoredres.MonitoredResource @@ -43,7 +32,7 @@ func newStackdriverLogger(ctx context.Context, logID, projectID, service, versio "version_id": version, }, } - if isGAE() { + if gcputils.IsGAE() { resource.Type = "gae_app" if logID == "" { logID = "app_logs" diff --git a/server/kit/stackdriver.go b/server/kit/stackdriver.go deleted file mode 100644 index 88161657b..000000000 --- a/server/kit/stackdriver.go +++ /dev/null @@ -1,66 +0,0 @@ -package kit - -import ( - "os" - - "contrib.go.opencensus.io/exporter/stackdriver" - "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" - "github.com/go-kit/kit/log" - "go.opencensus.io/stats/view" - "go.opencensus.io/trace" -) - -func sdExporterOptions(projectID, service, version string, lg log.Logger) *stackdriver.Options { - var mr monitoredresource.Interface - if m := monitoredresource.Autodetect(); m != nil { - mr = m - } else if isGAE() { - mr = gaeInterface{ - typ: "gae_app", - labels: map[string]string{ - "project_id": projectID, - }, - } - } - if mr == nil { - return nil - } - - return &stackdriver.Options{ - ProjectID: projectID, - MonitoredResource: mr, - OnError: func(err error) { - lg.Log("error", err, - "message", "tracing client encountered an error") - }, - DefaultMonitoringLabels: &stackdriver.Labels{}, - DefaultTraceAttributes: map[string]interface{}{ - "service": service, - "version": version, - }, - } -} - -func googleProjectID() string { - return os.Getenv("GOOGLE_CLOUD_PROJECT") -} - -func initSDExporter(opt stackdriver.Options) error { - exporter, err := stackdriver.NewExporter(opt) - if err != nil { - return err - } - trace.RegisterExporter(exporter) - view.RegisterExporter(exporter) - return nil -} - -// implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface -type gaeInterface struct { - typ string - labels map[string]string -} - -func (g gaeInterface) MonitoredResource() (string, map[string]string) { - return g.typ, g.labels -} From d49ac47fea2548be5f54ad6dbd3d2dc23df58c86 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Sat, 9 Feb 2019 14:07:43 -0500 Subject: [PATCH 02/13] remove s --- gcputils/gcputils.go => gcputil/gcputil.go | 2 +- server/kit/kitserver.go | 12 ++++++------ server/kit/log.go | 4 ++-- server/kit/sd_log.go | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) rename gcputils/gcputils.go => gcputil/gcputil.go (99%) diff --git a/gcputils/gcputils.go b/gcputil/gcputil.go similarity index 99% rename from gcputils/gcputils.go rename to gcputil/gcputil.go index bb45602fd..f939070a5 100644 --- a/gcputils/gcputils.go +++ b/gcputil/gcputil.go @@ -1,4 +1,4 @@ -package gcputils +package gcputil import ( "os" diff --git a/server/kit/kitserver.go b/server/kit/kitserver.go index 8649c2369..d3e20b21c 100644 --- a/server/kit/kitserver.go +++ b/server/kit/kitserver.go @@ -13,7 +13,7 @@ import ( "cloud.google.com/go/errorreporting" sdpropagation "contrib.go.opencensus.io/exporter/stackdriver/propagation" - "github.com/NYTimes/gizmo/gcputils" + "github.com/NYTimes/gizmo/gcputil" "github.com/go-kit/kit/log" httptransport "github.com/go-kit/kit/transport/http" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" @@ -90,10 +90,10 @@ func NewServer(svc Service) *Server { propr propagation.HTTPFormat ) - projectID := gcputils.GoogleProjectID() + projectID := gcputil.GoogleProjectID() var svcName, svcVersion string - if gcputils.IsGAE() { - _, svcName, svcVersion = gcputils.GetGAEInfo() + if gcputil.IsGAE() { + _, svcName, svcVersion = gcputil.GetGAEInfo() } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { svcName, svcVersion = n, v } @@ -102,8 +102,8 @@ func NewServer(svc Service) *Server { lg.Log("error", err, "message", "tracing client encountered an error") } - if opt := gcputils.SDExporterOptions(projectID, svcName, svcVersion, onErr); opt != nil { - err = gcputils.InitSDExporter(*opt) + if opt := gcputil.SDExporterOptions(projectID, svcName, svcVersion, onErr); opt != nil { + err = gcputil.InitSDExporter(*opt) if err != nil { lg.Log("error", err, "message", "unable to initiate error tracing exporter") diff --git a/server/kit/log.go b/server/kit/log.go index 8d883b085..ceef14a74 100644 --- a/server/kit/log.go +++ b/server/kit/log.go @@ -4,7 +4,7 @@ import ( "context" "os" - "github.com/NYTimes/gizmo/gcputils" + "github.com/NYTimes/gizmo/gcputil" "github.com/go-kit/kit/log" "github.com/go-kit/kit/transport/http" "google.golang.org/grpc/metadata" @@ -24,7 +24,7 @@ import ( // If an empty string is provided, "gae_log" will be used in App Engine and "stdout" elsewhere. // For more information about to use of logID see the documentation here: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#FIELDS.log_name func NewLogger(ctx context.Context, logID string) (log.Logger, func() error, error) { - projectID, serviceID, svcVersion := gcputils.GetGAEInfo() + projectID, serviceID, svcVersion := gcputil.GetGAEInfo() lg, cl, err := newStackdriverLogger(ctx, logID, projectID, serviceID, svcVersion) // if Stackdriver logger was not able to find information about monitored resource it returns nil. if err != nil { diff --git a/server/kit/sd_log.go b/server/kit/sd_log.go index a3654f0cd..a48b91ad2 100644 --- a/server/kit/sd_log.go +++ b/server/kit/sd_log.go @@ -10,7 +10,7 @@ import ( "cloud.google.com/go/logging" "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" - "github.com/NYTimes/gizmo/gcputils" + "github.com/NYTimes/gizmo/gcputil" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" @@ -32,7 +32,7 @@ func newStackdriverLogger(ctx context.Context, logID, projectID, service, versio "version_id": version, }, } - if gcputils.IsGAE() { + if gcputil.IsGAE() { resource.Type = "gae_app" if logID == "" { logID = "app_logs" From 0615a5e487759d1b9200dbf5a2c2918bd9718b50 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Sat, 9 Feb 2019 14:10:33 -0500 Subject: [PATCH 03/13] add pkg doc --- gcputil/gcputil.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gcputil/gcputil.go b/gcputil/gcputil.go index f939070a5..5133a4f16 100644 --- a/gcputil/gcputil.go +++ b/gcputil/gcputil.go @@ -1,3 +1,7 @@ +// Package gcputil provides functions +// that interact with various GCP platforms +// such as getting GAE metadata and registering +// stack driver monitoring package gcputil import ( From fff7ddeb3e87ea278edbd9596627596005922296 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Sat, 9 Feb 2019 14:50:39 -0500 Subject: [PATCH 04/13] return exporter --- gcputil/gcputil.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/gcputil/gcputil.go b/gcputil/gcputil.go index 5133a4f16..827dfad9a 100644 --- a/gcputil/gcputil.go +++ b/gcputil/gcputil.go @@ -16,7 +16,9 @@ import ( // RegisterOpenCensus will register // the tracing and metrics through // the stack driver exporter, if exists in the underlying platform. -func RegisterOpenCensus(projectID string, onErr func(error)) error { +// If exporter is registered, it returns the exporter so you can call Flush +// when the server terminates to ensure all metrics are uploaded. +func RegisterOpenCensus(projectID string, onErr func(error)) (*stackdriver.Exporter, error) { svcName, svcVersion := "", "" if IsGAE() { _, svcName, svcVersion = GetGAEInfo() @@ -25,9 +27,9 @@ func RegisterOpenCensus(projectID string, onErr func(error)) error { } opts := SDExporterOptions(projectID, svcName, svcVersion, onErr) if opts == nil { - return nil + return nil, nil } - return InitSDExporter(*opts) + return initSDExporterV2(*opts) } // GoogleProjectID returns the GCP Project ID @@ -92,6 +94,18 @@ func InitSDExporter(opts stackdriver.Options) error { return nil } +// TODO(marwan): this is so that I don't break server/kit +// see https://github.com/NYTimes/gizmo/issues/193 +func initSDExporterV2(opts stackdriver.Options) (*stackdriver.Exporter, error) { + exporter, err := stackdriver.NewExporter(opts) + if err != nil { + return nil, err + } + trace.RegisterExporter(exporter) + view.RegisterExporter(exporter) + return exporter, nil +} + // implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface type gaeInterface struct { typ string From af57a1b68733dcf8de0e27b43b0add025a711a99 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Sun, 10 Feb 2019 17:03:07 -0500 Subject: [PATCH 05/13] return flush func --- gcputil/gcputil.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/gcputil/gcputil.go b/gcputil/gcputil.go index 827dfad9a..70268f87e 100644 --- a/gcputil/gcputil.go +++ b/gcputil/gcputil.go @@ -13,12 +13,11 @@ import ( "go.opencensus.io/trace" ) -// RegisterOpenCensus will register -// the tracing and metrics through +// RegisterOpenCensus will register the tracing and metrics through // the stack driver exporter, if exists in the underlying platform. -// If exporter is registered, it returns the exporter so you can call Flush -// when the server terminates to ensure all metrics are uploaded. -func RegisterOpenCensus(projectID string, onErr func(error)) (*stackdriver.Exporter, error) { +// If exporter is registered, it returns the exporter's flush function so you can +// ensure all metrics are uploaded. The flush function is safe to call. +func RegisterOpenCensus(projectID string, onErr func(error)) (flush func(), err error) { svcName, svcVersion := "", "" if IsGAE() { _, svcName, svcVersion = GetGAEInfo() @@ -27,9 +26,13 @@ func RegisterOpenCensus(projectID string, onErr func(error)) (*stackdriver.Expor } opts := SDExporterOptions(projectID, svcName, svcVersion, onErr) if opts == nil { - return nil, nil + return func() {}, nil } - return initSDExporterV2(*opts) + exp, err := initSDExporterV2(*opts) + if err != nil { + return func() {}, nil + } + return exp.Flush, nil } // GoogleProjectID returns the GCP Project ID From ecee882ec20467eb6671bede7e906b52f61dfddc Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Mon, 11 Feb 2019 14:25:40 -0500 Subject: [PATCH 06/13] try another approach --- gcputil/gcputil.go | 28 +++------- ocexporters/ocexporters.go | 111 +++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 ocexporters/ocexporters.go diff --git a/gcputil/gcputil.go b/gcputil/gcputil.go index 70268f87e..4f3dfb217 100644 --- a/gcputil/gcputil.go +++ b/gcputil/gcputil.go @@ -13,11 +13,11 @@ import ( "go.opencensus.io/trace" ) -// RegisterOpenCensus will register the tracing and metrics through +// NewOpenCensusExporter will return the tracing and metrics through // the stack driver exporter, if exists in the underlying platform. -// If exporter is registered, it returns the exporter's flush function so you can -// ensure all metrics are uploaded. The flush function is safe to call. -func RegisterOpenCensus(projectID string, onErr func(error)) (flush func(), err error) { +// If exporter is registered, it returns the exporter so you can register +// it and ensure to call Flush on termination. +func NewOpenCensusExporter(projectID string, onErr func(error)) (*stackdriver.Exporter, error) { svcName, svcVersion := "", "" if IsGAE() { _, svcName, svcVersion = GetGAEInfo() @@ -26,13 +26,13 @@ func RegisterOpenCensus(projectID string, onErr func(error)) (flush func(), err } opts := SDExporterOptions(projectID, svcName, svcVersion, onErr) if opts == nil { - return func() {}, nil + return nil, nil } - exp, err := initSDExporterV2(*opts) + exp, err := stackdriver.NewExporter(*opts) if err != nil { - return func() {}, nil + return nil, err } - return exp.Flush, nil + return exp, nil } // GoogleProjectID returns the GCP Project ID @@ -97,18 +97,6 @@ func InitSDExporter(opts stackdriver.Options) error { return nil } -// TODO(marwan): this is so that I don't break server/kit -// see https://github.com/NYTimes/gizmo/issues/193 -func initSDExporterV2(opts stackdriver.Options) (*stackdriver.Exporter, error) { - exporter, err := stackdriver.NewExporter(opts) - if err != nil { - return nil, err - } - trace.RegisterExporter(exporter) - view.RegisterExporter(exporter) - return exporter, nil -} - // implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface type gaeInterface struct { typ string diff --git a/ocexporters/ocexporters.go b/ocexporters/ocexporters.go new file mode 100644 index 000000000..77d62d621 --- /dev/null +++ b/ocexporters/ocexporters.go @@ -0,0 +1,111 @@ +package ocexporters + +import ( + "fmt" + "os" + + "contrib.go.opencensus.io/exporter/stackdriver" + "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + "go.opencensus.io/exporter/prometheus" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" +) + +// NewOCExporters returns trace/client exporters +// based on the given environment. ProjectID +// is the GCP projectID for stackdriver (or the prometheus namespace) and backend +// specifies whether the backend is StackDriver or Prometheus +func NewOCExporters(projectID, backend string, onErr func(err error)) (trace.Exporter, view.Exporter, error) { + switch backend { + case "stackdriver": + return getSDExporter(projectID, onErr) + case "prometheus": + return getPrometheusExporter(projectID, onErr) + } + + return nil, nil, fmt.Errorf("unrecognized backend: %v", backend) +} + +func getPrometheusExporter(namespace string, onErr func(error)) (trace.Exporter, view.Exporter, error) { + pe, err := prometheus.NewExporter(prometheus.Options{ + Namespace: namespace, + OnError: onErr, + }) + return nil, pe, err +} + +func getSDExporter(projectID string, onErr func(error)) (trace.Exporter, view.Exporter, error) { + svcName, svcVersion := "", "" + if IsGAE() { + _, svcName, svcVersion = GetGAEInfo() + } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { + svcName, svcVersion = n, v + } + opts := getSDExporterOptions(projectID, svcName, svcVersion, onErr) + if opts == nil { + return nil, nil, nil + } + exp, err := stackdriver.NewExporter(*opts) + return exp, exp, err +} + +// getSDExporterOptions returns Stack Driver Options that you can pass directly +// to the OpenCensus exporter or other libraries. +func getSDExporterOptions(projectID, service, version string, onErr func(err error)) *stackdriver.Options { + var mr monitoredresource.Interface + if m := monitoredresource.Autodetect(); m != nil { + mr = m + } else if IsGAE() { + mr = gaeInterface{ + typ: "gae_app", + labels: map[string]string{ + "project_id": projectID, + }, + } + } + if mr == nil { + return nil + } + + return &stackdriver.Options{ + ProjectID: projectID, + MonitoredResource: mr, + OnError: onErr, + DefaultMonitoringLabels: &stackdriver.Labels{}, + DefaultTraceAttributes: map[string]interface{}{ + "service": service, + "version": version, + }, + } +} + +// implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface +type gaeInterface struct { + typ string + labels map[string]string +} + +func (g gaeInterface) MonitoredResource() (string, map[string]string) { + return g.typ, g.labels +} + +// GoogleProjectID returns the GCP Project ID +// that can be used to instantiate various +// GCP clients such as Stack Driver. +func GoogleProjectID() string { + return os.Getenv("GOOGLE_CLOUD_PROJECT") +} + +// IsGAE tells you whether your program is running +// within the App Engine platform. +func IsGAE() bool { + return os.Getenv("GAE_DEPLOYMENT_ID") != "" +} + +// GetGAEInfo returns the GCP Project ID, +// the service, and the version of the application. +func GetGAEInfo() (projectID, service, version string) { + return GoogleProjectID(), + os.Getenv("GAE_SERVICE"), + os.Getenv("GAE_VERSION") +} From e2aa9e665c9047b42aed0d40edf8cdee1e3b08c8 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Mon, 11 Feb 2019 16:37:19 -0500 Subject: [PATCH 07/13] bypass if check --- gcputil/gcputil.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gcputil/gcputil.go b/gcputil/gcputil.go index 4f3dfb217..c2b4f80c7 100644 --- a/gcputil/gcputil.go +++ b/gcputil/gcputil.go @@ -28,11 +28,7 @@ func NewOpenCensusExporter(projectID string, onErr func(error)) (*stackdriver.Ex if opts == nil { return nil, nil } - exp, err := stackdriver.NewExporter(*opts) - if err != nil { - return nil, err - } - return exp, nil + return stackdriver.NewExporter(*opts) } // GoogleProjectID returns the GCP Project ID From bfdbe4d04f631115b026c2c3bf5859ba60f7d9c3 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Mon, 11 Feb 2019 17:50:12 -0500 Subject: [PATCH 08/13] gcputil -> observe --- gcputil/gcputil.go => observe/observe.go | 32 ++++--- ocexporters/ocexporters.go | 111 ----------------------- server/kit/kitserver.go | 27 ++++-- server/kit/log.go | 4 +- server/kit/sd_log.go | 4 +- 5 files changed, 44 insertions(+), 134 deletions(-) rename gcputil/gcputil.go => observe/observe.go (75%) delete mode 100644 ocexporters/ocexporters.go diff --git a/gcputil/gcputil.go b/observe/observe.go similarity index 75% rename from gcputil/gcputil.go rename to observe/observe.go index c2b4f80c7..161b3df7a 100644 --- a/gcputil/gcputil.go +++ b/observe/observe.go @@ -1,23 +1,23 @@ -// Package gcputil provides functions -// that interact with various GCP platforms -// such as getting GAE metadata and registering -// stack driver monitoring -package gcputil +// Package observe provides functions +// that help with setting observability +// in cloud provides, mainly GCP. +package observe import ( "os" "contrib.go.opencensus.io/exporter/stackdriver" "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + "go.opencensus.io/exporter/prometheus" "go.opencensus.io/stats/view" "go.opencensus.io/trace" ) -// NewOpenCensusExporter will return the tracing and metrics through +// NewStackDriverExporter will return the tracing and metrics through // the stack driver exporter, if exists in the underlying platform. // If exporter is registered, it returns the exporter so you can register // it and ensure to call Flush on termination. -func NewOpenCensusExporter(projectID string, onErr func(error)) (*stackdriver.Exporter, error) { +func NewStackDriverExporter(projectID string, onErr func(error)) (*stackdriver.Exporter, error) { svcName, svcVersion := "", "" if IsGAE() { _, svcName, svcVersion = GetGAEInfo() @@ -31,6 +31,11 @@ func NewOpenCensusExporter(projectID string, onErr func(error)) (*stackdriver.Ex return stackdriver.NewExporter(*opts) } +// NewPrometheusExporter return a prometheus Exporter for OpenCensus. +func NewPrometheusExporter(opts prometheus.Options) (*prometheus.Exporter, error) { + return NewPrometheusExporter(opts) +} + // GoogleProjectID returns the GCP Project ID // that can be used to instantiate various // GCP clients such as Stack Driver. @@ -71,10 +76,9 @@ func SDExporterOptions(projectID, service, version string, onErr func(err error) } return &stackdriver.Options{ - ProjectID: projectID, - MonitoredResource: mr, - OnError: onErr, - DefaultMonitoringLabels: &stackdriver.Labels{}, + ProjectID: projectID, + MonitoredResource: mr, + OnError: onErr, DefaultTraceAttributes: map[string]interface{}{ "service": service, "version": version, @@ -82,6 +86,12 @@ func SDExporterOptions(projectID, service, version string, onErr func(err error) } } +// IsGCPAccessible returns whether the running application +// is inside GCP or has access to its products. +func IsGCPAccessible() bool { + return monitoredresource.Autodetect() != nil || IsGAE() +} + // InitSDExporter will initialize the OpenCensus tracing/metrics exporter func InitSDExporter(opts stackdriver.Options) error { exporter, err := stackdriver.NewExporter(opts) diff --git a/ocexporters/ocexporters.go b/ocexporters/ocexporters.go deleted file mode 100644 index 77d62d621..000000000 --- a/ocexporters/ocexporters.go +++ /dev/null @@ -1,111 +0,0 @@ -package ocexporters - -import ( - "fmt" - "os" - - "contrib.go.opencensus.io/exporter/stackdriver" - "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" - "go.opencensus.io/exporter/prometheus" - "go.opencensus.io/stats/view" - "go.opencensus.io/trace" -) - -// NewOCExporters returns trace/client exporters -// based on the given environment. ProjectID -// is the GCP projectID for stackdriver (or the prometheus namespace) and backend -// specifies whether the backend is StackDriver or Prometheus -func NewOCExporters(projectID, backend string, onErr func(err error)) (trace.Exporter, view.Exporter, error) { - switch backend { - case "stackdriver": - return getSDExporter(projectID, onErr) - case "prometheus": - return getPrometheusExporter(projectID, onErr) - } - - return nil, nil, fmt.Errorf("unrecognized backend: %v", backend) -} - -func getPrometheusExporter(namespace string, onErr func(error)) (trace.Exporter, view.Exporter, error) { - pe, err := prometheus.NewExporter(prometheus.Options{ - Namespace: namespace, - OnError: onErr, - }) - return nil, pe, err -} - -func getSDExporter(projectID string, onErr func(error)) (trace.Exporter, view.Exporter, error) { - svcName, svcVersion := "", "" - if IsGAE() { - _, svcName, svcVersion = GetGAEInfo() - } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { - svcName, svcVersion = n, v - } - opts := getSDExporterOptions(projectID, svcName, svcVersion, onErr) - if opts == nil { - return nil, nil, nil - } - exp, err := stackdriver.NewExporter(*opts) - return exp, exp, err -} - -// getSDExporterOptions returns Stack Driver Options that you can pass directly -// to the OpenCensus exporter or other libraries. -func getSDExporterOptions(projectID, service, version string, onErr func(err error)) *stackdriver.Options { - var mr monitoredresource.Interface - if m := monitoredresource.Autodetect(); m != nil { - mr = m - } else if IsGAE() { - mr = gaeInterface{ - typ: "gae_app", - labels: map[string]string{ - "project_id": projectID, - }, - } - } - if mr == nil { - return nil - } - - return &stackdriver.Options{ - ProjectID: projectID, - MonitoredResource: mr, - OnError: onErr, - DefaultMonitoringLabels: &stackdriver.Labels{}, - DefaultTraceAttributes: map[string]interface{}{ - "service": service, - "version": version, - }, - } -} - -// implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface -type gaeInterface struct { - typ string - labels map[string]string -} - -func (g gaeInterface) MonitoredResource() (string, map[string]string) { - return g.typ, g.labels -} - -// GoogleProjectID returns the GCP Project ID -// that can be used to instantiate various -// GCP clients such as Stack Driver. -func GoogleProjectID() string { - return os.Getenv("GOOGLE_CLOUD_PROJECT") -} - -// IsGAE tells you whether your program is running -// within the App Engine platform. -func IsGAE() bool { - return os.Getenv("GAE_DEPLOYMENT_ID") != "" -} - -// GetGAEInfo returns the GCP Project ID, -// the service, and the version of the application. -func GetGAEInfo() (projectID, service, version string) { - return GoogleProjectID(), - os.Getenv("GAE_SERVICE"), - os.Getenv("GAE_VERSION") -} diff --git a/server/kit/kitserver.go b/server/kit/kitserver.go index d3e20b21c..f2f50b8b2 100644 --- a/server/kit/kitserver.go +++ b/server/kit/kitserver.go @@ -13,13 +13,15 @@ import ( "cloud.google.com/go/errorreporting" sdpropagation "contrib.go.opencensus.io/exporter/stackdriver/propagation" - "github.com/NYTimes/gizmo/gcputil" + "github.com/NYTimes/gizmo/observe" "github.com/go-kit/kit/log" httptransport "github.com/go-kit/kit/transport/http" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/pkg/errors" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/plugin/ochttp" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" "go.opencensus.io/trace/propagation" ocontext "golang.org/x/net/context" "google.golang.org/grpc" @@ -29,6 +31,7 @@ import ( type Server struct { logger log.Logger logClose func() error + ocFlush func() errs *errorreporting.Client @@ -90,10 +93,10 @@ func NewServer(svc Service) *Server { propr propagation.HTTPFormat ) - projectID := gcputil.GoogleProjectID() + projectID := observe.GoogleProjectID() var svcName, svcVersion string - if gcputil.IsGAE() { - _, svcName, svcVersion = gcputil.GetGAEInfo() + if observe.IsGAE() { + _, svcName, svcVersion = observe.GetGAEInfo() } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { svcName, svcVersion = n, v } @@ -101,20 +104,22 @@ func NewServer(svc Service) *Server { onErr := func(err error) { lg.Log("error", err, "message", "tracing client encountered an error") } - - if opt := gcputil.SDExporterOptions(projectID, svcName, svcVersion, onErr); opt != nil { - err = gcputil.InitSDExporter(*opt) + ocFlush := func() {} + if observe.IsGCPAccessible() { + exp, err := observe.NewStackDriverExporter(projectID, onErr) if err != nil { lg.Log("error", err, "message", "unable to initiate error tracing exporter") } + ocFlush = exp.Flush + trace.RegisterExporter(exp) + view.RegisterExporter(exp) propr = &sdpropagation.HTTPFormat{} errs, err = errorreporting.NewClient(ctx, projectID, errorreporting.Config{ ServiceName: svcName, ServiceVersion: svcVersion, - OnError: func(err error) { lg.Log("error", err, "message", "error reporting client encountered an error") @@ -132,6 +137,7 @@ func NewServer(svc Service) *Server { exit: make(chan chan error), logger: lg, logClose: logClose, + ocFlush: ocFlush, errs: errs, } s.svr = &http.Server{ @@ -342,6 +348,11 @@ func (s *Server) start() error { s.logClose() } + // flush the stack driver exporter + if s.ocFlush != nil { + s.ocFlush() + } + if s.errs != nil { s.errs.Close() } diff --git a/server/kit/log.go b/server/kit/log.go index ceef14a74..bfd6ebf5a 100644 --- a/server/kit/log.go +++ b/server/kit/log.go @@ -4,7 +4,7 @@ import ( "context" "os" - "github.com/NYTimes/gizmo/gcputil" + "github.com/NYTimes/gizmo/observe" "github.com/go-kit/kit/log" "github.com/go-kit/kit/transport/http" "google.golang.org/grpc/metadata" @@ -24,7 +24,7 @@ import ( // If an empty string is provided, "gae_log" will be used in App Engine and "stdout" elsewhere. // For more information about to use of logID see the documentation here: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#FIELDS.log_name func NewLogger(ctx context.Context, logID string) (log.Logger, func() error, error) { - projectID, serviceID, svcVersion := gcputil.GetGAEInfo() + projectID, serviceID, svcVersion := observe.GetGAEInfo() lg, cl, err := newStackdriverLogger(ctx, logID, projectID, serviceID, svcVersion) // if Stackdriver logger was not able to find information about monitored resource it returns nil. if err != nil { diff --git a/server/kit/sd_log.go b/server/kit/sd_log.go index a48b91ad2..d3f1e278a 100644 --- a/server/kit/sd_log.go +++ b/server/kit/sd_log.go @@ -10,7 +10,7 @@ import ( "cloud.google.com/go/logging" "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" - "github.com/NYTimes/gizmo/gcputil" + "github.com/NYTimes/gizmo/observe" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" @@ -32,7 +32,7 @@ func newStackdriverLogger(ctx context.Context, logID, projectID, service, versio "version_id": version, }, } - if gcputil.IsGAE() { + if observe.IsGAE() { resource.Type = "gae_app" if logID == "" { logID = "app_logs" From 3b1b4c90f542cd8c98a08e5ba61ab9db693fa6ec Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Mon, 11 Feb 2019 17:51:31 -0500 Subject: [PATCH 09/13] fix doc --- observe/observe.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/observe/observe.go b/observe/observe.go index 161b3df7a..e1b374593 100644 --- a/observe/observe.go +++ b/observe/observe.go @@ -1,6 +1,6 @@ // Package observe provides functions -// that help with setting observability -// in cloud provides, mainly GCP. +// that help with setting tracing/metrics +// in cloud providers, mainly GCP. package observe import ( From fe2b58bf294b7098322df300dcdfbceb19642036 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Tue, 12 Feb 2019 13:28:27 -0500 Subject: [PATCH 10/13] pr fixes --- observe/observe.go | 45 ++++++++++++++++++----------------------- server/kit/kitserver.go | 12 ++--------- 2 files changed, 22 insertions(+), 35 deletions(-) diff --git a/observe/observe.go b/observe/observe.go index e1b374593..9f3a41374 100644 --- a/observe/observe.go +++ b/observe/observe.go @@ -9,8 +9,6 @@ import ( "contrib.go.opencensus.io/exporter/stackdriver" "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" "go.opencensus.io/exporter/prometheus" - "go.opencensus.io/stats/view" - "go.opencensus.io/trace" ) // NewStackDriverExporter will return the tracing and metrics through @@ -18,13 +16,8 @@ import ( // If exporter is registered, it returns the exporter so you can register // it and ensure to call Flush on termination. func NewStackDriverExporter(projectID string, onErr func(error)) (*stackdriver.Exporter, error) { - svcName, svcVersion := "", "" - if IsGAE() { - _, svcName, svcVersion = GetGAEInfo() - } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { - svcName, svcVersion = n, v - } - opts := SDExporterOptions(projectID, svcName, svcVersion, onErr) + _, svcName, svcVersion := GetServiceInfo() + opts := getSDOpts(projectID, svcName, svcVersion, onErr) if opts == nil { return nil, nil } @@ -33,7 +26,7 @@ func NewStackDriverExporter(projectID string, onErr func(error)) (*stackdriver.E // NewPrometheusExporter return a prometheus Exporter for OpenCensus. func NewPrometheusExporter(opts prometheus.Options) (*prometheus.Exporter, error) { - return NewPrometheusExporter(opts) + return prometheus.NewExporter(opts) } // GoogleProjectID returns the GCP Project ID @@ -57,9 +50,22 @@ func GetGAEInfo() (projectID, service, version string) { os.Getenv("GAE_VERSION") } -// SDExporterOptions returns Stack Driver Options that you can pass directly +// GetServiceInfo returns the GCP Project ID, +// the service name and version (gae or through +// GAE_SERVICE/GAE_VERSION env vars) +func GetServiceInfo() (projectID, service, version string) { + projectID = GoogleProjectID() + if IsGAE() { + _, service, version = GetGAEInfo() + } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { + service, version = n, v + } + return projectID, service, version +} + +// getSDOpts returns Stack Driver Options that you can pass directly // to the OpenCensus exporter or other libraries. -func SDExporterOptions(projectID, service, version string, onErr func(err error)) *stackdriver.Options { +func getSDOpts(projectID, service, version string, onErr func(err error)) *stackdriver.Options { var mr monitoredresource.Interface if m := monitoredresource.Autodetect(); m != nil { mr = m @@ -86,23 +92,12 @@ func SDExporterOptions(projectID, service, version string, onErr func(err error) } } -// IsGCPAccessible returns whether the running application +// IsGCPEnabled returns whether the running application // is inside GCP or has access to its products. -func IsGCPAccessible() bool { +func IsGCPEnabled() bool { return monitoredresource.Autodetect() != nil || IsGAE() } -// InitSDExporter will initialize the OpenCensus tracing/metrics exporter -func InitSDExporter(opts stackdriver.Options) error { - exporter, err := stackdriver.NewExporter(opts) - if err != nil { - return err - } - trace.RegisterExporter(exporter) - view.RegisterExporter(exporter) - return nil -} - // implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface type gaeInterface struct { typ string diff --git a/server/kit/kitserver.go b/server/kit/kitserver.go index f2f50b8b2..acd3f974b 100644 --- a/server/kit/kitserver.go +++ b/server/kit/kitserver.go @@ -8,7 +8,6 @@ import ( "net" "net/http" "net/http/pprof" - "os" "strings" "cloud.google.com/go/errorreporting" @@ -93,19 +92,12 @@ func NewServer(svc Service) *Server { propr propagation.HTTPFormat ) - projectID := observe.GoogleProjectID() - var svcName, svcVersion string - if observe.IsGAE() { - _, svcName, svcVersion = observe.GetGAEInfo() - } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { - svcName, svcVersion = n, v - } - + projectID, svcName, svcVersion := observe.GetServiceInfo() onErr := func(err error) { lg.Log("error", err, "message", "tracing client encountered an error") } ocFlush := func() {} - if observe.IsGCPAccessible() { + if observe.IsGCPEnabled() { exp, err := observe.NewStackDriverExporter(projectID, onErr) if err != nil { lg.Log("error", err, From a614cdea26162a2d53aff7ca934ca0ded0490254 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Tue, 12 Feb 2019 14:19:25 -0500 Subject: [PATCH 11/13] fix gae metrics --- observe/observe.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/observe/observe.go b/observe/observe.go index 9f3a41374..2f19f3299 100644 --- a/observe/observe.go +++ b/observe/observe.go @@ -4,11 +4,15 @@ package observe import ( + "context" "os" + traceapi "cloud.google.com/go/trace/apiv2" "contrib.go.opencensus.io/exporter/stackdriver" "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" "go.opencensus.io/exporter/prometheus" + "golang.org/x/oauth2/google" + "google.golang.org/api/option" ) // NewStackDriverExporter will return the tracing and metrics through @@ -67,24 +71,31 @@ func GetServiceInfo() (projectID, service, version string) { // to the OpenCensus exporter or other libraries. func getSDOpts(projectID, service, version string, onErr func(err error)) *stackdriver.Options { var mr monitoredresource.Interface + + // this is so that you can export views from your local server up to SD if you wish + creds, err := google.FindDefaultCredentials(context.Background(), traceapi.DefaultAuthScopes()...) + if err != nil { + return nil + } + canExport := IsGAE() if m := monitoredresource.Autodetect(); m != nil { mr = m - } else if IsGAE() { - mr = gaeInterface{ - typ: "gae_app", - labels: map[string]string{ - "project_id": projectID, - }, - } + canExport = true } - if mr == nil { + if !canExport { return nil } return &stackdriver.Options{ ProjectID: projectID, MonitoredResource: mr, - OnError: onErr, + MonitoringClientOptions: []option.ClientOption{ + option.WithCredentials(creds), + }, + TraceClientOptions: []option.ClientOption{ + option.WithCredentials(creds), + }, + OnError: onErr, DefaultTraceAttributes: map[string]interface{}{ "service": service, "version": version, @@ -97,13 +108,3 @@ func getSDOpts(projectID, service, version string, onErr func(err error)) *stack func IsGCPEnabled() bool { return monitoredresource.Autodetect() != nil || IsGAE() } - -// implements contrib.go.opencensus.io/exporter/stackdriver/monitoredresource.Interface -type gaeInterface struct { - typ string - labels map[string]string -} - -func (g gaeInterface) MonitoredResource() (string, map[string]string) { - return g.typ, g.labels -} From 121284e0e579c6e3ee3e6bf8c1ca0f69af5f2cf8 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Tue, 12 Feb 2019 14:45:32 -0500 Subject: [PATCH 12/13] pr fixes --- observe/observe.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/observe/observe.go b/observe/observe.go index 2f19f3299..cefc40601 100644 --- a/observe/observe.go +++ b/observe/observe.go @@ -58,13 +58,10 @@ func GetGAEInfo() (projectID, service, version string) { // the service name and version (gae or through // GAE_SERVICE/GAE_VERSION env vars) func GetServiceInfo() (projectID, service, version string) { - projectID = GoogleProjectID() if IsGAE() { - _, service, version = GetGAEInfo() - } else if n, v := os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION"); n != "" { - service, version = n, v + return GetGAEInfo() } - return projectID, service, version + return GoogleProjectID(), os.Getenv("SERVICE_NAME"), os.Getenv("SERVICE_VERSION") } // getSDOpts returns Stack Driver Options that you can pass directly From 56730697352be6897cc1d4a98af2bf1254881b27 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Tue, 12 Feb 2019 14:59:20 -0500 Subject: [PATCH 13/13] pr fixes --- observe/observe.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/observe/observe.go b/observe/observe.go index cefc40601..b4c31f549 100644 --- a/observe/observe.go +++ b/observe/observe.go @@ -55,8 +55,11 @@ func GetGAEInfo() (projectID, service, version string) { } // GetServiceInfo returns the GCP Project ID, -// the service name and version (gae or through -// GAE_SERVICE/GAE_VERSION env vars) +// the service name and version (GAE or through +// GAE_SERVICE/GAE_VERSION env vars). Note +// that GAE_SERVICE/GAE_VERSION are not standard but +// your application can pass them in as variables +// to be included in your trace attributes func GetServiceInfo() (projectID, service, version string) { if IsGAE() { return GetGAEInfo()