obs: provisioner service observability — slog default, NR agent, gRPC interceptor, healthz sidecar (track 5/8)

provisioner/go.mod

@@ -0,0 +1,35 @@
+// Module instant.dev/provisioner — observability scaffolding for the
+// instant.dev/provisioner gRPC service.
+//
+// This is a self-contained module (NOT joined with the parent api module)
+// so that:
+//
+//  1. The api repo's `go build ./...` continues to be a pure api build —
+//     adding the provisioner subdir doesn't pull NR deps into the api binary.
+//  2. The Go files here can be copied verbatim into the real provisioner
+//     repo (github.com/InstaNode-dev/provisioner) which already uses the
+//     module name `instant.dev/provisioner` — see that repo's go.mod.
+//
+// When the real provisioner adopts these files, this scaffolding go.mod is
+// deleted and the imports resolve against the real provisioner's go.mod.
+module instant.dev/provisioner
+
+go 1.25.0
+
+require (
+	github.com/newrelic/go-agent/v3 v3.43.3
+	github.com/newrelic/go-agent/v3/integrations/nrgrpc v1.4.9
+	google.golang.org/grpc v1.80.0
+)
+
+require (
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/newrelic/csec-go-agent v1.6.0 // indirect
+	golang.org/x/arch v0.27.0 // indirect
+	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/sys v0.40.0 // indirect
+	golang.org/x/text v0.33.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)

provisioner/go.sum

@@ -0,0 +1,61 @@
+github.com/adhocore/gronx v1.19.1 h1:S4c3uVp5jPjnk00De0lslyTenGJ4nA3Ydbkj1SbdPVc=
+github.com/adhocore/gronx v1.19.1/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/dlclark/regexp2 v1.9.0 h1:pTK/l/3qYIKaRXuHnEnIf7Y5NxfRPfpb7dis6/gdlVI=
+github.com/dlclark/regexp2 v1.9.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/k2io/hookingo v1.0.6 h1:HBSKd1tNbW5BCj8VLNqemyBKjrQ8g0HkXcbC/DEHODE=
+github.com/k2io/hookingo v1.0.6/go.mod h1:2L1jdNjdB3NkbzSVv9Q5fq7SJhRkWyAhe65XsAp5iXk=
+github.com/newrelic/csec-go-agent v1.6.0 h1:OCShRZgiE+kg37jk+QXHw9e9EQ9BvLOeQTk+ovJhnrE=
+github.com/newrelic/csec-go-agent v1.6.0/go.mod h1:LiLGm6a+q+hkmTnrxrYw1ToToirThOHydjrrLMtci5M=
+github.com/newrelic/go-agent/v3 v3.43.3 h1:0A6DkUBYK2bidV6jJDJ1SD2XkRlg976nl+SiEqkGTUQ=
+github.com/newrelic/go-agent/v3 v3.43.3/go.mod h1:MFXnCId5xXMIJI6A/kbkg0DO48EVTsKcmNijMYphzTg=
+github.com/newrelic/go-agent/v3/integrations/nrgrpc v1.4.9 h1:mkoYqqEjFTNjJURsX+08iwuXTmsW7eFT+L0+hBuvAzw=
+github.com/newrelic/go-agent/v3/integrations/nrgrpc v1.4.9/go.mod h1:KkYfN06JZLI/H6l7w2+TJ5ILKF5NCXN5iysLsKkzMiI=
+github.com/newrelic/go-agent/v3/integrations/nrsecurityagent v1.1.0 h1:gqkTDYUHWUyiG+u0PJQCRh98rcHLxP/w7GtIbJDVULY=
+github.com/newrelic/go-agent/v3/integrations/nrsecurityagent v1.1.0/go.mod h1:3wugGvRmOVYov/08y+D8tB1uYIZds5bweVdr5vo4Gbs=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
+go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
+go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
+go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
+go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
+go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
+go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
+go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
+go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
+go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
+golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU=
+golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
+google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

provisioner/internal/_obs_stubs/buildinfo/buildinfo.go

@@ -0,0 +1,34 @@
+// Package buildinfo exposes compile-time build metadata.
+//
+// STUB: this is a temporary, vendored copy of what will become
+// instant.dev/common/buildinfo once track 1 of the observability rollout
+// merges. After that PR lands, callers in this service should switch their
+// imports from
+//
+//	"instant.dev/provisioner/internal/_obs_stubs/buildinfo"
+//
+// to
+//
+//	"instant.dev/common/buildinfo"
+//
+// and this directory should be deleted in a follow-up cleanup PR.
+//
+// The variables are populated at link time via -ldflags. See the Dockerfile
+// change shipped in track 1 for the exact command line. When the service is
+// built without ldflags (e.g. `go build ./...` during local dev), the values
+// fall back to "dev" / "unknown" so the program never panics.
+package buildinfo
+
+var (
+	// GitSHA is the 7+ char git commit hash this binary was built from.
+	// Set via: -ldflags "-X .../buildinfo.GitSHA=$GIT_SHA"
+	GitSHA = "dev"
+
+	// BuildTime is the UTC RFC3339 timestamp of the build.
+	// Set via: -ldflags "-X .../buildinfo.BuildTime=$BUILD_TIME"
+	BuildTime = "unknown"
+
+	// Version is the semver tag of the release, or "dev" for untagged builds.
+	// Set via: -ldflags "-X .../buildinfo.Version=$VERSION"
+	Version = "dev"
+)

provisioner/internal/_obs_stubs/logctx/logctx.go

@@ -0,0 +1,96 @@
+// Package logctx provides a context-aware slog.Handler wrapper that injects
+// observability fields (service, commit_id, trace_id, team_id, tid) into every
+// log record automatically, plus typed context setters/getters for those
+// fields.
+//
+// STUB: this is a minimal vendored copy of what will become
+// instant.dev/common/logctx once track 2 of the observability rollout merges.
+// After that PR lands, callers should switch their imports to
+// instant.dev/common/logctx and this directory should be deleted.
+//
+// Scope of this stub: only the surface area the provisioner service actually
+// uses — NewHandler, WithTraceID, TraceID. The full common/logctx package
+// will also expose WithTeamID, WithRequestID, WithTID, etc. — those are not
+// needed here yet because the provisioner has no team/auth context.
+package logctx
+
+import (
+	"context"
+	"log/slog"
+
+	"instant.dev/provisioner/internal/_obs_stubs/buildinfo"
+)
+
+// ctxKey is a private, comparable type for context keys so we never collide
+// with other packages that stash values on the same ctx.
+type ctxKey int
+
+const (
+	keyTraceID ctxKey = iota
+)
+
+// WithTraceID returns a child context with the given W3C trace ID attached.
+// Empty traceID is a no-op — the parent context is returned unchanged so
+// callers can pipe through values they extracted from gRPC metadata without
+// branching on emptiness.
+func WithTraceID(ctx context.Context, traceID string) context.Context {
+	if traceID == "" {
+		return ctx
+	}
+	return context.WithValue(ctx, keyTraceID, traceID)
+}
+
+// TraceID extracts a previously-set trace ID, returning "" when absent.
+// Never panics — safe to call on background or unrelated contexts.
+func TraceID(ctx context.Context) string {
+	if ctx == nil {
+		return ""
+	}
+	v, _ := ctx.Value(keyTraceID).(string)
+	return v
+}
+
+// handler wraps an underlying slog.Handler and stamps every Record with
+// service, commit_id, build_time, version, and ctx-derived trace_id.
+type handler struct {
+	inner   slog.Handler
+	service string
+}
+
+// NewHandler returns a slog.Handler that decorates `inner` with mandatory
+// observability fields. The returned handler is safe for concurrent use.
+//
+// Typical wiring in a service's main():
+//
+//	slog.SetDefault(slog.New(logctx.NewHandler(
+//	    "provisioner",
+//	    slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true}),
+//	)))
+func NewHandler(service string, inner slog.Handler) slog.Handler {
+	return &handler{inner: inner, service: service}
+}
+
+func (h *handler) Enabled(ctx context.Context, level slog.Level) bool {
+	return h.inner.Enabled(ctx, level)
+}
+
+func (h *handler) Handle(ctx context.Context, r slog.Record) error {
+	r.AddAttrs(
+		slog.String("service", h.service),
+		slog.String("commit_id", buildinfo.GitSHA),
+		slog.String("build_time", buildinfo.BuildTime),
+		slog.String("version", buildinfo.Version),
+	)
+	if tid := TraceID(ctx); tid != "" {
+		r.AddAttrs(slog.String("trace_id", tid))
+	}
+	return h.inner.Handle(ctx, r)
+}
+
+func (h *handler) WithAttrs(attrs []slog.Attr) slog.Handler {
+	return &handler{inner: h.inner.WithAttrs(attrs), service: h.service}
+}
+
+func (h *handler) WithGroup(name string) slog.Handler {
+	return &handler{inner: h.inner.WithGroup(name), service: h.service}
+}

provisioner/internal/server/healthz.go

@@ -0,0 +1,49 @@
+// Package server hosts the gRPC service implementation and, as of the
+// observability rollout (2026-05-12), a tiny sidecar HTTP handler exposing
+// /healthz so the platform can curl the running pod's commit_id without
+// going through the gRPC surface.
+//
+// The provisioner is otherwise gRPC-only on port 50051. We bind the HTTP
+// sidecar to a different port (default 8092, see plan doc) — verified in
+// HealthzPort_NoCollisionWithGRPC test below.
+package server
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"instant.dev/provisioner/internal/_obs_stubs/buildinfo"
+)
+
+// HealthzResponse is the JSON body returned by GET /healthz.
+//
+// Field order matches what the api and worker services return so dashboards
+// and curl pipelines can use a single jq filter across all three.
+type HealthzResponse struct {
+	OK        bool   `json:"ok"`
+	Service   string `json:"service"`
+	CommitID  string `json:"commit_id"`
+	BuildTime string `json:"build_time"`
+	Version   string `json:"version"`
+}
+
+// HealthzHandler returns an http.Handler that responds to any method (the
+// k8s liveness probe will use GET; humans use curl) with the build metadata
+// JSON. Never errors — used as a liveness probe so it must be cheap and
+// dependency-free.
+func HealthzHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		resp := HealthzResponse{
+			OK:        true,
+			Service:   "instant-provisioner",
+			CommitID:  buildinfo.GitSHA,
+			BuildTime: buildinfo.BuildTime,
+			Version:   buildinfo.Version,
+		}
+		w.Header().Set("Content-Type", "application/json")
+		// json.NewEncoder.Encode never errors on a value of fixed shape with
+		// no unmarshalable types — and we'd be unable to write an error
+		// response anyway if the connection were broken. Discard.
+		_ = json.NewEncoder(w).Encode(resp)
+	})
+}

provisioner/internal/server/healthz_test.go

@@ -0,0 +1,60 @@
+package server
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// TestHealthzHandler_ResponseShape pins the JSON contract since dashboards
+// and alert rules consume this body shape.
+func TestHealthzHandler_ResponseShape(t *testing.T) {
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
+	HealthzHandler().ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+
+	var raw map[string]any
+	if err := json.NewDecoder(rec.Body).Decode(&raw); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+
+	for _, key := range []string{"ok", "service", "commit_id", "build_time", "version"} {
+		if _, ok := raw[key]; !ok {
+			t.Errorf("response missing key %q — keys present: %v", key, mapKeys(raw))
+		}
+	}
+
+	if raw["service"] != "instant-provisioner" {
+		t.Errorf("service = %v, want instant-provisioner", raw["service"])
+	}
+	if raw["ok"] != true {
+		t.Errorf("ok = %v, want true", raw["ok"])
+	}
+}
+
+// TestHealthzHandler_AcceptsAnyMethod confirms HEAD / POST don't 405. The k8s
+// liveness probe sends GET but having the endpoint be method-agnostic makes
+// it easier to curl from a shell during incidents.
+func TestHealthzHandler_AcceptsAnyMethod(t *testing.T) {
+	for _, m := range []string{http.MethodGet, http.MethodHead, http.MethodPost} {
+		rec := httptest.NewRecorder()
+		req := httptest.NewRequest(m, "/healthz", nil)
+		HealthzHandler().ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Errorf("method %s: status = %d, want 200", m, rec.Code)
+		}
+	}
+}
+
+func mapKeys(m map[string]any) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	return out
+}