Switch from Progrock to OpenTelemetry

.gitignore

@@ -45,4 +45,3 @@ go.work.sum
 
 # merged from dagger/examples repository
 **/node_modules
-**/env

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "telemetry/opentelemetry-proto"]
+	path = telemetry/opentelemetry-proto
+	url = https://github.com/open-telemetry/opentelemetry-proto

analytics/analytics.go

@@ -8,14 +8,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
 	"sync"
 	"time"
 
-	"github.com/dagger/dagger/core/pipeline"
 	"github.com/dagger/dagger/engine"
-	"github.com/vito/progrock"
+	"github.com/dagger/dagger/telemetry"
 )
 
 const (
@@ -78,30 +78,20 @@ func DoNotTrack() bool {
 
 type Config struct {
 	DoNotTrack bool
-	Labels     pipeline.Labels
+	Labels     telemetry.Labels
 	CloudToken string
 }
 
-func DefaultConfig() Config {
+func DefaultConfig(labels telemetry.Labels) Config {
 	cfg := Config{
 		DoNotTrack: DoNotTrack(),
 		CloudToken: os.Getenv("DAGGER_CLOUD_TOKEN"),
+		Labels:     labels,
 	}
 	// Backward compatibility with the old environment variable.
 	if cfg.CloudToken == "" {
 		cfg.CloudToken = os.Getenv("_EXPERIMENTAL_DAGGER_CLOUD_TOKEN")
 	}
-
-	workdir, err := os.Getwd()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to get cwd: %v\n", err)
-		return cfg
-	}
-
-	cfg.Labels.AppendCILabel()
-	cfg.Labels = append(cfg.Labels, pipeline.LoadVCSLabels(workdir)...)
-	cfg.Labels = append(cfg.Labels, pipeline.LoadClientLabels(engine.Version)...)
-
 	return cfg
 }
 
@@ -111,8 +101,7 @@ type queuedEvent struct {
 }
 
 type CloudTracker struct {
-	cfg    Config
-	labels map[string]string
+	cfg Config
 
 	closed bool
 	mu     sync.Mutex
@@ -128,15 +117,10 @@ func New(cfg Config) Tracker {
 
 	t := &CloudTracker{
 		cfg:    cfg,
-		labels: make(map[string]string),
 		stopCh: make(chan struct{}),
 		doneCh: make(chan struct{}),
 	}
 
-	for _, l := range cfg.Labels {
-		t.labels[l.Name] = l.Value
-	}
-
 	go t.start()
 
 	return t
@@ -155,19 +139,19 @@ func (t *CloudTracker) Capture(ctx context.Context, event string, properties map
 		Type:       event,
 		Properties: properties,
 
-		DeviceID: t.labels["dagger.io/client.machine_id"],
+		DeviceID: t.cfg.Labels["dagger.io/client.machine_id"],
 
-		ClientVersion: t.labels["dagger.io/client.version"],
-		ClientOS:      t.labels["dagger.io/client.os"],
-		ClientArch:    t.labels["dagger.io/client.arch"],
+		ClientVersion: t.cfg.Labels["dagger.io/client.version"],
+		ClientOS:      t.cfg.Labels["dagger.io/client.os"],
+		ClientArch:    t.cfg.Labels["dagger.io/client.arch"],
 
-		CI:       t.labels["dagger.io/ci"] == "true",
-		CIVendor: t.labels["dagger.io/ci.vendor"],
+		CI:       t.cfg.Labels["dagger.io/ci"] == "true",
+		CIVendor: t.cfg.Labels["dagger.io/ci.vendor"],
 	}
-	if remote := t.labels["dagger.io/git.remote"]; remote != "" {
+	if remote := t.cfg.Labels["dagger.io/git.remote"]; remote != "" {
 		ev.GitRemoteEncoded = fmt.Sprintf("%x", base64.StdEncoding.EncodeToString([]byte(remote)))
 	}
-	if author := t.labels["dagger.io/git.author.email"]; author != "" {
+	if author := t.cfg.Labels["dagger.io/git.author.email"]; author != "" {
 		ev.GitAuthorHashed = fmt.Sprintf("%x", sha256.Sum256([]byte(author)))
 	}
 	if clientMetadata, err := engine.ClientMetadataFromContext(ctx); err == nil {
@@ -203,34 +187,32 @@ func (t *CloudTracker) send() {
 	}
 
 	// grab the progrock recorder from the last event in the queue
-	rec := progrock.FromContext(queue[len(queue)-1].ctx)
-
 	payload := bytes.NewBuffer([]byte{})
 	enc := json.NewEncoder(payload)
 	for _, q := range queue {
 		err := enc.Encode(q.event)
 		if err != nil {
-			rec.Debug("analytics: encode failed", progrock.ErrorLabel(err))
+			slog.Debug("analytics: encode failed", "error", err)
 			continue
 		}
 	}
 
 	req, err := http.NewRequest(http.MethodPost, trackURL, bytes.NewReader(payload.Bytes()))
 	if err != nil {
-		rec.Debug("analytics: new request failed", progrock.ErrorLabel(err))
+		slog.Debug("analytics: new request failed", "error", err)
 		return
 	}
 	if t.cfg.CloudToken != "" {
 		req.SetBasicAuth(t.cfg.CloudToken, "")
 	}
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		rec.Debug("analytics: do request failed", progrock.ErrorLabel(err))
+		slog.Debug("analytics: do request failed", "error", err)
 		return
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusCreated {
-		rec.Debug("analytics: unexpected response", progrock.Labelf("status", resp.Status))
+		slog.Debug("analytics: unexpected response", "status", resp.Status)
 	}
 }
 

cmd/codegen/codegen.go

@@ -4,10 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"strings"
-	"time"
-
-	"github.com/vito/progrock"
 
 	"dagger.io/dagger"
 	"github.com/dagger/dagger/cmd/codegen/generator"
@@ -17,18 +15,14 @@ import (
 )
 
 func Generate(ctx context.Context, cfg generator.Config, dag *dagger.Client) (err error) {
-	var vtxName string
+	logsW := os.Stdout
+
 	if cfg.ModuleName != "" {
-		vtxName = fmt.Sprintf("generating %s module: %s", cfg.Lang, cfg.ModuleName)
+		fmt.Fprintf(logsW, "generating %s module: %s\n", cfg.Lang, cfg.ModuleName)
 	} else {
-		vtxName = fmt.Sprintf("generating %s SDK client", cfg.Lang)
+		fmt.Fprintf(logsW, "generating %s SDK client\n", cfg.Lang)
 	}
 
-	ctx, vtx := progrock.Span(ctx, time.Now().String(), vtxName)
-	defer func() { vtx.Done(err) }()
-
-	logsW := vtx.Stdout()
-
 	var introspectionSchema *introspection.Schema
 	if cfg.IntrospectionJSON != "" {
 		var resp introspection.Response
@@ -55,12 +49,12 @@ func Generate(ctx context.Context, cfg generator.Config, dag *dagger.Client) (er
 
 		for _, cmd := range generated.PostCommands {
 			cmd.Dir = cfg.OutputDir
-			cmd.Stdout = vtx.Stdout()
-			cmd.Stderr = vtx.Stderr()
-			task := vtx.Task(strings.Join(cmd.Args, " "))
+			cmd.Stdout = os.Stdout
+			cmd.Stderr = os.Stderr
+			fmt.Fprintln(logsW, "running post-command:", strings.Join(cmd.Args, " "))
 			err := cmd.Run()
-			task.Done(err)
 			if err != nil {
+				fmt.Fprintln(logsW, "post-command failed:", err)
 				return err
 			}
 		}

cmd/codegen/generator/go/generator.go

@@ -56,7 +56,11 @@ func (g *GoGenerator) Generate(ctx context.Context, schema *introspection.Schema
 
 	var overlay fs.FS = mfs
 	if g.Config.ModuleName != "" {
-		overlay = layerfs.New(mfs, &MountedFS{FS: dagger.QueryBuilder, Name: "internal"})
+		overlay = layerfs.New(
+			mfs,
+			&MountedFS{FS: dagger.QueryBuilder, Name: "internal"},
+			&MountedFS{FS: dagger.Telemetry, Name: "internal"},
+		)
 	}
 
 	genSt := &generator.GeneratedState{

cmd/codegen/generator/go/templates/module_interfaces.go

@@ -346,7 +346,7 @@ func (spec *parsedIfaceType) marshalJSONMethodCode() *Statement {
 		BlockFunc(func(g *Group) {
 			g.If(Id("r").Op("==").Nil()).Block(Return(Index().Byte().Parens(Lit(`""`)), Nil()))
 
-			g.List(Id("id"), Id("err")).Op(":=").Id("r").Dot("ID").Call(Qual("context", "Background").Call())
+			g.List(Id("id"), Id("err")).Op(":=").Id("r").Dot("ID").Call(Id("marshalCtx"))
 			g.If(Id("err").Op("!=").Nil()).Block(Return(Nil(), Id("err")))
 			g.Return(Id("json").Dot("Marshal").Call(Id("id")))
 		})

cmd/codegen/generator/go/templates/modules.go

@@ -44,7 +44,7 @@ from the Engine, calls the relevant function and returns the result. The generat
 on the object+function name, with each case doing json deserialization of the input arguments and calling the actual
 Go function.
 */
-func (funcs goTemplateFuncs) moduleMainSrc() (string, error) {
+func (funcs goTemplateFuncs) moduleMainSrc() (string, error) { //nolint: gocyclo
 	// HACK: the code in this func can be pretty flaky and tricky to debug -
 	// it's much easier to debug when we actually have stack traces, so we grab
 	// those on a panic
@@ -93,6 +93,12 @@ func (funcs goTemplateFuncs) moduleMainSrc() (string, error) {
 
 	tps := []types.Type{}
 	for _, obj := range objs {
+		// ignore any private definitions, they may be part of the runtime itself
+		// e.g. marshalCtx
+		if !obj.Exported() {
+			continue
+		}
+
 		// check if this is the constructor func, save it for later if so
 		if ok := ps.checkConstructor(obj); ok {
 			continue
@@ -230,58 +236,86 @@ const (
 	mainSrc = `func main() {
 	ctx := context.Background()
 
+	// Direct slog to the new stderr. This is only for dev time debugging, and
+	// runtime errors/warnings.
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+		Level: slog.LevelWarn,
+	})))
+
+	if err := dispatch(ctx); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(2)
+	}
+}
+
+func dispatch(ctx context.Context) error {
+	ctx = telemetry.InitEmbedded(ctx, resource.NewWithAttributes(
+		semconv.SchemaURL,
+		semconv.ServiceNameKey.String("dagger-go-sdk"),
+		// TODO version?
+	))
+	defer telemetry.Close()
+
+	ctx, span := Tracer().Start(ctx, "Go runtime",
+		trace.WithAttributes(
+			// In effect, the following two attributes hide the exec /runtime span.
+			//
+			// Replace the parent span,
+			attribute.Bool("dagger.io/ui.mask", true),
+			// and only show our children.
+			attribute.Bool("dagger.io/ui.passthrough", true),
+		))
+	defer span.End()
+
+	// A lot of the "work" actually happens when we're marshalling the return
+	// value, which entails getting object IDs, which happens in MarshalJSON,
+	// which has no ctx argument, so we use this lovely global variable.
+	setMarshalContext(ctx)
+
 	fnCall := dag.CurrentFunctionCall()
 	parentName, err := fnCall.ParentName(ctx)
 	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(2)
+		return fmt.Errorf("get parent name: %w", err)
 	}
 	fnName, err := fnCall.Name(ctx)
 	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(2)
+		return fmt.Errorf("get fn name: %w", err)
 	}
 	parentJson, err := fnCall.Parent(ctx)
 	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(2)
+		return fmt.Errorf("get fn parent: %w", err)
 	}
 	fnArgs, err := fnCall.InputArgs(ctx)
 	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(2)
+		return fmt.Errorf("get fn args: %w", err)
 	}
 
 	inputArgs := map[string][]byte{}
 	for _, fnArg := range fnArgs {
 		argName, err := fnArg.Name(ctx)
 		if err != nil {
-			fmt.Println(err.Error())
-			os.Exit(2)
+			return fmt.Errorf("get fn arg name: %w", err)
 		}
 		argValue, err := fnArg.Value(ctx)
 		if err != nil {
-			fmt.Println(err.Error())
-			os.Exit(2)
+			return fmt.Errorf("get fn arg value: %w", err)
 		}
 		inputArgs[argName] = []byte(argValue)
 	}
 
 	result, err := invoke(ctx, []byte(parentJson), parentName, fnName, inputArgs)
 	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(2)
+		return fmt.Errorf("invoke: %w", err)
 	}
 	resultBytes, err := json.Marshal(result)
 	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(2)
+		return fmt.Errorf("marshal: %w", err)
 	}
 	_, err = fnCall.ReturnValue(ctx, JSON(resultBytes))
 	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(2)
+		return fmt.Errorf("store return value: %w", err)
 	}
+	return nil
 }
 `
 	parentJSONVar  = "parentJSON"

cmd/codegen/generator/go/templates/src/_dagger.gen.go/alias.go.tmpl

@@ -1,15 +1,33 @@
 import (
+  "context"
+	"log/slog"
+
 	"{{.PackageImport}}/internal/dagger"
+
+	"go.opentelemetry.io/otel/trace"
 )
 
 var dag = dagger.Connect()
 
+func Tracer() trace.Tracer {
+	return otel.Tracer("dagger.io/sdk.go")
+}
+
+// used for local MarshalJSON implementations
+var marshalCtx = context.Background()
+
+// called by main()
+func setMarshalContext(ctx context.Context) {
+	marshalCtx = ctx
+	dagger.SetMarshalContext(ctx)
+}
+
 type DaggerObject = dagger.DaggerObject
 
 type ExecError = dagger.ExecError
 
 {{ range .Types }}
-{{ $name := .Name | FormatName }} 
+{{ $name := .Name | FormatName }}
 
 {{ .Description | Comment }}
 type {{ $name }} = dagger.{{ $name }}