Add hidden gcptrace flag

This also hoists the trace logic from the build subcommand up into the
root Command so that it's available for every subcommand.

Signed-off-by: Jon Johnson <jon.johnson@chainguard.dev>

go.mod

@@ -6,6 +6,7 @@ require (
 	chainguard.dev/apko v0.14.2-0.20240516182909-5d04baeb15df
 	cloud.google.com/go/storage v1.41.0
 	dagger.io/dagger v0.11.4
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.23.0
 	github.com/chainguard-dev/clog v1.3.1
 	github.com/chainguard-dev/go-apk v0.0.0-20240514202343-05db79c0242f
 	github.com/chainguard-dev/go-pkgconfig v0.0.0-20240404163941-6351b37b2a10
@@ -63,8 +64,10 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	cloud.google.com/go/iam v1.1.8 // indirect
+	cloud.google.com/go/trace v1.10.7 // indirect
 	github.com/99designs/gqlgen v0.17.46 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.47.0 // indirect
 	github.com/Khan/genqlient v0.7.0 // indirect
 	github.com/MakeNowJust/heredoc/v2 v2.0.1 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect

go.sum

@@ -11,8 +11,16 @@ cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2Qx
 cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0=
 cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE=
+cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw=
+cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE=
+cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU=
+cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng=
+cloud.google.com/go/monitoring v1.19.0 h1:NCXf8hfQi+Kmr56QJezXRZ6GPb80ZI7El1XztyUuLQI=
+cloud.google.com/go/monitoring v1.19.0/go.mod h1:25IeMR5cQ5BoZ8j1eogHE5VPJLlReQ7zFp5OiLgiGZw=
 cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0=
 cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80=
+cloud.google.com/go/trace v1.10.7 h1:gK8z2BIJQ3KIYGddw9RJLne5Fx0FEXkrEQzPaeEYVvk=
+cloud.google.com/go/trace v1.10.7/go.mod h1:qk3eiKmZX0ar2dzIJN/3QhY2PIFh1eqcIdaN5uEjQPM=
 dagger.io/dagger v0.11.4 h1:br81yt7S/KWrxieajFyzleOMBccevhi3TSSxmBp2lu0=
 dagger.io/dagger v0.11.4/go.mod h1:YyHN9NpFZ0U70vLfGR9RQw1ppwxzXG7q/DPn7kn25Oc=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
@@ -22,6 +30,12 @@ github.com/99designs/gqlgen v0.17.46/go.mod h1:qRtiAeVPgkBBSPzZtoZXRRl5WkNrUTpp1
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.23.0 h1:5A4O4OdC7yzkIEPl4GrS+PRYV15zsboaWBT52g3Hc0k=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.23.0/go.mod h1:zO73rmlwRYxQF/6Nul4PA/UIAYJo9BtDAMgPfMthXnw=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.47.0 h1:h3GAq9n95A6IYOGqAucbp+tGBxAQul2vj5ORkmRrGxg=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.47.0/go.mod h1:PdB0wkmILI+phhoBhWdrrB4LfORT9tHc03OOn+q3dWU=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.47.0 h1:TOjDcFzPkoglwb5sa6+704TXwYgs+XsN5HYc98ksK+M=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.47.0/go.mod h1:ZC7rjqRzdhRKDK223jQ7Tsz89ZtrSSLH/VFzf7k5Sb0=
 github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w=
 github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM=
 github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=

pkg/cli/build.go

@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
-	"os"
 	"path/filepath"
 	"runtime"
 	"time"
@@ -32,8 +31,6 @@ import (
 	"github.com/chainguard-dev/clog"
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
-	"go.opentelemetry.io/otel/sdk/trace"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -74,8 +71,6 @@ func Build() *cobra.Command {
 	var timeout time.Duration
 	var extraPackages []string
 
-	var traceFile string
-
 	cmd := &cobra.Command{
 		Use:     "build",
 		Short:   "Build a package from a YAML configuration file",
@@ -85,30 +80,6 @@ func Build() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			if traceFile != "" {
-				w, err := os.Create(traceFile)
-				if err != nil {
-					return fmt.Errorf("creating trace file: %w", err)
-				}
-				defer w.Close()
-				exporter, err := stdouttrace.New(stdouttrace.WithWriter(w))
-				if err != nil {
-					return fmt.Errorf("creating stdout exporter: %w", err)
-				}
-				tp := trace.NewTracerProvider(trace.WithBatcher(exporter))
-				otel.SetTracerProvider(tp)
-
-				defer func() {
-					if err := tp.Shutdown(context.WithoutCancel(ctx)); err != nil {
-						clog.FromContext(ctx).Errorf("shutting down trace provider: %v", err)
-					}
-				}()
-
-				tctx, span := otel.Tracer("melange").Start(ctx, "build")
-				defer span.End()
-				ctx = tctx
-			}
-
 			r, err := getRunner(ctx, runner)
 			if err != nil {
 				return err
@@ -203,7 +174,6 @@ func Build() *cobra.Command {
 	cmd.Flags().StringVar(&cpu, "cpu", "", "default CPU resources to use for builds")
 	cmd.Flags().StringVar(&memory, "memory", "", "default memory resources to use for builds")
 	cmd.Flags().DurationVar(&timeout, "timeout", 0, "default timeout for builds")
-	cmd.Flags().StringVar(&traceFile, "trace", "", "where to write trace output")
 
 	return cmd
 }

pkg/cli/commands.go

@@ -15,21 +15,39 @@
 package cli
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
+	"os"
+	"slices"
+	"time"
 
 	"chainguard.dev/apko/pkg/log"
+	gtrace "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
 	"github.com/chainguard-dev/clog/gcp"
 	charmlog "github.com/charmbracelet/log"
 	"github.com/spf13/cobra"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"sigs.k8s.io/release-utils/version"
 )
 
 func New() *cobra.Command {
 	var logPolicy []string
 	var level log.CharmLogLevel
 	var gcplog bool
+	var gcptrace bool
+	var traceFile string
+
+	// Approximate calling defer() across Persistent[Pre/Post]RunE method calls.
+	stack := []func(context.Context) error{}
+	deferred := func(f func(context.Context) error) {
+		stack = append(stack, f)
+	}
+
 	cmd := &cobra.Command{
 		Use:               "melange",
 		DisableAutoGenTag: true,
@@ -48,13 +66,77 @@ func New() *cobra.Command {
 				slog.SetDefault(slog.New(charmlog.NewWithOptions(out, charmlog.Options{ReportTimestamp: true, Level: charmlog.Level(level)})))
 			}
 
+			var (
+				err error
+				exp sdktrace.SpanExporter
+			)
+
+			if gcptrace {
+				exp, err = gtrace.New()
+				if err != nil {
+					return fmt.Errorf("creating gcp trace exporter: %w", err)
+				}
+			} else if traceFile != "" {
+				w, err := os.Create(traceFile)
+				if err != nil {
+					return fmt.Errorf("creating trace file: %w", err)
+				}
+
+				deferred(func(context.Context) error {
+					return w.Close()
+				})
+
+				exp, err = stdouttrace.New(stdouttrace.WithWriter(w))
+				if err != nil {
+					return fmt.Errorf("creating stdout exporter: %w", err)
+				}
+			}
+
+			if exp != nil {
+				tp := sdktrace.NewTracerProvider(sdktrace.WithBatcher(exp))
+				otel.SetTracerProvider(tp)
+
+				deferred(func(ctx context.Context) error {
+					return tp.Shutdown(ctx)
+				})
+
+				deferred(func(ctx context.Context) error {
+					return tp.ForceFlush(ctx)
+				})
+
+				ctx, span := otel.Tracer("melange").Start(cmd.Context(), cmd.CalledAs())
+				cmd.SetContext(ctx)
+
+				deferred(func(context.Context) error {
+					span.End()
+					return nil
+				})
+			}
+
 			return nil
 		},
+		PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
+			// It shouldn't take very long to flush spans but we don't want the whole build to fail if it takes a few seconds.
+			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+			defer cancel()
+
+			slices.Reverse(stack)
+
+			errs := []error{}
+			for _, f := range stack {
+				errs = append(errs, f(ctx))
+			}
+
+			return errors.Join(errs...)
+		},
 	}
 	cmd.PersistentFlags().StringSliceVar(&logPolicy, "log-policy", []string{"builtin:stderr"}, "log policy (e.g. builtin:stderr, /tmp/log/foo)")
 	cmd.PersistentFlags().Var(&level, "log-level", "log level (e.g. debug, info, warn, error)")
 	cmd.PersistentFlags().BoolVar(&gcplog, "gcplog", false, "use GCP logging")
 	_ = cmd.PersistentFlags().MarkHidden("gcplog")
+	cmd.PersistentFlags().BoolVar(&gcptrace, "gcptrace", false, "use GCP tracing")
+	_ = cmd.PersistentFlags().MarkHidden("gcptrace")
+	cmd.PersistentFlags().StringVar(&traceFile, "trace", "", "where to write trace output")
 
 	cmd.AddCommand(Build())
 	cmd.AddCommand(Bump())