[tmpnet] Add optional stack traces to errors originating from tmpnet

tests/fixture/stacktrace/stacktrace.go

@@ -0,0 +1,128 @@
+// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package stacktrace
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"runtime"
+)
+
+// If the environment variable STACK_TRACE_ERRORS=1 is set, errors
+// passing through the functions defined in this package will have a
+// stack trace added to them. The following equivalents to stdlib error
+// functions are provided:
+//
+// - `fmt.Errorf` -> `stacktrace.Errorf`
+// - `errors.New` -> `stacktrace.New`
+//
+// Additionally, a stack trace can be added to an existing error with
+// `stacktrace.Wrap(err)`.
+
+var stackTraceErrors bool
+
+func init() {
+	if os.Getenv("STACK_TRACE_ERRORS") == "1" {
+		stackTraceErrors = true
+	}
+}
+
+type StackTraceError struct {
+	StackTrace []runtime.Frame
+	Cause      error
+}
+
+func (e StackTraceError) Error() string {
+	result := e.Cause.Error()
+	if !stackTraceErrors {
+		return result
+	}
+
+	result += "\nStack trace:\n"
+	for _, frame := range e.StackTrace {
+		result += fmt.Sprintf("%s:%d: %s\n", frame.File, frame.Line, frame.Function)
+	}
+	return result
+}
+
+func (e StackTraceError) Unwrap() error {
+	return e.Cause
+}
+
+func New(msg string) error {
+	if !stackTraceErrors {
+		return errors.New(msg)
+	}
+	return wrap(errors.New(msg))
+}
+
+// Errorf adds a stack trace to the last argument provided if it is an
+// error and stack traces are enabled.
+func Errorf(format string, args ...any) error {
+	if !stackTraceErrors {
+		return fmt.Errorf(format, args...)
+	}
+
+	// Assume the last argument is an error requring a stack trace if it is of type error
+	err, ok := args[len(args)-1].(error)
+	if !ok {
+		return fmt.Errorf(format, args...)
+	}
+
+	newErr := fmt.Errorf(format, args...)
+
+	// If there's already a StackTraceError, preserve its stack but update the cause
+	var existingStackErr StackTraceError
+	if errors.As(err, &existingStackErr) {
+		existingStackErr.Cause = newErr
+		return existingStackErr
+	}
+
+	// No stack trace exists, capture one now
+	return wrap(newErr)
+}
+
+func Wrap(err error) error {
+	if !stackTraceErrors {
+		return err
+	}
+	return wrap(err)
+}
+
+// wrap adds a stack trace to err if stack traces are enabled and it
+// doesn't already have one.
+func wrap(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	// If there's already a StackTraceError in the chain, just return it
+	var existingStackErr StackTraceError
+	if errors.As(err, &existingStackErr) {
+		return err
+	}
+
+	// Need to capture a stack trace
+	const depth = 32
+	var pcs [depth]uintptr
+	skip := 3 // skip wrap, New/Wrap/Errorf, and runtime.Callers
+	n := runtime.Callers(skip, pcs[:])
+
+	frames := runtime.CallersFrames(pcs[:n])
+	var frameSlice []runtime.Frame
+
+	for {
+		frame, more := frames.Next()
+		frameSlice = append(frameSlice, frame)
+		if !more {
+			break
+		}
+	}
+
+	return StackTraceError{
+		StackTrace: frameSlice,
+		Cause:      err,
+	}
+}

tests/fixture/tmpnet/README.md

@@ -16,6 +16,8 @@ orchestrate the same temporary networks without the use of an rpc daemon.
   - [Simplifying usage with direnv](#simplifying-usage-with-direnv)
     - [Deprecated usage with e2e suite](#deprecated-usage-with-e2e-suite)
   - [Via code](#via-code)
+  - [Enabling errors with stack traces](#enabling-errors-with-stack-traces)
+    - [Ensuring stack trace support](#ensuring-stack-trace-support)
 - [Networking configuration](#networking-configuration)
 - [Configuration on disk](#configuration-on-disk)
   - [Common networking configuration](#common-networking-configuration)
@@ -188,6 +190,30 @@ uris := network.GetNodeURIs()
 network.Stop(context.Background())
 ```
 
+### Enabling errors with stack traces
+[Top](#table-of-contents)
+
+By default, errors originating from tmpnet will be standard golang
+errors. Setting `STACK_TRACE_ERRORS=1` in the environment in which
+tmpnet is run will ensure that such errors also include the stack trace
+from the point of failure. While there is some overhead associated
+with collecting and storing a stack trace, a testing tool like tmpnet
+is not performance critical.
+
+#### Ensuring stack trace support
+[Top](#table-of-contents)
+
+For an error originating from tmpnet to have a stack trace, the error
+must pass through one of the methods of the
+`github.com/ava-labs/avalanchego/tests/fixture/stacktrace`
+package. The following alternatives to stdlib functions are supported:
+
+| stdlib       | stacktrace             |
+|:-------------|:-----------------------|
+| `err`        | `stacktrace.Wrap(err)` |
+| `errors.New` | `stacktrace.New`       |
+| `fmt.Errorf` | `stacktrace.Errorf`    |
+
 ## Networking configuration
 [Top](#table-of-contents)
 

tests/fixture/tmpnet/check_monitoring.go

@@ -7,7 +7,6 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -23,6 +22,7 @@ import (
 	"github.com/prometheus/common/model"
 	"go.uber.org/zap"
 
+	"github.com/ava-labs/avalanchego/tests/fixture/stacktrace"
 	"github.com/ava-labs/avalanchego/utils/logging"
 )
 
@@ -51,7 +51,7 @@ func waitForCount(ctx context.Context, log logging.Logger, name string, getCount
 		},
 	)
 	if err != nil {
-		return fmt.Errorf("%s not found before timeout: %w", name, err)
+		return stacktrace.Errorf("%s not found before timeout: %w", name, err)
 	}
 	return nil
 }
@@ -62,17 +62,17 @@ func waitForCount(ctx context.Context, log logging.Logger, name string, getCount
 func CheckLogsExist(ctx context.Context, log logging.Logger, networkUUID string) error {
 	username, password, err := getCollectorCredentials(promtailCmd)
 	if err != nil {
-		return fmt.Errorf("failed to get collector credentials: %w", err)
+		return stacktrace.Errorf("failed to get collector credentials: %w", err)
 	}
 
 	url := getLokiURL()
 	if !strings.HasPrefix(url, "https") {
-		return fmt.Errorf("loki URL must be https for basic auth to be secure: %s", url)
+		return stacktrace.Errorf("loki URL must be https for basic auth to be secure: %s", url)
 	}
 
 	selectors, err := getSelectors(networkUUID)
 	if err != nil {
-		return err
+		return stacktrace.Wrap(err)
 	}
 	query := fmt.Sprintf("sum(count_over_time({%s}[1h]))", selectors)
 
@@ -81,14 +81,18 @@ func CheckLogsExist(ctx context.Context, log logging.Logger, networkUUID string)
 		zap.String("query", query),
 	)
 
-	return waitForCount(
+	err = waitForCount(
 		ctx,
 		log,
 		"logs",
 		func() (int, error) {
 			return queryLoki(ctx, url, username, password, query)
 		},
 	)
+	if err != nil {
+		return stacktrace.Errorf("failed to find logs: %w", err)
+	}
+	return nil
 }
 
 func queryLoki(
@@ -106,7 +110,7 @@ func queryLoki(
 	// Create request
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
 	if err != nil {
-		return 0, fmt.Errorf("failed to create request: %w", err)
+		return 0, stacktrace.Errorf("failed to create request: %w", err)
 	}
 
 	auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
@@ -115,18 +119,18 @@ func queryLoki(
 	// Execute request
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return 0, fmt.Errorf("failed to execute request: %w", err)
+		return 0, stacktrace.Errorf("failed to execute request: %w", err)
 	}
 	defer resp.Body.Close()
 
 	// Read and parse response
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return 0, fmt.Errorf("failed to read response: %w", err)
+		return 0, stacktrace.Errorf("failed to read response: %w", err)
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return 0, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body))
+		return 0, stacktrace.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body))
 	}
 
 	// Parse JSON response
@@ -140,25 +144,25 @@ func queryLoki(
 	}
 
 	if err := json.Unmarshal(body, &result); err != nil {
-		return 0, fmt.Errorf("failed to parse response: %w", err)
+		return 0, stacktrace.Errorf("failed to parse response: %w", err)
 	}
 
 	// Extract count value
 	if len(result.Data.Result) == 0 {
 		return 0, nil
 	}
 	if len(result.Data.Result[0].Value) != 2 {
-		return 0, errors.New("unexpected value format in response")
+		return 0, stacktrace.New("unexpected value format in response")
 	}
 	// Convert value to a string
 	valueStr, ok := result.Data.Result[0].Value[1].(string)
 	if !ok {
-		return 0, errors.New("value is not a string")
+		return 0, stacktrace.New("value is not a string")
 	}
 	// Convert string to float64 first to handle scientific notation
 	floatVal, err := strconv.ParseFloat(valueStr, 64)
 	if err != nil {
-		return 0, fmt.Errorf("parsing count value: %w", err)
+		return 0, stacktrace.Errorf("parsing count value: %w", err)
 	}
 	// Round to nearest integer
 	return int(math.Round(floatVal)), nil
@@ -169,17 +173,17 @@ func queryLoki(
 func CheckMetricsExist(ctx context.Context, log logging.Logger, networkUUID string) error {
 	username, password, err := getCollectorCredentials(prometheusCmd)
 	if err != nil {
-		return fmt.Errorf("failed to get collector credentials: %w", err)
+		return stacktrace.Errorf("failed to get collector credentials: %w", err)
 	}
 
 	url := getPrometheusURL()
 	if !strings.HasPrefix(url, "https") {
-		return fmt.Errorf("prometheus URL must be https for basic auth to be secure: %s", url)
+		return stacktrace.Errorf("prometheus URL must be https for basic auth to be secure: %s", url)
 	}
 
 	selectors, err := getSelectors(networkUUID)
 	if err != nil {
-		return err
+		return stacktrace.Wrap(err)
 	}
 	query := fmt.Sprintf("count({%s})", selectors)
 
@@ -216,7 +220,7 @@ func queryPrometheus(
 		},
 	})
 	if err != nil {
-		return 0, fmt.Errorf("failed to create client: %w", err)
+		return 0, stacktrace.Errorf("failed to create client: %w", err)
 	}
 
 	// Query Prometheus
@@ -226,7 +230,7 @@ func queryPrometheus(
 		Step:  time.Minute,
 	})
 	if err != nil {
-		return 0, fmt.Errorf("query failed: %w", err)
+		return 0, stacktrace.Errorf("query failed: %w", err)
 	}
 	if len(warnings) > 0 {
 		log.Warn("prometheus query warnings",
@@ -235,7 +239,7 @@ func queryPrometheus(
 	}
 
 	if matrix, ok := result.(model.Matrix); !ok {
-		return 0, fmt.Errorf("unexpected result type: %s", result.Type())
+		return 0, stacktrace.Errorf("unexpected result type: %s", result.Type())
 	} else if len(matrix) > 0 {
 		return int(matrix[0].Values[len(matrix[0].Values)-1].Value), nil
 	}
@@ -269,7 +273,7 @@ func getSelectors(networkUUID string) (string, error) {
 		}
 	}
 	if len(selectors) == 0 {
-		return "", errors.New("no GH_* env vars set to use for selectors")
+		return "", stacktrace.New("no GH_* env vars set to use for selectors")
 	}
 
 	return strings.Join(selectors, ","), nil

tests/fixture/tmpnet/flags/kube_runtime.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/spf13/pflag"
 
+	"github.com/ava-labs/avalanchego/tests/fixture/stacktrace"
 	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
 )
 
@@ -89,13 +90,13 @@ func (v *kubeRuntimeVars) register(stringVar varFunc[string], uintVar varFunc[ui
 
 func (v *kubeRuntimeVars) getKubeRuntimeConfig() (*tmpnet.KubeRuntimeConfig, error) {
 	if len(v.namespace) == 0 {
-		return nil, errKubeNamespaceRequired
+		return nil, stacktrace.Wrap(errKubeNamespaceRequired)
 	}
 	if len(v.image) == 0 {
-		return nil, errKubeImageRequired
+		return nil, stacktrace.Wrap(errKubeImageRequired)
 	}
 	if v.volumeSizeGB < tmpnet.MinimumVolumeSizeGB {
-		return nil, errKubeMinVolumeSizeRequired
+		return nil, stacktrace.Wrap(errKubeMinVolumeSizeRequired)
 	}
 	return &tmpnet.KubeRuntimeConfig{
 		ConfigPath:             v.config.Path,