writer: new option to log rejected spans

docs/config-schema.md

@@ -154,6 +154,7 @@ The **nested** `writer` config object has the following fields:
 | `logDatapoints` | no | bool | If the log level is set to `debug` and this is true, all datapoints generated by the agent will be logged. (**default:** `false`) |
 | `logEvents` | no | bool | The analogue of `logDatapoints` for events. (**default:** `false`) |
 | `logTraceSpans` | no | bool | The analogue of `logDatapoints` for trace spans. (**default:** `false`) |
+| `logTraceSpansFailedToShip` | no | bool | If `true`, traces and spans which weren't successfully received by the backend, will be logged as json (**default:** `false`) |
 | `logDimensionUpdates` | no | bool | If `true`, dimension updates will be logged at the INFO level. (**default:** `false`) |
 | `logDroppedDatapoints` | no | bool | If true, and the log level is `debug`, filtered out datapoints will be logged. (**default:** `false`) |
 | `addGlobalDimensionsAsSpanTags` | no | bool | If true, the dimensions specified in the top-level `globalDimensions` configuration will be added to the tag set of all spans that are emitted by the writer.  If this is false, only the "host id" dimensions such as `host`, `AwsUniqueId`, etc. are added to the span tags. (**default:** `false`) |
@@ -428,6 +429,7 @@ where applicable:
     logDatapoints: false
     logEvents: false
     logTraceSpans: false
+    logTraceSpansFailedToShip: false
     logDimensionUpdates: false
     logDroppedDatapoints: false
     addGlobalDimensionsAsSpanTags: false

pkg/core/config/writer.go

@@ -71,6 +71,9 @@ type WriterConfig struct {
 	LogEvents bool `yaml:"logEvents"`
 	// The analogue of `logDatapoints` for trace spans.
 	LogTraceSpans bool `yaml:"logTraceSpans"`
+	// If `true`, traces and spans which weren't successfully received by the
+	// backend, will be logged as json
+	LogTraceSpansFailedToShip bool `yaml:"logTraceSpansFailedToShip"`
 	// If `true`, dimension updates will be logged at the INFO level.
 	LogDimensionUpdates bool `yaml:"logDimensionUpdates"`
 	// If true, and the log level is `debug`, filtered out datapoints will be

pkg/core/writer/signalfx/spans.go

@@ -21,9 +21,21 @@ func (sw *Writer) sendSpans(ctx context.Context, spans []*trace.Span) error {
 		// This sends synchonously
 		err := sw.client.AddSpans(context.Background(), spans)
 		if err != nil {
-			log.WithFields(log.Fields{
-				"error": err,
-			}).Error("Error shipping spans to SignalFx")
+			var meta log.Fields
+			if sw.conf.LogTraceSpansFailedToShip {
+				jsonEncodedSpans, _ := json.Marshal(spans)
+				meta = log.Fields{
+					"error":   err,
+					"payload": string(jsonEncodedSpans),
+				}
+			} else {
+				meta = log.Fields{
+					"error": err,
+				}
+			}
+
+			log.WithFields(meta).Error("Error shipping spans to SignalFx")
+
 			// If there is an error sending spans then just forget about them.
 			return err
 		}

selfdescribe.json

@@ -58322,6 +58322,14 @@
               "type": "bool",
               "elementKind": ""
             },
+            {
+              "yamlName": "logTraceSpansFailedToShip",
+              "doc": "If `true`, traces and spans which weren't successfully received by the backend, will be logged as json",
+              "default": false,
+              "required": false,
+              "type": "bool",
+              "elementKind": ""
+            },
             {
               "yamlName": "logDimensionUpdates",
               "doc": "If `true`, dimension updates will be logged at the INFO level.",