From e6ffdc6826e1c02f2c4194b546c7c9160381365e Mon Sep 17 00:00:00 2001 From: Emilio Garcia Date: Wed, 7 Dec 2022 12:36:17 -0500 Subject: [PATCH] Notice Expected Errors A new API has been added that will allow users to notice errors without triggering their error alerts or metrics: NoticeExpectedError() ` --- v3/examples/server/main.go | 8 ++ v3/newrelic/errors_from_internal.go | 5 +- v3/newrelic/harvest.go | 6 +- v3/newrelic/harvest_test.go | 30 +++++++ .../internal_errors_stacktrace_test.go | 2 +- v3/newrelic/internal_errors_test.go | 2 +- v3/newrelic/internal_txn.go | 29 ++++--- v3/newrelic/intrinsics.go | 10 ++- v3/newrelic/metric_names.go | 4 +- v3/newrelic/tracing.go | 48 +++++++---- v3/newrelic/transaction.go | 86 +++++++++++++------ v3/newrelic/txn_trace.go | 2 +- 12 files changed, 171 insertions(+), 61 deletions(-) diff --git a/v3/examples/server/main.go b/v3/examples/server/main.go index 9919539c1..c012e349b 100644 --- a/v3/examples/server/main.go +++ b/v3/examples/server/main.go @@ -31,6 +31,13 @@ func noticeError(w http.ResponseWriter, r *http.Request) { txn.NoticeError(errors.New("my error message")) } +func noticeExpectedError(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "noticing an error") + + txn := newrelic.FromContext(r.Context()) + txn.NoticeExpectedError(errors.New("my expected error message")) +} + func noticeErrorWithAttributes(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "noticing an error") @@ -273,6 +280,7 @@ func main() { http.HandleFunc(newrelic.WrapHandleFunc(app, "/", index)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/version", versionHandler)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/notice_error", noticeError)) + http.HandleFunc(newrelic.WrapHandleFunc(app, "/notice_expected_error", noticeExpectedError)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/notice_error_with_attributes", noticeErrorWithAttributes)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/custom_event", customEvent)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/set_name", setName)) diff --git a/v3/newrelic/errors_from_internal.go b/v3/newrelic/errors_from_internal.go index 7813d102f..9b174f3f0 100644 --- a/v3/newrelic/errors_from_internal.go +++ b/v3/newrelic/errors_from_internal.go @@ -61,6 +61,7 @@ type errorData struct { Msg string Klass string SpanID string + Expect bool } // txnError combines error data with information about a transaction. txnError is used for @@ -113,7 +114,7 @@ func (h *tracedError) WriteJSON(buf *bytes.Buffer) { buf.WriteByte(',') buf.WriteString(`"intrinsics"`) buf.WriteByte(':') - intrinsicsJSON(&h.txnEvent, buf) + intrinsicsJSON(&h.txnEvent, buf, h.errorData.Expect) if nil != h.Stack { buf.WriteByte(',') buf.WriteString(`"stack_trace"`) @@ -152,7 +153,7 @@ func mergeTxnErrors(errors *harvestErrors, errs txnErrors, txnEvent txnEvent) { } func (errors harvestErrors) Data(agentRunID string, harvestStart time.Time) ([]byte, error) { - if 0 == len(errors) { + if len(errors) == 0 { return nil, nil } estimate := 1024 * len(errors) diff --git a/v3/newrelic/harvest.go b/v3/newrelic/harvest.go index 3a7e1cf61..72159e49a 100644 --- a/v3/newrelic/harvest.go +++ b/v3/newrelic/harvest.go @@ -327,12 +327,16 @@ func createTxnMetrics(args *txnData, metrics *metricTable) { } // Error Metrics - if args.HasErrors() { + if args.NoticeErrors() { metrics.addSingleCount(errorsRollupMetric.all, forced) metrics.addSingleCount(errorsRollupMetric.webOrOther(args.IsWeb), forced) metrics.addSingleCount(errorsPrefix+args.FinalName, forced) } + if args.HasExpectedErrors() { + metrics.addSingleCount(expectedErrorsRollupMetric.all, forced) + } + // Queueing Metrics if args.Queuing > 0 { metrics.addDuration(queueMetric, "", args.Queuing, args.Queuing, forced) diff --git a/v3/newrelic/harvest_test.go b/v3/newrelic/harvest_test.go index 0acbe6ae1..b175e2232 100644 --- a/v3/newrelic/harvest_test.go +++ b/v3/newrelic/harvest_test.go @@ -771,6 +771,7 @@ func TestCreateTxnMetrics(t *testing.T) { webName := "WebTransaction/zip/zap" backgroundName := "OtherTransaction/zip/zap" args := &txnData{} + args.noticeErrors = true args.Duration = 123 * time.Second args.TotalTime = 150 * time.Second args.ApdexThreshold = 2 * time.Second @@ -803,6 +804,7 @@ func TestCreateTxnMetrics(t *testing.T) { args.FinalName = webName args.IsWeb = true args.Errors = nil + args.noticeErrors = false args.Zone = apdexTolerating metrics = newMetricTable(100, time.Now()) createTxnMetrics(args, metrics) @@ -821,6 +823,7 @@ func TestCreateTxnMetrics(t *testing.T) { args.FinalName = backgroundName args.IsWeb = false args.Errors = txnErrors + args.noticeErrors = true args.Zone = apdexNone metrics = newMetricTable(100, time.Now()) createTxnMetrics(args, metrics) @@ -838,9 +841,32 @@ func TestCreateTxnMetrics(t *testing.T) { {Name: "ErrorsByCaller/Unknown/Unknown/Unknown/Unknown/allOther", Scope: "", Forced: false, Data: []float64{1, 0, 0, 0, 0, 0}}, }) + // Verify expected errors metrics + args.FinalName = backgroundName + args.IsWeb = false + args.Errors = txnErrors + args.noticeErrors = false + args.expectedErrors = true + args.Zone = apdexNone + metrics = newMetricTable(100, time.Now()) + createTxnMetrics(args, metrics) + expectMetrics(t, metrics, []internal.WantMetric{ + {Name: backgroundName, Scope: "", Forced: true, Data: []float64{1, 123, 0, 123, 123, 123 * 123}}, + {Name: backgroundRollup, Scope: "", Forced: true, Data: []float64{1, 123, 0, 123, 123, 123 * 123}}, + {Name: "OtherTransactionTotalTime", Scope: "", Forced: true, Data: []float64{1, 150, 150, 150, 150, 150 * 150}}, + {Name: "OtherTransactionTotalTime/zip/zap", Scope: "", Forced: false, Data: []float64{1, 150, 150, 150, 150, 150 * 150}}, + {Name: "ErrorsExpected/all", Scope: "", Forced: true, Data: []float64{1, 0, 0, 0, 0, 0}}, + {Name: "DurationByCaller/Unknown/Unknown/Unknown/Unknown/all", Scope: "", Forced: false, Data: []float64{1, 123, 123, 123, 123, 123 * 123}}, + {Name: "DurationByCaller/Unknown/Unknown/Unknown/Unknown/allOther", Scope: "", Forced: false, Data: []float64{1, 123, 123, 123, 123, 123 * 123}}, + {Name: "ErrorsByCaller/Unknown/Unknown/Unknown/Unknown/all", Scope: "", Forced: false, Data: []float64{1, 0, 0, 0, 0, 0}}, + {Name: "ErrorsByCaller/Unknown/Unknown/Unknown/Unknown/allOther", Scope: "", Forced: false, Data: []float64{1, 0, 0, 0, 0, 0}}, + }) + args.FinalName = backgroundName args.IsWeb = false args.Errors = nil + args.noticeErrors = false + args.expectedErrors = false args.Zone = apdexNone metrics = newMetricTable(100, time.Now()) createTxnMetrics(args, metrics) @@ -889,6 +915,7 @@ func TestCreateTxnMetricsOldCAT(t *testing.T) { args.FinalName = webName args.IsWeb = true args.Errors = txnErrors + args.noticeErrors = true args.Zone = apdexTolerating metrics := newMetricTable(100, time.Now()) createTxnMetrics(args, metrics) @@ -908,6 +935,7 @@ func TestCreateTxnMetricsOldCAT(t *testing.T) { args.FinalName = webName args.IsWeb = true args.Errors = nil + args.noticeErrors = false args.Zone = apdexTolerating metrics = newMetricTable(100, time.Now()) createTxnMetrics(args, metrics) @@ -924,6 +952,7 @@ func TestCreateTxnMetricsOldCAT(t *testing.T) { args.FinalName = backgroundName args.IsWeb = false args.Errors = txnErrors + args.noticeErrors = true args.Zone = apdexNone metrics = newMetricTable(100, time.Now()) createTxnMetrics(args, metrics) @@ -940,6 +969,7 @@ func TestCreateTxnMetricsOldCAT(t *testing.T) { args.FinalName = backgroundName args.IsWeb = false args.Errors = nil + args.noticeErrors = false args.Zone = apdexNone metrics = newMetricTable(100, time.Now()) createTxnMetrics(args, metrics) diff --git a/v3/newrelic/internal_errors_stacktrace_test.go b/v3/newrelic/internal_errors_stacktrace_test.go index 6ff3a9304..a7fae6fa6 100644 --- a/v3/newrelic/internal_errors_stacktrace_test.go +++ b/v3/newrelic/internal_errors_stacktrace_test.go @@ -64,7 +64,7 @@ func TestStackTrace(t *testing.T) { } for idx, tc := range testcases { - data, err := errDataFromError(tc.Error) + data, err := errDataFromError(tc.Error, false) if err != nil { t.Errorf("testcase %d: got error: %v", idx, err) continue diff --git a/v3/newrelic/internal_errors_test.go b/v3/newrelic/internal_errors_test.go index d5fa710cd..aa618d155 100644 --- a/v3/newrelic/internal_errors_test.go +++ b/v3/newrelic/internal_errors_test.go @@ -636,7 +636,7 @@ func TestErrorClass(t *testing.T) { } for idx, tc := range testcases { - data, err := errDataFromError(tc.Error) + data, err := errDataFromError(tc.Error, false) if err != nil { t.Errorf("testcase %d: got error: %v", idx, err) continue diff --git a/v3/newrelic/internal_txn.go b/v3/newrelic/internal_txn.go index ee89456f6..b98fb45c2 100644 --- a/v3/newrelic/internal_txn.go +++ b/v3/newrelic/internal_txn.go @@ -366,7 +366,7 @@ func headersJustWritten(thd *thread, code int, hdr http.Header) { if txn.appRun.responseCodeIsError(code) { e := txnErrorFromResponseCode(time.Now(), code) e.Stack = getStackTrace() - thd.noticeErrorInternal(e) + thd.noticeErrorInternal(e, false) } } @@ -425,7 +425,7 @@ func (thd *thread) End(recovered interface{}) error { if nil != recovered { e := txnErrorFromPanic(time.Now(), recovered) e.Stack = getStackTrace() - thd.noticeErrorInternal(e) + thd.noticeErrorInternal(e, false) log.Println(string(debug.Stack())) } @@ -447,7 +447,7 @@ func (thd *thread) End(recovered interface{}) error { txn.ApdexThreshold = internal.CalculateApdexThreshold(txn.Reply, txn.FinalName) if txn.getsApdex() { - if txn.HasErrors() { + if txn.HasErrors() && txn.NoticeErrors() { txn.Zone = apdexFailing } else { txn.Zone = calculateApdexZone(txn.ApdexThreshold, txn.Duration) @@ -461,7 +461,7 @@ func (thd *thread) End(recovered interface{}) error { "name": txn.FinalName, "duration_ms": txn.Duration.Seconds() * 1000.0, "ignored": txn.ignore, - "app_connected": "" != txn.Reply.RunID, + "app_connected": txn.Reply.RunID != "", }) } @@ -559,12 +559,18 @@ const ( securityPolicyErrorMsg = "message removed by security policy" ) -func (thd *thread) noticeErrorInternal(err errorData) error { +func (thd *thread) noticeErrorInternal(err errorData, expect bool) error { txn := thd.txn if !txn.Config.ErrorCollector.Enabled { return errorsDisabled } + if !expect { + thd.noticeErrors = true + } else { + thd.expectedErrors = true + } + if nil == txn.Errors { txn.Errors = newTxnErrors(maxTxnErrors) } @@ -643,12 +649,13 @@ func errorAttributesMethod(err error) map[string]interface{} { return nil } -func errDataFromError(input error) (data errorData, err error) { +func errDataFromError(input error, expect bool) (data errorData, err error) { cause := errorCause(input) data = errorData{ - When: time.Now(), - Msg: input.Error(), + When: time.Now(), + Msg: input.Error(), + Expect: expect, } if c := errorClassMethod(input); "" != c { @@ -700,7 +707,7 @@ func errDataFromError(input error) (data errorData, err error) { return data, nil } -func (thd *thread) NoticeError(input error) error { +func (thd *thread) NoticeError(input error, expect bool) error { txn := thd.txn txn.Lock() defer txn.Unlock() @@ -713,7 +720,7 @@ func (thd *thread) NoticeError(input error) error { return errNilError } - data, err := errDataFromError(input) + data, err := errDataFromError(input, expect) if nil != err { return err } @@ -722,7 +729,7 @@ func (thd *thread) NoticeError(input error) error { data.ExtraAttributes = nil } - return thd.noticeErrorInternal(data) + return thd.noticeErrorInternal(data, expect) } func (txn *txn) SetName(name string) error { diff --git a/v3/newrelic/intrinsics.go b/v3/newrelic/intrinsics.go index f626c8f41..39eff9648 100644 --- a/v3/newrelic/intrinsics.go +++ b/v3/newrelic/intrinsics.go @@ -7,13 +7,17 @@ import ( "bytes" ) +const ( + expectErrorAttr = "error.expected" +) + func addOptionalStringField(w *jsonFieldsWriter, key, value string) { if value != "" { w.stringField(key, value) } } -func intrinsicsJSON(e *txnEvent, buf *bytes.Buffer) { +func intrinsicsJSON(e *txnEvent, buf *bytes.Buffer, expect bool) { w := jsonFieldsWriter{buf: buf} buf.WriteByte('{') @@ -27,6 +31,10 @@ func intrinsicsJSON(e *txnEvent, buf *bytes.Buffer) { w.boolField("sampled", e.BetterCAT.Sampled) } + if expect { + w.stringField(expectErrorAttr, "true") + } + if e.CrossProcess.Used() { addOptionalStringField(&w, "client_cross_process_id", e.CrossProcess.ClientID) addOptionalStringField(&w, "trip_id", e.CrossProcess.TripID) diff --git a/v3/newrelic/metric_names.go b/v3/newrelic/metric_names.go index 7bb053e23..cc0883416 100644 --- a/v3/newrelic/metric_names.go +++ b/v3/newrelic/metric_names.go @@ -187,8 +187,8 @@ func (r rollupMetric) webOrOther(isWeb bool) string { } var ( - errorsRollupMetric = newRollupMetric("Errors/") - + errorsRollupMetric = newRollupMetric("Errors/") + expectedErrorsRollupMetric = newRollupMetric("ErrorsExpected/") // source.datanerd.us/agents/agent-specs/blob/master/APIs/external_segment.md // source.datanerd.us/agents/agent-specs/blob/master/APIs/external_cat.md // source.datanerd.us/agents/agent-specs/blob/master/Cross-Application-Tracing-PORTED.md diff --git a/v3/newrelic/tracing.go b/v3/newrelic/tracing.go index c17f2a2ad..d6a966401 100644 --- a/v3/newrelic/tracing.go +++ b/v3/newrelic/tracing.go @@ -63,21 +63,35 @@ func (bc *betterCAT) SetTraceAndTxnIDs(traceID string) { // txnData contains the recorded data of a transaction. type txnData struct { - txnEvent - IsWeb bool - Name string // Work in progress name. - Errors txnErrors // Lazily initialized. - Stop time.Time - ApdexThreshold time.Duration + IsWeb bool + SlowQueriesEnabled bool + noticeErrors bool // If errors are not expected or ignored, then true + expectedErrors bool stamp segmentStamp threadIDCounter uint64 + Name string // Work in progress name. + rootSpanID string + + txnEvent + TxnTrace txnTrace + + Stop time.Time + ApdexThreshold time.Duration + SlowQueryThreshold time.Duration + + SlowQueries *slowQueries + + // These better CAT supportability fields are left outside of + // TxnEvent.BetterCAT to minimize the size of transaction event memory. + DistributedTracingSupport distributedTracingSupport + TraceIDGenerator *internal.TraceIDGenerator ShouldCollectSpanEvents func() bool ShouldCreateSpanGUID func() bool - rootSpanID string rootSpanErrData *errorData + Errors txnErrors // Lazily initialized. SpanEvents []*spanEvent logs logEventHeap @@ -85,16 +99,6 @@ type txnData struct { datastoreSegments map[datastoreMetricKey]*metricData externalSegments map[externalMetricKey]*metricData messageSegments map[internal.MessageMetricKey]*metricData - - TxnTrace txnTrace - - SlowQueriesEnabled bool - SlowQueryThreshold time.Duration - SlowQueries *slowQueries - - // These better CAT supportability fields are left outside of - // TxnEvent.BetterCAT to minimize the size of transaction event memory. - DistributedTracingSupport distributedTracingSupport } func (t *txnData) saveTraceSegment(end segmentEnd, name string, attrs spanAttributeMap, externalGUID string) { @@ -320,11 +324,21 @@ const ( datastoreOperationUnknown = "other" ) +// NoticeErrors indicates whether the errors collected count towards error/ metrics +func (t *txnData) NoticeErrors() bool { + return t.noticeErrors +} + // HasErrors indicates whether the transaction had errors. func (t *txnData) HasErrors() bool { return len(t.Errors) > 0 } +// HasExpectedErrors is a special case where the txn has errors but we dont increment error metrics +func (t *txnData) HasExpectedErrors() bool { + return t.expectedErrors +} + func (t *txnData) time(now time.Time) segmentTime { // Update the stamp before using it so that a 0 stamp can be special. t.stamp++ diff --git a/v3/newrelic/transaction.go b/v3/newrelic/transaction.go index 78b3bed60..31f22b39e 100644 --- a/v3/newrelic/transaction.go +++ b/v3/newrelic/transaction.go @@ -43,14 +43,12 @@ func (txn *Transaction) End() { txn.thread.logAPIError(txn.thread.End(r), "end transaction", nil) } -// // SetOption allows the setting of some transaction TraceOption parameters // after the transaction has already been started, such as specifying a new // source code location for code-level metrics. // // The set of options should be the complete set you wish to have in effect, // just as if you were calling StartTransaction now with the same set of options. -// func (txn *Transaction) SetOption(options ...TraceOption) { if txn == nil || txn.thread == nil || txn.thread.txn == nil { return @@ -94,14 +92,14 @@ func (txn *Transaction) SetName(name string) { // NoticeError examines whether the error implements the following optional // methods: // -// // StackTrace records a stack trace -// StackTrace() []uintptr +// // StackTrace records a stack trace +// StackTrace() []uintptr // -// // ErrorClass sets the error's class -// ErrorClass() string +// // ErrorClass sets the error's class +// ErrorClass() string // -// // ErrorAttributes sets the errors attributes -// ErrorAttributes() map[string]interface{} +// // ErrorAttributes sets the errors attributes +// ErrorAttributes() map[string]interface{} // // The newrelic.Error type, which implements these methods, is the recommended // way to directly control the recorded error's message, class, stacktrace, @@ -113,7 +111,44 @@ func (txn *Transaction) NoticeError(err error) { if nil == txn.thread { return } - txn.thread.logAPIError(txn.thread.NoticeError(err), "notice error", nil) + txn.thread.logAPIError(txn.thread.NoticeError(err, false), "notice error", nil) +} + +// NoticeExpectedError records an error that was expected to occur. Errors recoreded with this +// method will not trigger any error alerts or count towards your error metrics. +// The Transaction saves the first five errors. +// For more control over the recorded error fields, see the +// newrelic.Error type. +// +// In certain situations, using this method may result in an error being +// recorded twice. Errors are automatically recorded when +// Transaction.WriteHeader receives a status code at or above 400 or strictly +// below 100 that is not in the IgnoreStatusCodes configuration list. This +// method is unaffected by the IgnoreStatusCodes configuration list. +// +// NoticeExpectedError examines whether the error implements the following optional +// methods: +// +// // StackTrace records a stack trace +// StackTrace() []uintptr +// +// // ErrorClass sets the error's class +// ErrorClass() string +// +// // ErrorAttributes sets the errors attributes +// ErrorAttributes() map[string]interface{} +// +// The newrelic.Error type, which implements these methods, is the recommended +// way to directly control the recorded error's message, class, stacktrace, +// and attributes. +func (txn *Transaction) NoticeExpectedError(err error) { + if nil == txn { + return + } + if nil == txn.thread { + return + } + txn.thread.logAPIError(txn.thread.NoticeError(err, true), "notice error", nil) } // AddAttribute adds a key value pair to the transaction event, errors, @@ -309,13 +344,11 @@ func (txn *Transaction) AcceptDistributedTraceHeaders(t TransportType, hdrs http txn.thread.logAPIError(txn.thread.AcceptDistributedTraceHeaders(t, hdrs), "accept trace payload", nil) } -// // AcceptDistributedTraceHeadersFromJSON works just like AcceptDistributedTraceHeaders(), except // that it takes the header data as a JSON string à la DistributedTraceHeadersFromJSON(). Additionally // (unlike AcceptDistributedTraceHeaders()) it returns an error if it was unable to successfully // convert the JSON string to http headers. There is no guarantee that the header data found in JSON // is correct beyond conforming to the expected types and syntax. -// func (txn *Transaction) AcceptDistributedTraceHeadersFromJSON(t TransportType, jsondata string) error { hdrs, err := DistributedTraceHeadersFromJSON(jsondata) if err != nil { @@ -325,7 +358,6 @@ func (txn *Transaction) AcceptDistributedTraceHeadersFromJSON(t TransportType, j return nil } -// // DistributedTraceHeadersFromJSON takes a set of distributed trace headers as a JSON-encoded string // and emits a http.Header value suitable for passing on to the // txn.AcceptDistributedTraceHeaders() function. @@ -336,27 +368,33 @@ func (txn *Transaction) AcceptDistributedTraceHeadersFromJSON(t TransportType, j // languages which may natively handle these header values as JSON strings. // // For example, given the input string -// `{"traceparent": "frob", "tracestate": "blorfl", "newrelic": "xyzzy"}` +// +// `{"traceparent": "frob", "tracestate": "blorfl", "newrelic": "xyzzy"}` +// // This will emit an http.Header value with headers "traceparent", "tracestate", and "newrelic". // Specifically: -// http.Header{ -// "Traceparent": {"frob"}, -// "Tracestate": {"blorfl"}, -// "Newrelic": {"xyzzy"}, -// } +// +// http.Header{ +// "Traceparent": {"frob"}, +// "Tracestate": {"blorfl"}, +// "Newrelic": {"xyzzy"}, +// } // // The JSON string must be a single object whose values may be strings or arrays of strings. // These are translated directly to http headers with singleton or multiple values. // In the case of multiple string values, these are translated to a multi-value HTTP // header. For example: -// `{"traceparent": "12345", "colors": ["red", "green", "blue"]}` +// +// `{"traceparent": "12345", "colors": ["red", "green", "blue"]}` +// // which produces -// http.Header{ -// "Traceparent": {"12345"}, -// "Colors": {"red", "green", "blue"}, -// } -// (Note that the HTTP headers are capitalized.) // +// http.Header{ +// "Traceparent": {"12345"}, +// "Colors": {"red", "green", "blue"}, +// } +// +// (Note that the HTTP headers are capitalized.) func DistributedTraceHeadersFromJSON(jsondata string) (hdrs http.Header, err error) { var raw interface{} hdrs = http.Header{} diff --git a/v3/newrelic/txn_trace.go b/v3/newrelic/txn_trace.go index 6050ba69e..2e925c5e4 100644 --- a/v3/newrelic/txn_trace.go +++ b/v3/newrelic/txn_trace.go @@ -281,7 +281,7 @@ func (trace *harvestTrace) writeJSON(buf *bytes.Buffer) { userAttributesJSON(trace.Attrs, buf, destTxnTrace, nil) buf.WriteByte(',') buf.WriteString(`"intrinsics":`) - intrinsicsJSON(&trace.txnEvent, buf) + intrinsicsJSON(&trace.txnEvent, buf, false) buf.WriteByte('}') // If the trace string pool is used, end another array here.