Skip to content

Commit

Permalink
Merge pull request #540 from mirackara/customeventlimitincrease
Browse files Browse the repository at this point in the history 
Harvest Limit Increase + Patch for DT Spans
  • Loading branch information
@nr-swilloughby
nr-swilloughby committed Aug 1, 2022
2 parents c9d5980 + 22bb213 commit eaaf111
Show file tree
Hide file tree
Showing 9 changed files with 47 additions and 33 deletions.
9 changes: 8 additions & 1 deletion v3/internal/connect_reply.go
View file
Expand Up @@ -104,7 +104,8 @@ type ConnectReply struct {
} `json:"agent_config"`

// Faster Event Harvest
EventData EventHarvestConfig `json:"event_harvest_config"`
EventData EventHarvestConfig `json:"event_harvest_config"`
SpanEventHarvestConfig `json:"span_event_harvest_config"`
}

// EventHarvestConfig contains fields relating to faster event harvest.
Expand All @@ -124,6 +125,12 @@ type EventHarvestConfig struct {
} `json:"harvest_limits"`
}

// SpanEventHarvestConfig contains the Reporting period time and the given harvest limit.
type SpanEventHarvestConfig struct {
ReportPeriod *uint `json:"report_period_ms"`
HarvestLimit *uint `json:"harvest_limit"`
}

// ConfigurablePeriod returns the Faster Event Harvest configurable reporting period if it is set, or the default
// report period otherwise.
func (r *ConnectReply) ConfigurablePeriod() time.Duration {
Expand Down
3 changes: 2 additions & 1 deletion v3/internal/connect_reply_test.go
View file
Expand Up @@ -5,6 +5,7 @@ package internal

import (
"encoding/json"
"fmt"
"testing"
"time"
)
Expand Down Expand Up @@ -178,7 +179,7 @@ func TestDefaultEventHarvestConfigJSON(t *testing.T) {
t.Error(err)
}

expect := `{"report_period_ms":60000,"harvest_limits":{"analytic_event_data":10000,"custom_event_data":10000,"log_event_data":10000,"error_event_data":100}}`
expect := fmt.Sprintf(`{"report_period_ms":60000,"harvest_limits":{"analytic_event_data":10000,"custom_event_data":%d,"log_event_data":%d,"error_event_data":100}}`, MaxCustomEvents, MaxLogEvents)
if string(js) != expect {
t.Errorf("DefaultEventHarvestConfig does not match expected valued:\nExpected:\t%s\nActual:\t\t%s", expect, string(js))
}
Expand Down
1 change: 0 additions & 1 deletion v3/internal/limits.go
View file
Expand Up @@ -13,7 +13,6 @@ const (
// MaxPayloadSizeInBytes specifies the maximum payload size in bytes that
// should be sent to any endpoint
MaxPayloadSizeInBytes = 1000 * 1000

// MaxCustomEvents is the maximum number of Transaction Events that can be captured
// per 60-second harvest cycle
MaxCustomEvents = 10 * 1000
Expand Down
2 changes: 1 addition & 1 deletion v3/newrelic/app_run.go
View file
Expand Up @@ -190,7 +190,7 @@ func (run *appRun) ptrTxnEvents() *uint { return run.Reply.EventData.Limits.T
func (run *appRun) ptrCustomEvents() *uint { return run.Reply.EventData.Limits.CustomEvents }
func (run *appRun) ptrLogEvents() *uint { return run.Reply.EventData.Limits.LogEvents }
func (run *appRun) ptrErrorEvents() *uint { return run.Reply.EventData.Limits.ErrorEvents }
func (run *appRun) ptrSpanEvents() *uint { return run.Reply.EventData.Limits.SpanEvents }
func (run *appRun) ptrSpanEvents() *uint { return run.Reply.SpanEventHarvestConfig.HarvestLimit }

func (run *appRun) MaxTxnEvents() int { return run.limit(run.Config.maxTxnEvents(), run.ptrTxnEvents) }
func (run *appRun) MaxCustomEvents() int {
Expand Down
28 changes: 15 additions & 13 deletions v3/newrelic/app_run_test.go
View file
Expand Up @@ -147,9 +147,12 @@ func TestEventHarvestFieldsAllPopulated(t *testing.T) {
"analytic_event_data": 1,
"custom_event_data": 2,
"log_event_data": 3,
"span_event_data": 4,
"error_event_data": 5
"error_event_data": 4
}
},
"span_event_harvest_config":{
"report_period_ms": 10000,
"harvest_limit": 5
}
}}`), internal.PreconnectReply{})
if nil != err {
Expand All @@ -160,8 +163,8 @@ func TestEventHarvestFieldsAllPopulated(t *testing.T) {
maxTxnEvents: 1,
maxCustomEvents: 2,
maxLogEvents: 3,
maxSpanEvents: 4,
maxErrorEvents: 5,
maxErrorEvents: 4,
maxSpanEvents: 5,
periods: map[harvestTypes]time.Duration{
harvestMetricsTraces: 60 * time.Second,
harvestTypesEvents: 5 * time.Second,
Expand All @@ -184,19 +187,19 @@ func TestZeroReportPeriod(t *testing.T) {
maxCustomEvents: internal.MaxCustomEvents,
maxLogEvents: internal.MaxLogEvents,
maxErrorEvents: internal.MaxErrorEvents,
maxSpanEvents: run.Config.DistributedTracer.ReservoirLimit,
maxSpanEvents: defaultMaxSpanEvents,
periods: map[harvestTypes]time.Duration{
harvestTypesAll: 60 * time.Second,
0: 60 * time.Second,
},
})
}

func TestEventHarvestFieldsOnlySpanEvents(t *testing.T) {
func TestConnectResponseOnlySpanEvents(t *testing.T) {
reply, err := internal.UnmarshalConnectReply([]byte(`{"return_value":{
"event_harvest_config": {
"report_period_ms": 5000,
"harvest_limits": { "span_event_data": 3 }
"span_event_harvest_config":{
"report_period_ms": 10000,
"harvest_limit": 3
}}}`), internal.PreconnectReply{})
if nil != err {
t.Fatal(err)
Expand All @@ -210,7 +213,7 @@ func TestEventHarvestFieldsOnlySpanEvents(t *testing.T) {
maxSpanEvents: 3,
periods: map[harvestTypes]time.Duration{
harvestTypesAll ^ harvestSpanEvents: 60 * time.Second,
harvestSpanEvents: 5 * time.Second,
2: 60 * time.Second,
},
})
}
Expand Down Expand Up @@ -283,7 +286,6 @@ func TestEventHarvestFieldsOnlyCustomEvents(t *testing.T) {
},
})
}

func TestConfigurableHarvestNegativeReportPeriod(t *testing.T) {
h, err := internal.UnmarshalConnectReply([]byte(`{"return_value":{
"event_harvest_config": {
Expand Down Expand Up @@ -379,8 +381,8 @@ type expectHarvestConfig struct {
periods map[harvestTypes]time.Duration
}

func errorExpectNotEqualActual(value string, expect, actual interface{}) error {
return fmt.Errorf("Expected %s value does not match actual; expected: %+v actual: %+v", value, expect, actual)
func errorExpectNotEqualActual(value string, actual, expect interface{}) error {
return fmt.Errorf("Expected %s value does not match actual; actual: %+v expect: %+v", value, actual, expect)
}
func assertHarvestConfig(t testing.TB, hc harvestConfig, expect expectHarvestConfig) {
if h, ok := t.(interface {
Expand Down
2 changes: 1 addition & 1 deletion v3/newrelic/config.go
View file
Expand Up @@ -790,7 +790,7 @@ func configConnectJSONInternal(c Config, pid int, util *utilization.Data, e envi
Util: util,
SecurityPolicies: securityPolicies,
Metadata: metadata,
EventData: internal.DefaultEventHarvestConfigWithDT(c.maxTxnEvents(), c.maxLogEvents(), c.maxCustomEvents(), c.DistributedTracer.ReservoirLimit, c.DistributedTracer.Enabled),
EventData: internal.DefaultEventHarvestConfigWithDT(c.TransactionEvents.MaxSamplesStored, c.ApplicationLogging.Forwarding.MaxSamplesStored, c.CustomInsightsEvents.MaxSamplesStored, c.DistributedTracer.ReservoirLimit, c.DistributedTracer.Enabled),
}})
}

Expand Down
4 changes: 4 additions & 0 deletions v3/newrelic/config_options.go
View file
Expand Up @@ -39,7 +39,11 @@ func ConfigDistributedTracerEnabled(enabled bool) ConfigOption {
// ConfigCustomInsightsEventsMaxSamplesStored alters the sample size allowing control
// of how many custom events are stored in an agent for a given harvest cycle.
// Alters the CustomInsightsEvents.MaxSamplesStored setting.
// Note: As of Jul 2022, the absolute maximum events that can be sent each minute is 100000.
func ConfigCustomInsightsEventsMaxSamplesStored(limit int) ConfigOption {
if limit > 100000 {
return func(cfg *Config) { cfg.CustomInsightsEvents.MaxSamplesStored = 100000 }
}
return func(cfg *Config) { cfg.CustomInsightsEvents.MaxSamplesStored = limit }
}

Expand Down
29 changes: 15 additions & 14 deletions v3/newrelic/config_test.go
View file
Expand Up @@ -5,6 +5,7 @@ package newrelic

import (
"encoding/json"
"fmt"
"net/http"
"os"
"reflect"
Expand Down Expand Up @@ -121,7 +122,7 @@ func TestCopyConfigReferenceFieldsPresent(t *testing.T) {
cfg.TransactionTracer.Segments.Attributes.Include[0] = "zap"
cfg.TransactionTracer.Segments.Attributes.Exclude[0] = "zap"

expect := internal.CompactJSONString(`[
expect := internal.CompactJSONString(fmt.Sprintf(`[
{
"pid":123,
"language":"go",
Expand All @@ -133,7 +134,7 @@ func TestCopyConfigReferenceFieldsPresent(t *testing.T) {
"Enabled":true,
"Forwarding": {
"Enabled": false,
"MaxSamplesStored": 10000
"MaxSamplesStored": %d
},
"LocalDecorating":{
"Enabled": false
Expand All @@ -151,7 +152,7 @@ func TestCopyConfigReferenceFieldsPresent(t *testing.T) {
"CrossApplicationTracer":{"Enabled":false},
"CustomInsightsEvents":{
"Enabled":true,
"MaxSamplesStored":10000
"MaxSamplesStored":%d
},
"DatastoreTracer":{
"DatabaseNameReporting":{"Enabled":true},
Expand Down Expand Up @@ -206,7 +207,7 @@ func TestCopyConfigReferenceFieldsPresent(t *testing.T) {
"TransactionEvents":{
"Attributes":{"Enabled":true,"Exclude":["4"],"Include":["3"]},
"Enabled":true,
"MaxSamplesStored": 10000
"MaxSamplesStored": %d
},
"TransactionTracer":{
"Attributes":{"Enabled":true,"Exclude":["8"],"Include":["7"]},
Expand Down Expand Up @@ -266,13 +267,13 @@ func TestCopyConfigReferenceFieldsPresent(t *testing.T) {
"report_period_ms": 60000,
"harvest_limits": {
"analytic_event_data": 10000,
"custom_event_data": 10000,
"log_event_data": 10000,
"custom_event_data": %d,
"log_event_data": %d,
"error_event_data": 100,
"span_event_data": 2000
}
}
}]`)
}]`, internal.MaxLogEvents, internal.MaxCustomEvents, internal.MaxTxnEvents, internal.MaxCustomEvents, internal.MaxTxnEvents))

securityPoliciesInput := []byte(`{
"record_sql": { "enabled": false, "required": false },
Expand Down Expand Up @@ -312,7 +313,7 @@ func TestCopyConfigReferenceFieldsAbsent(t *testing.T) {

cp := copyConfigReferenceFields(cfg)

expect := internal.CompactJSONString(`[
expect := internal.CompactJSONString(fmt.Sprintf(`[
{
"pid":123,
"language":"go",
Expand All @@ -324,7 +325,7 @@ func TestCopyConfigReferenceFieldsAbsent(t *testing.T) {
"Enabled": true,
"Forwarding": {
"Enabled": false,
"MaxSamplesStored": 10000
"MaxSamplesStored": %d
},
"LocalDecorating":{
"Enabled": false
Expand All @@ -346,7 +347,7 @@ func TestCopyConfigReferenceFieldsAbsent(t *testing.T) {
"CrossApplicationTracer":{"Enabled":false},
"CustomInsightsEvents":{
"Enabled":true,
"MaxSamplesStored":10000
"MaxSamplesStored":%d
},
"DatastoreTracer":{
"DatabaseNameReporting":{"Enabled":true},
Expand Down Expand Up @@ -399,7 +400,7 @@ func TestCopyConfigReferenceFieldsAbsent(t *testing.T) {
"TransactionEvents":{
"Attributes":{"Enabled":true,"Exclude":null,"Include":null},
"Enabled":true,
"MaxSamplesStored": 10000
"MaxSamplesStored": %d
},
"TransactionTracer":{
"Attributes":{"Enabled":true,"Exclude":null,"Include":null},
Expand Down Expand Up @@ -449,13 +450,13 @@ func TestCopyConfigReferenceFieldsAbsent(t *testing.T) {
"report_period_ms": 60000,
"harvest_limits": {
"analytic_event_data": 10000,
"custom_event_data": 10000,
"log_event_data": 10000,
"custom_event_data": %d,
"log_event_data": %d,
"error_event_data": 100,
"span_event_data": 2000
}
}
}]`)
}]`, internal.MaxLogEvents, internal.MaxCustomEvents, internal.MaxTxnEvents, internal.MaxCustomEvents, internal.MaxTxnEvents))

metadata := map[string]string{}
js, err := configConnectJSONInternal(cp, 123, &utilization.SampleData, sampleEnvironment, "0.2.2", nil, metadata)
Expand Down
2 changes: 1 addition & 1 deletion v3/newrelic/harvest_test.go
View file
Expand Up @@ -165,7 +165,7 @@ func TestCreateFinalMetrics(t *testing.T) {
{Name: "rename_me", Scope: "", Forced: false, Data: []float64{1.0, 0, 0, 0, 0, 0}},
{Name: "Supportability/EventHarvest/ReportPeriod", Scope: "", Forced: true, Data: []float64{1, 60, 60, 60, 60, 60 * 60}},
{Name: "Supportability/EventHarvest/AnalyticEventData/HarvestLimit", Scope: "", Forced: true, Data: []float64{1, 10 * 1000, 10 * 1000, 10 * 1000, 10 * 1000, 10 * 1000 * 10 * 1000}},
{Name: "Supportability/EventHarvest/CustomEventData/HarvestLimit", Scope: "", Forced: true, Data: []float64{1, 10 * 1000, 10 * 1000, 10 * 1000, 10 * 1000, 10 * 1000 * 10 * 1000}},
{Name: "Supportability/EventHarvest/CustomEventData/HarvestLimit", Scope: "", Forced: true, Data: []float64{1, internal.MaxCustomEvents, internal.MaxCustomEvents, internal.MaxCustomEvents, internal.MaxCustomEvents, internal.MaxCustomEvents * internal.MaxCustomEvents}},
{Name: "Supportability/EventHarvest/ErrorEventData/HarvestLimit", Scope: "", Forced: true, Data: []float64{1, 100, 100, 100, 100, 100 * 100}},
{Name: "Supportability/EventHarvest/SpanEventData/HarvestLimit", Scope: "", Forced: true, Data: []float64{1, 2000, 2000, 2000, 2000, 2000 * 2000}},
{Name: "Supportability/EventHarvest/LogEventData/HarvestLimit", Scope: "", Forced: true, Data: []float64{1, 10000, 10000, 10000, 10000, 10000 * 10000}},
Expand Down

0 comments on commit eaaf111

Please sign in to comment.