From dd617e76c79cfc6f92cb2e88ea8f2c487c231df2 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Fri, 20 Jan 2023 17:14:52 +0000 Subject: [PATCH 01/15] Add upstream connection metrics --- src/core/metrics/sources/nginx_access_log.go | 81 +++++++++++--- .../metrics/sources/nginx_access_log_test.go | 102 ++++++++++++------ src/core/metrics/sources/tailer/tailer.go | 24 ++--- .../metrics/sources/tailer/tailer_test.go | 29 ++--- 4 files changed, 152 insertions(+), 84 deletions(-) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index 6617354cc..72cdaae43 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -18,11 +18,11 @@ import ( "time" "github.com/nginx/agent/sdk/v2/proto" - log "github.com/sirupsen/logrus" - "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/core/metrics/sources/tailer" + + log "github.com/sirupsen/logrus" ) const ( @@ -78,7 +78,6 @@ func NewNginxAccessLog( func (c *NginxAccessLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { defer wg.Done() - c.collectLogStats(ctx, m) } @@ -155,13 +154,14 @@ func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.St func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string) { logPattern := convertLogFormat(logFormat) - log.Debugf("Collecting from: %s using format: %s", logFile, logFormat) - log.Debugf("Pattern used for tailing logs: %s", logPattern) + // TODO: Undo debugf->infof + log.Infof("Collecting from: %s using format: %s", logFile, logFormat) + log.Infof("Pattern used for tailing logs: %s", logPattern) counters := getDefaultCounters() - gzipRatios := []float64{} - requestLengths := []float64{} - requestTimes := []float64{} + genCounters := getDefaultGenCounters() + + gzipRatios, requestLengths, requestTimes, connectTimes := []float64{}, []float64{}, []float64{}, []float64{} mu := sync.Mutex{} t, err := tailer.NewPatternTailer(logFile, map[string]string{"DEFAULT": logPattern}) @@ -270,13 +270,13 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string case <-tick.C: c.baseDimensions.NginxType = c.nginxType c.baseDimensions.PublishedAPI = logFile - c.group = "http" mu.Lock() if len(requestLengths) > 0 { counters["request.length"] = getRequestLengthMetricValue(requestLengths) } + if len(gzipRatios) > 0 { counters["gzip.ratio"] = getGzipRatioMetricValue(gzipRatios) } @@ -285,14 +285,22 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string counters[key] = value } + for key, value := range c.getUpstreamConnectMetrics(connectTimes) { + genCounters[key] = value + } + + c.group = "http" simpleMetrics := c.convertSamplesToSimpleMetrics(counters) + + c.group = "" + simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(genCounters)...) + log.Tracef("Access log metrics collected: %v", simpleMetrics) // reset the counters counters = getDefaultCounters() - gzipRatios = []float64{} - requestLengths = []float64{} - requestTimes = []float64{} + genCounters = getDefaultGenCounters() + gzipRatios, requestLengths, requestTimes, connectTimes = []float64{}, []float64{}, []float64{}, []float64{} c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) @@ -373,16 +381,13 @@ func getRequestTimeMetrics(requestTimes []float64) map[string]float64 { if len(requestTimes) > 0 { // Calculate request time average - sort.Float64s(requestTimes) requestTimesSum := 0.0 for _, requestTime := range requestTimes { requestTimesSum += requestTime } - counters["request.time"] = requestTimesSum / float64(len(requestTimes)) // Calculate request time count - sort.Float64s(requestTimes) counters["request.time.count"] = float64(len(requestTimes)) // Calculate request time max @@ -405,6 +410,41 @@ func getRequestTimeMetrics(requestTimes []float64) map[string]float64 { return counters } +func (c *NginxAccessLog) getUpstreamConnectMetrics(connectTimes []float64) map[string]float64 { + counters := make(map[string]float64) + + if len(connectTimes) > 0 { + // Calculate upstream connect time average + connectTimesSum := 0.0 + for _, connectTime := range connectTimes { + connectTimesSum += connectTime + } + counters["upstream.connect.time"] = connectTimesSum / float64(len(connectTimes)) + + // Calculate upstream connect time count + counters["upstream.connect.time.count"] = float64(len(connectTimes)) + + // Calculate upstream connect time max + sort.Float64s(connectTimes) + counters["upstream.connect.time.max"] = connectTimes[len(connectTimes)-1] + + // Calculate upstream connect time median + mNumber := len(connectTimes) / 2 + if len(connectTimes)%2 != 0 { + counters["upstream.connect.time.median"] = connectTimes[mNumber] + } else { + counters["upstream.connect.time.median"] = (connectTimes[mNumber-1] + connectTimes[mNumber]) / 2 + } + + // Calculate upstream connect time 95 percentile + index := int(math.RoundToEven(float64(0.95)*float64(len(connectTimes)))) - 1 + counters["upstream.connect.time.pctl95"] = connectTimes[index] + } + + return counters +} + +// convertLogFormat converts log format into a log format including a pattern that can be parsed by the tailer func convertLogFormat(logFormat string) string { newLogFormat := strings.ReplaceAll(logFormat, "$remote_addr", "%{IPORHOST:remote_addr}") newLogFormat = strings.ReplaceAll(newLogFormat, "$remote_user", "%{USERNAME:remote_user}") @@ -421,6 +461,7 @@ func convertLogFormat(logFormat string) string { newLogFormat = strings.ReplaceAll(newLogFormat, "$request_time", "%{DATA:request_time}") newLogFormat = strings.ReplaceAll(newLogFormat, "\"$request\"", "\"%{DATA:request}\"") newLogFormat = strings.ReplaceAll(newLogFormat, "$request ", "%{DATA:request} ") + newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time} ") newLogFormat = strings.ReplaceAll(newLogFormat, "[", "\\[") newLogFormat = strings.ReplaceAll(newLogFormat, "]", "\\]") return newLogFormat @@ -472,3 +513,13 @@ func getDefaultCounters() map[string]float64 { "v2": 0, } } + +func getDefaultGenCounters() map[string]float64 { + return map[string]float64{ + "upstream.connect.time": 0, + "upstream.connect.time.count": 0, + "upstream.connect.time.max": 0, + "upstream.connect.time.median": 0, + "upstream.connect.time.pctl95": 0, + } +} diff --git a/src/core/metrics/sources/nginx_access_log_test.go b/src/core/metrics/sources/nginx_access_log_test.go index 4b42a3d44..a0d9a9399 100644 --- a/src/core/metrics/sources/nginx_access_log_test.go +++ b/src/core/metrics/sources/nginx_access_log_test.go @@ -9,7 +9,6 @@ package sources import ( "context" - "io/ioutil" "os" "sort" "testing" @@ -19,18 +18,18 @@ import ( "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" - testutils "github.com/nginx/agent/v2/test/utils" tutils "github.com/nginx/agent/v2/test/utils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAccessLogUpdate(t *testing.T) { - binary := testutils.NewMockNginxBinary() + binary := tutils.NewMockNginxBinary() binary.On("GetAccessLogs").Return(map[string]string{"/tmp/access.log": ""}).Once() binary.On("GetAccessLogs").Return(map[string]string{"/tmp/new_access.log": ""}).Once() - collectionDuration, _ := time.ParseDuration("300ms") - newCollectionDuration, _ := time.ParseDuration("500ms") + collectionDuration := time.Millisecond * 300 + newCollectionDuration := time.Millisecond * 500 nginxAccessLog := NewNginxAccessLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration) assert.Equal(t, "", nginxAccessLog.baseDimensions.InstanceTags) @@ -54,10 +53,10 @@ func TestAccessLogUpdate(t *testing.T) { } func TestAccessLogStop(t *testing.T) { - binary := testutils.NewMockNginxBinary() + binary := tutils.NewMockNginxBinary() binary.On("GetAccessLogs").Return(map[string]string{"/tmp/access.log": ""}).Once() - collectionDuration, _ := time.ParseDuration("300ms") + collectionDuration := time.Millisecond * 300 nginxAccessLog := NewNginxAccessLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration) _, ok := nginxAccessLog.logs["/tmp/access.log"] @@ -65,7 +64,7 @@ func TestAccessLogStop(t *testing.T) { nginxAccessLog.Stop() - assert.Equal(t, 0, len(nginxAccessLog.logs)) + assert.Len(t, nginxAccessLog.logs, 0) } func TestAccessLogStats(t *testing.T) { @@ -73,7 +72,6 @@ func TestAccessLogStats(t *testing.T) { name string logFormat string logLines []string - m chan *proto.StatsEntity expectedStats *proto.StatsEntity }{ { @@ -83,7 +81,6 @@ func TestAccessLogStats(t *testing.T) { "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\"\n", `127.0.0.1 - - [19/May/2022:09:30:39 +0000] "GET /user/register?ahref HTTP/1.1" 200 98 "-" "-" "-"`, }, - make(chan *proto.StatsEntity, 1), &proto.StatsEntity{ Simplemetrics: []*proto.SimpleMetric{ { @@ -218,29 +215,48 @@ func TestAccessLogStats(t *testing.T) { Name: "nginx.http.v2", Value: 0, }, + { + Name: "nginx.upstream.connect.time", + Value: 0, + }, + { + Name: "nginx.upstream.connect.time.count", + Value: 0, + }, + { + Name: "nginx.upstream.connect.time.max", + Value: 0, + }, + { + Name: "nginx.upstream.connect.time.median", + Value: 0, + }, + { + Name: "nginx.upstream.connect.time.pctl95", + Value: 0, + }, }, }, }, { "full_access_log_test", - `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$bytes_sent" "$request_length" "$request_time" "$gzip_ratio" "$server_protocol"`, + `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$bytes_sent" "$request_length" "$request_time" "$gzip_ratio" "$server_protocol" "$upstream_connect_time"`, []string{ - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"1\"\n", }, - make(chan *proto.StatsEntity, 1), &proto.StatsEntity{ Simplemetrics: []*proto.SimpleMetric{ { @@ -375,29 +391,47 @@ func TestAccessLogStats(t *testing.T) { Name: "nginx.http.v2", Value: 1, }, + { + Name: "nginx.upstream.connect.time", + Value: 0, + }, + { + Name: "nginx.upstream.connect.time.count", + Value: 0, + }, + { + Name: "nginx.upstream.connect.time.max", + Value: 0, + }, + { + Name: "nginx.upstream.connect.time.median", + Value: 0, + }, + { + Name: "nginx.upstream.connect.time.pctl95", + Value: 0, + }, }, }, }, } binary := core.NewNginxBinary(tutils.NewMockEnvironment(), &config.Config{}) - collectionDuration, _ := time.ParseDuration("300ms") - sleepDuration, _ := time.ParseDuration("100ms") + collectionDuration := time.Millisecond * 300 + sleepDuration := time.Millisecond * 100 for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - context := context.TODO() - accessLogFile, _ := ioutil.TempFile(os.TempDir(), "access.log") + accessLogFile, _ := os.CreateTemp(os.TempDir(), "access.log") nginxAccessLog := NewNginxAccessLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration) - go nginxAccessLog.logStats(context, accessLogFile.Name(), test.logFormat) + go nginxAccessLog.logStats(context.TODO(), accessLogFile.Name(), test.logFormat) time.Sleep(sleepDuration) + for _, logLine := range test.logLines { _, err := accessLogFile.WriteString(logLine) - if err != nil { - tt.Fatalf("Error writing data to access log") - } + require.NoError(t, err, "Error writing data to access log") } time.Sleep(collectionDuration) diff --git a/src/core/metrics/sources/tailer/tailer.go b/src/core/metrics/sources/tailer/tailer.go index 42a245b5d..0b93bcffd 100644 --- a/src/core/metrics/sources/tailer/tailer.go +++ b/src/core/metrics/sources/tailer/tailer.go @@ -13,9 +13,8 @@ import ( "github.com/mitchellh/mapstructure" "github.com/nxadm/tail" - "github.com/trivago/grok" - log "github.com/sirupsen/logrus" + "github.com/trivago/grok" ) var ( @@ -31,16 +30,17 @@ var ( // NginxAccessItem represents the decoded access log data type NginxAccessItem struct { - BodyBytesSent string `mapstructure:"body_bytes_sent"` - Status string `mapstructure:"status"` - RemoteAddress string `mapstructure:"remote_addr"` - HTTPUserAgent string `mapstructure:"http_user_agent"` - Request string `mapstructure:"request"` - BytesSent string `mapstructure:"bytes_sent"` - RequestLength string `mapstructure:"request_length"` - RequestTime string `mapstructure:"request_time"` - GzipRatio string `mapstructure:"gzip_ratio"` - ServerProtocol string `mapstructure:"server_protocol"` + BodyBytesSent string `mapstructure:"body_bytes_sent"` + Status string `mapstructure:"status"` + RemoteAddress string `mapstructure:"remote_addr"` + HTTPUserAgent string `mapstructure:"http_user_agent"` + Request string `mapstructure:"request"` + BytesSent string `mapstructure:"bytes_sent"` + RequestLength string `mapstructure:"request_length"` + RequestTime string `mapstructure:"request_time"` + GzipRatio string `mapstructure:"gzip_ratio"` + ServerProtocol string `mapstructure:"server_protocol"` + UpstreamConnectTime string `mapstructure:"upstream_connect_time"` } func NewNginxAccessItem(v map[string]string) (*NginxAccessItem, error) { diff --git a/src/core/metrics/sources/tailer/tailer_test.go b/src/core/metrics/sources/tailer/tailer_test.go index 70e24a3a7..6ddb5aecc 100644 --- a/src/core/metrics/sources/tailer/tailer_test.go +++ b/src/core/metrics/sources/tailer/tailer_test.go @@ -9,7 +9,6 @@ package tailer import ( "context" - "io/ioutil" "os" "testing" "time" @@ -62,16 +61,13 @@ func TestGrok(t *testing.T) { } func TestTailer(t *testing.T) { - errorLogFile, _ := ioutil.TempFile(os.TempDir(), "error.log") + errorLogFile, _ := os.CreateTemp(os.TempDir(), "error.log") logLine := `2015/07/15 05:56:30 [info] 28386#28386: *94160 client 10.196.158.41 closed keepalive connection` tailer, err := NewTailer(errorLogFile.Name()) require.Nil(t, err) - timeoutDuration, err := time.ParseDuration("300ms") - if err != nil { - t.Fatal("Error creating timeout duration") - } + timeoutDuration := time.Millisecond * 300 ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) defer cancel() @@ -79,11 +75,7 @@ func TestTailer(t *testing.T) { data := make(chan string, 100) go tailer.Tail(ctx, data) - sleepDuration, err := time.ParseDuration("100ms") - if err != nil { - t.Fatal("Error creating sleep duration") - } - time.Sleep(sleepDuration) + time.Sleep(time.Millisecond * 100) _, err = errorLogFile.WriteString(logLine) if err != nil { t.Fatalf("Error writing data to error log") @@ -109,29 +101,20 @@ T: } func TestPatternTailer(t *testing.T) { - accessLogFile, _ := ioutil.TempFile(os.TempDir(), "access.log") + accessLogFile, _ := os.CreateTemp(os.TempDir(), "access.log") logLine := "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\"\n" tailer, err := NewPatternTailer(accessLogFile.Name(), defaultPatterns) require.Nil(t, err) - timeoutDuration, err := time.ParseDuration("300ms") - if err != nil { - t.Fatal("Error creating timeout duration") - } - + timeoutDuration := time.Millisecond * 300 ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) defer cancel() data := make(chan map[string]string, 100) go tailer.Tail(ctx, data) - sleepDuration, err := time.ParseDuration("100ms") - if err != nil { - t.Fatal("Error creating sleep duration") - } - - time.Sleep(sleepDuration) + time.Sleep(time.Millisecond * 100) _, err = accessLogFile.WriteString(logLine) if err != nil { t.Fatalf("Error writing data to access log") From e408dff2f6f506c2dd3872f6eb95344fcb45d564 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Fri, 20 Jan 2023 17:15:17 +0000 Subject: [PATCH 02/15] Lint/Cleanup --- src/core/metrics/metrics_util.go | 3 +- src/core/metrics/sources/nginx_access_log.go | 2 +- .../metrics/sources/nginx_error_log_test.go | 28 ++++++++----------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/core/metrics/metrics_util.go b/src/core/metrics/metrics_util.go index 46f942f95..60a52ffe2 100644 --- a/src/core/metrics/metrics_util.go +++ b/src/core/metrics/metrics_util.go @@ -12,9 +12,10 @@ import ( "sync" "time" - "github.com/gogo/protobuf/types" "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core/config" + + "github.com/gogo/protobuf/types" ) type Collector interface { diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index 72cdaae43..a719cb174 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -461,7 +461,7 @@ func convertLogFormat(logFormat string) string { newLogFormat = strings.ReplaceAll(newLogFormat, "$request_time", "%{DATA:request_time}") newLogFormat = strings.ReplaceAll(newLogFormat, "\"$request\"", "\"%{DATA:request}\"") newLogFormat = strings.ReplaceAll(newLogFormat, "$request ", "%{DATA:request} ") - newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time} ") + newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time}") newLogFormat = strings.ReplaceAll(newLogFormat, "[", "\\[") newLogFormat = strings.ReplaceAll(newLogFormat, "]", "\\]") return newLogFormat diff --git a/src/core/metrics/sources/nginx_error_log_test.go b/src/core/metrics/sources/nginx_error_log_test.go index 3209e4f51..d7a57d2e4 100644 --- a/src/core/metrics/sources/nginx_error_log_test.go +++ b/src/core/metrics/sources/nginx_error_log_test.go @@ -19,18 +19,18 @@ import ( "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" - testutils "github.com/nginx/agent/v2/test/utils" tutils "github.com/nginx/agent/v2/test/utils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNginxErrorLogUpdate(t *testing.T) { - binary := testutils.NewMockNginxBinary() + binary := tutils.NewMockNginxBinary() binary.On("GetErrorLogs").Return(map[string]string{"/tmp/error.log": ""}).Once() binary.On("GetErrorLogs").Return(map[string]string{"/tmp/new_error.log": ""}).Once() - collectionDuration, _ := time.ParseDuration("300ms") - newCollectionDuration, _ := time.ParseDuration("500ms") + collectionDuration := time.Millisecond * 300 + newCollectionDuration := time.Millisecond * 500 nginxErrorLog := NewNginxErrorLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration) assert.Equal(t, "", nginxErrorLog.baseDimensions.InstanceTags) @@ -54,10 +54,10 @@ func TestNginxErrorLogUpdate(t *testing.T) { } func TestNginxErrorLogStop(t *testing.T) { - binary := testutils.NewMockNginxBinary() + binary := tutils.NewMockNginxBinary() binary.On("GetErrorLogs").Return(map[string]string{"/tmp/error.log": ""}).Once() - collectionDuration, _ := time.ParseDuration("300ms") + collectionDuration := time.Millisecond * 300 nginxErrorLog := NewNginxErrorLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration) _, ok := nginxErrorLog.logs["/tmp/error.log"] @@ -65,14 +65,13 @@ func TestNginxErrorLogStop(t *testing.T) { nginxErrorLog.Stop() - assert.Equal(t, 0, len(nginxErrorLog.logs)) + assert.Len(t, nginxErrorLog.logs, 0) } func TestErrorLogStats(t *testing.T) { tests := []struct { name string logLines []string - m chan *proto.StatsEntity expectedStats *proto.StatsEntity }{ { @@ -84,7 +83,6 @@ func TestErrorLogStats(t *testing.T) { `2022/05/24 13:18:37 [error] 21314#21314: *91 connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: , request: "GET /frontend1 HTTP/1.1", upstream: "http://127.0.0.1:9091/frontend1", host: "127.0.0.1:8081"`, `2022/05/24 13:18:37 [error] 21314#21314: client request body is buffered.`, }, - make(chan *proto.StatsEntity, 1), &proto.StatsEntity{ Simplemetrics: []*proto.SimpleMetric{ { @@ -109,23 +107,21 @@ func TestErrorLogStats(t *testing.T) { } binary := core.NewNginxBinary(tutils.NewMockEnvironment(), &config.Config{}) - collectionDuration, _ := time.ParseDuration("300ms") - sleepDuration, _ := time.ParseDuration("100ms") + collectionDuration := time.Millisecond * 300 + sleepDuration := time.Millisecond * 100 for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - context := context.TODO() errorLogFile, _ := ioutil.TempFile(os.TempDir(), "error.log") nginxErrorLog := NewNginxErrorLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration) - go nginxErrorLog.logStats(context, errorLogFile.Name()) + go nginxErrorLog.logStats(context.TODO(), errorLogFile.Name()) time.Sleep(sleepDuration) + for _, logLine := range test.logLines { _, err := errorLogFile.WriteString(logLine) - if err != nil { - tt.Fatalf("Error writing data to error log") - } + require.NoError(t, err, "Error writing data to error log") } time.Sleep(collectionDuration) From 3f74d51bdacfcc040611db7c7d35eb7f048fac81 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Fri, 20 Jan 2023 17:22:12 +0000 Subject: [PATCH 03/15] Add to calculation map --- src/core/metrics/metrics_util.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/metrics/metrics_util.go b/src/core/metrics/metrics_util.go index 60a52ffe2..d91485bf2 100644 --- a/src/core/metrics/metrics_util.go +++ b/src/core/metrics/metrics_util.go @@ -130,6 +130,11 @@ func GetCalculationMap() map[string]string { "nginx.http.v1_0": "sum", "nginx.http.v1_1": "sum", "nginx.http.v2": "sum", + "nginx.upstream.time": "avg", + "nginx.upstream.time.count": "sum", + "nginx.upstream.time.max": "avg", + "nginx.upstream.time.median": "avg", + "nginx.upstream.time.pctl95": "avg", "nginx.http.conn.handled": "sum", "nginx.http.conn.reading": "avg", "nginx.http.conn.writing": "avg", From f52610ac845898fc905f9252ca9b00537ec68bd4 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Fri, 20 Jan 2023 17:46:05 +0000 Subject: [PATCH 04/15] Revert func to method conversion. Lint --- src/core/metrics/sources/nginx_access_log.go | 4 +- .../metrics/sources/nginx_access_log_test.go | 69 ++++++++++--------- .../metrics/sources/nginx_error_log_test.go | 4 +- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index a719cb174..bc6b26b92 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -285,7 +285,7 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string counters[key] = value } - for key, value := range c.getUpstreamConnectMetrics(connectTimes) { + for key, value := range getUpstreamConnectMetrics(connectTimes) { genCounters[key] = value } @@ -410,7 +410,7 @@ func getRequestTimeMetrics(requestTimes []float64) map[string]float64 { return counters } -func (c *NginxAccessLog) getUpstreamConnectMetrics(connectTimes []float64) map[string]float64 { +func getUpstreamConnectMetrics(connectTimes []float64) map[string]float64 { counters := make(map[string]float64) if len(connectTimes) > 0 { diff --git a/src/core/metrics/sources/nginx_access_log_test.go b/src/core/metrics/sources/nginx_access_log_test.go index a0d9a9399..4c39c6723 100644 --- a/src/core/metrics/sources/nginx_access_log_test.go +++ b/src/core/metrics/sources/nginx_access_log_test.go @@ -19,6 +19,7 @@ import ( "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" tutils "github.com/nginx/agent/v2/test/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -242,20 +243,20 @@ func TestAccessLogStats(t *testing.T) { "full_access_log_test", `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$bytes_sent" "$request_length" "$request_time" "$gzip_ratio" "$server_protocol" "$upstream_connect_time"`, []string{ - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"1\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"1\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\"\n", }, &proto.StatsEntity{ Simplemetrics: []*proto.SimpleMetric{ @@ -391,26 +392,26 @@ func TestAccessLogStats(t *testing.T) { Name: "nginx.http.v2", Value: 1, }, - { - Name: "nginx.upstream.connect.time", - Value: 0, - }, - { - Name: "nginx.upstream.connect.time.count", - Value: 0, - }, - { - Name: "nginx.upstream.connect.time.max", - Value: 0, - }, - { - Name: "nginx.upstream.connect.time.median", - Value: 0, - }, - { - Name: "nginx.upstream.connect.time.pctl95", - Value: 0, - }, + // { + // Name: "nginx.upstream.connect.time", + // Value: 0, + // }, + // { + // Name: "nginx.upstream.connect.time.count", + // Value: 0, + // }, + // { + // Name: "nginx.upstream.connect.time.max", + // Value: 0, + // }, + // { + // Name: "nginx.upstream.connect.time.median", + // Value: 0, + // }, + // { + // Name: "nginx.upstream.connect.time.pctl95", + // Value: 0, + // }, }, }, }, diff --git a/src/core/metrics/sources/nginx_error_log_test.go b/src/core/metrics/sources/nginx_error_log_test.go index d7a57d2e4..8e5e4bdb9 100644 --- a/src/core/metrics/sources/nginx_error_log_test.go +++ b/src/core/metrics/sources/nginx_error_log_test.go @@ -9,7 +9,6 @@ package sources import ( "context" - "io/ioutil" "os" "sort" "testing" @@ -20,6 +19,7 @@ import ( "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" tutils "github.com/nginx/agent/v2/test/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -112,7 +112,7 @@ func TestErrorLogStats(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - errorLogFile, _ := ioutil.TempFile(os.TempDir(), "error.log") + errorLogFile, _ := os.CreateTemp(os.TempDir(), "error.log") nginxErrorLog := NewNginxErrorLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration) go nginxErrorLog.logStats(context.TODO(), errorLogFile.Name()) From 7a21538090d64cad78008fbe67d9cc569057b001 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Mon, 23 Jan 2023 14:56:50 +0000 Subject: [PATCH 05/15] Add test assertion --- src/core/metrics/sources/nginx_access_log.go | 3 +- .../metrics/sources/nginx_access_log_test.go | 40 +++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index bc6b26b92..b92606272 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -444,7 +444,7 @@ func getUpstreamConnectMetrics(connectTimes []float64) map[string]float64 { return counters } -// convertLogFormat converts log format into a log format including a pattern that can be parsed by the tailer +// convertLogFormat converts log format into a pattern that can be parsed by the tailer func convertLogFormat(logFormat string) string { newLogFormat := strings.ReplaceAll(logFormat, "$remote_addr", "%{IPORHOST:remote_addr}") newLogFormat = strings.ReplaceAll(newLogFormat, "$remote_user", "%{USERNAME:remote_user}") @@ -462,6 +462,7 @@ func convertLogFormat(logFormat string) string { newLogFormat = strings.ReplaceAll(newLogFormat, "\"$request\"", "\"%{DATA:request}\"") newLogFormat = strings.ReplaceAll(newLogFormat, "$request ", "%{DATA:request} ") newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time}") + newLogFormat = strings.ReplaceAll(newLogFormat, "\"$upstream_connect_time\"", "\"%{DATA:upstream_connect_time}\"") newLogFormat = strings.ReplaceAll(newLogFormat, "[", "\\[") newLogFormat = strings.ReplaceAll(newLogFormat, "]", "\\]") return newLogFormat diff --git a/src/core/metrics/sources/nginx_access_log_test.go b/src/core/metrics/sources/nginx_access_log_test.go index 4c39c6723..da9e71494 100644 --- a/src/core/metrics/sources/nginx_access_log_test.go +++ b/src/core/metrics/sources/nginx_access_log_test.go @@ -392,26 +392,26 @@ func TestAccessLogStats(t *testing.T) { Name: "nginx.http.v2", Value: 1, }, - // { - // Name: "nginx.upstream.connect.time", - // Value: 0, - // }, - // { - // Name: "nginx.upstream.connect.time.count", - // Value: 0, - // }, - // { - // Name: "nginx.upstream.connect.time.max", - // Value: 0, - // }, - // { - // Name: "nginx.upstream.connect.time.median", - // Value: 0, - // }, - // { - // Name: "nginx.upstream.connect.time.pctl95", - // Value: 0, - // }, + { + Name: "nginx.upstream.connect.time", + Value: 350, + }, + { + Name: "nginx.upstream.connect.time.count", + Value: 350, + }, + { + Name: "nginx.upstream.connect.time.max", + Value: 350, + }, + { + Name: "nginx.upstream.connect.time.median", + Value: 350, + }, + { + Name: "nginx.upstream.connect.time.pctl95", + Value: 350, + }, }, }, }, From e6d63219a66391800b1ee4912dfaeca732f3b106 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Mon, 23 Jan 2023 16:06:42 +0000 Subject: [PATCH 06/15] Fix reading of upstream conn metrics --- src/core/metrics/sources/nginx_access_log.go | 7 ++++++- .../metrics/sources/nginx_access_log_test.go | 20 +++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index b92606272..b7fde71e3 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -235,6 +235,12 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string } } + if v, err := strconv.ParseFloat(access.UpstreamConnectTime, 64); err == nil { + connectTimes = append(connectTimes, v) + } else { + log.Debugf("Error getting upstream_connect_time value from access logs, %v", err) + } + if access.ServerProtocol != "" { if strings.Count(access.ServerProtocol, "/") == 1 { httpProtocolVersion := strings.Split(access.ServerProtocol, "/")[1] @@ -462,7 +468,6 @@ func convertLogFormat(logFormat string) string { newLogFormat = strings.ReplaceAll(newLogFormat, "\"$request\"", "\"%{DATA:request}\"") newLogFormat = strings.ReplaceAll(newLogFormat, "$request ", "%{DATA:request} ") newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time}") - newLogFormat = strings.ReplaceAll(newLogFormat, "\"$upstream_connect_time\"", "\"%{DATA:upstream_connect_time}\"") newLogFormat = strings.ReplaceAll(newLogFormat, "[", "\\[") newLogFormat = strings.ReplaceAll(newLogFormat, "]", "\\]") return newLogFormat diff --git a/src/core/metrics/sources/nginx_access_log_test.go b/src/core/metrics/sources/nginx_access_log_test.go index da9e71494..02970af67 100644 --- a/src/core/metrics/sources/nginx_access_log_test.go +++ b/src/core/metrics/sources/nginx_access_log_test.go @@ -243,18 +243,18 @@ func TestAccessLogStats(t *testing.T) { "full_access_log_test", `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$bytes_sent" "$request_length" "$request_time" "$gzip_ratio" "$server_protocol" "$upstream_connect_time"`, []string{ - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"350\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\" \"350\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\"\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"100\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"2350\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\" \"350\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\" \"350\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"900\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\"\n", "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\"\n", }, @@ -266,7 +266,7 @@ func TestAccessLogStats(t *testing.T) { }, { Name: "nginx.http.request.time", - Value: 0.1857142857142857, + Value: 0.18571428571428572, }, { Name: "nginx.http.request.time.count", @@ -394,15 +394,15 @@ func TestAccessLogStats(t *testing.T) { }, { Name: "nginx.upstream.connect.time", - Value: 350, + Value: 514.2857142857143, }, { Name: "nginx.upstream.connect.time.count", - Value: 350, + Value: 14, }, { Name: "nginx.upstream.connect.time.max", - Value: 350, + Value: 2350, }, { Name: "nginx.upstream.connect.time.median", @@ -410,7 +410,7 @@ func TestAccessLogStats(t *testing.T) { }, { Name: "nginx.upstream.connect.time.pctl95", - Value: 350, + Value: 900, }, }, }, From 41c953373b49485ed35af8d4fa9eff8f99e6a98a Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Mon, 23 Jan 2023 16:06:54 +0000 Subject: [PATCH 07/15] Lint --- src/core/metrics/sources/cpu.go | 1 + src/core/metrics/sources/cpu_test.go | 1 + src/core/metrics/sources/nginx_error_log.go | 4 ++-- src/core/metrics/sources/nginx_oss.go | 4 ++-- src/core/metrics/sources/nginx_oss_test.go | 4 ++-- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/core/metrics/sources/cpu.go b/src/core/metrics/sources/cpu.go index 391913c53..def4a1404 100644 --- a/src/core/metrics/sources/cpu.go +++ b/src/core/metrics/sources/cpu.go @@ -15,6 +15,7 @@ import ( "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" cgroup "github.com/nginx/agent/v2/src/core/metrics/sources/cgroup" + "github.com/shirou/gopsutil/v3/cpu" log "github.com/sirupsen/logrus" ) diff --git a/src/core/metrics/sources/cpu_test.go b/src/core/metrics/sources/cpu_test.go index 5009e1144..2ab0330b9 100644 --- a/src/core/metrics/sources/cpu_test.go +++ b/src/core/metrics/sources/cpu_test.go @@ -18,6 +18,7 @@ import ( "github.com/nginx/agent/sdk/v2/proto" cgroup "github.com/nginx/agent/v2/src/core/metrics/sources/cgroup" tutils "github.com/nginx/agent/v2/test/utils" + "github.com/shirou/gopsutil/v3/cpu" "github.com/stretchr/testify/assert" ) diff --git a/src/core/metrics/sources/nginx_error_log.go b/src/core/metrics/sources/nginx_error_log.go index 6b1943956..b9889ed59 100644 --- a/src/core/metrics/sources/nginx_error_log.go +++ b/src/core/metrics/sources/nginx_error_log.go @@ -14,11 +14,11 @@ import ( "time" "github.com/nginx/agent/sdk/v2/proto" - log "github.com/sirupsen/logrus" - "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/core/metrics/sources/tailer" + + log "github.com/sirupsen/logrus" ) const ( diff --git a/src/core/metrics/sources/nginx_oss.go b/src/core/metrics/sources/nginx_oss.go index a82176f9f..1773f302c 100644 --- a/src/core/metrics/sources/nginx_oss.go +++ b/src/core/metrics/sources/nginx_oss.go @@ -13,10 +13,10 @@ import ( "sync" "github.com/nginx/agent/sdk/v2/proto" + "github.com/nginx/agent/v2/src/core/metrics" + "github.com/nginxinc/nginx-prometheus-exporter/client" log "github.com/sirupsen/logrus" - - "github.com/nginx/agent/v2/src/core/metrics" ) type NginxOSS struct { diff --git a/src/core/metrics/sources/nginx_oss_test.go b/src/core/metrics/sources/nginx_oss_test.go index db2db35d1..aa636273c 100644 --- a/src/core/metrics/sources/nginx_oss_test.go +++ b/src/core/metrics/sources/nginx_oss_test.go @@ -15,10 +15,10 @@ import ( "testing" "github.com/nginx/agent/sdk/v2/proto" - "github.com/stretchr/testify/assert" - "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" + + "github.com/stretchr/testify/assert" ) func TestNginxOSSUpdate(t *testing.T) { From a5f4ece31a1852e0c1cd4ef4d3dbefc3b5bb9ae6 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Mon, 23 Jan 2023 16:13:04 +0000 Subject: [PATCH 08/15] Support multiple conn times, failed TCP conns --- src/core/metrics/sources/nginx_access_log.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index b7fde71e3..ef29802f4 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -235,10 +235,15 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string } } - if v, err := strconv.ParseFloat(access.UpstreamConnectTime, 64); err == nil { - connectTimes = append(connectTimes, v) - } else { - log.Debugf("Error getting upstream_connect_time value from access logs, %v", err) + for _, cTime := range strings.Split(access.UpstreamConnectTime, ", ") { + // nginx uses '-' to represent TCP connection fails + cTime = strings.ReplaceAll(cTime, "-", "0") + + if v, err := strconv.ParseFloat(cTime, 64); err == nil { + connectTimes = append(connectTimes, v) + } else { + log.Debugf("Error getting upstream_connect_time value from access logs, %v", err) + } } if access.ServerProtocol != "" { From e398ccfa6431939583ab348af250ffd9164c4e99 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Mon, 23 Jan 2023 18:55:14 +0000 Subject: [PATCH 09/15] Refactor to use generic time metrics func --- src/core/metrics/sources/nginx_access_log.go | 136 ++++++++----------- 1 file changed, 56 insertions(+), 80 deletions(-) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index ef29802f4..1a8a208b0 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -154,12 +154,11 @@ func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.St func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string) { logPattern := convertLogFormat(logFormat) - // TODO: Undo debugf->infof - log.Infof("Collecting from: %s using format: %s", logFile, logFormat) - log.Infof("Pattern used for tailing logs: %s", logPattern) + log.Debugf("Collecting from: %s using format: %s", logFile, logFormat) + log.Debugf("Pattern used for tailing logs: %s", logPattern) - counters := getDefaultCounters() - genCounters := getDefaultGenCounters() + httpCounters := getDefaultHTTPCounters() + counters := getDefaultGenCounters() gzipRatios, requestLengths, requestTimes, connectTimes := []float64{}, []float64{}, []float64{}, []float64{} mu := sync.Mutex{} @@ -187,14 +186,14 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string mu.Lock() if v, err := strconv.Atoi(access.BodyBytesSent); err == nil { n := "request.body_bytes_sent" - counters[n] = float64(v) + counters[n] + httpCounters[n] = float64(v) + httpCounters[n] } else { log.Debugf("Error getting body_bytes_sent value from access logs, %v", err) } if v, err := strconv.Atoi(access.BytesSent); err == nil { n := "request.bytes_sent" - counters[n] = float64(v) + counters[n] + httpCounters[n] = float64(v) + httpCounters[n] } else { log.Debugf("Error getting bytes_sent value from access logs, %v", err) } @@ -223,20 +222,20 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string if isOtherMethod(n) { n = "method.others" } - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 if access.ServerProtocol == "" { if strings.Count(protocol, "/") == 1 { httpProtocolVersion := strings.Split(protocol, "/")[1] httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_") n = fmt.Sprintf("v%s", httpProtocolVersion) - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } } } for _, cTime := range strings.Split(access.UpstreamConnectTime, ", ") { - // nginx uses '-' to represent TCP connection fails + // nginx uses '-' to represent TCP connection failures cTime = strings.ReplaceAll(cTime, "-", "0") if v, err := strconv.ParseFloat(cTime, 64); err == nil { @@ -251,7 +250,7 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string httpProtocolVersion := strings.Split(access.ServerProtocol, "/")[1] httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_") n := fmt.Sprintf("v%s", httpProtocolVersion) - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } } @@ -259,18 +258,18 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string if c.nginxType == OSSNginxType { if v, err := strconv.Atoi(access.Status); err == nil { n := fmt.Sprintf("status.%dxx", v/100) - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 if v == 403 || v == 404 || v == 500 || v == 502 || v == 503 || v == 504 { n := fmt.Sprintf("status.%d", v) - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } if v == 499 { n := "status.discarded" - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } if v == 400 { n := "request.malformed" - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } } else { log.Debugf("Error getting status value from access logs, %v", err) @@ -285,32 +284,32 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string mu.Lock() if len(requestLengths) > 0 { - counters["request.length"] = getRequestLengthMetricValue(requestLengths) + httpCounters["request.length"] = getRequestLengthMetricValue(requestLengths) } if len(gzipRatios) > 0 { - counters["gzip.ratio"] = getGzipRatioMetricValue(gzipRatios) + httpCounters["gzip.ratio"] = getGzipRatioMetricValue(gzipRatios) } - for key, value := range getRequestTimeMetrics(requestTimes) { - counters[key] = value + for metricName := range httpCounters { + httpCounters[metricName] = getTimeMetrics(metricName, requestTimes) } - for key, value := range getUpstreamConnectMetrics(connectTimes) { - genCounters[key] = value + for metricName := range counters { + counters[metricName] = getTimeMetrics(metricName, connectTimes) } c.group = "http" - simpleMetrics := c.convertSamplesToSimpleMetrics(counters) + simpleMetrics := c.convertSamplesToSimpleMetrics(httpCounters) c.group = "" - simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(genCounters)...) + simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(counters)...) log.Tracef("Access log metrics collected: %v", simpleMetrics) // reset the counters - counters = getDefaultCounters() - genCounters = getDefaultGenCounters() + httpCounters = getDefaultHTTPCounters() + counters = getDefaultGenCounters() gzipRatios, requestLengths, requestTimes, connectTimes = []float64{}, []float64{}, []float64{}, []float64{} c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) @@ -387,72 +386,49 @@ func getGzipRatioMetricValue(gzipRatios []float64) float64 { return value } -func getRequestTimeMetrics(requestTimes []float64) map[string]float64 { - counters := make(map[string]float64) - - if len(requestTimes) > 0 { - // Calculate request time average - requestTimesSum := 0.0 - for _, requestTime := range requestTimes { - requestTimesSum += requestTime - } - counters["request.time"] = requestTimesSum / float64(len(requestTimes)) - - // Calculate request time count - counters["request.time.count"] = float64(len(requestTimes)) - - // Calculate request time max - sort.Float64s(requestTimes) - counters["request.time.max"] = requestTimes[len(requestTimes)-1] - - // Calculate request time median - mNumber := len(requestTimes) / 2 - if len(requestTimes)%2 != 0 { - counters["request.time.median"] = requestTimes[mNumber] - } else { - counters["request.time.median"] = (requestTimes[mNumber-1] + requestTimes[mNumber]) / 2 - } - - // Calculate request time 95 percentile - index := int(math.RoundToEven(float64(0.95)*float64(len(requestTimes)))) - 1 - counters["request.time.pctl95"] = requestTimes[index] +func getTimeMetrics(metricName string, times []float64) float64 { + if len(times) == 0 { + return 0 } - return counters -} - -func getUpstreamConnectMetrics(connectTimes []float64) map[string]float64 { - counters := make(map[string]float64) + metricType := metricName[strings.LastIndex(metricName, ".")+1:] - if len(connectTimes) > 0 { - // Calculate upstream connect time average - connectTimesSum := 0.0 - for _, connectTime := range connectTimes { - connectTimesSum += connectTime + switch metricType { + case "time": + // Calculate time average + sum := 0.0 + for _, t := range times { + sum += t } - counters["upstream.connect.time"] = connectTimesSum / float64(len(connectTimes)) + return sum / float64(len(times)) - // Calculate upstream connect time count - counters["upstream.connect.time.count"] = float64(len(connectTimes)) + case "count": + return float64(len(times)) - // Calculate upstream connect time max - sort.Float64s(connectTimes) - counters["upstream.connect.time.max"] = connectTimes[len(connectTimes)-1] + case "max": + sort.Float64s(times) + return times[len(times)-1] - // Calculate upstream connect time median - mNumber := len(connectTimes) / 2 - if len(connectTimes)%2 != 0 { - counters["upstream.connect.time.median"] = connectTimes[mNumber] + case "median": + sort.Float64s(times) + + mNumber := len(times) / 2 + if len(times)%2 != 0 { + return times[mNumber] } else { - counters["upstream.connect.time.median"] = (connectTimes[mNumber-1] + connectTimes[mNumber]) / 2 + return (times[mNumber-1] + times[mNumber]) / 2 } - // Calculate upstream connect time 95 percentile - index := int(math.RoundToEven(float64(0.95)*float64(len(connectTimes)))) - 1 - counters["upstream.connect.time.pctl95"] = connectTimes[index] + case "pctl95": + sort.Float64s(times) + + index := int(math.RoundToEven(float64(0.95)*float64(len(times)))) - 1 + return times[index] } - return counters + log.Debugf("Could not get time metrics for %s: invalid metric type", metricName) + + return 0 } // convertLogFormat converts log format into a pattern that can be parsed by the tailer @@ -487,7 +463,7 @@ func isOtherMethod(method string) bool { method != "method.options" } -func getDefaultCounters() map[string]float64 { +func getDefaultHTTPCounters() map[string]float64 { return map[string]float64{ "gzip.ratio": 0, "method.delete": 0, From 58f8afb340131226aba6b50c927e4f48d38e89ee Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Mon, 23 Jan 2023 19:05:27 +0000 Subject: [PATCH 10/15] Fix metrics calculation --- src/core/metrics/sources/nginx_access_log.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index 1a8a208b0..cc75e4903 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -152,13 +152,21 @@ func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.St c.buf = []*proto.StatsEntity{} } +var httpRequestMetrics = []string{ + "request.time", + "request.time.count", + "request.time.max", + "request.time.median", + "request.time.pctl95", +} + func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string) { logPattern := convertLogFormat(logFormat) log.Debugf("Collecting from: %s using format: %s", logFile, logFormat) log.Debugf("Pattern used for tailing logs: %s", logPattern) httpCounters := getDefaultHTTPCounters() - counters := getDefaultGenCounters() + counters := getDefaultCounters() gzipRatios, requestLengths, requestTimes, connectTimes := []float64{}, []float64{}, []float64{}, []float64{} mu := sync.Mutex{} @@ -291,7 +299,7 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string httpCounters["gzip.ratio"] = getGzipRatioMetricValue(gzipRatios) } - for metricName := range httpCounters { + for _, metricName := range httpRequestMetrics { httpCounters[metricName] = getTimeMetrics(metricName, requestTimes) } @@ -309,7 +317,7 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string // reset the counters httpCounters = getDefaultHTTPCounters() - counters = getDefaultGenCounters() + counters = getDefaultCounters() gzipRatios, requestLengths, requestTimes, connectTimes = []float64{}, []float64{}, []float64{}, []float64{} c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) @@ -395,7 +403,7 @@ func getTimeMetrics(metricName string, times []float64) float64 { switch metricType { case "time": - // Calculate time average + // Calculate times average sum := 0.0 for _, t := range times { sum += t @@ -501,7 +509,7 @@ func getDefaultHTTPCounters() map[string]float64 { } } -func getDefaultGenCounters() map[string]float64 { +func getDefaultCounters() map[string]float64 { return map[string]float64{ "upstream.connect.time": 0, "upstream.connect.time.count": 0, From 58d235115c6d221a98182b8c979c25078dbc895f Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Tue, 24 Jan 2023 16:13:10 +0000 Subject: [PATCH 11/15] Add upstream header metrics --- src/core/metrics/metrics_util.go | 5 ++ src/core/metrics/sources/nginx_access_log.go | 67 ++++++++++++------ .../metrics/sources/nginx_access_log_test.go | 70 +++++++++++++++---- src/core/metrics/sources/tailer/tailer.go | 1 + 4 files changed, 106 insertions(+), 37 deletions(-) diff --git a/src/core/metrics/metrics_util.go b/src/core/metrics/metrics_util.go index d91485bf2..003c2baee 100644 --- a/src/core/metrics/metrics_util.go +++ b/src/core/metrics/metrics_util.go @@ -135,6 +135,11 @@ func GetCalculationMap() map[string]string { "nginx.upstream.time.max": "avg", "nginx.upstream.time.median": "avg", "nginx.upstream.time.pctl95": "avg", + "nginx.upstream.header": "avg", + "nginx.upstream.header.count": "sum", + "nginx.upstream.header.max": "avg", + "nginx.upstream.header.median": "avg", + "nginx.upstream.header.pctl95": "avg", "nginx.http.conn.handled": "sum", "nginx.http.conn.reading": "avg", "nginx.http.conn.writing": "avg", diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index cc75e4903..e8be18a28 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -29,7 +29,7 @@ const ( spaceDelim = " " ) -// This metrics source is used to tail the NGINX access logs to retrieve http metrics. +// This metrics source is used to tail the NGINX access logs to retrieve metrics. type NginxAccessLog struct { baseDimensions *metrics.CommonDim @@ -49,7 +49,7 @@ func NewNginxAccessLog( binary core.NginxBinary, nginxType string, collectionInterval time.Duration) *NginxAccessLog { - log.Trace("Creating NewNginxAccessLog") + log.Trace("Creating NginxAccessLog") nginxAccessLog := &NginxAccessLog{ baseDimensions, @@ -165,10 +165,9 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string log.Debugf("Collecting from: %s using format: %s", logFile, logFormat) log.Debugf("Pattern used for tailing logs: %s", logPattern) - httpCounters := getDefaultHTTPCounters() - counters := getDefaultCounters() + httpCounters, connCounters, headerCounters := getDefaultCounters() + gzipRatios, requestLengths, requestTimes, connectTimes, headerTimes := []float64{}, []float64{}, []float64{}, []float64{}, []float64{} - gzipRatios, requestLengths, requestTimes, connectTimes := []float64{}, []float64{}, []float64{}, []float64{} mu := sync.Mutex{} t, err := tailer.NewPatternTailer(logFile, map[string]string{"DEFAULT": logPattern}) @@ -196,32 +195,32 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string n := "request.body_bytes_sent" httpCounters[n] = float64(v) + httpCounters[n] } else { - log.Debugf("Error getting body_bytes_sent value from access logs, %v", err) + log.Debugf("Error getting body_bytes_sent value from access logs: %v", err) } if v, err := strconv.Atoi(access.BytesSent); err == nil { n := "request.bytes_sent" httpCounters[n] = float64(v) + httpCounters[n] } else { - log.Debugf("Error getting bytes_sent value from access logs, %v", err) + log.Debugf("Error getting bytes_sent value from access logs: %v", err) } if v, err := strconv.Atoi(access.GzipRatio); err == nil { gzipRatios = append(gzipRatios, float64(v)) } else { - log.Debugf("Error getting gzip_ratio value from access logs, %v", err) + log.Debugf("Error getting gzip_ratio value from access logs: %v", err) } if v, err := strconv.Atoi(access.RequestLength); err == nil { requestLengths = append(requestLengths, float64(v)) } else { - log.Debugf("Error getting request_length value from access logs, %v", err) + log.Debugf("Error getting request_length value from access logs: %v", err) } if v, err := strconv.ParseFloat(access.RequestTime, 64); err == nil { requestTimes = append(requestTimes, v) } else { - log.Debugf("Error getting request_time value from access logs, %v", err) + log.Debugf("Error getting request_time value from access logs: %v", err) } if access.Request != "" { @@ -253,6 +252,17 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string } } + for _, hTime := range strings.Split(access.UpstreamHeaderTime, ", ") { + // nginx uses '-' to represent TCP connection failures + hTime = strings.ReplaceAll(hTime, "-", "0") + + if v, err := strconv.ParseFloat(hTime, 64); err == nil { + headerTimes = append(headerTimes, v) + } else { + log.Debugf("Error getting upstream_header_time value from access logs: %v", err) + } + } + if access.ServerProtocol != "" { if strings.Count(access.ServerProtocol, "/") == 1 { httpProtocolVersion := strings.Split(access.ServerProtocol, "/")[1] @@ -303,22 +313,26 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string httpCounters[metricName] = getTimeMetrics(metricName, requestTimes) } - for metricName := range counters { - counters[metricName] = getTimeMetrics(metricName, connectTimes) + for metricName := range connCounters { + connCounters[metricName] = getTimeMetrics(metricName, connectTimes) + } + + for metricName := range headerCounters { + headerCounters[metricName] = getTimeMetrics(metricName, headerTimes) } c.group = "http" simpleMetrics := c.convertSamplesToSimpleMetrics(httpCounters) c.group = "" - simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(counters)...) + simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(connCounters)...) + simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(headerCounters)...) log.Tracef("Access log metrics collected: %v", simpleMetrics) // reset the counters - httpCounters = getDefaultHTTPCounters() - counters = getDefaultCounters() - gzipRatios, requestLengths, requestTimes, connectTimes = []float64{}, []float64{}, []float64{}, []float64{} + httpCounters, connCounters, headerCounters = getDefaultCounters() + gzipRatios, requestLengths, requestTimes, connectTimes, headerTimes = []float64{}, []float64{}, []float64{}, []float64{}, []float64{} c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) @@ -403,7 +417,7 @@ func getTimeMetrics(metricName string, times []float64) float64 { switch metricType { case "time": - // Calculate times average + // Calculate average sum := 0.0 for _, t := range times { sum += t @@ -457,6 +471,7 @@ func convertLogFormat(logFormat string) string { newLogFormat = strings.ReplaceAll(newLogFormat, "\"$request\"", "\"%{DATA:request}\"") newLogFormat = strings.ReplaceAll(newLogFormat, "$request ", "%{DATA:request} ") newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time}") + newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_header_time", "%{DATA:upstream_header_time}") newLogFormat = strings.ReplaceAll(newLogFormat, "[", "\\[") newLogFormat = strings.ReplaceAll(newLogFormat, "]", "\\]") return newLogFormat @@ -471,8 +486,8 @@ func isOtherMethod(method string) bool { method != "method.options" } -func getDefaultHTTPCounters() map[string]float64 { - return map[string]float64{ +func getDefaultCounters() (map[string]float64, map[string]float64, map[string]float64) { + httpCounters := map[string]float64{ "gzip.ratio": 0, "method.delete": 0, "method.get": 0, @@ -507,14 +522,22 @@ func getDefaultHTTPCounters() map[string]float64 { "v1_1": 0, "v2": 0, } -} -func getDefaultCounters() map[string]float64 { - return map[string]float64{ + upstreamConnnectCounters := map[string]float64{ "upstream.connect.time": 0, "upstream.connect.time.count": 0, "upstream.connect.time.max": 0, "upstream.connect.time.median": 0, "upstream.connect.time.pctl95": 0, } + + upstreamHeaderCounters := map[string]float64{ + "upstream.header.time": 0, + "upstream.header.time.count": 0, + "upstream.header.time.max": 0, + "upstream.header.time.median": 0, + "upstream.header.time.pctl95": 0, + } + + return httpCounters, upstreamConnnectCounters, upstreamHeaderCounters } diff --git a/src/core/metrics/sources/nginx_access_log_test.go b/src/core/metrics/sources/nginx_access_log_test.go index 02970af67..dab5e819f 100644 --- a/src/core/metrics/sources/nginx_access_log_test.go +++ b/src/core/metrics/sources/nginx_access_log_test.go @@ -236,27 +236,47 @@ func TestAccessLogStats(t *testing.T) { Name: "nginx.upstream.connect.time.pctl95", Value: 0, }, + { + Name: "nginx.upstream.header.time", + Value: 0, + }, + { + Name: "nginx.upstream.header.time.count", + Value: 0, + }, + { + Name: "nginx.upstream.header.time.max", + Value: 0, + }, + { + Name: "nginx.upstream.header.time.median", + Value: 0, + }, + { + Name: "nginx.upstream.header.time.pctl95", + Value: 0, + }, }, }, }, { "full_access_log_test", - `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$bytes_sent" "$request_length" "$request_time" "$gzip_ratio" "$server_protocol" "$upstream_connect_time"`, + `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$bytes_sent" "$request_length" "$request_time" "$gzip_ratio" "$server_protocol" "$upstream_connect_time" "$upstream_header_time"`, []string{ - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"100\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"2350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"900\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\"\n", - "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"350\" \"500\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\" \"350\" \"730\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"500\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"500\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"100\" \"500\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"505\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"2000\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"2350\" \"250\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\" \"350\" \"500\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\" \"350\" \"500\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\" \"350\" \"590\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"900\" \"500\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\" \"170\"\n", + "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\" \"500\"\n", }, &proto.StatsEntity{ Simplemetrics: []*proto.SimpleMetric{ @@ -412,6 +432,26 @@ func TestAccessLogStats(t *testing.T) { Name: "nginx.upstream.connect.time.pctl95", Value: 900, }, + { + Name: "nginx.upstream.header.time", + Value: 588.9285714285714, + }, + { + Name: "nginx.upstream.header.time.count", + Value: 14, + }, + { + Name: "nginx.upstream.header.time.max", + Value: 2000, + }, + { + Name: "nginx.upstream.header.time.median", + Value: 500, + }, + { + Name: "nginx.upstream.header.time.pctl95", + Value: 730, + }, }, }, }, diff --git a/src/core/metrics/sources/tailer/tailer.go b/src/core/metrics/sources/tailer/tailer.go index 0b93bcffd..dfc6aeb3a 100644 --- a/src/core/metrics/sources/tailer/tailer.go +++ b/src/core/metrics/sources/tailer/tailer.go @@ -41,6 +41,7 @@ type NginxAccessItem struct { GzipRatio string `mapstructure:"gzip_ratio"` ServerProtocol string `mapstructure:"server_protocol"` UpstreamConnectTime string `mapstructure:"upstream_connect_time"` + UpstreamHeaderTime string `mapstructure:"upstream_header_time"` } func NewNginxAccessItem(v map[string]string) (*NginxAccessItem, error) { From acf900a21de9b0fca018a244c3f0ab91b6d6abb1 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Wed, 25 Jan 2023 14:09:49 +0000 Subject: [PATCH 12/15] Make deps --- .../agent/v2/src/core/metrics/metrics_util.go | 13 +- .../agent/v2/src/core/metrics/sources/cpu.go | 1 + .../core/metrics/sources/nginx_access_log.go | 187 ++++++++++++------ .../core/metrics/sources/nginx_error_log.go | 4 +- .../v2/src/core/metrics/sources/nginx_oss.go | 4 +- .../src/core/metrics/sources/tailer/tailer.go | 25 +-- 6 files changed, 158 insertions(+), 76 deletions(-) diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go index 46f942f95..003c2baee 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go @@ -12,9 +12,10 @@ import ( "sync" "time" - "github.com/gogo/protobuf/types" "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core/config" + + "github.com/gogo/protobuf/types" ) type Collector interface { @@ -129,6 +130,16 @@ func GetCalculationMap() map[string]string { "nginx.http.v1_0": "sum", "nginx.http.v1_1": "sum", "nginx.http.v2": "sum", + "nginx.upstream.time": "avg", + "nginx.upstream.time.count": "sum", + "nginx.upstream.time.max": "avg", + "nginx.upstream.time.median": "avg", + "nginx.upstream.time.pctl95": "avg", + "nginx.upstream.header": "avg", + "nginx.upstream.header.count": "sum", + "nginx.upstream.header.max": "avg", + "nginx.upstream.header.median": "avg", + "nginx.upstream.header.pctl95": "avg", "nginx.http.conn.handled": "sum", "nginx.http.conn.reading": "avg", "nginx.http.conn.writing": "avg", diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/cpu.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/cpu.go index 391913c53..def4a1404 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/cpu.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/cpu.go @@ -15,6 +15,7 @@ import ( "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" cgroup "github.com/nginx/agent/v2/src/core/metrics/sources/cgroup" + "github.com/shirou/gopsutil/v3/cpu" log "github.com/sirupsen/logrus" ) diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go index 6617354cc..e8be18a28 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go @@ -18,18 +18,18 @@ import ( "time" "github.com/nginx/agent/sdk/v2/proto" - log "github.com/sirupsen/logrus" - "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/core/metrics/sources/tailer" + + log "github.com/sirupsen/logrus" ) const ( spaceDelim = " " ) -// This metrics source is used to tail the NGINX access logs to retrieve http metrics. +// This metrics source is used to tail the NGINX access logs to retrieve metrics. type NginxAccessLog struct { baseDimensions *metrics.CommonDim @@ -49,7 +49,7 @@ func NewNginxAccessLog( binary core.NginxBinary, nginxType string, collectionInterval time.Duration) *NginxAccessLog { - log.Trace("Creating NewNginxAccessLog") + log.Trace("Creating NginxAccessLog") nginxAccessLog := &NginxAccessLog{ baseDimensions, @@ -78,7 +78,6 @@ func NewNginxAccessLog( func (c *NginxAccessLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { defer wg.Done() - c.collectLogStats(ctx, m) } @@ -153,15 +152,22 @@ func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.St c.buf = []*proto.StatsEntity{} } +var httpRequestMetrics = []string{ + "request.time", + "request.time.count", + "request.time.max", + "request.time.median", + "request.time.pctl95", +} + func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string) { logPattern := convertLogFormat(logFormat) log.Debugf("Collecting from: %s using format: %s", logFile, logFormat) log.Debugf("Pattern used for tailing logs: %s", logPattern) - counters := getDefaultCounters() - gzipRatios := []float64{} - requestLengths := []float64{} - requestTimes := []float64{} + httpCounters, connCounters, headerCounters := getDefaultCounters() + gzipRatios, requestLengths, requestTimes, connectTimes, headerTimes := []float64{}, []float64{}, []float64{}, []float64{}, []float64{} + mu := sync.Mutex{} t, err := tailer.NewPatternTailer(logFile, map[string]string{"DEFAULT": logPattern}) @@ -187,34 +193,34 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string mu.Lock() if v, err := strconv.Atoi(access.BodyBytesSent); err == nil { n := "request.body_bytes_sent" - counters[n] = float64(v) + counters[n] + httpCounters[n] = float64(v) + httpCounters[n] } else { - log.Debugf("Error getting body_bytes_sent value from access logs, %v", err) + log.Debugf("Error getting body_bytes_sent value from access logs: %v", err) } if v, err := strconv.Atoi(access.BytesSent); err == nil { n := "request.bytes_sent" - counters[n] = float64(v) + counters[n] + httpCounters[n] = float64(v) + httpCounters[n] } else { - log.Debugf("Error getting bytes_sent value from access logs, %v", err) + log.Debugf("Error getting bytes_sent value from access logs: %v", err) } if v, err := strconv.Atoi(access.GzipRatio); err == nil { gzipRatios = append(gzipRatios, float64(v)) } else { - log.Debugf("Error getting gzip_ratio value from access logs, %v", err) + log.Debugf("Error getting gzip_ratio value from access logs: %v", err) } if v, err := strconv.Atoi(access.RequestLength); err == nil { requestLengths = append(requestLengths, float64(v)) } else { - log.Debugf("Error getting request_length value from access logs, %v", err) + log.Debugf("Error getting request_length value from access logs: %v", err) } if v, err := strconv.ParseFloat(access.RequestTime, 64); err == nil { requestTimes = append(requestTimes, v) } else { - log.Debugf("Error getting request_time value from access logs, %v", err) + log.Debugf("Error getting request_time value from access logs: %v", err) } if access.Request != "" { @@ -223,24 +229,46 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string if isOtherMethod(n) { n = "method.others" } - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 if access.ServerProtocol == "" { if strings.Count(protocol, "/") == 1 { httpProtocolVersion := strings.Split(protocol, "/")[1] httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_") n = fmt.Sprintf("v%s", httpProtocolVersion) - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } } } + for _, cTime := range strings.Split(access.UpstreamConnectTime, ", ") { + // nginx uses '-' to represent TCP connection failures + cTime = strings.ReplaceAll(cTime, "-", "0") + + if v, err := strconv.ParseFloat(cTime, 64); err == nil { + connectTimes = append(connectTimes, v) + } else { + log.Debugf("Error getting upstream_connect_time value from access logs, %v", err) + } + } + + for _, hTime := range strings.Split(access.UpstreamHeaderTime, ", ") { + // nginx uses '-' to represent TCP connection failures + hTime = strings.ReplaceAll(hTime, "-", "0") + + if v, err := strconv.ParseFloat(hTime, 64); err == nil { + headerTimes = append(headerTimes, v) + } else { + log.Debugf("Error getting upstream_header_time value from access logs: %v", err) + } + } + if access.ServerProtocol != "" { if strings.Count(access.ServerProtocol, "/") == 1 { httpProtocolVersion := strings.Split(access.ServerProtocol, "/")[1] httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_") n := fmt.Sprintf("v%s", httpProtocolVersion) - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } } @@ -248,18 +276,18 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string if c.nginxType == OSSNginxType { if v, err := strconv.Atoi(access.Status); err == nil { n := fmt.Sprintf("status.%dxx", v/100) - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 if v == 403 || v == 404 || v == 500 || v == 502 || v == 503 || v == 504 { n := fmt.Sprintf("status.%d", v) - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } if v == 499 { n := "status.discarded" - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } if v == 400 { n := "request.malformed" - counters[n] = counters[n] + 1 + httpCounters[n] = httpCounters[n] + 1 } } else { log.Debugf("Error getting status value from access logs, %v", err) @@ -270,29 +298,41 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string case <-tick.C: c.baseDimensions.NginxType = c.nginxType c.baseDimensions.PublishedAPI = logFile - c.group = "http" mu.Lock() if len(requestLengths) > 0 { - counters["request.length"] = getRequestLengthMetricValue(requestLengths) + httpCounters["request.length"] = getRequestLengthMetricValue(requestLengths) } + if len(gzipRatios) > 0 { - counters["gzip.ratio"] = getGzipRatioMetricValue(gzipRatios) + httpCounters["gzip.ratio"] = getGzipRatioMetricValue(gzipRatios) } - for key, value := range getRequestTimeMetrics(requestTimes) { - counters[key] = value + for _, metricName := range httpRequestMetrics { + httpCounters[metricName] = getTimeMetrics(metricName, requestTimes) } - simpleMetrics := c.convertSamplesToSimpleMetrics(counters) + for metricName := range connCounters { + connCounters[metricName] = getTimeMetrics(metricName, connectTimes) + } + + for metricName := range headerCounters { + headerCounters[metricName] = getTimeMetrics(metricName, headerTimes) + } + + c.group = "http" + simpleMetrics := c.convertSamplesToSimpleMetrics(httpCounters) + + c.group = "" + simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(connCounters)...) + simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(headerCounters)...) + log.Tracef("Access log metrics collected: %v", simpleMetrics) // reset the counters - counters = getDefaultCounters() - gzipRatios = []float64{} - requestLengths = []float64{} - requestTimes = []float64{} + httpCounters, connCounters, headerCounters = getDefaultCounters() + gzipRatios, requestLengths, requestTimes, connectTimes, headerTimes = []float64{}, []float64{}, []float64{}, []float64{}, []float64{} c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) @@ -368,43 +408,52 @@ func getGzipRatioMetricValue(gzipRatios []float64) float64 { return value } -func getRequestTimeMetrics(requestTimes []float64) map[string]float64 { - counters := make(map[string]float64) +func getTimeMetrics(metricName string, times []float64) float64 { + if len(times) == 0 { + return 0 + } + + metricType := metricName[strings.LastIndex(metricName, ".")+1:] - if len(requestTimes) > 0 { - // Calculate request time average - sort.Float64s(requestTimes) - requestTimesSum := 0.0 - for _, requestTime := range requestTimes { - requestTimesSum += requestTime + switch metricType { + case "time": + // Calculate average + sum := 0.0 + for _, t := range times { + sum += t } + return sum / float64(len(times)) - counters["request.time"] = requestTimesSum / float64(len(requestTimes)) + case "count": + return float64(len(times)) - // Calculate request time count - sort.Float64s(requestTimes) - counters["request.time.count"] = float64(len(requestTimes)) + case "max": + sort.Float64s(times) + return times[len(times)-1] - // Calculate request time max - sort.Float64s(requestTimes) - counters["request.time.max"] = requestTimes[len(requestTimes)-1] + case "median": + sort.Float64s(times) - // Calculate request time median - mNumber := len(requestTimes) / 2 - if len(requestTimes)%2 != 0 { - counters["request.time.median"] = requestTimes[mNumber] + mNumber := len(times) / 2 + if len(times)%2 != 0 { + return times[mNumber] } else { - counters["request.time.median"] = (requestTimes[mNumber-1] + requestTimes[mNumber]) / 2 + return (times[mNumber-1] + times[mNumber]) / 2 } - // Calculate request time 95 percentile - index := int(math.RoundToEven(float64(0.95)*float64(len(requestTimes)))) - 1 - counters["request.time.pctl95"] = requestTimes[index] + case "pctl95": + sort.Float64s(times) + + index := int(math.RoundToEven(float64(0.95)*float64(len(times)))) - 1 + return times[index] } - return counters + log.Debugf("Could not get time metrics for %s: invalid metric type", metricName) + + return 0 } +// convertLogFormat converts log format into a pattern that can be parsed by the tailer func convertLogFormat(logFormat string) string { newLogFormat := strings.ReplaceAll(logFormat, "$remote_addr", "%{IPORHOST:remote_addr}") newLogFormat = strings.ReplaceAll(newLogFormat, "$remote_user", "%{USERNAME:remote_user}") @@ -421,6 +470,8 @@ func convertLogFormat(logFormat string) string { newLogFormat = strings.ReplaceAll(newLogFormat, "$request_time", "%{DATA:request_time}") newLogFormat = strings.ReplaceAll(newLogFormat, "\"$request\"", "\"%{DATA:request}\"") newLogFormat = strings.ReplaceAll(newLogFormat, "$request ", "%{DATA:request} ") + newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time}") + newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_header_time", "%{DATA:upstream_header_time}") newLogFormat = strings.ReplaceAll(newLogFormat, "[", "\\[") newLogFormat = strings.ReplaceAll(newLogFormat, "]", "\\]") return newLogFormat @@ -435,8 +486,8 @@ func isOtherMethod(method string) bool { method != "method.options" } -func getDefaultCounters() map[string]float64 { - return map[string]float64{ +func getDefaultCounters() (map[string]float64, map[string]float64, map[string]float64) { + httpCounters := map[string]float64{ "gzip.ratio": 0, "method.delete": 0, "method.get": 0, @@ -471,4 +522,22 @@ func getDefaultCounters() map[string]float64 { "v1_1": 0, "v2": 0, } + + upstreamConnnectCounters := map[string]float64{ + "upstream.connect.time": 0, + "upstream.connect.time.count": 0, + "upstream.connect.time.max": 0, + "upstream.connect.time.median": 0, + "upstream.connect.time.pctl95": 0, + } + + upstreamHeaderCounters := map[string]float64{ + "upstream.header.time": 0, + "upstream.header.time.count": 0, + "upstream.header.time.max": 0, + "upstream.header.time.median": 0, + "upstream.header.time.pctl95": 0, + } + + return httpCounters, upstreamConnnectCounters, upstreamHeaderCounters } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_error_log.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_error_log.go index 6b1943956..b9889ed59 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_error_log.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_error_log.go @@ -14,11 +14,11 @@ import ( "time" "github.com/nginx/agent/sdk/v2/proto" - log "github.com/sirupsen/logrus" - "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/core/metrics/sources/tailer" + + log "github.com/sirupsen/logrus" ) const ( diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_oss.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_oss.go index a82176f9f..1773f302c 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_oss.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_oss.go @@ -13,10 +13,10 @@ import ( "sync" "github.com/nginx/agent/sdk/v2/proto" + "github.com/nginx/agent/v2/src/core/metrics" + "github.com/nginxinc/nginx-prometheus-exporter/client" log "github.com/sirupsen/logrus" - - "github.com/nginx/agent/v2/src/core/metrics" ) type NginxOSS struct { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/tailer/tailer.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/tailer/tailer.go index 42a245b5d..dfc6aeb3a 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/tailer/tailer.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/tailer/tailer.go @@ -13,9 +13,8 @@ import ( "github.com/mitchellh/mapstructure" "github.com/nxadm/tail" - "github.com/trivago/grok" - log "github.com/sirupsen/logrus" + "github.com/trivago/grok" ) var ( @@ -31,16 +30,18 @@ var ( // NginxAccessItem represents the decoded access log data type NginxAccessItem struct { - BodyBytesSent string `mapstructure:"body_bytes_sent"` - Status string `mapstructure:"status"` - RemoteAddress string `mapstructure:"remote_addr"` - HTTPUserAgent string `mapstructure:"http_user_agent"` - Request string `mapstructure:"request"` - BytesSent string `mapstructure:"bytes_sent"` - RequestLength string `mapstructure:"request_length"` - RequestTime string `mapstructure:"request_time"` - GzipRatio string `mapstructure:"gzip_ratio"` - ServerProtocol string `mapstructure:"server_protocol"` + BodyBytesSent string `mapstructure:"body_bytes_sent"` + Status string `mapstructure:"status"` + RemoteAddress string `mapstructure:"remote_addr"` + HTTPUserAgent string `mapstructure:"http_user_agent"` + Request string `mapstructure:"request"` + BytesSent string `mapstructure:"bytes_sent"` + RequestLength string `mapstructure:"request_length"` + RequestTime string `mapstructure:"request_time"` + GzipRatio string `mapstructure:"gzip_ratio"` + ServerProtocol string `mapstructure:"server_protocol"` + UpstreamConnectTime string `mapstructure:"upstream_connect_time"` + UpstreamHeaderTime string `mapstructure:"upstream_header_time"` } func NewNginxAccessItem(v map[string]string) (*NginxAccessItem, error) { From 421c5e533c8f7ab7a6811a75e146d8ae9ddfa502 Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Thu, 26 Jan 2023 11:01:14 +0000 Subject: [PATCH 13/15] Fix typo --- src/core/metrics/sources/nginx_access_log.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index e8be18a28..124a39415 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -523,7 +523,7 @@ func getDefaultCounters() (map[string]float64, map[string]float64, map[string]fl "v2": 0, } - upstreamConnnectCounters := map[string]float64{ + upstreamConnectCounters := map[string]float64{ "upstream.connect.time": 0, "upstream.connect.time.count": 0, "upstream.connect.time.max": 0, @@ -539,5 +539,5 @@ func getDefaultCounters() (map[string]float64, map[string]float64, map[string]fl "upstream.header.time.pctl95": 0, } - return httpCounters, upstreamConnnectCounters, upstreamHeaderCounters + return httpCounters, upstreamConnectCounters, upstreamHeaderCounters } From faf7847186450341c90da4265a7dd509fc25afff Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Thu, 26 Jan 2023 14:33:48 +0000 Subject: [PATCH 14/15] Fixed aggregate metrics naming --- src/core/metrics/metrics_util.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/metrics/metrics_util.go b/src/core/metrics/metrics_util.go index 003c2baee..af9393d90 100644 --- a/src/core/metrics/metrics_util.go +++ b/src/core/metrics/metrics_util.go @@ -130,16 +130,16 @@ func GetCalculationMap() map[string]string { "nginx.http.v1_0": "sum", "nginx.http.v1_1": "sum", "nginx.http.v2": "sum", - "nginx.upstream.time": "avg", - "nginx.upstream.time.count": "sum", - "nginx.upstream.time.max": "avg", - "nginx.upstream.time.median": "avg", - "nginx.upstream.time.pctl95": "avg", - "nginx.upstream.header": "avg", - "nginx.upstream.header.count": "sum", - "nginx.upstream.header.max": "avg", - "nginx.upstream.header.median": "avg", - "nginx.upstream.header.pctl95": "avg", + "nginx.upstream.connect.time": "avg", + "nginx.upstream.connect.time.count": "sum", + "nginx.upstream.connect.time.max": "avg", + "nginx.upstream.connect.time.median": "avg", + "nginx.upstream.connect.time.pctl95": "avg", + "nginx.upstream.header.time": "avg", + "nginx.upstream.header.time.count": "sum", + "nginx.upstream.header.time.max": "avg", + "nginx.upstream.header.time.median": "avg", + "nginx.upstream.header.time.pctl95": "avg", "nginx.http.conn.handled": "sum", "nginx.http.conn.reading": "avg", "nginx.http.conn.writing": "avg", From 9da5c849a53bdd333e1d1e31948e09a6e9cf3eec Mon Sep 17 00:00:00 2001 From: Dean Coakley Date: Thu, 26 Jan 2023 14:49:35 +0000 Subject: [PATCH 15/15] Make deps --- test/integration/go.mod | 1 - .../agent/v2/src/core/metrics/metrics_util.go | 20 +++++++++---------- .../core/metrics/sources/nginx_access_log.go | 4 ++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/test/integration/go.mod b/test/integration/go.mod index 316b3f910..05f7e56da 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -149,7 +149,6 @@ require ( ) replace ( - // https://github.com/cucumber/godog/releases/tag/v0.12.0 - this version breaks github.com/cucumber/godog => github.com/cucumber/godog v0.11.0 github.com/docker/cli => github.com/docker/cli v20.10.3-0.20221013132413-1d6c6e2367e2+incompatible // 22.06 master branch diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go index 003c2baee..af9393d90 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go @@ -130,16 +130,16 @@ func GetCalculationMap() map[string]string { "nginx.http.v1_0": "sum", "nginx.http.v1_1": "sum", "nginx.http.v2": "sum", - "nginx.upstream.time": "avg", - "nginx.upstream.time.count": "sum", - "nginx.upstream.time.max": "avg", - "nginx.upstream.time.median": "avg", - "nginx.upstream.time.pctl95": "avg", - "nginx.upstream.header": "avg", - "nginx.upstream.header.count": "sum", - "nginx.upstream.header.max": "avg", - "nginx.upstream.header.median": "avg", - "nginx.upstream.header.pctl95": "avg", + "nginx.upstream.connect.time": "avg", + "nginx.upstream.connect.time.count": "sum", + "nginx.upstream.connect.time.max": "avg", + "nginx.upstream.connect.time.median": "avg", + "nginx.upstream.connect.time.pctl95": "avg", + "nginx.upstream.header.time": "avg", + "nginx.upstream.header.time.count": "sum", + "nginx.upstream.header.time.max": "avg", + "nginx.upstream.header.time.median": "avg", + "nginx.upstream.header.time.pctl95": "avg", "nginx.http.conn.handled": "sum", "nginx.http.conn.reading": "avg", "nginx.http.conn.writing": "avg", diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go index e8be18a28..124a39415 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go @@ -523,7 +523,7 @@ func getDefaultCounters() (map[string]float64, map[string]float64, map[string]fl "v2": 0, } - upstreamConnnectCounters := map[string]float64{ + upstreamConnectCounters := map[string]float64{ "upstream.connect.time": 0, "upstream.connect.time.count": 0, "upstream.connect.time.max": 0, @@ -539,5 +539,5 @@ func getDefaultCounters() (map[string]float64, map[string]float64, map[string]fl "upstream.header.time.pctl95": 0, } - return httpCounters, upstreamConnnectCounters, upstreamHeaderCounters + return httpCounters, upstreamConnectCounters, upstreamHeaderCounters }