diff --git a/Dockerfile b/Dockerfile index 114bb6a..89db60e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.12 +FROM golang:1.13 ENV CGO_ENABLED=0 @@ -19,13 +19,10 @@ RUN go mod download COPY *.go ./ +RUN go test -short -v ./... +RUN go test -run SetupModule -v ./... + RUN gofmt -e -s -d . 2>&1 | tee /gofmt.out && test ! -s /gofmt.out RUN go vet . RUN golint -set_exit_status - -RUN errcheck ./... - -RUN go install ./... - -RUN go test -short -v ./... -RUN go test -run SetupModule -v ./... \ No newline at end of file +RUN errcheck ./... \ No newline at end of file diff --git a/go.mod b/go.mod index 7cc5257..a196b54 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/section-io/module-metrics -go 1.12 +go 1.13 require ( github.com/pkg/errors v0.8.1 diff --git a/metrics.go b/metrics.go index f5e8a35..77b8673 100644 --- a/metrics.go +++ b/metrics.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "regexp" "strconv" "strings" "syscall" @@ -16,31 +17,65 @@ import ( const maxLabelValueLength = 80 var ( - filepath string + filepath string + isValidHostHeader = regexp.MustCompile(`^[a-z0-9.-]+$`).MatchString ) -func sanitizeValue(label string, value interface{}) string { +func sanitizeLabel(label string, value interface{}) (string, string) { - // Convert to a string, no matter what underlying type it is - var labelValue string - if value != nil { - labelValue = fmt.Sprintf("%v", value) + if value == nil || value == "" || value == "-" { + return label, "" } + // Convert to a string, no matter what underlying type it is + labelValue := fmt.Sprintf("%v", value) + labelValue = strings.TrimSpace(labelValue) + switch label { case "content_type": - labelValue = strings.Split(labelValue, ";")[0] + label = "content_type_bucket" + + labelValue = strings.ToLower(labelValue) + if strings.HasPrefix(labelValue, "image/") { + labelValue = "image" + } else if strings.HasPrefix(labelValue, "text/html") { + labelValue = "html" + } else if strings.HasPrefix(labelValue, "text/css") { + labelValue = "css" + } else if strings.Contains(labelValue, "javascript") { + labelValue = "javascript" + } else { + labelValue = "other" + } + case "hostname": labelValue = strings.Split(labelValue, ":")[0] - } + labelValue = strings.ToLower(labelValue) + if !isValidHostHeader(labelValue) { + labelValue = "" + } - labelValue = strings.TrimSpace(labelValue) + case "status": + statusInt, _ := strconv.Atoi(labelValue) + switch { + case statusInt >= 100 && statusInt <= 103: + case statusInt >= 200 && statusInt <= 208: + case statusInt >= 300 && statusInt <= 308: + case statusInt >= 400 && statusInt <= 431: + case statusInt == 499: + case statusInt >= 500 && statusInt <= 511: + default: + // If it matches any of the above cases, do nothing (leave labelValue as is) + // otherwise set to blank + labelValue = "" + } + } if len(labelValue) > maxLabelValueLength { labelValue = labelValue[0:maxLabelValueLength] } - return labelValue + return label, labelValue } func getBytes(l map[string]interface{}) int { @@ -117,8 +152,9 @@ func StartReader(file io.Reader, output io.Writer, errorWriter io.Writer) { } else { labelValues := map[string]string{} - for _, label := range p8sLabels { - labelValues[label] = sanitizeValue(label, logline[label]) + for _, label := range logFieldNames { + label, value := sanitizeLabel(label, logline[label]) + labelValues[label] = value } addRequest(labelValues, getBytes(logline)) } diff --git a/metrics_test.go b/metrics_test.go index 442e057..7763b53 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -30,72 +30,159 @@ func TestCreateLogFifoFails(t *testing.T) { assert.Error(t, err) } -func TestSanitizeContentType(t *testing.T) { - const expected = "text/html" - actual := sanitizeValue("content_type", "text/html; charset=iso-8859-1") +func TestUnknownLabelSanitize(t *testing.T) { + const expectedLabel = "some_other_field" + const expectedValue = "foobar" + actual, _ := sanitizeLabel("some_other_field", "foobar") + assert.Equal(t, expectedLabel, actual) +} + +func TestSanitizeContentTypeLabelName(t *testing.T) { + const expectedLabel = "content_type_bucket" + actual, _ := sanitizeLabel("content_type", "text/html") + assert.Equal(t, expectedLabel, actual) +} +func TestSanitizeContentTypeHTML(t *testing.T) { + const expected = "html" + _, actual := sanitizeLabel("content_type", "text/html; charset=iso-8859-1") + assert.Equal(t, expected, actual) + + _, actual = sanitizeLabel("content_type", "text/html") assert.Equal(t, expected, actual) } -func TestSanitizeContentTypeNoSemiColon(t *testing.T) { - const expected = "text/html" - actual := sanitizeValue("content_type", "text/html") +func TestSanitizeContentTypeImage(t *testing.T) { + const expected = "image" + _, actual := sanitizeLabel("content_type", "image/jpg") + assert.Equal(t, expected, actual) + _, actual = sanitizeLabel("content_type", "image/gif") + assert.Equal(t, expected, actual) +} + +func TestSanitizeContentTypeCSS(t *testing.T) { + const expected = "css" + _, actual := sanitizeLabel("content_type", "text/css") + assert.Equal(t, expected, actual) +} + +func TestSanitizeContentTypeJavascript(t *testing.T) { + const expected = "javascript" + _, actual := sanitizeLabel("content_type", "text/javascript") + assert.Equal(t, expected, actual) + + _, actual = sanitizeLabel("content_type", "application/javascript") assert.Equal(t, expected, actual) } func TestSanitizeContentTypeEmpty(t *testing.T) { const expected = "" - actual := sanitizeValue("content_type", "") + _, actual := sanitizeLabel("content_type", "") + assert.Equal(t, expected, actual) + + _, actual = sanitizeLabel("content_type", "-") + assert.Equal(t, expected, actual) +} + +func TestSanitizeContentTypeOther(t *testing.T) { + const expected = "other" + _, actual := sanitizeLabel("content_type", "foobar") + assert.Equal(t, expected, actual) + _, actual = sanitizeLabel("content_type", "text/rtf") assert.Equal(t, expected, actual) } func TestUnsanitizedLabel(t *testing.T) { const expected = "fooooo3iwac" - actual := sanitizeValue("some_unknown_type", " fooooo3iwac ") + _, actual := sanitizeLabel("some_unknown_type", " fooooo3iwac ") assert.Equal(t, expected, actual) } func TestSanitizeNil(t *testing.T) { const expected = "" - actual := sanitizeValue("foo", nil) + _, actual := sanitizeLabel("foo", nil) assert.Equal(t, expected, actual) } func TestSanitizeHostname(t *testing.T) { const expected = "www.foo.com" - actual := sanitizeValue("hostname", "www.foo.com") + _, actual := sanitizeLabel("hostname", "www.foo.com") assert.Equal(t, expected, actual) } func TestSanitizeHostnameWithPort(t *testing.T) { const expected = "www.foo.com" - actual := sanitizeValue("hostname", "www.foo.com:80") + _, actual := sanitizeLabel("hostname", "www.foo.com:80") assert.Equal(t, expected, actual) } func TestSanitizeHostnameWithSpaces(t *testing.T) { const expected = "www.foo.com" - actual := sanitizeValue("hostname", " www.foo.com ") + _, actual := sanitizeLabel("hostname", " www.foo.com ") assert.Equal(t, expected, actual) } func TestSanitizeHostnameMissing(t *testing.T) { const expected = "" - actual := sanitizeValue("hostname", nil) + _, actual := sanitizeLabel("hostname", nil) + assert.Equal(t, expected, actual) + + _, actual = sanitizeLabel("hostname", "-") + assert.Equal(t, expected, actual) + + _, actual = sanitizeLabel("hostname", " ") + assert.Equal(t, expected, actual) +} + +func TestSanitizeHostnameCasing(t *testing.T) { + const expected = "www.foo.com" + _, actual := sanitizeLabel("hostname", "WWw.FOo.COm") + + assert.Equal(t, expected, actual) +} + +func TestSanitizeHostnameInvalidChars(t *testing.T) { + const expected = "" + + _, actual := sanitizeLabel("hostname", "www.fi$h.com") + assert.Equal(t, expected, actual) + + _, actual = sanitizeLabel("hostname", "%(+ ") assert.Equal(t, expected, actual) } +func TestSanitizeStatus(t *testing.T) { + const expected = "200" + _, actual := sanitizeLabel("status", "200") + + assert.Equal(t, expected, actual) +} + +func TestSanitizeStatusInvalid(t *testing.T) { + const expected = "" + + _, actual := sanitizeLabel("status", "220") + assert.Equal(t, expected, actual) + + _, actual = sanitizeLabel("status", "foobar") + assert.Equal(t, expected, actual) + + _, actual = sanitizeLabel("status", "220foo") + assert.Equal(t, expected, actual) + +} + func TestSanitizeMaxLength(t *testing.T) { const expected = "01234567890123456789012345678901234567890123456789012345678901234567890123456789" - actual := sanitizeValue("hostname", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789") + _, actual := sanitizeLabel("hostname", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789") assert.Equal(t, expected, actual) } diff --git a/p8s.go b/p8s.go index 17714c3..2855720 100644 --- a/p8s.go +++ b/p8s.go @@ -26,8 +26,9 @@ var ( registry *prometheus.Registry httpServer *http.Server - defaultP8sLabels = []string{"hostname", "status"} - p8sLabels []string + defaultP8sLabels = []string{"hostname"} + logFieldNames []string + sanitizedP8sLabels []string p8sHTTPServerStarted = false @@ -44,7 +45,13 @@ func addRequest(labels map[string]string, bytes int) { // will reset any collected metrics. Returns the registry so additional metrics can be registered. func InitMetrics(additionalLabels ...string) *prometheus.Registry { - p8sLabels = append(defaultP8sLabels, additionalLabels...) + logFieldNames = append(defaultP8sLabels, additionalLabels...) + + sanitizedP8sLabels = defaultP8sLabels + for _, label := range additionalLabels { + label, _ = sanitizeLabel(label, "dummy") + sanitizedP8sLabels = append(sanitizedP8sLabels, label) + } const promeNamespace = "section" registry = prometheus.NewRegistry() @@ -54,14 +61,14 @@ func InitMetrics(additionalLabels ...string) *prometheus.Registry { Subsystem: promeSubsystem, Name: "request_count_total", Help: "Total count of HTTP requests.", - }, p8sLabels) + }, sanitizedP8sLabels) bytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: promeNamespace, Subsystem: promeSubsystem, Name: "bytes_total", Help: "Total sum of response bytes.", - }, p8sLabels) + }, sanitizedP8sLabels) jsonParseErrorTotal = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: promeNamespace, diff --git a/p8s_test.go b/p8s_test.go index 00ec89e..5aa4b46 100644 --- a/p8s_test.go +++ b/p8s_test.go @@ -83,8 +83,8 @@ func testCountersIncrease(t *testing.T, stdout *bytes.Buffer) { actual := gatherP8sResponse(t) - assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com",status="304"} 5`) - assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",status="304"} 1790`) + assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com"} 7`) + assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com"} 5875`) } func testBytesAndBytesSentAreRead(t *testing.T, stdout *bytes.Buffer) { @@ -100,8 +100,8 @@ func testBytesAndBytesSentAreRead(t *testing.T, stdout *bytes.Buffer) { actual := gatherP8sResponse(t) - assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com",status="200"} 2`) - assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",status="200"} 30`) + assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com"} 2`) + assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com"} 30`) } func testInvalidBytesAndBytesSent(t *testing.T, stdout *bytes.Buffer) { @@ -119,8 +119,8 @@ func testInvalidBytesAndBytesSent(t *testing.T, stdout *bytes.Buffer) { actual := gatherP8sResponse(t) - assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com",status="200"} 4`) - assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",status="200"} 30`) + assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com"} 4`) + assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com"} 30`) } func testJSONParseErrors(t *testing.T, stdout *bytes.Buffer) { @@ -161,8 +161,8 @@ func testP8sServer(t *testing.T, stdout *bytes.Buffer) { actual := getP8sHTTPResponse(t) - assert.Contains(t, actual, `section_http_request_count_total{hostname="bar.example.com",status="304"} 1`) - assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",status="304"} 1790`) + assert.Contains(t, actual, `section_http_request_count_total{hostname="bar.example.com"} 1`) + assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com"} 5875`) } func testAdditionalLabelsAreUsed(t *testing.T, stdout *bytes.Buffer) { @@ -178,8 +178,8 @@ func testAdditionalLabelsAreUsed(t *testing.T, stdout *bytes.Buffer) { actual := gatherP8sResponse(t) - assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com",http_accept_encoding="gzip",status="200"} 2`) - assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",http_accept_encoding="gzip",status="200"} 30`) + assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com",http_accept_encoding="gzip"} 2`) + assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",http_accept_encoding="gzip"} 30`) } func testAdditionalLabelsWhenMissingFromLogs(t *testing.T, stdout *bytes.Buffer) { @@ -195,8 +195,8 @@ func testAdditionalLabelsWhenMissingFromLogs(t *testing.T, stdout *bytes.Buffer) actual := gatherP8sResponse(t) - assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com",missing_field="",status="200"} 2`) - assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",missing_field="",status="200"} 30`) + assert.Contains(t, actual, `section_http_request_count_total{hostname="www.example.com",missing_field=""} 2`) + assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",missing_field=""} 30`) } func testNonStringProperties(t *testing.T, stdout *bytes.Buffer) { @@ -212,8 +212,8 @@ func testNonStringProperties(t *testing.T, stdout *bytes.Buffer) { actual := gatherP8sResponse(t) - assert.Contains(t, actual, `section_http_request_count_total{bool="",hostname="www.example.com",int="12345",status="200"} 1`) - assert.Contains(t, actual, `section_http_request_count_total{bool="true",hostname="www.example.com",int="",status="200"} 1`) + assert.Contains(t, actual, `section_http_request_count_total{bool="",hostname="www.example.com",int="12345"} 1`) + assert.Contains(t, actual, `section_http_request_count_total{bool="true",hostname="www.example.com",int=""} 1`) } func testAdditionalMetricsAfterInit(t *testing.T, stdout *bytes.Buffer) { @@ -258,28 +258,28 @@ func TestSetupModule(t *testing.T) { } var stdout bytes.Buffer - err := SetupModule(fifoFilePath, &stdout, os.Stderr) + err := SetupModule(fifoFilePath, &stdout, os.Stderr, "status", "content_type") if err != nil { t.Error(err) } logs := []string{ - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.070","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.070","upstream_connect_time":"0.052","upstream_response_time":"0.070","upstream_response_length":"0","upstream_bytes_received":"288","upstream_http_content_type":"-","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"cf99df8057b93ec96c0ee1253ba4c309"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.069","hostname":"foo.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.069","upstream_connect_time":"0.052","upstream_response_time":"0.069","upstream_response_length":"0","upstream_bytes_received":"288","upstream_http_content_type":"-","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"451e230222237f722eb49324d47142f6"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.070","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.070","upstream_connect_time":"0.052","upstream_response_time":"0.070","upstream_response_length":"0","upstream_bytes_received":"288","upstream_http_content_type":"-","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"4e189f278375962cd19d380562846296"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.075","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.075","upstream_connect_time":"0.056","upstream_response_time":"0.075","upstream_response_length":"0","upstream_bytes_received":"288","upstream_http_content_type":"-","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"6ef1b5083893627d2426e42206d78f70"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.077","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"200","bytes_sent":"1959","body_bytes_sent":"1498","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"200","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.077","upstream_connect_time":"0.057","upstream_response_time":"0.077","upstream_response_length":"1498","upstream_bytes_received":"1889","upstream_http_content_type":"application/javascript","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"gzip","upstream_http_transfer_encoding":"chunked","sent_http_content_length":"-","sent_http_content_encoding":"gzip","sent_http_transfer_encoding":"chunked","section-io-id":"b1ea9bc0be7edfc997bc18a9f6b20d68"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.073","hostname":"foo.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.073","upstream_connect_time":"0.055","upstream_response_time":"0.073","upstream_response_length":"0","upstream_bytes_received":"288","upstream_http_content_type":"-","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"ff3117bb0ac0307d8d0e78fc8b8ba5c7"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.070","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.070","upstream_connect_time":"0.052","upstream_response_time":"0.070","upstream_response_length":"0","upstream_bytes_received":"288","upstream_http_content_type":"-","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"85e833ae62745c50492c80b4d7b78016"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.072","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.072","upstream_connect_time":"0.054","upstream_response_time":"0.072","upstream_response_length":"0","upstream_bytes_received":"288","upstream_http_content_type":"-","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"a095e3c2c3a0f25b4bbca4c941babd76"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.071","hostname":"bar.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.071","upstream_connect_time":"0.053","upstream_response_time":"0.071","upstream_response_length":"0","upstream_bytes_received":"288","upstream_http_content_type":"-","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"8fb3941b35418bdfa1946ef02c90e8c7"}`, - `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.075","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"200","bytes_sent":"2126","body_bytes_sent":"1665","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"200","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.075","upstream_connect_time":"0.056","upstream_response_time":"0.075","upstream_response_length":"1665","upstream_bytes_received":"2056","upstream_http_content_type":"application/javascript","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"gzip","upstream_http_transfer_encoding":"chunked","sent_http_content_length":"-","sent_http_content_encoding":"gzip","sent_http_transfer_encoding":"chunked","section-io-id":"789addb393a18ff1caf5d776b53cf30e"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.070","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.070","upstream_connect_time":"0.052","upstream_response_time":"0.070","upstream_response_length":"0","upstream_bytes_received":"288","content_type":"text/html","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"cf99df8057b93ec96c0ee1253ba4c309"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.069","hostname":"foo.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.069","upstream_connect_time":"0.052","upstream_response_time":"0.069","upstream_response_length":"0","upstream_bytes_received":"288","content_type":"text/html","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"451e230222237f722eb49324d47142f6"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.070","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.070","upstream_connect_time":"0.052","upstream_response_time":"0.070","upstream_response_length":"0","upstream_bytes_received":"288","content_type":"text/html","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"4e189f278375962cd19d380562846296"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.075","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.075","upstream_connect_time":"0.056","upstream_response_time":"0.075","upstream_response_length":"0","upstream_bytes_received":"288","content_type":"text/html","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"6ef1b5083893627d2426e42206d78f70"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.077","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"200","bytes_sent":"1959","body_bytes_sent":"1498","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"200","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.077","upstream_connect_time":"0.057","upstream_response_time":"0.077","upstream_response_length":"1498","upstream_bytes_received":"1889","content_type":"application/javascript","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"gzip","upstream_http_transfer_encoding":"chunked","sent_http_content_length":"-","sent_http_content_encoding":"gzip","sent_http_transfer_encoding":"chunked","section-io-id":"b1ea9bc0be7edfc997bc18a9f6b20d68"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.073","hostname":"foo.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.073","upstream_connect_time":"0.055","upstream_response_time":"0.073","upstream_response_length":"0","upstream_bytes_received":"288","content_type":"text/html","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"ff3117bb0ac0307d8d0e78fc8b8ba5c7"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.070","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.070","upstream_connect_time":"0.052","upstream_response_time":"0.070","upstream_response_length":"0","upstream_bytes_received":"288","content_type":"text/html","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"85e833ae62745c50492c80b4d7b78016"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.072","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.072","upstream_connect_time":"0.054","upstream_response_time":"0.072","upstream_response_length":"0","upstream_bytes_received":"288","content_type":"text/html","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"a095e3c2c3a0f25b4bbca4c941babd76"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.071","hostname":"bar.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"304","bytes_sent":"358","body_bytes_sent":"0","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"304","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.071","upstream_connect_time":"0.053","upstream_response_time":"0.071","upstream_response_length":"0","upstream_bytes_received":"288","content_type":"text/html","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"-","upstream_http_transfer_encoding":"-","sent_http_content_length":"-","sent_http_content_encoding":"-","sent_http_transfer_encoding":"-","section-io-id":"8fb3941b35418bdfa1946ef02c90e8c7"}`, + `{"time":"2019-06-20T01:34:36+00:00","request_time":"0.075","hostname":"www.example.com","request":"GET /a/path HTTP/1.1","http_accept_encoding":"gzip","http_x_forwarded_proto":"https","http_upgrade":"-","http_connection":"-","status":"200","bytes_sent":"2126","body_bytes_sent":"1665","upstream_label":"default","upstream_addr":"198.51.100.1:443","upstream_status":"200","upstream_request_connection":"","upstream_request_host":"in.example.com","upstream_header_time":"0.075","upstream_connect_time":"0.056","upstream_response_time":"0.075","upstream_response_length":"1665","upstream_bytes_received":"2056","content_type":"application/javascript","upstream_http_cache_control":"max-age=60","upstream_http_content_length":"-","upstream_http_content_encoding":"gzip","upstream_http_transfer_encoding":"chunked","sent_http_content_length":"-","sent_http_content_encoding":"gzip","sent_http_transfer_encoding":"chunked","section-io-id":"789addb393a18ff1caf5d776b53cf30e"}`, } writeLogs(t, logs) actual := gatherP8sResponse(t) - assert.Contains(t, actual, `section_http_request_count_total{hostname="bar.example.com",status="304"} 1`) - assert.Contains(t, actual, `section_http_bytes_total{hostname="www.example.com",status="304"} 1790`) + assert.Contains(t, actual, `section_http_request_count_total{content_type_bucket="javascript",hostname="www.example.com",status="200"} 2`) + assert.Contains(t, actual, `section_http_bytes_total{content_type_bucket="html",hostname="www.example.com",status="304"} 1790`) }