-
Notifications
You must be signed in to change notification settings - Fork 66
/
common_logging_processors.go
90 lines (83 loc) · 3.33 KB
/
common_logging_processors.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package apps
import (
"context"
"fmt"
"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
)
const InstrumentationSourceLabel = confgenerator.InstrumentationSourceLabel
func instrumentationSourceValue(processorType string) *confgenerator.ModifyField {
val := fmt.Sprintf("agent.googleapis.com/%s", processorType)
return &confgenerator.ModifyField{
StaticValue: &val,
}
}
func genericAccessLogParser(ctx context.Context, processorType, tag, uid string) []fluentbit.Component {
c := confgenerator.LoggingProcessorParseRegex{
// Documentation:
// https://httpd.apache.org/docs/current/logs.html#accesslog
// https://docs.nginx.com/nginx/admin-guide/monitoring/logging/#setting-up-the-access-log
// Sample "common" line: 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
// Sample "combined" line: ::1 - - [26/Aug/2021:16:49:43 +0000] "GET / HTTP/1.1" 200 10701 "-" "curl/7.64.0" "0.5"
Regex: `^(?<http_request_remoteIp>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<http_request_requestMethod>\S+)(?: +(?<http_request_requestUrl>[^\"]*?)(?: +(?<http_request_protocol>\S+))?)?" (?<http_request_status>[^ ]*) (?<http_request_responseSize>[^ ]*)(?: "(?<http_request_referer>[^\"]*)" "(?<http_request_userAgent>[^\"]*)")?(?: "(?<gzip_ratio>[^\"]*)")?$`,
ParserShared: confgenerator.ParserShared{
TimeKey: "time",
TimeFormat: "%d/%b/%Y:%H:%M:%S %z",
Types: map[string]string{
"http_request_status": "integer",
// N.B. "http_request_responseSize" is a string containing an integer.
// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#HttpRequest.FIELDS.response_size
},
},
}.Components(ctx, tag, uid)
mf := confgenerator.LoggingProcessorModifyFields{
Fields: map[string]*confgenerator.ModifyField{
InstrumentationSourceLabel: instrumentationSourceValue(processorType),
},
}
// apache/nginx/varnish logs "-" when a field does not have a value. Remove the field entirely when this happens.
for _, field := range []string{
"jsonPayload.host",
"jsonPayload.user",
"jsonPayload.gzip_ratio",
} {
mf.Fields[field] = &confgenerator.ModifyField{
OmitIf: fmt.Sprintf(`%s = "-"`, field),
}
}
// Generate the httpRequest structure.
for _, field := range []string{
"remoteIp",
"requestMethod",
"requestUrl",
"protocol",
"status",
"responseSize",
"referer",
"userAgent",
} {
dest := fmt.Sprintf("httpRequest.%s", field)
src := fmt.Sprintf("jsonPayload.http_request_%s", field)
mf.Fields[dest] = &confgenerator.ModifyField{
MoveFrom: src,
}
if field == "referer" {
mf.Fields[dest].OmitIf = fmt.Sprintf(`%s = "-"`, src)
}
}
c = append(c, mf.Components(ctx, tag, uid)...)
return c
}