This repository has been archived by the owner on Apr 25, 2024. It is now read-only.
/
fluentd-configmap.yaml
351 lines (309 loc) · 10.2 KB
/
fluentd-configmap.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-config
namespace: logging
labels:
k8s-app: fluentd
data:
fluentd.conf: |
# Use the config specified by the FLUENTD_CONFIG environment variable, or
# default to fluentd-standalone.conf
@include "#{ENV['FLUENTD_CONFIG'] || 'fluentd-standalone.conf'}"
# A config for running Fluentd as a daemon which collects, filters, parses,
# and sends log to storage. No extra fluentd processes required.
fluentd-standalone.conf: |
# Common config
@include general.conf
@include prometheus.conf
# Input sources
@include kubernetes-input.conf
@include apiserver-audit-input.conf
# Parsing/Filtering
@include kubernetes-filter.conf
@include extra.conf
# Send to storage
@include output.conf
# A config for running Fluentd as a daemon which collects logs and forwards
# the logs using a forward_output to a Fluentd configured as an aggregator,
# with a forward_input.
fluentd-forwarder.conf: |
@include general.conf
@include prometheus.conf
@include apiserver-audit-input.conf
@include systemd-input.conf
@include kubernetes-input.conf
# Send to the aggregator
@include forward-output.conf
# A config for running Fluentd as HA ready deployment for receiving forwarded
# logs, and then applying filtering, and parsing before sending them to
# storage.
fluentd-aggregator.conf: |
# Receive from the forwarder
@include forward-input.conf
@include general.conf
@include prometheus.conf
@include systemd-filter.conf
@include kubernetes-filter.conf
@include extra.conf
# Send to storage
@include output.conf
forward-input.conf: |
<source>
@type forward
port 24224
bind 0.0.0.0
</source>
forward-output.conf: |
<match **>
@type forward
require_ack_response true
ack_response_timeout 30
recover_wait 10s
heartbeat_interval 1s
phi_threshold 16
send_timeout 10s
hard_timeout 10s
expire_dns_cache 15
heartbeat_type tcp
buffer_chunk_limit 2M
buffer_queue_limit 32
flush_interval 5s
max_retry_wait 15
disable_retry_limit
num_threads 8
<server>
name fluentd-aggregator
host fluentd-aggregator.logging.svc.cluster.local
weight 60
</server>
</match>
general.conf: |
# Prevent fluentd from handling records containing its own logs. Otherwise
# it can lead to an infinite loop, when error in sending one message generates
# another message which also fails to be sent and so on.
<match fluent.**>
type null
</match>
# Used for health checking
<source>
@type http
port 9880
bind 0.0.0.0
</source>
# Emits internal metrics to every minute, and also exposes them on port
# 24220. Useful for determining if an output plugin is retryring/erroring,
# or determining the buffer queue length.
<source>
@type monitor_agent
bind 0.0.0.0
port 24220
tag fluentd.monitor.metrics
</source>
prometheus.conf: |
# input plugin that is required to expose metrics by other prometheus
# plugins, such as the prometheus_monitor input below.
<source>
@type prometheus
bind 0.0.0.0
port 24231
metrics_path /metrics
</source>
# input plugin that collects metrics from MonitorAgent and exposes them
# as prometheus metrics
<source>
@type prometheus_monitor
# update the metrics every 5 seconds
interval 5
</source>
<source>
@type prometheus_output_monitor
interval 5
</source>
<source>
@type prometheus_tail_monitor
interval 5
</source>
systemd.conf: |
@include systemd-input.conf
@include systemd-filter.conf
systemd-input.conf: |
<source>
@type systemd
pos_file /var/log/fluentd-journald-systemd.pos
read_from_head true
strip_underscores true
tag systemd
</source>
systemd-filter.conf: |
<match systemd>
@type rewrite_tag_filter
rewriterule1 SYSTEMD_UNIT ^(.+).service$ systemd.$1
rewriterule2 SYSTEMD_UNIT !^(.+).service$ systemd.unmatched
</match>
<filter systemd.kubelet>
type parser
format kubernetes
reserve_data true
key_name MESSAGE
suppress_parse_error_log true
</filter>
<filter systemd.docker>
type parser
format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
reserve_data true
key_name MESSAGE
suppress_parse_error_log true
</filter>
# Filter filter ssh logs since it's mostly bots trying to login
<filter systemd.**>
@type grep
exclude1 SYSTEMD_UNIT (sshd@.*\.service)
</filter>
kubernetes.conf: |
@include kubernetes-input.conf
@include kubernetes-filter.conf
kubernetes-input.conf: |
# Capture Kubernetes pod logs
# The kubelet creates symlinks that capture the pod name, namespace,
# container name & Docker container ID to the docker logs for pods in the
# /var/log/containers directory on the host.
<source>
type tail
path /var/log/containers/*.log
pos_file /var/log/fluentd-containers.log.pos
time_format %Y-%m-%dT%H:%M:%S.%NZ
tag kubernetes.*
format json
read_from_head true
</source>
kubernetes-filter.conf: |
# Query the API for extra metadata.
<filter kubernetes.**>
type kubernetes_metadata
# If the logs begin with '{' and end with '}' then it's JSON so merge
# the JSON log field into the log event
merge_json_log true
preserve_json_log true
</filter>
# rewrite_tag_filter does not support nested fields like
# kubernetes.container_name, so this exists to flatten the fields
# so we can use them in our rewrite_tag_filter
<filter kubernetes.**>
@type record_transformer
enable_ruby true
<record>
kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]}
</record>
</filter>
# retag based on the container name of the log message
<match kubernetes.**>
@type rewrite_tag_filter
rewriterule1 kubernetes_namespace_container_name ^(.+)$ kube.$1
</match>
# Remove the unnecessary field as the information is already available on
# other fields.
<filter kube.**>
@type record_transformer
remove_keys kubernetes_namespace_container_name
</filter>
<filter kube.kube-system.**>
type parser
format kubernetes
reserve_data true
key_name log
suppress_parse_error_log true
</filter>
apiserver-audit-input.conf: |
# Example:
# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
<source>
type tail
format multiline
multiline_flush_interval 5s
format_firstline /^\S+\s+AUDIT:/
# Fields must be explicitly captured by name to be parsed into the record.
# Fields may not always be present, and order may change, so this just looks
# for a list of key="\"quoted\" value" pairs separated by spaces.
# Unknown fields are ignored.
# Note: We can't separate query/response lines as format1/format2 because
# they don't always come one after the other for a given query.
format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
time_format %FT%T.%L%Z
path /var/log/kubernetes/kube-apiserver-audit.log
pos_file /var/log/kube-apiserver-audit.log.pos
tag kube-apiserver-audit
</source>
output.conf: |
<match **>
# Plugin specific settings
type cloudwatch_logs
log_group_name kubernetes-logs
log_stream_name fluentd-cloudwatch
auto_create_stream true
# Buffer settings
buffer_chunk_limit 2M
buffer_queue_limit 32
flush_interval 10s
max_retry_wait 30
disable_retry_limit
num_threads 8
</match>
elasticsearch-template-es5x.json: |
{
"template" : "logstash-*",
"version" : 50001,
"settings" : {
"index.refresh_interval" : "5s",
"number_of_shards": 1
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true, "norms" : false},
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword" }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date", "include_in_all": false },
"@version": { "type": "keyword", "include_in_all": false },
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}
extra.conf: |
# Example filter that adds an extra field "cluster_name" to all log
# messages:
# <filter **>
# @type record_transformer
# <record>
# cluster_name "your_cluster_name"
# </record>
# </filter>