/
fluentbit-configmap-updated.yaml
115 lines (105 loc) · 4 KB
/
fluentbit-configmap-updated.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-config
namespace: logging
labels:
k8s-app: fluent-bit
data:
# Configuration files: server, input, filters, and output
# =======================================================
fluent-bit.conf: |
[SERVICE]
Flush 5
Grace 120
Log_Level info
Daemon off
Parsers_File parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
# Plugins_File /fluent-bit/etc/plugins.conf
@INCLUDE input-kubernetes.conf
@INCLUDE filter-kubernetes.conf
@INCLUDE output-stackdriver.conf
input-kubernetes.conf: |
[INPUT]
Name tail
Path /var/log/containers/*.log
# System logs should be captured by the default system logging already
Exclude_Path /var/log/containers/*_kube-system_*.log,/var/log/containers/*_istio-system_*.log,/var/log/containers/*_knative-serving_*.log,/var/log/containers/*_gke-system_*.log,/var/log/containers/*_config-management-system_*.log
Tag kube.*
Parser docker
DB /var/log/flb_kube.db
Mem_Buf_Limit 20MB
Skip_Long_Lines On
Refresh_Interval 10
# Control the log line length
Buffer_Chunk_Size 256k
Buffer_Max_Size 10240k
# Using the docker mode to deal with multiline messages emitted by docker
Docker_Mode On
replace_info.lua: |
function extract_json(tag, timestamp, record)
record["log"] = string.gsub(record["log"], ".-%s*({.*}).*", "%1")
return 2, timestamp, record
end
filter-kubernetes.conf: |
[FILTER]
Name kubernetes
Match kube.*
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Kube_Tag_Prefix kube.var.log.containers.
# Try to merge the log messages
Merge_Log On
Merge_Log_Key log_processed
K8S-Logging.Parser On
K8S-Logging.Exclude Off
[FILTER]
Name lua
Match kube.*
# lua script to extract data in log messages
script replace_info.lua
call extract_json
[FILTER]
Name parser
Match kube.*
Key_Name log
Parser json
[FILTER]
Name parser
Match kube.*
Key_Name log
Parser my_paser
output-stackdriver.conf: |
[OUTPUT]
# write the log records that still have the 'kube.*' tags to Cloud Logging
Name stackdriver
Match kube.*
# The following fields are necessary. They allow filtering in Cloud Logging
# based on resource types. Change them accordingly based on your setup.
resource k8s_container
k8s_cluster_name custom-fluentbit
k8s_cluster_location us-east1-b
parsers.conf: |
[PARSER]
Name my_paser
Format regex
Regex ^(?<time>.+) (?<output>.+ .+) data:(?<int>[^ ]+) (?<float>[^ ]+) (?<bool>[^ ]+) (?<string>.+)$
[PARSER]
Name json
Format json
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Time_Keep On
[PARSER]
Name syslog
Format regex
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
Time_Key time
Time_Format %b %d %H:%M:%S