Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

filter fluentd container logs #402

Merged
merged 21 commits into from
Feb 12, 2020
Merged

filter fluentd container logs #402

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
3f92c49
filter fluentd logs from container logs and only ingest specific warn…
vsinghal13 Feb 7, 2020
b725432
change default fluentd log level to info
vsinghal13 Feb 7, 2020
2d75ee6
Generate new 'fluentd-sumologic.yaml.tmpl'
Feb 7, 2020
9d221ff
remove extra fluent label and filter fluentd logs in the existing pip…
vsinghal13 Feb 10, 2020
65a1010
Merge branch 'vsinghal-filter-fluentd-container-logs' of https://gith…
vsinghal13 Feb 10, 2020
b16501c
Generate new 'fluentd-sumologic.yaml.tmpl'
Feb 10, 2020
7c1ad8a
Merge branch 'master' into vsinghal-filter-fluentd-container-logs
vsinghal13 Feb 11, 2020
464287d
Generate new overrides yaml/libsonnet file(s).
Feb 11, 2020
1fc689c
Merge branch 'master' into vsinghal-filter-fluentd-container-logs
vsinghal13 Feb 11, 2020
7ff8699
Merge branch 'vsinghal-filter-fluentd-container-logs' of https://gith…
vsinghal13 Feb 11, 2020
2dcd965
Generate new 'fluentd-sumologic.yaml.tmpl'
Feb 11, 2020
6fabcd5
Generate new overrides yaml/libsonnet file(s).
Feb 11, 2020
4c4a9fa
change log level in regex to error
vsinghal13 Feb 12, 2020
bd838ea
Merge branch 'master' into vsinghal-filter-fluentd-container-logs
vsinghal13 Feb 12, 2020
7c3abf8
Generate new 'fluentd-sumologic.yaml.tmpl'
Feb 12, 2020
775eb50
Merge branch 'master' into vsinghal-filter-fluentd-container-logs
vsinghal13 Feb 12, 2020
af8b97a
Merge branch 'vsinghal-filter-fluentd-container-logs' of https://gith…
vsinghal13 Feb 12, 2020
4143fdb
Generate new 'fluentd-sumologic.yaml.tmpl'
Feb 12, 2020
c79d659
Update kube-prometheus-sumo-logic-mixin.libsonnet
vsinghal13 Feb 12, 2020
a22351c
Update SideBySidePrometheus.md
vsinghal13 Feb 12, 2020
65f09e2
Update Troubleshoot_Collection.md
vsinghal13 Feb 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions deploy/helm/sumologic/conf/logs/logs.enhance.k8s.metadata.filter.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
cache_size "#{ENV['K8S_METADATA_FILTER_CACHE_SIZE']}"
cache_ttl "#{ENV['K8S_METADATA_FILTER_CACHE_TTL']}"
cache_refresh "#{ENV['K8S_METADATA_FILTER_CACHE_REFRESH']}"
in_namespace_path '$.kubernetes.namespace_name'
in_pod_path '$.kubernetes.pod_name'
data_type logs
11 changes: 11 additions & 0 deletions deploy/helm/sumologic/conf/logs/logs.kubernetes.metadata.filter.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
annotation_match ["sumologic\.com.*"]
de_dot false
watch "#{ENV['K8S_METADATA_FILTER_WATCH']}"
ca_file "#{ENV['K8S_METADATA_FILTER_CA_FILE']}"
verify_ssl "#{ENV['K8S_METADATA_FILTER_VERIFY_SSL']}"
client_cert "#{ENV['K8S_METADATA_FILTER_CLIENT_CERT']}"
client_key "#{ENV['K8S_METADATA_FILTER_CLIENT_KEY']}"
bearer_token_file "#{ENV['K8S_METADATA_FILTER_BEARER_TOKEN_FILE']}"
cache_size "#{ENV['K8S_METADATA_FILTER_CACHE_SIZE']}"
cache_ttl "#{ENV['K8S_METADATA_FILTER_CACHE_TTL']}"
tag_to_kubernetes_name_regexp '.+?\.containers\.(?<pod_name>[^_]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
14 changes: 14 additions & 0 deletions deploy/helm/sumologic/conf/logs/logs.kubernetes.sumologic.filter.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
source_name "#{ENV['SOURCE_NAME']}"
source_host "#{ENV['SOURCE_HOST']}"
log_format "#{ENV['LOG_FORMAT']}"
kubernetes_meta "#{ENV['KUBERNETES_META']}"
kubernetes_meta_reduce "#{ENV['KUBERNETES_META_REDUCE']}"
add_stream "#{ENV['ADD_STREAM']}"
add_time "#{ENV['ADD_TIME']}"
source_category "#{ENV['SOURCE_CATEGORY']}"
source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
source_category_replace_dash "#{ENV['SOURCE_CATEGORY_REPLACE_DASH']}"
exclude_namespace_regex "#{ENV['EXCLUDE_NAMESPACE_REGEX']}"
exclude_pod_regex "#{ENV['EXCLUDE_POD_REGEX']}"
exclude_container_regex "#{ENV['EXCLUDE_CONTAINER_REGEX']}"
exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
52 changes: 18 additions & 34 deletions deploy/helm/sumologic/conf/logs/logs.source.containers.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,53 +8,37 @@
time ${[record["log"].split(/[\n\t]+/)[0]].map! {|item| JSON.parse(item)["time"]}.join("")}
</record>
</filter>
# match all container logs and label them @NORMAL
<match containers.**>
@type relabel
@label @NORMAL
</match>
<label @NORMAL>
# only match fluentd logs based on fluentd container name
{{ printf "<filter **%s**>" (include "sumologic.fullname" .) }}
vsinghal13 marked this conversation as resolved.
Show resolved Hide resolved
# only ingest fluentd logs of levels: {error, fatal} and warning messages if buffer is full
@type grep
<regexp>
key log
pattern /\[error\]|\[fatal\]|drop_oldest_chunk/
</regexp>
</filter>
# third-party kubernetes metadata filter plugin
<filter containers.**>
@type kubernetes_metadata
@log_level warn
annotation_match ["sumologic\.com.*"]
de_dot false
watch "#{ENV['K8S_METADATA_FILTER_WATCH']}"
ca_file "#{ENV['K8S_METADATA_FILTER_CA_FILE']}"
verify_ssl "#{ENV['K8S_METADATA_FILTER_VERIFY_SSL']}"
client_cert "#{ENV['K8S_METADATA_FILTER_CLIENT_CERT']}"
client_key "#{ENV['K8S_METADATA_FILTER_CLIENT_KEY']}"
bearer_token_file "#{ENV['K8S_METADATA_FILTER_BEARER_TOKEN_FILE']}"
cache_size "#{ENV['K8S_METADATA_FILTER_CACHE_SIZE']}"
cache_ttl "#{ENV['K8S_METADATA_FILTER_CACHE_TTL']}"
tag_to_kubernetes_name_regexp '.+?\.containers\.(?<pod_name>[^_]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
@include logs.kubernetes.metadata.filter.conf
</filter>
<filter **>
# sumologic kubernetes metadata enrichment filter plugin
<filter containers.**>
Comment on lines -31 to +32
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@samjsong @rvmiller89 I suspect it is because of this change.

@type enhance_k8s_metadata
cache_size "#{ENV['K8S_METADATA_FILTER_CACHE_SIZE']}"
cache_ttl "#{ENV['K8S_METADATA_FILTER_CACHE_TTL']}"
cache_refresh "#{ENV['K8S_METADATA_FILTER_CACHE_REFRESH']}"
in_namespace_path '$.kubernetes.namespace_name'
in_pod_path '$.kubernetes.pod_name'
data_type logs
@include logs.kubernetes.enhance.k8s.metadata.filter.conf
</filter>
# kubernetes sumologic filter plugin
<filter containers.**>
@type kubernetes_sumologic
source_name "#{ENV['SOURCE_NAME']}"
source_host "#{ENV['SOURCE_HOST']}"
log_format "#{ENV['LOG_FORMAT']}"
kubernetes_meta "#{ENV['KUBERNETES_META']}"
kubernetes_meta_reduce "#{ENV['KUBERNETES_META_REDUCE']}"
add_stream "#{ENV['ADD_STREAM']}"
add_time "#{ENV['ADD_TIME']}"
source_category "#{ENV['SOURCE_CATEGORY']}"
source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
source_category_replace_dash "#{ENV['SOURCE_CATEGORY_REPLACE_DASH']}"
exclude_namespace_regex "#{ENV['EXCLUDE_NAMESPACE_REGEX']}"
exclude_pod_regex "#{ENV['EXCLUDE_POD_REGEX']}"
exclude_container_regex "#{ENV['EXCLUDE_CONTAINER_REGEX']}"
exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
@include logs.kubernetes.sumologic.filter.conf
</filter>
<match **>
<match containers.**>
@type sumologic
@id sumologic.endpoint.logs
@include logs.output.conf
Expand Down
5 changes: 3 additions & 2 deletions deploy/helm/sumologic/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,10 @@ sumologic:
excludePodRegex: ""


## Sets the fluentd log level. The default log level, if not specified, is error.
## Sets the fluentd log level. The default log level, if not specified, is info.
## Sumo will only ingest the error log level and some specific warnings, the info logs can be seen in kubectl logs.
## ref: https://docs.fluentd.org/deployment/logging
fluentdLogLevel: "error"
fluentdLogLevel: "info"

## Override Kubernetes resource types you want to get events for from different Kubernetes
## API versions. The key represents the name of the resource type and the value represents
Expand Down
90 changes: 54 additions & 36 deletions deploy/kubernetes/fluentd-sumologic.yaml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ data:
bind 0.0.0.0
</source>
<system>
log_level error
log_level info
</system>

metrics.conf: |-
Expand Down Expand Up @@ -158,6 +158,40 @@ data:
@include logs.source.containers.conf
@include logs.source.systemd.conf
@include logs.source.generic.conf
logs.enhance.k8s.metadata.filter.conf: |-
cache_size "#{ENV['K8S_METADATA_FILTER_CACHE_SIZE']}"
cache_ttl "#{ENV['K8S_METADATA_FILTER_CACHE_TTL']}"
cache_refresh "#{ENV['K8S_METADATA_FILTER_CACHE_REFRESH']}"
in_namespace_path '$.kubernetes.namespace_name'
in_pod_path '$.kubernetes.pod_name'
data_type logs
logs.kubernetes.metadata.filter.conf: |-
annotation_match ["sumologic\.com.*"]
de_dot false
watch "#{ENV['K8S_METADATA_FILTER_WATCH']}"
ca_file "#{ENV['K8S_METADATA_FILTER_CA_FILE']}"
verify_ssl "#{ENV['K8S_METADATA_FILTER_VERIFY_SSL']}"
client_cert "#{ENV['K8S_METADATA_FILTER_CLIENT_CERT']}"
client_key "#{ENV['K8S_METADATA_FILTER_CLIENT_KEY']}"
bearer_token_file "#{ENV['K8S_METADATA_FILTER_BEARER_TOKEN_FILE']}"
cache_size "#{ENV['K8S_METADATA_FILTER_CACHE_SIZE']}"
cache_ttl "#{ENV['K8S_METADATA_FILTER_CACHE_TTL']}"
tag_to_kubernetes_name_regexp '.+?\.containers\.(?<pod_name>[^_]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
logs.kubernetes.sumologic.filter.conf: |-
source_name "#{ENV['SOURCE_NAME']}"
source_host "#{ENV['SOURCE_HOST']}"
log_format "#{ENV['LOG_FORMAT']}"
kubernetes_meta "#{ENV['KUBERNETES_META']}"
kubernetes_meta_reduce "#{ENV['KUBERNETES_META_REDUCE']}"
add_stream "#{ENV['ADD_STREAM']}"
add_time "#{ENV['ADD_TIME']}"
source_category "#{ENV['SOURCE_CATEGORY']}"
source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
source_category_replace_dash "#{ENV['SOURCE_CATEGORY_REPLACE_DASH']}"
exclude_namespace_regex "#{ENV['EXCLUDE_NAMESPACE_REGEX']}"
exclude_pod_regex "#{ENV['EXCLUDE_POD_REGEX']}"
exclude_container_regex "#{ENV['EXCLUDE_CONTAINER_REGEX']}"
exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
logs.output.conf: |-
data_type logs
log_key log
Expand All @@ -178,53 +212,37 @@ data:
time ${[record["log"].split(/[\n\t]+/)[0]].map! {|item| JSON.parse(item)["time"]}.join("")}
</record>
</filter>
# match all container logs and label them @NORMAL
<match containers.**>
@type relabel
@label @NORMAL
</match>
<label @NORMAL>
# only match fluentd logs based on fluentd container name
<filter **collection-sumologic**>
# only ingest fluentd logs of levels: {error, fatal} and warning messages if buffer is full
@type grep
<regexp>
key log
pattern /\[error\]|\[fatal\]|drop_oldest_chunk/
</regexp>
</filter>
# third-party kubernetes metadata filter plugin
<filter containers.**>
@type kubernetes_metadata
@log_level warn
annotation_match ["sumologic\.com.*"]
de_dot false
watch "#{ENV['K8S_METADATA_FILTER_WATCH']}"
ca_file "#{ENV['K8S_METADATA_FILTER_CA_FILE']}"
verify_ssl "#{ENV['K8S_METADATA_FILTER_VERIFY_SSL']}"
client_cert "#{ENV['K8S_METADATA_FILTER_CLIENT_CERT']}"
client_key "#{ENV['K8S_METADATA_FILTER_CLIENT_KEY']}"
bearer_token_file "#{ENV['K8S_METADATA_FILTER_BEARER_TOKEN_FILE']}"
cache_size "#{ENV['K8S_METADATA_FILTER_CACHE_SIZE']}"
cache_ttl "#{ENV['K8S_METADATA_FILTER_CACHE_TTL']}"
tag_to_kubernetes_name_regexp '.+?\.containers\.(?<pod_name>[^_]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
@include logs.kubernetes.metadata.filter.conf
</filter>
<filter **>
# sumologic kubernetes metadata enrichment filter plugin
<filter containers.**>
@type enhance_k8s_metadata
cache_size "#{ENV['K8S_METADATA_FILTER_CACHE_SIZE']}"
cache_ttl "#{ENV['K8S_METADATA_FILTER_CACHE_TTL']}"
cache_refresh "#{ENV['K8S_METADATA_FILTER_CACHE_REFRESH']}"
in_namespace_path '$.kubernetes.namespace_name'
in_pod_path '$.kubernetes.pod_name'
data_type logs
@include logs.kubernetes.enhance.k8s.metadata.filter.conf
</filter>
# kubernetes sumologic filter plugin
<filter containers.**>
@type kubernetes_sumologic
source_name "#{ENV['SOURCE_NAME']}"
source_host "#{ENV['SOURCE_HOST']}"
log_format "#{ENV['LOG_FORMAT']}"
kubernetes_meta "#{ENV['KUBERNETES_META']}"
kubernetes_meta_reduce "#{ENV['KUBERNETES_META_REDUCE']}"
add_stream "#{ENV['ADD_STREAM']}"
add_time "#{ENV['ADD_TIME']}"
source_category "#{ENV['SOURCE_CATEGORY']}"
source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
source_category_replace_dash "#{ENV['SOURCE_CATEGORY_REPLACE_DASH']}"
exclude_namespace_regex "#{ENV['EXCLUDE_NAMESPACE_REGEX']}"
exclude_pod_regex "#{ENV['EXCLUDE_POD_REGEX']}"
exclude_container_regex "#{ENV['EXCLUDE_CONTAINER_REGEX']}"
exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
@include logs.kubernetes.sumologic.filter.conf
</filter>
<match **>
<match containers.**>
@type sumologic
@id sumologic.endpoint.logs
@include logs.output.conf
Expand Down Expand Up @@ -347,7 +365,7 @@ data:
</buffer>
</match>
<system>
log_level error
log_level info
</system>

buffer.output.conf: |-
Expand Down