LOG-1491: Generate vector config #1230

vimalk78 · 2021-10-29T17:17:45Z

Description

PR to generate bare minimal vector.toml

all Sources are read (container logs, journal logs, audit logs)
adds basic transformation of raw logs to adjust log level, and add some metadata
adds basic routing
- splits container logs into app and infra
- source to input ("application", "infrastructure", "audit")
- input to pipeline
- pipeline to outputs
- some nodes in the generated pipeline are no-op, these are there just to rename the stream to get names as per API
kafka output is supported (sasl username/password not supported, key/cert supported)

Sample Conf:

# Logs from containers (including openshift containers)
[sources.raw_container_logs]
type = "kubernetes_logs"
auto_partial_merge = true
exclude_paths_glob_patterns = ["/var/log/pods/openshift-logging_collector-*/*/*.log", "/var/log/pods/openshift-logging_elasticsearch-*/*/*.log", "/var/log/pods/openshift-logging_kibana-*/*/*.log"]

[sources.raw_journal_logs]
type = "journald"

# Logs from host audit
[sources.host_audit_logs]
type = "file"
ignore_older_secs = 600
include = ["/var/log/audit/audit.log"]

# Logs from kubernetes audit
[sources.k8s_audit_logs]
type = "file"
ignore_older_secs = 600
include = ["/var/log/kube-apiserver/audit.log"]

# Logs from openshift audit
[sources.openshift_audit_logs]
type = "file"
ignore_older_secs = 600
include = ["/var/log/oauth-apiserver.audit.log"]

[transforms.container_logs]
type = "remap"
inputs = ["raw_container_logs"]
source = '''
  level = "unknown"
  if match(.message,r'(Warning|WARN|W[0-9]+|level=warn|Value:warn|"level":"warn")'){
	level = "warn"
  } else if match(.message, r'Info|INFO|I[0-9]+|level=info|Value:info|"level":"info"'){
	level = "info"
  } else if match(.message, r'Error|ERROR|E[0-9]+|level=error|Value:error|"level":"error"'){
	level = "error"
  } else if match(.message, r'Debug|DEBUG|D[0-9]+|level=debug|Value:debug|"level":"debug"'){
	level = "debug"
  }
  .level = level

  .pipeline_metadata.collector.name = "vector"
  .pipeline_metadata.collector.version = "0.14.1"
  ip4, err = get_env_var("NODE_IPV4")
  .pipeline_metadata.collector.ipaddr4 = ip4
  received, err = format_timestamp(now(),"%+")
  .pipeline_metadata.collector.received_at = received
  .pipeline_metadata.collector.error = err
 '''

[transforms.journal_logs]
type = "remap"
inputs = ["raw_journal_logs"]
source = '''
  level = "unknown"
  if match(.message,r'(Warning|WARN|W[0-9]+|level=warn|Value:warn|"level":"warn")'){
	level = "warn"
  } else if match(.message, r'Info|INFO|I[0-9]+|level=info|Value:info|"level":"info"'){
	level = "info"
  } else if match(.message, r'Error|ERROR|E[0-9]+|level=error|Value:error|"level":"error"'){
	level = "error"
  } else if match(.message, r'Debug|DEBUG|D[0-9]+|level=debug|Value:debug|"level":"debug"'){
	level = "debug"
  }
  .level = level

  .pipeline_metadata.collector.name = "vector"
  .pipeline_metadata.collector.version = "0.14.1"
  ip4, err = get_env_var("NODE_IPV4")
  .pipeline_metadata.collector.ipaddr4 = ip4
  received, err = format_timestamp(now(),"%+")
  .pipeline_metadata.collector.received_at = received
  .pipeline_metadata.collector.error = err
 '''


[transforms.route_container_logs]
type = "route"
inputs = ["container_logs"]
route.app = '!(starts_with!(.kubernetes.pod_namespace,"kube") && starts_with!(.kubernetes.pod_namespace,"openshift") && .kubernetes.pod_namespace == "default")'
route.infra = 'starts_with!(.kubernetes.pod_namespace,"kube") || starts_with!(.kubernetes.pod_namespace,"openshift") || .kubernetes.pod_namespace == "default"'


# Rename log stream to "application"
[transforms.application]
type = "remap"
inputs = ["route_container_logs.app"]
source = """
.
"""


# Rename log stream to "infrastructure"
[transforms.infrastructure]
type = "remap"
inputs = ["route_container_logs.infra","journal_logs"]
source = """
.
"""


# Rename log stream to "audit"
[transforms.audit]
type = "remap"
inputs = ["host_audit_logs","k8s_audit_logs","openshift_audit_logs"]
source = """
.
"""


[transforms.pipeline]
type = "remap"
inputs = ["application","infrastructure","audit"]
source = """
.
"""

# Kafka config
[sinks.kafka_receiver]
type = "kafka"
inputs = ["pipeline"]
bootstrap_servers = "broker1-kafka.svc.messaging.cluster.local:9092"
topic = "topic"

[sinks.kafka_receiver.encoding]
codec = "json"
timestamp_format = "rfc3339"

# TLS Config
[sinks.kafka_receiver.tls]
key_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/tls.key"
crt_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/tls.crt"
ca_file = "/var/run/ocp-collector/secrets/kafka-receiver-1/ca-bundle.crt"
enabled = true

/cc @jcantrill @ajaygupta978
/assign

Links

JIRA: https://issues.redhat.com/browse/LOG-1491

vimalk78 · 2021-10-29T17:18:28Z

/hold
testing in progress

jcantrill

/approve

internal/generator/vector/conf_test.go

internal/generator/vector/output/kafka/kafka_test.go

openshift-ci · 2021-10-29T17:41:45Z

[APPROVALNOTIFIER] This PR is APPROVED

This pull-request has been approved by: jcantrill, vimalk78

The full list of commands accepted by this bot can be found here.

The pull request process is described here

Needs approval from an approver in each of these files:

~~OWNERS~~ [jcantrill,vimalk78]

Approvers can indicate their approval by writing /approve in a comment
Approvers can cancel approval by writing /approve cancel in a comment

ajaygupta978 · 2021-10-29T18:31:31Z

internal/generator/vector/output/kafka/passphrase.go

This is fluentd syntax?

not exactly, passphrase is from kafka sasl config

I meant it requires = sign

thanks, i updated the PR. it has an = sign now

PR to generate bare minimal vector.toml - all Sources are read (container logs, journal logs, audit logs) - adds basic transformation of raw logs to adjust log level, and add some metadata - adds basic routing - splits container logs into app and infra - source to input ("application", "infrastructure", "audit") - input to pipeline - pipeline to outputs - some nodes in the generated pipeline are no-op, these are there just to rename the stream to get names as per API - kafka output is supported (sasl username/password not supported, key/cert supported)

vimalk78 · 2021-10-29T20:42:11Z

e2e kafka test case works
Steps:

uncomment ENABLE_VECTOR_COLLECTOR in Dockerfile
set the collector type to vector in ClusterLogging spec test/helpers/clusterlogging.go

infra log message captured in kafka:

{
  "file": "/var/log/pods/openshift-kube-apiserver_kube-apiserver-crc-dzk9v-master-0_cbe6089f-177a-4598-89d7-2bd4e5bef0c1/kube-apiserver/0.log",
  "kubernetes": {
    "container_id": "cri-o://17b1bcd3f1f0a27f4f31cdd62861f8f34a0dcf65df40abd28ced7dedbc7d26c7",
    "container_image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:104105f5abbd3cdc28303d8dd97dad3f05b298bdbc3fc48ca7aa66f34b59329b",
    "container_name": "kube-apiserver",
    "pod_ip": "192.168.126.11",
    "pod_ips": [
      "192.168.126.11"
    ],
    "pod_labels": {
      "apiserver": "true",
      "app": "openshift-kube-apiserver",
      "revision": "8"
    },
    "pod_name": "kube-apiserver-crc-dzk9v-master-0",
    "pod_namespace": "openshift-kube-apiserver",
    "pod_node_name": "crc-dzk9v-master-0",
    "pod_uid": "d445e1ff-4ea4-4387-9e31-efc6cbe8e389"
  },
  "level": "info",
  "message": "I1027 06:36:35.574151      19 apiaccess_count_controller.go:147] finished updating top build.openshift.io/v1, Resource=buildconfigs APIRequest counts",
  "pipeline_metadata": {
    "collector": {
      "error": null,
      "ipaddr4": "192.168.126.11",
      "name": "vector",
      "received_at": "2021-10-29T20:32:58.644812416+00:00",
      "version": "0.14.1"
    }
  },
  "source_type": "kubernetes_logs",
  "stream": "stderr",
  "timestamp": "2021-10-27T06:36:35.574216207Z"
}

the test case fails because test case framework needs to be updated to parse the vector log data model, which is still evolving

vimalk78 · 2021-10-30T08:08:42Z

/hold cancel

vimalk78 · 2021-10-30T14:23:32Z

/retest

jcantrill · 2021-11-01T17:20:20Z

internal/generator/vector/conf_test.go

+	DescribeTable("Generate full vector.toml", f,
+		Entry("with complex spec", generator.ConfGenerateTest{
+			CLSpec: logging.ClusterLoggingSpec{
+				Forwarder: &logging.ForwarderSpec{


Maybe a cleanup item but this is not relevant to vector

jcantrill · 2021-11-01T17:21:26Z

internal/generator/vector/conf_test.go

+  .level = level
+
+  .pipeline_metadata.collector.name = "vector"
+  .pipeline_metadata.collector.version = "0.14.1"


In future we probably should grab this from an ENV since it is likely to change

jcantrill · 2021-11-02T16:03:49Z

/hold

for 5.3 FF

jcantrill · 2021-11-09T16:46:26Z

/lgtm

jcantrill · 2021-11-09T17:12:31Z

/hold cancel

LOG-1491: Generate vector config

openshift-ci bot requested review from ajaygupta978 and jcantrill October 29, 2021 17:17

openshift-ci bot added do-not-merge/hold Indicates that a PR should not merge because someone has issued a /hold command. approved Indicates a PR has been approved by an approver from all required OWNERS files. labels Oct 29, 2021

vimalk78 force-pushed the log-1491-gen-vector-conf branch from 26c4e87 to 6b42095 Compare October 29, 2021 17:27

jcantrill reviewed Oct 29, 2021

View reviewed changes

internal/generator/vector/conf_test.go Outdated Show resolved Hide resolved

internal/generator/vector/output/kafka/kafka_test.go Outdated Show resolved Hide resolved

vimalk78 force-pushed the log-1491-gen-vector-conf branch 2 times, most recently from 18d8d50 to 8bee3cc Compare October 29, 2021 18:29

ajaygupta978 reviewed Oct 29, 2021

View reviewed changes

vimalk78 force-pushed the log-1491-gen-vector-conf branch from 8bee3cc to 984c997 Compare October 29, 2021 18:44

vimalk78 force-pushed the log-1491-gen-vector-conf branch from 984c997 to 9efa752 Compare October 29, 2021 18:49

openshift-ci bot removed the do-not-merge/hold Indicates that a PR should not merge because someone has issued a /hold command. label Oct 30, 2021

jcantrill reviewed Nov 1, 2021

View reviewed changes

openshift-ci bot added the do-not-merge/hold Indicates that a PR should not merge because someone has issued a /hold command. label Nov 2, 2021

openshift-ci bot assigned jcantrill Nov 9, 2021

openshift-ci bot added the lgtm Indicates that a PR is ready to be merged. label Nov 9, 2021

openshift-ci bot removed the do-not-merge/hold Indicates that a PR should not merge because someone has issued a /hold command. label Nov 9, 2021

openshift-merge-robot merged commit 37d049c into openshift:master Nov 9, 2021

ajaygupta978 mentioned this pull request Dec 7, 2021

WIP: LOG-1491:Vector config generator #1211

Closed

pmoogi-redhat pushed a commit to pmoogi-redhat/cluster-logging-operator that referenced this pull request Apr 26, 2022

Merge pull request openshift#1230 from vimalk78/log-1491-gen-vector-conf

66153ad

LOG-1491: Generate vector config

LOG-1491: Generate vector config #1230

LOG-1491: Generate vector config #1230

Uh oh!

Conversation

vimalk78 commented Oct 29, 2021 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Description

Links

Uh oh!

vimalk78 commented Oct 29, 2021

Uh oh!

jcantrill left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

openshift-ci bot commented Oct 29, 2021

Uh oh!

ajaygupta978 Oct 29, 2021

Choose a reason for hiding this comment

Uh oh!

vimalk78 Oct 29, 2021

Choose a reason for hiding this comment

Uh oh!

ajaygupta978 Oct 29, 2021

Choose a reason for hiding this comment

Uh oh!

vimalk78 Oct 29, 2021 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

vimalk78 commented Oct 29, 2021

Uh oh!

vimalk78 commented Oct 30, 2021

Uh oh!

vimalk78 commented Oct 30, 2021

Uh oh!

jcantrill Nov 1, 2021

Choose a reason for hiding this comment

Uh oh!

jcantrill Nov 1, 2021

Choose a reason for hiding this comment

Uh oh!

jcantrill commented Nov 2, 2021

Uh oh!

jcantrill commented Nov 9, 2021

Uh oh!

jcantrill commented Nov 9, 2021

Uh oh!

Uh oh!

vimalk78 commented Oct 29, 2021 •

edited

Loading

vimalk78 Oct 29, 2021 •

edited

Loading