reference-config.yaml

#####################################################################################
# This file documents all the available config options and it's default values.
#
# All config options can be configured via environment variables as well.
# If you specify both the env variable and yaml option for the same configuration
# the environment variable will take precedence. If you want to use a YAML config
# file, specify the path to the config file by setting the env variable
# CONFIG_FILEPATH.
#
# The env variable name is auto generated by upper casing everything and adding
# an underscore for each indentation/level. Some examples:
# kafka.rackId => KAFKA_RACKID
# kafka.tls.caFilepath => KAFKA_TLS_CAFILEPATH
# minion.consumerGroups.allowedGroups => MINION_CONSUMERGROUPS_ALLOWEDGROUPS
#
# Env variables that expect array values can be provided by separting them using
# a comma: KAFKA_BROKERS = "broker1:9092,broker2:9092,broker3:9092"
#####################################################################################

logger:
  # Valid values are: debug, info, warn, error, fatal, panic
  level: info

kafka:
  brokers: [ ]
  clientId: "kminion"
  rackId: ""
  tls:
    enabled: false
    caFilepath: ""
    certFilepath: ""
    keyFilepath: ""
    # base64 encoded tls CA, cannot be set if 'caFilepath' is set
    ca: ""
    # base64 encoded tls cert, cannot be set if 'certFilepath' is set
    cert: ""
    # base64 encoded tls key, cannot be set if 'keyFilepath' is set
    key: ""
    passphrase: ""
    insecureSkipTlsVerify: false

  sasl:
    # Whether or not SASL authentication will be used for authentication
    enabled: false
    # Username to use for PLAIN or SCRAM mechanism
    username: ""
    # Password to use for PLAIN or SCRAM mechanism
    password: ""
    # Mechanism to use for SASL Authentication. Valid values are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, GSSAPI, OAUTHBEARER
    mechanism: "PLAIN"
    # GSSAPI / Kerberos config properties
    gssapi:
      authType: ""
      keyTabPath: ""
      kerberosConfigPath: ""
      serviceName: ""
      username: ""
      password: ""
      realm: ""
      enableFast: true
    # OAUTHBEARER config properties
    oauth:
      tokenEndpoint: ""
      clientId: ""
      clientSecret: ""
      scope: ""

minion:
  consumerGroups:
    # Enabled specifies whether consumer groups shall be scraped and exported or not.
    enabled: true
    # Mode specifies whether we export consumer group offsets using the Admin API or by consuming the internal
    # __consumer_offsets topic. Both modes have their advantages and disadvantages.
    # * adminApi:
    #     - Useful for managed kafka clusters that do not provide access to the offsets topic.
    # * offsetsTopic
    #     - Enables kminion_kafka_consumer_group_offset_commits_total metrics.
    #     - Processing the offsetsTopic requires slightly more memory and cpu than using the adminApi. The amount depends on the
    #       size and throughput of the offsets topic.
    scrapeMode: adminApi # Valid values: adminApi, offsetsTopic
    # Granularity can be per topic or per partition. If you want to reduce the number of exported metric series and
    # you aren't interested in per partition lags you could choose "topic" where all partition lags will be summed
    # and only topic lags will be exported.
    granularity: partition
    # AllowedGroups are regex strings of group ids that shall be exported
    # You can specify allowed groups by providing literals like "my-consumergroup-name" or by providing regex expressions
    # like "/internal-.*/".
    allowedGroups: [ ".*" ]
    # IgnoredGroups are regex strings of group ids that shall be ignored/skipped when exporting metrics. Ignored groups
    # take precedence over allowed groups.
    ignoredGroups: [ ]
  topics:
    # Enabled can be set to false in order to disable collecting any topic metrics.
    enabled: true
    # Granularity can be per topic or per partition. If you want to reduce the number of exported metric series and
    # you aren't interested in per partition metrics you could choose "topic".
    granularity: partition
    # AllowedTopics are regex strings of topic names whose topic metrics that shall be exported.
    # You can specify allowed topics by providing literals like "my-topic-name" or by providing regex expressions
    # like "/internal-.*/".
    allowedTopics: [ ".*" ]
    # IgnoredTopics are regex strings of topic names that shall be ignored/skipped when exporting metrics. Ignored topics
    # take precedence over allowed topics.
    ignoredTopics: [ ]
    # infoMetric is a configuration object for the kminion_kafka_topic_info metric
    infoMetric:
      # ConfigKeys are set of strings of Topic configs that you want to have exported as part of the metric
      configKeys: [ "cleanup.policy" ]
  logDirs:
    # Enabled specifies whether log dirs shall be scraped and exported or not. This should be disabled for clusters prior
    # to version 1.0.0 as describing log dirs was not supported back then.
    enabled: true

  # EndToEnd Metrics
  # When enabled, kminion creates a topic which it produces to and consumes from, to measure various advanced metrics. See docs for more info
  endToEnd:
    enabled: false
    # How often to send end-to-end test messages
    probeInterval: 100ms
    topicManagement:
      # You can disable topic management, without disabling the testing feature.
      # Only makes sense if you have multiple kminion instances, and for some reason only want one of them to create/configure the topic
      enabled: true

      # Name of the topic kminion uses to send its test messages
      # You do *not* need to change this if you are running multiple kminion instances on the same cluster.
      # Different instances are perfectly fine with sharing the same topic!
      name: kminion-end-to-end

      # How often kminion checks its topic to validate configuration, partition count, and partition assignments
      reconciliationInterval: 10m

      # Depending on the desired monitoring (e.g. you want to alert on broker failure vs. cluster that is not writable)
      # you may choose replication factor 1 or 3 most commonly.
      replicationFactor: 1

      # Rarely makes sense to change this, but maybe if you want some sort of cheap load test?
      # By default (1) every broker gets one partition
      partitionsPerBroker: 1

    producer:
      # This defines:
      # - Maximum time to wait for an ack response after producing a message
      # - Upper bound for histogram buckets in "produce_latency_seconds"
      ackSla: 5s
      # Can be to "all" (default) so kafka only reports an end-to-end test message as acknowledged if
      # the message was written to all in-sync replicas of the partition.
      # Or can be set to "leader" to only require to have written the message to its log.
      requiredAcks: all

    consumer:
      # Prefix kminion uses when creating its consumer groups. Current kminion instance id will be appended automatically
      groupIdPrefix: kminion-end-to-end

      # Whether KMinion should try to delete empty consumer groups with the same prefix. This can be used if you want
      # KMinion to cleanup it's old consumer groups. It should only be used if you use a unique prefix for KMinion.
      deleteStaleConsumerGroups: false

      # This defines:
      # - Upper bound for histogram buckets in "roundtrip_latency"
      # - Time limit beyond which a message is considered "lost" (failed the roundtrip)
      roundtripSla: 20s

      # - Upper bound for histogram buckets in "commit_latency_seconds"
      # - Maximum time an offset commit is allowed to take before considering it failed
      commitSla: 10s

exporter:
  # Namespace is the prefix for all exported Prometheus metrics
  namespace: "kminion"
  # Host that shall be used to bind the HTTP server on
  host: ""
  # Port that shall be used to bind the HTTP server on
  port: 8080