This document was auto generated by benthos_config_gen.
The environment variables config default.yaml is an
auto generated Benthos configuration where all fields can be set with
environment variables. The architecture of the config is a standard bridge
between N replicated sources, M replicated sinks and an optional buffer and
processing pipeline between them.
The original intent of this config is to be deployed within a docker image, but as it is a standard config it can be used in other deployments.
In order to use this config simply define your env vars and point Benthos to it. For example, to send Kafka data to RabbitMQ you can run:
INPUT_TYPE=kafka_balanced \
INPUT_KAFKA_ADDRESSES=localhost:9092 \
INPUT_KAFKA_TOPIC=foo-topic \
INPUT_KAFKA_CONSUMER_GROUP=foo-consumer \
OUTPUT_TYPE=amqp \
OUTPUT_AMQP_URL=amqp://guest:guest@localhost:5672/ \
OUTPUT_AMQP_EXCHANGE=foo-exchange \
OUTPUT_AMQP_EXCHANGE_TYPE=direct \
benthos -c ./config/env/default.yamlAll variables within the config are listed in this document.
HTTP_ADDRESS = 0.0.0.0:4195
HTTP_DEBUG_ENDPOINTS = false
HTTP_READ_TIMEOUT_MS = 5000
HTTP_ROOT_PATH = /benthos
INPUTS = 1
INPUT_TYPE = dynamic
INPUT_AMQP_CONSUMER_TAG = benthos-consumer
INPUT_AMQP_PREFETCH_COUNT = 10
INPUT_AMQP_PREFETCH_SIZE = 0
INPUT_AMQP_QUEUE = benthos-queue
INPUT_AMQP_QUEUE_DECLARE_DURABLE = true
INPUT_AMQP_QUEUE_DECLARE_ENABLED = false
INPUT_AMQP_TLS_CAS_FILE
INPUT_AMQP_TLS_ENABLED = false
INPUT_AMQP_TLS_SKIP_CERT_VERIFY = false
INPUT_AMQP_URL = amqp://guest:guest@localhost:5672/
INPUT_DYNAMIC_PREFIX
INPUT_DYNAMIC_TIMEOUT_MS = 5000
INPUT_FILES_PATH
INPUT_FILE_DELIMITER
INPUT_FILE_MAX_BUFFER = 1000000
INPUT_FILE_MULTIPART = false
INPUT_FILE_PATH
INPUT_HTTP_CLIENT_BACKOFF_ON = 429
INPUT_HTTP_CLIENT_BASIC_AUTH_ENABLED = false
INPUT_HTTP_CLIENT_BASIC_AUTH_PASSWORD
INPUT_HTTP_CLIENT_BASIC_AUTH_USERNAME
INPUT_HTTP_CLIENT_HEADERS_CONTENT_TYPE = application/octet-stream
INPUT_HTTP_CLIENT_MAX_RETRY_BACKOFF_MS = 300000
INPUT_HTTP_CLIENT_OAUTH_ACCESS_TOKEN
INPUT_HTTP_CLIENT_OAUTH_ACCESS_TOKEN_SECRET
INPUT_HTTP_CLIENT_OAUTH_CONSUMER_KEY
INPUT_HTTP_CLIENT_OAUTH_CONSUMER_SECRET
INPUT_HTTP_CLIENT_OAUTH_ENABLED = false
INPUT_HTTP_CLIENT_OAUTH_REQUEST_URL
INPUT_HTTP_CLIENT_PAYLOAD
INPUT_HTTP_CLIENT_RETRIES = 3
INPUT_HTTP_CLIENT_RETRY_PERIOD_MS = 1000
INPUT_HTTP_CLIENT_STREAM_DELIMITER
INPUT_HTTP_CLIENT_STREAM_ENABLED = false
INPUT_HTTP_CLIENT_STREAM_MAX_BUFFER = 1000000
INPUT_HTTP_CLIENT_STREAM_MULTIPART = false
INPUT_HTTP_CLIENT_STREAM_RECONNECT = true
INPUT_HTTP_CLIENT_TIMEOUT_MS = 5000
INPUT_HTTP_CLIENT_TLS_CAS_FILE
INPUT_HTTP_CLIENT_TLS_ENABLED = false
INPUT_HTTP_CLIENT_TLS_SKIP_CERT_VERIFY = false
INPUT_HTTP_CLIENT_URL = http://localhost:4195/get
INPUT_HTTP_CLIENT_VERB = GET
INPUT_HTTP_SERVER_ADDRESS
INPUT_HTTP_SERVER_CERT_FILE
INPUT_HTTP_SERVER_KEY_FILE
INPUT_HTTP_SERVER_PATH = /post
INPUT_HTTP_SERVER_TIMEOUT_MS = 5000
INPUT_HTTP_SERVER_WS_PATH = /post/ws
INPUT_INPROC
INPUT_KAFKA_ADDRESSES = localhost:9092
INPUT_KAFKA_BALANCED_ADDRESSES = localhost:9092
INPUT_KAFKA_BALANCED_CLIENT_ID = benthos_kafka_input
INPUT_KAFKA_BALANCED_COMMIT_PERIOD_MS = 1000
INPUT_KAFKA_BALANCED_CONSUMER_GROUP = benthos_consumer_group
INPUT_KAFKA_BALANCED_START_FROM_OLDEST = true
INPUT_KAFKA_BALANCED_TLS_CAS_FILE
INPUT_KAFKA_BALANCED_TLS_ENABLED = false
INPUT_KAFKA_BALANCED_TLS_SKIP_CERT_VERIFY = false
INPUT_KAFKA_BALANCED_TOPICS = benthos_stream
INPUT_KAFKA_CLIENT_ID = benthos_kafka_input
INPUT_KAFKA_COMMIT_PERIOD_MS = 1000
INPUT_KAFKA_CONSUMER_GROUP = benthos_consumer_group
INPUT_KAFKA_PARTITION = 0
INPUT_KAFKA_START_FROM_OLDEST = true
INPUT_KAFKA_TARGET_VERSION = 1.0.0
INPUT_KAFKA_TLS_CAS_FILE
INPUT_KAFKA_TLS_ENABLED = false
INPUT_KAFKA_TLS_SKIP_CERT_VERIFY = false
INPUT_KAFKA_TOPIC = benthos_stream
INPUT_MQTT_CLIENT_ID = benthos_input
INPUT_MQTT_QOS = 1
INPUT_MQTT_TOPICS = benthos_topic
INPUT_MQTT_URLS = tcp://localhost:1883
INPUT_NANOMSG_BIND = true
INPUT_NANOMSG_POLL_TIMEOUT_MS = 5000
INPUT_NANOMSG_REPLY_TIMEOUT_MS = 5000
INPUT_NANOMSG_SOCKET_TYPE = PULL
INPUT_NANOMSG_URLS = tcp://*:5555
INPUT_NATS_STREAM_CLIENT_ID = benthos_client
INPUT_NATS_STREAM_CLUSTER_ID = test-cluster
INPUT_NATS_STREAM_DURABLE_NAME = benthos_offset
INPUT_NATS_STREAM_QUEUE = benthos_queue
INPUT_NATS_STREAM_START_FROM_OLDEST = true
INPUT_NATS_STREAM_SUBJECT = benthos_messages
INPUT_NATS_STREAM_URLS = nats://localhost:4222
INPUT_NATS_SUBJECT = benthos_messages
INPUT_NATS_URLS = nats://localhost:4222
INPUT_NSQ_CHANNEL = benthos_stream
INPUT_NSQ_LOOKUPD_HTTP_ADDRESSES = localhost:4161
INPUT_NSQ_MAX_IN_FLIGHT = 100
INPUT_NSQ_NSQD_TCP_ADDRESSES = localhost:4150
INPUT_NSQ_TOPIC = benthos_messages
INPUT_NSQ_USER_AGENT = benthos_consumer
INPUT_REDIS_LIST_KEY = benthos_list
INPUT_REDIS_LIST_TIMEOUT_MS = 5000
INPUT_REDIS_LIST_URL = tcp://localhost:6379
INPUT_REDIS_PUBSUB_CHANNELS = benthos_chan
INPUT_REDIS_PUBSUB_URL = tcp://localhost:6379
INPUT_S3_BUCKET
INPUT_S3_CREDENTIALS_ID
INPUT_S3_CREDENTIALS_ROLE
INPUT_S3_CREDENTIALS_SECRET
INPUT_S3_CREDENTIALS_TOKEN
INPUT_S3_DELETE_OBJECTS = false
INPUT_S3_PREFIX
INPUT_S3_REGION = eu-west-1
INPUT_S3_RETRIES = 3
INPUT_S3_SQS_BODY_PATH = Records.s3.object.key
INPUT_S3_SQS_ENVELOPE_PATH
INPUT_S3_SQS_MAX_MESSAGES = 10
INPUT_S3_SQS_URL
INPUT_S3_TIMEOUT_S = 5
INPUT_SQS_CREDENTIALS_ID
INPUT_SQS_CREDENTIALS_ROLE
INPUT_SQS_CREDENTIALS_SECRET
INPUT_SQS_CREDENTIALS_TOKEN
INPUT_SQS_REGION = eu-west-1
INPUT_SQS_TIMEOUT_S = 5
INPUT_SQS_URL
INPUT_STDIN_DELIMITER
INPUT_STDIN_MAX_BUFFER = 1000000
INPUT_STDIN_MULTIPART = false
INPUT_WEBSOCKET_BASIC_AUTH_ENABLED = false
INPUT_WEBSOCKET_BASIC_AUTH_PASSWORD
INPUT_WEBSOCKET_BASIC_AUTH_USERNAME
INPUT_WEBSOCKET_OAUTH_ACCESS_TOKEN
INPUT_WEBSOCKET_OAUTH_ACCESS_TOKEN_SECRET
INPUT_WEBSOCKET_OAUTH_CONSUMER_KEY
INPUT_WEBSOCKET_OAUTH_CONSUMER_SECRET
INPUT_WEBSOCKET_OAUTH_ENABLED = false
INPUT_WEBSOCKET_OAUTH_REQUEST_URL
INPUT_WEBSOCKET_OPEN_MESSAGE
INPUT_WEBSOCKET_URL = ws://localhost:4195/get/ws
BUFFER_TYPE = none
BUFFER_MEMORY_LIMIT = 524288000
BUFFER_MMAP_FILE_CLEAN_UP = true
BUFFER_MMAP_FILE_DIRECTORY
BUFFER_MMAP_FILE_FILE_SIZE = 262144000
BUFFER_MMAP_FILE_RESERVED_DISK_SPACE = 104857600
BUFFER_MMAP_FILE_RETRY_PERIOD_MS = 1000
PROCESSOR_THREADS = 1
PROCESSOR_TYPE = noop
PROCESSOR_ARCHIVE_FORMAT = binary
PROCESSOR_ARCHIVE_PATH = ${!count:files}-${!timestamp_unix_nano}.txt
PROCESSOR_BATCH_BYTE_SIZE = 10000
PROCESSOR_BATCH_CONDITION_COUNT_ARG = 100
PROCESSOR_BATCH_CONDITION_JMESPATH_PART = 0
PROCESSOR_BATCH_CONDITION_JMESPATH_QUERY
PROCESSOR_BATCH_CONDITION_METADATA_ARG
PROCESSOR_BATCH_CONDITION_METADATA_KEY
PROCESSOR_BATCH_CONDITION_METADATA_OPERATOR = equals_cs
PROCESSOR_BATCH_CONDITION_METADATA_PART = 0
PROCESSOR_BATCH_CONDITION_RESOURCE
PROCESSOR_BATCH_CONDITION_STATIC = false
PROCESSOR_BATCH_CONDITION_TEXT_ARG
PROCESSOR_BATCH_CONDITION_TEXT_OPERATOR = equals_cs
PROCESSOR_BATCH_CONDITION_TEXT_PART = 0
PROCESSOR_BATCH_CONDITION_TYPE = static
PROCESSOR_BATCH_PERIOD_MS = 0
PROCESSOR_BOUNDS_CHECK_MAX_PARTS = 100
PROCESSOR_BOUNDS_CHECK_MAX_PART_SIZE = 1073741824
PROCESSOR_BOUNDS_CHECK_MIN_PARTS = 1
PROCESSOR_BOUNDS_CHECK_MIN_PART_SIZE = 1
PROCESSOR_COMBINE_PARTS = 2
PROCESSOR_COMPRESS_ALGORITHM = gzip
PROCESSOR_COMPRESS_LEVEL = -1
PROCESSOR_DECODE_SCHEME = base64
PROCESSOR_DECOMPRESS_ALGORITHM = gzip
PROCESSOR_ENCODE_SCHEME = base64
PROCESSOR_GROK_NAMED_CAPTURES_ONLY = true
PROCESSOR_GROK_OUTPUT_FORMAT = json
PROCESSOR_GROK_REMOVE_EMPTY_VALUES = true
PROCESSOR_GROK_USE_DEFAULT_PATTERNS = true
PROCESSOR_HASH_SAMPLE_PARTS = 0
PROCESSOR_HASH_SAMPLE_RETAIN_MAX = 10
PROCESSOR_HASH_SAMPLE_RETAIN_MIN = 0
PROCESSOR_HTTP_MAX_PARALLEL = 0
PROCESSOR_HTTP_PARALLEL = false
PROCESSOR_HTTP_REQUEST_BACKOFF_ON = 429
PROCESSOR_HTTP_REQUEST_BASIC_AUTH_ENABLED = false
PROCESSOR_HTTP_REQUEST_BASIC_AUTH_PASSWORD
PROCESSOR_HTTP_REQUEST_BASIC_AUTH_USERNAME
PROCESSOR_HTTP_REQUEST_HEADERS_CONTENT_TYPE = application/octet-stream
PROCESSOR_HTTP_REQUEST_MAX_RETRY_BACKOFF_MS = 300000
PROCESSOR_HTTP_REQUEST_OAUTH_ACCESS_TOKEN
PROCESSOR_HTTP_REQUEST_OAUTH_ACCESS_TOKEN_SECRET
PROCESSOR_HTTP_REQUEST_OAUTH_CONSUMER_KEY
PROCESSOR_HTTP_REQUEST_OAUTH_CONSUMER_SECRET
PROCESSOR_HTTP_REQUEST_OAUTH_ENABLED = false
PROCESSOR_HTTP_REQUEST_OAUTH_REQUEST_URL
PROCESSOR_HTTP_REQUEST_RETRIES = 3
PROCESSOR_HTTP_REQUEST_RETRY_PERIOD_MS = 1000
PROCESSOR_HTTP_REQUEST_TIMEOUT_MS = 5000
PROCESSOR_HTTP_REQUEST_TLS_CAS_FILE
PROCESSOR_HTTP_REQUEST_TLS_ENABLED = false
PROCESSOR_HTTP_REQUEST_TLS_SKIP_CERT_VERIFY = false
PROCESSOR_HTTP_REQUEST_URL = http://localhost:4195/post
PROCESSOR_HTTP_REQUEST_VERB = POST
PROCESSOR_INSERT_PART_CONTENT
PROCESSOR_INSERT_PART_INDEX = -1
PROCESSOR_JMESPATH_QUERY
PROCESSOR_JSON_OPERATOR = get
PROCESSOR_JSON_PATH
PROCESSOR_JSON_VALUE
PROCESSOR_MERGE_JSON_RETAIN_PARTS = false
PROCESSOR_METADATA_KEY = example
PROCESSOR_METADATA_OPERATOR = set
PROCESSOR_METADATA_VALUE = ${!hostname}
PROCESSOR_SAMPLE_RETAIN = 10
PROCESSOR_SAMPLE_SEED = 0
PROCESSOR_SELECT_PARTS_PARTS = 0
PROCESSOR_SPLIT_SIZE = 1
PROCESSOR_TEXT_ARG
PROCESSOR_TEXT_OPERATOR = trim_space
PROCESSOR_TEXT_VALUE
PROCESSOR_THROTTLE_PERIOD = 100us
PROCESSOR_UNARCHIVE_FORMAT = binary
OUTPUTS = 1
OUTPUTS_PATTERN = greedy
OUTPUT_TYPE = dynamic
OUTPUT_AMQP_EXCHANGE = benthos-exchange
OUTPUT_AMQP_EXCHANGE_DECLARE_DURABLE = true
OUTPUT_AMQP_EXCHANGE_DECLARE_ENABLED = false
OUTPUT_AMQP_EXCHANGE_DECLARE_TYPE = direct
OUTPUT_AMQP_IMMEDIATE = false
OUTPUT_AMQP_KEY = benthos-key
OUTPUT_AMQP_MANDATORY = false
OUTPUT_AMQP_PERSISTENT = false
OUTPUT_AMQP_TLS_CAS_FILE
OUTPUT_AMQP_TLS_ENABLED = false
OUTPUT_AMQP_TLS_SKIP_CERT_VERIFY = false
OUTPUT_AMQP_URL = amqp://guest:guest@localhost:5672/
OUTPUT_DYNAMIC_PREFIX
OUTPUT_DYNAMIC_TIMEOUT_MS = 5000
OUTPUT_ELASTICSEARCH_BASIC_AUTH_ENABLED = false
OUTPUT_ELASTICSEARCH_BASIC_AUTH_PASSWORD
OUTPUT_ELASTICSEARCH_BASIC_AUTH_USERNAME
OUTPUT_ELASTICSEARCH_ID = ${!count:elastic_ids}-${!timestamp_unix}
OUTPUT_ELASTICSEARCH_INDEX = benthos_index
OUTPUT_ELASTICSEARCH_TIMEOUT_MS = 5000
OUTPUT_ELASTICSEARCH_TYPE = doc
OUTPUT_ELASTICSEARCH_URLS = http://localhost:9200
OUTPUT_FILES_PATH = ${!count:files}-${!timestamp_unix_nano}.txt
OUTPUT_FILE_DELIMITER
OUTPUT_FILE_PATH
OUTPUT_HTTP_CLIENT_BACKOFF_ON = 429
OUTPUT_HTTP_CLIENT_BASIC_AUTH_ENABLED = false
OUTPUT_HTTP_CLIENT_BASIC_AUTH_PASSWORD
OUTPUT_HTTP_CLIENT_BASIC_AUTH_USERNAME
OUTPUT_HTTP_CLIENT_HEADERS_CONTENT_TYPE = application/octet-stream
OUTPUT_HTTP_CLIENT_MAX_RETRY_BACKOFF_MS = 300000
OUTPUT_HTTP_CLIENT_OAUTH_ACCESS_TOKEN
OUTPUT_HTTP_CLIENT_OAUTH_ACCESS_TOKEN_SECRET
OUTPUT_HTTP_CLIENT_OAUTH_CONSUMER_KEY
OUTPUT_HTTP_CLIENT_OAUTH_CONSUMER_SECRET
OUTPUT_HTTP_CLIENT_OAUTH_ENABLED = false
OUTPUT_HTTP_CLIENT_OAUTH_REQUEST_URL
OUTPUT_HTTP_CLIENT_RETRIES = 3
OUTPUT_HTTP_CLIENT_RETRY_PERIOD_MS = 1000
OUTPUT_HTTP_CLIENT_TIMEOUT_MS = 5000
OUTPUT_HTTP_CLIENT_TLS_CAS_FILE
OUTPUT_HTTP_CLIENT_TLS_ENABLED = false
OUTPUT_HTTP_CLIENT_TLS_SKIP_CERT_VERIFY = false
OUTPUT_HTTP_CLIENT_URL = http://localhost:4195/post
OUTPUT_HTTP_CLIENT_VERB = POST
OUTPUT_HTTP_SERVER_ADDRESS
OUTPUT_HTTP_SERVER_CERT_FILE
OUTPUT_HTTP_SERVER_KEY_FILE
OUTPUT_HTTP_SERVER_PATH = /get
OUTPUT_HTTP_SERVER_STREAM_PATH = /get/stream
OUTPUT_HTTP_SERVER_TIMEOUT_MS = 5000
OUTPUT_HTTP_SERVER_WS_PATH = /get/ws
OUTPUT_INPROC
OUTPUT_KAFKA_ACK_REPLICAS = false
OUTPUT_KAFKA_ADDRESSES = localhost:9092
OUTPUT_KAFKA_CLIENT_ID = benthos_kafka_output
OUTPUT_KAFKA_COMPRESSION = none
OUTPUT_KAFKA_KEY
OUTPUT_KAFKA_MAX_MSG_BYTES = 1000000
OUTPUT_KAFKA_ROUND_ROBIN_PARTITIONS = false
OUTPUT_KAFKA_TARGET_VERSION = 1.0.0
OUTPUT_KAFKA_TIMEOUT_MS = 5000
OUTPUT_KAFKA_TLS_CAS_FILE
OUTPUT_KAFKA_TLS_ENABLED = false
OUTPUT_KAFKA_TLS_SKIP_CERT_VERIFY = false
OUTPUT_KAFKA_TOPIC = benthos_stream
OUTPUT_MQTT_CLIENT_ID = benthos_output
OUTPUT_MQTT_QOS = 1
OUTPUT_MQTT_TOPIC = benthos_topic
OUTPUT_MQTT_URLS = tcp://localhost:1883
OUTPUT_NANOMSG_BIND = false
OUTPUT_NANOMSG_POLL_TIMEOUT_MS = 5000
OUTPUT_NANOMSG_SOCKET_TYPE = PUSH
OUTPUT_NANOMSG_URLS = tcp://localhost:5556
OUTPUT_NATS_STREAM_CLIENT_ID = benthos_client
OUTPUT_NATS_STREAM_CLUSTER_ID = test-cluster
OUTPUT_NATS_STREAM_SUBJECT = benthos_messages
OUTPUT_NATS_STREAM_URLS = nats://localhost:4222
OUTPUT_NATS_SUBJECT = benthos_messages
OUTPUT_NATS_URLS = nats://localhost:4222
OUTPUT_NSQ_MAX_IN_FLIGHT = 100
OUTPUT_NSQ_NSQD_TCP_ADDRESS = localhost:4150
OUTPUT_NSQ_TOPIC = benthos_messages
OUTPUT_NSQ_USER_AGENT = benthos_producer
OUTPUT_REDIS_LIST_KEY = benthos_list
OUTPUT_REDIS_LIST_URL = tcp://localhost:6379
OUTPUT_REDIS_PUBSUB_CHANNEL = benthos_chan
OUTPUT_REDIS_PUBSUB_URL = tcp://localhost:6379
OUTPUT_S3_BUCKET
OUTPUT_S3_CREDENTIALS_ID
OUTPUT_S3_CREDENTIALS_ROLE
OUTPUT_S3_CREDENTIALS_SECRET
OUTPUT_S3_CREDENTIALS_TOKEN
OUTPUT_S3_PATH = ${!count:files}-${!timestamp_unix_nano}.txt
OUTPUT_S3_REGION = eu-west-1
OUTPUT_S3_TIMEOUT_S = 5
OUTPUT_SQS_CREDENTIALS_ID
OUTPUT_SQS_CREDENTIALS_ROLE
OUTPUT_SQS_CREDENTIALS_SECRET
OUTPUT_SQS_CREDENTIALS_TOKEN
OUTPUT_SQS_REGION = eu-west-1
OUTPUT_SQS_URL
OUTPUT_STDOUT_DELIMITER
OUTPUT_WEBSOCKET_BASIC_AUTH_ENABLED = false
OUTPUT_WEBSOCKET_BASIC_AUTH_PASSWORD
OUTPUT_WEBSOCKET_BASIC_AUTH_USERNAME
OUTPUT_WEBSOCKET_OAUTH_ACCESS_TOKEN
OUTPUT_WEBSOCKET_OAUTH_ACCESS_TOKEN_SECRET
OUTPUT_WEBSOCKET_OAUTH_CONSUMER_KEY
OUTPUT_WEBSOCKET_OAUTH_CONSUMER_SECRET
OUTPUT_WEBSOCKET_OAUTH_ENABLED = false
OUTPUT_WEBSOCKET_OAUTH_REQUEST_URL
OUTPUT_WEBSOCKET_URL = ws://localhost:4195/post/ws
LOGGER_ADD_TIMESTAMP = true
LOGGER_JSON_FORMAT = true
LOGGER_LEVEL = INFO
LOGGER_PREFIX = benthos
METRICS_TYPE = http_server
METRICS_PREFIX = benthos
METRICS_STATSD_ADDRESS = localhost:4040
METRICS_STATSD_FLUSH_PERIOD = 100ms
METRICS_STATSD_NETWORK = udp