forked from redpanda-data/connect
-
Notifications
You must be signed in to change notification settings - Fork 0
/
default.yaml
856 lines (856 loc) · 38.8 KB
/
default.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
# BENTHOS LINT DISABLE
# This file was auto generated by benthos_config_gen.
http:
address: ${HTTP_ADDRESS:0.0.0.0:4195}
debug_endpoints: ${HTTP_DEBUG_ENDPOINTS:false}
read_timeout: ${HTTP_READ_TIMEOUT:5s}
root_path: ${HTTP_ROOT_PATH:/benthos}
input:
broker:
copies: ${INPUTS:1}
inputs:
- amqp:
consumer_tag: ${INPUT_AMQP_CONSUMER_TAG:benthos-consumer}
max_batch_count: ${INPUT_AMQP_MAX_BATCH_COUNT:1}
prefetch_count: ${INPUT_AMQP_PREFETCH_COUNT:10}
prefetch_size: ${INPUT_AMQP_PREFETCH_SIZE:0}
queue: ${INPUT_AMQP_QUEUE:benthos-queue}
queue_declare:
durable: ${INPUT_AMQP_QUEUE_DECLARE_DURABLE:true}
enabled: ${INPUT_AMQP_QUEUE_DECLARE_ENABLED:false}
tls:
enabled: ${INPUT_AMQP_TLS_ENABLED:false}
root_cas_file: ${INPUT_AMQP_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${INPUT_AMQP_TLS_SKIP_CERT_VERIFY:false}
url: ${INPUT_AMQP_URL:amqp://guest:guest@localhost:5672/}
amqp_0_9:
batching:
byte_size: ${INPUT_AMQP_0_9_BATCHING_BYTE_SIZE:0}
count: ${INPUT_AMQP_0_9_BATCHING_COUNT:1}
period: ${INPUT_AMQP_0_9_BATCHING_PERIOD}
consumer_tag: ${INPUT_AMQP_0_9_CONSUMER_TAG:benthos-consumer}
prefetch_count: ${INPUT_AMQP_0_9_PREFETCH_COUNT:10}
prefetch_size: ${INPUT_AMQP_0_9_PREFETCH_SIZE:0}
queue: ${INPUT_AMQP_0_9_QUEUE:benthos-queue}
queue_declare:
durable: ${INPUT_AMQP_0_9_QUEUE_DECLARE_DURABLE:true}
enabled: ${INPUT_AMQP_0_9_QUEUE_DECLARE_ENABLED:false}
tls:
enabled: ${INPUT_AMQP_0_9_TLS_ENABLED:false}
root_cas_file: ${INPUT_AMQP_0_9_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${INPUT_AMQP_0_9_TLS_SKIP_CERT_VERIFY:false}
url: ${INPUT_AMQP_0_9_URL:amqp://guest:guest@localhost:5672/}
dynamic:
prefix: ${INPUT_DYNAMIC_PREFIX}
timeout: ${INPUT_DYNAMIC_TIMEOUT:5s}
file:
delimiter: ${INPUT_FILE_DELIMITER}
max_buffer: ${INPUT_FILE_MAX_BUFFER:1000000}
multipart: ${INPUT_FILE_MULTIPART:false}
path: ${INPUT_FILE_PATH}
files:
path: ${INPUT_FILES_PATH}
gcp_pubsub:
batching:
byte_size: ${INPUT_GCP_PUBSUB_BATCHING_BYTE_SIZE:0}
count: ${INPUT_GCP_PUBSUB_BATCHING_COUNT:1}
period: ${INPUT_GCP_PUBSUB_BATCHING_PERIOD}
max_batch_count: ${INPUT_GCP_PUBSUB_MAX_BATCH_COUNT:1}
max_outstanding_bytes: ${INPUT_GCP_PUBSUB_MAX_OUTSTANDING_BYTES:1000000000}
max_outstanding_messages: ${INPUT_GCP_PUBSUB_MAX_OUTSTANDING_MESSAGES:1000}
project: ${INPUT_GCP_PUBSUB_PROJECT}
subscription: ${INPUT_GCP_PUBSUB_SUBSCRIPTION}
hdfs:
directory: ${INPUT_HDFS_DIRECTORY}
hosts:
- ${INPUT_HDFS_HOSTS:localhost:9000}
user: ${INPUT_HDFS_USER:benthos_hdfs}
http_client:
backoff_on:
- ${INPUT_HTTP_CLIENT_BACKOFF_ON:429}
basic_auth:
enabled: ${INPUT_HTTP_CLIENT_BASIC_AUTH_ENABLED:false}
password: ${INPUT_HTTP_CLIENT_BASIC_AUTH_PASSWORD}
username: ${INPUT_HTTP_CLIENT_BASIC_AUTH_USERNAME}
copy_response_headers: ${INPUT_HTTP_CLIENT_COPY_RESPONSE_HEADERS:false}
headers:
Content-Type: ${INPUT_HTTP_CLIENT_HEADERS_CONTENT_TYPE:application/octet-stream}
max_retry_backoff: ${INPUT_HTTP_CLIENT_MAX_RETRY_BACKOFF:300s}
oauth:
access_token: ${INPUT_HTTP_CLIENT_OAUTH_ACCESS_TOKEN}
access_token_secret: ${INPUT_HTTP_CLIENT_OAUTH_ACCESS_TOKEN_SECRET}
consumer_key: ${INPUT_HTTP_CLIENT_OAUTH_CONSUMER_KEY}
consumer_secret: ${INPUT_HTTP_CLIENT_OAUTH_CONSUMER_SECRET}
enabled: ${INPUT_HTTP_CLIENT_OAUTH_ENABLED:false}
request_url: ${INPUT_HTTP_CLIENT_OAUTH_REQUEST_URL}
payload: ${INPUT_HTTP_CLIENT_PAYLOAD}
rate_limit: ${INPUT_HTTP_CLIENT_RATE_LIMIT}
retries: ${INPUT_HTTP_CLIENT_RETRIES:3}
retry_period: ${INPUT_HTTP_CLIENT_RETRY_PERIOD:1s}
stream:
delimiter: ${INPUT_HTTP_CLIENT_STREAM_DELIMITER}
enabled: ${INPUT_HTTP_CLIENT_STREAM_ENABLED:false}
max_buffer: ${INPUT_HTTP_CLIENT_STREAM_MAX_BUFFER:1000000}
multipart: ${INPUT_HTTP_CLIENT_STREAM_MULTIPART:false}
reconnect: ${INPUT_HTTP_CLIENT_STREAM_RECONNECT:true}
timeout: ${INPUT_HTTP_CLIENT_TIMEOUT:5s}
tls:
enabled: ${INPUT_HTTP_CLIENT_TLS_ENABLED:false}
root_cas_file: ${INPUT_HTTP_CLIENT_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${INPUT_HTTP_CLIENT_TLS_SKIP_CERT_VERIFY:false}
url: ${INPUT_HTTP_CLIENT_URL:http://localhost:4195/get}
verb: ${INPUT_HTTP_CLIENT_VERB:GET}
http_server:
address: ${INPUT_HTTP_SERVER_ADDRESS}
cert_file: ${INPUT_HTTP_SERVER_CERT_FILE}
key_file: ${INPUT_HTTP_SERVER_KEY_FILE}
path: ${INPUT_HTTP_SERVER_PATH:/post}
rate_limit: ${INPUT_HTTP_SERVER_RATE_LIMIT}
timeout: ${INPUT_HTTP_SERVER_TIMEOUT:5s}
ws_path: ${INPUT_HTTP_SERVER_WS_PATH:/post/ws}
ws_rate_limit_message: ${INPUT_HTTP_SERVER_WS_RATE_LIMIT_MESSAGE}
ws_welcome_message: ${INPUT_HTTP_SERVER_WS_WELCOME_MESSAGE}
inproc: ${INPUT_INPROC}
kafka:
addresses:
- ${INPUT_KAFKA_ADDRESSES:localhost:9092}
batching:
byte_size: ${INPUT_KAFKA_BATCHING_BYTE_SIZE:0}
count: ${INPUT_KAFKA_BATCHING_COUNT:1}
period: ${INPUT_KAFKA_BATCHING_PERIOD}
client_id: ${INPUT_KAFKA_CLIENT_ID:benthos_kafka_input}
commit_period: ${INPUT_KAFKA_COMMIT_PERIOD:1s}
consumer_group: ${INPUT_KAFKA_CONSUMER_GROUP:benthos_consumer_group}
fetch_buffer_cap: ${INPUT_KAFKA_FETCH_BUFFER_CAP:256}
max_batch_count: ${INPUT_KAFKA_MAX_BATCH_COUNT:1}
max_processing_period: ${INPUT_KAFKA_MAX_PROCESSING_PERIOD:100ms}
partition: ${INPUT_KAFKA_PARTITION:0}
sasl:
enabled: ${INPUT_KAFKA_SASL_ENABLED:false}
password: ${INPUT_KAFKA_SASL_PASSWORD}
user: ${INPUT_KAFKA_SASL_USER}
start_from_oldest: ${INPUT_KAFKA_START_FROM_OLDEST:true}
target_version: ${INPUT_KAFKA_TARGET_VERSION:1.0.0}
tls:
enabled: ${INPUT_KAFKA_TLS_ENABLED:false}
root_cas_file: ${INPUT_KAFKA_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${INPUT_KAFKA_TLS_SKIP_CERT_VERIFY:false}
topic: ${INPUT_KAFKA_TOPIC:benthos_stream}
kafka_balanced:
addresses:
- ${INPUT_KAFKA_BALANCED_ADDRESSES:localhost:9092}
batching:
byte_size: ${INPUT_KAFKA_BALANCED_BATCHING_BYTE_SIZE:0}
count: ${INPUT_KAFKA_BALANCED_BATCHING_COUNT:1}
period: ${INPUT_KAFKA_BALANCED_BATCHING_PERIOD}
client_id: ${INPUT_KAFKA_BALANCED_CLIENT_ID:benthos_kafka_input}
commit_period: ${INPUT_KAFKA_BALANCED_COMMIT_PERIOD:1s}
consumer_group: ${INPUT_KAFKA_BALANCED_CONSUMER_GROUP:benthos_consumer_group}
fetch_buffer_cap: ${INPUT_KAFKA_BALANCED_FETCH_BUFFER_CAP:256}
group:
heartbeat_interval: ${INPUT_KAFKA_BALANCED_GROUP_HEARTBEAT_INTERVAL:3s}
rebalance_timeout: ${INPUT_KAFKA_BALANCED_GROUP_REBALANCE_TIMEOUT:60s}
session_timeout: ${INPUT_KAFKA_BALANCED_GROUP_SESSION_TIMEOUT:10s}
max_batch_count: ${INPUT_KAFKA_BALANCED_MAX_BATCH_COUNT:1}
max_processing_period: ${INPUT_KAFKA_BALANCED_MAX_PROCESSING_PERIOD:100ms}
sasl:
enabled: ${INPUT_KAFKA_BALANCED_SASL_ENABLED:false}
password: ${INPUT_KAFKA_BALANCED_SASL_PASSWORD}
user: ${INPUT_KAFKA_BALANCED_SASL_USER}
start_from_oldest: ${INPUT_KAFKA_BALANCED_START_FROM_OLDEST:true}
target_version: ${INPUT_KAFKA_BALANCED_TARGET_VERSION:1.0.0}
tls:
enabled: ${INPUT_KAFKA_BALANCED_TLS_ENABLED:false}
root_cas_file: ${INPUT_KAFKA_BALANCED_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${INPUT_KAFKA_BALANCED_TLS_SKIP_CERT_VERIFY:false}
topics:
- ${INPUT_KAFKA_BALANCED_TOPICS:benthos_stream}
kinesis:
batching:
byte_size: ${INPUT_KINESIS_BATCHING_BYTE_SIZE:0}
count: ${INPUT_KINESIS_BATCHING_COUNT:1}
period: ${INPUT_KINESIS_BATCHING_PERIOD}
client_id: ${INPUT_KINESIS_CLIENT_ID:benthos_consumer}
commit_period: ${INPUT_KINESIS_COMMIT_PERIOD:1s}
credentials:
id: ${INPUT_KINESIS_CREDENTIALS_ID}
profile: ${INPUT_KINESIS_CREDENTIALS_PROFILE}
role: ${INPUT_KINESIS_CREDENTIALS_ROLE}
role_external_id: ${INPUT_KINESIS_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${INPUT_KINESIS_CREDENTIALS_SECRET}
token: ${INPUT_KINESIS_CREDENTIALS_TOKEN}
dynamodb_table: ${INPUT_KINESIS_DYNAMODB_TABLE}
endpoint: ${INPUT_KINESIS_ENDPOINT}
limit: ${INPUT_KINESIS_LIMIT:100}
region: ${INPUT_KINESIS_REGION:eu-west-1}
shard: ${INPUT_KINESIS_SHARD:0}
start_from_oldest: ${INPUT_KINESIS_START_FROM_OLDEST:true}
stream: ${INPUT_KINESIS_STREAM}
timeout: ${INPUT_KINESIS_TIMEOUT:5s}
kinesis_balanced:
batching:
byte_size: ${INPUT_KINESIS_BALANCED_BATCHING_BYTE_SIZE:0}
count: ${INPUT_KINESIS_BALANCED_BATCHING_COUNT:1}
period: ${INPUT_KINESIS_BALANCED_BATCHING_PERIOD}
credentials:
id: ${INPUT_KINESIS_BALANCED_CREDENTIALS_ID}
profile: ${INPUT_KINESIS_BALANCED_CREDENTIALS_PROFILE}
role: ${INPUT_KINESIS_BALANCED_CREDENTIALS_ROLE}
role_external_id: ${INPUT_KINESIS_BALANCED_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${INPUT_KINESIS_BALANCED_CREDENTIALS_SECRET}
token: ${INPUT_KINESIS_BALANCED_CREDENTIALS_TOKEN}
dynamodb_billing_mode: ${INPUT_KINESIS_BALANCED_DYNAMODB_BILLING_MODE}
dynamodb_read_provision: ${INPUT_KINESIS_BALANCED_DYNAMODB_READ_PROVISION:0}
dynamodb_table: ${INPUT_KINESIS_BALANCED_DYNAMODB_TABLE}
dynamodb_write_provision: ${INPUT_KINESIS_BALANCED_DYNAMODB_WRITE_PROVISION:0}
endpoint: ${INPUT_KINESIS_BALANCED_ENDPOINT}
max_batch_count: ${INPUT_KINESIS_BALANCED_MAX_BATCH_COUNT:1}
region: ${INPUT_KINESIS_BALANCED_REGION:eu-west-1}
start_from_oldest: ${INPUT_KINESIS_BALANCED_START_FROM_OLDEST:true}
stream: ${INPUT_KINESIS_BALANCED_STREAM}
mqtt:
clean_session: ${INPUT_MQTT_CLEAN_SESSION:true}
client_id: ${INPUT_MQTT_CLIENT_ID:benthos_input}
password: ${INPUT_MQTT_PASSWORD}
qos: ${INPUT_MQTT_QOS:1}
topics:
- ${INPUT_MQTT_TOPICS:benthos_topic}
urls:
- ${INPUT_MQTT_URLS:tcp://localhost:1883}
user: ${INPUT_MQTT_USER}
nanomsg:
bind: ${INPUT_NANOMSG_BIND:true}
poll_timeout: ${INPUT_NANOMSG_POLL_TIMEOUT:5s}
reply_timeout: ${INPUT_NANOMSG_REPLY_TIMEOUT:5s}
socket_type: ${INPUT_NANOMSG_SOCKET_TYPE:PULL}
urls:
- ${INPUT_NANOMSG_URLS:tcp://*:5555}
nats:
prefetch_count: ${INPUT_NATS_PREFETCH_COUNT:32}
queue: ${INPUT_NATS_QUEUE:benthos_queue}
subject: ${INPUT_NATS_SUBJECT:benthos_messages}
urls:
- ${INPUT_NATS_URLS:nats://127.0.0.1:4222}
nats_stream:
ack_wait: ${INPUT_NATS_STREAM_ACK_WAIT:30s}
batching:
byte_size: ${INPUT_NATS_STREAM_BATCHING_BYTE_SIZE:0}
count: ${INPUT_NATS_STREAM_BATCHING_COUNT:1}
period: ${INPUT_NATS_STREAM_BATCHING_PERIOD}
client_id: ${INPUT_NATS_STREAM_CLIENT_ID:benthos_client}
cluster_id: ${INPUT_NATS_STREAM_CLUSTER_ID:test-cluster}
durable_name: ${INPUT_NATS_STREAM_DURABLE_NAME:benthos_offset}
max_inflight: ${INPUT_NATS_STREAM_MAX_INFLIGHT:1024}
queue: ${INPUT_NATS_STREAM_QUEUE:benthos_queue}
start_from_oldest: ${INPUT_NATS_STREAM_START_FROM_OLDEST:true}
subject: ${INPUT_NATS_STREAM_SUBJECT:benthos_messages}
unsubscribe_on_close: ${INPUT_NATS_STREAM_UNSUBSCRIBE_ON_CLOSE:false}
urls:
- ${INPUT_NATS_STREAM_URLS:nats://localhost:4222}
nsq:
batching:
byte_size: ${INPUT_NSQ_BATCHING_BYTE_SIZE:0}
count: ${INPUT_NSQ_BATCHING_COUNT:1}
period: ${INPUT_NSQ_BATCHING_PERIOD}
channel: ${INPUT_NSQ_CHANNEL:benthos_stream}
lookupd_http_addresses:
- ${INPUT_NSQ_LOOKUPD_HTTP_ADDRESSES:localhost:4161}
max_in_flight: ${INPUT_NSQ_MAX_IN_FLIGHT:100}
nsqd_tcp_addresses:
- ${INPUT_NSQ_NSQD_TCP_ADDRESSES:localhost:4150}
topic: ${INPUT_NSQ_TOPIC:benthos_messages}
user_agent: ${INPUT_NSQ_USER_AGENT:benthos_consumer}
redis_list:
key: ${INPUT_REDIS_LIST_KEY:benthos_list}
timeout: ${INPUT_REDIS_LIST_TIMEOUT:5s}
url: ${INPUT_REDIS_LIST_URL:tcp://localhost:6379}
redis_pubsub:
channels:
- ${INPUT_REDIS_PUBSUB_CHANNELS:benthos_chan}
url: ${INPUT_REDIS_PUBSUB_URL:tcp://localhost:6379}
use_patterns: ${INPUT_REDIS_PUBSUB_USE_PATTERNS:false}
redis_streams:
batching:
byte_size: ${INPUT_REDIS_STREAMS_BATCHING_BYTE_SIZE:0}
count: ${INPUT_REDIS_STREAMS_BATCHING_COUNT:1}
period: ${INPUT_REDIS_STREAMS_BATCHING_PERIOD}
body_key: ${INPUT_REDIS_STREAMS_BODY_KEY:body}
client_id: ${INPUT_REDIS_STREAMS_CLIENT_ID:benthos_consumer}
commit_period: ${INPUT_REDIS_STREAMS_COMMIT_PERIOD:1s}
consumer_group: ${INPUT_REDIS_STREAMS_CONSUMER_GROUP:benthos_group}
limit: ${INPUT_REDIS_STREAMS_LIMIT:10}
start_from_oldest: ${INPUT_REDIS_STREAMS_START_FROM_OLDEST:true}
streams:
- ${INPUT_REDIS_STREAMS_STREAMS:benthos_stream}
timeout: ${INPUT_REDIS_STREAMS_TIMEOUT:5s}
url: ${INPUT_REDIS_STREAMS_URL:tcp://localhost:6379}
s3:
bucket: ${INPUT_S3_BUCKET}
credentials:
id: ${INPUT_S3_CREDENTIALS_ID}
profile: ${INPUT_S3_CREDENTIALS_PROFILE}
role: ${INPUT_S3_CREDENTIALS_ROLE}
role_external_id: ${INPUT_S3_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${INPUT_S3_CREDENTIALS_SECRET}
token: ${INPUT_S3_CREDENTIALS_TOKEN}
delete_objects: ${INPUT_S3_DELETE_OBJECTS:false}
download_manager:
enabled: ${INPUT_S3_DOWNLOAD_MANAGER_ENABLED:true}
endpoint: ${INPUT_S3_ENDPOINT}
force_path_style_urls: ${INPUT_S3_FORCE_PATH_STYLE_URLS:false}
max_batch_count: ${INPUT_S3_MAX_BATCH_COUNT:1}
prefix: ${INPUT_S3_PREFIX}
region: ${INPUT_S3_REGION:eu-west-1}
retries: ${INPUT_S3_RETRIES:3}
sqs_body_path: ${INPUT_S3_SQS_BODY_PATH:Records.*.s3.object.key}
sqs_bucket_path: ${INPUT_S3_SQS_BUCKET_PATH}
sqs_endpoint: ${INPUT_S3_SQS_ENDPOINT}
sqs_envelope_path: ${INPUT_S3_SQS_ENVELOPE_PATH}
sqs_max_messages: ${INPUT_S3_SQS_MAX_MESSAGES:10}
sqs_url: ${INPUT_S3_SQS_URL}
timeout: ${INPUT_S3_TIMEOUT:5s}
sqs:
credentials:
id: ${INPUT_SQS_CREDENTIALS_ID}
profile: ${INPUT_SQS_CREDENTIALS_PROFILE}
role: ${INPUT_SQS_CREDENTIALS_ROLE}
role_external_id: ${INPUT_SQS_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${INPUT_SQS_CREDENTIALS_SECRET}
token: ${INPUT_SQS_CREDENTIALS_TOKEN}
delete_message: ${INPUT_SQS_DELETE_MESSAGE:true}
endpoint: ${INPUT_SQS_ENDPOINT}
max_number_of_messages: ${INPUT_SQS_MAX_NUMBER_OF_MESSAGES:1}
region: ${INPUT_SQS_REGION:eu-west-1}
timeout: ${INPUT_SQS_TIMEOUT:5s}
url: ${INPUT_SQS_URL}
stdin:
delimiter: ${INPUT_STDIN_DELIMITER}
max_buffer: ${INPUT_STDIN_MAX_BUFFER:1000000}
multipart: ${INPUT_STDIN_MULTIPART:false}
tcp:
address: ${INPUT_TCP_ADDRESS:localhost:4194}
delimiter: ${INPUT_TCP_DELIMITER}
max_buffer: ${INPUT_TCP_MAX_BUFFER:1000000}
multipart: ${INPUT_TCP_MULTIPART:false}
tcp_server:
address: ${INPUT_TCP_SERVER_ADDRESS:127.0.0.1:0}
delimiter: ${INPUT_TCP_SERVER_DELIMITER}
max_buffer: ${INPUT_TCP_SERVER_MAX_BUFFER:1000000}
multipart: ${INPUT_TCP_SERVER_MULTIPART:false}
type: ${INPUT_TYPE:dynamic}
udp_server:
address: ${INPUT_UDP_SERVER_ADDRESS:127.0.0.1:0}
delimiter: ${INPUT_UDP_SERVER_DELIMITER}
max_buffer: ${INPUT_UDP_SERVER_MAX_BUFFER:1000000}
websocket:
basic_auth:
enabled: ${INPUT_WEBSOCKET_BASIC_AUTH_ENABLED:false}
password: ${INPUT_WEBSOCKET_BASIC_AUTH_PASSWORD}
username: ${INPUT_WEBSOCKET_BASIC_AUTH_USERNAME}
oauth:
access_token: ${INPUT_WEBSOCKET_OAUTH_ACCESS_TOKEN}
access_token_secret: ${INPUT_WEBSOCKET_OAUTH_ACCESS_TOKEN_SECRET}
consumer_key: ${INPUT_WEBSOCKET_OAUTH_CONSUMER_KEY}
consumer_secret: ${INPUT_WEBSOCKET_OAUTH_CONSUMER_SECRET}
enabled: ${INPUT_WEBSOCKET_OAUTH_ENABLED:false}
request_url: ${INPUT_WEBSOCKET_OAUTH_REQUEST_URL}
open_message: ${INPUT_WEBSOCKET_OPEN_MESSAGE}
url: ${INPUT_WEBSOCKET_URL:ws://localhost:4195/get/ws}
type: broker
buffer:
memory:
limit: ${BUFFER_MEMORY_LIMIT:524288000}
type: ${BUFFER_TYPE:none}
pipeline:
processors:
- archive:
format: ${PROCESSOR_ARCHIVE_FORMAT:binary}
path: ${PROCESSOR_ARCHIVE_PATH:${!count:files}-${!timestamp_unix_nano}.txt}
avro:
encoding: ${PROCESSOR_AVRO_ENCODING:textual}
operator: ${PROCESSOR_AVRO_OPERATOR:to_json}
schema: ${PROCESSOR_AVRO_SCHEMA}
awk:
codec: ${PROCESSOR_AWK_CODEC:text}
program: ${PROCESSOR_AWK_PROGRAM:BEGIN { x = 0 } { print $0, x; x++ }}
batch:
byte_size: ${PROCESSOR_BATCH_BYTE_SIZE:0}
condition:
bounds_check:
max_part_size: ${PROCESSOR_BATCH_CONDITION_BOUNDS_CHECK_MAX_PART_SIZE:1073741824}
max_parts: ${PROCESSOR_BATCH_CONDITION_BOUNDS_CHECK_MAX_PARTS:100}
min_part_size: ${PROCESSOR_BATCH_CONDITION_BOUNDS_CHECK_MIN_PART_SIZE:1}
min_parts: ${PROCESSOR_BATCH_CONDITION_BOUNDS_CHECK_MIN_PARTS:1}
check_interpolation:
value: ${PROCESSOR_BATCH_CONDITION_CHECK_INTERPOLATION_VALUE}
count:
arg: ${PROCESSOR_BATCH_CONDITION_COUNT_ARG:100}
jmespath:
part: ${PROCESSOR_BATCH_CONDITION_JMESPATH_PART:0}
query: ${PROCESSOR_BATCH_CONDITION_JMESPATH_QUERY}
metadata:
arg: ${PROCESSOR_BATCH_CONDITION_METADATA_ARG}
key: ${PROCESSOR_BATCH_CONDITION_METADATA_KEY}
operator: ${PROCESSOR_BATCH_CONDITION_METADATA_OPERATOR:equals_cs}
part: ${PROCESSOR_BATCH_CONDITION_METADATA_PART:0}
number:
arg: ${PROCESSOR_BATCH_CONDITION_NUMBER_ARG:0}
operator: ${PROCESSOR_BATCH_CONDITION_NUMBER_OPERATOR:equals}
part: ${PROCESSOR_BATCH_CONDITION_NUMBER_PART:0}
processor_failed:
part: ${PROCESSOR_BATCH_CONDITION_PROCESSOR_FAILED_PART:0}
resource: ${PROCESSOR_BATCH_CONDITION_RESOURCE}
static: ${PROCESSOR_BATCH_CONDITION_STATIC:false}
text:
arg: ${PROCESSOR_BATCH_CONDITION_TEXT_ARG}
operator: ${PROCESSOR_BATCH_CONDITION_TEXT_OPERATOR:equals_cs}
part: ${PROCESSOR_BATCH_CONDITION_TEXT_PART:0}
type: ${PROCESSOR_BATCH_CONDITION_TYPE:static}
count: ${PROCESSOR_BATCH_COUNT:0}
period: ${PROCESSOR_BATCH_PERIOD}
bounds_check:
max_part_size: ${PROCESSOR_BOUNDS_CHECK_MAX_PART_SIZE:1073741824}
max_parts: ${PROCESSOR_BOUNDS_CHECK_MAX_PARTS:100}
min_part_size: ${PROCESSOR_BOUNDS_CHECK_MIN_PART_SIZE:1}
min_parts: ${PROCESSOR_BOUNDS_CHECK_MIN_PARTS:1}
cache:
cache: ${PROCESSOR_CACHE_CACHE}
key: ${PROCESSOR_CACHE_KEY}
operator: ${PROCESSOR_CACHE_OPERATOR:set}
value: ${PROCESSOR_CACHE_VALUE}
compress:
algorithm: ${PROCESSOR_COMPRESS_ALGORITHM:gzip}
level: ${PROCESSOR_COMPRESS_LEVEL:-1}
decode:
scheme: ${PROCESSOR_DECODE_SCHEME:base64}
decompress:
algorithm: ${PROCESSOR_DECOMPRESS_ALGORITHM:gzip}
encode:
scheme: ${PROCESSOR_ENCODE_SCHEME:base64}
grok:
named_captures_only: ${PROCESSOR_GROK_NAMED_CAPTURES_ONLY:true}
output_format: ${PROCESSOR_GROK_OUTPUT_FORMAT:json}
remove_empty_values: ${PROCESSOR_GROK_REMOVE_EMPTY_VALUES:true}
use_default_patterns: ${PROCESSOR_GROK_USE_DEFAULT_PATTERNS:true}
group_by_value:
value: ${PROCESSOR_GROUP_BY_VALUE_VALUE:${!metadata:example}}
hash:
algorithm: ${PROCESSOR_HASH_ALGORITHM:sha256}
hash_sample:
parts:
- ${PROCESSOR_HASH_SAMPLE_PARTS:0}
retain_max: ${PROCESSOR_HASH_SAMPLE_RETAIN_MAX:10}
retain_min: ${PROCESSOR_HASH_SAMPLE_RETAIN_MIN:0}
http:
max_parallel: ${PROCESSOR_HTTP_MAX_PARALLEL:0}
parallel: ${PROCESSOR_HTTP_PARALLEL:false}
request:
backoff_on:
- ${PROCESSOR_HTTP_REQUEST_BACKOFF_ON:429}
basic_auth:
enabled: ${PROCESSOR_HTTP_REQUEST_BASIC_AUTH_ENABLED:false}
password: ${PROCESSOR_HTTP_REQUEST_BASIC_AUTH_PASSWORD}
username: ${PROCESSOR_HTTP_REQUEST_BASIC_AUTH_USERNAME}
copy_response_headers: ${PROCESSOR_HTTP_REQUEST_COPY_RESPONSE_HEADERS:false}
headers:
Content-Type: ${PROCESSOR_HTTP_REQUEST_HEADERS_CONTENT_TYPE:application/octet-stream}
max_retry_backoff: ${PROCESSOR_HTTP_REQUEST_MAX_RETRY_BACKOFF:300s}
oauth:
access_token: ${PROCESSOR_HTTP_REQUEST_OAUTH_ACCESS_TOKEN}
access_token_secret: ${PROCESSOR_HTTP_REQUEST_OAUTH_ACCESS_TOKEN_SECRET}
consumer_key: ${PROCESSOR_HTTP_REQUEST_OAUTH_CONSUMER_KEY}
consumer_secret: ${PROCESSOR_HTTP_REQUEST_OAUTH_CONSUMER_SECRET}
enabled: ${PROCESSOR_HTTP_REQUEST_OAUTH_ENABLED:false}
request_url: ${PROCESSOR_HTTP_REQUEST_OAUTH_REQUEST_URL}
rate_limit: ${PROCESSOR_HTTP_REQUEST_RATE_LIMIT}
retries: ${PROCESSOR_HTTP_REQUEST_RETRIES:3}
retry_period: ${PROCESSOR_HTTP_REQUEST_RETRY_PERIOD:1s}
timeout: ${PROCESSOR_HTTP_REQUEST_TIMEOUT:5s}
tls:
enabled: ${PROCESSOR_HTTP_REQUEST_TLS_ENABLED:false}
root_cas_file: ${PROCESSOR_HTTP_REQUEST_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${PROCESSOR_HTTP_REQUEST_TLS_SKIP_CERT_VERIFY:false}
url: ${PROCESSOR_HTTP_REQUEST_URL:http://localhost:4195/post}
verb: ${PROCESSOR_HTTP_REQUEST_VERB:POST}
insert_part:
content: ${PROCESSOR_INSERT_PART_CONTENT}
index: ${PROCESSOR_INSERT_PART_INDEX:-1}
jmespath:
query: ${PROCESSOR_JMESPATH_QUERY}
json:
operator: ${PROCESSOR_JSON_OPERATOR:clean}
path: ${PROCESSOR_JSON_PATH}
value: ${PROCESSOR_JSON_VALUE}
lambda:
credentials:
id: ${PROCESSOR_LAMBDA_CREDENTIALS_ID}
profile: ${PROCESSOR_LAMBDA_CREDENTIALS_PROFILE}
role: ${PROCESSOR_LAMBDA_CREDENTIALS_ROLE}
role_external_id: ${PROCESSOR_LAMBDA_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${PROCESSOR_LAMBDA_CREDENTIALS_SECRET}
token: ${PROCESSOR_LAMBDA_CREDENTIALS_TOKEN}
endpoint: ${PROCESSOR_LAMBDA_ENDPOINT}
function: ${PROCESSOR_LAMBDA_FUNCTION}
parallel: ${PROCESSOR_LAMBDA_PARALLEL:false}
rate_limit: ${PROCESSOR_LAMBDA_RATE_LIMIT}
region: ${PROCESSOR_LAMBDA_REGION:eu-west-1}
retries: ${PROCESSOR_LAMBDA_RETRIES:3}
timeout: ${PROCESSOR_LAMBDA_TIMEOUT:5s}
log:
level: ${PROCESSOR_LOG_LEVEL:INFO}
message: ${PROCESSOR_LOG_MESSAGE}
merge_json:
retain_parts: ${PROCESSOR_MERGE_JSON_RETAIN_PARTS:false}
metadata:
key: ${PROCESSOR_METADATA_KEY:example}
operator: ${PROCESSOR_METADATA_OPERATOR:set}
value: ${PROCESSOR_METADATA_VALUE:${!hostname}}
metric:
path: ${PROCESSOR_METRIC_PATH}
type: ${PROCESSOR_METRIC_TYPE:counter}
value: ${PROCESSOR_METRIC_VALUE}
number:
operator: ${PROCESSOR_NUMBER_OPERATOR:add}
value: ${PROCESSOR_NUMBER_VALUE:0}
parallel:
cap: ${PROCESSOR_PARALLEL_CAP:0}
rate_limit:
resource: ${PROCESSOR_RATE_LIMIT_RESOURCE}
redis:
key: ${PROCESSOR_REDIS_KEY}
operator: ${PROCESSOR_REDIS_OPERATOR:scard}
retries: ${PROCESSOR_REDIS_RETRIES:3}
retry_period: ${PROCESSOR_REDIS_RETRY_PERIOD:500ms}
url: ${PROCESSOR_REDIS_URL:tcp://localhost:6379}
sample:
retain: ${PROCESSOR_SAMPLE_RETAIN:10}
seed: ${PROCESSOR_SAMPLE_SEED:0}
select_parts:
parts:
- ${PROCESSOR_SELECT_PARTS_PARTS:0}
sleep:
duration: ${PROCESSOR_SLEEP_DURATION:100us}
split:
byte_size: ${PROCESSOR_SPLIT_BYTE_SIZE:0}
size: ${PROCESSOR_SPLIT_SIZE:1}
sql:
driver: ${PROCESSOR_SQL_DRIVER:mysql}
dsn: ${PROCESSOR_SQL_DSN}
query: ${PROCESSOR_SQL_QUERY}
result_codec: ${PROCESSOR_SQL_RESULT_CODEC:none}
subprocess:
max_buffer: ${PROCESSOR_SUBPROCESS_MAX_BUFFER:65536}
name: ${PROCESSOR_SUBPROCESS_NAME:cat}
text:
arg: ${PROCESSOR_TEXT_ARG}
operator: ${PROCESSOR_TEXT_OPERATOR:trim_space}
value: ${PROCESSOR_TEXT_VALUE}
throttle:
period: ${PROCESSOR_THROTTLE_PERIOD:100us}
type: ${PROCESSOR_TYPE:noop}
unarchive:
format: ${PROCESSOR_UNARCHIVE_FORMAT:binary}
xml:
operator: ${PROCESSOR_XML_OPERATOR:to_json}
threads: ${PROCESSOR_THREADS:1}
output:
broker:
copies: ${OUTPUTS:1}
outputs:
- amqp:
exchange: ${OUTPUT_AMQP_EXCHANGE:benthos-exchange}
exchange_declare:
durable: ${OUTPUT_AMQP_EXCHANGE_DECLARE_DURABLE:true}
enabled: ${OUTPUT_AMQP_EXCHANGE_DECLARE_ENABLED:false}
type: ${OUTPUT_AMQP_EXCHANGE_DECLARE_TYPE:direct}
immediate: ${OUTPUT_AMQP_IMMEDIATE:false}
key: ${OUTPUT_AMQP_KEY:benthos-key}
mandatory: ${OUTPUT_AMQP_MANDATORY:false}
persistent: ${OUTPUT_AMQP_PERSISTENT:false}
tls:
enabled: ${OUTPUT_AMQP_TLS_ENABLED:false}
root_cas_file: ${OUTPUT_AMQP_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${OUTPUT_AMQP_TLS_SKIP_CERT_VERIFY:false}
url: ${OUTPUT_AMQP_URL:amqp://guest:guest@localhost:5672/}
cache:
key: ${OUTPUT_CACHE_KEY:${!count:items}-${!timestamp_unix_nano}}
target: ${OUTPUT_CACHE_TARGET}
dynamic:
prefix: ${OUTPUT_DYNAMIC_PREFIX}
timeout: ${OUTPUT_DYNAMIC_TIMEOUT:5s}
elasticsearch:
aws:
credentials:
id: ${OUTPUT_ELASTICSEARCH_AWS_CREDENTIALS_ID}
profile: ${OUTPUT_ELASTICSEARCH_AWS_CREDENTIALS_PROFILE}
role: ${OUTPUT_ELASTICSEARCH_AWS_CREDENTIALS_ROLE}
role_external_id: ${OUTPUT_ELASTICSEARCH_AWS_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${OUTPUT_ELASTICSEARCH_AWS_CREDENTIALS_SECRET}
token: ${OUTPUT_ELASTICSEARCH_AWS_CREDENTIALS_TOKEN}
enabled: ${OUTPUT_ELASTICSEARCH_AWS_ENABLED:false}
endpoint: ${OUTPUT_ELASTICSEARCH_AWS_ENDPOINT}
region: ${OUTPUT_ELASTICSEARCH_AWS_REGION:eu-west-1}
backoff:
initial_interval: ${OUTPUT_ELASTICSEARCH_BACKOFF_INITIAL_INTERVAL:1s}
max_elapsed_time: ${OUTPUT_ELASTICSEARCH_BACKOFF_MAX_ELAPSED_TIME:30s}
max_interval: ${OUTPUT_ELASTICSEARCH_BACKOFF_MAX_INTERVAL:5s}
basic_auth:
enabled: ${OUTPUT_ELASTICSEARCH_BASIC_AUTH_ENABLED:false}
password: ${OUTPUT_ELASTICSEARCH_BASIC_AUTH_PASSWORD}
username: ${OUTPUT_ELASTICSEARCH_BASIC_AUTH_USERNAME}
id: ${OUTPUT_ELASTICSEARCH_ID:${!count:elastic_ids}-${!timestamp_unix}}
index: ${OUTPUT_ELASTICSEARCH_INDEX:benthos_index}
max_retries: ${OUTPUT_ELASTICSEARCH_MAX_RETRIES:0}
pipeline: ${OUTPUT_ELASTICSEARCH_PIPELINE}
sniff: ${OUTPUT_ELASTICSEARCH_SNIFF:true}
timeout: ${OUTPUT_ELASTICSEARCH_TIMEOUT:5s}
type: ${OUTPUT_ELASTICSEARCH_TYPE:doc}
urls:
- ${OUTPUT_ELASTICSEARCH_URLS:http://localhost:9200}
file:
delimiter: ${OUTPUT_FILE_DELIMITER}
path: ${OUTPUT_FILE_PATH}
files:
path: ${OUTPUT_FILES_PATH:${!count:files}-${!timestamp_unix_nano}.txt}
gcp_pubsub:
project: ${OUTPUT_GCP_PUBSUB_PROJECT}
topic: ${OUTPUT_GCP_PUBSUB_TOPIC}
hdfs:
directory: ${OUTPUT_HDFS_DIRECTORY}
hosts:
- ${OUTPUT_HDFS_HOSTS:localhost:9000}
path: ${OUTPUT_HDFS_PATH:${!count:files}-${!timestamp_unix_nano}.txt}
user: ${OUTPUT_HDFS_USER:benthos_hdfs}
http_client:
backoff_on:
- ${OUTPUT_HTTP_CLIENT_BACKOFF_ON:429}
basic_auth:
enabled: ${OUTPUT_HTTP_CLIENT_BASIC_AUTH_ENABLED:false}
password: ${OUTPUT_HTTP_CLIENT_BASIC_AUTH_PASSWORD}
username: ${OUTPUT_HTTP_CLIENT_BASIC_AUTH_USERNAME}
copy_response_headers: ${OUTPUT_HTTP_CLIENT_COPY_RESPONSE_HEADERS:false}
headers:
Content-Type: ${OUTPUT_HTTP_CLIENT_HEADERS_CONTENT_TYPE:application/octet-stream}
max_retry_backoff: ${OUTPUT_HTTP_CLIENT_MAX_RETRY_BACKOFF:300s}
oauth:
access_token: ${OUTPUT_HTTP_CLIENT_OAUTH_ACCESS_TOKEN}
access_token_secret: ${OUTPUT_HTTP_CLIENT_OAUTH_ACCESS_TOKEN_SECRET}
consumer_key: ${OUTPUT_HTTP_CLIENT_OAUTH_CONSUMER_KEY}
consumer_secret: ${OUTPUT_HTTP_CLIENT_OAUTH_CONSUMER_SECRET}
enabled: ${OUTPUT_HTTP_CLIENT_OAUTH_ENABLED:false}
request_url: ${OUTPUT_HTTP_CLIENT_OAUTH_REQUEST_URL}
propagate_response: ${OUTPUT_HTTP_CLIENT_PROPAGATE_RESPONSE:false}
rate_limit: ${OUTPUT_HTTP_CLIENT_RATE_LIMIT}
retries: ${OUTPUT_HTTP_CLIENT_RETRIES:3}
retry_period: ${OUTPUT_HTTP_CLIENT_RETRY_PERIOD:1s}
timeout: ${OUTPUT_HTTP_CLIENT_TIMEOUT:5s}
tls:
enabled: ${OUTPUT_HTTP_CLIENT_TLS_ENABLED:false}
root_cas_file: ${OUTPUT_HTTP_CLIENT_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${OUTPUT_HTTP_CLIENT_TLS_SKIP_CERT_VERIFY:false}
url: ${OUTPUT_HTTP_CLIENT_URL:http://localhost:4195/post}
verb: ${OUTPUT_HTTP_CLIENT_VERB:POST}
http_server:
address: ${OUTPUT_HTTP_SERVER_ADDRESS}
cert_file: ${OUTPUT_HTTP_SERVER_CERT_FILE}
key_file: ${OUTPUT_HTTP_SERVER_KEY_FILE}
path: ${OUTPUT_HTTP_SERVER_PATH:/get}
stream_path: ${OUTPUT_HTTP_SERVER_STREAM_PATH:/get/stream}
timeout: ${OUTPUT_HTTP_SERVER_TIMEOUT:5s}
ws_path: ${OUTPUT_HTTP_SERVER_WS_PATH:/get/ws}
inproc: ${OUTPUT_INPROC}
kafka:
ack_replicas: ${OUTPUT_KAFKA_ACK_REPLICAS:false}
addresses:
- ${OUTPUT_KAFKA_ADDRESSES:localhost:9092}
backoff:
initial_interval: ${OUTPUT_KAFKA_BACKOFF_INITIAL_INTERVAL:0s}
max_elapsed_time: ${OUTPUT_KAFKA_BACKOFF_MAX_ELAPSED_TIME:5s}
max_interval: ${OUTPUT_KAFKA_BACKOFF_MAX_INTERVAL:1s}
client_id: ${OUTPUT_KAFKA_CLIENT_ID:benthos_kafka_output}
compression: ${OUTPUT_KAFKA_COMPRESSION:none}
key: ${OUTPUT_KAFKA_KEY}
max_msg_bytes: ${OUTPUT_KAFKA_MAX_MSG_BYTES:1000000}
max_retries: ${OUTPUT_KAFKA_MAX_RETRIES:0}
round_robin_partitions: ${OUTPUT_KAFKA_ROUND_ROBIN_PARTITIONS:false}
sasl:
enabled: ${OUTPUT_KAFKA_SASL_ENABLED:false}
password: ${OUTPUT_KAFKA_SASL_PASSWORD}
user: ${OUTPUT_KAFKA_SASL_USER}
target_version: ${OUTPUT_KAFKA_TARGET_VERSION:1.0.0}
timeout: ${OUTPUT_KAFKA_TIMEOUT:5s}
tls:
enabled: ${OUTPUT_KAFKA_TLS_ENABLED:false}
root_cas_file: ${OUTPUT_KAFKA_TLS_ROOT_CAS_FILE}
skip_cert_verify: ${OUTPUT_KAFKA_TLS_SKIP_CERT_VERIFY:false}
topic: ${OUTPUT_KAFKA_TOPIC:benthos_stream}
kinesis:
backoff:
initial_interval: ${OUTPUT_KINESIS_BACKOFF_INITIAL_INTERVAL:1s}
max_elapsed_time: ${OUTPUT_KINESIS_BACKOFF_MAX_ELAPSED_TIME:30s}
max_interval: ${OUTPUT_KINESIS_BACKOFF_MAX_INTERVAL:5s}
credentials:
id: ${OUTPUT_KINESIS_CREDENTIALS_ID}
profile: ${OUTPUT_KINESIS_CREDENTIALS_PROFILE}
role: ${OUTPUT_KINESIS_CREDENTIALS_ROLE}
role_external_id: ${OUTPUT_KINESIS_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${OUTPUT_KINESIS_CREDENTIALS_SECRET}
token: ${OUTPUT_KINESIS_CREDENTIALS_TOKEN}
endpoint: ${OUTPUT_KINESIS_ENDPOINT}
hash_key: ${OUTPUT_KINESIS_HASH_KEY}
max_retries: ${OUTPUT_KINESIS_MAX_RETRIES:0}
partition_key: ${OUTPUT_KINESIS_PARTITION_KEY}
region: ${OUTPUT_KINESIS_REGION:eu-west-1}
stream: ${OUTPUT_KINESIS_STREAM}
kinesis_firehose:
backoff:
initial_interval: ${OUTPUT_KINESIS_FIREHOSE_BACKOFF_INITIAL_INTERVAL:1s}
max_elapsed_time: ${OUTPUT_KINESIS_FIREHOSE_BACKOFF_MAX_ELAPSED_TIME:30s}
max_interval: ${OUTPUT_KINESIS_FIREHOSE_BACKOFF_MAX_INTERVAL:5s}
credentials:
id: ${OUTPUT_KINESIS_FIREHOSE_CREDENTIALS_ID}
profile: ${OUTPUT_KINESIS_FIREHOSE_CREDENTIALS_PROFILE}
role: ${OUTPUT_KINESIS_FIREHOSE_CREDENTIALS_ROLE}
role_external_id: ${OUTPUT_KINESIS_FIREHOSE_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${OUTPUT_KINESIS_FIREHOSE_CREDENTIALS_SECRET}
token: ${OUTPUT_KINESIS_FIREHOSE_CREDENTIALS_TOKEN}
endpoint: ${OUTPUT_KINESIS_FIREHOSE_ENDPOINT}
max_retries: ${OUTPUT_KINESIS_FIREHOSE_MAX_RETRIES:0}
region: ${OUTPUT_KINESIS_FIREHOSE_REGION:eu-west-1}
stream: ${OUTPUT_KINESIS_FIREHOSE_STREAM}
mqtt:
client_id: ${OUTPUT_MQTT_CLIENT_ID:benthos_output}
password: ${OUTPUT_MQTT_PASSWORD}
qos: ${OUTPUT_MQTT_QOS:1}
topic: ${OUTPUT_MQTT_TOPIC:benthos_topic}
urls:
- ${OUTPUT_MQTT_URLS:tcp://localhost:1883}
user: ${OUTPUT_MQTT_USER}
nanomsg:
bind: ${OUTPUT_NANOMSG_BIND:false}
poll_timeout: ${OUTPUT_NANOMSG_POLL_TIMEOUT:5s}
socket_type: ${OUTPUT_NANOMSG_SOCKET_TYPE:PUSH}
urls:
- ${OUTPUT_NANOMSG_URLS:tcp://localhost:5556}
nats:
subject: ${OUTPUT_NATS_SUBJECT:benthos_messages}
urls:
- ${OUTPUT_NATS_URLS:nats://127.0.0.1:4222}
nats_stream:
client_id: ${OUTPUT_NATS_STREAM_CLIENT_ID:benthos_client}
cluster_id: ${OUTPUT_NATS_STREAM_CLUSTER_ID:test-cluster}
subject: ${OUTPUT_NATS_STREAM_SUBJECT:benthos_messages}
urls:
- ${OUTPUT_NATS_STREAM_URLS:nats://localhost:4222}
nsq:
nsqd_tcp_address: ${OUTPUT_NSQ_NSQD_TCP_ADDRESS:localhost:4150}
topic: ${OUTPUT_NSQ_TOPIC:benthos_messages}
user_agent: ${OUTPUT_NSQ_USER_AGENT:benthos_producer}
redis_hash:
key: ${OUTPUT_REDIS_HASH_KEY}
url: ${OUTPUT_REDIS_HASH_URL:tcp://localhost:6379}
walk_json_object: ${OUTPUT_REDIS_HASH_WALK_JSON_OBJECT:false}
walk_metadata: ${OUTPUT_REDIS_HASH_WALK_METADATA:false}
redis_list:
key: ${OUTPUT_REDIS_LIST_KEY:benthos_list}
url: ${OUTPUT_REDIS_LIST_URL:tcp://localhost:6379}
redis_pubsub:
channel: ${OUTPUT_REDIS_PUBSUB_CHANNEL:benthos_chan}
url: ${OUTPUT_REDIS_PUBSUB_URL:tcp://localhost:6379}
redis_streams:
body_key: ${OUTPUT_REDIS_STREAMS_BODY_KEY:body}
max_length: ${OUTPUT_REDIS_STREAMS_MAX_LENGTH:0}
stream: ${OUTPUT_REDIS_STREAMS_STREAM:benthos_stream}
url: ${OUTPUT_REDIS_STREAMS_URL:tcp://localhost:6379}
s3:
bucket: ${OUTPUT_S3_BUCKET}
content_encoding: ${OUTPUT_S3_CONTENT_ENCODING}
content_type: ${OUTPUT_S3_CONTENT_TYPE:application/octet-stream}
credentials:
id: ${OUTPUT_S3_CREDENTIALS_ID}
profile: ${OUTPUT_S3_CREDENTIALS_PROFILE}
role: ${OUTPUT_S3_CREDENTIALS_ROLE}
role_external_id: ${OUTPUT_S3_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${OUTPUT_S3_CREDENTIALS_SECRET}
token: ${OUTPUT_S3_CREDENTIALS_TOKEN}
endpoint: ${OUTPUT_S3_ENDPOINT}
force_path_style_urls: ${OUTPUT_S3_FORCE_PATH_STYLE_URLS:false}
path: ${OUTPUT_S3_PATH:${!count:files}-${!timestamp_unix_nano}.txt}
region: ${OUTPUT_S3_REGION:eu-west-1}
timeout: ${OUTPUT_S3_TIMEOUT:5s}
sns:
credentials:
id: ${OUTPUT_SNS_CREDENTIALS_ID}
profile: ${OUTPUT_SNS_CREDENTIALS_PROFILE}
role: ${OUTPUT_SNS_CREDENTIALS_ROLE}
role_external_id: ${OUTPUT_SNS_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${OUTPUT_SNS_CREDENTIALS_SECRET}
token: ${OUTPUT_SNS_CREDENTIALS_TOKEN}
endpoint: ${OUTPUT_SNS_ENDPOINT}
region: ${OUTPUT_SNS_REGION:eu-west-1}
timeout: ${OUTPUT_SNS_TIMEOUT:5s}
topic_arn: ${OUTPUT_SNS_TOPIC_ARN}
sqs:
backoff:
initial_interval: ${OUTPUT_SQS_BACKOFF_INITIAL_INTERVAL:1s}
max_elapsed_time: ${OUTPUT_SQS_BACKOFF_MAX_ELAPSED_TIME:30s}
max_interval: ${OUTPUT_SQS_BACKOFF_MAX_INTERVAL:5s}
credentials:
id: ${OUTPUT_SQS_CREDENTIALS_ID}
profile: ${OUTPUT_SQS_CREDENTIALS_PROFILE}
role: ${OUTPUT_SQS_CREDENTIALS_ROLE}
role_external_id: ${OUTPUT_SQS_CREDENTIALS_ROLE_EXTERNAL_ID}
secret: ${OUTPUT_SQS_CREDENTIALS_SECRET}
token: ${OUTPUT_SQS_CREDENTIALS_TOKEN}
endpoint: ${OUTPUT_SQS_ENDPOINT}
max_retries: ${OUTPUT_SQS_MAX_RETRIES:0}
message_deduplication_id: ${OUTPUT_SQS_MESSAGE_DEDUPLICATION_ID}
message_group_id: ${OUTPUT_SQS_MESSAGE_GROUP_ID}
region: ${OUTPUT_SQS_REGION:eu-west-1}
url: ${OUTPUT_SQS_URL}
stdout:
delimiter: ${OUTPUT_STDOUT_DELIMITER}
tcp:
address: ${OUTPUT_TCP_ADDRESS:localhost:4194}
type: ${OUTPUT_TYPE:dynamic}
udp:
address: ${OUTPUT_UDP_ADDRESS:localhost:4194}
websocket:
basic_auth:
enabled: ${OUTPUT_WEBSOCKET_BASIC_AUTH_ENABLED:false}
password: ${OUTPUT_WEBSOCKET_BASIC_AUTH_PASSWORD}
username: ${OUTPUT_WEBSOCKET_BASIC_AUTH_USERNAME}
oauth:
access_token: ${OUTPUT_WEBSOCKET_OAUTH_ACCESS_TOKEN}
access_token_secret: ${OUTPUT_WEBSOCKET_OAUTH_ACCESS_TOKEN_SECRET}
consumer_key: ${OUTPUT_WEBSOCKET_OAUTH_CONSUMER_KEY}
consumer_secret: ${OUTPUT_WEBSOCKET_OAUTH_CONSUMER_SECRET}
enabled: ${OUTPUT_WEBSOCKET_OAUTH_ENABLED:false}
request_url: ${OUTPUT_WEBSOCKET_OAUTH_REQUEST_URL}
url: ${OUTPUT_WEBSOCKET_URL:ws://localhost:4195/post/ws}
pattern: ${OUTPUTS_PATTERN:greedy}
type: broker
logger:
add_timestamp: ${LOGGER_ADD_TIMESTAMP:true}
json_format: ${LOGGER_JSON_FORMAT:true}
level: ${LOGGER_LEVEL:INFO}
prefix: ${LOGGER_PREFIX:benthos}
metrics:
http_server:
prefix: ${METRICS_HTTP_SERVER_PREFIX:benthos}
prometheus:
prefix: ${METRICS_PROMETHEUS_PREFIX:benthos}
push_interval: ${METRICS_PROMETHEUS_PUSH_INTERVAL}
push_job_name: ${METRICS_PROMETHEUS_PUSH_JOB_NAME:benthos_push}
push_url: ${METRICS_PROMETHEUS_PUSH_URL}
statsd:
address: ${METRICS_STATSD_ADDRESS:localhost:4040}
flush_period: ${METRICS_STATSD_FLUSH_PERIOD:100ms}
network: ${METRICS_STATSD_NETWORK:udp}
prefix: ${METRICS_STATSD_PREFIX:benthos}
stdout:
flush_metrics: ${METRICS_STDOUT_FLUSH_METRICS:false}
push_interval: ${METRICS_STDOUT_PUSH_INTERVAL}
static_fields:
'@service': ${METRICS_STDOUT_STATIC_FIELDS_@SERVICE:benthos}
type: ${METRICS_TYPE:http_server}