-
Notifications
You must be signed in to change notification settings - Fork 56
/
hstream.yaml
323 lines (273 loc) · 10.7 KB
/
hstream.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
# Configuration File for HStreamDB
# Configuration for HStream Server
hserver:
# The identifier of a single HServer node,
# the value must be given and can be overwritten by cli option `--server-id`
id: 1
# Server port value, the value must be given and can be overwritten by cli
# option `--port`
port: 6570
# Server port value for internal communications between server nodes,
# the value must be given
# and can be overwritten by cli option `--internal-port`
internal-port: 6571
bind-address: 0.0.0.0
# Server listener address value, the value must be given.
# It should NOT be "127.0.0.1", if you intend to start a cluster or
# trying to connect to the server from a different network.
#
# This value can be overwritten by cli option `--advertised-address`
advertised-address: 127.0.0.1
#advertised-listeners:
# private:
# - address: "127.0.0.1"
# port: 6580
#
#listeners-security-protocol-map: private:plaintext
# The address used for internal communications between server nodes,
# if not given, it will use the value of `advertised-address`
#
#gossip-address:
# The server nodes in the same cluster shares a meta store unit
# , this is essential for a server to start.
#
# Use Comma separated host:port pairs, with the storage protocol
# e.g. use a zookeeper cluster as meta storage,
# zk://127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183
# e.g. use a Rqlite cluster as meta storage,
# rq://127.0.0.1:4001
# the value must be given
# and can be overwritten by cli option `--metastore-uri`
metastore-uri: zk://127.0.0.1:2181
connector-meta-store: zk://127.0.0.1:2181
# To start a cluster, a list of host (and port) pairs must be provided
# if the port is not specified,
# the server will use `internal-port` value specified above.
# As a string of comma separated host:internal-port pairs,
# each corresponding to server gossip listener.
# the value must be given
# and can be overwritten by cli option `--seed-nodes`
seed-nodes: 127.0.0.1:6571
# The max record payload size the hstream server would accept.
# The value only indicates that the server would not reject the value,
# but has no guarantee that a record smaller than the specified value will be written successfully.
# Optional; If not specified, the default value will be 1MB
#
#max-record-size: 1048576 # 1024*1024
# The options used to control logs print by the server node,
# the following values are the default values if the options is not specified.
# Also the default value will be overwritten by cli flag
# `--log-with-color` and option `--log-level`
#
# Optional, the default value will be true
#
#log-with-color: true
# Optional, the default value will be info
#
#log-level: info
# TLS options
# Enable tls, which requires tls-key-path and tls-cert-path options
#
#enable-tls: false
# Key file path for tls, can be generated by openssl
#
#tls-key-path: /path/to/the/server.key.pem
# The signed certificate by CA for the key(tls-key-path)
#
#tls-cert-path: /path/to/the/signed.server.cert.pem
# Optional for tls, if tls-ca-path is not empty, then enable TLS authentication,
# in handshake phase, server will request and verify client's certificate.
#
#tls-ca-path: /path/to/the/ca.cert.pem
# Cluster Gossip Options
#
#gossip:
# gossip-fanout: 3
# retransmit-mult: 4
# gossip-interval: 1000000 # 1 sec
# probe-interval: 2000000 # 2 sec
# roundtrip-timeout: 500000 # 0.5 sec
# TODO: Auth tokens
# - store tokens safely
#tokens: []
# HStream IO Options
hstream-io:
# the io tasks work directory
tasks-path: /tmp/io/tasks
# io tasks run as docker containers,
# so the tasks-network should be the network that can connect to HStreamDB and external systems.
tasks-network: host
extra-docker-args: "-m 4g"
# when restarting a connector, whether use fixed connector image(if images of config file was updated),
# true -> read image from metastore
# false -> read image from config file
fixed-connector-image: true
# source images
source-images:
mysql: hstreamdb/source-mysql:latest
postgresql: hstreamdb/source-postgresql:latest
sqlserver: hstreamdb/source-sqlserver:latest
mongodb: hstreamdb/source-mongodb:latest
generator: hstreamdb/source-generator:latest
# sink images
sink-images:
mysql: hstreamdb/sink-mysql:latest
postgresql: hstreamdb/sink-postgresql:latest
mongodb: hstreamdb/sink-mongodb:latest
blackhole: hstreamdb/sink-blackhole:latest
las: hstreamdb/sink-las:latest
elasticsearch: hstreamdb/sink-elasticsearch:latest
# Internal grpc settings (typically, there's no need to modify these)
#grpc:
# channel-args:
# GRPC_ARG_KEEPALIVE_TIME_MS: 7200000 # (2 hours)
# GRPC_ARG_KEEPALIVE_TIMEOUT_MS: 20000 # (20 seconds)
# GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS: 0 # (false)
# GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA: 2
# GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS: 300000 # (5 minutes)
# GRPC_ARG_HTTP2_MAX_PING_STRIKES: 2
# Configuration for HStream Kafka Server
kafka:
# The identifier of a single HServer node,
# the value must be given and can be overwritten by cli option `--server-id`
id: 1
# Server port value, the value must be given and can be overwritten by cli
# option `--port`
port: 9092
# Server port value for internal communications between server nodes,
# the value must be given
# and can be overwritten by cli option `--gossip-port`
gossip-port: 6571
# Server port value for export metrics and can be overwritten by cli option `--metrics-port`
metrics-port: 9700
bind-address: 0.0.0.0
# Server listener address value, the value must be given.
# It should NOT be "127.0.0.1", if you intend to start a cluster or
# trying to connect to the server from a different network.
#
# This value can be overwritten by cli option `--advertised-address`
advertised-address: 127.0.0.1
#advertised-listeners:
# private:
# - address: "127.0.0.1"
# port: 9192
#
# Listener security in format <listener_key>:<security_key>[, ...]
# <security_key> can be one of the following:
# "plaintext": no authentication and no encryption
# "tls": tls encryption but no authentication
# "sasl_plaintext": sasl authentication but no encryption
# "sasl_tls": sasl authentication and tls encryption
# For example, "public:tls,private:plaintext"
#listeners-security-protocol-map: private:plaintext
# The address used for internal communications between server nodes,
# if not given, it will use the value of `advertised-address`
#
#gossip-address:
# The server nodes in the same cluster shares a meta store unit
# , this is essential for a server to start.
#
# Use Comma separated host:port pairs, with the storage protocol
# e.g. use a zookeeper cluster as meta storage,
# zk://127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183
# e.g. use a Rqlite cluster as meta storage,
# rq://127.0.0.1:4001
# the value must be given
# and can be overwritten by cli option `--metastore-uri`
metastore-uri: zk://127.0.0.1:2181
# To start a cluster, a list of host (and port) pairs must be provided
# if the port is not specified,
# the server will use `internal-port` value specified above.
# As a string of comma separated host:internal-port pairs,
# each corresponding to server gossip listener.
# the value must be given
# and can be overwritten by cli option `--seed-nodes`
seed-nodes: 127.0.0.1:6571
# The max record payload size the hstream server would accept.
# The value only indicates that the server would not reject the value,
# but has no guarantee that a record smaller than the specified value will be written successfully.
# Optional; If not specified, the default value will be 1MB
#
#max-record-size: 1048576 # 1024*1024
# The options used to control logs print by the server node,
# the following values are the default values if the options is not specified.
# Also the default value will be overwritten by cli flag
# `--log-with-color` and option `--log-level`
#
# Optional, the default value will be true
#
#log-with-color: true
# Optional, the default value will be info
#
#log-level: info
# TLS options
# Enable tls, which requires tls-key-path and tls-cert-path options
#
#enable-tls: false
# Key file path for tls, can be generated by openssl
#
#tls-key-path: /path/to/the/server.key.pem
# The signed certificate by CA for the key(tls-key-path)
#
#tls-cert-path: /path/to/the/signed.server.cert.pem
# Optional for tls, if tls-ca-path is not empty, then enable TLS authentication,
# in handshake phase, server will request and verify client's certificate.
#
#tls-ca-path: /path/to/the/ca.cert.pem
# SASL options
# Enable SASL authentication. This requires `--enable-sasl` option.
#enable-sasl: False
# SASL config should be placed under `sasl:`
# <SASL> ::= [<mechs>]
# <mech> ::= { mechanism: <string>, auth-list: [<auth>] }
# <auth> ::= { username: <string>, password: <string> } (PLAIN mechanism)
# To start a server with SASL authentication, use `advertised-listeners:`
# and `listeners-security-protocol-map:` options above.
#sasl:
# - mechanism: "PLAIN"
# auth-list:
# - username: "admin"
# password: "password"
# - username: "user_a"
# password: "password_a"
# - mechanism: "SCRAM-SHA-256"
# auth-list:
# - username: "admin"
# password: "password"
# - username: "user_a"
# password: "password_a"
# Cluster Gossip Options
#
#gossip:
# gossip-fanout: 3
# retransmit-mult: 4
# gossip-interval: 1000000 # 1 sec
# probe-interval: 2000000 # 2 sec
# roundtrip-timeout: 500000 # 0.5 sec
# Broker options (compatible with Kafka)
#
#num.partitions: 1
#default.replication.factor: 1
#auto.create.topic.enable: true
#offsets.topic.replication.factor: 1
# Internal storage options
#
#storage:
# fetch-mode: 1 # TODO: Currently, only mode 1 is supported
# fetch-reader-timeout: 50 # 50ms, default timeout of each read, 0 means nonblocking
# fetch-maxlen: 1000 # default max size of each read
# Configuration for HStream Store
# The configuration for hstore is **Optional**. When the values are not provided,
# hstreamdb will use the following configuration as the default configuration.
hstore:
#log-level: info
# checkpoint-replication-factor: 1
## Store admin section specify the client config when connecting to the storage admin server
##
#store-admin:
# host: "127.0.0.1"
# port: 6440
# protocol-id: binary
# conn-timeout: 5000
# send-timeout: 5000
# recv-timeout: 5000