/
demo.txt
108 lines (91 loc) · 3.81 KB
/
demo.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir -p /tmp/weatherapp
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /tmp/weatherapp
flink run --jobmanager yarn-cluster --detached --parallelism 1 --yarnname WeatherApp \
-yt /opt/demo/conf/application.properties \
weather-1.0.jar \
--kafka.bootstrap.servers edge2ai-1.dim.local:9092 \
--kafkaTopic weathernj \
--hdfsOutput hdfs:///tmp/weatherapp \
--properties.file /opt/demo/conf/application.properties
# application.properties
hdfsOutput=/tmp/weatherapp
kafkaTopic=weathernj
kafka.bootstrap.servers=edge2ai-1.dim.local:9092
kafka.security.protocol=PLAINTEXT
#kafka.sasl.kerberos.service.name=kafka
#kafka.ssl.truststore.location=/home/test/truststore.jks
#kafka.ssl.keystore.location=/home/test/keystore.jks
#kafka.ssl.keystore.password=changeit
#kafka.ssl.key.password=changeit
schema.registry.url=http://edge2ai-1.dim.local:7788/api/v1
#schema.registry.client.ssl.trustStorePath=/home/test/truststore.jks
#schema.registry.client.ssl.trustStorePassword=changeit
#sensitive.keys=password
# sql-env.yaml
#==============================================================================
# Catalogs
#==============================================================================
# Define catalogs here.
catalogs:
- name: registry
type: cloudera-registry
# Registry Client standard properties
registry.properties.schema.registry.url: http://edge2ai-1.dim.local:7788/api/v1
# registry.properties.key:
# Registry Client SSL properties
# Kafka Connector properties
connector.properties.bootstrap.servers: edge2ai-1.dim.local:9092
connector.startup-mode: earliest-offset
connector.properties.group.id: flinkapp
connector.type: kafka
connector.version: universal
format.type: registry
format.registry.properties.schema.registry.url: http://edge2ai-1.dim.local:7788/api/v1
#=============================================================================
# Execution properties
#==============================================================================
# Properties that change the fundamental execution behavior of a table program.
execution:
# select the implementation responsible for planning table programs
# possible values are 'old' (used by default) or 'blink'
planner: blink
# 'batch' or 'streaming' execution
type: streaming
# allow 'event-time' or only 'processing-time' in sources
time-characteristic: event-time
# interval in ms for emitting periodic watermarks
periodic-watermarks-interval: 200
# 'changelog' or 'table' presentation of results
result-mode: table
# maximum number of maintained rows in 'table' presentation of results
max-table-result-rows: 1000000
# parallelism of the program
parallelism: 1
# maximum parallelism
max-parallelism: 128
# minimum idle state retention in ms
min-idle-state-retention: 0
# maximum idle state retention in ms
max-idle-state-retention: 0
# current catalog ('default_catalog' by default)
current-catalog: default_catalog
# current database of the current catalog (default database of the catalog by default)
current-database: default_database
# controls how table programs are restarted in case of a failures
restart-strategy:
# strategy type
# possible values are "fixed-delay", "failure-rate", "none", or "fallback" (default)
type: fallback
#==============================================================================
# Deployment properties
#==============================================================================
# Properties that describe the cluster to which table programs are submitted to.
deployment:
# general cluster communication timeout in ms
response-timeout: 5000
# (optional) address from cluster to gateway
gateway-address: ""
# (optional) port from cluster to gateway
gateway-port: 0
configuration:
execution.target: yarn-session