-
Notifications
You must be signed in to change notification settings - Fork 1
/
docker-compose-spark.yml
57 lines (54 loc) · 1.44 KB
/
docker-compose-spark.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
version: "3"
services:
# -------------------- SPARK CLUSTER --------------------
spark-master:
image: bde2020/spark-master:3.1.1-hadoop3.2
hostname: spark-master
container_name: spark-master
ports:
- "8081:8080"
- "7077:7077"
environment:
- INIT_DAEMON_STEP=setup_spark
spark-worker-1:
image: bde2020/spark-worker:3.1.1-hadoop3.2
hostname: spark-worker-1
container_name: spark-worker-1
depends_on:
- spark-master
ports:
- "8082:8081"
environment:
- "SPARK_MASTER=spark://spark-master:7077"
spark-worker-2:
image: bde2020/spark-worker:3.1.1-hadoop3.2
hostname: spark-worker-2
container_name: spark-worker-2
depends_on:
- spark-master
ports:
- "8083:8081"
environment:
- "SPARK_MASTER=spark://spark-master:7077"
# -------------------- SPARK THRIFTSERVER --------------------
thriftserver:
image: chimerasuite/thriftserver:3.1.1-hadoop3.2
hostname: thriftserver
container_name: thriftserver
depends_on:
- spark-master
ports:
- "10001:10000"
environment:
- "SPARK_MASTER=spark-master:7077"
- "TOTAL_EXECUTOR_CORES=2"
- "EXECUTOR_MEMORY=4G"
- "DRIVER_MAX_RESULT_SIZE=4G"
- "HADOOP_CONF_DIR=/spark/conf/hive-site.xml"
volumes:
- "./spark/conf/hive-site.xml:/spark/conf/hive-site.xml"
- "./spark/init:/init"
networks:
default:
external:
name: qwerty