-
Notifications
You must be signed in to change notification settings - Fork 9
/
docker-compose.yml
137 lines (125 loc) · 2.81 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
version: "3.9"
services:
de_mysql:
image: "mariadb"
container_name: de_mysql
volumes:
- ./mysql:/var/lib/mysql
ports:
- "3306:3306"
env_file:
- .env
networks:
- data_network
de_psql:
image: postgres:15
container_name: de_psql
volumes:
- ./psql:/var/lib/postgresql/data
ports:
- "5432:5432"
env_file:
- .env
networks:
- data_network
minio:
hostname: minio
image: "minio/minio"
container_name: minio
ports:
- "9001:9001"
- "9000:9000"
command: [ "server", "/data", "--console-address", ":9001" ]
volumes:
- ./minio/data:/data
env_file:
- .env
networks:
- data_network
hive-metastore:
container_name: hive-metastore
hostname: hive-metastore
image: "bitsondatadev/hive-metastore"
ports:
- "9083:9083"
volumes:
- ./hive-metastore/metastore-site.xml:/opt/apache-hive-metastore-3.0.0-bin/conf/metastore-site.xml:ro
environment:
METASTORE_DB_HOSTNAME: de_mysql
networks:
- data_network
depends_on:
- de_mysql
- minio
trino:
container_name: trino
image: "trinodb/trino"
hostname: trino
ports:
- "8080:8080"
volumes:
- ./trino:/etc/trino
networks:
- data_network
spark-master:
build:
context: ./spark
dockerfile: ./Dockerfile
container_name: "spark-master"
environment:
- SPARK_MODE=master
- SPARK_LOCAL_IP=spark-master
ports:
- "7077:7077"
volumes:
- ./spark/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
networks:
- data_network
spark-worker-1:
image: docker.io/bitnami/spark:3.3
container_name: "spark-worker-1"
env_file:
- .env
depends_on:
- spark-master
networks:
- data_network
spark-worker-2:
image: docker.io/bitnami/spark:3.3
container_name: "spark-worker-2"
env_file:
- .env
depends_on:
- spark-master
networks:
- data_network
spark-thrift-server:
build:
context: ./spark
dockerfile: ./Dockerfile
container_name: "spark-thrift-server"
depends_on:
- spark-master
- hive-metastore
ports:
- "4040:4040"
- "10000:10000"
command: sh -c "
sleep 10 && ./sbin/start-thriftserver.sh --driver-java-options '-Dhive.metastore.uris=thrift://hive-metastore:9083' --master spark://spark-master:7077"
volumes:
- ./spark/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
networks:
- data_network
metabase:
image: metabase/metabase:latest
container_name: "metabase"
ports:
- "3000:3000"
env_file:
- .env
networks:
- data_network
networks:
data_network:
driver: bridge
name: data_network