-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yml
executable file
·135 lines (123 loc) · 3.51 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
version: '3'
services:
redis:
image: redis:latest
container_name: superset_cache
restart: unless-stopped
networks:
- fintechstream_default
volumes:
- ./redis:/data
presto:
build:
context: .
dockerfile: ./docker/Dockerfile_presto
container_name: presto
ports:
- "8085:8080"
volumes:
- ./presto_properties:/opt/presto-server-0.284/etc/
networks:
- fintechstream_default
superset:
image: apache/superset:latest
user: root
restart: unless-stopped
container_name: superset
volumes:
- ./config/superset_config.py:/app/superset_config.py
- ./docker/docker-entrypoint-initdb.d:/app/docker-entrypoint-initdb.d
env_file: ./docker/.env-non-dev
ports:
- "8088:8088"
networks:
- fintechstream_default
depends_on:
- postgres
postgres:
image: postgres:latest
container_name: postgres
restart: unless-stopped
env_file: ./docker/.env-non-dev
volumes:
- ./db_home:/var/lib/postgresql/data
- ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
ports:
- "5432:5432"
networks:
- fintechstream_default
scylla:
image: scylladb/scylla:latest
container_name: scylladb
networks:
- fintechstream_default
volumes:
- ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
- ./data/scylla:/var/lib/scylla
command: ["--smp", "1", "--memory", "256M", "--experimental", "1"]
zookeeper:
image: confluentinc/cp-zookeeper:7.3.2
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
networks:
- fintechstream_default
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_SERVERS: zookeeper:2888:3888
broker:
image: confluentinc/cp-kafka:7.3.2
container_name: broker
ports:
- "9092:9092"
networks:
- fintechstream_default
depends_on:
- zookeeper
environment:
KAFKA_DELETE_TOPIC_ENABLE: "true"
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://172.17.0.1:9092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
spark-master:
image: bitnami/spark:3.5.0
command: bin/spark-class org.apache.spark.deploy.master.Master
ports:
- "7077:7077"
- "8080:8080"
environment:
- SPARK_MODE=master
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
volumes:
- ./data_pipeline:/opt/bitnami/data_pipeline:rw
networks:
- fintechstream_default
spark-worker:
image: bitnami/spark:3.5.0
command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
depends_on:
- spark-master
volumes:
- ./data_pipeline:/opt/bitnami/data_pipeline:rw
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark:7077
- SPARK_WORKER_MEMORY=2G
- SPARK_WORKER_CORES=2
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
networks:
- fintechstream_default
networks:
fintechstream_default: