-
Notifications
You must be signed in to change notification settings - Fork 3
/
docker-compose.yml
71 lines (70 loc) · 2.03 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
version: "3"
services:
sts:
image: archongum/spark:3.3.2-hadoop3
restart: always
depends_on:
- minio
ports:
- 10000:10000
- 4040:4040
volumes:
- ./conf/hive-site.xml:/etc/spark/hive-site.xml
entrypoint: /entrypoint-sts.sh
command:
- --master
- local[2]
- --conf
- spark.hadoop.fs.s3a.access.key=minioadmin
- --conf
- spark.hadoop.fs.s3a.secret.key=minioadmin
- --conf
- spark.hadoop.fs.s3a.endpoint=http://minio:9000
- --conf
- spark.hadoop.fs.s3a.path.style.access=true
- --conf
- spark.hadoop.fs.s3a.connection.ssl.enabled=false
- --conf
- spark.sql.warehouse.dir=s3a://warehouse/spark
- --conf
- spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
- --conf
- spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
- --conf
- spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
- --conf
- spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog
- --conf
- spark.sql.catalog.spark_catalog.type=hadoop
- --conf
- spark.sql.catalog.spark_catalog.warehouse=s3a://warehouse/spark
- --conf
- spark.sql.defaultCatalog=spark_catalog
minio:
image: minio/minio
restart: always
ports:
- 9000:9000
- 9001:9001
command:
- server
- /data
- --console-address
- ":9001"
volumes:
- ./docker-volume/minio:/data
environment:
MINIO_ROOT_USER: minioadmin
MINIO_ROOT_PASSWORD: minioadmin
mc:
image: minio/mc
depends_on:
- minio
entrypoint: >
/bin/sh -c "
until (/usr/bin/mc config host add minio http://minio:9000 minioadmin minioadmin) do echo '...waiting...' && sleep 1; done;
/usr/bin/mc rm -r --force minio/warehouse;
/usr/bin/mc mb minio/warehouse;
/usr/bin/mc policy set public minio/warehouse;
tail -f /dev/null
"