docker-compose-spark3.0.2.yml

version: "3"
services:
    namenode:
        image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
        container_name: namenode
        restart: always
        volumes:
            - hadoop_namenode:/hadoop/dfs/name
        environment:
            - CLUSTER_NAME=test
        ports:
            - 9870:9870
            - 9000:9000
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    datanode:
        image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
        container_name: datanode
        restart: always
        depends_on: 
            - namenode
        volumes:
            - hadoop_datanode:/hadoop/dfs/data
        environment:
            SERVICE_PRECONDITION: "namenode:9870"
        ports:
            - 9864:9864
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    resourcemanager:
        image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
        container_name: resourcemanager
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
        ports:
            - 8088:8088
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    nodemanager:
        image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
        container_name: nodemanager
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        ports:
            - 8042:8042
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    historyserver:
        image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
        container_name: historyserver
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        volumes:
            - hadoop_historyserver:/hadoop/yarn/timeline
        ports:
            - 8188:8188
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    hive-server:
        image: bde2020/hive:2.3.2-postgresql-metastore
        container_name: hive-server
        restart: always
        depends_on: 
            - namenode
            - datanode
        environment:
            HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
        ports:
            - 10000:10000
            - 10002:10002
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    hive-metastore:
        image: bde2020/hive:2.3.2-postgresql-metastore
        container_name: hive-metastore
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
        command: /opt/hive/bin/hive --service metastore
        ports:
            - 9083:9083
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    hive-metastore-postgresql:
        image: bde2020/hive-metastore-postgresql:2.3.0
        container_name: hive-metastore-postgresql
        volumes:
            - hive_metastore:/var/lib/postgresql/data
        networks:
            - hadoop
    
    presto-coordinator:
        image: shawnzhu/prestodb:0.181
        container_name: presto-coordinator
        ports:
            - 8090:8080
        networks:
            - hadoop
        
    spark-master:
        image: bde2020/spark-master:3.0.2-hadoop3.2
        container_name: spark-master
        depends_on:
            - namenode
            - datanode
        environment:
            - INIT_DAEMON_STEP=setup_spark
        ports:
            - 8080:8080
            - 7077:7077
            - 4040:4040
            - 18080:18080
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    spark-worker:
        image: bde2020/spark-worker:3.0.2-hadoop3.2
        container_name: spark-worker
        depends_on:
            - spark-master
        environment:
            - "SPARK_MASTER=spark://spark-master:7077"
        ports:
            - 8081:8081
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

volumes:
    hadoop_namenode:
    hadoop_datanode:
    hadoop_historyserver:
    hive_metastore:

networks:
    hadoop:
        name: hadoop