docker-compose.yml

version: "3"
services:
    namenode:
        image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
        container_name: namenode
        restart: always
        volumes:
            - hadoop_namenode:/hadoop/dfs/name
        environment:
            - CLUSTER_NAME=test
        ports:
            - 9870:9870
            - 9000:9000
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    datanode:
        image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
        container_name: datanode
        restart: always
        depends_on:
            - namenode
        volumes:
            - hadoop_datanode:/hadoop/dfs/data
        environment:
            SERVICE_PRECONDITION: "namenode:9870"
        ports:
            - 9864:9864
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    resourcemanager:
        image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
        container_name: resourcemanager
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
        ports:
            - 8088:8088
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    nodemanager:
        image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
        container_name: nodemanager
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        ports:
            - 8042:8042
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    historyserver:
        image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
        container_name: historyserver
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        volumes:
            - hadoop_historyserver:/hadoop/yarn/timeline
        ports:
            - 8188:8188
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    hive-server:
        image: bde2020/hive:2.3.2-postgresql-metastore
        container_name: hive-server
        restart: always
        depends_on:
            - namenode
            - datanode
        environment:
            HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
        ports:
            - 10000:10000
            - 10002:10002
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    hive-metastore:
        image: bde2020/hive:2.3.2-postgresql-metastore
        container_name: hive-metastore
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
        command: /opt/hive/bin/hive --service metastore
        ports:
            - 9083:9083
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    hive-metastore-postgresql:
        image: bde2020/hive-metastore-postgresql:2.3.0
        container_name: hive-metastore-postgresql
        volumes:
            - hive_metastore:/var/lib/postgresql/data
        networks:
            - hadoop

    presto-coordinator:
        image: shawnzhu/prestodb:0.181
        container_name: presto-coordinator
        ports:
            - 8090:8080
        networks:
            - hadoop

    spark-master:
        image: bde2020/spark-master:3.2.0-hadoop3.2
        container_name: spark-master
        depends_on:
            - namenode
            - datanode
        environment:
            - INIT_DAEMON_STEP=setup_spark
        ports:
            - 8080:8080
            - 7077:7077
            - 4040:4040
        env_file:
            - hadoop.env
        networks:
            - hadoop

    spark-worker:
        image: bde2020/spark-worker:3.2.0-hadoop3.2
        container_name: spark-worker
        depends_on:
            - spark-master
        ports:
            - "8081:8081"
        environment:
            - "SPARK_MASTER=spark://spark-master:7077"
        env_file:
            - hadoop.env
        networks:
            - hadoop
    spark-history-server:
        image: bde2020/spark-history-server:3.2.0-hadoop3.2
        container_name: spark-history-server
        depends_on:
            - spark-master
        ports:
            - 18081:18081
        volumes:
            - spark-events:/tmp/spark-events
        env_file:
            - hadoop.env
        networks:
            - hadoop
    R-data-analysis:
        image: rocker/tidyverse:latest
        container_name: R-data-analysis
        networks:
            - hadoop

    hbase:
        build: ./images/HBase-standalone
        container_name: hbase
        volumes:
            - hbase_data:/hbase-data
            - hbase_zookeeper_data:/zookeeper-data
        ports:
            - 16000:16000
            - 16010:16010
            - 16020:16020
            - 16030:16030
            - 2888:2888
            - 3888:3888
            - 2181:2181
        environment:
            SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

volumes:
    hadoop_namenode:
    hadoop_datanode:
    hadoop_historyserver:
    hive_metastore:
    hbase_data:
    hbase_zookeeper_data:
    spark-events:


networks:
    hadoop:
        name: hadoop