diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..78799e0 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,49 @@ +name: Deploy Docker image + +on: + release: + types: [published] + + # Do not run when README gets updated + paths-ignore: + - '**/README.md' + + # Can trigger action manually + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-deploy-image: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Log in to the container registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build & push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4db4622 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,62 @@ +# Ubuntu as the base image +FROM ubuntu:20.04 + +# Set working directory to /home +WORKDIR / + +# Install required dependencies +RUN apt-get update && apt-get install -y \ + openjdk-8-jdk \ + openssh-server \ + openssh-client \ + nano \ + && rm -rf /var/lib/apt/lists/* + +# Generate SSH key pair for password less login +RUN ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa \ + && cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys \ + && chmod 0600 ~/.ssh/authorized_keys + +# Download Hadoop 3.3.1 +RUN wget https://mirrors.estointernet.in/apache/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz + +# Unzip the .tar.gz +RUN tar xzf hadoop-3.3.1.tar.gz + +# Remove the .tar.gz file +RUN rm ./hadoop-3.3.1.tar.gz + +# Hadoop home +ENV HADOOP_HOME=/hadoop-3.3.1 + +# Other Hadoop environment variables +ENV HADOOP_INSTALL=${HADOOP_HOME} \ + HADOOP_MAPRED_HOME=${HADOOP_HOME} \ + HADOOP_COMMON_HOME=${HADOOP_HOME} \ + HADOOP_HDFS_HOME=${HADOOP_HOME} \ + YARN_HOME=${HADOOP_HOME} \ + HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_HOME}/lib/native \ + PATH=$PATH:${HADOOP_HOME}/sbin:${HADOOP_HOME}/bin \ + HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/nativ" \ + + # Java home + JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ \ + + # For start-all.sh + HDFS_NAMENODE_USER="root" \ + HDFS_DATANODE_USER="root" \ + HDFS_SECONDARYNAMENODE_USER="root" \ + YARN_RESOURCEMANAGER_USER="root" \ + YARN_NODEMANAGER_USER="root" + +# Dump environment variables since connecting +# to localhost via SSH wipes them out +RUN env | grep _ >> /etc/environment + +# Copy Hadoop configuration files to the "etc" directory +COPY /etc/* ${HADOOP_HOME}/etc/hadoop/ + +# Copy bootstrap.sh +COPY ./bootstrap.sh / + +CMD [ "bash", "./bootstrap.sh" ] diff --git a/LICENSE b/LICENSE index 261eeb9..dd61706 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2021 D. Kasi Pavan Kumar Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index f9981e1..f71007f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,41 @@ -# hadoop-docker -Apache Hadoop's Pseudo Distributed Mode using Docker. 🐳 +# Apache Hadoop using Docker 🐳 + +A Docker image to play around with [Apache Hadoop](https://hadoop.apache.org) in [Pseudo Distributed Mode](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SingleCluster.html) (single cluster mode). + +### Below are the steps to play around with this image using [Play with Docker](https://labs.play-with-docker.com). + +1. First of all, create an account on [Docker Hub](https://hub.docker.com/signup). +2. Login to [Play with Docker](https://labs.play-with-docker.com) using the Docker Hub account you just created. +3. You should see a green "Start" button, click on it to start a session. +4. Create an instance by clicking on "+ Add new instance" in the left pane, to create a VM. +5. A new terminal should show up in the right pane. Here, we need to pull the Docker image from _Github Container Registry (GHCR)_. +To do so, execute: + +```bash +docker pull ghcr.io/max-rocco/hadoop-docker:main +``` + +6. After the image has been pulled into the VM, we need to start a new container & switch into it's terminal (mostly bash). +To do so, execute: + +```bash +docker run -it ghcr.io/max-rocco/hadoop-docker:main +``` + +_At this stage, the image will be booting up by executing all the required steps to start Hadoop._ + +**From now on, you will be inside container's bash (terminal) and can start using Hadoop's filesystem commands.** 🚀 + +