added dockerfiles for different stack images of spark-master

Dockerfile.jupyter-all

@@ -0,0 +1,48 @@
+FROM jupyter/all-spark-notebook:ae885c0a6226
+
+ARG dev_mode=false
+
+USER root
+
+# This is needed because requests-kerberos fails to install on debian due to missing linux headers
+RUN conda install requests-kerberos -y
+
+USER $NB_USER
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade --ignore-installed setuptools
+
+#this is needed for the autovizwidget to work properly
+RUN pip install pandas --upgrade
+
+COPY examples /home/jovyan/work
+
+# Install sparkmagic - if DEV_MODE is set, use the one in the host directory.
+# Otherwise, just install from pip.
+COPY hdijupyterutils hdijupyterutils/
+COPY autovizwidget autovizwidget/
+COPY sparkmagic sparkmagic/
+RUN if [ "$dev_mode" = "true" ]; then \
+      cd hdijupyterutils && pip install . && cd ../ && \
+      cd autovizwidget && pip install . && cd ../ && \
+      cd sparkmagic && pip install . && cd ../ ; \
+    else pip install sparkmagic ; fi
+
+RUN mkdir /home/$NB_USER/.sparkmagic
+COPY sparkmagic/example_config.json /home/$NB_USER/.sparkmagic/config.json
+RUN sed -i 's/localhost/spark/g' /home/$NB_USER/.sparkmagic/config.json
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pyspark3kernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkrkernel
+RUN jupyter serverextension enable --py sparkmagic
+
+USER root
+RUN chown $NB_USER /home/$NB_USER/.sparkmagic/config.json
+RUN rm -rf hdijupyterutils/ autovizwidget/ sparkmagic/
+
+CMD ["start-notebook.sh", "--NotebookApp.iopub_data_rate_limit=1000000000"]
+
+USER $NB_USER
+

Dockerfile.jupyter-datascience

@@ -0,0 +1,45 @@
+FROM jupyter/datascience-notebook:ae885c0a6226
+
+ARG dev_mode=false
+
+USER root
+
+# This is needed because requests-kerberos fails to install on debian due to missing linux headers
+RUN conda install requests-kerberos -y
+
+USER $NB_USER
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade --ignore-installed setuptools
+
+COPY examples /home/jovyan/work
+
+# Install sparkmagic - if DEV_MODE is set, use the one in the host directory.
+# Otherwise, just install from pip.
+COPY hdijupyterutils hdijupyterutils/
+COPY autovizwidget autovizwidget/
+COPY sparkmagic sparkmagic/
+RUN if [ "$dev_mode" = "true" ]; then \
+      cd hdijupyterutils && pip install . && cd ../ && \
+      cd autovizwidget && pip install . && cd ../ && \
+      cd sparkmagic && pip install . && cd ../ ; \
+    else pip install sparkmagic ; fi
+
+RUN mkdir /home/$NB_USER/.sparkmagic
+COPY sparkmagic/example_config.json /home/$NB_USER/.sparkmagic/config.json
+RUN sed -i 's/localhost/spark/g' /home/$NB_USER/.sparkmagic/config.json
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pyspark3kernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkrkernel
+RUN jupyter serverextension enable --py sparkmagic
+
+USER root
+RUN chown $NB_USER /home/$NB_USER/.sparkmagic/config.json
+RUN rm -rf hdijupyterutils/ autovizwidget/ sparkmagic/
+
+CMD ["start-notebook.sh", "--NotebookApp.iopub_data_rate_limit=1000000000"]
+
+USER $NB_USER
+

Dockerfile.jupyter-minimal

@@ -0,0 +1,45 @@
+FROM jupyter/minimal-notebook:ae885c0a6226
+
+ARG dev_mode=false
+
+USER root
+
+# This is needed because requests-kerberos fails to install on debian due to missing linux headers
+RUN conda install requests-kerberos -y
+
+USER $NB_USER
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade --ignore-installed setuptools
+
+COPY examples /home/jovyan/work
+
+# Install sparkmagic - if DEV_MODE is set, use the one in the host directory.
+# Otherwise, just install from pip.
+COPY hdijupyterutils hdijupyterutils/
+COPY autovizwidget autovizwidget/
+COPY sparkmagic sparkmagic/
+RUN if [ "$dev_mode" = "true" ]; then \
+      cd hdijupyterutils && pip install . && cd ../ && \
+      cd autovizwidget && pip install . && cd ../ && \
+      cd sparkmagic && pip install . && cd ../ ; \
+    else pip install sparkmagic ; fi
+
+RUN mkdir /home/$NB_USER/.sparkmagic
+COPY sparkmagic/example_config.json /home/$NB_USER/.sparkmagic/config.json
+RUN sed -i 's/localhost/spark/g' /home/$NB_USER/.sparkmagic/config.json
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pyspark3kernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkrkernel
+RUN jupyter serverextension enable --py sparkmagic
+
+USER root
+RUN chown $NB_USER /home/$NB_USER/.sparkmagic/config.json
+RUN rm -rf hdijupyterutils/ autovizwidget/ sparkmagic/
+
+CMD ["start-notebook.sh", "--NotebookApp.iopub_data_rate_limit=1000000000"]
+
+USER $NB_USER
+

Dockerfile.jupyter-pyspark

@@ -0,0 +1,45 @@
+FROM jupyter/pyspark-notebook:ae885c0a6226
+
+ARG dev_mode=false
+
+USER root
+
+# This is needed because requests-kerberos fails to install on debian due to missing linux headers
+RUN conda install requests-kerberos -y
+
+USER $NB_USER
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade --ignore-installed setuptools
+
+COPY examples /home/jovyan/work
+
+# Install sparkmagic - if DEV_MODE is set, use the one in the host directory.
+# Otherwise, just install from pip.
+COPY hdijupyterutils hdijupyterutils/
+COPY autovizwidget autovizwidget/
+COPY sparkmagic sparkmagic/
+RUN if [ "$dev_mode" = "true" ]; then \
+      cd hdijupyterutils && pip install . && cd ../ && \
+      cd autovizwidget && pip install . && cd ../ && \
+      cd sparkmagic && pip install . && cd ../ ; \
+    else pip install sparkmagic ; fi
+
+RUN mkdir /home/$NB_USER/.sparkmagic
+COPY sparkmagic/example_config.json /home/$NB_USER/.sparkmagic/config.json
+RUN sed -i 's/localhost/spark/g' /home/$NB_USER/.sparkmagic/config.json
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pyspark3kernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkrkernel
+RUN jupyter serverextension enable --py sparkmagic
+
+USER root
+RUN chown $NB_USER /home/$NB_USER/.sparkmagic/config.json
+RUN rm -rf hdijupyterutils/ autovizwidget/ sparkmagic/
+
+CMD ["start-notebook.sh", "--NotebookApp.iopub_data_rate_limit=1000000000"]
+
+USER $NB_USER
+

Dockerfile.jupyter-r

@@ -0,0 +1,45 @@
+FROM jupyter/r-notebook:ae885c0a6226
+
+ARG dev_mode=false
+
+USER root
+
+# This is needed because requests-kerberos fails to install on debian due to missing linux headers
+RUN conda install requests-kerberos -y
+
+USER $NB_USER
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade --ignore-installed setuptools
+
+COPY examples /home/jovyan/work
+
+# Install sparkmagic - if DEV_MODE is set, use the one in the host directory.
+# Otherwise, just install from pip.
+COPY hdijupyterutils hdijupyterutils/
+COPY autovizwidget autovizwidget/
+COPY sparkmagic sparkmagic/
+RUN if [ "$dev_mode" = "true" ]; then \
+      cd hdijupyterutils && pip install . && cd ../ && \
+      cd autovizwidget && pip install . && cd ../ && \
+      cd sparkmagic && pip install . && cd ../ ; \
+    else pip install sparkmagic ; fi
+
+RUN mkdir /home/$NB_USER/.sparkmagic
+COPY sparkmagic/example_config.json /home/$NB_USER/.sparkmagic/config.json
+RUN sed -i 's/localhost/spark/g' /home/$NB_USER/.sparkmagic/config.json
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pyspark3kernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkrkernel
+RUN jupyter serverextension enable --py sparkmagic
+
+USER root
+RUN chown $NB_USER /home/$NB_USER/.sparkmagic/config.json
+RUN rm -rf hdijupyterutils/ autovizwidget/ sparkmagic/
+
+CMD ["start-notebook.sh", "--NotebookApp.iopub_data_rate_limit=1000000000"]
+
+USER $NB_USER
+

Dockerfile.jupyter-scipy

@@ -0,0 +1,45 @@
+FROM jupyter/scipy-notebook:ae885c0a6226
+
+ARG dev_mode=false
+
+USER root
+
+# This is needed because requests-kerberos fails to install on debian due to missing linux headers
+RUN conda install requests-kerberos -y
+
+USER $NB_USER
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade --ignore-installed setuptools
+
+COPY examples /home/jovyan/work
+
+# Install sparkmagic - if DEV_MODE is set, use the one in the host directory.
+# Otherwise, just install from pip.
+COPY hdijupyterutils hdijupyterutils/
+COPY autovizwidget autovizwidget/
+COPY sparkmagic sparkmagic/
+RUN if [ "$dev_mode" = "true" ]; then \
+      cd hdijupyterutils && pip install . && cd ../ && \
+      cd autovizwidget && pip install . && cd ../ && \
+      cd sparkmagic && pip install . && cd ../ ; \
+    else pip install sparkmagic ; fi
+
+RUN mkdir /home/$NB_USER/.sparkmagic
+COPY sparkmagic/example_config.json /home/$NB_USER/.sparkmagic/config.json
+RUN sed -i 's/localhost/spark/g' /home/$NB_USER/.sparkmagic/config.json
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pyspark3kernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkrkernel
+RUN jupyter serverextension enable --py sparkmagic
+
+USER root
+RUN chown $NB_USER /home/$NB_USER/.sparkmagic/config.json
+RUN rm -rf hdijupyterutils/ autovizwidget/ sparkmagic/
+
+CMD ["start-notebook.sh", "--NotebookApp.iopub_data_rate_limit=1000000000"]
+
+USER $NB_USER
+

Dockerfile.jupyter-tensorflow

@@ -0,0 +1,45 @@
+FROM jupyter/tensorflow-notebook:ae885c0a6226
+
+ARG dev_mode=false
+
+USER root
+
+# This is needed because requests-kerberos fails to install on debian due to missing linux headers
+RUN conda install requests-kerberos -y
+
+USER $NB_USER
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade --ignore-installed setuptools
+
+COPY examples /home/jovyan/work
+
+# Install sparkmagic - if DEV_MODE is set, use the one in the host directory.
+# Otherwise, just install from pip.
+COPY hdijupyterutils hdijupyterutils/
+COPY autovizwidget autovizwidget/
+COPY sparkmagic sparkmagic/
+RUN if [ "$dev_mode" = "true" ]; then \
+      cd hdijupyterutils && pip install . && cd ../ && \
+      cd autovizwidget && pip install . && cd ../ && \
+      cd sparkmagic && pip install . && cd ../ ; \
+    else pip install sparkmagic ; fi
+
+RUN mkdir /home/$NB_USER/.sparkmagic
+COPY sparkmagic/example_config.json /home/$NB_USER/.sparkmagic/config.json
+RUN sed -i 's/localhost/spark/g' /home/$NB_USER/.sparkmagic/config.json
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pyspark3kernel
+RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkrkernel
+RUN jupyter serverextension enable --py sparkmagic
+
+USER root
+RUN chown $NB_USER /home/$NB_USER/.sparkmagic/config.json
+RUN rm -rf hdijupyterutils/ autovizwidget/ sparkmagic/
+
+CMD ["start-notebook.sh", "--NotebookApp.iopub_data_rate_limit=1000000000"]
+
+USER $NB_USER
+

README.md

@@ -171,6 +171,40 @@ re-build the container. This will cause the container to install your
 local version of autovizwidget, hdijupyterutils, and sparkmagic. Make
 sure to re-run `docker-compose build` before each test run.
 
+### Alternative images
+
+If you want a different stack image for jupyter, just uncommnent the proper lines in the docker-compose.yml file.
+
+	jupyter:
+		image: jupyter/sparkmagic
+		#image: darkice01/sparkmagic-minimal
+		#image: darkice01/sparkmagic-scipy
+		#image: darkice01/sparkmagic-r
+		#image: darkice01/sparkmagic-tensorflow
+		#image: darkice01/sparkmagic-datascience
+		#image: darkice01/sparkmagic-pyspark
+		#image: darkice01/sparkmagic-all
+
+		build:
+		context: .
+
+		dockerfile: Dockerfile.jupyter
+		#dockerfile: Dockerfile.jupyter-minimal
+		#dockerfile: Dockerfile.jupyter-scipy
+		#dockerfile: Dockerfile.jupyter-r
+		#dockerfile: Dockerfile.jupyter-tensorflow
+		#dockerfile: Dockerfile.jupyter-datascience
+		#dockerfile: Dockerfile.jupyter-pyspark
+		#dockerfile: Dockerfile.jupyter-all
+
+		args:
+        		dev_mode: "false"
+
+and then simply run:
+
+	docker-compose build
+    	docker-compose up
+
 ## Server extension API
 
 ### `/reconnectsparkmagic`:

docker-compose.yml

@@ -10,9 +10,23 @@ services:
       - "8998:8998"
   jupyter:
     image: jupyter/sparkmagic
+    #image: darkice01/sparkmagic-minimal
+    #image: darkice01/sparkmagic-scipy
+    #image: darkice01/sparkmagic-r
+    #image: darkice01/sparkmagic-tensorflow
+    #image: darkice01/sparkmagic-datascience
+    #image: darkice01/sparkmagic-pyspark
+    #image: darkice01/sparkmagic-all
     build:
       context: .
       dockerfile: Dockerfile.jupyter
+      #dockerfile: Dockerfile.jupyter-minimal
+      #dockerfile: Dockerfile.jupyter-scipy
+      #dockerfile: Dockerfile.jupyter-r
+      #dockerfile: Dockerfile.jupyter-tensorflow
+      #dockerfile: Dockerfile.jupyter-datascience
+      #dockerfile: Dockerfile.jupyter-pyspark
+      #dockerfile: Dockerfile.jupyter-all
       args:
         dev_mode: "false"
     links: