Change getting-start docker file to use official spark image from outdated jupyter image

getting-started/spark/docker-compose.yml

@@ -52,4 +52,4 @@ services:
       AWS_REGION: us-west-2
       POLARIS_HOST: polaris
     volumes:
-      - ./notebooks:/home/jovyan/notebooks
+      - ./notebooks:/home/spark/notebooks

getting-started/spark/notebooks/Dockerfile

@@ -17,15 +17,32 @@
 # under the License.
 #
 
-FROM jupyter/all-spark-notebook:spark-3.5.0
+FROM docker.io/apache/spark:3.5.6-java17
 
-ENV LANGUAGE='en_US:en'
+ENV PYTHONPATH="${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.9.7-src.zip:/home/spark/venv/lib/python3.10/site-packages"
+ENV PYSPARK_PYTHON=/home/spark/venv/bin/python
 
-COPY --chown=jovyan client /home/jovyan/client
-COPY --chown=jovyan regtests/requirements.txt /tmp
-RUN pip install -r /tmp/requirements.txt && \
-    cd client/python && poetry lock && \
-    python3 -m poetry install && \
-    pip install -e .
+USER root
 
-WORKDIR /home/jovyan/
+RUN apt-get update -y && \
+    apt-get install -y python3-venv && \
+    mkdir -p /home/spark && \
+    chown -R spark /home/spark
+
+USER spark
+
+WORKDIR /home/spark
+
+COPY --chown=spark client /home/spark/client
+COPY --chown=spark regtests/requirements.txt /tmp
+COPY --chown=spark regtests/notebook_requirements.txt /tmp
+
+RUN python3 -m venv /home/spark/venv && \
+    . /home/spark/venv/bin/activate && \
+    pip install -r /tmp/requirements.txt -r /tmp/notebook_requirements.txt && \
+    cd client/python && \
+    poetry lock && \
+    poetry install --all-extras
+
+EXPOSE 8888
+CMD ["/home/spark/venv/bin/jupyter", "lab", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--NotebookApp.token=''"]

getting-started/spark/notebooks/SparkPolaris.ipynb

@@ -792,7 +792,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.10.12"
   },
   "toc-autonumbering": false,
   "toc-showmarkdowntxt": false,

plugins/spark/v3.5/getting-started/notebooks/Dockerfile

@@ -34,14 +34,13 @@ USER spark
 WORKDIR /home/spark
 
 COPY --chown=spark client /home/spark/client
-COPY --chown=spark plugins/spark/v3.5/getting-started/notebooks/requirements.txt /tmp/notebook_requirements.txt
 COPY --chown=spark regtests/requirements.txt /tmp
+COPY --chown=spark regtests/notebook_requirements.txt /tmp
 COPY --chown=spark plugins/spark/v3.5/spark/build/2.12/libs /home/spark/polaris_libs
 
-SHELL ["/bin/bash", "-c"]
 
 RUN python3 -m venv /home/spark/venv && \
-    source /home/spark/venv/bin/activate && \
+    . /home/spark/venv/bin/activate && \
     pip install -r /tmp/requirements.txt -r /tmp/notebook_requirements.txt && \
     cd client/python && \
     poetry lock && \