add bigdata=pyspark+pyflink

.github/workflows/docker.yml

@@ -20,15 +20,19 @@ env:
 
 jobs:
   qpod_pyflink:
-    name: qpod/pyflink
+    name: qpod/bigdata
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - run: source ./tool.sh && build_image pyflink latest         docker_pyflink/Dockerfile       && push_image
+      - uses: actions/checkout@v3
+      - run: |
+          source ./tool.sh
+          build_image bigdata latest         docker_bigdata/Dockerfile       && push_image
 
   qpod_elasticsearch:
     name: qpod/elasticsearch
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - run: source ./tool.sh && build_image elasticsearch latest   docker_elasticsearch/Dockerfile && push_image
+      - uses: actions/checkout@v3
+      - run: |
+          source ./tool.sh
+          build_image elasticsearch latest   docker_elasticsearch/Dockerfile && push_image

docker_bigdata/Dockerfile

@@ -0,0 +1,23 @@
+# Distributed under the terms of the Modified BSD License.
+
+ARG BASE_NAMESPACE
+ARG BASE_IMG="atom"
+FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
+
+LABEL maintainer="haobibo@gmail.com"
+
+COPY work /opt/utils/
+
+ENV CONDA_PREFIX=/opt/conda
+ENV JAVA_HOME=/opt/jdk
+ENV PATH=$PATH:${CONDA_PREFIX}/bin:${JAVA_HOME}/bin
+
+RUN source /opt/utils/script-setup.sh \
+ && install_apt /opt/utils/install_list_base.apt \
+ && echo "Install tini" && setup_tini \
+ # pyflink does not yet support Python > 3.8
+ && echo "Install Mamba, Python 3.8, and Conda:" && setup_mamba && setup_conda_with_mamba "3.8" \
+ && echo "Install JDK:" && VERSION_JDK=11 setup_java_base \
+ && echo "Install pyflink:" && install_pip /opt/utils/list_install_pip_pyflink.txt \
+ && echo "Install pyspark:" && install_pip /opt/utils/list_install_pip_pyspark.txt \
+ && echo "Clean up" && list_installed_packages && install__clean

docker_pyflink/work/list_install_pip_pyflink.txt → docker_bigdata/work/list_install_pip_pyflink.txt

@@ -1,4 +1,5 @@
 % from: https://github.com/apache/flink/blob/master/flink-python/setup.py
+apache-flink
 pandas
 pyarrow
 requests

docker_bigdata/work/list_install_pip_pyspark.txt

@@ -0,0 +1,4 @@
+% from: https://github.com/apache/spark/blob/master/python/setup.py#L262
+pyspark
+pandas
+pyarrow