t# This is a combination of 3 commits. (#88)

add pandas to spark base image Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> upper bound for numpy version Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> fix Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
feast-dev · Aug 13, 2021 · bddcae2 · bddcae2
1 parent 76451a3
commit bddcae2
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 6 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -46,13 +46,22 @@ jobs:
         run: make lint-python
 
   unit-test-python:
-    runs-on: [ubuntu-latest]
+    runs-on: ubuntu-latest
     needs: lint-python
-    container: gcr.io/kf-feast/feast-ci:latest
     env:
       PYSPARK_PYTHON: python3.7
     steps:
       - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v1
+        with:
+          java-version: '8'
+          java-package: jdk
+          architecture: x64
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+          architecture: 'x64'
       - name: Install python
         run: make install-python
       - name: Test python

diff --git a/infra/docker/spark/Dockerfile b/infra/docker/spark/Dockerfile
@@ -35,6 +35,10 @@ RUN echo 'spark.driver.extraJavaOptions="-Dcom.google.cloud.spark.bigquery.repac
 RUN echo 'spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
 RUN echo 'spark.executor.extraJavaOptions="-Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
 
+# python dependencies
+RUN pip3 install -U pip wheel
+RUN pip3 install pandas pyarrow==2.0.0 'numpy<1.20.0'
+
 # For logging to /dev/termination-log
 RUN mkdir -p /dev
 

diff --git a/infra/scripts/helm/k8s-jobservice.tpl.yaml b/infra/scripts/helm/k8s-jobservice.tpl.yaml
@@ -37,7 +37,7 @@ feast-jobservice:
           memory: "600m"
           labels:
             version: 3.0.2
-          javaOptions: "-Dio.netty.tryReflectionSetAccessible=true"
+          javaOptions: "-Dio.netty.tryReflectionSetAccessible=true -Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"
           secrets:
             - name: feast-gcp-service-account
               path: /mnt/secrets
@@ -48,7 +48,7 @@ feast-jobservice:
           memory: "800m"
           labels:
             version: 3.0.2
-          javaOptions: "-Dio.netty.tryReflectionSetAccessible=true"
+          javaOptions: "-Dio.netty.tryReflectionSetAccessible=true -Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"
           secrets:
             - name: feast-gcp-service-account
               path: /mnt/secrets

diff --git a/spark/ingestion/src/test/resources/python/setup.sh b/spark/ingestion/src/test/resources/python/setup.sh
@@ -8,11 +8,11 @@ if [[ -f "$DESTINATION/libs.tar.gz" ]]; then
     echo "$DESTINATION/libs.tar.gz exists."
 else
   tmp_dir=$(mktemp -d)
-  pip3 install -t ${tmp_dir}/libs great-expectations pyarrow==2.0.0
+  pip3 install -t ${tmp_dir}/libs great-expectations==0.13.2 pyarrow==2.0.0
   cd $tmp_dir && tar -czf libs.tar.gz libs/ && mv libs.tar.gz $DESTINATION/libs.tar.gz
 fi
 
 # 2. Pickle python udf
 cd $CURRENT_PATH
-pip3 install great-expectations setuptools pyspark==3.0.1
+pip3 install great-expectations==0.13.2 setuptools pyspark==3.0.1
 python3 udf.py $DESTINATION/udf.pickle