update version to 0.2.21 (#118)

Signed-off-by: Khor Shu Heng <khor.heng@gojek.com>

Co-authored-by: Khor Shu Heng <khor.heng@gojek.com>

Makefile

@@ -64,6 +64,9 @@ push-jobservice-docker:
 build-spark-docker:
 	docker build -t $(REGISTRY)/feast-spark:$(VERSION) --build-arg VERSION=$(VERSION) -f infra/docker/spark/Dockerfile .
 
+build-spark-docker-dev:
+	docker build -t $(REGISTRY)/feast-spark:$(VERSION) --build-arg VERSION=$(VERSION) -f infra/docker/spark/dev.Dockerfile .
+
 push-spark-docker:
 	docker push $(REGISTRY)/feast-spark:$(VERSION)
 

infra/charts/feast-spark/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: Feast Extension for running Ingestion on Spark
 name: feast-spark
-version: 0.2.20
+version: 0.2.21

infra/charts/feast-spark/README.md

@@ -1,6 +1,6 @@
 # feast-spark
 
-Feast Extension for running Ingestion on Spark 0.2.20
+Feast Extension for running Ingestion on Spark 0.2.21
 
 ## Installation
 
@@ -10,7 +10,7 @@ https://docs.feast.dev/v/master/getting-started/deploying-feast/kubernetes
 
 | Repository | Name | Version |
 |------------|------|---------|
-|  | feast-jobservice | 0.2.20 |
+|  | feast-jobservice | 0.2.21 |
 |  | prometheus-statsd-exporter | 0.1.2 |
 
 ## Values

infra/charts/feast-spark/charts/feast-jobservice/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: Feast Job Service manage ingestion jobs.
 name: feast-jobservice
-version: 0.2.20
+version: 0.2.21

infra/charts/feast-spark/charts/feast-jobservice/README.md

@@ -1,6 +1,6 @@
 # feast-jobservice
 
-![Version: 0.2.20](https://img.shields.io/badge/Version-0.2.20-informational?style=flat-square)
+![Version: 0.2.21](https://img.shields.io/badge/Version-0.2.21-informational?style=flat-square)
 
 Feast Job Service manage ingestion jobs.
 

infra/charts/feast-spark/requirements.lock

@@ -1,9 +1,9 @@
 dependencies:
 - name: feast-jobservice
   repository: ""
-  version: 0.2.20
+  version: 0.2.21
 - name: prometheus-statsd-exporter
   repository: ""
   version: 0.1.2
-digest: sha256:11bf58ab1eeccfc3897783678e8726ad39b919460cca4d0ffa354009b9c521f9
-generated: "2022-02-23T14:48:35.932913388+08:00"
+digest: sha256:8349dadd4eeb38d107e09b451b5b1992dcd9f163688cc16dff10116e23076ede
+generated: "2022-02-24T16:36:04.388305806+08:00"

infra/charts/feast-spark/requirements.yaml

@@ -1,6 +1,6 @@
 dependencies:
   - name: feast-jobservice
-    version: 0.2.20
+    version: 0.2.21
     condition: feast-jobservice.enabled
   - name: prometheus-statsd-exporter
     version: 0.1.2

infra/docker/spark/dev.Dockerfile

@@ -0,0 +1,36 @@
+FROM gcr.io/kf-feast/spark-py:3.0.2 as runtime
+
+ARG VERSION=dev
+
+ARG HADOOP_AWS_VERSION=3.2.1
+ARG AWS_JAVA_SDK_VERSION=1.11.874
+ARG TFRECORD_VERSION=0.3.0
+ARG GCS_CONNECTOR_VERSION=2.0.1
+ARG BQ_CONNECTOR_VERSION=0.18.1
+
+USER root
+# Add HADOOP_AWS_JAR and AWS_JAVA_SDK
+ADD https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_AWS_VERSION}/hadoop-aws-${HADOOP_AWS_VERSION}.jar /opt/spark/jars
+ADD https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_JAVA_SDK_VERSION}/aws-java-sdk-bundle-${AWS_JAVA_SDK_VERSION}.jar /opt/spark/jars
+ADD https://repo1.maven.org/maven2/com/linkedin/sparktfrecord/spark-tfrecord_2.12/${TFRECORD_VERSION}/spark-tfrecord_2.12-${TFRECORD_VERSION}.jar /opt/spark/jars
+ADD https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-${GCS_CONNECTOR_VERSION}.jar /opt/spark/jars
+ADD https://repo1.maven.org/maven2/com/google/cloud/spark/spark-bigquery-with-dependencies_2.12/${BQ_CONNECTOR_VERSION}/spark-bigquery-with-dependencies_2.12-${BQ_CONNECTOR_VERSION}.jar /opt/spark/jars
+
+# Fix arrow issue for jdk-11
+RUN mkdir -p /opt/spark/conf
+RUN echo 'spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
+RUN echo 'spark.driver.extraJavaOptions="-Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
+RUN echo 'spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
+RUN echo 'spark.executor.extraJavaOptions="-Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
+
+# python dependencies
+RUN pip3 install -U pip wheel
+RUN pip3 install pandas pyarrow==2.0.0 'numpy<1.20.0'
+
+# For logging to /dev/termination-log
+RUN mkdir -p /dev
+
+COPY spark/ingestion/target/feast-ingestion-spark-${VERSION}.jar /opt/spark/jars
+
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]

pom.xml

@@ -14,7 +14,7 @@
     </modules>
 
     <properties>
-        <revision>0.2.20</revision>
+        <revision>0.2.21</revision>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
         <scala.version>2.12</scala.version>

spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala

@@ -34,7 +34,7 @@ object IngestionJob {
 
   val parser = new scopt.OptionParser[IngestionJobConfig]("IngestionJob") {
     // ToDo: read version from Manifest
-    head("feastSpark.ingestion.IngestionJob", "0.2.20")
+    head("feastSpark.ingestion.IngestionJob", "0.2.21")
 
     opt[Modes]("mode")
       .action((x, c) => c.copy(mode = x))

spark/ingestion/src/main/scala/feast/ingestion/stores/redis/ClusterPipelineProvider.scala

@@ -41,5 +41,7 @@ case class ClusterPipelineProvider(endpoint: RedisEndpoint) extends PipelineProv
   /**
     * Close client connection
     */
-  override def close(): Unit = {}
+  override def close(): Unit = {
+    provider.close()
+  }
 }