Explicitly close spark session after batch ingestion is done

Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
feast-dev · Apr 27, 2021 · b8f0313 · b8f0313
1 parent d04ccd4
commit b8f0313
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala
@@ -114,6 +114,7 @@ object IngestionJob {
           case Modes.Offline =>
             val sparkSession = BasePipeline.createSparkSession(config)
             BatchPipeline.createPipeline(sparkSession, config)
+            sparkSession.close()
           case Modes.Online =>
             val sparkSession = BasePipeline.createSparkSession(config)
             StreamingPipeline.createPipeline(sparkSession, config).get.awaitTermination