Stopping spark at the end of the job can be turned off

leobenkel · May 22, 2020 · 75173d9 · 75173d9
1 parent 9a0da18
commit 75173d9
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/Library/src/main/scala/com/leobenkel/zparkio/ZparkioApp.scala b/Library/src/main/scala/com/leobenkel/zparkio/ZparkioApp.scala
@@ -60,6 +60,8 @@ trait ZparkioApp[C <: CLA.Service, ENV <: ZparkioApp.ZPEnv[C] with Logger, OUTPU
     } yield { makeEnvironment(cliService, logger.log, sparkService) }
   }
 
+  protected def stopSparkAtTheEnd: Boolean = true
+
   protected def app(args: List[String]): ZIO[zio.ZEnv, Throwable, OUTPUT] = {
     for {
       env <- buildEnv(args)
@@ -72,8 +74,15 @@ trait ZparkioApp[C <: CLA.Service, ENV <: ZparkioApp.ZPEnv[C] with Logger, OUTPU
       output <- runApp()
         .provide(env)
         .timeoutFail(ZparkioApplicationTimeoutException())(timedApplication)
-      _ = s.sparkContext.stop()
-      _ = s.stop()
+      _ <- if (stopSparkAtTheEnd) {
+        Task {
+          s.sparkContext.stop()
+          s.stop()
+          ()
+        }
+      } else {
+        Task(())
+      }
     } yield { output }
   }
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-v0.8.0
+v0.8.2