feat(core): add shutdown hook to avoid killing running jobs (#245)

rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/JobExecutor.groovy

@@ -23,4 +23,5 @@ interface JobExecutor {
   boolean jobExists(String jobId)
   BakeStatus updateJob(String jobId)
   void cancelJob(String jobId)
+  int runningJobCount()
 }

rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/config/LocalJobConfig.groovy

@@ -19,6 +19,7 @@ package com.netflix.spinnaker.rosco.jobs.config
 import com.netflix.spinnaker.rosco.jobs.JobExecutor
 import com.netflix.spinnaker.rosco.jobs.local.JobExecutorLocal
 import groovy.util.logging.Slf4j
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -30,6 +31,21 @@ class LocalJobConfig {
   @Bean
   @ConditionalOnMissingBean(JobExecutor)
   JobExecutor jobExecutorLocal() {
-    new JobExecutorLocal()
+    JobExecutor jobExecutor = new JobExecutorLocal()
+
+    Runtime.runtime.addShutdownHook(new Thread(new Runnable() {
+      @Override
+      void run() {
+        log.info("Running job executor shutdown hook...")
+        int jobCount = jobExecutor.runningJobCount()
+        while (jobCount > 0) {
+          log.info("Waiting on $jobCount jobs before shutting down...")
+          sleep(1000)
+          jobCount = jobExecutor.runningJobCount()
+        }
+      }
+    }))
+
+    return jobExecutor
   }
 }

rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/jobs/local/JobExecutorLocal.groovy

@@ -21,18 +21,23 @@ import com.netflix.spinnaker.rosco.api.BakeStatus
 import com.netflix.spinnaker.rosco.jobs.JobExecutor
 import com.netflix.spinnaker.rosco.jobs.JobRequest
 import groovy.util.logging.Slf4j
-import org.apache.commons.exec.*
+import org.apache.commons.exec.CommandLine
+import org.apache.commons.exec.DefaultExecuteResultHandler
+import org.apache.commons.exec.DefaultExecutor
+import org.apache.commons.exec.ExecuteWatchdog
+import org.apache.commons.exec.Executor
+import org.apache.commons.exec.PumpStreamHandler
+import org.apache.commons.exec.Watchdog
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Value
 import rx.Scheduler
 import rx.functions.Action0
 import rx.schedulers.Schedulers
 
+import javax.annotation.PostConstruct
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.ToDoubleFunction
 
-import javax.annotation.PostConstruct
-
 @Slf4j
 class JobExecutorLocal implements JobExecutor {
 
@@ -182,6 +187,11 @@ class JobExecutorLocal implements JobExecutor {
     // The next polling interval will be unable to retrieve the job status and will mark the bake as canceled.
   }
 
+  @Override
+  int runningJobCount() {
+    return jobIdToHandlerMap.keySet().size()
+  }
+
   @PostConstruct
   void initializeMetrics() {
     // We need to have at least one tag.