Output checkpoint

cmd/cloudexec/user_data.sh.tmpl

@@ -121,6 +121,18 @@ s3cmd() {
 		"$@"
 }
 
+upload_output() {
+
+	output_files="$(ls -A "${output_dir}/")"
+	if [[ -n ${output_files} ]]; then
+		echo "Uploading results..."
+		s3cmd put -r "${output_dir}"/* "s3://${BUCKET_NAME}/job-${JOB_ID}/output/"
+	else
+		echo "Skipping results upload, no files found in ${output_dir}"
+	fi
+
+}
+
 # Define a cleanup function that will be executed on signals or exit
 export COMPLETED=false
 export TIMEDOUT=false
@@ -130,13 +142,7 @@ cleanup() {
 		update_state "failed"
 	fi
 
-	output_files="$(ls -A "${output_dir}/")"
-	if [[ -n ${output_files} ]]; then
-		echo "Uploading results..."
-		s3cmd put -r "${output_dir}"/* "s3://${BUCKET_NAME}/job-${JOB_ID}/output/"
-	else
-		echo "Skipping results upload, no files found in ${output_dir}"
-	fi
+	upload_output
 
 	if [[ -s ${stdout_log} ]]; then
 		echo
@@ -252,10 +258,14 @@ wrapped_run_command="$(
 tmux new-session -d -s "${tmux_session}" "${wrapped_run_command}"
 
 start_time=$(date "+%s")
+sync_heartbeat=60
+next_sync=$(("${start_time}" + "${sync_heartbeat}"))
 pretty_start_time="$(fmtDate "${start_time}")"
 end_time=$(("${start_time}" + TIMEOUT))
 pretty_end_time="$(fmtDate "${end_time}")"
 echo "Workload is running, timer started at ${pretty_start_time}, we'll time out at ${pretty_end_time}"
+echo "================================================================================================"
+echo
 
 ########################################
 # Wait for job to finish
@@ -286,5 +296,11 @@ while true; do
 		break
 	fi
 
+	if [[ ${current_time} -gt ${next_sync} ]]; then
+		echo "Uploading output at ${current_time}"
+		upload_output
+		next_sync=$(("${next_sync}" + "${sync_heartbeat}"))
+	fi
+
 	sleep 1s
 done

pkg/state/state.go

@@ -199,12 +199,16 @@ func GetLatestCompletedJob(bucketName string, state *State) (*JobInfo, error) {
 	// Find the latest completed job
 	for i := len(state.Jobs) - 1; i >= 0; i-- {
 		job := state.Jobs[i]
-		if job.Status == Completed {
+		if job.Status == Completed || job.Status == Failed {
 			latestCompletedJob = &job
 			break
 		}
 	}
 
+	if latestCompletedJob == nil {
+		return nil, fmt.Errorf("No completed jobs available")
+	}
+
 	return latestCompletedJob, nil
 }