Skip to content
Browse files

warden: Support spawn/link semantics

This enables client connections to go away without killing the scripts
that it started. Clients can reconnect and re-attach to the scripts they
started to wait until completion, or to reap the status when the script
has already exited.

Change-Id: Id9b89f241d9bba89b5ea211b5737d957ca48c62a
  • Loading branch information...
1 parent 09aff5b commit 56411bb83a176da6d288e6b51fde6220fbc5a60f @pietern pietern committed
View
95 warden/lib/warden/container/base.rb
@@ -56,12 +56,20 @@ def new(conn)
def setup
# noop
end
+
+ # Generates process-wide unique job IDs
+ def generate_job_id
+ @job_id ||= 0
+ @job_id += 1
+ end
end
attr_reader :connections
+ attr_reader :jobs
def initialize
@connections = ::Set.new
+ @jobs = {}
@created = false
@destroyed = false
end
@@ -105,16 +113,6 @@ def container_path
File.join(root_path, ".instance-#{handle}")
end
- def sh(command)
- handler = ::EM.popen(command, ScriptHandler)
- yield handler if block_given?
- handler.yield # Yields fiber
-
- rescue WardenError
- error "error running: #{command.inspect}"
- raise
- end
-
def create
if @created
raise WardenError.new("container is already created")
@@ -161,7 +159,7 @@ def do_destroy
raise WardenError.new("not implemented")
end
- def run(script)
+ def spawn(script)
unless @created
raise WardenError.new("container is not yet created")
end
@@ -170,11 +168,80 @@ def run(script)
raise WardenError.new("container is already destroyed")
end
- do_run(script)
+ job = create_job(script)
+ jobs[job.job_id.to_s] = job
+
+ # Return job id to caller
+ job.job_id
end
- def do_run(script)
- raise WardenError.new("not implemented")
+ def link(job_id)
+ unless @created
+ raise WardenError.new("container is not yet created")
+ end
+
+ if @destroyed
+ raise WardenError.new("container is already destroyed")
+ end
+
+ job = jobs[job_id.to_s]
+ unless job
+ raise WardenError.new("no such job")
+ end
+
+ job.yield
+ end
+
+ def run(script)
+ link(spawn(script))
+ end
+
+ protected
+
+ def sh(command)
+ handler = ::EM.popen(command, ScriptHandler)
+ yield handler if block_given?
+ handler.yield # Yields fiber
+
+ rescue WardenError
+ error "error running: #{command.inspect}"
+ raise
+ end
+
+ class Job
+
+ attr_reader :container
+ attr_reader :job_id
+ attr_reader :path
+
+ def initialize(container)
+ @container = container
+ @job_id = container.class.generate_job_id
+ @path = File.join("tmp", job_id.to_s)
+
+ @status = nil
+ @yielded = []
+ end
+
+ def finish
+ exit_status_path = File.join(container.container_root_path, path, "exit_status")
+ stdout_path = File.join(container.container_root_path, path, "stdout")
+ stderr_path = File.join(container.container_root_path, path, "stderr")
+
+ exit_status = File.read(exit_status_path) if File.exist?(exit_status_path)
+ exit_status = exit_status.to_i if exit_status && !exit_status.empty?
+ stdout_path = nil unless File.exist?(stdout_path)
+ stderr_path = nil unless File.exist?(stderr_path)
+
+ @status = [exit_status, stdout_path, stderr_path]
+ @yielded.each { |f| f.resume(@status) }
+ end
+
+ def yield
+ return @status if @status
+ @yielded << Fiber.current
+ Fiber.yield
+ end
end
end
end
View
19 warden/lib/warden/container/insecure.rb
@@ -38,7 +38,7 @@ def do_destroy
debug "container destroyed"
end
- def do_run(script)
+ def create_job(script)
# Store script in temporary file. This is done because run.sh moves the
# subshell that actually runs the script to the background, and with
# that closes its stdin. In addition, we cannot capture stdin before
@@ -48,20 +48,13 @@ def do_run(script)
stdin.write(script)
stdin.close
- # Run script
- command = "#{container_path}/run.sh #{stdin.path}"
+ # Create new job and run script
+ job = Job.new(self)
+ command = "env job_path=#{container_root_path}/#{job.path} #{container_path}/run.sh #{stdin.path}"
handler = ::EM.popen(command, RemoteScriptHandler)
- result = handler.yield { error "runner unexpectedly terminated" }
- debug "runner successfully terminated: #{result.inspect}"
+ handler.callback { job.finish }
- # Mix in path to the container's root path
- status, stdout_path, stderr_path = result
- stdout_path = File.join(container_root_path, stdout_path) if stdout_path
- stderr_path = File.join(container_root_path, stderr_path) if stderr_path
- [status, stdout_path, stderr_path]
-
- ensure
- stdin.close!
+ job
end
end
end
View
18 warden/lib/warden/container/lxc.rb
@@ -63,13 +63,19 @@ def do_destroy
debug "container destroyed"
end
- def do_run(script)
+ def create_job(script)
socket_path = File.join(container_root_path, "/tmp/runner.sock")
unless File.exist?(socket_path)
error "socket does not exist: #{socket_path}"
end
+ job = Job.new(self)
handler = ::EM.connect_unix_domain(socket_path, RemoteScriptHandler)
+
+ # Send path to job artifact directory on the first line
+ handler.send_data(job.path + "\n")
+
+ # The remainder of stdin will be consumed by a subshell
handler.send_data(script + "\n")
# Make bash exit without losing the exit status. This can otherwise
@@ -77,15 +83,9 @@ def do_run(script)
# on stdin for the remote. However, EM doesn't do shutdown...
handler.send_data "exit $?\n"
- # Wait for bash to exit and eof.
- result = handler.yield { error "runner unexpectedly terminated" }
- debug "runner successfully terminated: #{result.inspect}"
+ handler.callback { job.finish }
- # Mix in path to the container's root path
- status, stdout_path, stderr_path = result
- stdout_path = File.join(container_root_path, stdout_path) if stdout_path
- stderr_path = File.join(container_root_path, stderr_path) if stderr_path
- [status, stdout_path, stderr_path]
+ job
end
end
end
View
14 warden/lib/warden/container/remote_script_handler.rb
@@ -6,18 +6,10 @@ module Container
class RemoteScriptHandler < ScriptHandler
+ # This handler is only interesting in knowning when the descriptor was
+ # closed. Success/failure is determined by other logic.
def unbind
- if buffer.empty?
- # The wrapper script was terminated before it could return anything.
- # It is likely that the container was destroyed while the script
- # was being executed.
- set_deferred_failure "execution aborted"
- else
- status, path = buffer.chomp.split
- stdout_path = File.join(path, "stdout") if path
- stderr_path = File.join(path, "stderr") if path
- set_deferred_success [status.to_i, stdout_path, stderr_path]
- end
+ set_deferred_success
end
end
end
View
12 warden/root/insecure/.instance-skeleton/run.sh
@@ -4,10 +4,18 @@
self=$(readlink -f ${0})
cd $(dirname ${self})
+# Determine artifact path for this job
+if [ -n "${job_path}" ]; then
+tmp=${job_path}
+else
+tmp=$(mktemp -d)
+fi
+
+mkdir -p ${tmp}
+
# Run script with PWD=root. Bash closes stdin for processes that is moves to
# the background so we need to pass the script via a temporary file.
cd root
-tmp=$(mktemp --tmpdir=. -d)
cat ${1:-/dev/null} > ${tmp}/stdin
env -i bash < ${tmp}/stdin 1> ${tmp}/stdout 2> ${tmp}/stderr &
cd ..
@@ -19,4 +27,4 @@ touch pids/${parent_pid}
wait ${child_pid} 2> /dev/null
child_exit_status=${?}
rm pids/${parent_pid}
-echo ${child_exit_status} ${tmp}
+echo ${child_exit_status} > ${tmp}/exit_status
View
14 warden/root/lxc/.instance-skeleton/setup.rb
@@ -57,10 +57,20 @@ def find_root(lines)
# Add runner script
write "usr/bin/runner", <<-EOS
-#!/bin/sh
+#!/bin/bash
+
+# Determine artifact path for this job
+read job_path
+if [ -n "${job_path}" ]; then
+tmp=${job_path}
+else
tmp=$(mktemp -d)
+fi
+
+mkdir -p ${tmp} || exit 1
+
sudo -u vcap env -i bash 1> ${tmp}/stdout 2> ${tmp}/stderr
-echo ${?} ${tmp}
+echo ${?} > ${tmp}/exit_status
EOS
script "chmod +x usr/bin/runner"
View
7 warden/spec/support/examples/warden_server.rb
@@ -80,10 +80,9 @@
reply = other_client.call("destroy", @handle)
reply.should == "ok"
- # Expect an error for the running command
- lambda {
- client.read
- }.should raise_error(/execution aborted/i)
+ # The command should not have exited cleanly
+ reply = client.read
+ reply[0].should_not == 0
end
end
end

0 comments on commit 56411bb

Please sign in to comment.
Something went wrong with that request. Please try again.