Skip to content

Commit

Permalink
Switched from using chroot jail to using dotcloud's docker framework
Browse files Browse the repository at this point in the history
  • Loading branch information
@ironcamel
ironcamel committed May 3, 2013
1 parent 2a7807c commit 5c85b21
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 164 deletions.
210 changes: 46 additions & 164 deletions bin/codejail.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,25 @@
use strict;
use warnings;
use v5.10;
use Data::Dump qw(dd dump);
use File::Copy::Recursive qw(rcopy);
use Data::Dump qw(dump);
use File::Slurp qw(write_file);
use FindBin qw($RealBin);
use IO::File;
use IPC::Run qw(run timeout);
use IPC::Run qw(run);
use JSON qw(decode_json encode_json);
use LWP::UserAgent;
use Net::Stomp;
use POSIX qw(setgid);
use Try::Tiny;
use YAML qw(LoadFile);

my $AGENT = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 });
my $AGENT = LWP::UserAgent->new;
my $config = LoadFile("$RealBin/../config.yml");
my $log_file = $config->{log_file} || "/tmp/codejail.log";
open my $LOG, '>>', $log_file or die "Could not open $log_file : $!\n";
$LOG->autoflush(1);
STDOUT->autoflush(1);
my $jail = $config->{jail_path} or die "No jail_path configured";
my $queue = $config->{queue} || '/queue/codejail';
my $docker_bin = "/opt/go/bin/docker";

my $stomp = Net::Stomp->new({
hostname => $config->{plugins}{Stomp}{default}{hostname},
Expand All @@ -32,28 +30,18 @@
$stomp->connect();
$stomp->subscribe({ destination => $queue, ack => 'client' });

$SIG{INT} = sub {
$stomp->disconnect;
say 'goodbye';
exit;
};
$SIG{PIPE} = 'IGNORE';

say "worker is starting";

# main loop --------------------------------------------------------------------

while (1) {
my $frame = $stomp->receive_frame;
my $msg = $frame->body;
my $data;
say "processing msg ...";
try {
$data = decode_json($msg);
process_msg($data);
process_msg($msg);
} catch {
debug("failed to process msg: $_");
post_result(0, $_, $data);
debug("failed to process job: $_");
} finally {
$stomp->ack({ frame => $frame });
};
Expand All @@ -62,155 +50,49 @@
# functions -------------------------------------------------------------------

sub process_msg {
my ($data) = @_;
log_msg($data);
my $out = run_code($data);
return post_result(-1, $out, $data) if $out =~ /^ERROR:/;
my $status = $out eq $data->{problem}{output} ? 1 : 0;
my $reason = $status == 1 ? 'Success' : 'Wrong answer';
post_result($status, $reason, $data);
my ($msg) = @_;

my $data = decode_json $msg;
debug($data);
my $dir = "/tmp/content";
run "rm -rf $dir";
run "mkdir $dir";
run "cp $RealBin/docker-agent.pl $dir";
write_file "$dir/data.json", $msg;
get_bundle($data->{env_bundle_url}, "$dir/env_bundle.tar")
if $data->{env_bundle_url};
get_bundle($data->{lib_bundle_url}, "$dir/env_bundle.tar")
if $data->{lib_bundle_url};

my $cmd = join ' ',
'tar -C /tmp -c content |',
"$docker_bin run -u sandbox -i -a stdin ironcamel/test1 /bin/bash -c",
qw("
cd /tmp;
tar -x;
perl ./content/docker-agent.pl;
tar -C ./content -c results;
");
my $docker_run_id = `$cmd`;
chomp $docker_run_id;
debug("docker_run_id: $docker_run_id");
sleep 3;
run "$docker_bin logs $docker_run_id > results.tar";
debug("Results are ready!");
}

sub run_code {
my ($data) = @_;

sys("rm -rf $jail/home/sandbox/runs/*");
#sys("tar -xf $jail.tar -C /var") or die "Could not build chroot jail: $!";

my $pid = open my $child_process, '-|';
if (!$pid) { # child process starts here
try {
run_in_chroot($data);
} catch {
say "ERROR: $_";
};
exit;
} # end of child process

# Read output from the child process
my @out = <$child_process>;
close $child_process;
my $result = join '', @out;
debug('output: ' . substr $result, 0, 100);
debug("full: $result");

# Copy result files out of the jail
if (my $results_path = $data->{copy_results_path}) {
my $run_id = $data->{run_id} || 'unknown';
my $target_dir = "$RealBin/../results/$run_id";
rcopy "$jail/$results_path", $target_dir;
}

# Lets clean up after ourselves.
#sys("rm -rf $jail/home/sandbox/runs/*");

return $result;
}

sub run_in_chroot {
my ($data) = @_;
my $code = $data->{code};
my $file_name = $data->{file_name} // 'foo';
my $problem = $data->{problem};
my $compile_cmd = $data->{compile_cmd};
my $run_cmd = $data->{run_cmd};
my $input = $problem->{input};

chdir $jail or die "Failed to chdir into $jail: $!";

if (not glob "$jail/proc/*") {
sys("mount -t proc proc proc/") or die "Could not mount /proc: $!";
}

my $run_dir = "/home/sandbox/runs";
my $chroot_run_dir = $jail . $run_dir;
if (not -d $chroot_run_dir) {
mkdir $chroot_run_dir or die "Could not mkdir $chroot_run_dir: $!";
chmod 0777, $chroot_run_dir or die "Could not chmod $chroot_run_dir";
}
if (my $url = $data->{env_bundle_url}) {
debug("getting bundle from $url");
my $bundle_path = "$chroot_run_dir/bundle.tar";
# For https: LWP::Protocol::https, libssl-dev, libnet-ssleay-perl
my $res = $AGENT->mirror($url, $bundle_path);
debug($res->status_line);
if (!$res->is_success and $res->code != 304) {
die "Could not download bundle: " . $res->status_line;
}
sys("tar xf $bundle_path -C $chroot_run_dir")
or die "Could not expand bundle $bundle_path $!";
}
if (my $url = $data->{lib_bundle_url}) {
debug("getting bundle from $url");
my $lib_dir = "$jail/usr/local/lib/codejail";
my $bundle_path = "$lib_dir/bundle.tar";
my $res = $AGENT->mirror($url, $bundle_path);
debug($res->status_line);
if (!$res->is_success and $res->code != 304) {
die "Could not download bundle: " . $res->status_line;
}
sys("tar xf $bundle_path -C $lib_dir")
or die "Could not expand bundle $bundle_path $!";
}

debug("chroot $jail");
chroot $jail or die "Could not chroot: $!";

# Drop root privileges immediately after successful chroot
my $new_uid = getpwnam('sandbox');
die "Can't find uid for sandbox user" unless defined $new_uid;
$< = $> = $new_uid; # Update the real and effective user id
setgid($new_uid); # Update the group id

debug("chdir $run_dir");
chdir $run_dir or die "Could not chdir to $run_dir: $!";

write_file $file_name, $code;
if ($compile_cmd) {
debug("compiling: ", $compile_cmd);
run $compile_cmd;
}

my ($out, $err) = ('', '');
debug("going to run: @$run_cmd");
try {
run $run_cmd, \$input, \$out, \$err, timeout(3);
} catch {
print "Took too long $_\n";
};
print $out;
}

sub post_result {
my ($status, $reason, $data) = @_;
print "posting result:";
return unless ref $data eq 'HASH' and $data->{cb_url};
my $result = {
status => $status,
reason => $reason,
run_id => $data->{run_id},
user_id => $data->{user_id},
problem => $data->{problem}{title},
};
dd $result;
debug($result);
debug("going to post result to callback url: $data->{cb_url}");
my $res = $AGENT->post($data->{cb_url}, content_type => 'application/json',
content => encode_json($result));
sub get_bundle {
my ($url, $bundle_path) = @_;
debug("getting bundle from $url");
my $res = $AGENT->mirror($url, $bundle_path);
debug($res->status_line);
if (!$res->is_success and $res->code != 304) {
die "Could not download bundle: " . $res->status_line;
}
}

sub debug { say $LOG '[' . localtime . '] (DEBUG) ' . dump(@_) }

sub log_msg {
my $data = shift;
my %copy = %$data;
$copy{problem} = $copy{problem}{title};
debug(\%copy)
}

sub sys {
my ($cmd) = @_;
debug($cmd);
return system($cmd) == 0;
sub debug {
my $msg = '[' . localtime . '] (DEBUG) ' . dump(@_);
say $LOG $msg;
say $msg;
}
57 changes: 57 additions & 0 deletions bin/docker-agent.pl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/usr/bin/env perl
use v5.12;
use warnings;
use File::Copy::Recursive qw(rcopy);
use File::Slurp qw(read_file write_file);
use FindBin qw($RealBin);
use IPC::Run qw(run timeout);
use JSON qw(decode_json encode_json);
use Try::Tiny;

chdir $RealBin;
my $results_dir = 'results';
mkdir $results_dir;

my $data = decode_json read_file 'data.json';
my $code = $data->{code};
my $file_name = $data->{file_name} // 'foo';
my $compile_cmd = $data->{compile_cmd};
my $run_cmd = $data->{run_cmd};
my $run_id = $data->{run_id} // 'unknown';
my $problem = $data->{problem} || {};
my $timeout = $data->{timeout} || 3;
my $input = $problem->{input};
my $copy_results_path = $data->{copy_results_path};
my ($out, $err, $compile_success);

write_file $file_name, $code;
run [qw(tar xf env_bundle.tar)], \'', \$out, \$err;
run [qw(tar xf lib_bundle.tar -C /usr/local/lib/codejail)], \'', \$out, \$err;

if ($compile_cmd) {
$compile_success = try {
run $compile_cmd, \'', \$out, \$err, timeout($timeout);
} catch {
$err = "Compile command took too long $_";
};
write_file "$results_dir/compile_out.log", $out // '';
write_file "$results_dir/compile_err.log", $err // '';
}

exit unless $compile_success;

try {
run $run_cmd, \$input, \$out, \$err, timeout($timeout);
} catch {
$err = "Run command took too long $_";
};
write_file "$results_dir/run_out.log", $out // '';
write_file "$results_dir/run_err.log", $err // '';

# Copy out a custom dir, such as junit results for example
if ($copy_results_path) {
my $dir = "$results_dir/extra";
mkdir $dir;
rcopy $copy_results_path => $dir;
}

0 comments on commit 5c85b21

Please sign in to comment.