Permalink
Browse files

Switched from using chroot jail to using dotcloud's docker framework

  • Loading branch information...
ironcamel committed May 1, 2013
1 parent 2a7807c commit 5c85b21b4226c04e5004f6fe0d45f3e2004fb859
Showing with 103 additions and 164 deletions.
  1. +46 −164 bin/codejail.pl
  2. +57 −0 bin/docker-agent.pl
View
@@ -2,27 +2,25 @@
use strict;
use warnings;
use v5.10;
-use Data::Dump qw(dd dump);
-use File::Copy::Recursive qw(rcopy);
+use Data::Dump qw(dump);
use File::Slurp qw(write_file);
use FindBin qw($RealBin);
use IO::File;
-use IPC::Run qw(run timeout);
+use IPC::Run qw(run);
use JSON qw(decode_json encode_json);
use LWP::UserAgent;
use Net::Stomp;
-use POSIX qw(setgid);
use Try::Tiny;
use YAML qw(LoadFile);
-my $AGENT = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 });
+my $AGENT = LWP::UserAgent->new;
my $config = LoadFile("$RealBin/../config.yml");
my $log_file = $config->{log_file} || "/tmp/codejail.log";
open my $LOG, '>>', $log_file or die "Could not open $log_file : $!\n";
$LOG->autoflush(1);
STDOUT->autoflush(1);
-my $jail = $config->{jail_path} or die "No jail_path configured";
my $queue = $config->{queue} || '/queue/codejail';
+my $docker_bin = "/opt/go/bin/docker";
my $stomp = Net::Stomp->new({
hostname => $config->{plugins}{Stomp}{default}{hostname},
@@ -32,28 +30,18 @@
$stomp->connect();
$stomp->subscribe({ destination => $queue, ack => 'client' });
-$SIG{INT} = sub {
- $stomp->disconnect;
- say 'goodbye';
- exit;
-};
-$SIG{PIPE} = 'IGNORE';
-
say "worker is starting";
# main loop --------------------------------------------------------------------
while (1) {
my $frame = $stomp->receive_frame;
my $msg = $frame->body;
- my $data;
say "processing msg ...";
try {
- $data = decode_json($msg);
- process_msg($data);
+ process_msg($msg);
} catch {
- debug("failed to process msg: $_");
- post_result(0, $_, $data);
+ debug("failed to process job: $_");
} finally {
$stomp->ack({ frame => $frame });
};
@@ -62,155 +50,49 @@
# functions -------------------------------------------------------------------
sub process_msg {
- my ($data) = @_;
- log_msg($data);
- my $out = run_code($data);
- return post_result(-1, $out, $data) if $out =~ /^ERROR:/;
- my $status = $out eq $data->{problem}{output} ? 1 : 0;
- my $reason = $status == 1 ? 'Success' : 'Wrong answer';
- post_result($status, $reason, $data);
+ my ($msg) = @_;
+
+ my $data = decode_json $msg;
+ debug($data);
+ my $dir = "/tmp/content";
+ run "rm -rf $dir";
+ run "mkdir $dir";
+ run "cp $RealBin/docker-agent.pl $dir";
+ write_file "$dir/data.json", $msg;
+ get_bundle($data->{env_bundle_url}, "$dir/env_bundle.tar")
+ if $data->{env_bundle_url};
+ get_bundle($data->{lib_bundle_url}, "$dir/env_bundle.tar")
+ if $data->{lib_bundle_url};
+
+ my $cmd = join ' ',
+ 'tar -C /tmp -c content |',
+ "$docker_bin run -u sandbox -i -a stdin ironcamel/test1 /bin/bash -c",
+ qw("
+ cd /tmp;
+ tar -x;
+ perl ./content/docker-agent.pl;
+ tar -C ./content -c results;
+ ");
+ my $docker_run_id = `$cmd`;
+ chomp $docker_run_id;
+ debug("docker_run_id: $docker_run_id");
+ sleep 3;
+ run "$docker_bin logs $docker_run_id > results.tar";
+ debug("Results are ready!");
}
-sub run_code {
- my ($data) = @_;
-
- sys("rm -rf $jail/home/sandbox/runs/*");
- #sys("tar -xf $jail.tar -C /var") or die "Could not build chroot jail: $!";
-
- my $pid = open my $child_process, '-|';
- if (!$pid) { # child process starts here
- try {
- run_in_chroot($data);
- } catch {
- say "ERROR: $_";
- };
- exit;
- } # end of child process
-
- # Read output from the child process
- my @out = <$child_process>;
- close $child_process;
- my $result = join '', @out;
- debug('output: ' . substr $result, 0, 100);
- debug("full: $result");
-
- # Copy result files out of the jail
- if (my $results_path = $data->{copy_results_path}) {
- my $run_id = $data->{run_id} || 'unknown';
- my $target_dir = "$RealBin/../results/$run_id";
- rcopy "$jail/$results_path", $target_dir;
- }
-
- # Lets clean up after ourselves.
- #sys("rm -rf $jail/home/sandbox/runs/*");
-
- return $result;
-}
-
-sub run_in_chroot {
- my ($data) = @_;
- my $code = $data->{code};
- my $file_name = $data->{file_name} // 'foo';
- my $problem = $data->{problem};
- my $compile_cmd = $data->{compile_cmd};
- my $run_cmd = $data->{run_cmd};
- my $input = $problem->{input};
-
- chdir $jail or die "Failed to chdir into $jail: $!";
-
- if (not glob "$jail/proc/*") {
- sys("mount -t proc proc proc/") or die "Could not mount /proc: $!";
- }
-
- my $run_dir = "/home/sandbox/runs";
- my $chroot_run_dir = $jail . $run_dir;
- if (not -d $chroot_run_dir) {
- mkdir $chroot_run_dir or die "Could not mkdir $chroot_run_dir: $!";
- chmod 0777, $chroot_run_dir or die "Could not chmod $chroot_run_dir";
- }
- if (my $url = $data->{env_bundle_url}) {
- debug("getting bundle from $url");
- my $bundle_path = "$chroot_run_dir/bundle.tar";
- # For https: LWP::Protocol::https, libssl-dev, libnet-ssleay-perl
- my $res = $AGENT->mirror($url, $bundle_path);
- debug($res->status_line);
- if (!$res->is_success and $res->code != 304) {
- die "Could not download bundle: " . $res->status_line;
- }
- sys("tar xf $bundle_path -C $chroot_run_dir")
- or die "Could not expand bundle $bundle_path $!";
- }
- if (my $url = $data->{lib_bundle_url}) {
- debug("getting bundle from $url");
- my $lib_dir = "$jail/usr/local/lib/codejail";
- my $bundle_path = "$lib_dir/bundle.tar";
- my $res = $AGENT->mirror($url, $bundle_path);
- debug($res->status_line);
- if (!$res->is_success and $res->code != 304) {
- die "Could not download bundle: " . $res->status_line;
- }
- sys("tar xf $bundle_path -C $lib_dir")
- or die "Could not expand bundle $bundle_path $!";
- }
-
- debug("chroot $jail");
- chroot $jail or die "Could not chroot: $!";
-
- # Drop root privileges immediately after successful chroot
- my $new_uid = getpwnam('sandbox');
- die "Can't find uid for sandbox user" unless defined $new_uid;
- $< = $> = $new_uid; # Update the real and effective user id
- setgid($new_uid); # Update the group id
-
- debug("chdir $run_dir");
- chdir $run_dir or die "Could not chdir to $run_dir: $!";
-
- write_file $file_name, $code;
- if ($compile_cmd) {
- debug("compiling: ", $compile_cmd);
- run $compile_cmd;
- }
-
- my ($out, $err) = ('', '');
- debug("going to run: @$run_cmd");
- try {
- run $run_cmd, \$input, \$out, \$err, timeout(3);
- } catch {
- print "Took too long $_\n";
- };
- print $out;
-}
-
-sub post_result {
- my ($status, $reason, $data) = @_;
- print "posting result:";
- return unless ref $data eq 'HASH' and $data->{cb_url};
- my $result = {
- status => $status,
- reason => $reason,
- run_id => $data->{run_id},
- user_id => $data->{user_id},
- problem => $data->{problem}{title},
- };
- dd $result;
- debug($result);
- debug("going to post result to callback url: $data->{cb_url}");
- my $res = $AGENT->post($data->{cb_url}, content_type => 'application/json',
- content => encode_json($result));
+sub get_bundle {
+ my ($url, $bundle_path) = @_;
+ debug("getting bundle from $url");
+ my $res = $AGENT->mirror($url, $bundle_path);
debug($res->status_line);
+ if (!$res->is_success and $res->code != 304) {
+ die "Could not download bundle: " . $res->status_line;
+ }
}
-sub debug { say $LOG '[' . localtime . '] (DEBUG) ' . dump(@_) }
-
-sub log_msg {
- my $data = shift;
- my %copy = %$data;
- $copy{problem} = $copy{problem}{title};
- debug(\%copy)
-}
-
-sub sys {
- my ($cmd) = @_;
- debug($cmd);
- return system($cmd) == 0;
+sub debug {
+ my $msg = '[' . localtime . '] (DEBUG) ' . dump(@_);
+ say $LOG $msg;
+ say $msg;
}
View
@@ -0,0 +1,57 @@
+#!/usr/bin/env perl
+use v5.12;
+use warnings;
+use File::Copy::Recursive qw(rcopy);
+use File::Slurp qw(read_file write_file);
+use FindBin qw($RealBin);
+use IPC::Run qw(run timeout);
+use JSON qw(decode_json encode_json);
+use Try::Tiny;
+
+chdir $RealBin;
+my $results_dir = 'results';
+mkdir $results_dir;
+
+my $data = decode_json read_file 'data.json';
+my $code = $data->{code};
+my $file_name = $data->{file_name} // 'foo';
+my $compile_cmd = $data->{compile_cmd};
+my $run_cmd = $data->{run_cmd};
+my $run_id = $data->{run_id} // 'unknown';
+my $problem = $data->{problem} || {};
+my $timeout = $data->{timeout} || 3;
+my $input = $problem->{input};
+my $copy_results_path = $data->{copy_results_path};
+my ($out, $err, $compile_success);
+
+write_file $file_name, $code;
+run [qw(tar xf env_bundle.tar)], \'', \$out, \$err;
+run [qw(tar xf lib_bundle.tar -C /usr/local/lib/codejail)], \'', \$out, \$err;
+
+if ($compile_cmd) {
+ $compile_success = try {
+ run $compile_cmd, \'', \$out, \$err, timeout($timeout);
+ } catch {
+ $err = "Compile command took too long $_";
+ };
+ write_file "$results_dir/compile_out.log", $out // '';
+ write_file "$results_dir/compile_err.log", $err // '';
+}
+
+exit unless $compile_success;
+
+try {
+ run $run_cmd, \$input, \$out, \$err, timeout($timeout);
+} catch {
+ $err = "Run command took too long $_";
+};
+write_file "$results_dir/run_out.log", $out // '';
+write_file "$results_dir/run_err.log", $err // '';
+
+# Copy out a custom dir, such as junit results for example
+if ($copy_results_path) {
+ my $dir = "$results_dir/extra";
+ mkdir $dir;
+ rcopy $copy_results_path => $dir;
+}
+

0 comments on commit 5c85b21

Please sign in to comment.