/
jobrunner.php
354 lines (322 loc) · 15.3 KB
/
jobrunner.php
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
<?php
// This file is part of CodeRunner - http://coderunner.org.nz/
//
// CodeRunner is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// CodeRunner is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with CodeRunner. If not, see <http://www.gnu.org/licenses/>.
/*
* @package qtype
* @subpackage coderunner
* @copyright 2016 Richard Lobb, University of Canterbury
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
defined('MOODLE_INTERNAL') || die();
require_once($CFG->dirroot . '/question/type/coderunner/Twig/Autoloader.php');
require_once($CFG->dirroot . '/question/type/coderunner/questiontype.php');
// The qtype_coderunner_jobrunner class contains all code concerned with running a question
// in the sandbox and grading the result.
class qtype_coderunner_jobrunner {
private $grader = null; // The grader instance, if it's NOT a custom one.
private $twig = null; // The template processor environment.
private $sandbox = null; // The sandbox we're using.
private $code = null; // The code we're running.
private $question = null; // The question that we're running code for.
private $testcases = null; // The testcases (a subset of those in the question).
private $allruns = null; // Array of the source code for all runs.
private $precheck = null; // True if this is a precheck run.
// Check the correctness of a student's code as an answer to the given
// question and and a given set of test cases (which may be empty or a
// subset of the question's set of testcases. $isprecheck is true if
// this is a run triggered by the student clicking the Precheck button.
// Returns a TestingOutcome object.
public function run_tests($question, $code, $testcases, $isprecheck) {
global $CFG, $USER;
$this->question = $question;
$this->code = $code;
$this->testcases = $testcases;
$this->isprecheck = $isprecheck;
$this->grader = $question->get_grader();
$this->sandbox = $question->get_sandbox();
$this->template = $question->get_template();
$this->files = $question->get_files();
$this->sandboxparams = $question->get_sandbox_params();
$this->language = $question->get_language();
Twig_Autoloader::register();
$loader = new Twig_Loader_String();
$this->twig = new Twig_Environment($loader, array(
'debug' => true,
'autoescape' => false,
'optimizations' => 0
));
$twigcore = $this->twig->getExtension('core');
$twigcore->setEscaper('py', 'qtype_coderunner_escapers::python');
$twigcore->setEscaper('python', 'qtype_coderunner_escapers::python');
$twigcore->setEscaper('c', 'qtype_coderunner_escapers::java');
$twigcore->setEscaper('java', 'qtype_coderunner_escapers::java');
$twigcore->setEscaper('ml', 'qtype_coderunner_escapers::matlab');
$twigcore->setEscaper('matlab', 'qtype_coderunner_escapers::matlab');
$this->allruns = array();
$this->templateparams = array(
'STUDENT_ANSWER' => $code,
'ESCAPED_STUDENT_ANSWER' => qtype_coderunner_escapers::python(null, $code, null),
'MATLAB_ESCAPED_STUDENT_ANSWER' => qtype_coderunner_escapers::matlab(null, $code, null),
'IS_PRECHECK' => $isprecheck ? "1" : "0",
'QUESTION' => $question,
'STUDENT' => new qtype_coderunner_student($USER)
);
if ($question->get_is_combinator() and
($this->has_no_stdins() || $question->allow_multiple_stdins())) {
$outcome = $this->run_combinator($isprecheck);
} else {
$outcome = null;
}
// If that failed for any reason (e.g. timeout or signal), or if the
// template isn't a combinator, run the tests individually. Any compilation
// errors or stderr output in individual tests bomb the whole test process,
// but otherwise we should finish with a TestingOutcome object containing
// a test result for each test case.
if ($outcome == null) {
$outcome = $this->run_tests_singly($isprecheck);
}
$this->sandbox->close();
if ($question->get_show_source()) {
$outcome->sourcecodelist = $this->allruns;
}
return $outcome;
}
// If the template is a combinator, try running all the tests in a single
// go.
//
// Special template parameters are STUDENT_ANSWER, the raw submitted code,
// IS_PRECHECK, which is true if this is a precheck run, TESTCASES,
// a list of all the test cases and QUESTION, the original question object.
// Return the testing outcome object if successful else null.
private function run_combinator($isprecheck) {
$numtests = count($this->testcases);
$this->templateparams['TESTCASES'] = $this->testcases;
$maxmark = $this->maximum_possible_mark();
$outcome = new qtype_coderunner_testing_outcome($maxmark, $numtests, $isprecheck);
try {
$testprog = $this->twig->render($this->template, $this->templateparams);
} catch (Exception $e) {
$outcome->set_status(
qtype_coderunner_testing_outcome::STATUS_SYNTAX_ERROR,
get_string('templateerror', 'qtype_coderunner') . $e->getMessage());
return $outcome;
}
$this->allruns[] = $testprog;
$run = $this->sandbox->execute($testprog, $this->language,
null, $this->files, $this->sandboxparams);
// If it's a template grader, we pass the result to the
// do_combinator_grading method. Otherwise we deal with syntax errors or
// a successful result without accompanying stderr.
// In all other cases (runtime error etc) we give up
// on the combinator.
if ($run->error !== qtype_coderunner_sandbox::OK) {
$outcome->set_status(
qtype_coderunner_testing_outcome::STATUS_SANDBOX_ERROR,
qtype_coderunner_sandbox::error_string($run));
} else if ($this->grader->name() === 'TemplateGrader') {
$outcome = $this->do_combinator_grading($run, $isprecheck);
} else if ($run->result === qtype_coderunner_sandbox::RESULT_COMPILATION_ERROR) {
$outcome->set_status(
qtype_coderunner_testing_outcome::STATUS_SYNTAX_ERROR,
$run->cmpinfo);
} else if ($run->result === qtype_coderunner_sandbox::RESULT_SUCCESS) {
$outputs = preg_split($this->question->get_test_splitter_re(), $run->output);
if (count($outputs) === $numtests) {
$i = 0;
foreach ($this->testcases as $testcase) {
$outcome->add_test_result($this->grade($outputs[$i], $testcase));
$i++;
}
} else { // Error: wrong number of tests after splitting.
$error = get_string('brokencombinator', 'qtype_coderunner',
array('numtests' => $numtests, 'numresults' => count($outputs)));
$outcome->set_status(qtype_coderunner_testing_outcome::STATUS_BAD_COMBINATOR, $error);
}
} else {
$outcome = null; // Something broke badly.
}
return $outcome;
}
// Run all tests one-by-one on the sandbox.
private function run_tests_singly($isprecheck) {
$maxmark = $this->maximum_possible_mark($this->testcases);
if ($maxmark == 0) {
$maxmark = 1; // Something silly is happening. Probably running a prototype with no tests.
}
$numtests = count($this->testcases);
$outcome = new qtype_coderunner_testing_outcome($maxmark, $numtests, $isprecheck);
foreach ($this->testcases as $testcase) {
if ($this->question->iscombinatortemplate) {
$this->templateparams['TESTCASES'] = array($testcase);
} else {
$this->templateparams['TEST'] = $testcase;
}
try {
$testprog = $this->twig->render($this->template, $this->templateparams);
} catch (Exception $e) {
$outcome->set_status(
qtype_coderunner_testing_outcome::STATUS_SYNTAX_ERROR,
'TEMPLATE ERROR: ' . $e->getMessage());
break;
}
$input = isset($testcase->stdin) ? $testcase->stdin : '';
$this->allruns[] = $testprog;
$run = $this->sandbox->execute($testprog, $this->language,
$input, $this->files, $this->sandboxparams);
if ($run->error !== qtype_coderunner_sandbox::OK) {
$outcome->set_status(
qtype_coderunner_testing_outcome::STATUS_SANDBOX_ERROR,
qtype_coderunner_sandbox::error_string($run));
break;
} else if ($run->result === qtype_coderunner_sandbox::RESULT_COMPILATION_ERROR) {
$outcome->set_status(
qtype_coderunner_testing_outcome::STATUS_SYNTAX_ERROR,
$run->cmpinfo);
break;
} else if ($run->result != qtype_coderunner_sandbox::RESULT_SUCCESS) {
$errormessage = $this->make_error_message($run);
$iserror = true;
$outcome->add_test_result($this->grade($errormessage, $testcase, $iserror));
break;
} else {
$testresult = $this->grade($run->output, $testcase);
$aborting = false;
if (isset($testresult->abort) && $testresult->abort) { // Templategrader abort request?
$testresult->awarded = 0; // Mark it wrong regardless.
$testresult->iscorrect = false;
$aborting = true;
}
$outcome->add_test_result($testresult);
if ($aborting) {
break;
}
}
}
return $outcome;
}
// Grade a given test result by calling the grader.
private function grade($output, $testcase, $isbad = false) {
return $this->grader->grade($output, $testcase, $isbad);
}
/**
* Given the result of a sandbox run with the combinator template,
* build and return a testingOutcome object with a status of
* STATUS_COMBINATOR_TEMPLATE_GRADER and attributes of prelude and/or
* and/or testresults and/or epiloguehtml.
*
* @param int $maxmark The maximum mark for this question
* @param JSON $run The JSON-encoded output from the run.
* @return \qtype_coderunner_testing_outcome the outcome object ready
* for display by the renderer. This will have an actualmark and zero or more of
* prologuehtml, testresults and epiloguehtml. The last three are: some
* html for display before the result table, the test results table (an
* array of pseudo-test_result objects) and some html for display after
* the result table.
*/
private function do_combinator_grading($run, $isprecheck) {
$outcome = new qtype_coderunner_combinator_grader_outcome($isprecheck);
if ($run->result !== qtype_coderunner_sandbox::RESULT_SUCCESS) {
$error = get_string('brokentemplategrader', 'qtype_coderunner',
array('output' => $run->cmpinfo . "\n" . $run->stderr));
$outcome->set_status(qtype_coderunner_testing_outcome::STATUS_BAD_COMBINATOR, $error);
} else {
$result = json_decode($run->output);
if ($result === null || !isset($result->fraction) ||
!is_numeric($result->fraction)) {
// Bad combinator output.
$error = get_string('badjsonorfraction', 'qtype_coderunner',
array('output' => $run->output));
$outcome->set_status(qtype_coderunner_testing_outcome::STATUS_BAD_COMBINATOR, $error);
} else {
// A successful combinator run.
$fract = $result->fraction;
$feedback = array();
if (isset($result->feedback_html)) { // Legacy combinator grader?
$result->feedbackhtml = $result->feedback_html; // Change to modern version.
unset($result->feedback_html);
}
foreach (array('prologuehtml', 'testresults', 'epiloguehtml', 'feedbackhtml') as $key) {
if (isset($result->$key)) {
if ($key === 'feedbackhtml' || $key === 'feedback_html') {
// For compatibility with older combinator graders.
$feedback['epiloguehtml'] = $result->$key;
} else {
$feedback[$key] = $result->$key;
}
}
}
$outcome->set_mark_and_feedback($fract, $feedback);
}
}
return $outcome;
}
/* Return a $sep-separated string of the non-empty elements
of the array $strings. Similar to implode except empty strings
are ignored. */
private function merge($sep, $strings) {
$s = '';
foreach ($strings as $el) {
if (trim($el)) {
if ($s !== '') {
$s .= $sep;
}
$s .= $el;
}
}
return $s;
}
// Return the maximum possible mark from the set of testcases we're running.
private function maximum_possible_mark() {
$total = 0;
foreach ($this->testcases as $testcase) {
$total += $testcase->mark;
}
if ($total == 0) {
$total = 1; // Something silly is happening. Probably running a prototype with no tests.
}
return $total;
}
private function make_error_message($run) {
$err = "***" . qtype_coderunner_sandbox::result_string($run->result) . "***";
if ($run->result === qtype_coderunner_sandbox::RESULT_RUNTIME_ERROR) {
$sig = $run->signal;
if ($sig) {
$err .= " (signal $sig)";
}
}
return $this->merge("\n", array($run->cmpinfo, $run->output, $err, $run->stderr));
}
/** True IFF no testcases have nonempty stdin. */
private function has_no_stdins() {
foreach ($this->testcases as $testcase) {
if ($testcase->stdin != '') {
return false;
}
}
return true;
}
// Count the number of errors in the given array of test results.
// TODO -- figure out how to eliminate either this one or the identical
// version in renderer.php.
private function count_errors($testresults) {
$errors = 0;
foreach ($testresults as $tr) {
if (!$tr->iscorrect) {
$errors++;
}
}
return $errors;
}
}