Skip to content

Commit

Permalink
MDL-31065 question stats: fix analysis of responses not matching a gi…
Browse files Browse the repository at this point in the history
…ven answer

When shortanswer, numerical, calculated and calculatedsimple questions
did not have a '*' match-anything answer, then any student response that
did not match any of the teacher-given answers were classified as
'[No response]', which was not right.

This patch fixes that. Such responses are now classified as
[Did not match any answer].

While I was doing this, I noticed that the display of tolerance
intervals for numerical questions in the response analysis was horrible,
so I improved it.
  • Loading branch information
timhunt committed Jan 20, 2012
1 parent 2a040c2 commit e6dedfe
Show file tree
Hide file tree
Showing 15 changed files with 300 additions and 16 deletions.
1 change: 1 addition & 0 deletions lang/en/question.php
Expand Up @@ -97,6 +97,7 @@
$string['deletequestionscheck'] = 'Are you absolutely sure you want to delete the following questions?<br /><br />{$a}';
$string['deletingbehaviour'] = 'Deleting question behaviour \'{$a}\'';
$string['deletingqtype'] = 'Deleting question type \'{$a}\'';
$string['didnotmatchanyanswer'] = '[Did not match any answer]';
$string['disabled'] = 'Disabled';
$string['disterror'] = 'The distribution {$a} caused problems';
$string['donothing'] = 'Don\'t copy or move files or change links.';
Expand Down
1 change: 1 addition & 0 deletions question/type/calculated/lang/en/qtype_calculated.php
Expand Up @@ -30,6 +30,7 @@
$string['addsets'] = 'Add set(s)';
$string['answerhdr'] = 'Answer';
$string['answerstoleranceparam'] = 'Answers tolerance parameters';
$string['answerwithtolerance'] = '{$a->answer} (±{$a->tolerance} {$a->tolerancetype})';
$string['anyvalue'] = 'Any value';
$string['atleastoneanswer'] = 'You need to provide at least one answer.';
$string['atleastonerealdataset']='There should be at least one real dataset in question text';
Expand Down
36 changes: 28 additions & 8 deletions question/type/calculated/questiontype.php
Expand Up @@ -690,6 +690,15 @@ public function delete_question($questionid, $contextid) {
parent::delete_question($questionid, $contextid);
}

public function get_random_guess_score($questiondata) {
foreach ($questiondata->options->answers as $aid => $answer) {
if ('*' == trim($answer->answer)) {
return max($answer->fraction - $questiondata->options->unitpenalty, 0);
}
}
return 0;
}

public function supports_dataset_item_generation() {
// Calcualted support generation of randomly distributed number data
return true;
Expand Down Expand Up @@ -1204,7 +1213,7 @@ public function construct_dataset_menus($form, $mandatorydatasets,

public function substitute_variables($str, $dataset) {
global $OUTPUT;
// testing for wrong numerical values
// testing for wrong numerical values
// all calculations used this function so testing here should be OK

foreach ($dataset as $name => $value) {
Expand All @@ -1224,6 +1233,7 @@ public function substitute_variables($str, $dataset) {
}
return $str;
}

public function evaluate_equations($str, $dataset) {
$formula = $this->substitute_variables($str, $dataset);
if ($error = qtype_calculated_find_formula_errors($formula)) {
Expand All @@ -1232,7 +1242,6 @@ public function evaluate_equations($str, $dataset) {
return $str;
}


public function substitute_variables_and_eval($str, $dataset) {
$formula = $this->substitute_variables($str, $dataset);
if ($error = qtype_calculated_find_formula_errors($formula)) {
Expand Down Expand Up @@ -1797,21 +1806,32 @@ public function get_possible_responses($questiondata) {
$virtualqtype = $this->get_virtual_qtype();
$unit = $virtualqtype->get_default_numerical_unit($questiondata);

$tolerancetypes = $this->tolerance_types();

$starfound = false;
foreach ($questiondata->options->answers as $aid => $answer) {
$responseclass = $answer->answer;

if ($responseclass != '*') {
$responseclass = $virtualqtype->add_unit($questiondata, $responseclass, $unit);
if ($responseclass === '*') {
$starfound = true;
} else {
$a = new stdClass();
$a->answer = $virtualqtype->add_unit($questiondata, $responseclass, $unit);
$a->tolerance = $answer->tolerance;
$a->tolerancetype = $tolerancetypes[$answer->tolerancetype];

$ans = new qtype_numerical_answer($answer->id, $answer->answer, $answer->fraction,
$answer->feedback, $answer->feedbackformat, $answer->tolerance);
list($min, $max) = $ans->get_tolerance_interval();
$responseclass .= " ($min..$max)";
$responseclass = get_string('answerwithtolerance', 'qtype_calculated', $a);
}

$responses[$aid] = new question_possible_response($responseclass,
$answer->fraction);
}

if (!$starfound) {
$responses[0] = new question_possible_response(
get_string('didnotmatchanyanswer', 'question'), 0);
}

$responses[null] = question_possible_response::no_response();

return array($questiondata->id => $responses);
Expand Down
37 changes: 37 additions & 0 deletions question/type/calculated/simpletest/helper.php
Expand Up @@ -77,6 +77,43 @@ public function make_calculated_question_sum() {

return $q;
}

/**
* Makes a calculated question about summing two numbers.
* @return qtype_calculated_question
*/
public function get_calculated_question_data_sum() {
question_bank::load_question_definition_classes('calculated');
$qdata = new stdClass();
test_question_maker::initialise_question_data($qdata);

$qdata->qtype = 'calculated';
$qdata->name = 'Simple sum';
$qdata->questiontext = 'What is {a} + {b}?';
$qdata->generalfeedback = 'Generalfeedback: {={a} + {b}} is the right answer.';

$qdata->options = new stdClass();
$qdata->options->unitgradingtype = 0;
$qdata->options->unitpenalty = 0.0;
$qdata->options->showunits = qtype_numerical::UNITNONE;
$qdata->options->unitsleft = 0;
$qdata->options->synchronize = 0;

$qdata->options->answers = array(
13 => new qtype_numerical_answer(13, '{a} + {b}', 1.0, 'Very good.', FORMAT_HTML, 0.001),
14 => new qtype_numerical_answer(14, '{a} - {b}', 0.0, 'Add. not subtract!.',
FORMAT_HTML, 0.001),
17 => new qtype_numerical_answer(17, '*', 0.0, 'Completely wrong.', FORMAT_HTML, 0),
);
foreach ($qdata->options->answers as $answer) {
$answer->correctanswerlength = 2;
$answer->correctanswerformat = 1;
}

$qdata->options->units = array();

return $qdata;
}
}


Expand Down
20 changes: 20 additions & 0 deletions question/type/calculated/simpletest/testquestion.php
Expand Up @@ -109,6 +109,26 @@ public function test_classify_response() {
$question->classify_response(array('answer' => '')));
}

public function test_classify_response_no_star() {
$question = test_question_maker::make_question('calculated');
unset($question->answers[17]);
$question->start_attempt(new question_attempt_step(), 1);
$values = $question->vs->get_values();

$this->assertEqual(array(
new question_classified_response(13, $values['a'] + $values['b'], 1.0)),
$question->classify_response(array('answer' => $values['a'] + $values['b'])));
$this->assertEqual(array(
new question_classified_response(14, $values['a'] - $values['b'], 0.0)),
$question->classify_response(array('answer' => $values['a'] - $values['b'])));
$this->assertEqual(array(
new question_classified_response(0, 7 * $values['a'], 0.0)),
$question->classify_response(array('answer' => 7 * $values['a'])));
$this->assertEqual(array(
question_classified_response::no_response()),
$question->classify_response(array('answer' => '')));
}

public function test_get_variants_selection_seed_q_not_synchronised() {
$question = test_question_maker::make_question('calculated');
$this->assertEqual($question->stamp, $question->get_variants_selection_seed());
Expand Down
107 changes: 107 additions & 0 deletions question/type/calculated/simpletest/testquestiontype.php
@@ -0,0 +1,107 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

/**
* Unit tests for (some of) question/type/calculated/questiontype.php.
*
* @package qtype_calculated
* @copyright 2012 The Open University
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/


defined('MOODLE_INTERNAL') || die();

require_once($CFG->dirroot . '/question/type/calculated/questiontype.php');


/**
* Unit tests for question/type/calculated/questiontype.php.
*
* @copyright 2012 The Open University
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class qtype_calculated_test extends UnitTestCase {
public static $includecoverage = array(
'question/type/questiontypebase.php',
'question/type/calculated/questiontype.php'
);

protected $tolerance = 0.00000001;
protected $qtype;

public function setUp() {
$this->qtype = new qtype_calculated();
}

public function tearDown() {
$this->qtype = null;
}

public function test_name() {
$this->assertEqual($this->qtype->name(), 'calculated');
}

public function test_can_analyse_responses() {
$this->assertTrue($this->qtype->can_analyse_responses());
}

public function test_get_random_guess_score() {
$q = test_question_maker::get_question_data('calculated');
$q->options->answers[17]->fraction = 0.1;
$this->assertEqual(0.1, $this->qtype->get_random_guess_score($q));
}

protected function get_possible_response($ans, $tolerance, $type) {
$a = new stdClass();
$a->answer = $ans;
$a->tolerance = $tolerance;
$a->tolerancetype = get_string($type, 'qtype_numerical');
return get_string('answerwithtolerance', 'qtype_calculated', $a);
}

public function test_get_possible_responses() {
$q = test_question_maker::get_question_data('calculated');

$this->assertEqual(array(
$q->id => array(
13 => new question_possible_response(
$this->get_possible_response('{a} + {b}', 0.001, 'nominal'), 1.0),
14 => new question_possible_response(
$this->get_possible_response('{a} - {b}', 0.001, 'nominal'), 0.0),
17 => new question_possible_response('*', 0.0),
null => question_possible_response::no_response()
),
), $this->qtype->get_possible_responses($q));
}

public function test_get_possible_responses_no_star() {
$q = test_question_maker::get_question_data('calculated');
unset($q->options->answers[17]);

$this->assertEqual(array(
$q->id => array(
13 => new question_possible_response(
$this->get_possible_response('{a} + {b}', 0.001, 'nominal'), 1),
14 => new question_possible_response(
$this->get_possible_response('{a} - {b}', 0.001, 'nominal'), 0),
0 => new question_possible_response(
get_string('didnotmatchanyanswer', 'question'), 0),
null => question_possible_response::no_response()
),
), $this->qtype->get_possible_responses($q));
}
}
7 changes: 4 additions & 3 deletions question/type/numerical/question.php
Expand Up @@ -258,15 +258,16 @@ public function classify_response(array $response) {
}
list($value, $unit, $multiplier) = $this->ap->apply_units($response['answer'], $selectedunit);
$ans = $this->get_matching_answer($value, $multiplier);
if (!$ans) {
return array($this->id => question_classified_response::no_response());
}

$resp = $response['answer'];
if ($this->has_separate_unit_field()) {
$resp = $this->ap->add_unit($resp, $unit);
}

if (!$ans) {
return array($this->id => new question_classified_response(0, $resp, 0));
}

return array($this->id => new question_classified_response($ans->id,
$resp,
$this->apply_unit_penalty($ans->fraction, $ans->unitisright)));
Expand Down
11 changes: 10 additions & 1 deletion question/type/numerical/questiontype.php
Expand Up @@ -418,10 +418,13 @@ public function get_possible_responses($questiondata) {

$unit = $this->get_default_numerical_unit($questiondata);

$starfound = false;
foreach ($questiondata->options->answers as $aid => $answer) {
$responseclass = $answer->answer;

if ($responseclass != '*') {
if ($responseclass === '*') {
$starfound = true;
} else {
$responseclass = $this->add_unit($questiondata, $responseclass, $unit);

$ans = new qtype_numerical_answer($answer->id, $answer->answer, $answer->fraction,
Expand All @@ -433,6 +436,12 @@ public function get_possible_responses($questiondata) {
$responses[$aid] = new question_possible_response($responseclass,
$answer->fraction);
}

if (!$starfound) {
$responses[0] = new question_possible_response(
get_string('didnotmatchanyanswer', 'question'), 0);
}

$responses[null] = question_possible_response::no_response();

return array($questiondata->id => $responses);
Expand Down
3 changes: 1 addition & 2 deletions question/type/numerical/simpletest/helper.php
Expand Up @@ -72,8 +72,7 @@ public function make_numerical_question_pi() {
}

/**
* Makes a numerical question with correct ansewer 3.14, and various incorrect
* answers with different feedback.
* Makes a numerical question with a choice (select menu) of units.
* @return qtype_numerical_question
*/
public function make_numerical_question_unit() {
Expand Down
35 changes: 35 additions & 0 deletions question/type/numerical/simpletest/testquestion.php
Expand Up @@ -213,6 +213,22 @@ public function test_classify_response() {
$num->classify_response(array('answer' => '')));
}

public function test_classify_response_no_star() {
$num = test_question_maker::make_question('numerical');
unset($num->answers[17]);
$num->start_attempt(new question_attempt_step(), 1);

$this->assertEqual(array(
new question_classified_response(15, '3.1', 0.0)),
$num->classify_response(array('answer' => '3.1')));
$this->assertEqual(array(
new question_classified_response(0, '42', 0.0)),
$num->classify_response(array('answer' => '42')));
$this->assertEqual(array(
question_classified_response::no_response()),
$num->classify_response(array('answer' => '')));
}

public function test_classify_response_unit() {
$num = test_question_maker::make_question('numerical', 'unit');
$num->start_attempt(new question_attempt_step(), 1);
Expand Down Expand Up @@ -240,6 +256,25 @@ public function test_classify_response_unit() {
$num->classify_response(array('answer' => '')));
}

public function test_classify_response_unit_no_star() {
$num = test_question_maker::make_question('numerical', 'unit');
unset($num->answers[17]);
$num->start_attempt(new question_attempt_step(), 1);

$this->assertEqual(array(
new question_classified_response(0, '42 cm', 0)),
$num->classify_response(array('answer' => '42', 'unit' => 'cm')));
$this->assertEqual(array(
new question_classified_response(0, '3.0', 0)),
$num->classify_response(array('answer' => '3.0', 'unit' => '')));
$this->assertEqual(array(
new question_classified_response(0, '3.0 m', 0)),
$num->classify_response(array('answer' => '3.0', 'unit' => 'm')));
$this->assertEqual(array(
question_classified_response::no_response()),
$num->classify_response(array('answer' => '', 'unit' => '')));
}

public function test_classify_response_currency() {
$num = test_question_maker::make_question('numerical', 'currency');
$num->start_attempt(new question_attempt_step(), 1);
Expand Down

0 comments on commit e6dedfe

Please sign in to comment.