Quiz Stats - Multiple Answers

Ports the generation of stats for that question type to the CQS gem.

Changes:

  - no longer exposing "user_ids"
  - can now identify students who skipped the question

Closes CNVS-13089

TEST PLAN
---- ----

  - create a quiz with multiple-answer question(s)
  - take it by a number of students and cover the following cases:
    - answer correctly by picking only the right choices
    - answer almost correctly by:
      1. picking only 1 right choice
      2. picking 1 right and 1 wrong choices
      3. picking everything
    - answer incorrectly by picking only the incorrect choice(s)
    - don't answer at all
  - get the stats from the API:
    - for "responses", "correct", and "partially_correct", verify they
      meet the specification in the docs
      - also for the "responses" field in each document in "answers"
    - verify that there is an answer document with "none" for an id with
      "responses" that reflect the number of students that skipped the
      question
  - visit ember quiz stats:
    - verify the "Attempts: X out of Y" should read the "responses"
      field out of the total quiz participant count
    - verify the donut chart reads the correct "correct" response ratio
    - verify there is a "No-Answer" bar
    - expand the question details:
      - verify that all the choices are displayed, and the correct
        choices are highlighted in GREEEN

Change-Id: Ibc08b6f521f9cae35dd16950c68c164d7e27d95d
Reviewed-on: https://gerrit.instructure.com/35736
QA-Review: Caleb Guanzon <cguanzon@instructure.com>
Reviewed-by: Derek DeVries <ddevries@instructure.com>
Tested-by: Jenkins <jenkins@instructure.com>
Product-Review: Ahmad Amireh <ahmad@instructure.com>

app/coffeescripts/ember/quizzes/models/question_statistics.coffee

@@ -20,6 +20,9 @@ define [
     responseValues: attr()
     unexpectedResponseValues: attr()
 
+    correct: attr('number')
+    partiallyCorrect: attr('number')
+
     # MC/TF stats
     topStudentCount: attr()
     middleStudentCount: attr()

app/coffeescripts/ember/quizzes/models/question_statistics/response_ratio_calculator.coffee

@@ -1,5 +1,5 @@
-define [ 'ember', 'underscore' ], (Em, _) ->
-  {intersection, flatten, difference} = _
+define [ 'ember' ], (Em) ->
+  MULTIPLE_ANSWERS = 'multiple_answers_question'
 
   # A utility class for calculating response ratios for a given question
   # statistics object.
@@ -21,12 +21,11 @@ define [ 'ember', 'underscore' ], (Em, _) ->
     #
     #   {
     #     "responses": 0,
-    #     "correct": true,
-    #     "user_ids": []
+    #     "correct": true
     #   }
     #
     # Most question types will have these defined in the top-level "answers" set,
-    # but for some others that support multiple-answers, these could be found in
+    # but for some others that support answer sets, these could be found in
     # answer_sets.@each.answer_matches.
     #
     # @note
@@ -43,7 +42,7 @@ define [ 'ember', 'underscore' ], (Em, _) ->
 
       return 0 if participantCount <= 0
 
-      if hasMultipleAnswers(@get('questionType'))
+      if isMultipleAnswers(@get('questionType'))
         return ratioForMultipleAnswers.call(this)
 
       correctResponses = @get('answerPool').reduce (sum, answer) ->
@@ -55,10 +54,8 @@ define [ 'ember', 'underscore' ], (Em, _) ->
     ).property('answerPool', 'participantCount')
 
   # @private
-  hasMultipleAnswers = (questionType) ->
-    Em.A([
-      'multiple_answers_question'
-    ]).contains(questionType)
+  isMultipleAnswers = (questionType) ->
+    MULTIPLE_ANSWERS == questionType
 
   # @private
   #
@@ -67,20 +64,6 @@ define [ 'ember', 'underscore' ], (Em, _) ->
   # correct. As such, a "partially" correct response does not count towards
   # the correct response ratio.
   ratioForMultipleAnswers = () ->
-    respondentsFor = (answers) ->
-      respondents = Em.A(answers).mapBy('user_ids')
-
-    participantCount = @get('participantCount')
-    correctAnswers = @get('answerPool').filterBy 'correct', true
-    distractors = @get('answerPool').filterBy 'correct', false
-
-    # we need students who have picked all correct answers:
-    correctRespondents = intersection.apply(_, respondentsFor(correctAnswers))
-
-    # and none of the wrong ones:
-    correctRespondents = difference(correctRespondents,
-      flatten(respondentsFor(distractors)))
-
-    correctRespondents.length / participantCount
+    @get('correct') / @get('participantCount')
 
   Calculator

app/coffeescripts/ember/quizzes/tests/models/question_statistics/response_ratio_calculator.spec.coffee

@@ -43,27 +43,15 @@ define [
       equal subject.get('ratio'), 0
 
 
-  test '#correctMultipleResponseRatio', ->
+  test '#correctMultipleAnswerRatio', ->
     run ->
       subject.set 'questionType', 'multiple_answers_question'
-      subject.set 'participantCount', 10
-      subject.set 'answerPool', [
-        { user_ids: [3], correct: true },
-        { user_ids: [ ], correct: false },
-        { user_ids: [3], correct: true },
-      ]
-
-      equal subject.get('ratio'), 0.1,
-        'it counts only students who have picked all correct answers and nothing else'
-
-      subject.set 'answerPool', [
-        { user_ids: [3], correct: true },
-        { user_ids: [3], correct: false },
-        { user_ids: [3], correct: true },
-      ]
 
-      equal subject.get('ratio'), 0,
-        "it doesn't count students who picked a wrong answer and a correct one"
+      subject.set 'correct', 1
+      subject.set 'participantCount', 5
 
+      equal subject.get('ratio'), 0.2
 
+      subject.set 'participantCount', 0
 
+      equal subject.get('ratio'), 0

doc/examples/question_specific_statistics.md

@@ -196,6 +196,64 @@ may include extra metrics. You can find these metrics below.
 }
 ```
 
+#### Multiple Answers
+
+```javascript
+{
+  // Number of students who have picked any choice.
+  "responses": 3,
+
+  // Number of students who have picked all the right choices.
+  "correct": 1,
+
+  // Number of students who have picked at least one of the right choices,
+  // but may have also picked a wrong one.
+  "partially_correct": 2,
+
+  "answers": [
+    {
+      // Unique ID of this answer choice.
+      "id": "5514",
+
+      // Displayable choice text.
+      "text": "A",
+
+      // Number of students who picked this choice.
+      "responses": 3,
+
+      // Whether this choice is part of the answer.
+      "correct": true
+    },
+    // Here's the second part of the correct answer:
+    {
+      "id": "4261",
+      "text": "B",
+      "responses": 1,
+      "correct": true
+    },
+
+    // And here's a distractor:
+    {
+      "id": "3322",
+      "text": "C",
+      "responses": 2,
+      "correct": false
+    },
+
+    // "Missing" answers:
+    //
+    // This is an auto-generated answer to account for all students who
+    // left this question unanswered.
+    {
+      "id": "none",
+      "text": "No Answer",
+      "responses": 0,
+      "correct": false
+    }
+  ]
+}
+```
+
 #### Multiple Dropdowns
 
 Multiple Dropdown question statistics look just like the statistics for [Fill In Multiple Blanks](#fimb-question-stats).
@@ -425,4 +483,4 @@ Formula question statistics look just like the statistics for [Essays](#essay-qu
 
 #### True/False
 
-True/False question statistics look just like the statistics for [Multiple-Choice](#multiple-choice-question-stats).
+True/False question statistics look just like the statistics for [Multiple-Choice](#multiple-choice-question-stats).

gems/canvas_quiz_statistics/lib/canvas_quiz_statistics/analyzers.rb

@@ -45,6 +45,7 @@ def self.inherited(klass)
   require 'canvas_quiz_statistics/analyzers/concerns/has_answers'
   require 'canvas_quiz_statistics/analyzers/essay'
   require 'canvas_quiz_statistics/analyzers/fill_in_multiple_blanks'
+  require 'canvas_quiz_statistics/analyzers/multiple_answers'
   require 'canvas_quiz_statistics/analyzers/multiple_choice'
   require 'canvas_quiz_statistics/analyzers/multiple_dropdowns'
   require 'canvas_quiz_statistics/analyzers/file_upload'

gems/canvas_quiz_statistics/lib/canvas_quiz_statistics/analyzers/multiple_answers.rb

@@ -0,0 +1,136 @@
+#
+# Copyright (C) 2014 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+module CanvasQuizStatistics::Analyzers
+  require 'canvas_quiz_statistics/analyzers/fill_in_multiple_blanks'
+
+  # Generates statistics for a set of student responses to a multiple-answers
+  # question.
+  #
+  # Response is expected to look something like this:
+  #
+  # ```javascript
+  # {
+  #   "correct": "partial",
+  #   "points": 0.5,
+  #   "question_id": 17,
+  #   "text": "",
+  #   "answer_5514": "1",
+  #   "answer_4261": "0",
+  #   "answer_3322": "1"
+  # }
+  # ```
+  class MultipleAnswers < Base
+    include Concerns::HasAnswers
+
+    # Number of students who have answered this question by picking any choice.
+    #
+    # @return [Integer]
+    metric :responses do |responses|
+      responses.select(&method(:answer_present?)).length
+    end
+
+    inherit :correct, :partially_correct, from: :fill_in_multiple_blanks
+
+    # Statistics for the answers.
+    #
+    # Example output:
+    #
+    # ```json
+    # {
+    #   "answers": [
+    #     // First part of the correct answer:
+    #     {
+    #       "id": "5514",
+    #       "text": "A",
+    #       "responses": 3,
+    #       "correct": true
+    #     },
+    #     // The second part of the correct answer:
+    #     {
+    #       "id": "4261",
+    #       "text": "B",
+    #       "responses": 0,
+    #       "correct": true
+    #     },
+    #     // A wrong choice:
+    #     {
+    #       "id": "3322",
+    #       "text": "C",
+    #       "responses": 0,
+    #       "correct": false
+    #     },
+    #     // Students who didn't make any choice:
+    #     {
+    #       "id": "none",
+    #       "text": "No Answer",
+    #       "responses": 1,
+    #       "correct": false
+    #     }
+    #   ]
+    # }
+    metric :answers do |responses|
+      answers = parse_answers do |answer, answer_stats|
+        answer_stats.merge!({ responses: 0 })
+      end
+
+      answers.tap { calculate_responses(responses, answers) }
+    end
+
+    private
+
+    def build_context(responses)
+      {}.tap do |ctx|
+        ctx[:grades] = responses.map { |r| r.fetch(:correct, nil) }.map(&:to_s)
+      end
+    end
+
+    def answer_present?(response)
+      answer_ids.any? { |id| chosen?(response[answer_key(id)]) }
+    end
+
+    def answer_ids
+      @answer_ids ||= question_data[:answers].map { |a| "#{a[:id]}" }
+    end
+
+    def answer_key(id)
+      :"answer_#{id}"
+    end
+
+    def chosen?(value)
+      value.to_s == '1'
+    end
+
+    def extract_chosen_choices(response, answers)
+      answers.select do |answer|
+        chosen?(response[answer_key(answer[:id])])
+      end
+    end
+
+    def calculate_responses(responses, answers, *args)
+      responses.each do |response|
+        choices = extract_chosen_choices(response, answers, *args)
+
+        if choices.empty?
+          choices = [ generate_missing_answer(answers) ]
+        end
+
+        choices.each { |answer| answer[:responses] += 1 }
+      end
+    end
+  end
+end

gems/canvas_quiz_statistics/spec/canvas_quiz_statistics/analyzers/multiple_answers_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+describe CanvasQuizStatistics::Analyzers::MultipleAnswers do
+  Constants = CanvasQuizStatistics::Analyzers::Base::Constants
+
+  let(:question_data) { QuestionHelpers.fixture('multiple_answers_question') }
+
+  subject { described_class.new(question_data) }
+
+  it 'should not blow up when no responses are provided' do
+    expect { subject.run([]).should be_present }.to_not raise_error
+  end
+
+  describe '[:responses]' do
+    it 'should count students who picked any answer' do
+      subject.run([{ answer_5514: '1' }])[:responses].should == 1
+    end
+
+    it 'should not count those who did not' do
+      subject.run([{}])[:responses].should == 0
+      subject.run([{ answer_5514: '0' }])[:responses].should == 0
+    end
+
+    it 'should not get confused by an imaginary answer' do
+      subject.run([{ answer_1234: '1' }])[:responses].should == 0
+    end
+  end
+
+  it_behaves_like '[:correct]'
+  it_behaves_like '[:partially_correct]'
+
+  describe '[:answers][]' do
+    it 'generate "none" answer for those who picked no choice at all' do
+      stats = subject.run([{}])
+
+      answer = stats[:answers].detect do |answer|
+        answer[:id] == Constants::MissingAnswerKey
+      end
+
+      answer.should be_present
+      answer[:responses].should == 1
+    end
+  end
+
+  describe '[:answers][]' do
+    describe '[:responses]' do
+      it 'should count students who picked this answer' do
+        stats = subject.run([{ answer_5514: '1' }])
+        stats[:answers].detect { |a| a[:id] == '5514' }[:responses].should == 1
+      end
+
+      it 'should not count those who did not' do
+        stats = subject.run([{ answer_5514: '1', answer_4261: '0' }])
+        stats[:answers].detect { |a| a[:id] == '4261' }[:responses].should == 0
+      end
+    end
+  end
+end