Multi-question submission emits only a single xAPI statement #219
Comments
|
@ayub02 shared that this is a known issue. xAPI doesn't support multiple submissions in a single statement and dis-integrating the score for a multi-submission on the tracking event side is unreliable given weighting and other complexity. There's another general problem with this model of page -- multiple problems with a single submit -- that was raised in a separate conversation with a team working on learning analytics. The issue is that you cannot effectively measure time-on-task for the questions individually. This prevents performing some important measures of problem efficacy. |
|
@pomegranited while you're pondering on this you may also run across this: #311 Just a heads up in case the fixes are adjacent. |
|
I found a recommendation from adlnet for how best to deal with this issue in xAPI:
We can also use StatementRefs to connect the sub-problem "evaluated" xAPI statement to the parent problem's "evaluated" event (see Appendix A: Example Statements (3rd example down)). This lets us keep the parent problem xAPI statement mostly intact, and issue separate-but-linked xAPI statements for each sub-problem. I've explained in detail below -- let me know what you think? Parent multi-part problem xAPI statement Similar to the original problem xAPI statement, with the following changes:
{
"id": "e903feae-9a7a-44a4-8ec1-ac9f25c417cb",
"result": {
"score": {
"scaled": 1.0,
"raw": 3.0,
"min": 0.0,
"max": 3.0
},
"success": true,
},
"version": "1.0.3",
"actor": {
"objectType": "Agent",
"account": {
"name": "7c27fd76-cef8-452f-8cac-7ddf0c8ec593",
"homePage": "http://local.overhang.io:8000"
}
},
"verb": {
"id": "https://w3id.org/xapi/acrossx/verbs/evaluated",
"display": {
"en": "evaluated"
}
},
"object": {
"id": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
"objectType": "Group", // was "Activity"
"definition": {
"description": {
"en-US": "Multiple Choice Questions"
},
"type": "http://adlnet.gov/expapi/activities/cmi.interaction",
"interactionType": "other", // was "choice"
"extensions": {
"http://id.tincanapi.com/extension/attempt-id": 7
}
}
},
"timestamp": "2022-07-19T15:18:51.610141+00:00",
"context": {
"contextActivities": {
"parent": [
{
"id": "http://local.overhang.io:8000/course/course-v1:edX+DemoX+Demo_Course",
"objectType": "Activity",
"definition": {
"name": {
"en-US": "Demonstration Course"
},
"type": "http://adlnet.gov/expapi/activities/course"
}
}
]
},
"extensions": {
"https://github.com/edx/event-routing-backends/blob/master/docs/xapi-extensions/eventVersion.rst": "1.0"
}
}
}And issue Activity statements for each of the parts of the problem, which are like our original Activity xAPI statement, with the following changes:
Question 1 xAPI statement {
"id": "<new uuid>",
"result": {
"response": "['blue']",
"success": true,
},
"version": "1.0.3",
"actor": {
"objectType": "Agent",
"account": {
"name": "7c27fd76-cef8-452f-8cac-7ddf0c8ec593",
"homePage": "http://local.overhang.io:8000"
}
},
"verb": {
"id": "https://w3id.org/xapi/acrossx/verbs/evaluated",
"display": {
"en": "evaluated"
}
},
"object": {
"objectType":"Activity",
"id" :"http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4_2_1",
"definition": {
"description": {
"en-US": "Question 1"
},
"type": "http://adlnet.gov/expapi/activities/cmi.interaction",
"interactionType": "choice",
}
},
"timestamp": "2022-07-19T15:18:51.610141+00:00",
"context": {
"statement": {
"id" :"e903feae-9a7a-44a4-8ec1-ac9f25c417cb", // the Group object's event ID above
"objectType": "StatementRef",
},
"contextActivities": {
"parent": [
{ // "object" stanza from referenced parent statement
"id": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
"objectType": "Group",
"definition": {
"description": {
"en-US": "Multiple Choice Questions"
},
"type": "http://adlnet.gov/expapi/activities/cmi.interaction",
"interactionType": "choice",
"extensions": {
"http://id.tincanapi.com/extension/attempt-id": 7
}
}
}
],
"grouping": [
{ // "parent" course stanza from referenced parent statement
"id": "http://local.overhang.io:8000/course/course-v1:edX+DemoX+Demo_Course",
"objectType": "Activity",
"definition": {
"name": {
"en-US": "Demonstration Course"
},
"type": "http://adlnet.gov/expapi/activities/course"
}
}
]
},
"extensions": {
"https://github.com/edx/event-routing-backends/blob/master/docs/xapi-extensions/eventVersion.rst": "1.0"
}
}
}Question 3 xAPI statement Similar to Question 1 above, but with Question 3's "object.id" and submitted "result.response". {
"id": "<new uuid>",
"result": {
"response": "['a piano', 'a guitar']",
"success": true,
},
"version": "1.0.3",
"actor": {
"objectType": "Agent",
"account": {
"name": "7c27fd76-cef8-452f-8cac-7ddf0c8ec593",
"homePage": "http://local.overhang.io:8000"
}
},
"verb": {
"id": "https://w3id.org/xapi/acrossx/verbs/evaluated",
"display": {
"en": "evaluated"
}
},
"object": {
"objectType":"Activity",
"id" :"http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4_4_1",
"definition": {
"description": {
"en-US": "Question 3"
},
"type": "http://adlnet.gov/expapi/activities/cmi.interaction",
"interactionType": "choice",
}
},
"timestamp": "2022-07-19T15:18:51.610141+00:00",
"context": {
"statement": {
"id" :"e903feae-9a7a-44a4-8ec1-ac9f25c417cb",
"objectType": "StatementRef",
},
"contextActivities": {
"parent": [
{
"id": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
"objectType": "Group",
"definition": {
"description": {
"en-US": "Multiple Choice Questions"
},
"type": "http://adlnet.gov/expapi/activities/cmi.interaction",
"interactionType": "choice",
"extensions": {
"http://id.tincanapi.com/extension/attempt-id": 7
}
}
}
],
"grouping": [
{
"id": "http://local.overhang.io:8000/course/course-v1:edX+DemoX+Demo_Course",
"objectType": "Activity",
"definition": {
"name": {
"en-US": "Demonstration Course"
},
"type": "http://adlnet.gov/expapi/activities/course"
}
}
]
},
"extensions": {
"https://github.com/edx/event-routing-backends/blob/master/docs/xapi-extensions/eventVersion.rst": "1.0"
}
}
}Issues/questions about this approach:
Notes Some other things I picked up, which are unrelated to this issue of splitting multi-problem events:
|
|
This makes sense to me from a data model standpoint, and seems really complicated from a query standpoint, but I think it is a good way forward. I think our current suite of reports cares more about the individual questions, but it will be good to be able to model these things in a way that makes sense. |
|
@bmtcril Ok cool -- is there anything I can do to make the querying simpler? These points relate to that question:
Ok to proceed with dev here? |
Yep, go for it unless anyone else has other input! |
|
@bmtcril The first thing I'm trying to do here is to find a place to wedge in the "some events get transformed into multiple events" logic. Do you have opinions on where this should go? I'm considering the following approach, but it doesn't feel very clean..
But:
The only alternative I came up with was refactoring |
|
I see what you're saying, and it's definitely a pain either way. Either way you're suggesting would be ok with me, but overall I have a preference for "doing things one way" instead of having the cognitive load of having one special case. I think you'd have to do most of the plumbing for the To the best of my knowledge no one is using this in production, let alone writing custom processor chains. @ziafazal might know better. The only other process loop I know if is the management command which is pretty well plumbed for multiple statements already so hopefully not too bad. Another thing to keep in mind is that Vector can read statements off the logs, so we'd need to make sure the loggers for xAPI and Caliper both emit one statement per line like: |
Ahh... that's a big gotcha. I'll make sure that's still working, and put a giant warning comment around those log statements :) And from what you said when we talked, I'll see if I can make everything use the bulk_import, and remove the single-event submission code. |
|
I meant the bulk send method in the router vs send fwiw |
This page in the demo course represents a pretty standard page model where there are multiple questions, but a single button to submit all the problems.
When one submits answers to these three questions, a tracking event with all three responses is created.
When that tracking event is mapped to an xAPI statement, however:
Though they are long, I'm including the json documents here for reference.
Tracking event
{ "name": "problem_check", "context": { "course_id": "course-v1:edX+DemoX+Demo_Course", "course_user_tags": { }, "user_id": 4, "path": "/courses/course-v1:edX+DemoX+Demo_Course/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4/handler/xmodule_handler/problem_check", "org_id": "edX", "enterprise_uuid": "", "module": { "display_name": "Multiple Choice Questions", "usage_key": "block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4" }, "asides": { } }, "username": "e0d", "session": "97662bef7c463c187b8fd91e0f580468", "ip": "172.18.0.1", "agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36", "host": "local.overhang.io:8000", "referer": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@vertical+block@54bb9b142c6c4c22afc62bcb628f0e68?show_title=0&show_bookmark_button=0&recheck_access=1&view=student_view&format=Homework", "accept_language": "en-US,en;q=0.9", "event": { "state": { "seed": 1, "student_answers": { "a0effb954cca4759994f1ac9e9434bf4_3_1": "choice_2", "a0effb954cca4759994f1ac9e9434bf4_4_1": [ "choice_0", "choice_2" ], "a0effb954cca4759994f1ac9e9434bf4_2_1": "blue" }, "has_saved_answers": false, "correct_map": { "a0effb954cca4759994f1ac9e9434bf4_2_1": { "correctness": "correct", "npoints": null, "msg": "", "hint": "", "hintmode": null, "queuestate": null, "answervariable": null }, "a0effb954cca4759994f1ac9e9434bf4_3_1": { "correctness": "correct", "npoints": null, "msg": "", "hint": "", "hintmode": null, "queuestate": null, "answervariable": null }, "a0effb954cca4759994f1ac9e9434bf4_4_1": { "correctness": "correct", "npoints": null, "msg": "", "hint": "", "hintmode": null, "queuestate": null, "answervariable": null } }, "input_state": { "a0effb954cca4759994f1ac9e9434bf4_2_1": { }, "a0effb954cca4759994f1ac9e9434bf4_3_1": { }, "a0effb954cca4759994f1ac9e9434bf4_4_1": { } }, "done": true }, "problem_id": "block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4", "answers": { "a0effb954cca4759994f1ac9e9434bf4_4_1": [ "choice_0", "choice_2" ], "a0effb954cca4759994f1ac9e9434bf4_3_1": "choice_2", "a0effb954cca4759994f1ac9e9434bf4_2_1": "yellow" }, "grade": 2, "max_grade": 3, "correct_map": { "a0effb954cca4759994f1ac9e9434bf4_2_1": { "correctness": "incorrect", "npoints": null, "msg": "", "hint": "", "hintmode": null, "queuestate": null, "answervariable": null }, "a0effb954cca4759994f1ac9e9434bf4_3_1": { "correctness": "correct", "npoints": null, "msg": "", "hint": "", "hintmode": null, "queuestate": null, "answervariable": null }, "a0effb954cca4759994f1ac9e9434bf4_4_1": { "correctness": "correct", "npoints": null, "msg": "", "hint": "", "hintmode": null, "queuestate": null, "answervariable": null } }, "success": "incorrect", "attempts": 9, "submission": { "a0effb954cca4759994f1ac9e9434bf4_4_1": { "question": "", "answer": [ "a piano", "a guitar" ], "response_type": "choiceresponse", "input_type": "checkboxgroup", "correct": true, "variant": "", "group_label": "" }, "a0effb954cca4759994f1ac9e9434bf4_3_1": { "question": "", "answer": "a chair", "response_type": "multiplechoiceresponse", "input_type": "choicegroup", "correct": true, "variant": "", "group_label": "" }, "a0effb954cca4759994f1ac9e9434bf4_2_1": { "question": "", "answer": "yellow", "response_type": "optionresponse", "input_type": "optioninput", "correct": false, "variant": "", "group_label": "" } } }, "time": "2022-07-19T15:29:01.188319+00:00", "event_type": "problem_check", "event_source": "server", "page": "x_module" }xAPI Statement
{ "id": "e903feae-9a7a-44a4-8ec1-ac9f25c417cb", "result": { "score": { "scaled": 1.0, "raw": 3.0, "min": 0.0, "max": 3.0 }, "success": true, "response": "['a piano', 'a guitar']" }, "version": "1.0.3", "actor": { "objectType": "Agent", "account": { "name": "7c27fd76-cef8-452f-8cac-7ddf0c8ec593", "homePage": "http://local.overhang.io:8000" } }, "verb": { "id": "https://w3id.org/xapi/acrossx/verbs/evaluated", "display": { "en": "evaluated" } }, "object": { "id": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4", "objectType": "Activity", "definition": { "description": { "en-US": "" }, "type": "http://adlnet.gov/expapi/activities/cmi.interaction", "interactionType": "choice", "extensions": { "http://id.tincanapi.com/extension/attempt-id": 7 } } }, "timestamp": "2022-07-19T15:18:51.610141+00:00", "context": { "contextActivities": { "parent": [ { "id": "http://local.overhang.io:8000/course/course-v1:edX+DemoX+Demo_Course", "objectType": "Activity", "definition": { "name": { "en-US": "Demonstration Course" }, "type": "http://adlnet.gov/expapi/activities/course" } } ] }, "extensions": { "https://github.com/edx/event-routing-backends/blob/master/docs/xapi-extensions/eventVersion.rst": "1.0" } } }The text was updated successfully, but these errors were encountered: