Skip to content
Permalink
Browse files

add type inference and variable value generators for JME part

Numbas.jme.inferVariableTypes finds assignments of types to free
variables in an expression which should allow the expression to be
evaluated.

For example, in `k*det(a)`, `a` should be a matrix, and `k` can be
anything that can be multiplied by a number.

The type inference is used to establish types for free variables in the
answers to mathematical expression parts, so we can pick random values
for them and evaluate, to compare against the student's answer. There
are default expressions generating values for the built-in types for
which that makes sense (i.e. things like number and matrix, but not
list, dict or html).

Now, parts with answers like `k*det(a)` and `a and b` can be marked
without any further setup by the question author.

Question authors can optionally give a value-generating expression for
a variable used in the correct answer. The expression is given a value
`vRange` representing the checking range parameter for the part.

Now that the type inference is in, I'm struggling to come up with a
question where the custom expressions are needed. Maybe an expression
whose domain is x>1 and y<-1?

fixes #398
  • Loading branch information...
christianp committed Feb 4, 2019
1 parent 29af192 commit a55aa677295f38163587d190a7ed6bf239b78833
@@ -696,6 +696,7 @@ class JMEPart(Part):
vsetRangeEnd = 1
vsetRangePoints = 5
checkVariableNames = False
valueGenerators = []

def __init__(self,marks=0,prompt=''):
Part.__init__(self,marks,prompt)
@@ -745,12 +746,16 @@ def loadDATA(self, builder, data):
self.vsetRangeStart = vsetrange[0]
self.vsetRangeEnd = vsetrange[1]

if haskey(data,'valuegenerators'):
self.valueGenerators = case_insensitive_get(data,'valuegenerators')

def toxml(self):
part = super(JMEPart,self).toxml()
part.append(makeTree(['answer',
['correctanswer',['math']],
['checking',
['range']
['range'],
['valuegenerators'],
]
]))

@@ -770,13 +775,20 @@ def toxml(self):
'failurerate': strcons_fix(self.failureRate)
}
checking.find('range').attrib = {'start': strcons_fix(self.vsetRangeStart), 'end': strcons_fix(self.vsetRangeEnd), 'points': strcons_fix(self.vsetRangePoints)}

valueGenerators = checking.find('valuegenerators')
for g in self.valueGenerators:
generator = etree.Element('generator')
generator.attrib = {'name': g['name'], 'value': g['value']}
valueGenerators.append(generator)

answer.append(self.maxLength.toxml())
answer.append(self.minLength.toxml())
answer.append(self.mustHave.toxml())
answer.append(self.notAllowed.toxml())
answer.append(self.expectedVariableNames.toxml())
answer.append(self.mustMatchPattern.toxml())

return part

class Restriction:
@@ -192,6 +192,7 @@
"part.jme.not-allowed several": "Your answer must not contain any of: {{strings}}",
"part.jme.unexpected variable name": "Your answer was interpreted to use the unexpected variable name <code>{{name}}</code>.",
"part.jme.unexpected variable name suggestion": "Your answer was interpreted to use the unexpected variable name <code>{{name}}</code>. Did you mean <code>{{suggestion}}</code>?",
"part.jme.invalid value generator expression": "Invalid value generator expression for variable <code>{{name}}</code>: {{-message}}",
"part.patternmatch.display answer missing": "Display answer is missing",
"part.patternmatch.correct except case": "Your answer is correct, except for the case.",
"part.numberentry.correct except decimal": "Your answer is within the allowed range, but decimal numbers are not allowed.",
@@ -100,11 +100,38 @@ requiredStringsPenalty:
)

vRange (The range to pick variable values from):
settings["vsetRangeStart"]..settings["vsetRangeEnd"]#0
settings["vsetRangeStart"]..settings["vsetRangeEnd"] # 0

answerVariables (Variables used in either the correct answer or the student's answer):
correctVariables or studentVariables

value_generators (Expressions which generate values for each variable in the answer):
dict(map(
[name, get(settings["valueGenerators"],name,default_value_generator[get(variable_types,name,"number")])],
name,
answerVariables
))

variable_types (Inferred types for each of the variables):
infer_variable_types(correctExpr)[0]

default_value_generator:
[
"number": expression("random(vRange)"),
"matrix": expression("matrix(repeat(repeat(random(vRange),3),3))"),
"vector": expression("vector(repeat(random(vRange),3))"),
"boolean": expression("random(true,false)"),
"set": expression("set(repeat(random(vRange),5))")
]

vset (The sets of variable values to test against):
repeat(
dict(map([x,random(vRange)],x,correctVariables or studentVariables)),
dict(map([
x,
let(generator,value_generators[x],
eval(generator,["vrange":vRange])
)
],x,answerVariables)),
settings["vsetRangePoints"]
)

@@ -1493,6 +1493,14 @@ newBuiltin('resultsequal',['?','?',TString,TNum],TBool,null, {
}
});

newBuiltin('infer_variable_types',[TExpression],TDict,null, {
evaluate: function(args, scope) {
var expr = args[0];
var assignments = jme.inferVariableTypes(expr.tree,scope);
return jme.wrapValue(assignments);
}
});

/** Helper function for the JME `match` function
* @param {Numbas.jme.tree} expr
* @param {String} pattern
@@ -2238,7 +2238,7 @@ var funcObjAcc = 0; //accumulator for ids for funcObjs, so they can be sorted
* @memberof Numbas.jme
* @constructor
* @param {String} name
* @param {Array.<Function|String>} intype - A list of data type constructors for the function's paramters' types. Use the string '?' to match any type. Or, give the type's name with a '*' in front to match any number of that type. If `null`, then `options.typecheck` is used.
* @param {Array.<Function|String>} intype - A list of data type constructors for the function's parameters' types. Use the string '?' to match any type. Or, give the type's name with a '*' in front to match any number of that type. If `null`, then `options.typecheck` is used.
* @param {Function} outcons - The constructor for the output value of the function
* @param {Numbas.jme.evaluate_fn} fn - JavaScript code which evaluates the function.
* @param {Numbas.jme.funcObj_options} options
@@ -2851,4 +2851,128 @@ var compareTrees = jme.compareTrees = function(a,b) {
}
return sign_a==sign_b ? 0 : sign_a ? 1 : -1;
}

/** Infer the types of variables in an expression, by trying all definitions of functions and returning only those that can be satisfied by an assignment of types to variable names.
* Doesn't work well on functions with unknown return type, like `if` and `switch`. In these cases, it assumes the return type of the function is whatever it needs to be, even if that is inconsistent with what the function would actually do.
* Returns a list of possible assignments of types, arranged in order of preference: numbers are preferred to matrices, for example, and then by specificity (number of variables assigned a type).
* @param {Numbas.jme.tree} tree
* @param {Numbas.jme.Scope} scope
* @param {String} outtype - the desired return type of the expression
* @param {Object.<String>} assigned_types - types of variables which have already been inferred
* @returns {Array.<Object.<String>>}
*/
var inferVariableTypes = jme.inferVariableTypes = function(tree, scope, outtype, assigned_types) {
/** Deduplicate a list of objects
* Return only items with unique sets of entries
* @param {Array.<Object>} objs
* @returns {Arra.<Object>}
*/
function dedup(objs) {
if(!objs.length) {
return [];
}
function compare(a,b) {
var as = Object.entries(a).sort();
var bs = Object.entries(b).sort();
for(var i=0;i<Math.max(as.length,bs.length);i++) {
if(i>=as.length) {
return -1;
}
if(i>=bs.length) {
return 1;
}
var ai = as[i][0];
var bi = bs[i][0];
var av = as[i][1];
var bv = bs[i][1];
var r = ai>bi ? 1 : ai<bi ? -1 : av>bv ? 1 : av<bv ? -1 : 0;
if(r!=0) {
return r;
}
}
return 0;
}
objs.sort(compare);
var out = [objs[0]];

for(var i=1;i<objs.length;i++) {
if(compare(objs[i],objs[i-1])!=0) {
out.push(objs[i]);
}
}
return out;
}

assigned_types = assigned_types || {};
//console.log(`infer for ${Numbas.jme.display.treeToJME(tree)}, want ${outtype}, assigned ${JSON.stringify(assigned_types)}`);
if(outtype=='?') {
outtype = undefined;
}
switch(tree.tok.type) {
case 'name':
var name = tree.tok.name.toLowerCase();
if(assigned_types[name] !== undefined) {
if(!outtype || outtype==assigned_types[name]) {
return [assigned_types];
} else {
return [];
}
} else {
var assignment = util.copyobj(assigned_types);
if(outtype) {
assignment[name] = outtype;
}
return [assignment];
}
case 'op':
case 'function':
var name = tree.tok.name.toLowerCase();
var functions = scope.getFunction(name.toLowerCase());
var assignments = [];
functions.forEach(function(fn) {
//console.log(`try ${name} : ${fn.intype} -> ${fn.outtype}`);
var fn_assignments = [assigned_types];
for(var i=0;i<tree.args.length;i++) {
var nassignments = [];
fn_assignments.forEach(function(assignment) {
var res = inferVariableTypes(tree.args[i],scope,fn.intype[i],assignment);
nassignments = nassignments.concat(res);
});
fn_assignments = nassignments;
}
assignments = assignments.concat(fn_assignments);
});
assignments = dedup(assignments);
//console.log('Decision:');
//assignments.forEach(r=>console.log(`* ${JSON.stringify(r)}`));
var type_preference_order = ['number','matrix','vector','boolean','set'];
function preference(type) {
var i = type_preference_order.indexOf(type);
return i==-1 ? Infinity : i;
}
assignments.sort(function(a,b) {
var as = Object.keys(a).sort();
var bs = Object.keys(b).sort();
for(var i=0;i<Math.max(as.length,bs.length);i++) {
if(i>=as.length) {
return -1;
}
if(i>=bs.length) {
return 1;
}
var ai = preference(a[as[i]]);
var bi = preference(b[bs[i]]);
var r = ai>bi ? 1 : ai<bi ? -1 : 0;
if(r!=0) {
return r;
}
}
return 0;
});
return assignments;
default:
return [assigned_types];
}
}

});
@@ -32,6 +32,7 @@ var JMEPart = Numbas.parts.JMEPart = function(path, question, parentPart)
{
var settings = this.settings;
util.copyinto(JMEPart.prototype.settings,settings);
settings.valueGenerators = {};
settings.mustHave = [];
settings.notAllowed = [];
settings.expectedVariableNames = [];
@@ -52,6 +53,17 @@ JMEPart.prototype = /** @lends Numbas.JMEPart.prototype */
var parametersPath = 'answer';
tryGetAttribute(settings,xml,parametersPath+'/checking',['type','accuracy','failurerate'],['checkingType','checkingAccuracy','failureRate']);
tryGetAttribute(settings,xml,parametersPath+'/checking/range',['start','end','points'],['vsetRangeStart','vsetRangeEnd','vsetRangePoints']);

var valueGeneratorsNode = xml.selectSingleNode('answer/checking/valuegenerators');
if(valueGeneratorsNode) {
var valueGenerators = valueGeneratorsNode.selectNodes('generator');
for(var i=0;i<valueGenerators.length;i++) {
var generator = {};
tryGetAttribute(generator,xml,valueGenerators[i],['name','value']);
this.addValueGenerator(generator.name, generator.value);
}
}

//max length and min length
tryGetAttribute(settings,xml,parametersPath+'/maxlength',['length','partialcredit'],['maxLength','maxLengthPC']);
var messageNode = xml.selectSingleNode('answer/maxlength/message');
@@ -121,16 +133,30 @@ JMEPart.prototype = /** @lends Numbas.JMEPart.prototype */
}
},
loadFromJSON: function(data) {
var p = this;
var settings = this.settings;
var tryLoad = Numbas.json.tryLoad;
var tryGet = Numbas.json.tryGet;
tryLoad(data, ['answer', 'answerSimplification'], settings, ['correctAnswerString', 'answerSimplificationString']);
tryLoad(data, ['checkingType', 'checkingAccuracy', 'failureRate'], settings, ['checkingType', 'checkingAccuracy', 'failureRate']);
tryLoad(data, ['vsetRangePoints'], settings);
var vsetRange = tryGet(data,'vsetRange');
if(vsetRange) {
settings.vsetRangeStart = vsetRange[0];
settings.vsetRangeEnd = vsetRange[1];
}
tryLoad(data.maxlength, ['length', 'partialCredit', 'message'], settings, ['maxLength', 'maxLengthPC', 'maxLengthMessage']);
tryLoad(data.minlength, ['length', 'partialCredit', 'message'], settings, ['minLength', 'minLengthPC', 'minLengthMessage']);
tryLoad(data.musthave, ['strings', 'showStrings', 'partialCredit', 'message'], settings, ['mustHave', 'mustHaveShowStrings', 'mustHavePC', 'mustHaveMessage']);
tryLoad(data.notallowed, ['strings', 'showStrings', 'partialCredit', 'message'], settings, ['notAllowed', 'notAllowedShowStrings', 'notAllowedPC', 'notAllowedMessage']);
tryLoad(data.mustmatchpattern, ['pattern', 'partialCredit', 'message', 'nameToCompare'], settings, ['mustMatchPattern', 'mustMatchPC', 'mustMatchMessage', 'nameToCompare']);
tryLoad(data, ['checkVariableNames', 'expectedVariableNames', 'showPreview'], settings);
var valuegenerators = tryGet(data,'valuegenerators');
if(valuegenerators) {
valuegenerators.forEach(function(g) {
p.addValueGenerator(g.name,g.value);
});
}
},
resume: function() {
if(!this.store) {
@@ -262,6 +288,21 @@ JMEPart.prototype = /** @lends Numbas.JMEPart.prototype */
*/
rawStudentAnswerAsJME: function() {
return new Numbas.jme.types.TString(this.studentAnswer);
},

/** Add a value generator expression to the list in this part's settings.
* @param {String} name
* @param {JME} expr
*/
addValueGenerator: function(name, expr) {
try {
var expression = new jme.types.TExpression(expr);
if(expression.tree) {
this.settings.valueGenerators[name] = expression;
}
} catch(e) {
this.error('part.jme.invalid value generator expression',{name: name, expr: expr, message: e.message}, e);
}
}
};
['resume','finaliseLoad','loadFromXML','loadFromJSON'].forEach(function(method) {
Oops, something went wrong.

0 comments on commit a55aa67

Please sign in to comment.
You can’t perform that action at this time.