Skip to content

Commit edeb158

Browse files
christianpchristianp
committed
The shunt step looks for any opening bracket character when trying to find a matching bracket.
While I was hacking together a parser that uses curly braces, I noticed that the actions for the other bracket types assume that `(` and `[` are the only possible opening brackets. This changes them to look for anything matching /^\p{Ps}$/u, any single opening bracket character.
1 parent 36c567b commit edeb158

File tree

3 files changed

+110
-56
lines changed

3 files changed

+110
-56
lines changed

‎runtime/scripts/jme.js‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -282,8 +282,8 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
282282
compileList: function(expr) {
283283
expr += ''; //make sure expression is a string and not a number or anything like that
284284
if(!expr.trim().length) {
285-
return null;
286-
}
285+
return null;
286+
}
287287
//typecheck
288288
//tokenise expression
289289
var tokens = jme.tokenise(expr);
@@ -455,18 +455,18 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
455455
argbrackets = true;
456456
const si = i + 1;
457457
while(i < s.length && s.charAt(i) != ']') {
458-
i++;
459-
}
458+
i++;
459+
}
460460
if(i == s.length) {
461-
throw(new Numbas.Error('jme.texsubvars.no right bracket', {op:cmd}));
462-
} else {
461+
throw(new Numbas.Error('jme.texsubvars.no right bracket', {op:cmd}));
462+
} else {
463463
args = s.slice(si, i);
464464
i++;
465465
}
466466
}
467467
if(!argbrackets) {
468-
args = 'all';
469-
}
468+
args = 'all';
469+
}
470470
out.push(args);
471471
if(s.charAt(i) != '{') {
472472
throw(new Numbas.Error('jme.texsubvars.missing parameter', {op:cmd, parameter:s}));
@@ -476,14 +476,14 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
476476
while(i < s.length - 1 && brackets > 0) {
477477
i++;
478478
if(s.charAt(i) == '{') {
479-
brackets++;
480-
} else if(s.charAt(i) == '}') {
481-
brackets--;
482-
}
479+
brackets++;
480+
} else if(s.charAt(i) == '}') {
481+
brackets--;
482+
}
483483
}
484484
if(i == s.length - 1 && brackets > 0) {
485-
throw(new Numbas.Error('jme.texsubvars.no right brace', {op:cmd}));
486-
}
485+
throw(new Numbas.Error('jme.texsubvars.no right brace', {op:cmd}));
486+
}
487487
var expr = s.slice(si, i);
488488
s = s.slice(i + 1);
489489
out.push(expr);
@@ -1470,6 +1470,24 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
14701470
return jme.normaliseName(op, this.options);
14711471
},
14721472

1473+
/** Is this token an opening bracket, such as `(` or `[`?
1474+
*
1475+
* @param {Numbas.jme.token} tok
1476+
* @returns {boolean}
1477+
*/
1478+
is_opening_bracket: function(tok) {
1479+
return tok.type.match(/^\p{Ps}$/u);
1480+
},
1481+
1482+
/** Is this token a closing bracket, such as `(` or `[`?
1483+
*
1484+
* @param {Numbas.jme.token} tok
1485+
* @returns {boolean}
1486+
*/
1487+
is_closing_bracket: function(tok) {
1488+
return tok.type.match(/^\p{Pe}$/u);
1489+
},
1490+
14731491
/** Descriptions of kinds of token that the tokeniser can match.
14741492
* `re` is a regular expression matching the token.
14751493
* `parse` is a function which takes a RegEx match object, the tokens produced up to this point, the input expression, and the current position in the expression.
@@ -1549,7 +1567,7 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
15491567
var postfix = false;
15501568
var prefix = false;
15511569
name = this.opSynonym(name);
1552-
if(tokens.length == 0 || (nt = tokens.at(-1).type) == '(' || nt == ',' || nt == '[' || nt == ['lambda'] || (nt == 'op' && !tokens.at(-1).postfix) || nt == 'keypair') {
1570+
if(tokens.length == 0 || this.is_opening_bracket(nt = tokens.at(-1)) || nt.type == ',' || nt.type == 'lambda' || (nt.type == 'op' && !nt.postfix) || nt.type == 'keypair') {
15531571
var prefixForm = this.getPrefixForm(name);
15541572
if(prefixForm !== undefined) {
15551573
name = prefixForm;
@@ -1735,11 +1753,11 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
17351753
}
17361754
},
17371755
',': function(tok) {
1738-
if(this.tokens[this.i - 1].type == '(' || this.tokens[this.i - 1].type == '[') {
1756+
if(this.is_opening_bracket(this.tokens.at(this.i-1))) {
17391757
throw(new Numbas.Error('jme.shunt.expected argument before comma'));
17401758
}
17411759
//reached end of expression defining function parameter, so pop all of its operations off stack and onto output
1742-
while(this.stack.length > 0 && this.stack.at(-1).type != "(" && this.stack.at(-1).type != '[') {
1760+
while(this.stack.length > 0 && !this.is_opening_bracket(this.stack.at(-1))) {
17431761
this.addoutput(this.popstack())
17441762
}
17451763
this.numvars[this.numvars.length - 1]++;
@@ -1785,7 +1803,7 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
17851803
var i = this.i;
17861804
var tokens = this.tokens;
17871805
var last_token = i == 0 ? null : tokens[i - 1].type;
1788-
if(i == 0 || last_token == '(' || last_token == '[' || last_token == ',' || last_token == 'op' || last_token == 'keypair' || last_token == 'lambda') {
1806+
if(i == 0 || this.is_opening_bracket(tokens.at(i-1)) || last_token == ',' || last_token == 'op' || last_token == 'keypair' || last_token == 'lambda') {
17891807
this.listmode.push('new');
17901808
} else {
17911809
this.listmode.push('index');

‎tests/jme-runtime.js‎

Lines changed: 37 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9097,8 +9097,8 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
90979097
compileList: function(expr) {
90989098
expr += ''; //make sure expression is a string and not a number or anything like that
90999099
if(!expr.trim().length) {
9100-
return null;
9101-
}
9100+
return null;
9101+
}
91029102
//typecheck
91039103
//tokenise expression
91049104
var tokens = jme.tokenise(expr);
@@ -9270,18 +9270,18 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
92709270
argbrackets = true;
92719271
const si = i + 1;
92729272
while(i < s.length && s.charAt(i) != ']') {
9273-
i++;
9274-
}
9273+
i++;
9274+
}
92759275
if(i == s.length) {
9276-
throw(new Numbas.Error('jme.texsubvars.no right bracket', {op:cmd}));
9277-
} else {
9276+
throw(new Numbas.Error('jme.texsubvars.no right bracket', {op:cmd}));
9277+
} else {
92789278
args = s.slice(si, i);
92799279
i++;
92809280
}
92819281
}
92829282
if(!argbrackets) {
9283-
args = 'all';
9284-
}
9283+
args = 'all';
9284+
}
92859285
out.push(args);
92869286
if(s.charAt(i) != '{') {
92879287
throw(new Numbas.Error('jme.texsubvars.missing parameter', {op:cmd, parameter:s}));
@@ -9291,14 +9291,14 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
92919291
while(i < s.length - 1 && brackets > 0) {
92929292
i++;
92939293
if(s.charAt(i) == '{') {
9294-
brackets++;
9295-
} else if(s.charAt(i) == '}') {
9296-
brackets--;
9297-
}
9294+
brackets++;
9295+
} else if(s.charAt(i) == '}') {
9296+
brackets--;
9297+
}
92989298
}
92999299
if(i == s.length - 1 && brackets > 0) {
9300-
throw(new Numbas.Error('jme.texsubvars.no right brace', {op:cmd}));
9301-
}
9300+
throw(new Numbas.Error('jme.texsubvars.no right brace', {op:cmd}));
9301+
}
93029302
var expr = s.slice(si, i);
93039303
s = s.slice(i + 1);
93049304
out.push(expr);
@@ -10285,6 +10285,24 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
1028510285
return jme.normaliseName(op, this.options);
1028610286
},
1028710287

10288+
/** Is this token an opening bracket, such as `(` or `[`?
10289+
*
10290+
* @param {Numbas.jme.token} tok
10291+
* @returns {boolean}
10292+
*/
10293+
is_opening_bracket: function(tok) {
10294+
return tok.type.match(/^\p{Ps}$/u);
10295+
},
10296+
10297+
/** Is this token a closing bracket, such as `(` or `[`?
10298+
*
10299+
* @param {Numbas.jme.token} tok
10300+
* @returns {boolean}
10301+
*/
10302+
is_closing_bracket: function(tok) {
10303+
return tok.type.match(/^\p{Pe}$/u);
10304+
},
10305+
1028810306
/** Descriptions of kinds of token that the tokeniser can match.
1028910307
* `re` is a regular expression matching the token.
1029010308
* `parse` is a function which takes a RegEx match object, the tokens produced up to this point, the input expression, and the current position in the expression.
@@ -10364,7 +10382,7 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
1036410382
var postfix = false;
1036510383
var prefix = false;
1036610384
name = this.opSynonym(name);
10367-
if(tokens.length == 0 || (nt = tokens.at(-1).type) == '(' || nt == ',' || nt == '[' || nt == ['lambda'] || (nt == 'op' && !tokens.at(-1).postfix) || nt == 'keypair') {
10385+
if(tokens.length == 0 || this.is_opening_bracket(nt = tokens.at(-1)) || nt.type == ',' || nt.type == 'lambda' || (nt.type == 'op' && !nt.postfix) || nt.type == 'keypair') {
1036810386
var prefixForm = this.getPrefixForm(name);
1036910387
if(prefixForm !== undefined) {
1037010388
name = prefixForm;
@@ -10550,11 +10568,11 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
1055010568
}
1055110569
},
1055210570
',': function(tok) {
10553-
if(this.tokens[this.i - 1].type == '(' || this.tokens[this.i - 1].type == '[') {
10571+
if(this.is_opening_bracket(this.tokens.at(this.i-1))) {
1055410572
throw(new Numbas.Error('jme.shunt.expected argument before comma'));
1055510573
}
1055610574
//reached end of expression defining function parameter, so pop all of its operations off stack and onto output
10557-
while(this.stack.length > 0 && this.stack.at(-1).type != "(" && this.stack.at(-1).type != '[') {
10575+
while(this.stack.length > 0 && !this.is_opening_bracket(this.stack.at(-1))) {
1055810576
this.addoutput(this.popstack())
1055910577
}
1056010578
this.numvars[this.numvars.length - 1]++;
@@ -10600,7 +10618,7 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
1060010618
var i = this.i;
1060110619
var tokens = this.tokens;
1060210620
var last_token = i == 0 ? null : tokens[i - 1].type;
10603-
if(i == 0 || last_token == '(' || last_token == '[' || last_token == ',' || last_token == 'op' || last_token == 'keypair' || last_token == 'lambda') {
10621+
if(i == 0 || this.is_opening_bracket(tokens.at(i-1)) || last_token == ',' || last_token == 'op' || last_token == 'keypair' || last_token == 'lambda') {
1060410622
this.listmode.push('new');
1060510623
} else {
1060610624
this.listmode.push('index');
@@ -18648,7 +18666,7 @@ var texOps = jme.display.texOps = {
1864818666
var list = tree.args[0];
1864918667
var items;
1865018668
items = list.tok ? list.args : list.value.map(tok => { return {tok} });
18651-
return '\\left\\{ ' + items.map(item => this.render(item)).join(Numbas.locale.default_list_separator+' ') + ' \\right\\}';
18669+
return '\\left\\{ ' + items.map(item => this.render(item)).join(Numbas.locale.default_list_separator + ' ') + ' \\right\\}';
1865218670
} else {
1865318671
return '\\left\\{ ' + texArgs.join(Numbas.locale.default_list_separator + ' ') + ' \\right\\}';
1865418672
}

‎tests/numbas-runtime.js‎

Lines changed: 37 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8433,8 +8433,8 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
84338433
compileList: function(expr) {
84348434
expr += ''; //make sure expression is a string and not a number or anything like that
84358435
if(!expr.trim().length) {
8436-
return null;
8437-
}
8436+
return null;
8437+
}
84388438
//typecheck
84398439
//tokenise expression
84408440
var tokens = jme.tokenise(expr);
@@ -8606,18 +8606,18 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
86068606
argbrackets = true;
86078607
const si = i + 1;
86088608
while(i < s.length && s.charAt(i) != ']') {
8609-
i++;
8610-
}
8609+
i++;
8610+
}
86118611
if(i == s.length) {
8612-
throw(new Numbas.Error('jme.texsubvars.no right bracket', {op:cmd}));
8613-
} else {
8612+
throw(new Numbas.Error('jme.texsubvars.no right bracket', {op:cmd}));
8613+
} else {
86148614
args = s.slice(si, i);
86158615
i++;
86168616
}
86178617
}
86188618
if(!argbrackets) {
8619-
args = 'all';
8620-
}
8619+
args = 'all';
8620+
}
86218621
out.push(args);
86228622
if(s.charAt(i) != '{') {
86238623
throw(new Numbas.Error('jme.texsubvars.missing parameter', {op:cmd, parameter:s}));
@@ -8627,14 +8627,14 @@ var jme = Numbas.jme = /** @lends Numbas.jme */ {
86278627
while(i < s.length - 1 && brackets > 0) {
86288628
i++;
86298629
if(s.charAt(i) == '{') {
8630-
brackets++;
8631-
} else if(s.charAt(i) == '}') {
8632-
brackets--;
8633-
}
8630+
brackets++;
8631+
} else if(s.charAt(i) == '}') {
8632+
brackets--;
8633+
}
86348634
}
86358635
if(i == s.length - 1 && brackets > 0) {
8636-
throw(new Numbas.Error('jme.texsubvars.no right brace', {op:cmd}));
8637-
}
8636+
throw(new Numbas.Error('jme.texsubvars.no right brace', {op:cmd}));
8637+
}
86388638
var expr = s.slice(si, i);
86398639
s = s.slice(i + 1);
86408640
out.push(expr);
@@ -9621,6 +9621,24 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
96219621
return jme.normaliseName(op, this.options);
96229622
},
96239623

9624+
/** Is this token an opening bracket, such as `(` or `[`?
9625+
*
9626+
* @param {Numbas.jme.token} tok
9627+
* @returns {boolean}
9628+
*/
9629+
is_opening_bracket: function(tok) {
9630+
return tok.type.match(/^\p{Ps}$/u);
9631+
},
9632+
9633+
/** Is this token a closing bracket, such as `(` or `[`?
9634+
*
9635+
* @param {Numbas.jme.token} tok
9636+
* @returns {boolean}
9637+
*/
9638+
is_closing_bracket: function(tok) {
9639+
return tok.type.match(/^\p{Pe}$/u);
9640+
},
9641+
96249642
/** Descriptions of kinds of token that the tokeniser can match.
96259643
* `re` is a regular expression matching the token.
96269644
* `parse` is a function which takes a RegEx match object, the tokens produced up to this point, the input expression, and the current position in the expression.
@@ -9700,7 +9718,7 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
97009718
var postfix = false;
97019719
var prefix = false;
97029720
name = this.opSynonym(name);
9703-
if(tokens.length == 0 || (nt = tokens.at(-1).type) == '(' || nt == ',' || nt == '[' || nt == ['lambda'] || (nt == 'op' && !tokens.at(-1).postfix) || nt == 'keypair') {
9721+
if(tokens.length == 0 || this.is_opening_bracket(nt = tokens.at(-1)) || nt.type == ',' || nt.type == 'lambda' || (nt.type == 'op' && !nt.postfix) || nt.type == 'keypair') {
97049722
var prefixForm = this.getPrefixForm(name);
97059723
if(prefixForm !== undefined) {
97069724
name = prefixForm;
@@ -9886,11 +9904,11 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
98869904
}
98879905
},
98889906
',': function(tok) {
9889-
if(this.tokens[this.i - 1].type == '(' || this.tokens[this.i - 1].type == '[') {
9907+
if(this.is_opening_bracket(this.tokens.at(this.i-1))) {
98909908
throw(new Numbas.Error('jme.shunt.expected argument before comma'));
98919909
}
98929910
//reached end of expression defining function parameter, so pop all of its operations off stack and onto output
9893-
while(this.stack.length > 0 && this.stack.at(-1).type != "(" && this.stack.at(-1).type != '[') {
9911+
while(this.stack.length > 0 && !this.is_opening_bracket(this.stack.at(-1))) {
98949912
this.addoutput(this.popstack())
98959913
}
98969914
this.numvars[this.numvars.length - 1]++;
@@ -9936,7 +9954,7 @@ jme.Parser.prototype = /** @lends Numbas.jme.Parser.prototype */ {
99369954
var i = this.i;
99379955
var tokens = this.tokens;
99389956
var last_token = i == 0 ? null : tokens[i - 1].type;
9939-
if(i == 0 || last_token == '(' || last_token == '[' || last_token == ',' || last_token == 'op' || last_token == 'keypair' || last_token == 'lambda') {
9957+
if(i == 0 || this.is_opening_bracket(tokens.at(i-1)) || last_token == ',' || last_token == 'op' || last_token == 'keypair' || last_token == 'lambda') {
99409958
this.listmode.push('new');
99419959
} else {
99429960
this.listmode.push('index');
@@ -17984,7 +18002,7 @@ var texOps = jme.display.texOps = {
1798418002
var list = tree.args[0];
1798518003
var items;
1798618004
items = list.tok ? list.args : list.value.map(tok => { return {tok} });
17987-
return '\\left\\{ ' + items.map(item => this.render(item)).join(Numbas.locale.default_list_separator+' ') + ' \\right\\}';
18005+
return '\\left\\{ ' + items.map(item => this.render(item)).join(Numbas.locale.default_list_separator + ' ') + ' \\right\\}';
1798818006
} else {
1798918007
return '\\left\\{ ' + texArgs.join(Numbas.locale.default_list_separator + ' ') + ' \\right\\}';
1799018008
}

0 commit comments

Comments
 (0)