-
Notifications
You must be signed in to change notification settings - Fork 1.1k
/
filters.js
392 lines (373 loc) · 12.6 KB
/
filters.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
/*\
title: $:/core/modules/filters.js
type: application/javascript
module-type: wikimethod
Adds tiddler filtering methods to the $tw.Wiki object.
\*/
(function(){
/*jslint node: true, browser: true */
/*global $tw: false */
"use strict";
var widgetClass = require("$:/core/modules/widgets/widget.js").widget;
/* Maximum permitted filter recursion depth */
var MAX_FILTER_DEPTH = 300;
/*
Parses an operation (i.e. a run) within a filter string
operators: Array of array of operator nodes into which results should be inserted
filterString: filter string
p: start position within the string
Returns the new start position, after the parsed operation
*/
function parseFilterOperation(operators,filterString,p) {
var nextBracketPos, operator;
// Skip the starting square bracket
if(filterString.charAt(p++) !== "[") {
throw "Missing [ in filter expression";
}
// Process each operator in turn
do {
operator = {};
// Check for an operator prefix
if(filterString.charAt(p) === "!") {
operator.prefix = filterString.charAt(p++);
}
// Get the operator name
nextBracketPos = filterString.substring(p).search(/[\[\{<\/]/);
if(nextBracketPos === -1) {
throw "Missing [ in filter expression";
}
nextBracketPos += p;
var bracket = filterString.charAt(nextBracketPos);
operator.operator = filterString.substring(p,nextBracketPos);
// Any suffix?
var colon = operator.operator.indexOf(':');
if(colon > -1) {
// The raw suffix for older filters
operator.suffix = operator.operator.substring(colon + 1);
operator.operator = operator.operator.substring(0,colon) || "field";
// The processed suffix for newer filters
operator.suffixes = [];
$tw.utils.each(operator.suffix.split(":"),function(subsuffix) {
operator.suffixes.push([]);
$tw.utils.each(subsuffix.split(","),function(entry) {
entry = $tw.utils.trim(entry);
if(entry) {
operator.suffixes[operator.suffixes.length - 1].push(entry);
}
});
});
}
// Empty operator means: title
else if(operator.operator === "") {
operator.operator = "title";
}
operator.operands = [];
var parseOperand = function(bracketType) {
var operand = {};
switch (bracketType) {
case "{": // Curly brackets
operand.indirect = true;
nextBracketPos = filterString.indexOf("}",p);
break;
case "[": // Square brackets
nextBracketPos = filterString.indexOf("]",p);
break;
case "<": // Angle brackets
operand.variable = true;
nextBracketPos = filterString.indexOf(">",p);
break;
case "/": // regexp brackets
var rex = /^((?:[^\\\/]|\\.)*)\/(?:\(([mygi]+)\))?/g,
rexMatch = rex.exec(filterString.substring(p));
if(rexMatch) {
operator.regexp = new RegExp(rexMatch[1], rexMatch[2]);
// DEPRECATION WARNING
console.log("WARNING: Filter",operator.operator,"has a deprecated regexp operand",operator.regexp);
nextBracketPos = p + rex.lastIndex - 1;
}
else {
throw "Unterminated regular expression in filter expression";
}
break;
}
if(nextBracketPos === -1) {
throw "Missing closing bracket in filter expression";
}
if(operator.regexp) {
operand.text = "";
} else {
operand.text = filterString.substring(p,nextBracketPos);
}
operator.operands.push(operand);
p = nextBracketPos + 1;
}
p = nextBracketPos + 1;
parseOperand(bracket);
// Check for multiple operands
while(filterString.charAt(p) === ",") {
p++;
if(/^[\[\{<\/]/.test(filterString.substring(p))) {
nextBracketPos = p;
p++;
parseOperand(filterString.charAt(nextBracketPos));
} else {
throw "Missing [ in filter expression";
}
}
// Push this operator
operators.push(operator);
} while(filterString.charAt(p) !== "]");
// Skip the ending square bracket
if(filterString.charAt(p++) !== "]") {
throw "Missing ] in filter expression";
}
// Return the parsing position
return p;
}
/*
Parse a filter string
*/
exports.parseFilter = function(filterString) {
filterString = filterString || "";
var results = [], // Array of arrays of operator nodes {operator:,operand:}
p = 0, // Current position in the filter string
match;
var whitespaceRegExp = /(\s+)/mg,
operandRegExp = /((?:\+|\-|~|=|\:(\w+)(?:\:([\w\:, ]*))?)?)(?:(\[)|(?:"([^"]*)")|(?:'([^']*)')|([^\s\[\]]+))/mg;
while(p < filterString.length) {
// Skip any whitespace
whitespaceRegExp.lastIndex = p;
match = whitespaceRegExp.exec(filterString);
if(match && match.index === p) {
p = p + match[0].length;
}
// Match the start of the operation
if(p < filterString.length) {
operandRegExp.lastIndex = p;
match = operandRegExp.exec(filterString);
if(!match || match.index !== p) {
throw $tw.language.getString("Error/FilterSyntax");
}
var operation = {
prefix: "",
operators: []
};
if(match[1]) {
operation.prefix = match[1];
p = p + operation.prefix.length;
if(match[2]) {
operation.namedPrefix = match[2];
}
if(match[3]) {
operation.suffixes = [];
$tw.utils.each(match[3].split(":"),function(subsuffix) {
operation.suffixes.push([]);
$tw.utils.each(subsuffix.split(","),function(entry) {
entry = $tw.utils.trim(entry);
if(entry) {
operation.suffixes[operation.suffixes.length -1].push(entry);
}
});
});
}
}
if(match[4]) { // Opening square bracket
p = parseFilterOperation(operation.operators,filterString,p);
} else {
p = match.index + match[0].length;
}
if(match[5] || match[6] || match[7]) { // Double quoted string, single quoted string or unquoted title
operation.operators.push(
{operator: "title", operands: [{text: match[5] || match[6] || match[7]}]}
);
}
results.push(operation);
}
}
return results;
};
exports.getFilterOperators = function() {
if(!this.filterOperators) {
$tw.Wiki.prototype.filterOperators = {};
$tw.modules.applyMethods("filteroperator",this.filterOperators);
}
return this.filterOperators;
};
exports.getFilterRunPrefixes = function() {
if(!this.filterRunPrefixes) {
$tw.Wiki.prototype.filterRunPrefixes = {};
$tw.modules.applyMethods("filterrunprefix",this.filterRunPrefixes);
}
return this.filterRunPrefixes;
}
exports.filterTiddlers = function(filterString,widget,source) {
var fn = this.compileFilter(filterString);
return fn.call(this,source,widget);
};
/*
Compile a filter into a function with the signature fn(source,widget) where:
source: an iterator function for the source tiddlers, called source(iterator), where iterator is called as iterator(tiddler,title)
widget: an optional widget node for retrieving the current tiddler etc.
*/
exports.compileFilter = function(filterString) {
var self = this;
// Set up the filter function cache
if(!this.filterCache) {
this.filterCache = Object.create(null);
this.filterCacheCount = 0;
}
// Use the cached version of this filter function if it exists
if(this.filterCache[filterString] !== undefined) {
return this.filterCache[filterString];
}
// Parse the filter string
var filterParseTree;
try {
filterParseTree = this.parseFilter(filterString);
} catch(e) {
// We do not cache this result, so it adjusts along with localization changes
return function(source,widget) {
return [$tw.language.getString("Error/Filter") + ": " + e];
};
}
// Get the filter function
var fnFilter = this.optimiseFilter && this.optimiseFilter(filterString,filterParseTree);
if(!fnFilter) {
fnFilter = this.compileFilterToJavaScript(filterParseTree);
}
// Add recursion detection
var fnGuardedFilter = function guardedFilterFunction(source,widget) {
var results;
self.filterRecursionCount = (self.filterRecursionCount || 0) + 1;
if(self.filterRecursionCount < MAX_FILTER_DEPTH) {
results = fnFilter(source,widget);
} else {
results = ["/**-- Excessive filter recursion --**/"];
}
self.filterRecursionCount = self.filterRecursionCount - 1;
return results;
}
// Add performance measurement
var fnMeasured = $tw.perf.measure("filter: " + filterString,fnGuardedFilter);
// Cache the final filter function
if(this.filterCacheCount >= 2000) {
// To prevent memory leak, we maintain an upper limit for cache size.
// Reset if exceeded. This should give us 95% of the benefit
// that no cache limit would give us.
this.filterCache = Object.create(null);
this.filterCacheCount = 0;
}
this.filterCache[filterString] = fnMeasured;
this.filterCacheCount++;
return fnMeasured;
};
exports.compileFilterToJavaScript = function(filterParseTree) {
var operationFunctions = [];
// Get the hashmap of filter operator functions
var filterOperators = this.getFilterOperators();
// Step through the operations
var self = this;
$tw.utils.each(filterParseTree,function(operation) {
// Create a function for the chain of operators in the operation
var operationSubFunction = function(source,widget) {
var accumulator = source,
results = [],
currTiddlerTitle = widget && widget.getVariable("currentTiddler");
$tw.utils.each(operation.operators,function(operator) {
var operands = [],
operatorFunction;
if(!operator.operator) {
// Use the "title" operator if no operator is specified
operatorFunction = filterOperators.title;
} else if(!filterOperators[operator.operator]) {
// Unknown operators treated as "[unknown]" - at run time we can distinguish between a custom operator and falling back to the default "field" operator
operatorFunction = filterOperators["[unknown]"];
} else {
// Use the operator function
operatorFunction = filterOperators[operator.operator];
}
$tw.utils.each(operator.operands,function(operand) {
if(operand.indirect) {
operand.value = self.getTextReference(operand.text,"",currTiddlerTitle);
} else if(operand.variable) {
var varTree = $tw.utils.parseFilterVariable(operand.text);
operand.value = widgetClass.evaluateVariable(widget,varTree.name,{params: varTree.params, source: source})[0] || "";
} else {
operand.value = operand.text;
}
operands.push(operand.value);
});
// Invoke the appropriate filteroperator module
results = operatorFunction(accumulator,{
operator: operator.operator,
operand: operands.length > 0 ? operands[0] : undefined,
operands: operands,
prefix: operator.prefix,
suffix: operator.suffix,
suffixes: operator.suffixes,
regexp: operator.regexp
},{
wiki: self,
widget: widget
});
if($tw.utils.isArray(results)) {
accumulator = self.makeTiddlerIterator(results);
} else {
accumulator = results;
}
});
if($tw.utils.isArray(results)) {
return results;
} else {
var resultArray = [];
results(function(tiddler,title) {
resultArray.push(title);
});
return resultArray;
}
};
var filterRunPrefixes = self.getFilterRunPrefixes();
// Wrap the operator functions in a wrapper function that depends on the prefix
operationFunctions.push((function() {
var options = {wiki: self, suffixes: operation.suffixes || []};
switch(operation.prefix || "") {
case "": // No prefix means that the operation is unioned into the result
return filterRunPrefixes["or"](operationSubFunction, options);
case "=": // The results of the operation are pushed into the result without deduplication
return filterRunPrefixes["all"](operationSubFunction, options);
case "-": // The results of this operation are removed from the main result
return filterRunPrefixes["except"](operationSubFunction, options);
case "+": // This operation is applied to the main results so far
return filterRunPrefixes["and"](operationSubFunction, options);
case "~": // This operation is unioned into the result only if the main result so far is empty
return filterRunPrefixes["else"](operationSubFunction, options);
default:
if(operation.namedPrefix && filterRunPrefixes[operation.namedPrefix]) {
return filterRunPrefixes[operation.namedPrefix](operationSubFunction, options);
} else {
return function(results,source,widget) {
results.clear();
results.push($tw.language.getString("Error/FilterRunPrefix"));
};
}
}
})());
});
// Make the filter function
return function filterFunction(source,widget) {
if(!source) {
source = self.each;
} else if(typeof source === "object") { // Array or hashmap
source = self.makeTiddlerIterator(source);
}
if(!widget) {
widget = $tw.rootWidget;
}
var results = new $tw.utils.LinkedList();
$tw.utils.each(operationFunctions,function(operationFunction) {
operationFunction(results,source,widget);
});
return results.toArray();
};
};
})();