Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
fix tokenizer on ie8
changes $arrayTokens to use exec instead of split
 1 it works on ie8
 2 it is faster on chrome http://jsperf.com/capturing-split-vs-exec
unfortunately exec is slow on firefox but ff regexes are a bit rusty atm (v21), and whole tokenizer is slow on it anyway.
  • Loading branch information
nightwing committed Feb 16, 2013
1 parent 118c80c commit 77b5ae1
Show file tree
Hide file tree
Showing 6 changed files with 51 additions and 13 deletions.
2 changes: 1 addition & 1 deletion lib/ace/keyboard/vim/commands.js
Expand Up @@ -310,7 +310,7 @@ var actions = exports.actions = {
fn: function(editor, range, count, param) {
editor.modifyNumber(count || 1);
}
},
}
};

var inputBuffer = exports.inputBuffer = {
Expand Down
8 changes: 4 additions & 4 deletions lib/ace/mode/coffee_highlight_rules.js
Expand Up @@ -99,7 +99,7 @@ define(function(require, exports, module) {
token : "string", regex : "'''", next : [
{token : "string", regex : "'''", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"},
{defaultToken: "string"}
]
}, {
stateName: "qqdoc",
Expand All @@ -115,21 +115,21 @@ define(function(require, exports, module) {
token : "string", regex : "'", next : [
{token : "string", regex : "'", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"},
{defaultToken: "string"}
]
}, {
stateName: "qqstring",
token : "string.start", regex : '"', next : [
{token : "string.end", regex : '"', next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"},
{defaultToken: "string"}
]
}, {
stateName: "js",
token : "string", regex : "`", next : [
{token : "string", regex : "`", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"},
{defaultToken: "string"}
]
}, {
token : "string.regex",
Expand Down
6 changes: 3 additions & 3 deletions lib/ace/mode/tmsnippet.js
Expand Up @@ -46,19 +46,19 @@ var SnippetHighlightRules = function() {
{regex: /\[/, token: "regex.start", next: "charClass"},
{regex: "/", token: "string.regex", next: "format"},
//{"default": "string.regex"},
{"token": "string.regex", regex:"."},
{"token": "string.regex", regex:"."}
],
charClass : [
{regex: "\\.", token: "escape"},
{regex: "\\]", token: "regex.end", next: "regexp"},
{"token": "string.regex", regex:"."},
{"token": "string.regex", regex:"."}
],
"format" : [
{regex: /\\[ulULE]/, token: "keyword"},
{regex: /\$\d+/, token: "variable"},
{regex: "/[gim]*:?", token: "string.regex", next: "start"},
// {"default": "string"},
{"token": "string", regex:"."},
{"token": "string", regex:"."}
]
};
};
Expand Down
2 changes: 1 addition & 1 deletion lib/ace/search_test.js
Expand Up @@ -451,7 +451,7 @@ module.exports = {
assert.position(ranges[1].end, 0, 11);
assert.position(ranges[0].start, 0, 0);
assert.position(ranges[0].end, 0, 3);
},
}
};

});
Expand Down
8 changes: 4 additions & 4 deletions lib/ace/tokenizer.js
Expand Up @@ -111,12 +111,12 @@ var Tokenizer = function(rules) {
this.$arrayTokens = function(str) {
if (!str)
return [];
var values = str.split(this.splitRegex)
var values = this.splitRegex.exec(str);
var tokens = [];
var types = this.tokenArray;
if (types.length != values.length - 2) {
if (types.length != values.length - 1) {
if (window.console)
console.error(types.length , values.length - 2, str, this.splitRegex);
console.error(types , values, str, this.splitRegex, this);
return [{type: "error.invalid", value: str}];
}
for (var i = 0; i < types.length; i++) {
Expand All @@ -140,7 +140,7 @@ var Tokenizer = function(rules) {

this.createSplitterRegexp = function(src, flag) {
src = src.replace(/\(\?=([^()]|\\.|\(([^()]|\\.)*?\))*\)(?=\)*$)/, "");
return new RegExp(src, flag);
return new RegExp(src, (flag||"").replace("g", ""));
};

/**
Expand Down
38 changes: 38 additions & 0 deletions tool/lib.js
@@ -0,0 +1,38 @@
var plist = require("plist");
var util = require("util");
exports.parsePlist = function(themeXml, callback) {
var result = ""
plist.parseString(themeXml, function(_, theme) {
result = theme[0];
callback && callback(theme[0]);
});
return result;
}

exports.formatJSON = function(object, initialIndent) {
return util.inspect(object, false, 40).replace(/^/gm, initialIndent||"")

}


exports.fillTemplate = function(template, replacements) {
return template.replace(/%(.+?)%/g, function(str, m) {
return replacements[m] || "";
});
}

exports.hyphenate = function(str) {
return str.replace(/([A-Z])/g, "-$1").replace(/_/g, "-").toLowerCase();
}

exports.quoteString = function(str) {
return '"' + str.replace(/\\/, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\\n") + '"';
}


exports.restoreJSONComments = function(objStr) {
return objStr.replace(/^(\s*)comment: '(.*)'/gm, function(_, i, c) {
return i + "//" + c.replace(/\\n(\\t)*/g, "\n" + i + "//") + "\n" + i
}).replace(/ \/\/ ERROR/g, '", // ERROR');
}

0 comments on commit 77b5ae1

Please sign in to comment.