Permalink
Browse files

fix tokenizer on ie8

changes $arrayTokens to use exec instead of split
 1 it works on ie8
 2 it is faster on chrome http://jsperf.com/capturing-split-vs-exec
unfortunately exec is slow on firefox but ff regexes are a bit rusty atm (v21), and whole tokenizer is slow on it anyway.
  • Loading branch information...
1 parent 118c80c commit 77b5ae10621fdf07d2cfe7334d2d0af352baf362 @nightwing nightwing committed Feb 12, 2013
@@ -310,7 +310,7 @@ var actions = exports.actions = {
fn: function(editor, range, count, param) {
editor.modifyNumber(count || 1);
}
- },
+ }
};
var inputBuffer = exports.inputBuffer = {
@@ -99,7 +99,7 @@ define(function(require, exports, module) {
token : "string", regex : "'''", next : [
{token : "string", regex : "'''", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
- {defaultToken: "string"},
+ {defaultToken: "string"}
]
}, {
stateName: "qqdoc",
@@ -115,21 +115,21 @@ define(function(require, exports, module) {
token : "string", regex : "'", next : [
{token : "string", regex : "'", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
- {defaultToken: "string"},
+ {defaultToken: "string"}
]
}, {
stateName: "qqstring",
token : "string.start", regex : '"', next : [
{token : "string.end", regex : '"', next : "start"},
{token : "constant.language.escape", regex : stringEscape},
- {defaultToken: "string"},
+ {defaultToken: "string"}
]
}, {
stateName: "js",
token : "string", regex : "`", next : [
{token : "string", regex : "`", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
- {defaultToken: "string"},
+ {defaultToken: "string"}
]
}, {
token : "string.regex",
@@ -46,19 +46,19 @@ var SnippetHighlightRules = function() {
{regex: /\[/, token: "regex.start", next: "charClass"},
{regex: "/", token: "string.regex", next: "format"},
//{"default": "string.regex"},
- {"token": "string.regex", regex:"."},
+ {"token": "string.regex", regex:"."}
],
charClass : [
{regex: "\\.", token: "escape"},
{regex: "\\]", token: "regex.end", next: "regexp"},
- {"token": "string.regex", regex:"."},
+ {"token": "string.regex", regex:"."}
],
"format" : [
{regex: /\\[ulULE]/, token: "keyword"},
{regex: /\$\d+/, token: "variable"},
{regex: "/[gim]*:?", token: "string.regex", next: "start"},
// {"default": "string"},
- {"token": "string", regex:"."},
+ {"token": "string", regex:"."}
]
};
};
View
@@ -451,7 +451,7 @@ module.exports = {
assert.position(ranges[1].end, 0, 11);
assert.position(ranges[0].start, 0, 0);
assert.position(ranges[0].end, 0, 3);
- },
+ }
};
});
View
@@ -111,12 +111,12 @@ var Tokenizer = function(rules) {
this.$arrayTokens = function(str) {
if (!str)
return [];
- var values = str.split(this.splitRegex)
+ var values = this.splitRegex.exec(str);
var tokens = [];
var types = this.tokenArray;
- if (types.length != values.length - 2) {
+ if (types.length != values.length - 1) {
if (window.console)
- console.error(types.length , values.length - 2, str, this.splitRegex);
+ console.error(types , values, str, this.splitRegex, this);
return [{type: "error.invalid", value: str}];
}
for (var i = 0; i < types.length; i++) {
@@ -140,7 +140,7 @@ var Tokenizer = function(rules) {
this.createSplitterRegexp = function(src, flag) {
src = src.replace(/\(\?=([^()]|\\.|\(([^()]|\\.)*?\))*\)(?=\)*$)/, "");
- return new RegExp(src, flag);
+ return new RegExp(src, (flag||"").replace("g", ""));
};
/**
View
@@ -0,0 +1,38 @@
+var plist = require("plist");
+var util = require("util");
+exports.parsePlist = function(themeXml, callback) {
+ var result = ""
+ plist.parseString(themeXml, function(_, theme) {
+ result = theme[0];
+ callback && callback(theme[0]);
+ });
+ return result;
+}
+
+exports.formatJSON = function(object, initialIndent) {
+ return util.inspect(object, false, 40).replace(/^/gm, initialIndent||"")
+
+}
+
+
+exports.fillTemplate = function(template, replacements) {
+ return template.replace(/%(.+?)%/g, function(str, m) {
+ return replacements[m] || "";
+ });
+}
+
+exports.hyphenate = function(str) {
+ return str.replace(/([A-Z])/g, "-$1").replace(/_/g, "-").toLowerCase();
+}
+
+exports.quoteString = function(str) {
+ return '"' + str.replace(/\\/, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\\n") + '"';
+}
+
+
+exports.restoreJSONComments = function(objStr) {
+ return objStr.replace(/^(\s*)comment: '(.*)'/gm, function(_, i, c) {
+ return i + "//" + c.replace(/\\n(\\t)*/g, "\n" + i + "//") + "\n" + i
+ }).replace(/ \/\/ ERROR/g, '", // ERROR');
+}
+

0 comments on commit 77b5ae1

Please sign in to comment.