improve syntax highlighting in error messages

- Support hexadecimal, octal and binary number literals, template strings, and
  tokenize everything more robustly in general.
- Tokens spanning over several lines (such as multiline strings and comments) no
  longer leak their color into the line number gutter.
- The color scheme is preserved.
- The line numbers are now right-aligned instead of left-aligned, since that's
  how practically every editor does it.
- Superfluos space in the line number gutter has been removed.

Author: lydell

lib/babel/helpers/code-frame.js

@@ -1,42 +1,57 @@
-// syntax highlighting based on https://github.com/dominictarr/ansi-highlight by the fantastic Dominic Tarr
-
-var repeating = require("repeating");
-var tokenize  = require("js-tokenizer");
-var chalk     = require("chalk");
+var repeating   = require("repeating");
+var jsTokens    = require("js-tokens");
+var isJSKeyword = require("is-keyword-js");
+var chalk       = require("chalk");
+var lineNumbers = require("line-numbers");
+var ary         = require("lodash/function/ary");
 
 var defs = {
-  string1:  "red",
-  string2:  "red",
-  punct:    ["white", "bold"],
-  curly:    "green",
-  parens:   ["blue", "bold"],
-  square:   ["yellow"],
-  name:     "white",
-  keyword:  ["cyan"],
-  number:   "magenta",
-  regexp:   "magenta",
-  comment1: "grey",
-  comment2: "grey"
+  string:      chalk.red,
+  punctuation: chalk.white.bold,
+  operator:    chalk.white.bold,
+  curly:       chalk.green,
+  parens:      chalk.blue.bold,
+  square:      chalk.yellow,
+  name:        chalk.white,
+  keyword:     chalk.cyan,
+  number:      chalk.magenta,
+  regex:       chalk.magenta,
+  comment:     chalk.grey,
+  invalid:     chalk.inverse
 };
 
-var highlight = function (text) {
-  var colorize = function (str, col) {
-    if (!col) return str;
+var newline = /\r\n|[\n\r\u2028\u2029]/;
 
-    if (Array.isArray(col)) {
-      col.forEach(function (col) {
-        str = chalk[col](str);
-      });
-    } else {
-      str = chalk[col](str);
+var highlight = function (text) {
+  var tokenType = function (match) {
+    var token = jsTokens.matchToToken(match);
+    if (token.type === "name" && isJSKeyword(token.value)) {
+      return "keyword";
     }
-    return str;
+    if (token.type === "punctuation") {
+      switch (token.value) {
+        case "{":
+        case "}":
+          return "curly";
+        case "(":
+        case ")":
+          return "parens";
+        case "[":
+        case "]":
+          return "square";
+      }
+    }
+    return token.type;
   };
 
-  return tokenize(text, true).map(function (str) {
-    var type = tokenize.type(str);
-    return colorize(str, defs[type]);
-  }).join("");
+  return text.replace(jsTokens, function (match) {
+    var type = tokenType(arguments);
+    if (type in defs) {
+      var colorize = ary(defs[type], 1);
+      return match.split(newline).map(colorize).join("\n");
+    }
+    return match;
+  });
 };
 
 module.exports = function (lines, lineNumber, colNumber) {
@@ -46,33 +61,29 @@ module.exports = function (lines, lineNumber, colNumber) {
     lines = highlight(lines);
   }
 
-  lines = lines.split(/\r\n|[\n\r\u2028\u2029]/);
+  lines = lines.split(newline);
 
   var start = Math.max(lineNumber - 3, 0);
   var end   = Math.min(lines.length, lineNumber + 3);
-  var width = (end + "").length;
 
   if (!lineNumber && !colNumber) {
     start = 0;
     end = lines.length;
   }
 
-  return "\n" + lines.slice(start, end).map(function (line, i) {
-    var curr = i + start + 1;
-
-    var gutter = curr === lineNumber ? "> " : "  ";
-
-    var sep = curr + repeating(" ", width + 1);
-    gutter += sep + "| ";
-
-    var str = gutter + line;
-
-    if (colNumber && curr === lineNumber) {
-      str += "\n";
-      str += repeating(" ", gutter.length - 2);
-      str += "|" + repeating(" ", colNumber) + "^";
+  return "\n" + lineNumbers(lines.slice(start, end), {
+    start: start + 1,
+    before: "  ",
+    after: " | ",
+    transform: function (params) {
+      if (params.number !== lineNumber) {
+        return;
+      }
+      if (colNumber) {
+        params.line += "\n" + params.before + repeating(" ", params.width) +
+                       params.after + repeating(" ", colNumber - 1) + "^";
+      }
+      params.before = params.before.replace(/^./, ">");
     }
-
-    return str;
   }).join("\n");
 };

package.json

@@ -49,8 +49,10 @@
     "fs-readdir-recursive": "^0.1.0",
     "globals": "^6.2.0",
     "is-integer": "^1.0.4",
-    "js-tokenizer": "^1.3.3",
+    "is-keyword-js": "^1.0.3",
+    "js-tokens": "~0.4.1",
     "leven": "^1.0.1",
+    "line-numbers": "~0.2.0",
     "lodash": "^3.2.0",
     "output-file-sync": "^1.1.0",
     "path-is-absolute": "^1.0.0",