Fix tokenizing of various keywords within ternary expression

Supports proper tokenizing of these keywords when they appear as
subexpression `b` within ternary expression `a ? b : c`:

  yield, true, false, null, [CONSTANT], super, this, debugger,
  module, exports, __filename, __dirname, global, process

Author: mislav

grammars/javascript.cson

@@ -468,12 +468,16 @@
     ]
   }
   {
-    'match': '(?<!\\.)\\b(yield)(?!\\s*:)\\b(?:\\s*(\\*))?',
+    'match': '(?<!\\.)\\b(yield)(?!\\s*:)\\b(?:\\s*(\\*))?|(?<=\\?)(?:\\s*)(yield)(?=\\s*:)',
     'captures':
       '1':
         'name': 'keyword.control.js'
       '2':
         'name': 'storage.modifier.js'
+      '3':
+        'name': 'keyword.control.js'
+      '4':
+        'name': 'storage.modifier.js'
     'name': 'meta.control.yield.js'
   }
   {
@@ -489,28 +493,44 @@
     'name': 'keyword.operator.js'
   }
   {
-    'match': '(?<!\\.)\\btrue(?!\\s*:)\\b'
-    'name': 'constant.language.boolean.true.js'
-  }
-  {
-    'match': '(?<!\\.)\\bfalse(?!\\s*:)\\b'
-    'name': 'constant.language.boolean.false.js'
+    'match': '(?<!\\.)\\b(true|false)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(true|false)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'constant.language.boolean.$1.js'
+      '2':
+        'name': 'constant.language.boolean.$2.js'
   }
   {
-    'match': '(?<!\\.)\\bnull(?!\\s*:)\\b'
-    'name': 'constant.language.null.js'
+    'match': '(?<!\\.)\\b(null)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(null)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'constant.language.null.js'
+      '2':
+        'name': 'constant.language.null.js'
   }
   {
-    'match': '(?<!\\.)\\b([A-Z][A-Z0-9_]+)(?!\\s*:)\\b'
-    'name': 'constant.other.js'
+    'match': '(?<!\\.)\\b([A-Z][A-Z0-9_]+)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)([A-Z][A-Z0-9_]+)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'constant.other.js'
+      '2':
+        'name': 'constant.other.js'
   }
   {
-    'match': '(?<!\\.)\\b(super|this)(?!\\s*:)\\b'
-    'name': 'variable.language.js'
+    'match': '(?<!\\.)\\b(super|this)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(super|this)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'variable.language.js'
+      '2':
+        'name': 'variable.language.js'
   }
   {
-    'match': '(?<!\\.)\\b(debugger)(?!\\s*:)\\b'
-    'name': 'keyword.other.js'
+    'match': '(?<!\\.)\\b(debugger)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(debugger)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'keyword.other.js'
+      '2':
+        'name': 'keyword.other.js'
   }
   {
     'match': '(?<!\\$)\\b(Anchor|Applet|Area|Array|Boolean|Button|Checkbox|Date|document|event|FileUpload|Form|Frame|Function|Hidden|History|Image|JavaArray|JavaClass|JavaObject|JavaPackage|java|Layer|Link|Location|Math|MimeType|Number|navigator|netscape|Object|Option|Packages|Password|Plugin|Radio|RegExp|Reset|Select|String|Style|Submit|screen|sun|Text|Textarea|window|XMLHttpRequest)\\b'
@@ -545,8 +565,13 @@
     'name': 'support.function.js'
   }
   {
-    'match': '(?<!\\.)\\b(module|exports|__filename|__dirname|global|process)(?!\\s*:)\\b'
-    'name': 'support.variable.js'
+    'match': '(?<!\\.)\\b()(?!\\s*:)\\b'
+    'match': '(?<!\\.)\\b(module|exports|__filename|__dirname|global|process)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(module|exports|__filename|__dirname|global|process)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'support.variable.js'
+      '2':
+        'name': 'support.variable.js'
   }
   {
     'match': '\\b(Infinity|NaN|undefined)\\b'

spec/javascript-spec.coffee

@@ -53,6 +53,28 @@ describe "Javascript grammar", ->
       {tokens} = grammar.tokenizeLine('with')
       expect(tokens[0]).toEqual value: 'with', scopes: ['source.js', 'keyword.control.js']
 
+    map =
+      super: 'variable.language.js'
+      this: 'variable.language.js'
+      null: 'constant.language.null.js'
+      true: 'constant.language.boolean.true.js'
+      false: 'constant.language.boolean.false.js'
+      debugger: 'keyword.other.js'
+      exports: 'support.variable.js'
+      __filename: 'support.variable.js'
+
+    for keyword, scope of map
+      do (keyword, scope) ->
+        it "does not tokenize `#{keyword}` when object key", ->
+          {tokens} = grammar.tokenizeLine("#{keyword}: 1")
+          expect(tokens[0]).toEqual value: keyword, scopes: ['source.js']
+          expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
+
+        it "tokenizes `#{keyword}` in ternary expression", ->
+          {tokens} = grammar.tokenizeLine("a ? #{keyword} : b")
+          expect(tokens[2]).toEqual value: ' ', scopes: ['source.js']
+          expect(tokens[3]).toEqual value: keyword, scopes: ['source.js', scope]
+
   describe "built-in globals", ->
     it "tokenizes them as support classes", ->
       {tokens} = grammar.tokenizeLine('window')
@@ -336,6 +358,15 @@ describe "Javascript grammar", ->
       expect(tokens[4]).toEqual value: 'systemLanguage', scopes: ['source.js', 'support.constant.js']
       expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']
 
+    it "does not tokenize constant when object key", ->
+      {tokens} = grammar.tokenizeLine('FOO: 1')
+      expect(tokens[0]).toEqual value: 'FOO', scopes: ['source.js']
+      expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
+
+    it "tokenizes constant in ternary expression", ->
+      {tokens} = grammar.tokenizeLine('a ? FOO : b')
+      expect(tokens[3]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js']
+
   describe "ES6 string templates", ->
     it "tokenizes them as strings", ->
       {tokens} = grammar.tokenizeLine('`hey ${name}`')
@@ -399,6 +430,15 @@ describe "Javascript grammar", ->
       expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js']
       expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.control.yield.js', 'storage.modifier.js']
 
+    it "does not tokenize yield when object key", ->
+      {tokens} = grammar.tokenizeLine('yield: 1')
+      expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js']
+      expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
+
+    it "tokenizes yield in ternary expression", ->
+      {tokens} = grammar.tokenizeLine('a ? yield : b')
+      expect(tokens[3]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js']
+
   describe "default: in a switch statement", ->
     it "tokenizes it as a keyword", ->
       {tokens} = grammar.tokenizeLine('default: ')