From 2538ed34689f1d816934065c559eafcc21dd8057 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mislav=20Marohni=C4=87?= <mislav.marohnic@gmail.com>
Date: Mon, 21 Sep 2015 17:43:21 +0200
Subject: [PATCH 1/2] Fix tokenizing of various keywords within ternary
 expression

Supports proper tokenizing of these keywords when they appear as
subexpression `b` within ternary expression `a ? b : c`:

  yield, true, false, null, [CONSTANT], super, this, debugger,
  module, exports, __filename, __dirname, global, process
---
 grammars/javascript.cson    | 58 ++++++++++++++++++++++++++-----------
 spec/javascript-spec.coffee | 40 +++++++++++++++++++++++++
 2 files changed, 81 insertions(+), 17 deletions(-)

diff --git a/grammars/javascript.cson b/grammars/javascript.cson
index acd8ada0..d6fb2ad1 100644
--- a/grammars/javascript.cson
+++ b/grammars/javascript.cson
@@ -468,12 +468,16 @@
     ]
   }
   {
-    'match': '(?<!\\.)\\b(yield)(?!\\s*:)\\b(?:\\s*(\\*))?',
+    'match': '(?<!\\.)\\b(yield)(?!\\s*:)\\b(?:\\s*(\\*))?|(?<=\\?)(?:\\s*)(yield)(?=\\s*:)',
     'captures':
       '1':
         'name': 'keyword.control.js'
       '2':
         'name': 'storage.modifier.js'
+      '3':
+        'name': 'keyword.control.js'
+      '4':
+        'name': 'storage.modifier.js'
     'name': 'meta.control.yield.js'
   }
   {
@@ -489,28 +493,44 @@
     'name': 'keyword.operator.js'
   }
   {
-    'match': '(?<!\\.)\\btrue(?!\\s*:)\\b'
-    'name': 'constant.language.boolean.true.js'
-  }
-  {
-    'match': '(?<!\\.)\\bfalse(?!\\s*:)\\b'
-    'name': 'constant.language.boolean.false.js'
+    'match': '(?<!\\.)\\b(true|false)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(true|false)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'constant.language.boolean.$1.js'
+      '2':
+        'name': 'constant.language.boolean.$2.js'
   }
   {
-    'match': '(?<!\\.)\\bnull(?!\\s*:)\\b'
-    'name': 'constant.language.null.js'
+    'match': '(?<!\\.)\\b(null)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(null)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'constant.language.null.js'
+      '2':
+        'name': 'constant.language.null.js'
   }
   {
-    'match': '(?<!\\.)\\b([A-Z][A-Z0-9_]+)(?!\\s*:)\\b'
-    'name': 'constant.other.js'
+    'match': '(?<!\\.)\\b([A-Z][A-Z0-9_]+)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)([A-Z][A-Z0-9_]+)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'constant.other.js'
+      '2':
+        'name': 'constant.other.js'
   }
   {
-    'match': '(?<!\\.)\\b(super|this)(?!\\s*:)\\b'
-    'name': 'variable.language.js'
+    'match': '(?<!\\.)\\b(super|this)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(super|this)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'variable.language.js'
+      '2':
+        'name': 'variable.language.js'
   }
   {
-    'match': '(?<!\\.)\\b(debugger)(?!\\s*:)\\b'
-    'name': 'keyword.other.js'
+    'match': '(?<!\\.)\\b(debugger)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(debugger)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'keyword.other.js'
+      '2':
+        'name': 'keyword.other.js'
   }
   {
     'match': '(?<!\\$)\\b(Anchor|Applet|Area|Array|Boolean|Button|Checkbox|Date|document|event|FileUpload|Form|Frame|Function|Hidden|History|Image|JavaArray|JavaClass|JavaObject|JavaPackage|java|Layer|Link|Location|Math|MimeType|Number|navigator|netscape|Object|Option|Packages|Password|Plugin|Radio|RegExp|Reset|Select|String|Style|Submit|screen|sun|Text|Textarea|window|XMLHttpRequest)\\b'
@@ -545,8 +565,12 @@
     'name': 'support.function.js'
   }
   {
-    'match': '(?<!\\.)\\b(module|exports|__filename|__dirname|global|process)(?!\\s*:)\\b'
-    'name': 'support.variable.js'
+    'match': '(?<!\\.)\\b(module|exports|__filename|__dirname|global|process)(?!\\s*:)\\b|(?<=\\?)(?:\\s*)(module|exports|__filename|__dirname|global|process)(?=\\s*:)'
+    'captures':
+      '1':
+        'name': 'support.variable.js'
+      '2':
+        'name': 'support.variable.js'
   }
   {
     'match': '\\b(Infinity|NaN|undefined)\\b'
diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee
index ca4c1665..064362f6 100644
--- a/spec/javascript-spec.coffee
+++ b/spec/javascript-spec.coffee
@@ -53,6 +53,28 @@ describe "Javascript grammar", ->
       {tokens} = grammar.tokenizeLine('with')
       expect(tokens[0]).toEqual value: 'with', scopes: ['source.js', 'keyword.control.js']
 
+    map =
+      super: 'variable.language.js'
+      this: 'variable.language.js'
+      null: 'constant.language.null.js'
+      true: 'constant.language.boolean.true.js'
+      false: 'constant.language.boolean.false.js'
+      debugger: 'keyword.other.js'
+      exports: 'support.variable.js'
+      __filename: 'support.variable.js'
+
+    for keyword, scope of map
+      do (keyword, scope) ->
+        it "does not tokenize `#{keyword}` when it is an object key", ->
+          {tokens} = grammar.tokenizeLine("#{keyword}: 1")
+          expect(tokens[0]).toEqual value: keyword, scopes: ['source.js']
+          expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
+
+        it "tokenizes `#{keyword}` in ternary expressions", ->
+          {tokens} = grammar.tokenizeLine("a ? #{keyword} : b")
+          expect(tokens[2]).toEqual value: ' ', scopes: ['source.js']
+          expect(tokens[3]).toEqual value: keyword, scopes: ['source.js', scope]
+
   describe "built-in globals", ->
     it "tokenizes them as support classes", ->
       {tokens} = grammar.tokenizeLine('window')
@@ -336,6 +358,15 @@ describe "Javascript grammar", ->
       expect(tokens[4]).toEqual value: 'systemLanguage', scopes: ['source.js', 'support.constant.js']
       expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']
 
+    it "does not tokenize constants when they are object keys", ->
+      {tokens} = grammar.tokenizeLine('FOO: 1')
+      expect(tokens[0]).toEqual value: 'FOO', scopes: ['source.js']
+      expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
+
+    it "tokenizes constants in ternary expressions", ->
+      {tokens} = grammar.tokenizeLine('a ? FOO : b')
+      expect(tokens[3]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js']
+
   describe "ES6 string templates", ->
     it "tokenizes them as strings", ->
       {tokens} = grammar.tokenizeLine('`hey ${name}`')
@@ -399,6 +430,15 @@ describe "Javascript grammar", ->
       expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js']
       expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.control.yield.js', 'storage.modifier.js']
 
+    it "does not tokenize yield when it is an object key", ->
+      {tokens} = grammar.tokenizeLine('yield: 1')
+      expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js']
+      expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
+
+    it "tokenizes yield in ternary expressions", ->
+      {tokens} = grammar.tokenizeLine('a ? yield : b')
+      expect(tokens[3]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js']
+
   describe "default: in a switch statement", ->
     it "tokenizes it as a keyword", ->
       {tokens} = grammar.tokenizeLine('default: ')

From 246bfea00ffecdf50d6e21e1dba6f739e1a339f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mislav=20Marohni=C4=87?= <mislav.marohnic@gmail.com>
Date: Thu, 24 Sep 2015 19:19:39 +0200
Subject: [PATCH 2/2] Add tests for different positions within ternary
 expressions

---
 spec/javascript-spec.coffee | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee
index 064362f6..eb7bc59f 100644
--- a/spec/javascript-spec.coffee
+++ b/spec/javascript-spec.coffee
@@ -70,11 +70,16 @@ describe "Javascript grammar", ->
           expect(tokens[0]).toEqual value: keyword, scopes: ['source.js']
           expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
 
-        it "tokenizes `#{keyword}` in ternary expressions", ->
+        it "tokenizes `#{keyword}` in the middle of ternary expressions", ->
           {tokens} = grammar.tokenizeLine("a ? #{keyword} : b")
           expect(tokens[2]).toEqual value: ' ', scopes: ['source.js']
           expect(tokens[3]).toEqual value: keyword, scopes: ['source.js', scope]
 
+        it "tokenizes `#{keyword}` at the end of ternary expressions", ->
+          {tokens} = grammar.tokenizeLine("a ? b : #{keyword}")
+          expect(tokens[4]).toEqual value: ' ', scopes: ['source.js']
+          expect(tokens[5]).toEqual value: keyword, scopes: ['source.js', scope]
+
   describe "built-in globals", ->
     it "tokenizes them as support classes", ->
       {tokens} = grammar.tokenizeLine('window')
@@ -363,10 +368,14 @@ describe "Javascript grammar", ->
       expect(tokens[0]).toEqual value: 'FOO', scopes: ['source.js']
       expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
 
-    it "tokenizes constants in ternary expressions", ->
+    it "tokenizes constants in the middle of ternary expressions", ->
       {tokens} = grammar.tokenizeLine('a ? FOO : b')
       expect(tokens[3]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js']
 
+    it "tokenizes constants at the end of ternary expressions", ->
+      {tokens} = grammar.tokenizeLine('a ? b : FOO')
+      expect(tokens[5]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js']
+
   describe "ES6 string templates", ->
     it "tokenizes them as strings", ->
       {tokens} = grammar.tokenizeLine('`hey ${name}`')
@@ -435,10 +444,14 @@ describe "Javascript grammar", ->
       expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js']
       expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js']
 
-    it "tokenizes yield in ternary expressions", ->
+    it "tokenizes yield in the middle of ternary expressions", ->
       {tokens} = grammar.tokenizeLine('a ? yield : b')
       expect(tokens[3]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js']
 
+    it "tokenizes yield at the end of ternary expressions", ->
+      {tokens} = grammar.tokenizeLine('a ? b : yield')
+      expect(tokens[5]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js']
+
   describe "default: in a switch statement", ->
     it "tokenizes it as a keyword", ->
       {tokens} = grammar.tokenizeLine('default: ')