Make operators scoping more granular (to match Atom language-python)

MagicStack · Oct 24, 2015 · 9f7e839 · 9f7e839
1 parent 13bf534
commit 9f7e839
Show file tree

Hide file tree

Showing 29 changed files with 214 additions and 96 deletions.
diff --git a/grammars/MagicPython.cson b/grammars/MagicPython.cson
@@ -445,20 +445,34 @@ repository:
       
     '''
   operator:
-    name: "keyword.operator.python"
     match: '''
       (?x)
-          \\b(?<!\\.)(
-              and | or | not | in | is | for | if | else | await
-              | (yield(\\s+from)?)
-          )(?!\\s*:)\\b
+          \\b(?<!\\.)
+            (?:
+              (and | or | not | in | is)                         (?# 1)
+              |
+              (for | if | else | await | (?:yield(?:\\s+from)?))  (?# 2)
+            )
+          (?!\\s*:)\\b
+      
+          | (<< | >> | & | \\| | \\^ | ~)                          (?# 3)
+      
+          | (\\*\\* | \\* | \\+ | - | % | // | / | @)                (?# 4)
       
-          | != | == | >= | <= | << | >>
-          | && | \\|\\| | \\*\\* | //
-          | > | < | \\+ | - | \\* | % | \\| | & | \\^
-          | ~ | / | @
+          | (!= | == | >= | <= | < | >)                          (?# 5)
       
     '''
+    captures:
+      "1":
+        name: "keyword.operator.logical.python"
+      "2":
+        name: "keyword.control.flow.python"
+      "3":
+        name: "keyword.operator.bitwise.python"
+      "4":
+        name: "keyword.operator.arithmetic.python"
+      "5":
+        name: "keyword.operator.comparison.python"
   literal:
     patterns: [
       {
@@ -1394,16 +1408,36 @@ repository:
       }
     ]
   "function-arguments":
-    begin: "(\\()"
+    begin: '''
+      (?x)
+        (?:
+          (\\()
+          (?:\\s*(\\*\\*|\\*))?
+        )
+      
+    '''
     end: "(?=\\))(?!\\)\\s*\\()"
     beginCaptures:
       "1":
         name: "punctuation.definition.arguments.begin.python"
+      "2":
+        name: "keyword.operator.unpacking.arguments.python"
     contentName: "meta.function-call.arguments.python"
     patterns: [
       {
-        name: "punctuation.separator.arguments.python"
-        match: ","
+        match: '''
+          (?x)
+            (?:
+              (,)
+              (?:\\s*(\\*\\*|\\*))?
+            )
+          
+        '''
+        captures:
+          "1":
+            name: "punctuation.separator.arguments.python"
+          "2":
+            name: "keyword.operator.unpacking.arguments.python"
       }
       {
         include: "#lambda-incomplete"

diff --git a/grammars/MagicPython.syntax.yaml b/grammars/MagicPython.syntax.yaml
@@ -327,18 +327,28 @@ repository:
           | =
 
   operator:
-    name: keyword.operator.python
     match: |
       (?x)
-          \b(?<!\.)(
-              and | or | not | in | is | for | if | else | await
-              | (yield(\s+from)?)
-          )(?!\s*:)\b
+          \b(?<!\.)
+            (?:
+              (and | or | not | in | is)                         (?# 1)
+              |
+              (for | if | else | await | (?:yield(?:\s+from)?))  (?# 2)
+            )
+          (?!\s*:)\b
+
+          | (<< | >> | & | \| | \^ | ~)                          (?# 3)
 
-          | != | == | >= | <= | << | >>
-          | && | \|\| | \*\* | //
-          | > | < | \+ | - | \* | % | \| | & | \^
-          | ~ | / | @
+          | (\*\* | \* | \+ | - | % | // | / | @)                (?# 4)
+
+          | (!= | == | >= | <= | < | >)                          (?# 5)
+
+    captures:
+      '1': {name: keyword.operator.logical.python}
+      '2': {name: keyword.control.flow.python}
+      '3': {name: keyword.operator.bitwise.python}
+      '4': {name: keyword.operator.arithmetic.python}
+      '5': {name: keyword.operator.comparison.python}
 
   literal:
     patterns:
@@ -936,14 +946,27 @@ repository:
             \b ([[:alpha:]_]\w*) \b
 
   function-arguments:
-    begin: (\()
+    begin: |
+      (?x)
+        (?:
+          (\()
+          (?:\s*(\*\*|\*))?
+        )
     end: (?=\))(?!\)\s*\()
     beginCaptures:
       '1': {name: punctuation.definition.arguments.begin.python}
+      '2': {name: keyword.operator.unpacking.arguments.python}
     contentName: meta.function-call.arguments.python
     patterns:
-      - name: punctuation.separator.arguments.python
-        match: ','
+      - match: |
+          (?x)
+            (?:
+              (,)
+              (?:\s*(\*\*|\*))?
+            )
+        captures:
+          '1': {name: punctuation.separator.arguments.python}
+          '2': {name: keyword.operator.unpacking.arguments.python}
       - include: '#lambda-incomplete'
       - include: '#illegal-names'
       - match: '\b([[:alpha:]_]\w*)\s*(=)'

diff --git a/grammars/MagicPython.tmLanguage b/grammars/MagicPython.tmLanguage
@@ -699,20 +699,50 @@
       </dict>
       <key>operator</key>
       <dict>
-        <key>name</key>
-        <string>keyword.operator.python</string>
         <key>match</key>
         <string>(?x)
-    \b(?&lt;!\.)(
-        and | or | not | in | is | for | if | else | await
-        | (yield(\s+from)?)
-    )(?!\s*:)\b
+    \b(?&lt;!\.)
+      (?:
+        (and | or | not | in | is)                         (?# 1)
+        |
+        (for | if | else | await | (?:yield(?:\s+from)?))  (?# 2)
+      )
+    (?!\s*:)\b
+
+    | (&lt;&lt; | &gt;&gt; | &amp; | \| | \^ | ~)                          (?# 3)
 
-    | != | == | &gt;= | &lt;= | &lt;&lt; | &gt;&gt;
-    | &amp;&amp; | \|\| | \*\* | //
-    | &gt; | &lt; | \+ | - | \* | % | \| | &amp; | \^
-    | ~ | / | @
+    | (\*\* | \* | \+ | - | % | // | / | @)                (?# 4)
+
+    | (!= | == | &gt;= | &lt;= | &lt; | &gt;)                          (?# 5)
 </string>
+        <key>captures</key>
+        <dict>
+          <key>1</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.operator.logical.python</string>
+          </dict>
+          <key>2</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.control.flow.python</string>
+          </dict>
+          <key>3</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.operator.bitwise.python</string>
+          </dict>
+          <key>4</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.operator.arithmetic.python</string>
+          </dict>
+          <key>5</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.operator.comparison.python</string>
+          </dict>
+        </dict>
       </dict>
       <key>literal</key>
       <dict>
@@ -2235,7 +2265,12 @@ it&apos;s not tokenized as ellipsis.
       <key>function-arguments</key>
       <dict>
         <key>begin</key>
-        <string>(\()</string>
+        <string>(?x)
+  (?:
+    (\()
+    (?:\s*(\*\*|\*))?
+  )
+</string>
         <key>end</key>
         <string>(?=\))(?!\)\s*\()</string>
         <key>beginCaptures</key>
@@ -2245,16 +2280,37 @@ it&apos;s not tokenized as ellipsis.
             <key>name</key>
             <string>punctuation.definition.arguments.begin.python</string>
           </dict>
+          <key>2</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.operator.unpacking.arguments.python</string>
+          </dict>
         </dict>
         <key>contentName</key>
         <string>meta.function-call.arguments.python</string>
         <key>patterns</key>
         <array>
           <dict>
-            <key>name</key>
-            <string>punctuation.separator.arguments.python</string>
             <key>match</key>
-            <string>,</string>
+            <string>(?x)
+  (?:
+    (,)
+    (?:\s*(\*\*|\*))?
+  )
+</string>
+            <key>captures</key>
+            <dict>
+              <key>1</key>
+              <dict>
+                <key>name</key>
+                <string>punctuation.separator.arguments.python</string>
+              </dict>
+              <key>2</key>
+              <dict>
+                <key>name</key>
+                <string>keyword.operator.unpacking.arguments.python</string>
+              </dict>
+            </dict>
           </dict>
           <dict>
             <key>include</key>

diff --git a/misc/scopes b/misc/scopes
@@ -40,17 +40,22 @@ invalid.illegal.prefix.python
 keyword.codetag.notation.python
 keyword.control.flow.python
 keyword.invalid.illegal.name.python
+keyword.operator.arithmetic.python
 keyword.operator.assignment.python
+keyword.operator.bitwise.python
+keyword.operator.comparison.python
 keyword.operator.conditional.negative.regexp
 keyword.operator.conditional.regexp
 keyword.operator.disjunction.regexp
+keyword.operator.logical.python
 keyword.operator.lookahead.negative.regexp
 keyword.operator.lookahead.regexp
 keyword.operator.lookbehind.negative.regexp
 keyword.operator.lookbehind.regexp
 keyword.operator.negation.regexp
 keyword.operator.python
 keyword.operator.quantifier.regexp
+keyword.operator.unpacking.arguments.python
 meta.backreference.named.regexp
 meta.backreference.regexp
 meta.character.set.regexp

diff --git a/test/calls/call1.py b/test/calls/call1.py
@@ -15,7 +15,7 @@
 ]             : meta.function-call.arguments.python, meta.function-call.python, meta.item-access.python, punctuation.definition.arguments.end.python, source.python
 ,             : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python
               : meta.function-call.arguments.python, meta.function-call.python, source.python
-*             : keyword.operator.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+*             : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function-call.python, source.python
 args          : meta.function-call.arguments.python, meta.function-call.python, source.python
 ,             : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python
               : meta.function-call.arguments.python, meta.function-call.python, source.python
@@ -32,6 +32,6 @@
 }             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.dict.end.python, source.python
 ,             : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python
               : meta.function-call.arguments.python, meta.function-call.python, source.python
-**            : keyword.operator.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+**            : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function-call.python, source.python
 kwargs        : meta.function-call.arguments.python, meta.function-call.python, source.python
 )             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
diff --git a/test/calls/call4.py b/test/calls/call4.py
@@ -23,7 +23,7 @@
 .             : source.python
 None          : keyword.invalid.illegal.name.python, source.python
               : source.python
-and           : keyword.operator.python, source.python
+and           : keyword.operator.logical.python, source.python
  foo.         : source.python
 None          : constant.language.python, source.python
 .             : source.python

diff --git a/test/calls/print1.py b/test/calls/print1.py
@@ -18,7 +18,7 @@
 "             : punctuation.definition.string.end.python, source.python, string.quoted.single.python
 ,             : source.python
 2             : constant.numeric.dec.python, source.python
-*             : keyword.operator.python, source.python
+*             : keyword.operator.arithmetic.python, source.python
 2             : constant.numeric.dec.python, source.python
 print         : meta.function-call.python, source.python, support.function.builtin.python
 (             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python
@@ -28,7 +28,7 @@
 ,             : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python
               : meta.function-call.arguments.python, meta.function-call.python, source.python
 2             : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python
-*             : keyword.operator.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+*             : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, source.python
 2             : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python
 )             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
 print         : source.python, support.function.builtin.python
@@ -50,7 +50,7 @@
 )             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
 print         : source.python, support.function.builtin.python
               : source.python
->>            : keyword.operator.python, source.python
+>>            : keyword.operator.bitwise.python, source.python
 sys.stderr,   : source.python
 "             : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
 er            : source.python, string.quoted.single.python

diff --git a/test/classes/class12.py b/test/classes/class12.py
@@ -61,7 +61,7 @@ def           : meta.function.python, source.python, storage.type.function.pytho
               : source.python
 cls           : source.python, variable.language.special.cls.python
               : source.python
-+             : keyword.operator.python, source.python
++             : keyword.operator.arithmetic.python, source.python
               : source.python
 1             : constant.numeric.dec.python, source.python
         a     : source.python

diff --git a/test/docstrings/oneline1.py b/test/docstrings/oneline1.py
@@ -20,19 +20,19 @@
 >>>           : keyword.control.flow.python, source.python, string.quoted.docstring.multi.python
 print('''docstring''') : source.python, string.quoted.docstring.multi.python
 """           : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python
-await         : keyword.operator.python, source.python
+await         : keyword.control.flow.python, source.python
 """           : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python
 \n            : constant.character.python, source.python, string.quoted.docstring.multi.python
 >>> print('''docstring''') : source.python, string.quoted.docstring.multi.python
 """           : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python
-await         : keyword.operator.python, source.python
+await         : keyword.control.flow.python, source.python
 """           : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python
               : source.python, string.quoted.docstring.multi.python
 >>>           : keyword.control.flow.python, source.python, string.quoted.docstring.multi.python
 print('''docstring''') : source.python, string.quoted.docstring.multi.python
 """           : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python
-await         : keyword.operator.python, source.python
+await         : keyword.control.flow.python, source.python
 """           : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python
  1  >>> print('''docstring''') : source.python, string.quoted.docstring.multi.python
 """           : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python
-await         : keyword.operator.python, source.python
+await         : keyword.control.flow.python, source.python