Merge pull request #18 from ananthakumaran/stepped-range

add support for stepped ranges

.tool-versions

@@ -0,0 +1,2 @@
+elixir 1.12.0-rc.1-otp-23
+erlang 23.1

Makefile

@@ -15,7 +15,7 @@ parse-all:
 	./node_modules/.bin/tree-sitter parse '../elixir/**/*.ex*' --quiet --stat
 
 parse:
-	./node_modules/.bin/tree-sitter parse 'test.ex'
+	./node_modules/.bin/tree-sitter parse -x 'test.ex'
 
 debug-graph:
 	./node_modules/.bin/tree-sitter parse 'test.ex' --debug-graph

grammar.js

@@ -20,17 +20,14 @@ function atleastOnce(rule) {
   return seq(rule, repeat(rule));
 }
 
-function binaryOp($, assoc, precedence, operator, bare_keyword) {
-  const right = bare_keyword
-    ? choice($._expression, $.keyword_list)
-    : $._expression;
+function binaryOp($, assoc, precedence, operator, right) {
   return assoc(
     precedence,
     seq(
       field("left", $._expression),
       field("operator", operator),
       optional($._terminator),
-      field("right", right)
+      field("right", right || $._expression)
     )
   );
 }
@@ -107,7 +104,8 @@ const OPERATORS = [
   "|",
   "::",
   "<-",
-  "\\\\"
+  "\\\\",
+  "..//"
 ];
 
 const PREC = {
@@ -263,34 +261,37 @@ module.exports = grammar({
     binary_op: $ =>
       choice(
         binaryOp($, prec.left, 40, choice("\\\\", "<-")),
-        binaryOp($, prec.right, 50, alias($._when, "when"), true),
+        binaryOp(
+          $,
+          prec.right,
+          50,
+          alias($._when, "when"),
+          choice($._expression, $.keyword_list)
+        ),
         binaryOp($, prec.right, 60, "::"),
-        binaryOp($, prec.right, 70, "|", true),
+        binaryOp($, prec.right, 70, "|", choice($._expression, $.keyword_list)),
         binaryOp($, prec.right, 80, "=>"),
         binaryOp($, prec.right, 100, "="),
-        binaryOp($, prec.left, 130, choice("||", "|||", alias($._or, "or"))),
-        binaryOp($, prec.left, 140, choice("&&", "&&&", alias($._and, "and"))),
-        binaryOp($, prec.left, 150, choice("==", "!=", "=~", "===", "!==")),
-        binaryOp($, prec.left, 160, choice("<", ">", "<=", ">=")),
+        binaryOp($, prec.left, 120, choice("||", "|||", alias($._or, "or"))),
+        binaryOp($, prec.left, 130, choice("&&", "&&&", alias($._and, "and"))),
+        binaryOp($, prec.left, 140, choice("==", "!=", "=~", "===", "!==")),
+        binaryOp($, prec.left, 150, choice("<", ">", "<=", ">=")),
         binaryOp(
           $,
           prec.left,
-          170,
+          160,
           choice("|>", "<<<", ">>>", "<<~", "~>>", "<~", "~>", "<~>", "<|>")
         ),
         binaryOp(
           $,
           prec.left,
-          180,
+          170,
           choice(alias($._in, "in"), alias($._not_in, "not in"))
         ),
-        binaryOp($, prec.left, 190, choice("^^^")),
-        binaryOp(
-          $,
-          prec.right,
-          200,
-          choice("++", "--", "..", "<>", "+++", "---")
-        ),
+        binaryOp($, prec.left, 180, choice("^^^")),
+        binaryOp($, prec.right, 200, choice("++", "--", "<>", "+++", "---")),
+        binaryOp($, prec.right, 190, choice("//")),
+        binaryOp($, prec.right, 200, ".."),
         binaryOp($, prec.left, 210, choice("+", "-")),
         binaryOp($, prec.left, 220, choice("*", "/")),
         $._op_capture

src/scanner.cc

@@ -329,9 +329,16 @@ struct Scanner {
       if (lexer->lookahead != '.') return true;
       advance(lexer);
       lexer->mark_end(lexer);
-      if (lexer->lookahead != '.') return true;
-      advance(lexer);
-      lexer->mark_end(lexer);
+      if (lexer->lookahead == '/') {
+        advance(lexer);
+        if (lexer->lookahead == '/') {
+          advance(lexer);
+          lexer->mark_end(lexer);
+        }
+      } else if (lexer->lookahead == '.') {
+        advance(lexer);
+        lexer->mark_end(lexer);
+      }
       return true;
     case ':':
       advance(lexer);
@@ -584,6 +591,7 @@ struct Scanner {
             is_whitespace(lexer->lookahead)) {
           return is_valid(lexer, valid_symbols, KEYWORD_LITERAL);
         }
+        return false;
       }
       if (lexer->lookahead != '.') return false;
       advance(lexer);
@@ -593,7 +601,22 @@ struct Scanner {
             is_whitespace(lexer->lookahead)) {
           return is_valid(lexer, valid_symbols, KEYWORD_LITERAL);
         }
+        return false;
       }
+      if (lexer->lookahead == '/') {
+        advance(lexer);
+        if (lexer->lookahead != '/') return false;
+        advance(lexer);
+        if (lexer->lookahead == ':') {
+          advance(lexer);
+          if (is_newline(lexer->lookahead) ||
+              is_whitespace(lexer->lookahead)) {
+            return is_valid(lexer, valid_symbols, KEYWORD_LITERAL);
+          }
+        }
+        return false;
+      }
+
       if (lexer->lookahead != '.') return false;
       advance(lexer);
       lexer->mark_end(lexer);
@@ -604,6 +627,7 @@ struct Scanner {
             is_whitespace(lexer->lookahead)) {
           return is_valid(lexer, valid_symbols, KEYWORD_LITERAL);
         }
+        return false;
       }
 
       return is_valid(lexer, valid_symbols, IDENTIFIER, false);

test/corpus/data_types.txt

@@ -175,7 +175,7 @@ atoms
 operator atom
 ================================================================================
 
-[:@, :., :+, :-, :!, :^, :not, :~~~, :*, :/, :+, :-, :++, :--, :.., :..., :<>, :+++, :---, :^^^, :in, :not, :|>, :<<<, :>>>, :<<~, :~>>, :<~, :~>, :<~>, :<|>, :<, :>, :<=, :>=, :==, :!=, :=~, :===, :!==, :&&, :&&&, :and, :||, :|||, :=, :&, :=>, :|, :::, :when, :<-, :\\, :%, :%{}, :{}, :->, :<<>>]
+[:@, :., :+, :-, :!, :^, :not, :~~~, :*, :/, :+, :-, :++, :--, :.., :..., :<>, :+++, :---, :^^^, :in, :not, :|>, :<<<, :>>>, :<<~, :~>>, :<~, :~>, :<~>, :<|>, :<, :>, :<=, :>=, :==, :!=, :=~, :===, :!==, :&&, :&&&, :and, :||, :|||, :=, :&, :=>, :|, :::, :when, :<-, :\\, :%, :%{}, :{}, :->, :<<>>, :..//]
 
 --------------------------------------------------------------------------------
 
@@ -295,6 +295,8 @@ operator atom
       (atom_literal))
     (atom
       (atom_literal))
+    (atom
+      (atom_literal))
     (atom
       (atom_literal))))
 
@@ -861,6 +863,9 @@ linebreak followed by operator keyword
 [
 ...: 1]
 
+[
+..//: 1]
+
 [
 <>: 1]
 
@@ -1214,6 +1219,11 @@ in: 1]
       (keyword
         (keyword_literal))
       (integer)))
+  (list
+    (keyword_list
+      (keyword
+        (keyword_literal))
+      (integer)))
   (list
     (keyword_list
       (keyword

test/corpus/expression.txt

@@ -131,3 +131,19 @@ Unary operator with newline
 (program
   (unary_op
     (integer)))
+
+================================================================================
+stepped range
+================================================================================
+
+1..2
+// 4
+
+--------------------------------------------------------------------------------
+
+(program
+  (binary_op
+    (binary_op
+      (integer)
+      (integer))
+    (integer)))

test/corpus/operator.txt

@@ -67,3 +67,98 @@ a
   (binary_op
     (identifier)
     (identifier)))
+
+================================================================================
+stepped ranges
+================================================================================
+
+1 .. 2 // 3
+(1..2//3).step
+1..2//3
+0..1//-1
+--------------------------------------------------------------------------------
+
+(program
+  (binary_op
+    (binary_op
+      (integer)
+      (integer))
+    (integer))
+  (dot_call
+    (paren_expr
+      (binary_op
+        (binary_op
+          (integer)
+          (integer))
+        (integer)))
+    (function_identifier))
+  (binary_op
+    (binary_op
+      (integer)
+      (integer))
+    (integer))
+  (binary_op
+    (binary_op
+      (integer)
+      (integer))
+    (unary_op
+      (integer))))
+
+================================================================================
+stepped blocks
+================================================================================
+
+foo do end..bar do end//baz do end
+1..(2//3)
+(1..2)//3
+(1..2)//(3)
+
+--------------------------------------------------------------------------------
+
+(program
+  (binary_op
+    (binary_op
+      (call
+        (function_identifier)
+        (do_block))
+      (call
+        (function_identifier)
+        (do_block)))
+    (call
+      (function_identifier)
+      (do_block)))
+  (binary_op
+    (integer)
+    (block
+      (binary_op
+        (integer)
+        (integer))))
+  (binary_op
+    (block
+      (binary_op
+        (integer)
+        (integer)))
+    (integer))
+  (binary_op
+    (block
+      (binary_op
+        (integer)
+        (integer)))
+    (block
+      (integer))))
+
+================================================================================
+multiline stepped blocks
+================================================================================
+
+1..2
+// 4
+
+--------------------------------------------------------------------------------
+
+(program
+  (binary_op
+    (binary_op
+      (integer)
+      (integer))
+    (integer)))

test/highlight/sandbox.ex

@@ -1065,6 +1065,10 @@ use Bitwise
 2 <<< 3
 # ^ operator
 
+1..10//3
+#^ operator
+#    ^ operator
+
 # Protocols
 defprotocol Useless do
 #<- keyword