Fix lexing of messages, nested comments, missing operators and decima…

…ls in JSL lexer (rouge-ruby#1638)

This commit adds support to the JSL lexer for:

- messages that don't end with `(` or `;`;
- nested block comments;
- the missing operators `` ` `` and `^`;
- decimals that start with `.`.

lib/rouge/lexers/jsl.rb

@@ -14,22 +14,20 @@ class JSL < RegexLexer
         rule %r/\s+/m, Text::Whitespace
 
         rule %r(//.*?$), Comment::Single
-        rule %r(/\*.*?\*/)m, Comment::Multiline
+        rule %r'/[*].*', Comment::Multiline, :comment
 
         # messages
-        rule %r/(<<)(.*?)(\(|;)/ do |m|
-          groups Operator, Name::Function, Punctuation
-        end
+        rule %r/<</, Operator, :message
 
         # covers built-in and custom functions
         rule %r/([a-z_][\w\s'%.\\]*)(\()/i do |m|
           groups Keyword, Punctuation
         end
 
-        rule %r/\b[+-]?(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+|\.)(?:e[+-]?[0-9]+)?i?\b/i, Num
-
         rule %r/\d{2}(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d{2}(\d{2})?(:\d{2}:\d{2}(:\d{2}(\.\d*)?)?)?/i, Literal::Date
 
+        rule %r/-?(?:[0-9]+(?:[.][0-9]+)?|[.][0-9]*)(?:e[+-]?[0-9]+)?i?/i, Num
+
         rule %r/::[a-z_][\w\s'%.\\]*/i, Name::Variable
         rule %r/:\w+/, Name
         rule %r/[a-z_][\w\s'%.\\]*/i, Name::Variable
@@ -40,16 +38,30 @@ class JSL < RegexLexer
         end
         rule %r/"/, Str::Double, :dq
 
-        rule %r/[-+*\/!%&<>\|=:]/, Operator
+        rule %r/[-+*\/!%&<>\|=:`^]/, Operator
         rule %r/[\[\](){},;]/, Punctuation
       end
 
+      state :message do
+        rule %r/\s+/m, Text::Whitespace
+        rule %r/[a-z_][\w\s'%.\\]*/i, Name::Function
+        rule %r/[(),;]/, Punctuation, :pop!
+        rule %r/[&|!=<>]/, Operator, :pop!
+      end
+
       state :dq do
         rule %r/\\![btrnNf0\\"]/, Str::Escape
         rule %r/\\/, Str::Double
         rule %r/"/, Str::Double, :pop!
         rule %r/[^\\"]+/m, Str::Double
       end
+
+      state :comment do
+        rule %r'/[*]', Comment::Multiline, :comment
+        rule %r'[*]/', Comment::Multiline, :pop!
+        rule %r'[^/*]+', Comment::Multiline
+        rule %r'[/*]', Comment::Multiline
+      end
     end
   end
 end

spec/visual/samples/jsl

@@ -5,6 +5,14 @@ dt << Distribution( Column( :age ), Histograms Only( 1 ) );
     Multi-line comment
 */
 
+/*
+    Nested
+    /*
+        Comments
+    */
+    Work
+*/
+
 escapeSequence = "This is an \!b escaped sequence";
 escapeQuote = "This is a \!" quotation mark";
 escapeStr = "\[This is """"""" an escaped string]\"
@@ -16,9 +24,12 @@ a name with spaces = 5;
 
 scientificNotation = 5e9;
 decimal = 1.234;
+missing = .;
 date = 01jan00;
 dateTime = 12dec1999:12:30:00.45;
 
 New Window( "Rouge Test",
-    Text Box( "Syntax highlighting is great!" )
+    tb = Text Box( "Syntax highlighting is great!" )
 );
+
+If(tb << Get Text != "", "I'm still formatted correctly!");