Merge pull request #9 from appcypher/ch-implement-lexer-156977385

#156977385 Implement Lexer

docs/language/summary.ast

@@ -103,7 +103,7 @@ COMMAND NOTATION
 
     print [9] # Invalid!
     print {9} # Invalid!
-    print /9/ # Invalid!
+    print //9// # Invalid!
     print +5 # Invalid!
 
 TYPE DEFINITION (1)
@@ -636,7 +636,7 @@ FOR LOOP
         return i
 
     # The `end` block executes when a loop completes without interruption from `break`, `return` or `throw`.
-    for line in file where /lagbaja/ in line:
+    for line in file where //lagbaja// in line:
         print "name found!"
         break
     end:
@@ -671,8 +671,8 @@ IN
         print 'Aaargh! Everyone hates me'
 
     # Regex matching.
-    if /dollar[s]?/ in sentence:
-        sentence.replace(/dollar[s]?/, 'pounds')
+    if //dollar[s]?// in sentence:
+        sentence.replace(//dollar[s]?//, 'pounds')
 
 MOD
     # `mod` is a keyword-based infix operator alias of `%` operator.
@@ -718,7 +718,7 @@ WHILE LOOP & LOOP
     ..
 
     # Like `for` loops, `while` and `loop` can have a closing `end` block.
-    while file.hasNext() where /\t+/ in file.next():
+    while file.hasNext() where //\t+// in file.next():
         break 'File contains tabs!'
     end: 'File is clean!'
 
@@ -1301,11 +1301,11 @@ SYMBOL
 
 REGEX
     # Regex are used to find and/or capture patterns in strings.
-    var numberPattern = /\d+(.\d+)?/
+    var numberPattern = //\d+(.\d+)?//
     numberPattern in 'π = 3.14159265' # true
 
     # Regex operations.
-    var = /\d/ + re.'\[a-z]' + /\d/ # Concatenation
+    var = //\d// + //[a-z]// + //\d// # Concatenation
 
 COEFFICIENT EXPRESSION
     # Coefficient expression as the name implies allows coefficients to be taken as multiplication.

src/compiler/syntax/grammar-and-ast.peg

@@ -9,6 +9,9 @@ spacechar =
 spaces =
   | space+
 
+newline =
+  | '\r'? '\n'
+
 noname =
   | '_'
 
@@ -83,6 +86,10 @@ floatdecimalliteral  =
   | digitdecimal ('_'? digitdecimal)* 'e' [-+]? digitdecimal ('_'? digitdecimal)*
   { token, kind, startLine, stopLine, startColumn, stopColumn }
 
+floatLiteralnomantissa =
+  | (integerbinaryliteral | integeroctalliteral | integerhexadecimalliteral | integerdecimalliteral) '.' !(operator)
+  { token, kind, startLine, stopLine, startColumn, stopColumn }
+
 singlequotestringchars =
   | (!(newline | "'") .)+ // TODO
 
@@ -95,21 +102,21 @@ singlelinestringliteral =
   { token, kind, startLine, stopLine, startColumn, stopColumn }
 
 triplesinglequotestringchars =
-  | (!(newline | "'''") .)+ // TODO
+  | (!"'''" .)+ // TODO
 
 tripledoublequotestringchars =
-  | (!(newline | '"""') .)+ // TODO
+  | (!'"""' .)+ // TODO
 
 multilinestringliteral =
   | "'''" triplesinglequotestringchars? (nextline samedent triplesinglequotestringchars?)* "'''"
   | '"""' tripledoublequotestringchars? (nextline samedent tripledoublequotestringchars?)* '"""'
   { token, kind, startLine, stopLine, startColumn, stopColumn }
 
 regexchars =
-  | (!(newline | '/') .)+ // TODO
+  | (!(newline | '//') .)+ // TODO
 
 regexliteral =
-  | '/' regexchars? '/'
+  | '//' regexchars? '//'
   { token, kind, startLine, stopLine, startColumn, stopColumn }
 
 booleanliteral =
@@ -138,6 +145,9 @@ multilinecomment =
 
 /*************** PARSER ***************/
 
+// NOTE: multilinestringliteral has a special parsing stage
+multilinestringliteral
+
 integerliteral =
   | integerbinaryliteral
   | integeroctalliteral
@@ -189,9 +199,6 @@ dedent = // TODO. adding !space
   | ' '+
   { level }
 
-newline =
-  | '\r'? '\n'
-
 nextline =
   | newline (_? newline)*
 
@@ -299,8 +306,8 @@ namedtupleliteral =
 
 symbolliteral =
   | '$' identifier
-  | '$' '(' _? expression _? ')' // ignorenewline
-  | '$' '(' block ')'
+  | '$' '{' _? expression _? '}' // ignorenewline
+  | '$' '{' block '}'
   { kind, expressions }
 
 listcomprehension =