scanner!: 1) detect line sep by CR and replaces to EOL. 2) remove END…

… spaces.
gad-lang · Jan 16, 2024 · ce24c73 · ce24c73
1 parent 4f38a25
commit ce24c73
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 8 deletions.
diff --git a/parser/parser_test.go b/parser/parser_test.go
@@ -2924,6 +2924,16 @@ if true {
 return`)
 }
 
+func TestParseLinesSep(t *testing.T) {
+	expectParseString(t, "\r\r1+\r\r2+\r\r\r3\r\r\n  \t", `((1 + 2) + 3)`)
+	expectParseString(t, "1+\n2+\n3", `((1 + 2) + 3)`)
+	expectParseString(t, "1+\r\n2+\n3", `((1 + 2) + 3)`)
+	expectParseString(t, "1+\r2+\n3", `((1 + 2) + 3)`)
+	expectParseString(t, "1+\r2+\r3", `((1 + 2) + 3)`)
+	expectParseString(t, "\r\r1+\r2+\r3", `((1 + 2) + 3)`)
+	expectParseString(t, "\r\r1+\r\r2+\r\r\r3", `((1 + 2) + 3)`)
+}
+
 type pfn func(int, int) Pos          // position conversion function
 type expectedFn func(pos pfn) []Stmt // callback function to return expected results
 

diff --git a/parser/scanner.go b/parser/scanner.go
@@ -15,6 +15,7 @@
 package parser
 
 import (
+	"bytes"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -230,6 +231,17 @@ func NewScanner(
 			file.Size, len(src)))
 	}
 
+	isSpace := func(r rune) bool {
+		switch r {
+		case ' ', '\t', '\n', '\r':
+			return true
+		default:
+			return false
+		}
+	}
+
+	src = bytes.TrimRightFunc(src, isSpace)
+
 	if opts == nil {
 		opts = &ScannerOptions{}
 	}
@@ -238,6 +250,18 @@ func NewScanner(
 		opts.MixedExprRune = '#'
 	}
 
+	last := len(src) - 1
+	if pos := bytes.IndexByte(src, '\r'); pos >= 0 {
+		// if line sep is only CR, replaces to EOL
+		if pos < last && src[pos] != '\n' {
+			for i, b := range src {
+				if b == '\r' && i < last && src[i+1] != '\n' {
+					src[i] = '\n'
+				}
+			}
+		}
+	}
+
 	s := &Scanner{
 		File:          file,
 		Src:           src,
@@ -713,7 +737,11 @@ func (s *Scanner) NextTo(v string) {
 }
 
 func (s *Scanner) Next() {
-	var newLineEscape bool
+	var (
+		newLineEscape bool
+		r             rune
+		w             int
+	)
 next:
 	if s.ReadOffset < len(s.Src) {
 		s.Offset = s.ReadOffset
@@ -730,7 +758,13 @@ next:
 			defer s.CallPostLineEndHandlers()
 		}
 
-		r, w := rune(s.Src[s.ReadOffset]), 1
+		r, w = rune(s.Src[s.ReadOffset]), 1
+
+		for r == '\r' {
+			s.ReadOffset++
+			r = rune(s.Src[s.ReadOffset])
+		}
+
 		switch {
 		case r == 0:
 			s.Error(s.Offset, "illegal character NUL")

diff --git a/parser/scanner_test.go b/parser/scanner_test.go
@@ -183,6 +183,17 @@ func TestScanner_Scan(t *testing.T) {
 func testScan(t *testing.T, mode parser.ScanMode, addLines bool, testCases []struct {
 	token   token.Token
 	literal string
+}) {
+	t.Helper()
+	testScanI(t, mode, addLines, "\n", testCases)
+	if addLines {
+		testScanI(t, mode, addLines, "\r\n", testCases)
+	}
+}
+
+func testScanI(t *testing.T, mode parser.ScanMode, addLines bool, lineSep string, testCases []struct {
+	token   token.Token
+	literal string
 }) {
 	// combine
 	var lines []string
@@ -277,9 +288,9 @@ func testScan(t *testing.T, mode parser.ScanMode, addLines bool, testCases []str
 		}
 	}
 
-	scanExpect(t, strings.Join(lines, "\n"),
+	scanExpect(t, strings.Join(lines, lineSep),
 		parser.ScanComments|parser.DontInsertSemis|mode, expected...)
-	scanExpect(t, strings.Join(lines, "\n"),
+	scanExpect(t, strings.Join(lines, lineSep),
 		parser.DontInsertSemis|mode, expectedSkipComments...)
 }
 

diff --git a/parser/testdata/trace.golden b/parser/testdata/trace.golden
@@ -1396,7 +1396,7 @@
   610:    49: 26: . . . . )
   610:    49: 26: . . . )
   610:    49: 26: . . . ";"
-  611:    49: 27: . . )
-  611:    49: 27: . )
-  611:    49: 27: . EOF
-  611:    49: 27: )
+  610:    49: 26: . . )
+  610:    49: 26: . )
+  610:    49: 26: . EOF
+  610:    49: 26: )