FIX: do not allow LF or CR characters in double-quoted string literal…

…s anymore.
red · Jun 11, 2020 · 9a6b7ea · 9a6b7ea
1 parent a34d552
commit 9a6b7ea
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 3 deletions.
diff --git a/docs/lexer/lexer-FSM.csv b/docs/lexer/lexer-FSM.csv
@@ -1,7 +1,7 @@
 ;C_BLANK;C_LINE;C_DIGIT;C_ZERO;C_BLOCK_OP;C_BLOCK_CL;C_PAREN_OP;C_PAREN_CL;C_STRING_OP;C_STRING_CL;C_DBL_QUOTE;C_SHARP;C_QUOTE;C_COLON;C_X;C_T;C_H;C_E_LOW;C_E_UP;C_ALPHAL;C_ALPHAU;C_SLASH;C_BSLASH;C_LESSER;C_GREATER;C_EQUAL;C_PERCENT;C_COMMA;C_SEMICOL;C_AT;C_DOT;C_MONEY;C_PLUS;C_MINUS;C_CARET;C_BIN;C_WORD;C_ILLEGAL;C_EOF
 S_START;S_START;S_START;S_NUMBER;S_NUMBER;T_BLK_OP;T_BLK_CL;T_PAR_OP;T_PAR_CL;T_MSTR_OP;T_ERROR;S_LINE_STR;S_SHARP;S_WORD_1ST;S_WORD_1ST;S_WORD;S_WORD;S_WORD;S_WORD;S_HEX;S_WORD;S_HEX;S_SLASH;T_ERROR;S_LESSER;S_WORD;S_WORD;S_FILE_1ST;T_ERROR;S_LINE_CMT;S_REF;S_DOTWORD;S_MONEY_1ST;S_SIGN;S_SIGN;S_WORD;S_WORD;S_WORD;T_ERROR;T_EOF
 S_LINE_CMT;S_LINE_CMT;T_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;S_LINE_CMT;T_ERROR;T_EOF
-S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;T_STRING;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_SKIP_STR;S_LINE_STR;S_LINE_STR;T_ERROR;T_ERROR
+S_LINE_STR;S_LINE_STR;T_ERROR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;T_STRING;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_SKIP_STR;S_LINE_STR;S_LINE_STR;T_ERROR;T_ERROR
 S_SKIP_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;S_LINE_STR;T_ERROR;T_EOF
 S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;T_MSTR_OP;T_MSTR_CL;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_SKIP_MSTR;S_M_STRING;S_M_STRING;T_ERROR;T_ERROR
 S_SKIP_MSTR;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;S_M_STRING;T_ERROR;T_ERROR

diff --git a/docs/lexer/lexer-FSM.xlsx b/docs/lexer/lexer-FSM.xlsx
diff --git a/runtime/lexer-transitions.reds b/runtime/lexer-transitions.reds
@@ -111,7 +111,7 @@ Red/System [
     transitions: #{
 000016163F404142433E020F2F2F303030302530250D3E283030063E01352D22
 2C2C3030303E3D01480101010101010101010101010101010101010101010101
-0101010101010101010101013E3D020202020202020202024902020202020202
+0101010101010101010101013E3D023E02020202020202024902020202020202
 020202020202020202020202020202020302023E3E0202020202020202020202
 02020202020202020202020202020202020202020202020202023E3D04040404
 0404040443440404040404040404040404040404040404040404040404040504

diff --git a/runtime/lexer.reds b/runtime/lexer.reds
@@ -175,7 +175,7 @@ lexer: context [
 		C_LINE 											;-- 0A		LF
 		C_BIN											;-- 0B
 		C_BIN											;-- 0C
-		C_BLANK											;-- 0D		CR
+		C_LINE											;-- 0D		CR
 		C_BIN C_BIN C_BIN C_BIN C_BIN C_BIN C_BIN C_BIN	;-- 0E-15
 		C_BIN C_BIN C_BIN C_BIN C_BIN C_BIN C_BIN C_BIN	;-- 16-1D
 		C_BIN C_BIN										;-- 1E-1F

diff --git a/tests/source/units/lexer-test.red b/tests/source/units/lexer-test.red
@@ -609,6 +609,9 @@ Red [
 	--test-- "tro-142"  --assert -1000000000 == transcode/one "-1000000000"
 
 	--test-- "tro-143"  --assert #"^@" == transcode/one {#""}
+	--test-- "tro-144"  --assert error? try [transcode/one {"hello^/world"}]
+	--test-- "tro-145"  --assert error? try [transcode/one {"hello^Mworld"}]
+	--test-- "tro-146"  --assert "hello^-world" == transcode/one {"hello^-world"}
 
 ===end-group===
 ===start-group=== "transcode/next"
@@ -700,6 +703,9 @@ Red [
 	--test-- "scan-59" --assert error!	 = scan "30/02/2020"
 	--test-- "scan-60" --assert none? 	   scan ""
 	--test-- "scan-61" --assert char!	 = scan {#""}
+	--test-- "scan-62" --assert error!	 = scan {"hello^/world"}
+	--test-- "scan-63" --assert error!	 = scan {"hello^Mworld"}
+	--test-- "scan-64" --assert string!	 = scan {"hello^-world"}
 
 ===end-group===
 ===start-group=== "scan/fast"