#234 Allow non-ASCII here-doc delimiters.

ScintillaOrg · Apr 5, 2024 · 34b7758 · 34b7758
1 parent 5dad19b
commit 34b7758
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 3 deletions.
diff --git a/doc/LexillaHistory.html b/doc/LexillaHistory.html
@@ -615,6 +615,10 @@ <h3>
 	HTML: Match standard handling of comments.
 	<a href="https://github.com/ScintillaOrg/lexilla/issues/232">Issue #232</a>.
 	</li>
+	<li>
+	Ruby: Allow non-ASCII here-doc delimiters.
+	<a href="https://github.com/ScintillaOrg/lexilla/issues/234">Issue #234</a>.
+	</li>
     </ul>
     <h3>
        <a href="https://www.scintilla.org/lexilla531.zip">Release 5.3.1</a>

diff --git a/lexers/LexRuby.cxx b/lexers/LexRuby.cxx
@@ -48,6 +48,10 @@ inline bool isSafeAlpha(char ch) noexcept {
     return (isSafeASCII(ch) && isalpha(ch)) || ch == '_';
 }
 
+inline bool isSafeAlphaOrHigh(char ch) noexcept {
+	return isHighBitChar(ch) || isalpha(ch) || ch == '_';
+}
+
 inline bool isSafeAlnum(char ch) noexcept {
     return (isSafeASCII(ch) && isalnum(ch)) || ch == '_';
 }
@@ -613,7 +617,7 @@ bool sureThisIsNotHeredoc(Sci_Position lt2StartPos, Accessor &styler) {
         j += 1;
     }
 
-    if (isSafeAlnum(styler[j])) {
+    if (isSafeAlnumOrHigh(styler[j])) {
         // Init target_end because some compilers think it won't
         // be initialized by the time it's used
         target_start = target_end = j;
@@ -622,7 +626,7 @@ bool sureThisIsNotHeredoc(Sci_Position lt2StartPos, Accessor &styler) {
         return definitely_not_a_here_doc;
     }
     for (; j < lengthDoc; j++) {
-        if (!isSafeAlnum(styler[j])) {
+        if (!isSafeAlnumOrHigh(styler[j])) {
             if (target_quote && styler[j] != target_quote) {
                 // unquoted end
                 return definitely_not_a_here_doc;
@@ -945,7 +949,7 @@ void ColouriseRbDoc(Sci_PositionU startPos, Sci_Position length, int initStyle,
                 chNext = chNext2;
                 styler.ColourTo(i, SCE_RB_OPERATOR);
 
-                if (!(strchr("\"\'`_-~", chNext2) || isSafeAlpha(chNext2))) {
+                if (!(strchr("\"\'`_-~", chNext2) || isSafeAlphaOrHigh(chNext2))) {
                     // It's definitely not a here-doc,
                     // based on Ruby's lexer/parser in the
                     // heredoc_identifier routine.

diff --git a/test/examples/ruby/234HereDoc.rb b/test/examples/ruby/234HereDoc.rb
@@ -0,0 +1,8 @@
+# encoding: utf-8
+puts <<A中
+#{1+2}
+A中
+
+puts <<中
+#{1+2}
+中
diff --git a/test/examples/ruby/234HereDoc.rb.folded b/test/examples/ruby/234HereDoc.rb.folded
@@ -0,0 +1,9 @@
+ 0 400   0   # encoding: utf-8
+ 2 400   0 + puts <<A中
+ 0 401   0 | #{1+2}
+ 0 401   0 | A中
+ 1 400   0   
+ 2 400   0 + puts <<中
+ 0 401   0 | #{1+2}
+ 0 401   0 | 中
+ 0 400   0   
diff --git a/test/examples/ruby/234HereDoc.rb.styled b/test/examples/ruby/234HereDoc.rb.styled
@@ -0,0 +1,8 @@
+{2}# encoding: utf-8{0}
+{11}puts{0} {10}<<{20}A中{22}
+{10}#{{4}1{10}+{4}2{10}}{22}
+{20}A中{0}
+
+{11}puts{0} {10}<<{20}中{22}
+{10}#{{4}1{10}+{4}2{10}}{22}
+{20}中{0}