fix: handling escaped whitespace in a %w list

Introduces a new flavor of unescaping, YP_UNESCAPE_WHITESPACE, which
is the same as MINIMAL but also unescapes whitespace.

Note that a spanning_heredoc.txt fixture test is updated to be less
wrong, but YARP's behavior doesn't yet fully match Ruby in this case.

Fixes #1505

Author: flavorjones

ext/yarp/extension.c

@@ -491,6 +491,12 @@ unescape_minimal(VALUE self, VALUE source) {
     return unescape(source, YP_UNESCAPE_MINIMAL);
 }
 
+// Escape the given string minimally plus whitespace. Returns the unescaped string.
+static VALUE
+unescape_whitespace(VALUE self, VALUE source) {
+    return unescape(source, YP_UNESCAPE_WHITESPACE);
+}
+
 // Unescape everything in the given string. Return the unescaped string.
 static VALUE
 unescape_all(VALUE self, VALUE source) {
@@ -608,6 +614,7 @@ Init_yarp(void) {
     rb_define_singleton_method(rb_cYARPDebug, "named_captures", named_captures, 1);
     rb_define_singleton_method(rb_cYARPDebug, "unescape_none", unescape_none, 1);
     rb_define_singleton_method(rb_cYARPDebug, "unescape_minimal", unescape_minimal, 1);
+    rb_define_singleton_method(rb_cYARPDebug, "unescape_whitespace", unescape_whitespace, 1);
     rb_define_singleton_method(rb_cYARPDebug, "unescape_all", unescape_all, 1);
     rb_define_singleton_method(rb_cYARPDebug, "memsize", memsize, 1);
     rb_define_singleton_method(rb_cYARPDebug, "profile_file", profile_file, 1);

include/yarp/unescape.h

@@ -24,9 +24,13 @@ typedef enum {
     // single quotes and backslashes.
     YP_UNESCAPE_MINIMAL,
 
+    // When we're unescaping a string list, in addition to MINIMAL, we need to
+    // unescape whitespace.
+    YP_UNESCAPE_WHITESPACE,
+
     // When we're unescaping a double-quoted string, we need to unescape all
     // escapes.
-    YP_UNESCAPE_ALL
+    YP_UNESCAPE_ALL,
 } yp_unescape_type_t;
 
 // Unescape the contents of the given token into the given string using the given unescape mode.

src/unescape.c

@@ -509,7 +509,17 @@ yp_unescape_manipulate_string_or_char_literal(yp_parser_t *parser, yp_string_t *
                 cursor = backslash + 2;
                 break;
             default:
-                if (unescape_type == YP_UNESCAPE_MINIMAL) {
+                if (unescape_type == YP_UNESCAPE_WHITESPACE) {
+                    if (backslash[1] == '\r' && backslash[2] == '\n') {
+                        cursor = backslash + 2;
+                        break;
+                    }
+                    if (yp_strspn_whitespace(backslash + 1, 1)) {
+                        cursor = backslash + 1;
+                        break;
+                    }
+                }
+                if (unescape_type == YP_UNESCAPE_WHITESPACE || unescape_type == YP_UNESCAPE_MINIMAL) {
                     // In this case we're escaping something that doesn't need escaping.
                     dest[dest_length++] = '\\';
                     cursor = backslash + 1;
@@ -579,7 +589,16 @@ yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *backslash,
         case '\'':
             return 2;
         default: {
-            if (unescape_type == YP_UNESCAPE_MINIMAL) {
+            if (unescape_type == YP_UNESCAPE_WHITESPACE) {
+                if (backslash[1] == '\r' && backslash[2] == '\n') {
+                    return 2;
+                }
+                size_t whitespace = yp_strspn_whitespace(backslash + 1, 1);
+                if (whitespace > 0) {
+                    return whitespace;
+                }
+            }
+            if (unescape_type == YP_UNESCAPE_WHITESPACE || unescape_type == YP_UNESCAPE_MINIMAL) {
                 return 1 + yp_char_width(parser, backslash + 1, parser->end);
             }
 

src/yarp.c

@@ -12958,7 +12958,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) {
 
                 yp_token_t opening = not_provided(parser);
                 yp_token_t closing = not_provided(parser);
-                yp_node_t *string = (yp_node_t *) yp_string_node_create_and_unescape(parser, &opening, &parser->previous, &closing, YP_UNESCAPE_MINIMAL);
+                yp_node_t *string = (yp_node_t *) yp_string_node_create_and_unescape(parser, &opening, &parser->previous, &closing, YP_UNESCAPE_WHITESPACE);
                 yp_array_node_elements_append(array, string);
             }
 

test/yarp/snapshots/spanning_heredoc.txt

@@ -136,6 +136,13 @@ def test_escaping_normal_characters
       assert_unescape_all("g", "\\g")
     end
 
+    def test_whitespace_escaping_string_list
+      assert_equal("a b", Debug.unescape_whitespace("a\\ b"))
+      assert_equal("a\tb", Debug.unescape_whitespace("a\\\tb"))
+      assert_equal("a\nb", Debug.unescape_whitespace("a\\\nb"))
+      assert_equal("a\nb", Debug.unescape_whitespace("a\\\r\nb"))
+    end
+
     private
 
     def unescape_all(source)