Fix line continuations in literals (#3362)

* Fix line continuations in literals

Signed-off-by: Alexandre Terrasa <alexandre.terrasa@shopify.com>

* Add tests for line continuations in literals

Signed-off-by: Alexandre Terrasa <alexandre.terrasa@shopify.com>

test/whitequark/test_slash_slash_n_escaping_in_literals_0.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+'a\
+b'

test/whitequark/test_slash_slash_n_escaping_in_literals_0.rb.parse-tree-whitequark.exp

@@ -0,0 +1,4 @@
+s(:dstr,
+  s(:str, "a\
+"),
+  s(:str, "b"))

test/whitequark/test_slash_slash_n_escaping_in_literals_1.rb

@@ -0,0 +1,6 @@
+# typed: strict
+
+<<-'HERE'
+a\
+b
+HERE

test/whitequark/test_slash_slash_n_escaping_in_literals_1.rb.parse-tree-whitequark.exp

@@ -0,0 +1,5 @@
+s(:dstr,
+  s(:str, "a\
+"),
+  s(:str, "b
+"))

test/whitequark/test_slash_slash_n_escaping_in_literals_10.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%I{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_10.rb.parse-tree-whitequark.exp

@@ -0,0 +1,2 @@
+s(:array,
+  s(:sym, :a\nb))

test/whitequark/test_slash_slash_n_escaping_in_literals_11.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+:'a\
+b'

test/whitequark/test_slash_slash_n_escaping_in_literals_11.rb.parse-tree-whitequark.exp

@@ -0,0 +1,4 @@
+s(:dsym,
+  s(:str, "a\
+"),
+  s(:str, "b"))

test/whitequark/test_slash_slash_n_escaping_in_literals_12.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%s{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_12.rb.parse-tree-whitequark.exp

@@ -0,0 +1,4 @@
+s(:dsym,
+  s(:str, "a\
+"),
+  s(:str, "b"))

test/whitequark/test_slash_slash_n_escaping_in_literals_13.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+:"a\
+b"

test/whitequark/test_slash_slash_n_escaping_in_literals_13.rb.parse-tree-whitequark.exp

@@ -0,0 +1 @@
+s(:sym, :ab)

test/whitequark/test_slash_slash_n_escaping_in_literals_14.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+/a\
+b/

test/whitequark/test_slash_slash_n_escaping_in_literals_14.rb.parse-tree-whitequark.exp

@@ -0,0 +1,3 @@
+s(:regexp,
+  s(:str, "ab"),
+  s(:regopt, ""))

test/whitequark/test_slash_slash_n_escaping_in_literals_15.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%r{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_15.rb.parse-tree-whitequark.exp

@@ -0,0 +1,3 @@
+s(:regexp,
+  s(:str, "ab"),
+  s(:regopt, ""))

test/whitequark/test_slash_slash_n_escaping_in_literals_16.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%x{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_16.rb.parse-tree-whitequark.exp

@@ -0,0 +1,2 @@
+s(:xstr,
+  s(:str, "ab"))

test/whitequark/test_slash_slash_n_escaping_in_literals_17.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+`a\
+b`

test/whitequark/test_slash_slash_n_escaping_in_literals_17.rb.parse-tree-whitequark.exp

@@ -0,0 +1,2 @@
+s(:xstr,
+  s(:str, "ab"))

test/whitequark/test_slash_slash_n_escaping_in_literals_18.rb

@@ -0,0 +1,6 @@
+# typed: strict
+
+<<-`HERE`
+a\
+b
+HERE

test/whitequark/test_slash_slash_n_escaping_in_literals_18.rb.parse-tree-whitequark.exp

@@ -0,0 +1,3 @@
+s(:xstr,
+  s(:str, "ab
+"))

test/whitequark/test_slash_slash_n_escaping_in_literals_19.rb

@@ -0,0 +1,6 @@
+# typed: strict
+
+<<-HERE
+a\
+b
+HERE

test/whitequark/test_slash_slash_n_escaping_in_literals_19.rb.parse-tree-whitequark.exp

@@ -0,0 +1,2 @@
+s(:str, "ab
+")

test/whitequark/test_slash_slash_n_escaping_in_literals_2.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%q{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_2.rb.parse-tree-whitequark.exp

@@ -0,0 +1,4 @@
+s(:dstr,
+  s(:str, "a\
+"),
+  s(:str, "b"))

test/whitequark/test_slash_slash_n_escaping_in_literals_20.rb

@@ -0,0 +1,6 @@
+# typed: strict
+
+<<~HERE
+a\
+b
+HERE

test/whitequark/test_slash_slash_n_escaping_in_literals_20.rb.parse-tree-whitequark.exp

@@ -0,0 +1,2 @@
+s(:str, "ab
+")

test/whitequark/test_slash_slash_n_escaping_in_literals_21.rb

@@ -0,0 +1,6 @@
+# typed: strict
+
+<<~HERE
+  a\
+  b
+HERE

test/whitequark/test_slash_slash_n_escaping_in_literals_21.rb.parse-tree-whitequark.exp

@@ -0,0 +1,2 @@
+s(:str, "a  b
+")

test/whitequark/test_slash_slash_n_escaping_in_literals_3.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+"a\
+b"

test/whitequark/test_slash_slash_n_escaping_in_literals_3.rb.parse-tree-whitequark.exp

@@ -0,0 +1 @@
+s(:str, "ab")

test/whitequark/test_slash_slash_n_escaping_in_literals_4.rb

@@ -0,0 +1,6 @@
+# typed: strict
+
+<<-"HERE"
+a\
+b
+HERE

test/whitequark/test_slash_slash_n_escaping_in_literals_4.rb.parse-tree-whitequark.exp

@@ -0,0 +1,2 @@
+s(:str, "ab
+")

test/whitequark/test_slash_slash_n_escaping_in_literals_5.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_5.rb.parse-tree-whitequark.exp

@@ -0,0 +1 @@
+s(:str, "ab")

test/whitequark/test_slash_slash_n_escaping_in_literals_6.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%Q{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_6.rb.parse-tree-whitequark.exp

@@ -0,0 +1 @@
+s(:str, "ab")

test/whitequark/test_slash_slash_n_escaping_in_literals_7.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%w{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_7.rb.parse-tree-whitequark.exp

@@ -0,0 +1,3 @@
+s(:array,
+  s(:str, "a
+b"))

test/whitequark/test_slash_slash_n_escaping_in_literals_8.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%W{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_8.rb.parse-tree-whitequark.exp

@@ -0,0 +1,3 @@
+s(:array,
+  s(:str, "a
+b"))

test/whitequark/test_slash_slash_n_escaping_in_literals_9.rb

@@ -0,0 +1,4 @@
+# typed: strict
+
+%i{a\
+b}

test/whitequark/test_slash_slash_n_escaping_in_literals_9.rb.parse-tree-whitequark.exp

@@ -0,0 +1,2 @@
+s(:array,
+  s(:sym, :a\nb))

third_party/parser/cc/lexer.rl

@@ -1028,7 +1028,7 @@ void lexer::set_state_expr_value() {
 
   action extend_string_escaped {
     auto& current_literal = literal_();
-
+    // Get the first character after the backslash.
     // TODO multibyte
     auto escaped_char = *escape_s;
 
@@ -1071,7 +1071,26 @@ void lexer::set_state_expr_value() {
       }
     } else {
       // It does not. So this is an actual escape sequence, yay!
-      if (current_literal.regexp()) {
+      if (current_literal.squiggly_heredoc() && escaped_char == '\n') {
+        // Squiggly heredocs like
+        //   <<~-HERE
+        //     1\
+        //     2
+        //   HERE
+        // treat '\' as a line continuation, but still dedent the body, so the heredoc above becomes "12\n".
+        // This information is emitted as is, without escaping,
+        // later this escape sequence (\\\n) gets handled manually in the dedenter
+        std::string str = gsub(tok(), "\\\n", "");
+        current_literal.extend_string(str, ts, te);
+      } else if (current_literal.support_line_continuation_via_slash() && escaped_char == '\n') {
+        // Heredocs, regexp and a few other types of literals support line
+        // continuation via \\\n sequence. The code like
+        //   "a\
+        //   b"
+        // must be parsed as "ab"
+        std::string str = gsub(tok(), "\\\n", "");
+        current_literal.extend_string(str, ts, te);
+      } else if (current_literal.regexp()) {
         // Regular expressions should include escape sequences in their
         // escaped form. On the other hand, escaped newlines are removed.
         std::string str = gsub(tok(), "\\\n", "");

third_party/parser/cc/literal.cc

@@ -82,6 +82,14 @@ bool literal::heredoc() const {
   return heredoc_e != nullptr;
 }
 
+bool literal::squiggly_heredoc() const {
+  return heredoc() && dedent_body;
+}
+
+bool literal::support_line_continuation_via_slash() const {
+  return !words() && interpolate();
+}
+
 token_type literal::start_token_type() const {
   switch (_type) {
     case literal_type::SQUOTE_STRING:

third_party/parser/include/ruby_parser/literal.hh

@@ -68,6 +68,8 @@ namespace ruby_parser {
     bool interpolate() const;
     bool regexp() const;
     bool heredoc() const;
+    bool squiggly_heredoc() const;
+    bool support_line_continuation_via_slash() const;
 
     token_type start_token_type() const;