[test/ysh-every-string] Add escaping test cases

Still need to change J8 notation to use single rather than double
quotes.

Remove confusing \' escape from lexer def.

frontend/lexer_def.py

@@ -568,7 +568,7 @@ def R(pat, tok_type):
     R(r'[\x01-\x1F]', Id.Char_AsciiControl),
 
     # Note: This will match INVALID UTF-8.  UTF-8 validation is another step.
-    R(r'[^\\"\'\0]+', Id.Char_Literals),
+    R(r'''[^\\"'\0]+''', Id.Char_Literals),
 
     # Should match control chars
     R(r'[^\0]', Id.Unknown_Tok),

osh/word_parse.py

@@ -812,8 +812,10 @@ def _ReadUnquotedLeftParts(self, triple_out):
                 self._SetNext(lex_mode_e.ShCommand)
                 self._GetToken()
 
-                # HACK: magically transform the third ' in r''' to
-                # Id.Left_RTSingleQuote, so that ''' is the terminator
+                # TODO: multi-line strings must he Id.Left_SingleQuote
+
+                # HACK: magically transform the third ' in ''' to
+                # Id.Left_TSingleQuote, so that ''' is the terminator
                 left_sq_token = self.cur_token
                 left_sq_token.id = new_id
                 triple_out.b = True  # let caller know we got it

test/ysh-every-string.sh

@@ -22,17 +22,14 @@ test-legacy-osh-ysh() {
     log "     test-legacy with $sh"
 
     $sh <<'EOF'
-  echo 'foo
+  echo 'foo \ "
 ---'
 
-  echo "foo
+  echo "foo \\ \" '
 ---"
 
-  echo $'fo\x6f
+  echo $'fo\x6f \\ \" \'
 ---'
-
-  #echo u'fooz
-#---'
 EOF
   done
 }
@@ -44,12 +41,12 @@ test-legacy-expr() {
 ---'
   echo $x
 
-  var x = "foo
+  var x = "foo \\ \" '
 ---"
   echo $x
 
   # I guess this is useful for refactoring
-  var x = $'f\u006f\x6f
+  var x = $'f\u006f\x6f \\ \" \'
 ---'
   echo $x
 
@@ -63,12 +60,12 @@ test-legacy-multiline() {
   for sh in $YSH; do
     $sh <<'EOF'
   echo """
-  foo
+  foo \\ \" '
   ---
   """
 
   var x = """
-  foo
+  foo \\ \" '
   ---
   """
   echo $x
@@ -129,13 +126,15 @@ EOF
 test-j8() {
   ### J8 strings are allowed in YSH
 
+  # TODO: double to single quotes - Add " \' here
+
   #for sh in $OSH $YSH; do
   for sh in $YSH; do
     $sh <<'EOF'
 
   # Command mode
 
-  echo u'fo\u{6f}
+  echo u'fo\u{6f} \\ \"
 ---'
 
   echo b'f\u{6f}\y6f