[spec/ysh-string] Failing tests for multi-line J8 strings

Update expression mode lexer for multi-line strings.

frontend/id_kind_def.py

@@ -443,6 +443,9 @@ def AddKinds(spec):
             'TDoubleQuote',  # """ """
             'TSingleQuote',  # ''' '''
             'RTSingleQuote',  # r''' '''
+            'UTSingleQuote',  # u''' '''
+            'BTSingleQuote',  # b''' '''
+
             'DollarTSingleQuote',  # $''' '''
             'Backtick',  # `
             'DollarParen',  # $(

frontend/lexer_def.py

@@ -803,12 +803,18 @@ def R(pat, tok_type):
     # In expression mode, we add the r'' and c'' prefixes for '' and $''.
     C("'", Id.Left_SingleQuote),
     C("r'", Id.Left_RSingleQuote),
-    C("$'", Id.Left_DollarSingleQuote),
+    C("u'", Id.Left_USingleQuote),
+    C("b'", Id.Left_BSingleQuote),
+    C("$'", Id.Left_DollarSingleQuote),  # TODO: remove
+
     C('"""', Id.Left_TDoubleQuote),
     # In expression mode, we add the r'' and c'' prefixes for '' and $''.
     C("'''", Id.Left_TSingleQuote),
     C("r'''", Id.Left_RTSingleQuote),
-    C("$'''", Id.Left_DollarTSingleQuote),
+    C("u'''", Id.Left_UTSingleQuote),
+    C("b'''", Id.Left_BTSingleQuote),
+    C("$'''", Id.Left_DollarTSingleQuote),  # TODO: remove from YSH expressions
+
     C('@(', Id.Left_AtParen),  # Split Command Sub
     C('^(', Id.Left_CaretParen),  # Block literals in expression mode
     C('^[', Id.Left_CaretBracket),  # Expr literals

osh/word_parse.py

@@ -725,7 +725,6 @@ def _ReadYshSingleQuoted(self, left_id):
 
         #log('AFT self.cur_token %s', self.cur_token)
 
-        # TODO: Change this to Left_USingleQuote, BSingleQuote
         sq_part.left.id = left_id
 
         # Advance and validate
@@ -773,14 +772,15 @@ def _ReadUnquotedLeftParts(self, triple_out):
                 new_id = Id.Left_TSingleQuote
 
             sq_part = self._ReadSingleQuoted(self.cur_token, lexer_mode)
-            # read empty '' or r'' or $''
             if triple_out and len(sq_part.tokens) == 0:
+                # Read empty '' or r'' or $''.  Now test if there's a triple
+                # quote.
                 if self.lexer.ByteLookAhead() == "'":
                     self._SetNext(lex_mode_e.ShCommand)
                     self._GetToken()
 
                     # HACK: magically transform the third ' in r''' to
-                    # Id.Left_TSingleQuote, so that ''' is the terminator
+                    # Id.Left_RTSingleQuote, so that ''' is the terminator
                     left_sq_token = self.cur_token
                     left_sq_token.id = new_id
                     triple_out.b = True  # let caller know we got it
@@ -1916,11 +1916,12 @@ def _ReadWord(self, word_mode):
 
                         # TODO: This makes triple quoted strings fail
 
-                        #self._SetNext(lex_mode_e.ShCommand)
-                        #self._GetToken()
-                        #assert self.token_type == Id.Left_SingleQuote, self.token_type
+                        if 0:
+                            #self._SetNext(lex_mode_e.ShCommand)
+                            self._GetToken()
+                            #assert self.token_type == Id.Left_SingleQuote, self.token_type
 
-                        #return self._ReadYshSingleQuoted(Id.Left_RSingleQuote)
+                            return self._ReadYshSingleQuoted(Id.Left_RSingleQuote)
 
                     # When shopt -s parse_j8_string
                     #     echo u'\u{3bc}' b'\yff' works

spec/ysh-string.test.sh

@@ -1,5 +1,5 @@
 ## our_shell: ysh
-## oils_failures_allowed: 1
+## oils_failures_allowed: 2
 
 #### single quoted -- implicit and explicit raw
 var x = 'foo bar'
@@ -67,14 +67,41 @@ hello
 write --end '' -- u'\u{3bc}' | od -A n -t x1
 write --end '' -- b'\yff' | od -A n -t x1
 
-# Should be illegal?
-#echo u'hello \u03bc'
+# TODO: make this be illegal
+# echo u'hello \u03bc'
 
 ## STDOUT:
  ce bc
  ff
 ## END
 
+#### J8-style multi-line strings u''' b''' in command mode
+
+write --end '' -- u'''
+--
+\u{61}
+--
+'''
+write --end '' -- b'''
+--
+\y61
+--
+'''
+
+# Should be illegal?
+#echo u'hello \u03bc'
+
+## STDOUT:
+
+--
+a
+--
+
+--
+a
+--
+## END
+
 #### Double Quoted
 var name = 'World'
 var g = "Hello $name"