Better way to handle \exp_not:n in \text_expand:n (fixes #875)

l3kernel/l3text.dtx

@@ -637,8 +637,10 @@
 % \begin{macro}[EXP]{\@@_expand_replace:N}
 % \begin{macro}[EXP]{\@@_expand_replace:n}
 % \begin{macro}[EXP]{\@@_expand_cs_expand:N}
-% \begin{macro}[EXP]{\@@_expand_noexpand:w}
-% \begin{macro}[EXP]{\@@_expand_noexpand:nn}
+% \begin{macro}[EXP]{\@@_expand_unexpanded:w}
+% \begin{macro}[EXP]{\@@_expand_unexpanded_test:w}
+% \begin{macro}[EXP]{\@@_expand_unexpanded:N}
+% \begin{macro}[EXP]{\@@_expand_unexpanded:n}
 %   After precautions against |&| tokens, start a simple loop: that of
 %   course means that \enquote{text} cannot contain the two recursion
 %   quarks. The loop here must be \texttt{f}-type expandable; we have
@@ -997,32 +999,61 @@
 %   Finally, expand any macros which can be: this then loops back around to
 %   deal with what they produce. The only issue is if the token is
 %   \cs{exp_not:n}, as that must apply to the following balanced text.
-%   Expand everything that follows such an \cs{exp_not:n} (in principle
-%   we should actually expand tokens and discard any spaces or \cs{scan_stop:},
-%   but this is messy to ensure safely).
 %    \begin{macrocode}
 \cs_new:Npn \@@_expand_cs_expand:N #1
   {
     \@@_if_expandable:NTF #1
       {
         \token_if_eq_meaning:NNTF #1 \exp_not:n
-          {
-            \exp_after:wN \@@_expand_noexpand:w
-            \exp:w \exp_end_continue_f:w
-          }
+          { \@@_expand_unexpanded:w }
           { \exp_after:wN \@@_expand_loop:w #1 }
       }
       {
         \@@_expand_store:n {#1}
         \@@_expand_loop:w
       }
   }
-\cs_new:Npn \@@_expand_noexpand:w #1#
-  { \@@_expand_noexpand:nn {#1} }
-\cs_new:Npn \@@_expand_noexpand:nn #1#2
+%    \end{macrocode}
+%   Since \cs{exp_not:n} is actually a primitive, it allows a strange syntax
+%   and it particular the primitive expands what follows and discards spaces
+%   and \cs{scan_stop:} until finding a braced argument (the opening brace
+%   can be implicit but we will not support this here).  Here, we repeatedly
+%   |f|-expand after such an \cs{exp_not:n}, and test what follows.  If
+%   it is a brace group, then we found the intended argument of
+%   \cs{exp_not:n}.  If it is a space, then the next |f|-expansion will
+%   eliminate it.  If it is an |N|-type token then
+%   \cs{@@_expand_unexpanded:N} leaves the token to be expanded if it is
+%   expandable, and otherwise removes it, assuming that it is
+%   \cs{scan_stop:}.  This silently hides errors when \cs{exp_not:n} is
+%   incorrectly followed by some non-expandable token other than
+%   \cs{scan_stop:}, but this should be pretty rare, and there is no good
+%   error recovery anyways.
+%    \begin{macrocode}
+\cs_new:Npn \@@_expand_unexpanded:w
+  {
+    \exp_after:wN \@@_expand_unexpanded_test:w
+    \exp:w \exp_end_continue_f:w
+  }
+\cs_new:Npn \@@_expand_unexpanded_test:w #1 \q_@@_recursion_stop
   {
-    \exp_after:wN \@@_expand_store:n \exp_after:wN
-      { \__kernel_exp_not:w #1 {#2} }
+    \tl_if_head_is_group:nTF {#1}
+      { \@@_expand_unexpanded:n }
+      {
+        \@@_expand_unexpanded:w
+        \tl_if_head_is_N_type:nT {#1} { \@@_expand_unexpanded:N }
+      }
+    #1 \q_@@_recursion_stop
+  }
+\cs_new:Npn \@@_expand_unexpanded:N #1
+  {
+    \exp_after:wN \if_meaning:w \exp_not:N #1 #1
+    \else:
+      \exp_after:wN #1
+    \fi:
+  }
+\cs_new:Npn \@@_expand_unexpanded:n #1
+  {
+    \@@_expand_store:n {#1}
     \@@_expand_loop:w
   }
 %    \end{macrocode}
@@ -1058,6 +1089,8 @@
 % \end{macro}
 % \end{macro}
 % \end{macro}
+% \end{macro}
+% \end{macro}
 %
 % \begin{macro}
 %   {

l3kernel/testfiles/m3text001.lvt

@@ -87,6 +87,7 @@
   {
     \tl_show:x { \text_expand:n { \exp_not:n { ~ \abc \foo } } }
     \tl_show:x { \text_expand:n { \tl_tail:n { ~ \abc \foo } } }
+    \tl_show:x { \text_expand:n { \exp_not:n \scan_stop: \use:n { ~ \use_none:n } { X } { \foo } } }
   }
 
 \END

l3kernel/testfiles/m3text001.tlg

@@ -60,4 +60,7 @@ l. ...  }
 > \foo .
 <recently read> }
 l. ...  }
+> \foo .
+<recently read> }
+l. ...  }
 ============================================================