avoid crashing on devious uses of \output and write braces, from DRF …

…(pdftex and xetex only)

git-svn-id: svn://tug.org/texlive/trunk/Build/source@70173 c570f23f-e606-0410-a88d-b1316a301751

texk/web2c/ChangeLog

@@ -1,3 +1,11 @@
+2024-02-26  Karl Berry  <karl@freefriends.org>
+
+	* unbalanced-braces.ch: avoid crashing with devious uses of
+	\output and \write braces. From DRF.
+	* pdftexdir/am/pdftex.am (pdftex_ch_srcs),
+	* xetexdir/am/xetex.am (xetex_ch_srcs): include unbalanced-braces.ch.
+	* tests/unbalanced-braces.test: doc file for tests (not runnable).
+
 2024-02-25  Karl Berry  <karl@freefriends.org>
 
 	* locnull-optimize.ch: inner loop optimization (removing one

texk/web2c/Makefile.in

@@ -4736,6 +4736,7 @@ pdftex_ch_srcs = \
 	enctexdir/enctex1.ch \
 	enctexdir/enctex-pdftex.ch \
 	enctexdir/enctex2.ch \
+	unbalanced-braces.ch \
 	$(pdftex_ch_synctex) \
 	pdftexdir/pdftex.ch \
 	pdftexdir/char-warning-pdftex.ch \
@@ -5432,6 +5433,7 @@ xetex_ch_srcs = \
 	partoken-102.ch \
 	partoken.ch \
 	locnull-optimize.ch \
+	unbalanced-braces.ch \
 	showstream.ch \
 	$(xetex_ch_synctex) \
 	xetexdir/xetex.ch \

texk/web2c/pdftexdir/am/pdftex.am

@@ -79,6 +79,7 @@ pdftex_ch_srcs = \
 	enctexdir/enctex1.ch \
 	enctexdir/enctex-pdftex.ch \
 	enctexdir/enctex2.ch \
+	unbalanced-braces.ch \
 	$(pdftex_ch_synctex) \
 	pdftexdir/pdftex.ch \
 	pdftexdir/char-warning-pdftex.ch \

texk/web2c/tests/unbalanced-braces.test

@@ -0,0 +1,73 @@
+$Id$
+
+This is (clearly) not an actual test file that can be run.
+It exists to record some of the nefarious input files that caused the
+problems that unbalanced-braces.ch tries to resolve.
+
+% https://tug.org/pipermail/tex-k/2022-July/003851.html
+% Tyge Tiessen
+\catcode`\{=1 \catcode`\}=2 \catcode`\#=6
+\outer\def\someouter{}
+\def\weird{\expandafter\weirda\expandafter{\iffalse}\fi}
+\def\weirda#1{\ifx}
+\immediate\write16{\weird}\someouter
+
+% https://tex.stackexchange.com/questions/609423
+% user202729
+\catcode`\{=1 \catcode`\}=2 \catcode`\#=6
+\def\weird{\expandafter\weirda\expandafter{\iffalse}\fi}
+\def\weirda#1{\expandafter\weirdb\noexpand}
+\def\weirdb#1{\iffalse{\fi}#1\edef\mycontainendwrite{\noexpand#1}}
+\immediate\write16{\weird}
+\expandafter\let\expandafter\myendwrite\mycontainendwrite
+\end
+
+% https://codegolf.stackexchange.com/questions/4399/shortest-code-that-raises-a-sigsegv/4409#4409
+% -> https://groups.google.com/g/comp.text.tex/c/SEHJUKtksU8
+% Bruno Le Floch
+\catcode`\{=1 \catcode`\}=2 \catcode`\#=6 \catcode`\~=13
+\def~#1{\meaning}\write0{\expandafter~\string}\end
+
+% https://topanswers.xyz/tex?q=5286#a5394
+% user202729
+\catcode`\{=1 \catcode`\}=2 \catcode`\#=6
+\outer\def\a{}
+\def\weird{\expandafter\weirda\string}
+\def\weirda#1{\ifx}
+\immediate\write-1{\weird}\a
+\end
+
+% (From drf)
+% Below is the only test file I didn't get directly from others; the thing
+% I like about it is that it shows the case where the \extra token of the
+% \output routine has been "backed_up" so it's no longer part of the
+% output_text when we get to <Resume the page builder after an output
+% routine has come to an end>.  (The parsing of "\box255" reads the
+% following token, and sees it's not a digit, so pushes it back.) And if
+% you remove the "\extra", you get the case where the actual right brace
+% in the output routine is backed_up, which is legal and expected, so
+% shouldn't be messed up by my changes.
+%
+\catcode`\{=1 \catcode`\}=2
+\def\makepage{
+ \hrule width 2in height 1in
+ \vskip 1in plus 10in
+ \hrule width 2in height 1in
+ \penalty-10000
+}
+\output{\global\advance\count0 by 1 \shipout\box255\extra}
+\let\lb={
+\let\rb=}
+\message{RELAX}
+\let\extra=\relax
+\makepage
+\message{DONE_RELAX}
+\message{EXTRA_RIGHT_BRACE}
+\let\extra=}
+\makepage
+\message{DONE_EXTRA_RIGHTBRACE}
+\message{EXTRA_LEFT_BRACE}
+\let\extra={
+\makepage
+\message{DONE_EXTRA_LEFT_BRACE}
+\end

texk/web2c/unbalanced-braces.ch

@@ -0,0 +1,141 @@
+% $Id$
+% Fix overrun/underrun of \write and \output. David Fuchs, 2024.
+% Public domain.
+%
+% Changes for full defense against over-running (or under-running)
+% an \output or \write and ending up in all sorts of ill-defined
+% trouble.  (Including the case of \output=\toks with no braces.)
+% 
+% For some problematic input files, see tests/unbalanced-braces.test
+% (not a runnable test).
+%
+% The idea is that when it's time to run/evaluate each \output or \write,
+% they have to come to an end exactly as expected: at the right brace that
+% came from when they were first scanned to begin with; no amount of
+% monkey business with "funny braces" or \noexpand or \expandafter or \let
+% or whatever other trick you can think of should be able to get around
+% it. And you can't get away with going even one token past where you were
+% supposed to stop.
+%
+% To reiterate, fatal errors from this result in the terminal show just:
+%   ! Emergency stop.
+% with no details on what/why.  You have to look in the .log file for
+% the (terse) specifics:
+%   Unbalanced output routine
+% 
+% It's rather draconian, calling fatal_error when there's a problem,
+% but users really have no business trying to get anywhere near this
+% sort of thing on purpose.  I sure hope nobody has found any use for
+% such undefined behaviors.
+
+% The changes assume locnull-optimize.ch has been applied. Unfortunately
+% the semantically-related changes have to be broken up to apply in
+% tex.web order.
+
+% emacs-page
+%% Catch extra left braces in \output right when finished scanning it.
+%
+@x [23.324] l.7000 p.B139
+    if token_type=macro then {parameters must be flushed}
+      while param_ptr>param_start do
+        begin decr(param_ptr);
+        flush_list(param_stack[param_ptr]);
+        end;
+@y
+    if token_type=macro then {parameters must be flushed}
+      while param_ptr>param_start do
+        begin decr(param_ptr);
+        flush_list(param_stack[param_ptr]);
+        end
+    else if (token_type=output_text)and(output_active) then
+      fatal_error("Unbalanced output routine");
+@.Unbalanced output routine@>
+@z
+
+%% Catch extra right braces in the \output routine.
+% <Resume the page builder...> tried, but needs to be more robust against
+% a backed-up right (funny) brace in the middle of \output masquerading
+% as the end-of-\output right brace.
+%
+% Reorder these so that end_token_list sees output_active=false.
+% 1) In back_input:
+@x [23.325] l.7025 p.B139
+begin while (loc=null)and(token_type<>v_template) do
+  end_token_list; {conserve stack space}
+@y
+begin while (loc=null)and(token_type<>v_template)
+            and(token_type<>output_text) do
+  end_token_list; {conserve stack space}
+@z
+
+%% Catch extra left braces finishing scanning a \write_text.
+%
+% In <Input from token list...> don't allow end_write while we're
+% still scanning through the write_text.
+@x [24.357] l.7488 p.B150
+      else check_outer_validity;
+@y
+      else
+        begin
+        if (cur_cs=end_write)and(mode=0) then
+          fatal_error("Unbalanced write command");
+        check_outer_validity;
+        end;
+@z
+
+%% Returning to catching extra right braces.
+%
+% 2) In <Feed the macro body and its parameters to the scanner>:
+@x [25.390] l.7983 p.B161
+while (loc=null)and(token_type<>v_template) do
+  end_token_list; {conserve stack space}
+@y
+while (loc=null)and(token_type<>v_template)
+      and(token_type<>output_text) do
+  end_token_list; {conserve stack space}
+@z
+
+% We know we've just scanned a right brace that seems to be the end
+% of the \output routine.  But maybe it had been backed-up over,
+% and we've lost the output_text in the call to back_input. So,
+% the checking gets sloppy.  But now we're sure to keep the
+% (finished) output_text level around, so we can always check
+% that we were just finished with it, so it's where the brace
+% came from.
+
+% In <Resume the page builder after an output routine has come to an end>:
+@x [45.1026] l.19938 p.B432
+begin if (loc<>null) or
+ ((token_type<>output_text)and(token_type<>backed_up)) then
+  @<Recover from an unbalanced output routine@>;
+@y
+begin
+  while (state=token_list)and(loc=null)and(token_type=backed_up) do
+    end_token_list; {output-ending brace may have been backed-up}
+  if (state<>token_list)or(loc<>null)or(token_type<>output_text) then
+    @<Recover from an unbalanced output routine@>;
+@z
+
+% In <Resume the page builder after an output routine has come to an end>:
+@x [45.1026] l.19941 p.B432
+end_token_list; {conserve stack space in case more outputs are triggered}
+end_graf; unsave; output_active:=false; insert_penalties:=0;@/
+@y
+end_graf; unsave; output_active:=false; insert_penalties:=0;@/
+end_token_list; {conserve stack space in case more outputs are triggered}
+@z
+
+% <Expand macros in the token list and...> had set mode:=0 while
+% expanding the \write token list.
+
+% Reorder these statements so that the final get_token that's supposed to
+% scan off the end_write_token will have mode<>0 if everything lined up:
+@x [53.1371] l.24884 p.B546
+get_token;@+if cur_tok<>end_write_token then
+  @<Recover from an unbalanced write command@>;
+mode:=old_mode;
+@y
+mode:=old_mode;
+get_token;@+if cur_tok<>end_write_token then
+  @<Recover from an unbalanced write command@>;
+@z

texk/web2c/xetexdir/am/xetex.am

@@ -117,6 +117,7 @@ xetex_ch_srcs = \
 	partoken-102.ch \
 	partoken.ch \
 	locnull-optimize.ch \
+	unbalanced-braces.ch \
 	showstream.ch \
 	$(xetex_ch_synctex) \
 	xetexdir/xetex.ch \