diff --git a/classprims.go b/classprims.go index 9da7766..7441fd3 100644 --- a/classprims.go +++ b/classprims.go @@ -243,8 +243,15 @@ const LaTeX2eClassLead = ` \expandafter\def\csname[ \endcsname{\relax\ifmmode\@badmath\else$$\fi} \expandafter\def\csname] \endcsname{\relax\ifmmode$$\else\@badmath\fi} % ── generic list (best-effort: each item on its own line with its label) ───── -\def\list#1#2{\par} -\def\endlist{\par} +\def\list#1#2{\@trivlist} +% \endlist ends the innermost list by ending its trivlist, as ltlists.dtx does — +% \def\endlist{\global\advance\@listdepth\m@ne \endtrivlist}. That chain is what a +% class hooks: beamer patches \endtrivlist to run \beamer@closeitem, which closes the +% overlay wrappers (\begin{actionenv}\begin{uncoverenv}\begin{altenv}) that its LAST +% \item left open — every earlier item is closed by the NEXT \item. With \endlist a bare +% \par those three stayed open past \end{itemize}, and every \end after them closed one +% group too high. +\def\endlist{\endtrivlist} % \trivlist opens a group so a real class's redefined \trivlist (amsart's % \maketitle author block: \trivlist … \item\relax … \endtrivlist, which calls % \@trivlist) contains its material and CLOSES cleanly at \endtrivlist instead of diff --git a/counters.go b/counters.go index 3d20da2..2f10687 100644 --- a/counters.go +++ b/counters.go @@ -42,16 +42,16 @@ func (e *Engine) doNewcounter() { code = m.code // already allocated (\newcounter of an existing counter): reuse } else if e.allocCnt < 256 { code = e.allocCnt - e.define(ctr, &meaning{kind: mCountRef, code: code}, false) + e.define(ctr, &meaning{kind: mCountRef, code: code}, true) e.allocCnt++ } // \the := \arabic{name}, the LaTeX default representation. body := append([]tok{csTok("arabic"), chTok('{', catBegin)}, stringToToks(name)...) body = append(body, chTok('}', catEnd)) - e.define("the"+name, &meaning{kind: mMacro, body: body}, false) + e.define("the"+name, &meaning{kind: mMacro, body: body}, true) // An empty reset list, so \stepcounter{name} always has one to run. if e.eq["cl@"+name] == nil { - e.define("cl@"+name, &meaning{kind: mMacro}, false) + e.define("cl@"+name, &meaning{kind: mMacro}, true) } if hasWithin && code >= 0 { if within := strings.TrimSpace(e.toksToString(withinToks)); within != "" { diff --git a/engine.go b/engine.go index 4451966..bc8e7cf 100644 --- a/engine.go +++ b/engine.go @@ -168,12 +168,13 @@ type Engine struct { footnoteCounter int pendingFootnotes []*boxNode buildingFootnote bool - noBase bool // when true, getNext does not fall through to the base string - negateNextIf int // pending \unless prefixes (e-TeX): reverse the next conditional - allocCnt int // next free \count register handed out by \newcount - allocDim int // next free \dimen register handed out by \newdimen - allocSkp int // next free \skip register handed out by \newskip - allocBox int // next free \box register handed out by \newsavebox + levels []mouthLevel // the input levels below this one (see pushInputLevel) + noBase bool // when true, getNext does not fall through to the base string + negateNextIf int // pending \unless prefixes (e-TeX): reverse the next conditional + allocCnt int // next free \count register handed out by \newcount + allocDim int // next free \dimen register handed out by \newdimen + allocSkp int // next free \skip register handed out by \newskip + allocBox int // next free \box register handed out by \newsavebox // token registers (see toks.go): \toks / \newtoks-allocated registers store // a token list each. A class's title/mark machinery (amsart's \andify, \toks@, @@ -239,8 +240,6 @@ type Engine struct { uccode map[rune]int // \uccode: what \uppercase maps a character to afterGroup [][]tok // \aftergroup tokens, one list per open group xpEnvArgs [][][]tok // xparse \NewDocumentEnvironment arguments, one frame per open such environment - inputNL []int // newlines of each \input file still being read (see endInput) - loadedNL int // newlines in fully-loaded class/package files, subtracted from the document's source lines (see setSrcPos) // runaway guard: a bound on macro expansion so a pathological input (an // infinite \def loop, or a tolerantly-skipped arg-consuming command that @@ -462,8 +461,7 @@ func (e *Engine) Run(src string) (string, error) { e.bpos = 0 e.progBpos = 0 // fresh document: reset the no-progress guard e.noProgSteps = 0 // (e.bpos is monotonic within a Run, so this is a clean baseline) - e.loadedNL = 0 // fresh document: no loaded-file lines discounted yet - e.inputNL = nil + e.levels = nil // fresh document: no file is open e.afterGroup = nil e.buildLineStarts() e.mainLoop() @@ -481,25 +479,37 @@ func (e *Engine) push(ts []tok) { } // getNext returns the next raw token (no expansion), or ok=false at end. +// +// It reads the pending token lists first, then the current input LEVEL's character +// buffer, and when that runs out it pops back to the level underneath (tex.web §537 +// end_file_reading). The pop is what makes a file a level of its own: the file is +// finished before whatever was being read when it was opened resumes. func (e *Engine) getNext() (tok, bool) { - for len(e.lists) > 0 { - top := e.lists[len(e.lists)-1] - if len(top) == 0 { - e.lists = e.lists[:len(e.lists)-1] - continue + for { + for len(e.lists) > 0 { + top := e.lists[len(e.lists)-1] + if len(top) == 0 { + e.lists = e.lists[:len(e.lists)-1] + continue + } + t := top[0] + if rest := top[1:]; len(rest) == 0 { + e.lists = e.lists[:len(e.lists)-1] // drop the drained list eagerly so + } else { // len(e.lists) reflects real nesting + e.lists[len(e.lists)-1] = rest + } + return t, true } - t := top[0] - if rest := top[1:]; len(rest) == 0 { - e.lists = e.lists[:len(e.lists)-1] // drop the drained list eagerly so - } else { // len(e.lists) reflects real nesting - e.lists[len(e.lists)-1] = rest + if e.noBase { + return tok{}, false + } + if t, ok := e.scan(); ok { + return t, true + } + if !e.popInputLevel() { + return tok{}, false } - return t, true - } - if e.noBase { - return tok{}, false } - return e.scan() } // back pushes a single token back onto the input. diff --git a/envgroup_test.go b/envgroup_test.go new file mode 100644 index 0000000..39bdfd5 --- /dev/null +++ b/envgroup_test.go @@ -0,0 +1,214 @@ +// Copyright (c) the go-tex/engine authors. +// SPDX-License-Identifier: BSD-3-Clause + +package engine + +import ( + "strings" + "testing" +) + +// \begin{env} … \end{env} is a GROUP. ltmiscen.dtx: +// +// \protected\def\begin#1{… \begingroup\@endpefalse\reserved@a} +// where \reserved@a is \def\@currenvir{#1}… \csname #1\endcsname +// \def\end#1{\csname end#1\endcsname\@checkend{#1}\expandafter\endgroup …} +// +// so \begingroup comes first and everything the environment defines — \@currenvir +// included — is local to it. + +func TestEnvironmentIsAGroup(t *testing.T) { + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\def\x{dehors}\newenvironment{env}{}{}` + + `\begin{env}\def\x{dedans}\message{[\x]}\end{env}\message{[\x]}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[dedans] [dehors]" { + t.Errorf("= %q, want a definition made inside the environment to end with it", got) + } +} + +// \@currenvir names the environment being run and is restored when it ends. beamer +// picks between \begin{frame}…\end{frame} and the command form \frame{…} with +// \ifx\@currenvir\beamer@frametext, so a stale value sends the command form down the +// environment path. +func TestCurrentEnvIsRestored(t *testing.T) { + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\newenvironment{env}{}{}\newenvironment{autre}{}{}` + + `\message{[\@currenvir]}` + + `\begin{env}\message{[\@currenvir]}\begin{autre}\message{[\@currenvir]}\end{autre}` + + `\message{[\@currenvir]}\end{env}\message{[\@currenvir]}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[] [env] [autre] [env] []" { + t.Errorf("= %q, want \\@currenvir to follow the nesting", got) + } +} + +// A class may leave an environment early by closing its group itself, and then read +// \@currenvir to find it is no longer inside. beamer's fragile frame does exactly +// that: \beamer@checkforfragile ends with \endgroup% end environment, then calls +// \frame — which must take the COMMAND path. +func TestClosingTheGroupLeavesTheEnvironment(t *testing.T) { + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\newenvironment{env}{\endgroup\message{[\@currenvir]}}{}` + + `\begin{env}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[]" { + t.Errorf("= %q, want \\endgroup to leave the environment", got) + } +} + +// Allocation is GLOBAL. ltplain.dtx's \e@alloc ends with +// `\global#2#6\allocationnumber`, and ltcounts.dtx's \@definecounter makes \cl@, +// \p@ and \the global too — so a counter declared inside an environment (or +// inside \begin{document}, which is one) is still there afterwards. \setcounter is +// global as well; the register's VALUE follows TeX's ordinary scoping, so the +// assignment here is explicitly \global. +func TestAllocationSurvivesTheEnvironment(t *testing.T) { + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\newenvironment{env}{}{}` + + `\begin{env}\newcounter{compte}\newcount\reg \global\reg=7 \setcounter{compte}{4}\end{env}` + + `\message{[\arabic{compte}][\the\reg]}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[4][7]" { + t.Errorf("= %q, want the counter and the register to outlive the environment", got) + } +} + +// Every alignment entry is a group (tex.web §791: a template's u-part and v-part are +// inserted inside braces), so a font switch in one cell stops at that cell. +func TestAlignmentCellIsAGroup(t *testing.T) { + e, err := buildEngine(Options{Lenient: true}, true) + if err != nil { + t.Fatalf("buildEngine: %v", err) + } + if _, err := e.Run(`\hsize=300pt\begin{tabular}{ll}` + + `\bfseries A & B \\` + + `\end{tabular}`); err != nil { + t.Fatalf("Run: %v", err) + } + // The table must have been built: a cell group that leaked would have been + // reported as a stray brace by the box builder. + if d := e.Diagnostics(); d.OpenGroups != 0 { + t.Errorf("a tabular left %d group(s) open", d.OpenGroups) + } +} + +// An environment the engine implements in Go swallows its own \end, so \end — and +// with it the \endgroup — never runs. Each such environment closes the group itself; +// this checks the whole family at once. +func TestGoSideEnvironmentsCloseTheirGroup(t *testing.T) { + for _, c := range []struct{ nom, src string }{ + {"tabular", `\begin{tabular}{ll}A & B\\\end{tabular}`}, + {"tabularx", `\begin{tabularx}{200pt}{lX}A & B\\\end{tabularx}`}, + {"verbatim", "\\begin{verbatim}\nbrut\n\\end{verbatim}"}, + {"equation", `\begin{equation}x\end{equation}`}, + {"align", `\begin{align}x&=y\end{align}`}, + {"minipage", `\begin{minipage}{100pt}texte\end{minipage}`}, + {"comment", `\excludecomment{comment}\begin{comment}rien\end{comment}`}, + } { + e, err := buildEngine(Options{Lenient: true}, true) + if err != nil { + t.Fatalf("%s: buildEngine: %v", c.nom, err) + } + if _, err := e.Run(`\hsize=300pt` + c.src + `\message{[fin]}`); err != nil { + t.Fatalf("%s: Run: %v", c.nom, err) + } + if d := e.Diagnostics(); d.OpenGroups != 0 { + t.Errorf("%s left %d group(s) open", c.nom, d.OpenGroups) + } + } +} + +// A group left open by an environment shows up as text later: the check above reads +// the count, this one reads the consequence. +func TestAnEnvironmentDoesNotLeakItsScope(t *testing.T) { + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\newenvironment{env}{}{}\count0=1 ` + + `\begin{env}\count0=2 \end{env}\message{[\the\count0]}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); !strings.Contains(got, "[1]") { + t.Errorf("= %q, want the register assignment to end with the environment", got) + } +} + +// TeX's alignment scanner EXPANDS as it looks for & and \cr, so a class may split a +// table from inside it. NeurIPS's style does: +// +// \def\And{\end{tabular}\hfil\linebreak[0]\hfil\begin{tabular}[t]{c}…} +// +// used inside \begin{tabular}[t]{c}…\@author\end{tabular}, so \author{A \And B} carries +// an \end{tabular} two levels down — inside \@author, inside \And. Read raw, it never +// reached the body scanner and only surfaced while the CELL was being typeset, in the +// middle of a box: measured, a paper of 55 232 glyphs rendered 221. +func TestTabularSplitByAMacroCarryingItsEnd(t *testing.T) { + e, err := buildEngine(Options{Lenient: true}, true) + if err != nil { + t.Fatalf("buildEngine: %v", err) + } + out, err := e.Run(`\hsize=300pt` + + `\def\And{\end{tabular}\begin{tabular}{c}}` + + `\def\auteurs{A \And B}` + + `\begin{tabular}{c}\auteurs\end{tabular}` + + `\message{[suite]}`) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !strings.Contains(out, "[suite]") { + t.Errorf("= %q, want the document to carry on past the split table", trimNL(out)) + } + if d := e.Diagnostics(); d.OpenGroups != 0 { + t.Errorf("a table split by \\And left %d group(s) open", d.OpenGroups) + } +} + +// \endlist ends the list's trivlist, as ltlists.dtx does: +// +// \def\endlist{\global\advance\@listdepth\m@ne \endtrivlist} +// +// That chain is what a class hooks. beamer patches \endtrivlist to run +// \beamer@closeitem, which closes the overlay wrappers its LAST \item left open — +// every earlier item is closed by the next \item. With \endlist a bare \par those +// three environments stayed open past \end{itemize}, every \end after them closed one +// group too high, and the stack grew by two per slide: four lines of beamer left six +// groups open where TeX leaves none. +func TestListAndEndlistAreAGroupPair(t *testing.T) { + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\count0=1 \list{}{}\count0=2 \message{[\the\count0]}\endlist\message{[\the\count0]}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[2] [1]" { + t.Errorf("= %q, want \\list … \\endlist to scope what it contains", got) + } + if d := e.Diagnostics(); d.OpenGroups != 0 { + t.Errorf("\\list … \\endlist left %d group(s) open", d.OpenGroups) + } +} diff --git a/equation.go b/equation.go index 80bd903..3d5828f 100644 --- a/equation.go +++ b/equation.go @@ -52,6 +52,11 @@ func (e *Engine) doEquationBody() { row = append(row, box) } e.placeDisplay([]*boxNode{hpackSP(row, packTo, e.hsize)}) + // collectMathUntilEnd read this environment's own \end, so \end — and the + // \gotex@endenv that closes the group \begin opened — never runs. Close it here, + // AFTER the labels are recorded: \end closes the group only once \endequation + // has finished, and the reference metadata is part of that. + e.endEnvGroup() } // doEquationStar handles \begin{equation*} (an unnumbered single-line display): the @@ -77,6 +82,7 @@ func (e *Engine) doEquationStar(name string) { row = append(row, box) } e.placeDisplay([]*boxNode{hpackSP(row, packTo, e.hsize)}) + e.endEnvGroup() // see doEquationBody } // eqNumberBox builds the number box for a display line: the \tag (parenthesised diff --git a/halign.go b/halign.go index 59bc75d..a475294 100644 --- a/halign.go +++ b/halign.go @@ -131,7 +131,15 @@ func (e *Engine) buildCellHList(toks []tok) []node { seq = append(seq, toks...) seq = append(seq, chTok('}', catEnd)) // sentinel to terminate buildBoxList e.push(seq) + // The sentinel closes a real group, as the cell's own braces would: TeX makes + // every alignment entry a group (tex.web §791 — the u-part and v-part of a + // template are inserted inside braces), so a font or colour switch in a cell + // stops at the cell. Without the group the sentinel was a STRAY brace, which + // the stomach reports the moment any \begingroup is open — and \begin{env} now + // opens one. + e.beginGroupKind(boxGroup) list := e.buildBoxList() + e.endGroup() e.noBase = saved return list } diff --git a/inputlevels_test.go b/inputlevels_test.go new file mode 100644 index 0000000..12f47c6 --- /dev/null +++ b/inputlevels_test.go @@ -0,0 +1,136 @@ +// Copyright (c) the go-tex/engine authors. +// SPDX-License-Identifier: BSD-3-Clause + +package engine + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// tex.web §537: start_input does begin_file_reading, "set up cur_file and new level +// of input". The file is a LEVEL of its own — it is read to its end before whatever +// was being read when it was opened resumes. +// +// This mouth reads pending token lists before the character buffer, so a file +// merged into that buffer while a macro was still expanding arrived AFTER the rest +// of the macro's body. beamer's [fragile] frame ends by expanding +// \frame<*>[…]{\begingroup\input{\jobname.vrb}\endgroup}: the frame's body came back +// after the frame had closed. + +func TestInputFromAMacroIsReadBeforeTheRestOfIt(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "vu.tex"), []byte(`\gdef\vu{}`), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("TEXINPUTS", dir) + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\def\x{\input{vu.tex}\ifdefined\vu\message{[dans l'ordre]}\else\message{[trop tard]}\fi}\x`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[dans l'ordre]" { + t.Errorf("= %q, want the file read before the rest of the macro's body", got) + } +} + +// What the file leaves pending is read before the level it interrupted: the two +// belong to different levels, and the outer one only resumes when the file is done. +func TestWhatAFileLeavesPendingIsReadBeforeTheLevelUnderIt(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "suite.tex"), []byte(`\message{[a]}\suite`), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("TEXINPUTS", dir) + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\def\suite{\message{[b]}}\def\x{\input{suite.tex}\message{[c]}}\x`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[a] [b] [c]" { + t.Errorf("= %q, want [a] [b] [c]", got) + } +} + +// A file read from inside a file returns to the RIGHT level: the inner one finishes, +// then the outer one, then the document. +func TestNestedInputReturnsThroughEachLevel(t *testing.T) { + dir := t.TempDir() + for name, body := range map[string]string{ + "interne.tex": `\message{[interne]}`, + "externe.tex": `\message{[externe-debut]}\input{interne.tex}\message{[externe-fin]}`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + t.Setenv("TEXINPUTS", dir) + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\input{externe.tex}\message{[document]}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[externe-debut] [interne] [externe-fin] [document]" { + t.Errorf("= %q", got) + } +} + +// The no-progress guard measures forward progress by watching the mouth's position +// in the buffer it is reading. A new level starts that position again at 0, so its +// baseline travels with the level: without that, every file read after the first +// looked like an expansion loop that was getting nowhere, and long documents were +// cut off mid-way. +func TestALongFileAfterALongOneIsNotMistakenForALoop(t *testing.T) { + dir := t.TempDir() + long := strings.Repeat("mot mot mot mot mot\n", 400) + for _, n := range []string{"un.tex", "deux.tex"} { + if err := os.WriteFile(filepath.Join(dir, n), []byte(long+`\message{[`+n+`]}`), 0o644); err != nil { + t.Fatal(err) + } + } + t.Setenv("TEXINPUTS", dir) + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\input{un.tex}\input{deux.tex}\message{[fin]}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[un.tex] [deux.tex] [fin]" { + t.Errorf("= %q, want both files read to their end", got) + } +} + +// \InputIfFileExists runs its then-code BEFORE the file, which is what ltfiles.dtx +// does: \IfFileExists{#1}{#2\@addtofilelist{#1}\@@input\@filef@und}. A package +// announces itself, or sets what the file it is about to read expects to find. +func TestInputIfFileExistsRunsItsThenCodeFirst(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "apres.tex"), []byte(`\message{[fichier]}`), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("TEXINPUTS", dir) + e := New() + if err := e.LoadLaTeX(); err != nil { + t.Fatal(err) + } + out, err := e.Run(`\InputIfFileExists{apres.tex}{\message{[alors]}}{\message{[sinon]}}\message{[suite]}`) + if err != nil { + t.Fatal(err) + } + if got := trimNL(out); got != "[alors] [fichier] [suite]" { + t.Errorf("= %q", got) + } +} diff --git a/io.go b/io.go index 0778d0f..36ecb97 100644 --- a/io.go +++ b/io.go @@ -101,33 +101,80 @@ func (e *Engine) doInput() { e.spliceInputFile(data) } -// spliceInputFile puts a file's text into the input at the mouth's position, -// followed by a marker that ends it. EVERY way of reading a file in must use -// this: the marker is where \endinput stops (it means "stop reading THIS file", -// so a file spliced without one lets an \endinput inside it run on to the -// enclosing file's marker and swallow the rest of that file), and it is what -// discounts the file's lines from the document's own numbering, so an error or -// an editor's source map still points at the right line. The marker's name has -// no @ in it, since a file may be read in wherever the catcode of @ happens to -// be. -func (e *Engine) spliceInputFile(data []byte) { - body := normalizeEOL(string(data)) - e.inputNL = append(e.inputNL, strings.Count(body, "\n")) - insert := []rune(body + " \\gotexendinput ") // TeX appends a space at end of file - tail := append(insert, e.base[e.bpos:]...) - e.base = append(e.base[:e.bpos:e.bpos], tail...) - e.buildLineStarts() // the splice shifted every offset; rebuild the line table +// mouthLevel is one level of TeX's input stack. tex.web §537 start_input does +// begin_file_reading — "set up cur_file and new level of input" — and the file is +// read to its end before the level underneath resumes; §329 end_file_reading pops +// back. A level holds the character buffer being read, where the mouth stands in +// it, and the state of the level it interrupted. +type mouthLevel struct { + base []rune + bpos int + lineStarts []int + lists [][]tok + progBpos int + srcPos int + srcLine int + srcCol int } -// endInput discounts the lines of the \input file whose end the mouth has just -// reached, so the document's own source lines keep their numbers. -func (e *Engine) endInput() { - if n := len(e.inputNL); n > 0 { - e.loadedNL += e.inputNL[n-1] - e.inputNL = e.inputNL[:n-1] +// pushInputLevel makes text the current input, to be read to its end before +// anything that was pending resumes. +// +// The pending token lists go WITH the interrupted level: this mouth reads lists +// before the character buffer, so a file merged into the buffer while a macro was +// still expanding would arrive after the rest of that macro's body. That is not a +// corner case — it is how beamer renders a [fragile] frame: +// +// \frame<*>[…][{…,fragile=false}]{\begingroup\input{\jobname.vrb}\endgroup} +// +// The frame's body is written out verbatim and read back here; arriving late, it +// was typeset AFTER the frame had been contributed to the page, into a box nothing +// ever placed. Measured, the frame lost its body and its page. +// +// The no-progress guard watches e.bpos, which starts again at 0 in the new buffer, +// so its baseline is saved and reset with the level — otherwise every level below +// the first would look like an expansion loop making no headway. +func (e *Engine) pushInputLevel(text string) { + e.levels = append(e.levels, mouthLevel{ + base: e.base, bpos: e.bpos, lineStarts: e.lineStarts, lists: e.lists, + progBpos: e.progBpos, srcPos: e.srcPos, srcLine: e.curSrcLine, srcCol: e.curSrcCol, + }) + e.base, e.bpos, e.lists = []rune(text), 0, nil + e.lineStarts, e.progBpos, e.noProgSteps = nil, 0, 0 + e.buildLineStarts() +} + +// popInputLevel returns to the level under the current one, and reports whether +// there was one. Anything the finished level left pending stays on top of what the +// level underneath had pending, so an unbalanced file cannot strand it. +func (e *Engine) popInputLevel() bool { + n := len(e.levels) + if n == 0 { + return false } + l := e.levels[n-1] + e.levels = e.levels[:n-1] + e.base, e.bpos, e.lineStarts = l.base, l.bpos, l.lineStarts + e.lists = append(l.lists, e.lists...) + e.progBpos, e.noProgSteps = l.progBpos, 0 + e.srcPos, e.curSrcLine, e.curSrcCol = l.srcPos, l.srcLine, l.srcCol + return true } +// spliceInputFile reads a file in as a new input level, followed by a marker that +// ends it. EVERY way of reading a file in must use this: the marker is where +// \endinput stops (it means "stop reading THIS file"), and TeX appends a space at +// the end of a file. The marker's name has no @ in it, since a file may be read in +// wherever the catcode of @ happens to be. +func (e *Engine) spliceInputFile(data []byte) { + e.pushInputLevel(normalizeEOL(string(data)) + " \\gotexendinput ") +} + +// endInput is what the marker at the end of an \input file runs. The level pops on +// its own when the buffer is exhausted, one token later, so there is nothing left +// to do here — it stays a primitive because \endinput jumps to it. +func (e *Engine) endInput() {} + // scanFileName reads a filename: optional spaces, then either a {braced} name // (spaces allowed) or characters up to the next space / control sequence. // diff --git a/latex.go b/latex.go index 3ecf9fb..bd0ecf0 100644 --- a/latex.go +++ b/latex.go @@ -16,12 +16,33 @@ import "strings" // the Plain macros). const MiniLaTeXKernel = ` \catcode64=11 -% \@currenvir (set by \gotex@checkenv, see doCheckEnv) names the environment being -% opened. \begin stays fully EXPANDABLE — the engine relies on that, so the name is -% recorded from the Go side rather than with a \def here. +% \begin{env} … \end{env} is a GROUP. ltmiscen.dtx: +% +% \protected\def\begin#1{… \begingroup\@endpefalse\reserved@a} +% where \reserved@a is \def\@currenvir{#1}… \csname #1\endcsname +% \def\end#1{\csname end#1\endcsname\@checkend{#1}\expandafter\endgroup …} +% +% so \begingroup comes FIRST and \@currenvir is defined INSIDE it. Without the +% group, \@currenvir was never restored, and a class that leaves an environment +% early by closing its group could not: beamer's fragile frame does exactly that — +% +% \def\beamer@checkforfragile#1fragile#2\relax{… \endgroup% end environment +% \expandafter\beamer@framecommand\beamer@frameoptions\bgroup} +% +% and then calls \frame, which picks its syntax with \ifx\@currenvir\beamer@frametext. +% With \@currenvir still "frame" the command form took the ENVIRONMENT path a second +% time, and \beamer@doseveralframes was handed the bare \bgroup instead of the +% frame's body — the frame went on the page empty and its body was typeset after it, +% into a box nothing places. +% +% The group is opened and closed by the Go-side probes \gotex@checkenv and +% \gotex@endenv rather than by \begingroup/\endgroup tokens, because \begin and +% \end must stay fully EXPANDABLE here: written as tokens, they would print as +% literal text wherever \begin is only expanded (inside \message or \edef). +% \@currenvir is defined inside that group (see doCheckEnv), as LaTeX's own \def is. \def\@currenvir{} \def\begin#1{\gotex@checkenv{#1}\csname #1\endcsname} -\def\end#1{\csname end#1\endcsname} +\def\end#1{\csname end#1\endcsname\gotex@endenv{#1}} \def\document{\catcode64=12 } \def\enddocument{\par\vfill\penalty-10000 } \def\rm{} @@ -1044,6 +1065,9 @@ func (e *Engine) gobbleEnvBody(name string, placeholder bool) { if placeholder { e.emitPicturePlaceholder(name) } + // This \end was swallowed here, so \end — and the \gotex@endenv + // that closes the group \begin opened — never runs. + e.endEnvGroup() return } } diff --git a/lengths.go b/lengths.go index 9f5ee0c..b584a70 100644 --- a/lengths.go +++ b/lengths.go @@ -45,7 +45,7 @@ func (e *Engine) doNewlength() { e.fail("No room for a new \\skip (\\newlength)") return } - e.define(target.cs, &meaning{kind: mSkipRef, code: e.allocSkp}, false) + e.define(target.cs, &meaning{kind: mSkipRef, code: e.allocSkp}, true) // global, see doNewcount e.allocSkp++ } diff --git a/listings.go b/listings.go index 1e7c32a..f0372de 100644 --- a/listings.go +++ b/listings.go @@ -105,6 +105,7 @@ func (e *Engine) doLstlisting() { content = rest[:idx] e.bpos += len([]rune(rest[:idx])) + len([]rune(end)) } + e.endEnvGroup() // this environment reads its own \end, so it closes \begin's group leadingNL := strings.HasPrefix(content, "\n") content = strings.TrimPrefix(content, "\n") content = strings.TrimSuffix(content, "\n") diff --git a/mathalign.go b/mathalign.go index 5276875..e471c38 100644 --- a/mathalign.go +++ b/mathalign.go @@ -291,6 +291,7 @@ func (e *Engine) collectAlignBody(name string) []alignRow { case depth == 0 && t.cs_ && t.cs == "end": if e.readBraceName() == name { endRow() + e.endEnvGroup() // this \end was read here, so \end's \endgroup will not run return rows } case depth == 0 && t.cs_ && t.cs == `\`: // \\ row separator diff --git a/minipage.go b/minipage.go index 7702ea3..4554b8e 100644 --- a/minipage.go +++ b/minipage.go @@ -62,7 +62,8 @@ func (e *Engine) collectEnvBody(name string) []tok { n := e.readBraceName() switch { case n == name && depth == 0: - return body // the matching \end{name}: consume it and stop + e.endEnvGroup() // this \end was read here, so \end's \endgroup will not run + return body // the matching \end{name}: consume it and stop case n == name: depth-- // a nested instance closes; re-emit for the nested typeset body = append(body, csTok("end")) diff --git a/packages.go b/packages.go index 87c239d..482aac8 100644 --- a/packages.go +++ b/packages.go @@ -25,7 +25,6 @@ import ( type loadFrame struct { atcat cat // catcode of @ to restore when the file ends name string // package/class base name (for the loaded registry + \CurrentOption) - nlCount int // newlines in the spliced file (added to loadedNL when it ends) endHook string // \@endofpackagehook / \@endofclasshook to reset after the file passed []string // options requested for this file declared map[string][]tok // \DeclareOption{name}{code} @@ -225,7 +224,6 @@ func (e *Engine) loadTeXFile(data []byte, name, ext string, passed []string) { // the engine treats only \n as end-of-line, so a CRLF file (e.g. a .cls checked // out on Windows) would otherwise typeset stray \r characters. body := normalizeEOL(string(data)) - e.loadStack[len(e.loadStack)-1].nlCount = strings.Count(body, "\n") // The 2020 format lets a package register code to run around ANOTHER file's // loading: \AddToHook{package/amsmath/after} (beamer's overlay layer does exactly // this) and \AddToHook{file/.sty/before}. Fire those four hooks around the @@ -245,10 +243,7 @@ func (e *Engine) loadTeXFile(data []byte, name, ext string, passed []string) { // input: reading it BEFORE splicing the file would put the token that is not a // "[" back on the token stack, ahead of the file about to be spliced into the // character buffer — which silently swallowed everything after the \usepackage. - insert := []rune(pre + body + "\\" + endHook + post + "\\@gotex@endload \\gotexeatdate ") - tail := append(insert, e.base[e.bpos:]...) - e.base = append(e.base[:e.bpos:e.bpos], tail...) - e.buildLineStarts() + e.pushInputLevel(pre + body + "\\" + endHook + post + "\\@gotex@endload \\gotexeatdate ") } // normalizeEOL converts CRLF and lone CR line endings to LF. The engine's mouth @@ -271,10 +266,6 @@ func (e *Engine) endLoad() { fr := e.loadStack[len(e.loadStack)-1] e.loadStack = e.loadStack[:len(e.loadStack)-1] e.catcode['@'] = fr.atcat - // The file's lines are now behind the mouth: exclude them from the document's - // source-line numbering (see setSrcPos), so loading a class does not shift the - // lines the editor/error reporter attributes to the user's own document. - e.loadedNL += fr.nlCount if fr.endHook != "" { e.define(fr.endHook, &meaning{kind: mMacro}, true) // \let\@endof…hook\@empty } @@ -733,8 +724,18 @@ func (e *Engine) readBraceNameX() string { } // doInputIfFileExists implements \InputIfFileExists{name}{then}{else}: when name -// resolves it splices the file (with the then-code queued to run after it), else it -// runs the else-code. +// resolves it runs the then-code and then reads the file, else it runs the +// else-code. +// +// The order is LaTeX's, and it is that way round. ltfiles.dtx: +// +// \long\def\InputIfFileExists#1#2{% +// \IfFileExists{#1}% +// {#2\@addtofilelist{#1}\@@input \@filef@und}} +// +// so #2 runs BEFORE the file is read — a package announces itself, or sets what the +// file it is about to read expects to find. The then-code is pushed onto the file's +// own level, where it is read before the file's text and cannot outlive it. func (e *Engine) doInputIfFileExists() { name := e.readBraceNameX() then := e.readBraceToksRaw() @@ -744,8 +745,6 @@ func (e *Engine) doInputIfFileExists() { e.push(els) return } - if len(then) > 0 { - e.push(then) - } e.spliceInputFile(data) + e.push(then) } diff --git a/primitives.go b/primitives.go index 120e4b5..49a812d 100644 --- a/primitives.go +++ b/primitives.go @@ -51,6 +51,7 @@ var expandableSet = map[string]bool{ // \gotex@checkenv acts in the gullet exactly like \csname (into which \begin // feeds), so \begin keeps expanding cleanly wherever it did before. "gotex@checkenv": true, + "gotex@endenv": true, } func isExpandable(name string) bool { return expandableSet[name] } @@ -386,7 +387,9 @@ func (e *Engine) doNewcount() { if name == "" || e.allocCnt >= 256 { return } - e.define(name, &meaning{kind: mCountRef, code: e.allocCnt}, false) + // Allocation is GLOBAL: ltplain.dtx's \e@alloc ends with `\global#2#6\allocationnumber`, + // so \newcount\x inside a group leaves \x a register afterwards. + e.define(name, &meaning{kind: mCountRef, code: e.allocCnt}, true) e.allocCnt++ } @@ -459,7 +462,7 @@ func (e *Engine) doNewdimen() { if name == "" || e.allocDim >= 256 { return } - e.define(name, &meaning{kind: mDimenRef, code: e.allocDim}, false) + e.define(name, &meaning{kind: mDimenRef, code: e.allocDim}, true) // global, see doNewcount e.allocDim++ } @@ -513,7 +516,7 @@ func (e *Engine) doNewskip() { if name == "" || e.allocSkp >= 256 { return } - e.define(name, &meaning{kind: mSkipRef, code: e.allocSkp}, false) + e.define(name, &meaning{kind: mSkipRef, code: e.allocSkp}, true) // global, see doNewcount e.allocSkp++ } @@ -855,6 +858,7 @@ func (e *Engine) doCheckEnv() { if name == "" { return // no braced argument, or an empty one: nothing meaningful to probe } + e.beginGroupKind(semiSimpleGroup) // \begin{env} … \end{env} is a group e.setCurrentEnv(name) if e.envUndefined(name) { if e.undefinedEnvs == nil { @@ -885,6 +889,29 @@ func (e *Engine) setCurrentEnv(name string) { e.define("@currenvir", &meaning{kind: mMacro, body: body}, false) } +// doEndEnv implements \gotex@endenv{env}, the probe \end runs after \end: it +// closes the group \begin opened (ltmiscen.dtx ends \end with \endgroup). Like +// \gotex@checkenv it produces no tokens and runs in the gullet, so \end stays +// expandable. +func (e *Engine) doEndEnv() { + if e.readBraceName() == "" { + return + } + e.endEnvGroup() +} + +// endEnvGroup closes the group \begin{env} opened, for an environment the engine +// implements in Go and which therefore swallows its own \end{env} instead of letting +// \end run (see ltmiscen.dtx: \begin ends with \begingroup, \end with \endgroup). +// Without it such an environment leaves a group open for the rest of the document. +// +// It closes whatever the innermost group is, as \endgroup does — including, through +// closeSemiSimple's "Missing } inserted" recovery, a box. beamer DEPENDS on that: its +// frame is \global\setbox\beamer@framebox=\vbox\bgroup …, and the \end that ends the +// frame meets that box. Guarding this to refuse a box, or to close only the group its +// own \begin opened, costs 140 pages over 200 talks — both were measured. +func (e *Engine) endEnvGroup() { e.closeSemiSimple() } + // envUndefined reports whether \name is not a real environment's opening control // sequence. It is true when \name has no meaning at all AND when it is \relax: // \csname coerces a missing \env to \relax the FIRST time \begin{env} runs (and @@ -1537,11 +1564,12 @@ func (e *Engine) loadMore() { e.grabUndelimited() e.grabUndelimited() }) - e.prim("[", func(e *Engine) { e.doDelimitedMath("]", true) }) // \[ … \] display math - e.prim("(", func(e *Engine) { e.doDelimitedMath(")", false) }) // \( … \) inline math - e.prim("]", func(e *Engine) {}) // consumed by \[ - e.prim(")", func(e *Engine) {}) // consumed by \( - e.prim("gotex@checkenv", func(e *Engine) { e.doCheckEnv() }) // \begin's undefined-environment probe (see doCheckEnv) + e.prim("[", func(e *Engine) { e.doDelimitedMath("]", true) }) // \[ … \] display math + e.prim("(", func(e *Engine) { e.doDelimitedMath(")", false) }) // \( … \) inline math + e.prim("]", func(e *Engine) {}) // consumed by \[ + e.prim(")", func(e *Engine) {}) // consumed by \( + e.prim("gotex@checkenv", func(e *Engine) { e.doCheckEnv() }) + e.prim("gotex@endenv", func(e *Engine) { e.doEndEnv() }) // \begin's undefined-environment probe (see doCheckEnv) e.prim("@equationbody", func(e *Engine) { e.doEquationBody() }) // \begin{equation} body + number e.prim("equation*", func(e *Engine) { e.doEquationStar("equation*") }) e.prim("endequation*", func(e *Engine) {}) diff --git a/srcmap.go b/srcmap.go index 1427d5f..b36782b 100644 --- a/srcmap.go +++ b/srcmap.go @@ -36,14 +36,13 @@ func (e *Engine) buildLineStarts() { // input, updating the cached (line, column). Line is 1-based, column 0-based. func (e *Engine) setSrcPos(pos int) { e.srcPos = pos - line, col := e.lineColAt(pos) - // Discount the lines of already-loaded class/package files (spliced into the - // base ahead of this position) so the reported line is the user's own document - // line — loading article.cls must not shift where a glyph says it came from. - if line -= e.loadedNL; line < 1 { - line = 1 - } - e.curSrcLine, e.curSrcCol = line, col + // A loaded file is an input LEVEL of its own (see pushInputLevel), not text + // merged into the document's buffer, so its lines never shift the document's: + // the position is read straight off the level being read. Inside a class or + // package that means the line in THAT file, which is where a diagnostic about + // it belongs; the document's own position is saved with the level and comes + // back when the file ends. + e.curSrcLine, e.curSrcCol = e.lineColAt(pos) } // lineColAt converts a base-input rune offset into a 1-based line and 0-based diff --git a/tabular.go b/tabular.go index e39aa19..2dae92b 100644 --- a/tabular.go +++ b/tabular.go @@ -621,16 +621,27 @@ func (e *Engine) collectTabularBody(env string) []tabItem { // would. Only at brace depth 0: a conditional nested inside a cell's {…} // stays raw and is evaluated when that cell is typeset. e.eq[t.cs].prim(e) - case depth == 0 && t.cs_ && t.cs != "end" && e.expandsToEnd(t): - // A user macro standing in for \end{...} (e.g. \newcommand\etab{\end{tabular}}): - // read raw here it hides the \end, so the body scanner would run to EOF and - // swallow the rest of the document. Expand it in place so the real \end token - // surfaces next iteration. Narrow: only a parameterless macro whose body - // begins with \end (see expandsToEnd), so verbatim cell content is untouched. + case depth == 0 && t.cs_ && t.cs != "end" && e.hidesTabularEnd(t): + // A macro that hides this environment's \end: read raw it never surfaces, so + // the body scanner runs to EOF and swallows the rest of the document. Expand + // it in place and the \end shows up on the next iteration. + // + // TeX's own alignment scanner EXPANDS as it looks for & and \cr, which is why + // a class may split a table from inside it. NeurIPS's style does: + // + // \def\And{\end{tabular}\hfil\linebreak[0]\hfil\begin{tabular}[t]{c}…} + // + // used inside \begin{tabular}…\@author\end{tabular}, so \author{A \And B} + // carries an \end{tabular} two levels down — inside \@author, inside \And. + // Read raw, it only surfaced while the CELL was being typeset, in the middle + // of a box (issue #106). e.expandMacro(e.meaningOf(t)) case depth == 0 && t.cs_ && t.cs == "end": if name := e.readBraceName(); name == env { endRow() + // The scanner read this environment's own \end, so \end — and the + // \gotex@endenv that closes \begin's group — never runs: close it here. + e.endEnvGroup() return items } case depth == 0 && t.cs_ && t.cs == "hline": @@ -675,6 +686,28 @@ func (e *Engine) collectTabularBody(env string) []tabItem { return items } +// hidesTabularEnd reports whether the control sequence t is a parameterless macro +// that leads to this environment's \end, directly or through ONE more parameterless +// macro. Two levels is what the reference case needs (\@author → \And → \end) and it +// keeps the check as narrow as expandsToEnd's: no parameters anywhere on the path, and +// the \end must be the first token of the macro that carries it — so an ordinary +// content macro, or a verbatim cell that merely mentions \end, is never expanded. +func (e *Engine) hidesTabularEnd(t tok) bool { + if e.expandsToEnd(t) { + return true + } + m := e.meaningOf(t) + if m == nil || m.kind != mMacro || len(m.params) != 0 { + return false + } + for _, b := range m.body { + if b.cs_ && e.expandsToEnd(b) { + return true + } + } + return false +} + // readBraceName reads a {name} group and returns its text (used for \end{name}). func (e *Engine) readBraceName() string { e.skipOptSpace() diff --git a/theorem.go b/theorem.go index f54b7ad..01b3978 100644 --- a/theorem.go +++ b/theorem.go @@ -51,7 +51,7 @@ func (e *Engine) doNewtheorem() { // Allocate a fresh \count register, exactly as \newcount would. if e.allocCnt < 256 { ctrCode = e.allocCnt - e.define(ctr, &meaning{kind: mCountRef, code: ctrCode}, false) + e.define(ctr, &meaning{kind: mCountRef, code: ctrCode}, true) e.allocCnt++ } } @@ -66,7 +66,7 @@ func (e *Engine) doNewtheorem() { } else { numBody = []tok{csTok("the"), csTok(ctr)} } - e.define("the"+env, &meaning{kind: mMacro, body: numBody}, false) + e.define("the"+env, &meaning{kind: mMacro, body: numBody}, true) // A within-numbered environment resets on its parent counter's step. if hasWithin && !hasShared && ctrCode >= 0 { @@ -87,10 +87,10 @@ func (e *Engine) doNewtheorem() { } body = append(body, head...) body = append(body, chTok('}', catEnd), chTok('{', catBegin), csTok("the"+env), chTok('}', catEnd)) - e.define(env, &meaning{kind: mMacro, body: body}, false) + e.define(env, &meaning{kind: mMacro, body: body}, true) // \end: the fixed closing macro (end paragraph, close group, vertical space). - e.define("end"+env, &meaning{kind: mMacro, body: []tok{csTok("@endtheorem")}}, false) + e.define("end"+env, &meaning{kind: mMacro, body: []tok{csTok("@endtheorem")}}, true) } // addToReset arranges for the count register ctrCode to be zeroed whenever the diff --git a/verbatim.go b/verbatim.go index 8a63879..9d12446 100644 --- a/verbatim.go +++ b/verbatim.go @@ -28,6 +28,7 @@ func (e *Engine) doVerbatim() { content = rest[:idx] e.bpos += len([]rune(rest[:idx])) + len([]rune(end)) } + e.endEnvGroup() // this environment reads its own \end, so it closes \begin's group // Verbatim ignores the newline right after \begin{verbatim} and the one just // before \end{verbatim}. leadingNL := strings.HasPrefix(content, "\n")