engine: a file is an input LEVEL, and \begin{env} is a group

classprims.go

@@ -243,8 +243,15 @@ const LaTeX2eClassLead = `
 \expandafter\def\csname[ \endcsname{\relax\ifmmode\@badmath\else$$\fi}
 \expandafter\def\csname] \endcsname{\relax\ifmmode$$\else\@badmath\fi}
 % ── generic list (best-effort: each item on its own line with its label) ─────
-\def\list#1#2{\par}
-\def\endlist{\par}
+\def\list#1#2{\@trivlist}
+% \endlist ends the innermost list by ending its trivlist, as ltlists.dtx does —
+% \def\endlist{\global\advance\@listdepth\m@ne \endtrivlist}. That chain is what a
+% class hooks: beamer patches \endtrivlist to run \beamer@closeitem, which closes the
+% overlay wrappers (\begin{actionenv}\begin{uncoverenv}\begin{altenv}) that its LAST
+% \item left open — every earlier item is closed by the NEXT \item. With \endlist a bare
+% \par those three stayed open past \end{itemize}, and every \end after them closed one
+% group too high.
+\def\endlist{\endtrivlist}
 % \trivlist opens a group so a real class's redefined \trivlist (amsart's
 % \maketitle author block: \trivlist … \item\relax … \endtrivlist, which calls
 % \@trivlist) contains its material and CLOSES cleanly at \endtrivlist instead of

counters.go

@@ -42,16 +42,16 @@ func (e *Engine) doNewcounter() {
 		code = m.code // already allocated (\newcounter of an existing counter): reuse
 	} else if e.allocCnt < 256 {
 		code = e.allocCnt
-		e.define(ctr, &meaning{kind: mCountRef, code: code}, false)
+		e.define(ctr, &meaning{kind: mCountRef, code: code}, true)
 		e.allocCnt++
 	}
 	// \the<name> := \arabic{name}, the LaTeX default representation.
 	body := append([]tok{csTok("arabic"), chTok('{', catBegin)}, stringToToks(name)...)
 	body = append(body, chTok('}', catEnd))
-	e.define("the"+name, &meaning{kind: mMacro, body: body}, false)
+	e.define("the"+name, &meaning{kind: mMacro, body: body}, true)
 	// An empty reset list, so \stepcounter{name} always has one to run.
 	if e.eq["cl@"+name] == nil {
-		e.define("cl@"+name, &meaning{kind: mMacro}, false)
+		e.define("cl@"+name, &meaning{kind: mMacro}, true)
 	}
 	if hasWithin && code >= 0 {
 		if within := strings.TrimSpace(e.toksToString(withinToks)); within != "" {

engine.go

@@ -168,12 +168,13 @@ type Engine struct {
 	footnoteCounter  int
 	pendingFootnotes []*boxNode
 	buildingFootnote bool
-	noBase           bool // when true, getNext does not fall through to the base string
-	negateNextIf     int  // pending \unless prefixes (e-TeX): reverse the next conditional
-	allocCnt         int  // next free \count register handed out by \newcount
-	allocDim         int  // next free \dimen register handed out by \newdimen
-	allocSkp         int  // next free \skip register handed out by \newskip
-	allocBox         int  // next free \box register handed out by \newsavebox
+	levels           []mouthLevel // the input levels below this one (see pushInputLevel)
+	noBase           bool         // when true, getNext does not fall through to the base string
+	negateNextIf     int          // pending \unless prefixes (e-TeX): reverse the next conditional
+	allocCnt         int          // next free \count register handed out by \newcount
+	allocDim         int          // next free \dimen register handed out by \newdimen
+	allocSkp         int          // next free \skip register handed out by \newskip
+	allocBox         int          // next free \box register handed out by \newsavebox
 
 	// token registers (see toks.go): \toks<n> / \newtoks-allocated registers store
 	// a token list each. A class's title/mark machinery (amsart's \andify, \toks@,
@@ -239,8 +240,6 @@ type Engine struct {
 	uccode         map[rune]int // \uccode: what \uppercase maps a character to
 	afterGroup     [][]tok      // \aftergroup tokens, one list per open group
 	xpEnvArgs      [][][]tok    // xparse \NewDocumentEnvironment arguments, one frame per open such environment
-	inputNL        []int        // newlines of each \input file still being read (see endInput)
-	loadedNL       int          // newlines in fully-loaded class/package files, subtracted from the document's source lines (see setSrcPos)
 
 	// runaway guard: a bound on macro expansion so a pathological input (an
 	// infinite \def loop, or a tolerantly-skipped arg-consuming command that
@@ -462,8 +461,7 @@ func (e *Engine) Run(src string) (string, error) {
 	e.bpos = 0
 	e.progBpos = 0    // fresh document: reset the no-progress guard
 	e.noProgSteps = 0 // (e.bpos is monotonic within a Run, so this is a clean baseline)
-	e.loadedNL = 0    // fresh document: no loaded-file lines discounted yet
-	e.inputNL = nil
+	e.levels = nil    // fresh document: no file is open
 	e.afterGroup = nil
 	e.buildLineStarts()
 	e.mainLoop()
@@ -481,25 +479,37 @@ func (e *Engine) push(ts []tok) {
 }
 
 // getNext returns the next raw token (no expansion), or ok=false at end.
+//
+// It reads the pending token lists first, then the current input LEVEL's character
+// buffer, and when that runs out it pops back to the level underneath (tex.web §537
+// end_file_reading). The pop is what makes a file a level of its own: the file is
+// finished before whatever was being read when it was opened resumes.
 func (e *Engine) getNext() (tok, bool) {
-	for len(e.lists) > 0 {
-		top := e.lists[len(e.lists)-1]
-		if len(top) == 0 {
-			e.lists = e.lists[:len(e.lists)-1]
-			continue
+	for {
+		for len(e.lists) > 0 {
+			top := e.lists[len(e.lists)-1]
+			if len(top) == 0 {
+				e.lists = e.lists[:len(e.lists)-1]
+				continue
+			}
+			t := top[0]
+			if rest := top[1:]; len(rest) == 0 {
+				e.lists = e.lists[:len(e.lists)-1] // drop the drained list eagerly so
+			} else { //                              len(e.lists) reflects real nesting
+				e.lists[len(e.lists)-1] = rest
+			}
+			return t, true
 		}
-		t := top[0]
-		if rest := top[1:]; len(rest) == 0 {
-			e.lists = e.lists[:len(e.lists)-1] // drop the drained list eagerly so
-		} else { //                              len(e.lists) reflects real nesting
-			e.lists[len(e.lists)-1] = rest
+		if e.noBase {
+			return tok{}, false
+		}
+		if t, ok := e.scan(); ok {
+			return t, true
+		}
+		if !e.popInputLevel() {
+			return tok{}, false
 		}
-		return t, true
-	}
-	if e.noBase {
-		return tok{}, false
 	}
-	return e.scan()
 }
 
 // back pushes a single token back onto the input.

envgroup_test.go

@@ -0,0 +1,214 @@
+// Copyright (c) the go-tex/engine authors.
+// SPDX-License-Identifier: BSD-3-Clause
+
+package engine
+
+import (
+	"strings"
+	"testing"
+)
+
+// \begin{env} … \end{env} is a GROUP. ltmiscen.dtx:
+//
+//	\protected\def\begin#1{… \begingroup\@endpefalse\reserved@a}
+//	  where \reserved@a is \def\@currenvir{#1}… \csname #1\endcsname
+//	\def\end#1{\csname end#1\endcsname\@checkend{#1}\expandafter\endgroup …}
+//
+// so \begingroup comes first and everything the environment defines — \@currenvir
+// included — is local to it.
+
+func TestEnvironmentIsAGroup(t *testing.T) {
+	e := New()
+	if err := e.LoadLaTeX(); err != nil {
+		t.Fatal(err)
+	}
+	out, err := e.Run(`\def\x{dehors}\newenvironment{env}{}{}` +
+		`\begin{env}\def\x{dedans}\message{[\x]}\end{env}\message{[\x]}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := trimNL(out); got != "[dedans] [dehors]" {
+		t.Errorf("= %q, want a definition made inside the environment to end with it", got)
+	}
+}
+
+// \@currenvir names the environment being run and is restored when it ends. beamer
+// picks between \begin{frame}…\end{frame} and the command form \frame{…} with
+// \ifx\@currenvir\beamer@frametext, so a stale value sends the command form down the
+// environment path.
+func TestCurrentEnvIsRestored(t *testing.T) {
+	e := New()
+	if err := e.LoadLaTeX(); err != nil {
+		t.Fatal(err)
+	}
+	out, err := e.Run(`\newenvironment{env}{}{}\newenvironment{autre}{}{}` +
+		`\message{[\@currenvir]}` +
+		`\begin{env}\message{[\@currenvir]}\begin{autre}\message{[\@currenvir]}\end{autre}` +
+		`\message{[\@currenvir]}\end{env}\message{[\@currenvir]}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := trimNL(out); got != "[] [env] [autre] [env] []" {
+		t.Errorf("= %q, want \\@currenvir to follow the nesting", got)
+	}
+}
+
+// A class may leave an environment early by closing its group itself, and then read
+// \@currenvir to find it is no longer inside. beamer's fragile frame does exactly
+// that: \beamer@checkforfragile ends with \endgroup% end environment, then calls
+// \frame — which must take the COMMAND path.
+func TestClosingTheGroupLeavesTheEnvironment(t *testing.T) {
+	e := New()
+	if err := e.LoadLaTeX(); err != nil {
+		t.Fatal(err)
+	}
+	out, err := e.Run(`\newenvironment{env}{\endgroup\message{[\@currenvir]}}{}` +
+		`\begin{env}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := trimNL(out); got != "[]" {
+		t.Errorf("= %q, want \\endgroup to leave the environment", got)
+	}
+}
+
+// Allocation is GLOBAL. ltplain.dtx's \e@alloc ends with
+// `\global#2#6\allocationnumber`, and ltcounts.dtx's \@definecounter makes \cl@<c>,
+// \p@<c> and \the<c> global too — so a counter declared inside an environment (or
+// inside \begin{document}, which is one) is still there afterwards. \setcounter is
+// global as well; the register's VALUE follows TeX's ordinary scoping, so the
+// assignment here is explicitly \global.
+func TestAllocationSurvivesTheEnvironment(t *testing.T) {
+	e := New()
+	if err := e.LoadLaTeX(); err != nil {
+		t.Fatal(err)
+	}
+	out, err := e.Run(`\newenvironment{env}{}{}` +
+		`\begin{env}\newcounter{compte}\newcount\reg \global\reg=7 \setcounter{compte}{4}\end{env}` +
+		`\message{[\arabic{compte}][\the\reg]}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := trimNL(out); got != "[4][7]" {
+		t.Errorf("= %q, want the counter and the register to outlive the environment", got)
+	}
+}
+
+// Every alignment entry is a group (tex.web §791: a template's u-part and v-part are
+// inserted inside braces), so a font switch in one cell stops at that cell.
+func TestAlignmentCellIsAGroup(t *testing.T) {
+	e, err := buildEngine(Options{Lenient: true}, true)
+	if err != nil {
+		t.Fatalf("buildEngine: %v", err)
+	}
+	if _, err := e.Run(`\hsize=300pt\begin{tabular}{ll}` +
+		`\bfseries A & B \\` +
+		`\end{tabular}`); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	// The table must have been built: a cell group that leaked would have been
+	// reported as a stray brace by the box builder.
+	if d := e.Diagnostics(); d.OpenGroups != 0 {
+		t.Errorf("a tabular left %d group(s) open", d.OpenGroups)
+	}
+}
+
+// An environment the engine implements in Go swallows its own \end, so \end — and
+// with it the \endgroup — never runs. Each such environment closes the group itself;
+// this checks the whole family at once.
+func TestGoSideEnvironmentsCloseTheirGroup(t *testing.T) {
+	for _, c := range []struct{ nom, src string }{
+		{"tabular", `\begin{tabular}{ll}A & B\\\end{tabular}`},
+		{"tabularx", `\begin{tabularx}{200pt}{lX}A & B\\\end{tabularx}`},
+		{"verbatim", "\\begin{verbatim}\nbrut\n\\end{verbatim}"},
+		{"equation", `\begin{equation}x\end{equation}`},
+		{"align", `\begin{align}x&=y\end{align}`},
+		{"minipage", `\begin{minipage}{100pt}texte\end{minipage}`},
+		{"comment", `\excludecomment{comment}\begin{comment}rien\end{comment}`},
+	} {
+		e, err := buildEngine(Options{Lenient: true}, true)
+		if err != nil {
+			t.Fatalf("%s: buildEngine: %v", c.nom, err)
+		}
+		if _, err := e.Run(`\hsize=300pt` + c.src + `\message{[fin]}`); err != nil {
+			t.Fatalf("%s: Run: %v", c.nom, err)
+		}
+		if d := e.Diagnostics(); d.OpenGroups != 0 {
+			t.Errorf("%s left %d group(s) open", c.nom, d.OpenGroups)
+		}
+	}
+}
+
+// A group left open by an environment shows up as text later: the check above reads
+// the count, this one reads the consequence.
+func TestAnEnvironmentDoesNotLeakItsScope(t *testing.T) {
+	e := New()
+	if err := e.LoadLaTeX(); err != nil {
+		t.Fatal(err)
+	}
+	out, err := e.Run(`\newenvironment{env}{}{}\count0=1 ` +
+		`\begin{env}\count0=2 \end{env}\message{[\the\count0]}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := trimNL(out); !strings.Contains(got, "[1]") {
+		t.Errorf("= %q, want the register assignment to end with the environment", got)
+	}
+}
+
+// TeX's alignment scanner EXPANDS as it looks for & and \cr, so a class may split a
+// table from inside it. NeurIPS's style does:
+//
+//	\def\And{\end{tabular}\hfil\linebreak[0]\hfil\begin{tabular}[t]{c}…}
+//
+// used inside \begin{tabular}[t]{c}…\@author\end{tabular}, so \author{A \And B} carries
+// an \end{tabular} two levels down — inside \@author, inside \And. Read raw, it never
+// reached the body scanner and only surfaced while the CELL was being typeset, in the
+// middle of a box: measured, a paper of 55 232 glyphs rendered 221.
+func TestTabularSplitByAMacroCarryingItsEnd(t *testing.T) {
+	e, err := buildEngine(Options{Lenient: true}, true)
+	if err != nil {
+		t.Fatalf("buildEngine: %v", err)
+	}
+	out, err := e.Run(`\hsize=300pt` +
+		`\def\And{\end{tabular}\begin{tabular}{c}}` +
+		`\def\auteurs{A \And B}` +
+		`\begin{tabular}{c}\auteurs\end{tabular}` +
+		`\message{[suite]}`)
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if !strings.Contains(out, "[suite]") {
+		t.Errorf("= %q, want the document to carry on past the split table", trimNL(out))
+	}
+	if d := e.Diagnostics(); d.OpenGroups != 0 {
+		t.Errorf("a table split by \\And left %d group(s) open", d.OpenGroups)
+	}
+}
+
+// \endlist ends the list's trivlist, as ltlists.dtx does:
+//
+//	\def\endlist{\global\advance\@listdepth\m@ne \endtrivlist}
+//
+// That chain is what a class hooks. beamer patches \endtrivlist to run
+// \beamer@closeitem, which closes the overlay wrappers its LAST \item left open —
+// every earlier item is closed by the next \item. With \endlist a bare \par those
+// three environments stayed open past \end{itemize}, every \end after them closed one
+// group too high, and the stack grew by two per slide: four lines of beamer left six
+// groups open where TeX leaves none.
+func TestListAndEndlistAreAGroupPair(t *testing.T) {
+	e := New()
+	if err := e.LoadLaTeX(); err != nil {
+		t.Fatal(err)
+	}
+	out, err := e.Run(`\count0=1 \list{}{}\count0=2 \message{[\the\count0]}\endlist\message{[\the\count0]}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := trimNL(out); got != "[2] [1]" {
+		t.Errorf("= %q, want \\list … \\endlist to scope what it contains", got)
+	}
+	if d := e.Diagnostics(); d.OpenGroups != 0 {
+		t.Errorf("\\list … \\endlist left %d group(s) open", d.OpenGroups)
+	}
+}

equation.go

@@ -52,6 +52,11 @@ func (e *Engine) doEquationBody() {
 		row = append(row, box)
 	}
 	e.placeDisplay([]*boxNode{hpackSP(row, packTo, e.hsize)})
+	// collectMathUntilEnd read this environment's own \end, so \end — and the
+	// \gotex@endenv that closes the group \begin opened — never runs. Close it here,
+	// AFTER the labels are recorded: \end closes the group only once \endequation
+	// has finished, and the reference metadata is part of that.
+	e.endEnvGroup()
 }
 
 // doEquationStar handles \begin{equation*} (an unnumbered single-line display): the
@@ -77,6 +82,7 @@ func (e *Engine) doEquationStar(name string) {
 		row = append(row, box)
 	}
 	e.placeDisplay([]*boxNode{hpackSP(row, packTo, e.hsize)})
+	e.endEnvGroup() // see doEquationBody
 }
 
 // eqNumberBox builds the number box for a display line: the \tag (parenthesised

halign.go

@@ -131,7 +131,15 @@ func (e *Engine) buildCellHList(toks []tok) []node {
 	seq = append(seq, toks...)
 	seq = append(seq, chTok('}', catEnd)) // sentinel to terminate buildBoxList
 	e.push(seq)
+	// The sentinel closes a real group, as the cell's own braces would: TeX makes
+	// every alignment entry a group (tex.web §791 — the u-part and v-part of a
+	// template are inserted inside braces), so a font or colour switch in a cell
+	// stops at the cell. Without the group the sentinel was a STRAY brace, which
+	// the stomach reports the moment any \begingroup is open — and \begin{env} now
+	// opens one.
+	e.beginGroupKind(boxGroup)
 	list := e.buildBoxList()
+	e.endGroup()
 	e.noBase = saved
 	return list
 }