[dev.unified] cmd/compile: more Unified IR docs and review

This adds more documentation throughout the core Unified IR logic and
removes their UNREVIEWED notices.

Updates #48194.

Change-Id: Iddd30edaee1c6ea8a05a5a7e013480e02be00d29
Reviewed-on: https://go-review.googlesource.com/c/go/+/411917
Auto-Submit: Matthew Dempsky <mdempsky@google.com>
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
Reviewed-by: Cuong Manh Le <cuong.manhle.vn@gmail.com>
Reviewed-by: David Chase <drchase@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

src/cmd/compile/internal/noder/codes.go

@@ -1,5 +1,3 @@
-// UNREVIEWED
-
 // Copyright 2021 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
@@ -8,6 +6,7 @@ package noder
 
 import "internal/pkgbits"
 
+// A codeStmt distinguishes among statement encodings.
 type codeStmt int
 
 func (c codeStmt) Marker() pkgbits.SyncMarker { return pkgbits.SyncStmt1 }
@@ -31,6 +30,7 @@ const (
 	stmtSelect
 )
 
+// A codeExpr distinguishes among expression encodings.
 type codeExpr int
 
 func (c codeExpr) Marker() pkgbits.SyncMarker { return pkgbits.SyncExpr }
@@ -66,6 +66,7 @@ const (
 	assignExpr
 )
 
+// A codeDecl distinguishes among declaration encodings.
 type codeDecl int
 
 func (c codeDecl) Marker() pkgbits.SyncMarker { return pkgbits.SyncDecl }

src/cmd/compile/internal/noder/linker.go

@@ -1,5 +1,3 @@
-// UNREVIEWED
-
 // Copyright 2021 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
@@ -34,13 +32,19 @@ import (
 // low-level linking details can be moved there, but the logic for
 // handling extension data needs to stay in the compiler.
 
+// A linker combines a package's stub export data with any referenced
+// elements from imported packages into a single, self-contained
+// export data file.
 type linker struct {
 	pw pkgbits.PkgEncoder
 
 	pkgs  map[string]pkgbits.Index
 	decls map[*types.Sym]pkgbits.Index
 }
 
+// relocAll ensures that all elements specified by pr and relocs are
+// copied into the output export data file, and returns the
+// corresponding indices in the output.
 func (l *linker) relocAll(pr *pkgReader, relocs []pkgbits.RelocEnt) []pkgbits.RelocEnt {
 	res := make([]pkgbits.RelocEnt, len(relocs))
 	for i, rent := range relocs {
@@ -50,6 +54,8 @@ func (l *linker) relocAll(pr *pkgReader, relocs []pkgbits.RelocEnt) []pkgbits.Re
 	return res
 }
 
+// relocIdx ensures a single element is copied into the output export
+// data file, and returns the corresponding index in the output.
 func (l *linker) relocIdx(pr *pkgReader, k pkgbits.RelocKind, idx pkgbits.Index) pkgbits.Index {
 	assert(pr != nil)
 
@@ -85,10 +91,19 @@ func (l *linker) relocIdx(pr *pkgReader, k pkgbits.RelocKind, idx pkgbits.Index)
 	return newidx
 }
 
+// relocString copies the specified string from pr into the output
+// export data file, deduplicating it against other strings.
 func (l *linker) relocString(pr *pkgReader, idx pkgbits.Index) pkgbits.Index {
 	return l.pw.StringIdx(pr.StringIdx(idx))
 }
 
+// relocPkg copies the specified package from pr into the output
+// export data file, rewriting its import path to match how it was
+// imported.
+//
+// TODO(mdempsky): Since CL 391014, we already have the compilation
+// unit's import path, so there should be no need to rewrite packages
+// anymore.
 func (l *linker) relocPkg(pr *pkgReader, idx pkgbits.Index) pkgbits.Index {
 	path := pr.PeekPkgPath(idx)
 
@@ -114,6 +129,9 @@ func (l *linker) relocPkg(pr *pkgReader, idx pkgbits.Index) pkgbits.Index {
 	return w.Flush()
 }
 
+// relocObj copies the specified object from pr into the output export
+// data file, rewriting its compiler-private extension data (e.g.,
+// adding inlining cost and escape analysis results for functions).
 func (l *linker) relocObj(pr *pkgReader, idx pkgbits.Index) pkgbits.Index {
 	path, name, tag := pr.PeekObj(idx)
 	sym := types.NewPkg(path, "").Lookup(name)
@@ -184,6 +202,8 @@ func (l *linker) relocObj(pr *pkgReader, idx pkgbits.Index) pkgbits.Index {
 	return w.Idx
 }
 
+// relocCommon copies the specified element from pr into w,
+// recursively relocating any referenced elements as well.
 func (l *linker) relocCommon(pr *pkgReader, w *pkgbits.Encoder, k pkgbits.RelocKind, idx pkgbits.Index) {
 	r := pr.NewDecoderRaw(k, idx)
 	w.Relocs = l.relocAll(pr, r.Relocs)

src/cmd/compile/internal/noder/quirks.go

@@ -1,5 +1,3 @@
-// UNREVIEWED
-
 // Copyright 2021 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
@@ -12,12 +10,12 @@ import (
 	"cmd/compile/internal/syntax"
 )
 
-// This file defines helper functions useful for satisfying toolstash
-// -cmp when compared against the legacy frontend behavior, but can be
-// removed after that's no longer a concern.
-
 // typeExprEndPos returns the position that noder would leave base.Pos
 // after parsing the given type expression.
+//
+// Deprecated: This function exists to emulate position semantics from
+// Go 1.17, necessary for compatibility with the backend DWARF
+// generation logic that assigns variables to their appropriate scope.
 func typeExprEndPos(expr0 syntax.Expr) syntax.Pos {
 	for {
 		switch expr := expr0.(type) {

src/cmd/compile/internal/noder/reader.go

@@ -1,5 +1,3 @@
-// UNREVIEWED
-
 // Copyright 2021 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
@@ -26,15 +24,25 @@ import (
 	"cmd/internal/src"
 )
 
+// This file implements cmd/compile backend's reader for the Unified
+// IR export data.
+
+// A pkgReader reads Unified IR export data.
 type pkgReader struct {
 	pkgbits.PkgDecoder
 
+	// Indices for encoded things; lazily populated as needed.
+	//
+	// Note: Objects (i.e., ir.Names) are lazily instantiated by
+	// populating their types.Sym.Def; see objReader below.
+
 	posBases []*src.PosBase
 	pkgs     []*types.Pkg
 	typs     []*types.Type
 
-	// offset for rewriting the given index into the output,
-	// but bitwise inverted so we can detect if we're missing the entry or not.
+	// offset for rewriting the given (absolute!) index into the output,
+	// but bitwise inverted so we can detect if we're missing the entry
+	// or not.
 	newindex []pkgbits.Index
 }
 
@@ -50,6 +58,8 @@ func newPkgReader(pr pkgbits.PkgDecoder) *pkgReader {
 	}
 }
 
+// A pkgReaderIndex compactly identifies an index (and its
+// corresponding dictionary) within a package's export data.
 type pkgReaderIndex struct {
 	pr   *pkgReader
 	idx  pkgbits.Index
@@ -69,6 +79,7 @@ func (pr *pkgReader) newReader(k pkgbits.RelocKind, idx pkgbits.Index, marker pk
 	}
 }
 
+// A writer provides APIs for reading an individual element.
 type reader struct {
 	pkgbits.Decoder
 
@@ -162,6 +173,7 @@ func setValue(name *ir.Name, val constant.Value) {
 
 // @@@ Positions
 
+// pos reads a position from the bitstream.
 func (r *reader) pos() src.XPos {
 	return base.Ctxt.PosTable.XPos(r.pos0())
 }
@@ -178,10 +190,13 @@ func (r *reader) pos0() src.Pos {
 	return src.MakePos(posBase, line, col)
 }
 
+// posBase reads a position base from the bitstream.
 func (r *reader) posBase() *src.PosBase {
 	return r.inlPosBase(r.p.posBaseIdx(r.Reloc(pkgbits.RelocPosBase)))
 }
 
+// posBaseIdx returns the specified position base, reading it first if
+// needed.
 func (pr *pkgReader) posBaseIdx(idx pkgbits.Index) *src.PosBase {
 	if b := pr.posBases[idx]; b != nil {
 		return b
@@ -222,6 +237,7 @@ func (pr *pkgReader) posBaseIdx(idx pkgbits.Index) *src.PosBase {
 	return b
 }
 
+// TODO(mdempsky): Document this.
 func (r *reader) inlPosBase(oldBase *src.PosBase) *src.PosBase {
 	if r.inlCall == nil {
 		return oldBase
@@ -236,36 +252,23 @@ func (r *reader) inlPosBase(oldBase *src.PosBase) *src.PosBase {
 	return newBase
 }
 
+// TODO(mdempsky): Document this.
 func (r *reader) updatePos(xpos src.XPos) src.XPos {
 	pos := base.Ctxt.PosTable.Pos(xpos)
 	pos.SetBase(r.inlPosBase(pos.Base()))
 	return base.Ctxt.PosTable.XPos(pos)
 }
 
-func (r *reader) origPos(xpos src.XPos) src.XPos {
-	if r.inlCall == nil {
-		return xpos
-	}
-
-	pos := base.Ctxt.PosTable.Pos(xpos)
-	for old, new := range r.inlPosBases {
-		if pos.Base() == new {
-			pos.SetBase(old)
-			return base.Ctxt.PosTable.XPos(pos)
-		}
-	}
-
-	base.FatalfAt(xpos, "pos base missing from inlPosBases")
-	panic("unreachable")
-}
-
 // @@@ Packages
 
+// pkg reads a package reference from the bitstream.
 func (r *reader) pkg() *types.Pkg {
 	r.Sync(pkgbits.SyncPkg)
 	return r.p.pkgIdx(r.Reloc(pkgbits.RelocPkg))
 }
 
+// pkgIdx returns the specified package from the export data, reading
+// it first if needed.
 func (pr *pkgReader) pkgIdx(idx pkgbits.Index) *types.Pkg {
 	if pkg := pr.pkgs[idx]; pkg != nil {
 		return pkg
@@ -276,6 +279,7 @@ func (pr *pkgReader) pkgIdx(idx pkgbits.Index) *types.Pkg {
 	return pkg
 }
 
+// doPkg reads a package definition from the bitstream.
 func (r *reader) doPkg() *types.Pkg {
 	path := r.String()
 	switch path {
@@ -530,8 +534,12 @@ func (r *reader) param() (*types.Pkg, *types.Field) {
 
 // @@@ Objects
 
+// objReader maps qualified identifiers (represented as *types.Sym) to
+// a pkgReader and corresponding index that can be used for reading
+// that object's definition.
 var objReader = map[*types.Sym]pkgReaderIndex{}
 
+// obj reads an instantiated object reference from the bitstream.
 func (r *reader) obj() ir.Node {
 	r.Sync(pkgbits.SyncObject)
 
@@ -567,6 +575,8 @@ func (r *reader) obj() ir.Node {
 	return r.p.objIdx(idx, implicits, explicits)
 }
 
+// objIdx returns the specified object from the bitstream,
+// instantiated with the given type arguments, if any.
 func (pr *pkgReader) objIdx(idx pkgbits.Index, implicits, explicits []*types.Type) ir.Node {
 	rname := pr.newReader(pkgbits.RelocName, idx, pkgbits.SyncObject1)
 	_, sym := rname.qualifiedIdent()
@@ -707,6 +717,7 @@ func (r *reader) mangle(sym *types.Sym) *types.Sym {
 	return sym.Pkg.Lookup(buf.String())
 }
 
+// objDictIdx reads and returns the specified object dictionary.
 func (pr *pkgReader) objDictIdx(sym *types.Sym, idx pkgbits.Index, implicits, explicits []*types.Type) *readerDict {
 	r := pr.newReader(pkgbits.RelocObjDict, idx, pkgbits.SyncObject1)
 
@@ -955,6 +966,8 @@ var bodyReader = map[*ir.Func]pkgReaderIndex{}
 // constructed.
 var todoBodies []*ir.Func
 
+// addBody reads a function body reference from the element bitstream,
+// and associates it with fn.
 func (r *reader) addBody(fn *ir.Func) {
 	pri := pkgReaderIndex{r.p, r.Reloc(pkgbits.RelocBody), r.dict}
 	bodyReader[fn] = pri
@@ -978,6 +991,8 @@ func (pri pkgReaderIndex) funcBody(fn *ir.Func) {
 	r.funcBody(fn)
 }
 
+// funcBody reads a function body definition from the element
+// bitstream, and populates fn with it.
 func (r *reader) funcBody(fn *ir.Func) {
 	r.curfn = fn
 	r.closureVars = fn.ClosureVars
@@ -1995,6 +2010,8 @@ func (r *reader) pkgObjs(target *ir.Package) []*ir.Name {
 
 var inlgen = 0
 
+// InlineCall implements inline.NewInline by re-reading the function
+// body from its Unified IR export data.
 func InlineCall(call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
 	// TODO(mdempsky): Turn callerfn into an explicit parameter.
 	callerfn := ir.CurFunc
@@ -2281,18 +2298,21 @@ func (r *reader) needWrapper(typ *types.Type) {
 	}
 }
 
+// importedDef reports whether r is reading from an imported and
+// non-generic element.
+//
+// If a type was found in an imported package, then we can assume that
+// package (or one of its transitive dependencies) already generated
+// method wrappers for it.
+//
+// Exception: If we're instantiating an imported generic type or
+// function, we might be instantiating it with type arguments not
+// previously seen before.
+//
+// TODO(mdempsky): Distinguish when a generic function or type was
+// instantiated in an imported package so that we can add types to
+// haveWrapperTypes instead.
 func (r *reader) importedDef() bool {
-	// If a type was found in an imported package, then we can assume
-	// that package (or one of its transitive dependencies) already
-	// generated method wrappers for it.
-	//
-	// Exception: If we're instantiating an imported generic type or
-	// function, we might be instantiating it with type arguments not
-	// previously seen before.
-	//
-	// TODO(mdempsky): Distinguish when a generic function or type was
-	// instantiated in an imported package so that we can add types to
-	// haveWrapperTypes instead.
 	return r.p != localPkgReader && !r.hasTypeParams()
 }
 

src/cmd/compile/internal/noder/unified.go

@@ -1,5 +1,3 @@
-// UNREVIEWED
-
 // Copyright 2021 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
@@ -226,6 +224,8 @@ func freePackage(pkg *types2.Package) {
 	base.Fatalf("package never finalized")
 }
 
+// readPackage reads package export data from pr to populate
+// importpkg.
 func readPackage(pr *pkgReader, importpkg *types.Pkg) {
 	r := pr.newReader(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
 
@@ -252,6 +252,8 @@ func readPackage(pr *pkgReader, importpkg *types.Pkg) {
 	}
 }
 
+// writeUnifiedExport writes to `out` the finalized, self-contained
+// Unified IR export data file for the current compilation unit.
 func writeUnifiedExport(out io.Writer) {
 	l := linker{
 		pw: pkgbits.NewPkgEncoder(base.Debug.SyncFrames),