Skip to content

Commit

Permalink
[dev.link] cmd/link: drop hash maps after loading
Browse files Browse the repository at this point in the history 
The hash maps are used to deduplicate hashed symbols. Once we
loaded all the symbols, we no longer need the hash maps. Drop
them.

Linking cmd/compile,

name         old live-B     new live-B     delta
Loadlib_GC      13.1M ± 0%     11.3M ± 0%   -13.62%  (p=0.008 n=5+5)

Change-Id: I4bb1f84e1111a56d9e777cd6a68f7d974b60e321
Reviewed-on: https://go-review.googlesource.com/c/go/+/245721
Reviewed-by: Jeremy Faller <jeremy@golang.org>
  • Loading branch information
@cherrymui
cherrymui committed Aug 3, 2020
1 parent cecb7a1 commit f204ca3
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 20 deletions.
54 changes: 35 additions & 19 deletions src/cmd/link/internal/loader/loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,10 +204,8 @@ type Loader struct {

objSyms []objSym // global index mapping to local index

hashed64Syms map[uint64]symAndSize // short hashed (content-addressable) symbols, keyed by content hash
hashedSyms map[goobj2.HashType]symAndSize // hashed (content-addressable) symbols, keyed by content hash
symsByName [2]map[string]Sym // map symbol name to index, two maps are for ABI0 and ABIInternal
extStaticSyms map[nameVer]Sym // externally defined static symbols, keyed by name
symsByName [2]map[string]Sym // map symbol name to index, two maps are for ABI0 and ABIInternal
extStaticSyms map[nameVer]Sym // externally defined static symbols, keyed by name

extReader *oReader // a dummy oReader, for external symbols
payloadBatch []extSymPayload
Expand Down Expand Up @@ -285,7 +283,8 @@ type Loader struct {

errorReporter *ErrorReporter

npkgsyms int // number of package symbols, for accounting
npkgsyms int // number of package symbols, for accounting
nhashedsyms int // number of hashed symbols, for accounting
}

const (
Expand Down Expand Up @@ -332,8 +331,6 @@ func NewLoader(flags uint32, elfsetstring elfsetstringFunc, reporter *ErrorRepor
objs: []objIdx{{}, {extReader, 0}}, // reserve index 0 for nil symbol, 1 for external symbols
objSyms: make([]objSym, 1, 100000), // reserve index 0 for nil symbol
extReader: extReader,
hashed64Syms: make(map[uint64]symAndSize, 10000), // TODO: adjust preallocation sizes
hashedSyms: make(map[goobj2.HashType]symAndSize, 20000), // TODO: adjust preallocation sizes
symsByName: [2]map[string]Sym{make(map[string]Sym, 80000), make(map[string]Sym, 50000)}, // preallocate ~2MB for ABI0 and ~1MB for ABI1 symbols
objByPkg: make(map[string]*oReader),
outer: make(map[Sym]Sym),
Expand Down Expand Up @@ -388,7 +385,8 @@ func (l *Loader) addObj(pkg string, r *oReader) Sym {

// Add a symbol from an object file, return the global index.
// If the symbol already exist, it returns the index of that symbol.
func (l *Loader) addSym(name string, ver int, r *oReader, li uint32, kind int, osym *goobj2.Sym) Sym {
func (st *loadState) addSym(name string, ver int, r *oReader, li uint32, kind int, osym *goobj2.Sym) Sym {
l := st.l
if l.extStart != 0 {
panic("addSym called after external symbol is created")
}
Expand Down Expand Up @@ -429,17 +427,17 @@ func (l *Loader) addSym(name string, ver int, r *oReader, li uint32, kind int, o
if kind == hashed64Def {
checkHash = func() (symAndSize, bool) {
h64 = r.Hash64(li - uint32(r.ndef))
s, existed := l.hashed64Syms[h64]
s, existed := st.hashed64Syms[h64]
return s, existed
}
addToHashMap = func(ss symAndSize) { l.hashed64Syms[h64] = ss }
addToHashMap = func(ss symAndSize) { st.hashed64Syms[h64] = ss }
} else {
checkHash = func() (symAndSize, bool) {
h = r.Hash(li - uint32(r.ndef+r.nhashed64def))
s, existed := l.hashedSyms[*h]
s, existed := st.hashedSyms[*h]
return s, existed
}
addToHashMap = func(ss symAndSize) { l.hashedSyms[*h] = ss }
addToHashMap = func(ss symAndSize) { st.hashedSyms[*h] = ss }
}
siz := osym.Siz()
if s, existed := checkHash(); existed {
Expand Down Expand Up @@ -2070,16 +2068,25 @@ func (l *Loader) Preload(localSymVersion int, f *bio.Reader, lib *sym.Library, u
}

l.addObj(lib.Pkg, or)
l.preloadSyms(or, pkgDef)
st := loadState{l: l}
st.preloadSyms(or, pkgDef)

// The caller expects us consuming all the data
f.MustSeek(length, os.SEEK_CUR)

return r.Fingerprint()
}

// Holds the loader along with temporary states for loading symbols.
type loadState struct {
l *Loader
hashed64Syms map[uint64]symAndSize // short hashed (content-addressable) symbols, keyed by content hash
hashedSyms map[goobj2.HashType]symAndSize // hashed (content-addressable) symbols, keyed by content hash
}

// Preload symbols of given kind from an object.
func (l *Loader) preloadSyms(r *oReader, kind int) {
func (st *loadState) preloadSyms(r *oReader, kind int) {
l := st.l
var start, end uint32
switch kind {
case pkgDef:
Expand Down Expand Up @@ -2121,7 +2128,7 @@ func (l *Loader) preloadSyms(r *oReader, kind int) {
}
v = abiToVer(osym.ABI(), r.version)
}
gi := l.addSym(name, v, r, i, kind, osym)
gi := st.addSym(name, v, r, i, kind, osym)
r.syms[i] = gi
if osym.TopFrame() {
l.SetAttrTopFrame(gi, true)
Expand Down Expand Up @@ -2152,11 +2159,20 @@ func (l *Loader) preloadSyms(r *oReader, kind int) {
// references to external symbols (which are always named).
func (l *Loader) LoadNonpkgSyms(arch *sys.Arch) {
l.npkgsyms = l.NSym()
// Preallocate some space (a few hundreds KB) for some symbols.
// As of Go 1.15, linking cmd/compile has ~8000 hashed64 symbols and
// ~13000 hashed symbols.
st := loadState{
l: l,
hashed64Syms: make(map[uint64]symAndSize, 10000),
hashedSyms: make(map[goobj2.HashType]symAndSize, 15000),
}
for _, o := range l.objs[goObjStart:] {
l.preloadSyms(o.r, hashed64Def)
l.preloadSyms(o.r, hashedDef)
l.preloadSyms(o.r, nonPkgDef)
st.preloadSyms(o.r, hashed64Def)
st.preloadSyms(o.r, hashedDef)
st.preloadSyms(o.r, nonPkgDef)
}
l.nhashedsyms = len(st.hashed64Syms) + len(st.hashedSyms)
for _, o := range l.objs[goObjStart:] {
loadObjRefs(l, o.r, arch)
}
Expand Down Expand Up @@ -2575,7 +2591,7 @@ func (l *Loader) Errorf(s Sym, format string, args ...interface{}) {
func (l *Loader) Stat() string {
s := fmt.Sprintf("%d symbols, %d reachable\n", l.NSym(), l.NReachableSym())
s += fmt.Sprintf("\t%d package symbols, %d hashed symbols, %d non-package symbols, %d external symbols\n",
l.npkgsyms, len(l.hashed64Syms)+len(l.hashedSyms), int(l.extStart)-l.npkgsyms-len(l.hashed64Syms)-len(l.hashedSyms), l.NSym()-int(l.extStart))
l.npkgsyms, l.nhashedsyms, int(l.extStart)-l.npkgsyms-l.nhashedsyms, l.NSym()-int(l.extStart))
return s
}

Expand Down
3 changes: 2 additions & 1 deletion src/cmd/link/internal/loader/loader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ import (
// data or relocations).
func addDummyObjSym(t *testing.T, ldr *Loader, or *oReader, name string) Sym {
idx := uint32(len(ldr.objSyms))
return ldr.addSym(name, 0, or, idx, nonPkgDef, &goobj2.Sym{})
st := loadState{l: ldr}
return st.addSym(name, 0, or, idx, nonPkgDef, &goobj2.Sym{})
}

func mkLoader() *Loader {
Expand Down

0 comments on commit f204ca3

Please sign in to comment.