Skip to content

Add more test programs #83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 22, 2023
Merged

Add more test programs #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4798b9e
dwarf: fix
achille-roussel May 18, 2023
a96dbf8
go mod tidy
achille-roussel May 18, 2023
dcfc463
add more test files
achille-roussel May 18, 2023
581107a
go mod tidy
achille-roussel May 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@
*~

testdata/rust/*/target/*
testdata/.sysroot/*
3 changes: 3 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "wasi-libc"]
path = testdata/.wasi-libc
url = https://github.com/WebAssembly/wasi-libc
59 changes: 59 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
.PHONY: all clean test testdata wasi-libc

testdata.c.src = $(wildcard testdata/c/*.c)
testdata.c.wasm = $(testdata.c.src:.c=.wasm)

testdata.go.src = $(wildcard testdata/go/*.go)
testdata.go.wasm = $(testdata.go.src:.go=.wasm)

testdata.tinygo.src = $(wildcard testdata/tinygo/*.go)
testdata.tinygo.wasm = $(testdata.tinygo.src:.go=.wasm)

testdata.wat.src = $(wildcard testdata/wat/*.go)
testdata.wat.wasm = $(testdata.wat.src:.wat=.wasm)

testdata.files = \
$(testdata.c.wasm) \
$(testdata.go.wasm) \
$(testdata.tinygo.wasm) \
$(testdata.wat.wasm)

all: test

clean:
rm -f $(testdata.files)

test: testdata
go test ./...

testdata: wasi-libc $(testdata.files)

testdata/.sysroot:
mkdir -p testdata/.sysroot

testdata/.wasi-libc: testdata/.wasi-libc/.git

testdata/.wasi-libc/.git: .gitmodules
git submodule update --init --recursive -- testdata/.wasi-libc

testdata/.sysroot/lib/wasm32-wasi/libc.a: testdata/.wasi-libc
make -j4 -C testdata/.wasi-libc install INSTALL_DIR=../.sysroot

testdata/c/%.c: wasi-libc
testdata/c/%.wasm: testdata/c/%.c
clang $< -o $@ -Wall -Os -target wasm32-unknown-wasi --sysroot testdata/.sysroot

testdata/go/%.wasm: testdata/go/%.go
GOARCH=wasm GOOS=wasip1 gotip build -o $@ $<

testdata/tinygo/%.wasm: testdata/tinygo/%.go
tinygo build -target=wasi -o $@ $<

testdata/wat/%.wasm: testdata/wat/%.wat
wat2wasm -o $@ $<

wasi-libc: testdata/.sysroot/lib/wasm32-wasi/libc.a

.gitmodules:
git submodule add --name wasi-libc -- \
'https://github.com/WebAssembly/wasi-libc' testdata/.wasi-libc
68 changes: 50 additions & 18 deletions cmd/wzprof/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ import (
"crypto/rand"
"flag"
"fmt"
"io"
"log"
"math"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
Expand All @@ -28,7 +30,7 @@ func main() {
defer cancel()

if err := run(ctx); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
stderr.Print(err)
os.Exit(1)
}
}
Expand Down Expand Up @@ -60,13 +62,18 @@ func (prog *program) run(ctx context.Context) error {

var listeners []experimental.FunctionListenerFactory
if prog.cpuProfile != "" || prog.pprofAddr != "" {
stdout.Printf("enabling cpu profiler")
listeners = append(listeners, cpu)
}
if prog.memProfile != "" || prog.pprofAddr != "" {
stdout.Printf("enabling memory profiler")
listeners = append(listeners, mem)
}
for i, lstn := range listeners {
listeners[i] = wzprof.Sample(prog.sampleRate, lstn)
if prog.sampleRate < 1 {
stdout.Printf("configuring sampling rate to %.2g%%", prog.sampleRate)
for i, lstn := range listeners {
listeners[i] = wzprof.Sample(prog.sampleRate, lstn)
}
}

ctx = context.WithValue(ctx,
Expand All @@ -78,24 +85,29 @@ func (prog *program) run(ctx context.Context) error {
WithDebugInfoEnabled(true).
WithCustomSections(true))

stdout.Printf("compiling wasm module %s", prog.filePath)
compiledModule, err := runtime.CompileModule(ctx, wasmCode)
if err != nil {
return fmt.Errorf("compiling wasm module: %w", err)
}

stdout.Printf("building dwarf symbolizer from compiled wasm module")
symbols, err := wzprof.BuildDwarfSymbolizer(compiledModule)
if err != nil {
symbols = nil
log.Println("warning: symbolizing wasm module:", err)
}

if prog.pprofAddr != "" {
u := &url.URL{Scheme: "http", Host: prog.pprofAddr, Path: "/debug/pprof"}
stdout.Printf("starting prrof http sever at %s", u)

server := http.NewServeMux()
server.Handle("/debug/pprof/", wzprof.Handler(prog.sampleRate, symbols, cpu, mem))

go func() {
if err := http.ListenAndServe(prog.pprofAddr, server); err != nil {
log.Println(err)
stderr.Println(err)
}
}()
}
Expand All @@ -107,7 +119,7 @@ func (prog *program) run(ctx context.Context) error {
return err
}
startCPUProfile(f)
defer stopCPUProfile()
defer stopCPUProfile(f)
}

if prog.memProfile != "" {
Expand All @@ -124,7 +136,7 @@ func (prog *program) run(ctx context.Context) error {
defer func() {
p := cpu.StopProfile(prog.sampleRate, symbols)
if !prog.hostProfile {
writeProfile(wasmName, prog.cpuProfile, p)
writeProfile("cpu", wasmName, prog.cpuProfile, p)
}
}()
}
Expand All @@ -133,14 +145,15 @@ func (prog *program) run(ctx context.Context) error {
defer func() {
p := mem.NewProfile(prog.sampleRate, symbols)
if !prog.hostProfile {
writeProfile(wasmName, prog.memProfile, p)
writeProfile("memory", wasmName, prog.memProfile, p)
}
}()
}

ctx, cancel := context.WithCancelCause(ctx)
go func() {
defer cancel(nil)
stdout.Printf("instantiating host module: wasi_snapshot_preview1")
wasi_snapshot_preview1.MustInstantiate(ctx, runtime)

config := wazero.NewModuleConfig().
Expand All @@ -154,13 +167,18 @@ func (prog *program) run(ctx context.Context) error {
WithArgs(append([]string{wasmName}, prog.args...)...).
WithFSConfig(createFSConfig(prog.mounts))

moduleName := compiledModule.Name()
if moduleName == "" {
moduleName = wasmName
}
stdout.Printf("instantiating guest module: %s", moduleName)
instance, err := runtime.InstantiateModule(ctx, compiledModule, config)
if err != nil {
cancel(fmt.Errorf("instantiating module: %w", err))
cancel(fmt.Errorf("instantiating guest module: %w", err))
return
}
if err := instance.Close(ctx); err != nil {
cancel(fmt.Errorf("closing module: %w", err))
cancel(fmt.Errorf("closing guest module: %w", err))
return
}
}()
Expand All @@ -184,21 +202,24 @@ var (
hostProfile bool
hostTime bool
inuseMemory bool
verbose bool
mounts string
printVersion bool
)

var version = "dev"
version = "dev"
stdout = log.Default()
stderr = log.New(os.Stderr, "ERROR: ", 0)
)

func init() {
log.Default().SetOutput(os.Stderr)
flag.StringVar(&pprofAddr, "pprof-addr", "", "Address where to expose a pprof HTTP endpoint.")
flag.StringVar(&cpuProfile, "cpuprofile", "", "Write a CPU profile to the specified file before exiting.")
flag.StringVar(&memProfile, "memprofile", "", "Write a memory profile to the specified file before exiting.")
flag.Float64Var(&sampleRate, "sample", defaultSampleRate, "Set the profile sampling rate (0-1).")
flag.BoolVar(&hostProfile, "host", false, "Generate profiles of the host instead of the guest application.")
flag.BoolVar(&hostTime, "iowait", false, "Include time spent waiting on I/O in guest CPU profile.")
flag.BoolVar(&inuseMemory, "inuse", false, "Include snapshots of memory in use (experimental).")
flag.BoolVar(&verbose, "verbose", false, "Enable more output")
flag.StringVar(&mounts, "mount", "", "Comma-separated list of directories to mount (e.g. /tmp:/tmp:ro).")
flag.BoolVar(&printVersion, "version", false, "Print the wzprof version.")
}
Expand All @@ -217,6 +238,14 @@ func run(ctx context.Context) error {
return fmt.Errorf("usage: wzprof </path/to/app.wasm>")
}

if verbose {
log.SetPrefix("==> ")
log.SetFlags(0)
log.SetOutput(os.Stdout)
} else {
log.SetOutput(io.Discard)
}

filePath := args[0]

rate := int(math.Ceil(1 / sampleRate))
Expand Down Expand Up @@ -246,25 +275,28 @@ func split(s string) []string {

func startCPUProfile(f *os.File) {
if err := pprof.StartCPUProfile(f); err != nil {
log.Print("ERROR: starting CPU profile:", err)
stderr.Print("starting CPU profile:", err)
}
}

func stopCPUProfile() {
func stopCPUProfile(f *os.File) {
stdout.Printf("writing host cpu profile to %s", f.Name())
pprof.StopCPUProfile()
}

func writeHeapProfile(f *os.File) {
stdout.Printf("writing host memory profile to %s", f.Name())
if err := pprof.WriteHeapProfile(f); err != nil {
log.Print("ERROR: writing heap profile:", err)
stderr.Print("writing memory profile:", err)
}
}

func writeProfile(wasmName, path string, prof *profile.Profile) {
func writeProfile(profileName, wasmName, path string, prof *profile.Profile) {
m := &profile.Mapping{ID: 1, File: wasmName}
prof.Mapping = []*profile.Mapping{m}
stdout.Printf("writing guest %s profile to %s", profileName, path)
if err := wzprof.WriteProfile(path, prof); err != nil {
log.Print("ERROR: writing profile:", err)
stderr.Print("writing profile:", err)
}
}

Expand All @@ -273,7 +305,7 @@ func createFSConfig(mounts []string) wazero.FSConfig {
for _, m := range mounts {
parts := strings.Split(m, ":")
if len(parts) < 2 {
log.Fatalf("invalid mount: %s", m)
stderr.Fatalf("invalid mount: %s", m)
}

var mode string
Expand Down
18 changes: 14 additions & 4 deletions dwarf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"log"
"math"
"sort"
"sync"

"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
Expand Down Expand Up @@ -36,12 +37,15 @@ type subprogramRange struct {
type dwarfmapper struct {
d *dwarf.Data
subprograms []subprogramRange
// once value used to limit the logging output on error
onceSourceOffsetNotFound sync.Once
}

func newDwarfmapper(sections []api.CustomSection) (*dwarfmapper, error) {
var info, line, ranges, str, abbrev []byte

for _, section := range sections {
log.Printf("dwarf: found section %s", section.Name())
switch section.Name() {
case ".debug_info":
info = section.Data()
Expand All @@ -65,6 +69,7 @@ func newDwarfmapper(sections []api.CustomSection) (*dwarfmapper, error) {

p := dwarfparser{d: d, r: r}
subprograms := p.Parse()
log.Printf("dwarf: parsed %d subprogramm ranges", len(subprograms))

dm := &dwarfmapper{
d: d,
Expand Down Expand Up @@ -161,7 +166,7 @@ func (d *dwarfparser) parseSubprogram(cu *dwarf.Entry, ns string, e *dwarf.Entry

ranges, err := d.d.Ranges(e)
if err != nil {
log.Printf("profiler: dwarf: failed to read ranges: %s\n", err)
log.Printf("dwarf: failed to read ranges: %s\n", err)
return
}

Expand Down Expand Up @@ -206,12 +211,15 @@ func (d *dwarfmapper) LocationsForSourceOffset(offset uint64) []Location {
}

if spgm == nil {
d.onceSourceOffsetNotFound.Do(func() {
log.Printf("dwarf: no subprogram ranges found for source offset %d (silencing similar errors now)", offset)
})
return nil
}

lr, err := d.d.LineReader(spgm.CU)
if err != nil || lr == nil {
log.Printf("profiler: dwarf: failed to read lines: %s\n", err)
log.Printf("dwarf: failed to read lines: %s\n", err)
return nil
}

Expand All @@ -225,7 +233,7 @@ func (d *dwarfmapper) LocationsForSourceOffset(offset uint64) []Location {
break
}
if err != nil {
log.Printf("profiler: dwarf: failed to iterate on lines: %s\n", err)
log.Printf("dwarf: failed to iterate on lines: %s\n", err)
break
}
lines = append(lines, line{Pos: pos, Address: le.Address})
Expand All @@ -235,6 +243,7 @@ func (d *dwarfmapper) LocationsForSourceOffset(offset uint64) []Location {
i := sort.Search(len(lines), func(i int) bool { return lines[i].Address >= offset })
if i == len(lines) {
// no line information for this source offset.
log.Printf("dwarf: no line information for source offset %d", offset)
return nil
}

Expand All @@ -249,6 +258,7 @@ func (d *dwarfmapper) LocationsForSourceOffset(offset uint64) []Location {
// https://github.com/gimli-rs/addr2line/blob/3a2dbaf84551a06a429f26e9c96071bb409b371f/src/lib.rs#L236-L242
// https://github.com/kateinoigakukun/wasminspect/blob/f29f052f1b03104da9f702508ac0c1bbc3530ae4/crates/debugger/src/dwarf/mod.rs#L453-L459
if i-1 < 0 {
log.Printf("dwarf: first line address does not match source (line=%d offset=%d)", l.Address, offset)
return nil
}
l = lines[i-1]
Expand All @@ -259,7 +269,7 @@ func (d *dwarfmapper) LocationsForSourceOffset(offset uint64) []Location {
if err != nil {
// l.Pos was created from parsing dwarf, should not
// happen.
panic("bug")
panic("BUG: l.Pos was created from parsing dwarf but got error: " + err.Error())
}

human, stable := d.namesForSubprogram(spgm.Entry, spgm)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ go 1.20

require (
github.com/google/pprof v0.0.0-20230406165453-00490a63f317
github.com/tetratelabs/wazero v1.1.1-0.20230512012700-13e38966b6e0
github.com/tetratelabs/wazero v1.1.1-0.20230522055633-256b7a4bf970
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
)
8 changes: 2 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
github.com/google/pprof v0.0.0-20230406165453-00490a63f317 h1:hFhpt7CTmR3DX+b4R19ydQFtofxT0Sv3QsKNMVQYTMQ=
github.com/google/pprof v0.0.0-20230406165453-00490a63f317/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
github.com/tetratelabs/wazero v1.1.0 h1:EByoAhC+QcYpwSZJSs/aV0uokxPwBgKxfiokSUwAknQ=
github.com/tetratelabs/wazero v1.1.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
github.com/tetratelabs/wazero v1.1.1-0.20230511035210-78c35acd6e1c h1:3vKw5AyUv+16qm9tIVfvm205aDIpDmrjxPydJ87GBKY=
github.com/tetratelabs/wazero v1.1.1-0.20230511035210-78c35acd6e1c/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
github.com/tetratelabs/wazero v1.1.1-0.20230512012700-13e38966b6e0 h1:HlJHKQkBSPsHxseQausjzcbvU+njYSkagrWdCS7x4kU=
github.com/tetratelabs/wazero v1.1.1-0.20230512012700-13e38966b6e0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
github.com/tetratelabs/wazero v1.1.1-0.20230522055633-256b7a4bf970 h1:X5OOeHRjoLA8XhVc7biEbh1/hnTzpYpPn7HuyarMslQ=
github.com/tetratelabs/wazero v1.1.1-0.20230522055633-256b7a4bf970/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
1 change: 1 addition & 0 deletions testdata/.wasi-libc
Open in desktop
Copy link
Contributor

@pelletier pelletier May 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is that submodule checked in on purpose?

Copy link
Contributor Author

@achille-roussel achille-roussel May 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the C/Rust builds are going to depend on it, so we can automate the full compilation.

Submodule .wasi-libc added at a6f871
Loading