tools/syz-runtest: add tool for program unit testing

The tool is run as:

$ syz-runtest -config manager.config

This runs all programs from sys/*/test/* in different modes
on actual VMs and checks results.

Fixes #603

Makefile

@@ -79,7 +79,7 @@ ifeq ("$(TARGETOS)", "akaros")
 endif
 
 .PHONY: all host target \
-	manager fuzzer executor \
+	manager runtest fuzzer executor \
 	ci hub \
 	execprog mutate prog2c stress repro upgrade db \
 	bin/syz-sysgen bin/syz-extract bin/syz-fmt \
@@ -97,7 +97,7 @@ all: host target
 
 host:
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) install ./syz-manager
-	$(MAKE) manager repro mutate prog2c db upgrade
+	$(MAKE) manager runtest repro mutate prog2c db upgrade
 
 target:
 	GOOS=$(TARGETGOOS) GOARCH=$(TARGETGOARCH) $(GO) install ./syz-fuzzer
@@ -113,6 +113,9 @@ executor:
 manager:
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-manager github.com/google/syzkaller/syz-manager
 
+runtest:
+	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-runtest github.com/google/syzkaller/tools/syz-runtest
+
 fuzzer:
 	GOOS=$(TARGETGOOS) GOARCH=$(TARGETGOARCH) $(GO) build $(GOTARGETFLAGS) -o ./bin/$(TARGETOS)_$(TARGETVMARCH)/syz-fuzzer$(EXE) github.com/google/syzkaller/syz-fuzzer
 

pkg/instance/instance.go

@@ -267,7 +267,7 @@ func (inst *inst) testInstance() error {
 	}
 
 	cmd := FuzzerCmd(fuzzerBin, executorBin, "test", inst.cfg.TargetOS, inst.cfg.TargetArch, fwdAddr,
-		inst.cfg.Sandbox, 0, 0, false, false, true)
+		inst.cfg.Sandbox, 0, 0, false, false, true, false)
 	outc, errc, err := inst.vm.Run(5*time.Minute, nil, cmd)
 	if err != nil {
 		return fmt.Errorf("failed to run binary in VM: %v", err)
@@ -364,7 +364,7 @@ func (inst *inst) testProgram(command string, testTime time.Duration) error {
 }
 
 func FuzzerCmd(fuzzer, executor, name, OS, arch, fwdAddr, sandbox string, procs, verbosity int,
-	cover, debug, test bool) string {
+	cover, debug, test, runtest bool) string {
 	osArg := ""
 	if OS == "akaros" {
 		// Only akaros needs OS, because the rest assume host OS.
@@ -373,9 +373,9 @@ func FuzzerCmd(fuzzer, executor, name, OS, arch, fwdAddr, sandbox string, procs,
 		osArg = " -os=" + OS
 	}
 	return fmt.Sprintf("%v -executor=%v -name=%v -arch=%v%v -manager=%v -sandbox=%v"+
-		" -procs=%v -v=%d -cover=%v -debug=%v -test=%v",
+		" -procs=%v -v=%d -cover=%v -debug=%v -test=%v -runtest=%v",
 		fuzzer, executor, name, arch, osArg, fwdAddr, sandbox,
-		procs, verbosity, cover, debug, test)
+		procs, verbosity, cover, debug, test, runtest)
 }
 
 func ExecprogCmd(execprog, executor, OS, arch, sandbox string, repeat, threaded, collide bool,

pkg/rpctype/rpctype.go

@@ -7,6 +7,7 @@ package rpctype
 
 import (
 	"github.com/google/syzkaller/pkg/host"
+	"github.com/google/syzkaller/pkg/ipc"
 	"github.com/google/syzkaller/pkg/signal"
 )
 
@@ -31,14 +32,15 @@ type ConnectRes struct {
 	EnabledCalls   []int
 	GitRevision    string
 	TargetRevision string
+	AllSandboxes   bool
 	CheckResult    *CheckArgs
 }
 
 type CheckArgs struct {
 	Name          string
 	Error         string
-	EnabledCalls  []int
-	DisabledCalls []SyscallReason
+	EnabledCalls  map[string][]int
+	DisabledCalls map[string][]SyscallReason
 	Features      *host.Features
 }
 
@@ -103,3 +105,24 @@ type HubSyncRes struct {
 	// if >0 manager should do sync again.
 	More int
 }
+
+type RunTestPollReq struct {
+	Name string
+}
+
+type RunTestPollRes struct {
+	ID     int
+	Bin    []byte
+	Prog   []byte
+	Cfg    *ipc.Config
+	Opts   *ipc.ExecOpts
+	Repeat int
+}
+
+type RunTestDoneArgs struct {
+	Name   string
+	ID     int
+	Output []byte
+	Info   [][]ipc.CallInfo
+	Error  string
+}

pkg/runtest/run.go

@@ -21,7 +21,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"syscall"
 	"time"
 
 	"github.com/google/syzkaller/pkg/csource"
@@ -206,19 +205,20 @@ func (ctx *Context) parseProg(filename string) (*prog.Prog, []int, error) {
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to deserialize %v: %v", filename, err)
 	}
+	errnos := map[string]int{
+		"":       0,
+		"EPERM":  1,
+		"EBADF":  9,
+		"ENOMEM": 12,
+		"EINVAL": 22,
+	}
 	results := make([]int, len(p.Calls))
 	for i, call := range p.Calls {
-		switch call.Comment {
-		case "":
-		case "EINVAL":
-			results[i] = int(syscall.EINVAL)
-		case "EBADF":
-			results[i] = int(syscall.EBADF)
-		case "ENOMEM":
-			results[i] = int(syscall.ENOMEM)
-		default:
+		res, ok := errnos[call.Comment]
+		if !ok {
 			return nil, nil, fmt.Errorf("%v: unknown comment %q", filename, call.Comment)
 		}
+		results[i] = res
 	}
 	return p, results, nil
 }

sys/akaros/test/openat

@@ -0,0 +1,3 @@
+r0 = openat(0xffffffffffffff9c, &(0x7f0000000000)='./file0\x00', 0x7, 0x43, 0x1a4)
+write(r0, &(0x7f0000001000)="112233", 0x3)
+close(r0)

sys/fuchsia/test/fd_assignment

@@ -0,0 +1,9 @@
+# Tests that fd's are always allocated starting from 3.
+
+close(0x3)	# EBADF
+close(0x4)	# EBADF
+close(0x5)	# EBADF
+pipe(&(0x7f0000000000)={<r0=>0x0, <r1=>0x0})
+close(0x3)
+close(0x4)
+close(0x5)	# EBADF

sys/fuchsia/test/pipe

@@ -0,0 +1,3 @@
+pipe(&(0x7f0000000000)={<r0=>0x0, <r1=>0x0})
+close(r0)
+close(r1)

sys/linux/test/fd_assignment

@@ -0,0 +1,9 @@
+# Tests that fd's are always allocated starting from 3.
+
+close(0x3)	# EBADF
+close(0x4)	# EBADF
+close(0x5)	# EBADF
+pipe(&(0x7f0000000000)={<r0=>0x0, <r1=>0x0})
+close(0x3)
+close(0x4)
+close(0x5)	# EBADF

sys/linux/test/fuse_deadlock

@@ -0,0 +1,11 @@
+# Test for deadlock in fuse.
+# Some calls are commented out for now, because it actually deadlocks kernel.
+
+mkdirat(0xffffffffffffff9c, &(0x7f0000000000)='./file0\x00', 0x0)
+r0 = openat$fuse(0xffffffffffffff9c, &(0x7f0000000640)='/dev/fuse\x00', 0x2, 0x0)
+mount$fuse(0x0, &(0x7f0000000200)='./file0\x00', &(0x7f0000000300)='fuse\x00', 0x0, &(0x7f0000000400)={{'fd', 0x3d, r0}, 0x2c, {'rootmode', 0x3d, 0x4000}, 0x2c, {'user_id', 0x3d}, 0x2c, {'group_id', 0x3d}, 0x2c})
+#read$FUSE(r0, &(0x7f0000002000), 0x1000)
+#pread64(r0, &(0x7f0000000540)=""/236, 0xec, 0x0)
+#write$FUSE_INIT(r0, &(0x7f0000000100)={0x50, 0x0, 0x1, {0x7, 0x1b}}, 0x50)
+#mkdirat(0xffffffffffffff9c, &(0x7f0000000500)='./file0/file0\x00', 0x0)
+#write$FUSE_NOTIFY_INVAL_ENTRY(r0, &(0x7f00000000c0)={0x29, 0x3, 0x0, {0x1, 0x8, 0x0, 'group_id'}}, 0x29)

sys/linux/test/pipe

@@ -0,0 +1,3 @@
+pipe(&(0x7f0000000000)={<r0=>0x0, <r1=>0x0})
+close(r0)
+close(r1)

sys/linux/test/vnet_tun

@@ -0,0 +1,2 @@
+syz_emit_ethernet()
+syz_emit_ethernet()

syz-fuzzer/fuzzer.go

@@ -100,7 +100,8 @@ func main() {
 		flagProcs   = flag.Int("procs", 1, "number of parallel test processes")
 		flagOutput  = flag.String("output", "stdout", "write programs to none/stdout/dmesg/file")
 		flagPprof   = flag.String("pprof", "", "address to serve pprof profiles")
-		flagTest    = flag.Bool("test", false, "enable image testing mode") // used by syz-ci
+		flagTest    = flag.Bool("test", false, "enable image testing mode")      // used by syz-ci
+		flagRunTest = flag.Bool("runtest", false, "enable program testing mode") // used by pkg/runtest
 	)
 	flag.Parse()
 	outputType := parseOutputType(*flagOutput)
@@ -165,6 +166,7 @@ func main() {
 		checkArgs.gitRevision = r.GitRevision
 		checkArgs.targetRevision = r.TargetRevision
 		checkArgs.enabledCalls = r.EnabledCalls
+		checkArgs.allSandboxes = r.AllSandboxes
 		r.CheckResult, err = checkMachine(checkArgs)
 		if err != nil {
 			r.CheckResult = &rpctype.CheckArgs{
@@ -197,6 +199,11 @@ func main() {
 		config.Flags |= ipc.FlagEnableFault
 	}
 
+	if *flagRunTest {
+		runTest(target, manager, *flagName, config.Executor)
+		return
+	}
+
 	needPoll := make(chan struct{}, 1)
 	needPoll <- struct{}{}
 	fuzzer := &Fuzzer{
@@ -216,7 +223,7 @@ func main() {
 	for i := 0; fuzzer.poll(i == 0, nil); i++ {
 	}
 	calls := make(map[*prog.Syscall]bool)
-	for _, id := range r.CheckResult.EnabledCalls {
+	for _, id := range r.CheckResult.EnabledCalls[sandbox] {
 		calls[target.Syscalls[id]] = true
 	}
 	prios := target.CalculatePriorities(fuzzer.corpus)

syz-fuzzer/testing.go

@@ -14,6 +14,7 @@ import (
 	"github.com/google/syzkaller/pkg/log"
 	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/pkg/rpctype"
+	"github.com/google/syzkaller/pkg/runtest"
 	"github.com/google/syzkaller/prog"
 	"github.com/google/syzkaller/sys"
 )
@@ -24,6 +25,7 @@ type checkArgs struct {
 	gitRevision    string
 	targetRevision string
 	enabledCalls   []int
+	allSandboxes   bool
 	ipcConfig      *ipc.Config
 	ipcExecOpts    *ipc.ExecOpts
 }
@@ -40,6 +42,64 @@ func testImage(hostAddr string, args *checkArgs) {
 	}
 }
 
+func runTest(target *prog.Target, manager *rpctype.RPCClient, name, executor string) {
+	pollReq := &rpctype.RunTestPollReq{Name: name}
+	for {
+		req := new(rpctype.RunTestPollRes)
+		if err := manager.Call("Manager.Poll", pollReq, req); err != nil {
+			log.Fatalf("Manager.Poll call failed: %v", err)
+		}
+		if len(req.Bin) == 0 && len(req.Prog) == 0 {
+			return
+		}
+		test := convertTestReq(target, req)
+		if test.Err == nil {
+			runtest.RunTest(test, executor)
+		}
+		reply := &rpctype.RunTestDoneArgs{
+			Name:   name,
+			ID:     req.ID,
+			Output: test.Output,
+			Info:   test.Info,
+		}
+		if test.Err != nil {
+			reply.Error = test.Err.Error()
+		}
+		if err := manager.Call("Manager.Done", reply, nil); err != nil {
+			log.Fatalf("Manager.Done call failed: %v", err)
+		}
+	}
+}
+
+func convertTestReq(target *prog.Target, req *rpctype.RunTestPollRes) *runtest.RunRequest {
+	test := &runtest.RunRequest{
+		Cfg:    req.Cfg,
+		Opts:   req.Opts,
+		Repeat: req.Repeat,
+	}
+	if len(req.Bin) != 0 {
+		bin, err := osutil.TempFile("syz-runtest")
+		if err != nil {
+			test.Err = err
+			return test
+		}
+		if err := osutil.WriteExecFile(bin, req.Bin); err != nil {
+			test.Err = err
+			return test
+		}
+		test.Bin = bin
+	}
+	if len(req.Prog) != 0 {
+		p, err := target.Deserialize(req.Prog)
+		if err != nil {
+			test.Err = err
+			return test
+		}
+		test.P = p
+	}
+	return test
+}
+
 func checkMachine(args *checkArgs) (*rpctype.CheckArgs, error) {
 	// Machine checking can be very slow on some machines (qemu without kvm, KMEMLEAK linux, etc),
 	// so print periodic heartbeats for vm.MonitorExecution so that it does not decide that we are dead.
@@ -79,14 +139,39 @@ func checkMachine(args *checkArgs) (*rpctype.CheckArgs, error) {
 	if err := checkSimpleProgram(args); err != nil {
 		return nil, err
 	}
-	enabledCalls, disabledCalls, err := buildCallList(args.target, args.enabledCalls, args.sandbox)
-	if err != nil {
-		return nil, err
-	}
 	res := &rpctype.CheckArgs{
-		EnabledCalls:  enabledCalls,
-		DisabledCalls: disabledCalls,
 		Features:      features,
+		EnabledCalls:  make(map[string][]int),
+		DisabledCalls: make(map[string][]rpctype.SyscallReason),
+	}
+	sandboxes := []string{args.sandbox}
+	if args.allSandboxes {
+		if features[host.FeatureSandboxSetuid].Enabled {
+			sandboxes = append(sandboxes, "setuid")
+		}
+		if features[host.FeatureSandboxNamespace].Enabled {
+			sandboxes = append(sandboxes, "namespace")
+		}
+	}
+	for _, sandbox := range sandboxes {
+		enabledCalls, disabledCalls, err := buildCallList(args.target, args.enabledCalls, sandbox)
+		if err != nil {
+			return nil, err
+		}
+		res.EnabledCalls[sandbox] = enabledCalls
+		res.DisabledCalls[sandbox] = disabledCalls
+	}
+	if args.allSandboxes {
+		var enabled []int
+		for _, id := range res.EnabledCalls["none"] {
+			switch args.target.Syscalls[id].Name {
+			default:
+				enabled = append(enabled, id)
+			case "syz_emit_ethernet", "syz_extract_tcp_res":
+				// Tun is not setup without sandbox, this is a hacky way to workaround this.
+			}
+		}
+		res.EnabledCalls[""] = enabled
 	}
 	return res, nil
 }

syz-manager/html.go

@@ -251,7 +251,7 @@ func (mgr *Manager) httpCoverFallback(w http.ResponseWriter, r *http.Request) {
 		calls[id] = append(calls[id], errno)
 	}
 	data := &UIFallbackCoverData{}
-	for _, id := range mgr.checkResult.EnabledCalls {
+	for _, id := range mgr.checkResult.EnabledCalls[mgr.cfg.Sandbox] {
 		errnos := calls[id]
 		sort.Ints(errnos)
 		successful := 0

syz-manager/hub.go

@@ -20,7 +20,7 @@ func (mgr *Manager) hubSyncLoop() {
 		cfg:           mgr.cfg,
 		target:        mgr.target,
 		stats:         mgr.stats,
-		enabledCalls:  mgr.checkResult.EnabledCalls,
+		enabledCalls:  mgr.checkResult.EnabledCalls[mgr.cfg.Sandbox],
 		fresh:         mgr.fresh,
 		hubReproQueue: mgr.hubReproQueue,
 	}