dashapi.go

// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package dashapi defines data structures used in dashboard communication
// and provides client interface.
package dashapi

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Dashboard struct {
	Client       string
	Addr         string
	Key          string
	ctor         RequestCtor
	doer         RequestDoer
	logger       RequestLogger
	errorHandler func(error)
}

func New(client, addr, key string) *Dashboard {
	return NewCustom(client, addr, key, http.NewRequest, http.DefaultClient.Do, nil, nil)
}

type (
	RequestCtor   func(method, url string, body io.Reader) (*http.Request, error)
	RequestDoer   func(req *http.Request) (*http.Response, error)
	RequestLogger func(msg string, args ...interface{})
)

func NewCustom(client, addr, key string, ctor RequestCtor, doer RequestDoer,
	logger RequestLogger, errorHandler func(error)) *Dashboard {
	return &Dashboard{
		Client:       client,
		Addr:         addr,
		Key:          key,
		ctor:         ctor,
		doer:         doer,
		logger:       logger,
		errorHandler: errorHandler,
	}
}

// Build describes all aspects of a kernel build.
type Build struct {
	Manager             string
	ID                  string
	OS                  string
	Arch                string
	VMArch              string
	SyzkallerCommit     string
	SyzkallerCommitDate time.Time
	CompilerID          string
	KernelRepo          string
	KernelBranch        string
	KernelCommit        string
	KernelCommitTitle   string
	KernelCommitDate    time.Time
	KernelConfig        []byte
	Commits             []string // see BuilderPoll
	FixCommits          []Commit
}

type Commit struct {
	Hash       string
	Title      string
	Author     string
	AuthorName string
	CC         []string
	BugIDs     []string // ID's extracted from Reported-by tags
	Date       time.Time
}

func (dash *Dashboard) UploadBuild(build *Build) error {
	return dash.Query("upload_build", build, nil)
}

// BuilderPoll request is done by kernel builder before uploading a new build
// with UploadBuild request. Response contains list of commit titles that
// dashboard is interested in (i.e. commits that fix open bugs) and email that
// appears in Reported-by tags for bug ID extraction. When uploading a new build
// builder will pass subset of the commit titles that are present in the build
// in Build.Commits field and list of {bug ID, commit title} pairs extracted
// from git log.

type BuilderPollReq struct {
	Manager string
}

type BuilderPollResp struct {
	PendingCommits []string
	ReportEmail    string
}

func (dash *Dashboard) BuilderPoll(manager string) (*BuilderPollResp, error) {
	req := &BuilderPollReq{
		Manager: manager,
	}
	resp := new(BuilderPollResp)
	err := dash.Query("builder_poll", req, resp)
	return resp, err
}

// Jobs workflow:
//   - syz-ci sends JobPollReq periodically to check for new jobs,
//     request contains list of managers that this syz-ci runs.
//   - dashboard replies with JobPollResp that contains job details,
//     if no new jobs available ID is set to empty string.
//   - when syz-ci finishes the job, it sends JobDoneReq which contains
//     job execution result (Build, Crash or Error details),
//     ID must match JobPollResp.ID.

type JobPollReq struct {
	PatchTestManagers []string
	BisectManagers    []string
}

type JobPollResp struct {
	ID                string
	Type              JobType
	Manager           string
	KernelRepo        string
	KernelBranch      string
	KernelCommit      string
	KernelCommitTitle string
	KernelCommitDate  time.Time
	KernelConfig      []byte
	SyzkallerCommit   string
	Patch             []byte
	ReproOpts         []byte
	ReproSyz          []byte
	ReproC            []byte
}

type JobDoneReq struct {
	ID          string
	Build       Build
	Error       []byte
	Log         []byte // bisection log
	CrashTitle  string
	CrashLog    []byte
	CrashReport []byte
	// Bisection results:
	// If there is 0 commits:
	//  - still happens on HEAD for fix bisection
	//  - already happened on the oldest release
	// If there is 1 commits: bisection result (cause or fix).
	// If there are more than 1: suspected commits due to skips (broken build/boot).
	Commits []Commit
}

type JobType int

const (
	JobTestPatch JobType = iota
	JobBisectCause
	JobBisectFix
)

func (dash *Dashboard) JobPoll(req *JobPollReq) (*JobPollResp, error) {
	resp := new(JobPollResp)
	err := dash.Query("job_poll", req, resp)
	return resp, err
}

func (dash *Dashboard) JobDone(req *JobDoneReq) error {
	return dash.Query("job_done", req, nil)
}

type BuildErrorReq struct {
	Build Build
	Crash Crash
}

func (dash *Dashboard) ReportBuildError(req *BuildErrorReq) error {
	return dash.Query("report_build_error", req, nil)
}

type CommitPollResp struct {
	ReportEmail string
	Repos       []Repo
	Commits     []string
}

type CommitPollResultReq struct {
	Commits []Commit
}

type Repo struct {
	URL    string
	Branch string
}

func (dash *Dashboard) CommitPoll() (*CommitPollResp, error) {
	resp := new(CommitPollResp)
	err := dash.Query("commit_poll", nil, resp)
	return resp, err
}

func (dash *Dashboard) UploadCommits(commits []Commit) error {
	if len(commits) == 0 {
		return nil
	}
	return dash.Query("upload_commits", &CommitPollResultReq{commits}, nil)
}

// Crash describes a single kernel crash (potentially with repro).
type Crash struct {
	BuildID     string // refers to Build.ID
	Title       string
	Corrupted   bool // report is corrupted (corrupted title, no stacks, etc)
	Maintainers []string
	Log         []byte
	Report      []byte
	// The following is optional and is filled only after repro.
	ReproOpts []byte
	ReproSyz  []byte
	ReproC    []byte
}

type ReportCrashResp struct {
	NeedRepro bool
}

func (dash *Dashboard) ReportCrash(crash *Crash) (*ReportCrashResp, error) {
	resp := new(ReportCrashResp)
	err := dash.Query("report_crash", crash, resp)
	return resp, err
}

// CrashID is a short summary of a crash for repro queries.
type CrashID struct {
	BuildID   string
	Title     string
	Corrupted bool
}

type NeedReproResp struct {
	NeedRepro bool
}

// NeedRepro checks if dashboard needs a repro for this crash or not.
func (dash *Dashboard) NeedRepro(crash *CrashID) (bool, error) {
	resp := new(NeedReproResp)
	err := dash.Query("need_repro", crash, resp)
	return resp.NeedRepro, err
}

// ReportFailedRepro notifies dashboard about a failed repro attempt for the crash.
func (dash *Dashboard) ReportFailedRepro(crash *CrashID) error {
	return dash.Query("report_failed_repro", crash, nil)
}

type LogEntry struct {
	Name string
	Text string
}

// Centralized logging on dashboard.
func (dash *Dashboard) LogError(name, msg string, args ...interface{}) {
	req := &LogEntry{
		Name: name,
		Text: fmt.Sprintf(msg, args...),
	}
	dash.Query("log_error", req, nil)
}

// BugReport describes a single bug.
// Used by dashboard external reporting.
type BugReport struct {
	Type              ReportType
	Namespace         string
	Config            []byte
	ID                string
	JobID             string
	ExtID             string // arbitrary reporting ID forwarded from BugUpdate.ExtID
	First             bool   // Set for first report for this bug (Type == ReportNew).
	Moderation        bool
	Title             string
	Link              string // link to the bug on dashboard
	CreditEmail       string // email for the Reported-by tag
	Maintainers       []string
	CC                []string // additional CC emails
	OS                string
	Arch              string
	VMArch            string
	UserSpaceArch     string // user-space arch as kernel developers know it (rather than Go names)
	CompilerID        string
	KernelRepo        string
	KernelRepoAlias   string
	KernelBranch      string
	KernelCommit      string
	KernelCommitTitle string
	KernelCommitDate  time.Time
	KernelConfig      []byte
	KernelConfigLink  string
	Log               []byte
	LogLink           string
	Report            []byte
	ReportLink        string
	ReproC            []byte
	ReproCLink        string
	ReproSyz          []byte
	ReproSyzLink      string
	CrashID           int64 // returned back in BugUpdate
	NumCrashes        int64
	HappenedOn        []string // list of kernel repo aliases

	CrashTitle     string // job execution crash title
	Error          []byte // job execution error
	ErrorLink      string
	ErrorTruncated bool // full Error text is too large and was truncated
	PatchLink      string
	BisectCause    *BisectResult
	BisectFix      *BisectResult
}

type BisectResult struct {
	Commit          *Commit   // for conclusive bisection
	Commits         []*Commit // for inconclusive bisection
	LogLink         string
	CrashLogLink    string
	CrashReportLink string
}

type BugUpdate struct {
	ID           string // copied from BugReport
	JobID        string // copied from BugReport
	ExtID        string
	Link         string
	Status       BugStatus
	ReproLevel   ReproLevel
	DupOf        string
	OnHold       bool     // If set for open bugs, don't upstream this bug.
	Notification bool     // Reply to a notification.
	FixCommits   []string // Titles of commits that fix this bug.
	CC           []string // Additional emails to add to CC list in future emails.
	CrashID      int64
}

type BugUpdateReply struct {
	// Bug update can fail for 2 reason:
	//  - update does not pass logical validataion, in this case OK=false
	//  - internal/datastore error, in this case Error=true
	OK    bool
	Error bool
	Text  string
}

type PollBugsRequest struct {
	Type string
}

type PollBugsResponse struct {
	Reports []*BugReport
}

type BugNotification struct {
	Type        BugNotif
	Namespace   string
	Config      []byte
	ID          string
	ExtID       string // arbitrary reporting ID forwarded from BugUpdate.ExtID
	Title       string
	Text        string   // meaning depends on Type
	CC          []string // additional CC emails
	Maintainers []string
	// Public is what we want all involved people to see (e.g. if we notify about a wrong commit title,
	// people need to see it and provide the right title). Not public is what we want to send only
	// to a minimal set of recipients (our mailing list) (e.g. notification about an obsoleted bug
	// is mostly "for the record").
	Public bool
}

type PollNotificationsRequest struct {
	Type string
}

type PollNotificationsResponse struct {
	Notifications []*BugNotification
}

type PollClosedRequest struct {
	IDs []string
}

type PollClosedResponse struct {
	IDs []string
}

func (dash *Dashboard) ReportingPollBugs(typ string) (*PollBugsResponse, error) {
	req := &PollBugsRequest{
		Type: typ,
	}
	resp := new(PollBugsResponse)
	if err := dash.Query("reporting_poll_bugs", req, resp); err != nil {
		return nil, err
	}
	return resp, nil
}

func (dash *Dashboard) ReportingPollNotifications(typ string) (*PollNotificationsResponse, error) {
	req := &PollNotificationsRequest{
		Type: typ,
	}
	resp := new(PollNotificationsResponse)
	if err := dash.Query("reporting_poll_notifs", req, resp); err != nil {
		return nil, err
	}
	return resp, nil
}

func (dash *Dashboard) ReportingPollClosed(ids []string) ([]string, error) {
	req := &PollClosedRequest{
		IDs: ids,
	}
	resp := new(PollClosedResponse)
	if err := dash.Query("reporting_poll_closed", req, resp); err != nil {
		return nil, err
	}
	return resp.IDs, nil
}

func (dash *Dashboard) ReportingUpdate(upd *BugUpdate) (*BugUpdateReply, error) {
	resp := new(BugUpdateReply)
	if err := dash.Query("reporting_update", upd, resp); err != nil {
		return nil, err
	}
	return resp, nil
}

type ManagerStatsReq struct {
	Name string
	Addr string

	// Current level:
	UpTime time.Duration
	Corpus uint64
	Cover  uint64

	// Delta since last sync:
	FuzzingTime time.Duration
	Crashes     uint64
	Execs       uint64
}

func (dash *Dashboard) UploadManagerStats(req *ManagerStatsReq) error {
	return dash.Query("manager_stats", req, nil)
}

type (
	BugStatus  int
	BugNotif   int
	ReproLevel int
	ReportType int
)

const (
	BugStatusOpen BugStatus = iota
	BugStatusUpstream
	BugStatusInvalid
	BugStatusDup
	BugStatusUpdate // aux info update (i.e. ExtID/Link/CC)
	BugStatusUnCC   // don't CC sender on any future communication
)

const (
	// Upstream bug into next reporting.
	// If the action succeeds, reporting sends BugStatusUpstream update.
	BugNotifUpstream BugNotif = iota
	// Bug needs to be closed as obsoleted.
	// If the action succeeds, reporting sends BugStatusInvalid update.
	BugNotifObsoleted
	// Bug fixing commit can't be discovered (wrong commit title).
	BugNotifBadCommit
)

const (
	ReproLevelNone ReproLevel = iota
	ReproLevelSyz
	ReproLevelC
)

const (
	ReportNew         ReportType = iota // First report for this bug in the reporting stage.
	ReportRepro                         // Found repro for an already reported bug.
	ReportTestPatch                     // Patch testing result.
	ReportBisectCause                   // Cause bisection result for an already reported bug.
	ReportBisectFix                     // Fix bisection result for an already reported bug.
)

func (dash *Dashboard) Query(method string, req, reply interface{}) error {
	if dash.logger != nil {
		dash.logger("API(%v): %#v", method, req)
	}
	err := dash.queryImpl(method, req, reply)
	if err != nil {
		if dash.logger != nil {
			dash.logger("API(%v): ERROR: %v", method, err)
		}
		if dash.errorHandler != nil {
			dash.errorHandler(err)
		}
		return err
	}
	if dash.logger != nil {
		dash.logger("API(%v): REPLY: %#v", method, reply)
	}
	return nil
}

func (dash *Dashboard) queryImpl(method string, req, reply interface{}) error {
	if reply != nil {
		// json decoding behavior is somewhat surprising
		// (see // https://github.com/golang/go/issues/21092).
		// To avoid any surprises, we zero the reply.
		typ := reflect.TypeOf(reply)
		if typ.Kind() != reflect.Ptr {
			return fmt.Errorf("resp must be a pointer")
		}
		reflect.ValueOf(reply).Elem().Set(reflect.New(typ.Elem()).Elem())
	}
	values := make(url.Values)
	values.Add("client", dash.Client)
	values.Add("key", dash.Key)
	values.Add("method", method)
	if req != nil {
		data, err := json.Marshal(req)
		if err != nil {
			return fmt.Errorf("failed to marshal request: %v", err)
		}
		buf := new(bytes.Buffer)
		gz := gzip.NewWriter(buf)
		if _, err := gz.Write(data); err != nil {
			return err
		}
		if err := gz.Close(); err != nil {
			return err
		}
		values.Add("payload", buf.String())
	}
	r, err := dash.ctor("POST", fmt.Sprintf("%v/api", dash.Addr), strings.NewReader(values.Encode()))
	if err != nil {
		return err
	}
	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := dash.doer(r)
	if err != nil {
		return fmt.Errorf("http request failed: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		data, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("request failed with %v: %s", resp.Status, data)
	}
	if reply != nil {
		if err := json.NewDecoder(resp.Body).Decode(reply); err != nil {
			return fmt.Errorf("failed to unmarshal response: %v", err)
		}
	}
	return nil
}