entities.go

// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package dash

import (
	"fmt"
	"regexp"
	"strconv"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/hash"
	"golang.org/x/net/context"
	"google.golang.org/appengine/datastore"
)

// This file contains definitions of entities stored in datastore.

const (
	maxTextLen   = 200
	MaxStringLen = 1024

	maxCrashes = 40
)

type Manager struct {
	Namespace      string
	Name           string
	Link           string
	CurrentBuild   string
	FailedBuildBug string
	LastAlive      time.Time
	CurrentUpTime  time.Duration
}

// ManagerStats holds per-day manager runtime stats.
// Has Manager as parent entity. Keyed by Date.
type ManagerStats struct {
	Date             int // YYYYMMDD
	MaxCorpus        int64
	MaxCover         int64
	TotalFuzzingTime time.Duration
	TotalCrashes     int64
	TotalExecs       int64
}

type Build struct {
	Namespace         string
	Manager           string
	ID                string // unique ID generated by syz-ci
	Type              BuildType
	Time              time.Time
	OS                string
	Arch              string
	VMArch            string
	SyzkallerCommit   string
	CompilerID        string
	KernelRepo        string
	KernelBranch      string
	KernelCommit      string
	KernelCommitTitle string    `datastore:",noindex"`
	KernelCommitDate  time.Time `datastore:",noindex"`
	KernelConfig      int64     // reference to KernelConfig text entity
}

type Bug struct {
	Namespace      string
	Seq            int64 // sequences of the bug with the same title
	Title          string
	Status         int
	DupOf          string
	NumCrashes     int64
	NumRepro       int64
	ReproLevel     dashapi.ReproLevel
	HasReport      bool
	FirstTime      time.Time
	LastTime       time.Time
	LastSavedCrash time.Time
	LastReproTime  time.Time
	Closed         time.Time
	Reporting      []BugReporting
	Commits        []string
	HappenedOn     []string `datastore:",noindex"` // list of managers
	PatchedOn      []string `datastore:",noindex"` // list of managers
}

type BugReporting struct {
	Name       string // refers to Reporting.Name
	ID         string // unique ID per BUG/BugReporting used in commucation with external systems
	ExtID      string // arbitrary reporting ID that is passed back in dashapi.BugReport
	Link       string
	CC         string // additional emails added to CC list (|-delimited list)
	CrashID    int64  // crash that we've last reported in this reporting
	ReproLevel dashapi.ReproLevel
	Reported   time.Time
	Closed     time.Time
}

type Crash struct {
	Manager     string
	BuildID     string
	Time        time.Time
	Reported    time.Time // set if this crash was ever reported
	Maintainers []string  `datastore:",noindex"`
	Log         int64     // reference to CrashLog text entity
	Report      int64     // reference to CrashReport text entity
	ReproOpts   []byte    `datastore:",noindex"`
	ReproSyz    int64     // reference to ReproSyz text entity
	ReproC      int64     // reference to ReproC text entity
	// Custom crash priority for reporting (greater values are higher priority).
	// For example, a crash in mainline kernel has higher priority than a crash in a side branch.
	// For historical reasons this is called ReportLen.
	ReportLen int
}

// ReportingState holds dynamic info associated with reporting.
type ReportingState struct {
	Entries []ReportingStateEntry
}

type ReportingStateEntry struct {
	Namespace string
	Name      string
	// Current reporting quota consumption.
	Sent int
	Date int // YYYYMMDD
}

// Job represent a single patch testing job for syz-ci.
// Later we may want to extend this to other types of jobs (hense the generic name):
//   - test of a committed fix
//   - reproduce crash
//   - test that crash still happens on HEAD
//   - crash bisect
// Job has Bug as parent entity.
type Job struct {
	Created   time.Time
	User      string
	CC        []string
	Reporting string
	ExtID     string // email Message-ID
	Link      string // web link for the job (e.g. email in the group)
	Namespace string
	Manager   string
	BugTitle  string
	CrashID   int64

	// Provided by user:
	KernelRepo   string
	KernelBranch string
	Patch        int64 // reference to Patch text entity

	Attempts int // number of times we tried to execute this job
	Started  time.Time
	Finished time.Time // if set, job is finished

	// Result of execution:
	CrashTitle  string // if empty, we did not hit crash during testing
	CrashLog    int64  // reference to CrashLog text entity
	CrashReport int64  // reference to CrashReport text entity
	BuildID     string
	Error       int64 // reference to Error text entity, if set job failed

	Reported bool // have we reported result back to user?
}

// Text holds text blobs (crash logs, reports, reproducers, etc).
type Text struct {
	Namespace string
	Text      []byte `datastore:",noindex"` // gzip-compressed text
}

const (
	textCrashLog     = "CrashLog"
	textCrashReport  = "CrashReport"
	textReproSyz     = "ReproSyz"
	textReproC       = "ReproC"
	textKernelConfig = "KernelConfig"
	textPatch        = "Patch"
	textError        = "Error"
)

const (
	BugStatusOpen = iota
)

const (
	BugStatusFixed = 1000 + iota
	BugStatusInvalid
	BugStatusDup
)

const (
	ReproLevelNone = dashapi.ReproLevelNone
	ReproLevelSyz  = dashapi.ReproLevelSyz
	ReproLevelC    = dashapi.ReproLevelC
)

type BuildType int

const (
	BuildNormal BuildType = iota
	BuildFailed
	BuildJob
)

// updateManager does transactional compare-and-swap on the manager and its current stats.
func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats)) error {
	date := timeDate(timeNow(c))
	tx := func(c context.Context) error {
		mgr := new(Manager)
		mgrKey := datastore.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil)
		if err := datastore.Get(c, mgrKey, mgr); err != nil {
			if err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("failed to get manager %v/%v: %v", ns, name, err)
			}
			mgr = &Manager{
				Namespace: ns,
				Name:      name,
			}
		}
		stats := new(ManagerStats)
		statsKey := datastore.NewKey(c, "ManagerStats", "", int64(date), mgrKey)
		if err := datastore.Get(c, statsKey, stats); err != nil {
			if err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("failed to get stats %v/%v/%v: %v", ns, name, date, err)
			}
			stats = &ManagerStats{
				Date: date,
			}
		}

		fn(mgr, stats)

		if _, err := datastore.Put(c, mgrKey, mgr); err != nil {
			return fmt.Errorf("failed to put manager: %v", err)
		}
		if _, err := datastore.Put(c, statsKey, stats); err != nil {
			return fmt.Errorf("failed to put manager stats: %v", err)
		}
		return nil
	}
	return datastore.RunInTransaction(c, tx, &datastore.TransactionOptions{Attempts: 10})
}

func loadAllManagers(c context.Context) ([]*Manager, []*datastore.Key, error) {
	var managers []*Manager
	keys, err := datastore.NewQuery("Manager").
		GetAll(c, &managers)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to query managers: %v", err)
	}
	var result []*Manager
	var resultKeys []*datastore.Key

	for i, mgr := range managers {
		if config.Namespaces[mgr.Namespace].Managers[mgr.Name].Decommissioned {
			continue
		}
		result = append(result, mgr)
		resultKeys = append(resultKeys, keys[i])
	}
	return result, resultKeys, nil
}

func buildKey(c context.Context, ns, id string) *datastore.Key {
	if ns == "" {
		panic("requesting build key outside of namespace")
	}
	h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id)))
	return datastore.NewKey(c, "Build", h, 0, nil)
}

func loadBuild(c context.Context, ns, id string) (*Build, error) {
	build := new(Build)
	if err := datastore.Get(c, buildKey(c, ns, id), build); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return nil, fmt.Errorf("unknown build %v/%v", ns, id)
		}
		return nil, fmt.Errorf("failed to get build %v/%v: %v", ns, id, err)
	}
	return build, nil
}

func lastManagerBuild(c context.Context, ns, manager string) (*Build, error) {
	var builds []*Build
	_, err := datastore.NewQuery("Build").
		Filter("Namespace=", ns).
		Filter("Manager=", manager).
		Filter("Type=", BuildNormal).
		Order("-Time").
		Limit(1).
		GetAll(c, &builds)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch manager build: %v", err)
	}
	if len(builds) == 0 {
		return nil, fmt.Errorf("failed to fetch manager build: no builds")
	}
	return builds[0], nil
}

func (bug *Bug) displayTitle() string {
	if bug.Seq == 0 {
		return bug.Title
	}
	return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1)
}

var displayTitleRe = regexp.MustCompile(`^(.*) \(([0-9]+)\)$`)

func splitDisplayTitle(display string) (string, int64, error) {
	match := displayTitleRe.FindStringSubmatchIndex(display)
	if match == nil {
		return display, 0, nil
	}
	title := display[match[2]:match[3]]
	seqStr := display[match[4]:match[5]]
	seq, err := strconv.ParseInt(seqStr, 10, 64)
	if err != nil {
		return "", 0, fmt.Errorf("failed to parse bug title: %v", err)
	}
	if seq <= 0 || seq > 1e6 {
		return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq)
	}
	return title, seq - 1, nil
}

func canonicalBug(c context.Context, bug *Bug) (*Bug, error) {
	for {
		if bug.Status != BugStatusDup {
			return bug, nil
		}
		canon := new(Bug)
		bugKey := datastore.NewKey(c, "Bug", bug.DupOf, 0, nil)
		if err := datastore.Get(c, bugKey, canon); err != nil {
			return nil, fmt.Errorf("failed to get dup bug %q for %q: %v",
				bug.DupOf, bugKeyHash(bug.Namespace, bug.Title, bug.Seq), err)
		}
		bug = canon
	}
}

func bugKeyHash(ns, title string, seq int64) string {
	return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", config.Namespaces[ns].Key, ns, title, seq)))
}

func bugReportingHash(bugHash, reporting string) string {
	// Since these IDs appear in Reported-by tags in commit, we slightly limit their size.
	const hashLen = 20
	return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:hashLen]
}

func kernelRepoInfo(build *Build) KernelRepo {
	return kernelRepoInfoRaw(build.KernelRepo, build.KernelBranch)
}

func kernelRepoInfoRaw(repo, branch string) KernelRepo {
	repoID := repo
	if branch != "" {
		repoID += "/" + branch
	}
	info := config.KernelRepos[repoID]
	if info.Alias == "" {
		info.Alias = repoID
	}
	return info
}

func textLink(tag string, id int64) string {
	if id == 0 {
		return ""
	}
	return fmt.Sprintf("/text?tag=%v&x=%v", tag, strconv.FormatUint(uint64(id), 16))
}

// timeDate returns t's date as a single int YYYYMMDD.
func timeDate(t time.Time) int {
	year, month, day := t.Date()
	return year*10000 + int(month)*100 + day
}