dashboard: introduce an emergency stop mode

Add an emergency stop button that can be used by any admin. After it's clicked two times, syzbot stops all reporting and recoding of new bugs. It's assumed that the stop mode is revoked by manually deleting an entry from the database.
google · Jan 9, 2024 · b438bd6 · b438bd6
1 parent 83b32fe
commit b438bd6
Show file tree

Hide file tree

Showing 8 changed files with 242 additions and 5 deletions.
diff --git a/dashboard/app/admin.html b/dashboard/app/admin.html
@@ -13,6 +13,12 @@
 </head>
 <body>
 	{{template "header" .Header}}
+	{{if $.Stopped}}
+		<div class="emergency-stopped">Syzbot is in the emergency stop state</div>
+	{{else}}
+		<div class="emergency-stop">Syzbot is reporting too many bugs? {{link $.StopLink "Emergency stop"}} [click {{$.MoreStopClicks}} more times]<br />
+		In this mode, syzbot will stop all reporting and won't record any new findings.</div>
+	{{end}}
 
 	<a class="plain" href="#log"><div id="log"><b>Error log:</b></div></a>
 	<textarea id="log_textarea" readonly rows="20" wrap=off>{{printf "%s" .Log}}</textarea>
@@ -21,7 +27,6 @@
 		textarea.scrollTop = textarea.scrollHeight;
 	</script>
 	<br><br>
-
 	{{with $.MemcacheStats}}
 	<table class="list_table">
 		<caption><a href="https://pkg.go.dev/google.golang.org/appengine/memcache?tab=doc#Item" target="_blank">Memcache stats:</a></caption>

diff --git a/dashboard/app/api.go b/dashboard/app/api.go
@@ -30,6 +30,7 @@ import (
 	"google.golang.org/appengine/v2"
 	db "google.golang.org/appengine/v2/datastore"
 	"google.golang.org/appengine/v2/log"
+	"google.golang.org/appengine/v2/user"
 )
 
 func initAPIHandlers() {
@@ -363,6 +364,10 @@ func addCommitInfoToBugImpl(c context.Context, bug *Bug, com dashapi.Commit) (bo
 }
 
 func apiJobPoll(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		// The bot's operation was aborted. Don't accept new crash reports.
+		return &dashapi.JobPollResp{}, err
+	}
 	req := new(dashapi.JobPollReq)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -699,6 +704,10 @@ const (
 )
 
 func apiReportCrash(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		// The bot's operation was aborted. Don't accept new crash reports.
+		return &dashapi.ReportCrashResp{}, err
+	}
 	req := new(dashapi.Crash)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -1632,3 +1641,23 @@ func apiSaveDiscussion(c context.Context, r *http.Request, payload []byte) (inte
 	}
 	return nil, mergeDiscussion(c, d)
 }
+
+func emergentlyStopped(c context.Context) (bool, error) {
+	keys, err := db.NewQuery("EmergencyStop").
+		Limit(1).
+		KeysOnly().
+		GetAll(c, nil)
+	if err != nil {
+		return false, err
+	}
+	return len(keys) > 0, nil
+}
+
+func recordEmergencyStop(c context.Context) error {
+	key := db.NewKey(c, "EmergencyStop", "all", 0, nil)
+	_, err := db.Put(c, key, &EmergencyStop{
+		Time: timeNow(c),
+		User: user.Current(c).Email,
+	})
+	return err
+}
diff --git a/dashboard/app/api_test.go b/dashboard/app/api_test.go
@@ -5,6 +5,9 @@ package main
 
 import (
 	"testing"
+	"time"
+
+	"github.com/google/syzkaller/dashboard/dashapi"
 )
 
 func TestClientSecretOK(t *testing.T) {
@@ -63,3 +66,139 @@ func TestClientNamespaceOK(t *testing.T) {
 		t.Errorf("Unexpected error %v %v", got, err)
 	}
 }
+
+func TestEmergentlyStoppedEmail(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.publicClient
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	client.ReportCrash(crash)
+
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	// There should be no email.
+	c.advanceTime(time.Hour)
+	c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedReproEmail(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.publicClient
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	client.ReportCrash(crash)
+	c.pollEmailBug()
+
+	crash2 := testCrash(build, 1)
+	crash2.ReproOpts = []byte("repro opts")
+	crash2.ReproSyz = []byte("getpid()")
+	client.ReportCrash(crash2)
+
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	// There should be no email.
+	c.advanceTime(time.Hour)
+	c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedExternalReport(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.client
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	client.ReportCrash(crash)
+
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	// There should be no email.
+	c.advanceTime(time.Hour)
+	client.pollBugs(0)
+}
+
+func TestEmergentlyStoppedEmailJob(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.publicClient
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	crash.ReproOpts = []byte("repro opts")
+	crash.ReproSyz = []byte("getpid()")
+	client.ReportCrash(crash)
+	sender := c.pollEmailBug().Sender
+	c.incomingEmail(sender, "#syz upstream\n")
+	sender = c.pollEmailBug().Sender
+
+	// Send a patch testing request.
+	c.advanceTime(time.Hour)
+	c.incomingEmail(sender, syzTestGitBranchSamplePatch,
+		EmailOptMessageID(1), EmailOptFrom("test@requester.com"),
+		EmailOptCC([]string{"somebody@else.com", "test@syzkaller.com"}))
+	c.expectNoEmail()
+
+	// Emulate a finished job.
+	pollResp := client.pollJobs(build.Manager)
+	c.expectEQ(pollResp.Type, dashapi.JobTestPatch)
+
+	c.advanceTime(time.Hour)
+	jobDoneReq := &dashapi.JobDoneReq{
+		ID:          pollResp.ID,
+		Build:       *build,
+		CrashTitle:  "test crash title",
+		CrashLog:    []byte("test crash log"),
+		CrashReport: []byte("test crash report"),
+	}
+	client.JobDone(jobDoneReq)
+
+	// Now we emergently stop syzbot.
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	// There should be no email.
+	c.advanceTime(time.Hour)
+	c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedCrashReport(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.publicClient
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	// Now we emergently stop syzbot.
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	crash := testCrash(build, 1)
+	crash.ReproOpts = []byte("repro opts")
+	crash.ReproSyz = []byte("getpid()")
+	client.ReportCrash(crash)
+
+	listResp, err := client.BugList()
+	c.expectOK(err)
+	c.expectEQ(len(listResp.List), 0)
+}
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
@@ -986,6 +986,13 @@ func (bug *Bug) dashapiStatus() (dashapi.BugStatus, error) {
 	return status, nil
 }
 
+// If an entity of type EmergencyStop exists, syzbot's operation is paused until
+// a support engineer deletes it from the DB.
+type EmergencyStop struct {
+	Time time.Time
+	User string
+}
+
 func addCrashReference(c context.Context, crashID int64, bugKey *db.Key, ref CrashReference) error {
 	crash := new(Crash)
 	crashKey := db.NewKey(c, "Crash", "", crashID, bugKey)

diff --git a/dashboard/app/main.go b/dashboard/app/main.go
@@ -203,6 +203,9 @@ type uiAdminPage struct {
 	CauseBisectionsLink string
 	JobOverviewLink     string
 	MemcacheStats       *memcache.Statistics
+	Stopped             bool
+	StopLink            string
+	MoreStopClicks      int
 }
 
 type uiManagerPage struct {
@@ -904,6 +907,10 @@ func handleAdmin(c context.Context, w http.ResponseWriter, r *http.Request) erro
 		}
 	case "invalidate_bisection":
 		return handleInvalidateBisection(c, w, r)
+	case "emergency_stop":
+		if err := recordEmergencyStop(c); err != nil {
+			return fmt.Errorf("failed to record an emergency stop: %w", err)
+		}
 	default:
 		return fmt.Errorf("%w: unknown action %q", ErrClientBadRequest, action)
 	}
@@ -963,15 +970,28 @@ func handleAdmin(c context.Context, w http.ResponseWriter, r *http.Request) erro
 			return err
 		})
 	}
+	alreadyStopped := false
+	g.Go(func() error {
+		var err error
+		alreadyStopped, err = emergentlyStopped(c)
+		return err
+	})
 	err = g.Wait()
 	if err != nil {
 		return err
 	}
 	data := &uiAdminPage{
-		Header:        hdr,
-		Log:           errorLog,
-		Managers:      makeManagerList(managers, hdr.Namespace),
-		MemcacheStats: memcacheStats,
+		Header:         hdr,
+		Log:            errorLog,
+		Managers:       makeManagerList(managers, hdr.Namespace),
+		MemcacheStats:  memcacheStats,
+		Stopped:        alreadyStopped,
+		MoreStopClicks: 2,
+		StopLink:       html.AmendURL("/admin", "stop_clicked", "1"),
+	}
+	if r.FormValue("stop_clicked") != "" {
+		data.MoreStopClicks = 1
+		data.StopLink = html.AmendURL("/admin", "action", "emergency_stop")
 	}
 	if r.FormValue("job_type") != "" {
 		data.TypeJobs = &uiJobList{Title: "Last jobs:", Jobs: typeJobs}

diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
@@ -105,6 +105,16 @@ func (cfg *EmailConfig) Validate() error {
 // handleEmailPoll is called by cron and sends emails for new bugs, if any.
 func handleEmailPoll(w http.ResponseWriter, r *http.Request) {
 	c := appengine.NewContext(r)
+	stop, err := emergentlyStopped(c)
+	if err != nil {
+		log.Errorf(c, "emergency stop querying failed: %v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if stop {
+		log.Errorf(c, "aborting email poll due to an emergency stop")
+		return
+	}
 	if err := emailPollJobs(c); err != nil {
 		log.Errorf(c, "job poll failed: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -476,8 +486,14 @@ func handleIncomingMail(w http.ResponseWriter, r *http.Request) {
 	}
 	log.Infof(c, "received email at %q, source %q", myEmail, source)
 	if source == dashapi.NoDiscussion {
+		if stop, err := emergentlyStopped(c); err != nil || stop {
+			log.Errorf(c, "abort email processing due to emergency stop (stop %v, err %v)",
+				stop, err)
+			return
+		}
 		err = processIncomingEmail(c, msg)
 	} else {
+		// Discussions are safe to handle even during an emergency stop.
 		err = processDiscussionEmail(c, msg, source)
 	}
 	if err != nil {

diff --git a/dashboard/app/reporting_external.go b/dashboard/app/reporting_external.go
@@ -19,6 +19,9 @@ import (
 // and report back bug status updates with apiReportingUpdate.
 
 func apiReportingPollBugs(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		return &dashapi.PollBugsResponse{}, err
+	}
 	req := new(dashapi.PollBugsRequest)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -36,6 +39,9 @@ func apiReportingPollBugs(c context.Context, r *http.Request, payload []byte) (i
 }
 
 func apiReportingPollNotifications(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		return &dashapi.PollNotificationsResponse{}, err
+	}
 	req := new(dashapi.PollNotificationsRequest)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -48,6 +54,9 @@ func apiReportingPollNotifications(c context.Context, r *http.Request, payload [
 }
 
 func apiReportingPollClosed(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		return &dashapi.PollClosedResponse{}, err
+	}
 	req := new(dashapi.PollClosedRequest)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)

diff --git a/pkg/html/pages/style.css b/pkg/html/pages/style.css
@@ -409,3 +409,15 @@ aside {
 .collapsible-show .show-icon {
 	display: none;
 }
+
+.emergency-stop {
+	background-color: yellow;
+	padding: 5pt;
+	margin-bottom: 5pt;
+}
+
+.emergency-stopped {
+	background-color: coral;
+	padding: 5pt;
+	margin-bottom: 5pt;
+}