Merge #32140

32140: roachtest: restart stopped nodes before exiting chaos r=tschottdorf a=petermattis

When chaos exits due to the stopper channel becoming ready, restart any
stopped nodes. Some tests (notably `scaledata/jobcoordinator`) do not
finish when a down node is present.

Add a timeout to the `scaledata/*` tests which is twice their expected
10m duration.

Fixes #32125
Fixes #32126

Release note: None

Co-authored-by: Peter Mattis <petermattis@gmail.com>

pkg/cmd/roachtest/chaos.go

@@ -94,6 +94,8 @@ func (ch *Chaos) Runner(c *cluster, m *monitor) func(context.Context) error {
 
 			select {
 			case <-ch.Stopper:
+				l.Printf("restarting %v (chaos is done)\n", target)
+				c.Start(ctx, c.t.(*test), target)
 				return nil
 			case <-ctx.Done():
 				return ctx.Err()

pkg/cmd/roachtest/scaledata.go

@@ -46,9 +46,10 @@ func registerScaleData(r *registry) {
 		const duration = 10 * time.Minute
 		for _, n := range []int{3, 6} {
 			r.Add(testSpec{
-				Name:   fmt.Sprintf("scaledata/%s/nodes=%d", app, n),
-				Nodes:  nodes(n + 1),
-				Stable: true, // DO NOT COPY to new tests
+				Name:    fmt.Sprintf("scaledata/%s/nodes=%d", app, n),
+				Timeout: 2 * duration,
+				Nodes:   nodes(n + 1),
+				Stable:  true, // DO NOT COPY to new tests
 				Run: func(ctx context.Context, t *test, c *cluster) {
 					runSqlapp(ctx, t, c, app, flags, duration)
 				},