roachtest: skip post-test assertions on failure

pkg/cmd/roachtest/test_runner.go

@@ -1087,16 +1087,19 @@ func (r *testRunner) runTest(
 		t.ReplaceL(logger)
 	}
 
-	// Awkward file name to keep it close to test.log.
-	l.Printf("running post test assertions (test-post-assertions.log)")
-	replaceLogger("test-post-assertions")
-
-	// We still want to run the post-test assertions even if the test timed out as it
-	// might provide useful information about the health of the nodes. Any assertion failures
-	// will will be recorded against, and eventually fail, the test.
-	// TODO (miral): consider not running these assertions if the test has already failed
-	if err := r.postTestAssertions(ctx, t, c, 10*time.Minute); err != nil {
-		l.Printf("error during post test assertions: %v; see test-post-assertions.log for details", err)
+	if !t.Failed() {
+		// Awkward file name to keep it close to test.log.
+		l.Printf("running post test assertions (test-post-assertions.log)")
+		replaceLogger("test-post-assertions")
+
+		// We still want to run the post-test assertions even if the test timed out as it
+		// might provide useful information about the health of the nodes. Any assertion failures
+		// will will be recorded against, and eventually fail, the test.
+		if err := r.postTestAssertions(ctx, t, c, 10*time.Minute); err != nil {
+			l.Printf("error during post test assertions: %v; see test-post-assertions.log for details", err)
+		}
+	} else {
+		l.Printf("skipping post test assertions as test failed")
 	}
 
 	// From now on, all logging goes to test-teardown.log to give a clear separation between

pkg/roachprod/install/cluster_synced.go

@@ -2412,8 +2412,8 @@ func WithDisplay(display string) ParallelOption {
 }
 
 // Parallel runs a user-defined function across the nodes in the
-// cluster. If any of the commands fail, Parallel will log an error
-// and exit the program.
+// cluster. If any of the commands fail, Parallel will log each failure
+// and return an error.
 //
 // A user may also pass in a RunRetryOpts to control how the function is retried
 // in the case of a failure.