Skip to content

Commit

Permalink
Merge pull request #18046 from serathius/robustness-trigger-fail
Browse files Browse the repository at this point in the history
Allow gofail trigger to fail as long as the member stops running
  • Loading branch information
serathius committed May 21, 2024
2 parents 31adb72 + 3fb36d9 commit f8d8f6a
Showing 1 changed file with 20 additions and 12 deletions.
32 changes: 20 additions & 12 deletions tests/robustness/failpoint/gofail.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,30 +90,38 @@ func (f goPanicFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logg
lg.Info("goFailpoint setup failed", zap.String("failpoint", f.Name()), zap.Error(err))
continue
}
if !member.IsRunning() {
// TODO: Check member logs that etcd not running is caused panic caused by proper gofailpoint.
break
}
if f.trigger != nil {
break
}

if f.trigger != nil {
for member.IsRunning() {
select {
case <-ctx.Done():
return reports, ctx.Err()
default:
}
var r []report.ClientReport
lg.Info("Triggering gofailpoint", zap.String("failpoint", f.Name()))
r, err = f.trigger.Trigger(ctx, t, member, clus, baseTime, ids)
if err != nil {
lg.Info("gofailpoint trigger failed", zap.String("failpoint", f.Name()), zap.Error(err))
continue
}
if r != nil {
reports = append(reports, r...)
}
break
}
lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name))
err = member.Wait(ctx)
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err))
return reports, fmt.Errorf("member didn't exit as expected: %v", err)
}
lg.Info("Member exited as expected", zap.String("member", member.Config().Name))
}

lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name))
err = member.Wait(ctx)
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err))
return reports, fmt.Errorf("member didn't exit as expected: %v", err)
}
lg.Info("Member exited as expected", zap.String("member", member.Config().Name))

if lazyfs := member.LazyFS(); lazyfs != nil {
lg.Info("Removing data that was not fsynced")
err := lazyfs.ClearCache(ctx)
Expand Down

0 comments on commit f8d8f6a

Please sign in to comment.