Skip to content

Commit

Permalink
Allow gofail trigger to fail as long as the member stops running
Browse files Browse the repository at this point in the history
This is required for compaction based failpoint, to allow the traffic
send compaction request causing etcd to crash before failpoint executes
the trigger.

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
  • Loading branch information
serathius committed May 21, 2024
1 parent 31adb72 commit 3fb36d9
Showing 1 changed file with 20 additions and 12 deletions.
32 changes: 20 additions & 12 deletions tests/robustness/failpoint/gofail.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,30 +90,38 @@ func (f goPanicFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logg
lg.Info("goFailpoint setup failed", zap.String("failpoint", f.Name()), zap.Error(err))
continue
}
if !member.IsRunning() {
// TODO: Check member logs that etcd not running is caused panic caused by proper gofailpoint.
break
}
if f.trigger != nil {
break
}

if f.trigger != nil {
for member.IsRunning() {
select {
case <-ctx.Done():
return reports, ctx.Err()
default:
}
var r []report.ClientReport
lg.Info("Triggering gofailpoint", zap.String("failpoint", f.Name()))
r, err = f.trigger.Trigger(ctx, t, member, clus, baseTime, ids)
if err != nil {
lg.Info("gofailpoint trigger failed", zap.String("failpoint", f.Name()), zap.Error(err))
continue
}
if r != nil {
reports = append(reports, r...)
}
break
}
lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name))
err = member.Wait(ctx)
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err))
return reports, fmt.Errorf("member didn't exit as expected: %v", err)
}
lg.Info("Member exited as expected", zap.String("member", member.Config().Name))
}

lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name))
err = member.Wait(ctx)
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err))
return reports, fmt.Errorf("member didn't exit as expected: %v", err)
}
lg.Info("Member exited as expected", zap.String("member", member.Config().Name))

if lazyfs := member.LazyFS(); lazyfs != nil {
lg.Info("Removing data that was not fsynced")
err := lazyfs.ClearCache(ctx)
Expand Down

0 comments on commit 3fb36d9

Please sign in to comment.