diff --git a/test/backup_test.go b/test/backup_test.go index 52d3977e..ddbbbdc6 100644 --- a/test/backup_test.go +++ b/test/backup_test.go @@ -36,6 +36,11 @@ import ( driver "github.com/arangodb/go-driver" ) +func waitForHealthyClusterAfterBackup(t *testing.T, client driver.Client) { + time.Sleep(5 * time.Second) + waitForHealthyCluster(t, client, 2*time.Second).RetryT(t, 125*time.Millisecond, 10*time.Second) +} + var backupAPIAvailable *bool func setBackupAvailable(av bool) { @@ -421,6 +426,7 @@ func TestBackupRestore(t *testing.T) { if err := b.Restore(ctx, id, nil); err != nil { t.Fatalf("Failed to restore backup: %s", describe(err)) } + defer waitForHealthyClusterAfterBackup(t, c) if isSingle { waitctx, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -672,6 +678,7 @@ func TestBackupCompleteCycle(t *testing.T) { if err := b.Restore(ctx, id, nil); err != nil { t.Fatalf("Failed to restore backup: %s", describe(err)) } + defer waitForHealthyClusterAfterBackup(t, c) if isSingle { waitctx, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -759,75 +766,6 @@ func TestBackupCreateManyBackupsFast(t *testing.T) { } } -func TestBackupCreateRestoreParallel(t *testing.T) { - c := createClientFromEnv(t, true) - skipIfNoBackup(c, t) - - ctx := context.Background() - b := c.Backup() - id := ensureBackup(ctx, b, t) - - isSingle := false - if role, err := c.ServerRole(ctx); err != nil { - t.Fatalf("Failed to obtain server role: %s", describe(err)) - } else { - isSingle = role == driver.ServerRoleSingle - } - - errchan := make(chan error) - defer close(errchan) - var wg sync.WaitGroup - - wg.Add(1) - go func() { - defer wg.Done() - if _, _, err := b.Create(ctx, nil); err == nil { - errchan <- nil - } else { - errchan <- err - t.Logf("Create failed: %s", describe(err)) - } - }() - wg.Add(1) - go func() { - defer wg.Done() - if err := b.Restore(ctx, id, nil); err == nil { - errchan <- nil - if isSingle { - waitctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - c = waitForServerRestart(waitctx, c, t) - } - } else { - errchan <- err - t.Logf("Restore failed: %s", describe(err)) - } - }() - - errCount := 0 - for i := 0; i < 2; i++ { - err := <-errchan - if err != nil { - errCount++ - } - } - - wg.Wait() - - if errCount >= 2 { - t.Fatalf("Both operation failed!") - } -} - -func ensureRemoteBackup(ctx context.Context, b driver.ClientBackup, t *testing.T) driver.BackupID { - id := ensureBackup(ctx, b, t) - uploadBackupWaitForCompletion(ctx, id, b, t) - if err := b.Delete(ctx, id); err != nil { - t.Fatalf("Failed to remove backup: %s", err) - } - return id -} - func TestBackupRestoreWithViews(t *testing.T) { c := createClientFromEnv(t, true) skipIfNoBackup(c, t) @@ -890,6 +828,7 @@ func TestBackupRestoreWithViews(t *testing.T) { if err := b.Restore(ctx, id, nil); err != nil { t.Fatalf("Failed to restore backup: %s", describe(err)) } + defer waitForHealthyClusterAfterBackup(t, c) if isSingle { waitctx, cancel := context.WithTimeout(ctx, 30*time.Second) diff --git a/test/util.go b/test/util.go index 57b2d7d5..90a36b96 100644 --- a/test/util.go +++ b/test/util.go @@ -229,3 +229,48 @@ func getCallerFunctionName() string { return function[0] + "_" + uniuri.NewLen(6) } + +func waitForHealthyCluster(t *testing.T, client driver.Client, timeout time.Duration) retryFunc { + return func() error { + ctx, c := context.WithTimeout(context.Background(), timeout) + defer c() + + cluster, err := client.Cluster(ctx) + if err != nil { + if !driver.IsPreconditionFailed(err) { + t.Logf("Unable to get cluster: %s", err.Error()) + return nil + } + // We are on single, check version + + _, err = client.Version(ctx) + if err == nil { + return interrupt{} + } + + t.Logf("Unable to get version: %s", err.Error()) + return nil + } + + health, err := cluster.Health(ctx) + if err != nil { + t.Logf("Unable to get health: %s", err.Error()) + return nil + } + + healthy := true + + for id, m := range health.Health { + if m.Status != driver.ServerStatusGood { + healthy = false + t.Logf("Server %s in bad health: %s", id, m.Status) + } + } + + if healthy { + return interrupt{} + } + + return nil + } +}