Skip to content
Permalink
Browse files
fix: retry restore if blocked by pending restore (#119)
Retry the restore operation in the integration test if the restore is
blocked by another pending restore operation. Wait 1 minute between
each retry, and retry at most 10 times. After that, the test will be
skipped.

Fixes #118
  • Loading branch information
olavloite committed Mar 20, 2020
1 parent 74c667d commit 220653d8e25c518d0df447bf777a7fcbf04a01ca
Showing with 34 additions and 6 deletions.
  1. +34 −6 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java
@@ -21,6 +21,7 @@

import com.google.api.gax.longrunning.OperationFuture;
import com.google.api.gax.paging.Page;
import com.google.api.gax.rpc.FailedPreconditionException;
import com.google.cloud.Timestamp;
import com.google.cloud.spanner.Backup;
import com.google.cloud.spanner.BackupId;
@@ -475,13 +476,40 @@ private void testRestore(Backup backup, OperationFuture<Backup, CreateBackupMeta
throws InterruptedException, ExecutionException {
// Restore the backup to a new database.
String restoredDb = testHelper.getUniqueDatabaseId();
logger.info(
String.format(
"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb));
OperationFuture<Database, RestoreDatabaseMetadata> restoreOp =
backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb));
String restoreOperationName;
OperationFuture<Database, RestoreDatabaseMetadata> restoreOp;
int attempts = 0;
while (true) {
try {
logger.info(
String.format(
"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb));
restoreOp = backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb));
restoreOperationName = restoreOp.getName();
break;
} catch (ExecutionException e) {
if (e.getCause() instanceof FailedPreconditionException
&& e.getCause()
.getMessage()
.contains("Please retry the operation once the pending restores complete")) {
attempts++;
if (attempts == 10) {
logger.info(
"Restore operation failed 10 times because of other pending restores. Skipping restore test.");
return;
}
// wait and then retry.
logger.info(
String.format(
"Restoring backup %s to database %s must wait because of other pending restore operation",
backup.getId().getBackup(), restoredDb));
Thread.sleep(60_000L);
} else {
throw e;
}
}
}
databases.add(restoredDb);
final String restoreOperationName = restoreOp.getName();
logger.info(String.format("Restore operation %s running", restoreOperationName));
RestoreDatabaseMetadata metadata = restoreOp.getMetadata().get();
assertThat(metadata.getBackupInfo().getBackup()).isEqualTo(backup.getId().getName());

0 comments on commit 220653d

Please sign in to comment.