Skip to content
Permalink
Browse files
feat!: Point In Time Recovery (PITR) (#452)
* feat: exposes new pitr-lite database fields

Exposes earliest version time and version retention period fields in the
database class.

* feat: adds it tests for updating version retention

Adds integration tests for updating the version retention period.

* feat: adds new create database tests for pitr

Adds create database tests for PITR and refactors the integration test
class.

* chore: refactors tests

Separates PITR database tests into 2 files for clarity.

* fix: disables pitr-lite tests in emulator

The feature is not supported in the emulator currently.

* fix: closes result set in test

Addresses PR comment.

* fix: updates DatabaseInfo equals/hashcode

To compare version retention period and earliest version time.

* fix: formatting

Fixes formatting of the DatabaseInfo

* feature: adds test for throttled pitr field

Adds test to check for the throttled field in the update database ddl
metadata.

* fix: explain further the pitr-lite params in docs

Adds more explanations to the purpose of the added params for pitr-lite:
version_retention_period and earliest_version_time.

* feat: adds version time to backups

Adds PITR-lite version time to backups. This should make it possible to
specify the consistent time for copying the database.

* test: adds integration tests for pitr backups

* test: adds tests for pitr restore

* test: fixes integration test for pitr restore

* test: fixes backup unit test

* test: fixes npe on pitr backup test

* chore: fixes clirr errors

* chore: refactors / addresses pr comments

* test: fixes the it pitr sad cases tests

* test: fixes pitr backup and restore tests

* test: skips pitr backup and restore tests

This is because the backend for these features is not ready yet.
  • Loading branch information
thiagotnunes committed Feb 17, 2021
1 parent 44aa384 commit ab14a5ec2dc2b7e2141305b5326f436eb6eee76f
Showing with 935 additions and 88 deletions.
  1. +13 −2 google-cloud-spanner/clirr-ignored-differences.xml
  2. +2 −1 google-cloud-spanner/src/main/java/com/google/cloud/spanner/Backup.java
  3. +31 −2 google-cloud-spanner/src/main/java/com/google/cloud/spanner/BackupInfo.java
  4. +9 −1 google-cloud-spanner/src/main/java/com/google/cloud/spanner/Database.java
  5. +26 −0 google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClient.java
  6. +23 −7 google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClientImpl.java
  7. +51 −3 google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseInfo.java
  8. +10 −0 google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/RemoteSpannerHelper.java
  9. +41 −0 google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/TimestampHelper.java
  10. +23 −1 google-cloud-spanner/src/test/java/com/google/cloud/spanner/BackupTest.java
  11. +46 −2 google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientImplTest.java
  12. +42 −27 google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientTest.java
  13. +11 −3 google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseTest.java
  14. +12 −39 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java
  15. +244 −0 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrBackupAndRestore.java
  16. +144 −0 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrCreateDatabaseTest.java
  17. +207 −0 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrUpdateDatabaseTest.java
@@ -453,7 +453,7 @@
<className>com/google/cloud/spanner/TransactionContext</className>
<method>com.google.api.core.ApiFuture executeUpdateAsync(com.google.cloud.spanner.Statement)</method>
</difference>

<!-- Support for CommitStats added -->
<difference>
<differenceType>7012</differenceType>
@@ -475,5 +475,16 @@
<className>com/google/cloud/spanner/TransactionRunner</className>
<method>com.google.cloud.spanner.CommitResponse getCommitResponse()</method>
</difference>


<!-- PITR -->
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/spanner/BackupInfo$Builder</className>
<method>com.google.cloud.spanner.BackupInfo$Builder setVersionTime(com.google.cloud.Timestamp)</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/DatabaseAdminClient</className>
<method>com.google.api.gax.longrunning.OperationFuture createBackup(com.google.cloud.spanner.Backup)</method>
</difference>
</differences>
@@ -65,7 +65,7 @@ public OperationFuture<Backup, CreateBackupMetadata> create() {
getExpireTime() != null, "Cannot create a backup without an expire time");
Preconditions.checkState(
getDatabase() != null, "Cannot create a backup without a source database");
return dbClient.createBackup(instance(), backup(), sourceDatabase(), getExpireTime());
return dbClient.createBackup(this);
}

/**
@@ -182,6 +182,7 @@ static Backup fromProto(
.setState(fromProtoState(proto.getState()))
.setSize(proto.getSizeBytes())
.setExpireTime(Timestamp.fromProto(proto.getExpireTime()))
.setVersionTime(Timestamp.fromProto(proto.getVersionTime()))
.setDatabase(DatabaseId.of(proto.getDatabase()))
.setProto(proto)
.build();
@@ -18,6 +18,7 @@

import com.google.api.client.util.Preconditions;
import com.google.cloud.Timestamp;
import com.google.spanner.admin.database.v1.Database;
import java.util.Objects;
import javax.annotation.Nullable;

@@ -40,6 +41,17 @@ public abstract static class Builder {
*/
public abstract Builder setExpireTime(Timestamp expireTime);

/**
* Optional for creating a new backup.
*
* <p>Specifies the timestamp to have an externally consistent copy of the database. If no
* version time is specified, it will be automatically set to the backup create time.
*
* <p>The version time can be as far in the past as specified by the database earliest version
* time (see {@link Database#getEarliestVersionTime()}).
*/
public abstract Builder setVersionTime(Timestamp versionTime);

/**
* Required for creating a new backup.
*
@@ -55,6 +67,7 @@ abstract static class BuilderImpl extends Builder {
protected final BackupId id;
private State state = State.UNSPECIFIED;
private Timestamp expireTime;
private Timestamp versionTime;
private DatabaseId database;
private long size;
private com.google.spanner.admin.database.v1.Backup proto;
@@ -67,6 +80,7 @@ abstract static class BuilderImpl extends Builder {
this.id = other.id;
this.state = other.state;
this.expireTime = other.expireTime;
this.versionTime = other.versionTime;
this.database = other.database;
this.size = other.size;
this.proto = other.proto;
@@ -84,6 +98,12 @@ public Builder setExpireTime(Timestamp expireTime) {
return this;
}

@Override
public Builder setVersionTime(Timestamp versionTime) {
this.versionTime = versionTime;
return this;
}

@Override
public Builder setDatabase(DatabaseId database) {
Preconditions.checkArgument(
@@ -119,6 +139,7 @@ public enum State {
private final BackupId id;
private final State state;
private final Timestamp expireTime;
private final Timestamp versionTime;
private final DatabaseId database;
private final long size;
private final com.google.spanner.admin.database.v1.Backup proto;
@@ -128,6 +149,7 @@ public enum State {
this.state = builder.state;
this.size = builder.size;
this.expireTime = builder.expireTime;
this.versionTime = builder.versionTime;
this.database = builder.database;
this.proto = builder.proto;
}
@@ -157,6 +179,11 @@ public Timestamp getExpireTime() {
return expireTime;
}

/** Returns the version time of the backup. */
public Timestamp getVersionTime() {
return versionTime;
}

/** Returns the id of the database that was used to create the backup. */
public DatabaseId getDatabase() {
return database;
@@ -180,17 +207,19 @@ public boolean equals(Object o) {
&& state == that.state
&& size == that.size
&& Objects.equals(expireTime, that.expireTime)
&& Objects.equals(versionTime, that.versionTime)
&& Objects.equals(database, that.database);
}

@Override
public int hashCode() {
return Objects.hash(id, state, size, expireTime, database);
return Objects.hash(id, state, size, expireTime, versionTime, database);
}

@Override
public String toString() {
return String.format(
"Backup[%s, %s, %d, %s, %s]", id.getName(), state, size, expireTime, database);
"Backup[%s, %s, %d, %s, %s, %s]",
id.getName(), state, size, expireTime, versionTime, database);
}
}
@@ -118,8 +118,14 @@ public OperationFuture<Backup, CreateBackupMetadata> backup(Backup backup) {
Preconditions.checkArgument(
backup.getInstanceId().equals(getId().getInstanceId()),
"The instance of the backup must be equal to the instance of this database.");

return dbClient.createBackup(
instance(), backup.getId().getBackup(), database(), backup.getExpireTime());
dbClient
.newBackupBuilder(backup.getId())
.setDatabase(getId())
.setExpireTime(backup.getExpireTime())
.setVersionTime(backup.getVersionTime())
.build());
}

/**
@@ -177,6 +183,8 @@ static Database fromProto(
.setState(fromProtoState(proto.getState()))
.setCreateTime(Timestamp.fromProto(proto.getCreateTime()))
.setRestoreInfo(RestoreInfo.fromProtoOrNullIfDefaultInstance(proto.getRestoreInfo()))
.setVersionRetentionPeriod(proto.getVersionRetentionPeriod())
.setEarliestVersionTime(Timestamp.fromProto(proto.getEarliestVersionTime()))
.setProto(proto)
.build();
}
@@ -101,6 +101,32 @@ OperationFuture<Backup, CreateBackupMetadata> createBackup(
String sourceInstanceId, String backupId, String databaseId, Timestamp expireTime)
throws SpannerException;

/**
* Creates a new backup from a database in a Cloud Spanner instance.
*
* <p>Example to create a backup.
*
* <pre>{@code
* BackupId backupId = BackupId.of("project", "instance", "backup-id");
* DatabaseId databaseId = DatabaseId.of("project", "instance", "database-id");
* Timestamp expireTime = Timestamp.ofTimeMicroseconds(expireTimeMicros);
* Timestamp versionTime = Timestamp.ofTimeMicroseconds(versionTimeMicros);
*
* Backup backupToCreate = dbAdminClient
* .newBackupBuilder(backupId)
* .setDatabase(databaseId)
* .setExpireTime(expireTime)
* .setVersionTime(versionTime)
* .build();
*
* OperationFuture<Backup, CreateBackupMetadata> op = dbAdminClient.createBackup(backupToCreate);
* Backup createdBackup = op.get();
* }</pre>
*
* @param backup the backup to be created
*/
OperationFuture<Backup, CreateBackupMetadata> createBackup(Backup backup) throws SpannerException;

/**
* Restore a database from a backup. The database that is restored will be created and may not
* already exist.
@@ -114,16 +114,31 @@ public Database apply(Exception e) {
public OperationFuture<Backup, CreateBackupMetadata> createBackup(
String instanceId, String backupId, String databaseId, Timestamp expireTime)
throws SpannerException {
com.google.spanner.admin.database.v1.Backup backup =
final Backup backup =
newBackupBuilder(BackupId.of(projectId, instanceId, backupId))
.setDatabase(DatabaseId.of(projectId, instanceId, databaseId))
.setExpireTime(expireTime)
.build();
return createBackup(backup);
}

@Override
public OperationFuture<Backup, CreateBackupMetadata> createBackup(final Backup backup) {
final String instanceId = backup.getInstanceId().getInstance();
final String databaseId = backup.getDatabase().getDatabase();
final String backupId = backup.getId().getBackup();
final com.google.spanner.admin.database.v1.Backup.Builder backupBuilder =
com.google.spanner.admin.database.v1.Backup.newBuilder()
.setDatabase(getDatabaseName(instanceId, databaseId))
.setExpireTime(expireTime.toProto())
.build();
String instanceName = getInstanceName(instanceId);
OperationFuture<com.google.spanner.admin.database.v1.Backup, CreateBackupMetadata>
rawOperationFuture = rpc.createBackup(instanceName, backupId, backup);
.setExpireTime(backup.getExpireTime().toProto());
if (backup.getVersionTime() != null) {
backupBuilder.setVersionTime(backup.getVersionTime().toProto());
}
final String instanceName = getInstanceName(instanceId);
final OperationFuture<com.google.spanner.admin.database.v1.Backup, CreateBackupMetadata>
rawOperationFuture = rpc.createBackup(instanceName, backupId, backupBuilder.build());

return new OperationFutureImpl<Backup, CreateBackupMetadata>(
return new OperationFutureImpl<>(
rawOperationFuture.getPollingFuture(),
rawOperationFuture.getInitialFuture(),
new ApiFunction<OperationSnapshot, Backup>() {
@@ -137,6 +152,7 @@ public Backup apply(OperationSnapshot snapshot) {
com.google.spanner.admin.database.v1.Backup.newBuilder(proto)
.setName(proto.getName())
.setExpireTime(proto.getExpireTime())
.setVersionTime(proto.getVersionTime())
.setState(proto.getState())
.build(),
DatabaseAdminClientImpl.this);
@@ -30,6 +30,10 @@ public abstract static class Builder {

abstract Builder setRestoreInfo(RestoreInfo restoreInfo);

abstract Builder setVersionRetentionPeriod(String versionRetentionPeriod);

abstract Builder setEarliestVersionTime(Timestamp earliestVersionTime);

abstract Builder setProto(com.google.spanner.admin.database.v1.Database proto);

/** Builds the database from this builder. */
@@ -41,6 +45,8 @@ abstract static class BuilderImpl extends Builder {
private State state = State.UNSPECIFIED;
private Timestamp createTime;
private RestoreInfo restoreInfo;
private String versionRetentionPeriod;
private Timestamp earliestVersionTime;
private com.google.spanner.admin.database.v1.Database proto;

BuilderImpl(DatabaseId id) {
@@ -52,6 +58,8 @@ abstract static class BuilderImpl extends Builder {
this.state = other.state;
this.createTime = other.createTime;
this.restoreInfo = other.restoreInfo;
this.versionRetentionPeriod = other.versionRetentionPeriod;
this.earliestVersionTime = other.earliestVersionTime;
this.proto = other.proto;
}

@@ -73,6 +81,18 @@ Builder setRestoreInfo(@Nullable RestoreInfo restoreInfo) {
return this;
}

@Override
Builder setVersionRetentionPeriod(String versionRetentionPeriod) {
this.versionRetentionPeriod = versionRetentionPeriod;
return this;
}

@Override
Builder setEarliestVersionTime(Timestamp earliestVersionTime) {
this.earliestVersionTime = earliestVersionTime;
return this;
}

@Override
Builder setProto(@Nullable com.google.spanner.admin.database.v1.Database proto) {
this.proto = proto;
@@ -96,13 +116,17 @@ public enum State {
private final State state;
private final Timestamp createTime;
private final RestoreInfo restoreInfo;
private final String versionRetentionPeriod;
private final Timestamp earliestVersionTime;
private final com.google.spanner.admin.database.v1.Database proto;

public DatabaseInfo(DatabaseId id, State state) {
this.id = id;
this.state = state;
this.createTime = null;
this.restoreInfo = null;
this.versionRetentionPeriod = null;
this.earliestVersionTime = null;
this.proto = null;
}

@@ -111,6 +135,8 @@ public DatabaseInfo(DatabaseId id, State state) {
this.state = builder.state;
this.createTime = builder.createTime;
this.restoreInfo = builder.restoreInfo;
this.versionRetentionPeriod = builder.versionRetentionPeriod;
this.earliestVersionTime = builder.earliestVersionTime;
this.proto = builder.proto;
}

@@ -129,6 +155,23 @@ public Timestamp getCreateTime() {
return createTime;
}

/**
* Returns the version retention period of the database. This is the period for which Cloud
* Spanner retains all versions of data for the database. For instance, if set to 3 days, Cloud
* Spanner will retain data versions that are up to 3 days old.
*/
public String getVersionRetentionPeriod() {
return versionRetentionPeriod;
}

/**
* Returns the earliest version time of the database. This is the oldest timestamp that can be
* used to read old versions of the data.
*/
public Timestamp getEarliestVersionTime() {
return earliestVersionTime;
}

/**
* Returns the {@link RestoreInfo} of the database if any is available, or <code>null</code> if no
* {@link RestoreInfo} is available for this database.
@@ -154,16 +197,21 @@ public boolean equals(Object o) {
return id.equals(that.id)
&& state == that.state
&& Objects.equals(createTime, that.createTime)
&& Objects.equals(restoreInfo, that.restoreInfo);
&& Objects.equals(restoreInfo, that.restoreInfo)
&& Objects.equals(versionRetentionPeriod, that.versionRetentionPeriod)
&& Objects.equals(earliestVersionTime, that.earliestVersionTime);
}

@Override
public int hashCode() {
return Objects.hash(id, state, createTime, restoreInfo);
return Objects.hash(
id, state, createTime, restoreInfo, versionRetentionPeriod, earliestVersionTime);
}

@Override
public String toString() {
return String.format("Database[%s, %s, %s, %s]", id.getName(), state, createTime, restoreInfo);
return String.format(
"Database[%s, %s, %s, %s, %s, %s]",
id.getName(), state, createTime, restoreInfo, versionRetentionPeriod, earliestVersionTime);
}
}

0 comments on commit ab14a5e

Please sign in to comment.