feat(spanner): database backup and restore (#5053)

* backup and restore apis, tests and docs

google-cloud-spanner/acceptance/spanner/backup_operations_test.rb

@@ -0,0 +1,99 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "spanner_helper"
+
+describe "Spanner Database Backup Operations", :spanner do
+  let(:instance_id) { $spanner_instance_id }
+  let(:database_id) { $spanner_database_id }
+  let(:backup_id) { "#{$spanner_database_id}-ops" }
+  let(:expire_time) { Time.now + 36000 }
+
+  it "list backup operations" do
+    instance = spanner.instance instance_id
+    instance.wont_be :nil?
+
+    database = instance.database database_id
+    database.wont_be :nil?
+
+    job = database.create_backup backup_id, expire_time
+    job.wait_until_done!
+
+    # All
+    jobs = instance.backup_operations.all.to_a
+    jobs.wont_be :empty?
+
+    jobs.each do |job|
+      job.must_be_kind_of Google::Cloud::Spanner::Backup::Job
+
+      unless job.error?
+        job.backup.must_be_kind_of Google::Cloud::Spanner::Backup
+      end
+
+      job.progress_percent.must_be :>=, 0
+      job.start_time.must_be_kind_of Time
+    end
+
+    job = jobs.first
+    job.reload!.must_be_kind_of Google::Cloud::Spanner::Backup::Job
+
+    # Filter completed jobs
+    filter = "done:true"
+    jobs = instance.backup_operations(filter: filter).all.to_a
+    jobs.wont_be :empty?
+    jobs.each do |job|
+      job.must_be :done?
+    end
+
+    # Filter by database name
+    filter = "metadata.database:#{database_id}"
+    jobs = instance.backup_operations(filter: filter).all.to_a
+    jobs.wont_be :empty?
+    jobs.each do |job|
+      job.backup.database_id.must_equal database_id unless job.error?
+    end
+
+    # Filter by metdata type
+    filter = "metadata.@type:CreateBackupMetadata"
+    jobs = instance.backup_operations(filter: filter).all.to_a
+    jobs.wont_be :empty?
+    jobs.each do |job|
+      job.grpc.metadata.must_be_kind_of Google::Spanner::Admin::Database::V1::CreateBackupMetadata
+    end
+
+    # Filter by job start time
+    time = (Time.now - 360000)
+    filter = "metadata.progress.start_time > \"#{time.iso8601}\""
+    jobs = instance.backup_operations(filter: filter).all.to_a
+    jobs.wont_be :empty?
+    jobs.each do |job|
+      job.start_time.must_be :>, time
+    end
+
+    # Filer - AND
+    time = (Time.now - 360000)
+    filter = [
+      "metadata.database:#{database_id}",
+      "metadata.progress.start_time > \"#{time.iso8601}\""
+    ].map{|f| "(#{f})"}.join(" AND ")
+
+    jobs = instance.backup_operations(filter: filter).all.to_a
+    jobs.wont_be :empty?
+    jobs.each do |job|
+      job.backup.database_id.must_equal database_id unless job.error?
+      job.start_time.must_be :>, time
+    end
+  end
+end

google-cloud-spanner/acceptance/spanner/backup_test.rb

@@ -0,0 +1,156 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "spanner_helper"
+
+describe "Spanner Database Backup", :spanner do
+  let(:instance_id) { $spanner_instance_id }
+  let(:database_id) { $spanner_database_id }
+  let(:expire_time) { Time.now + 36000 }
+
+  it "creates, get, updates, restore and delete a database backup" do
+    backup_id = "#{$spanner_database_id}-crud"
+    database = spanner.database instance_id, database_id
+    database.wont_be :nil?
+
+    # Create
+    job = database.create_backup backup_id, expire_time
+
+    job.must_be_kind_of Google::Cloud::Spanner::Backup::Job
+    job.wont_be :done?
+    job.wait_until_done!
+
+    job.must_be :done?
+    job.error.must_be :nil?
+
+    backup = job.backup
+    backup.wont_be :nil?
+    backup.must_be_kind_of Google::Cloud::Spanner::Backup
+    backup.backup_id.must_equal backup_id
+    backup.database_id.must_equal database_id
+    backup.instance_id.must_equal instance_id
+    backup.project_id.must_equal spanner.project
+    backup.expire_time.to_i.must_equal expire_time.to_i
+    backup.create_time.must_be_kind_of Time
+    backup.size_in_bytes.must_be :>, 0
+
+    # Get
+    instance = spanner.instance instance_id
+    backup = instance.backup backup_id
+
+    backup.wont_be :nil?
+    backup.must_be_kind_of Google::Cloud::Spanner::Backup
+    backup.backup_id.must_equal backup_id
+    backup.database_id.must_equal database_id
+    backup.instance_id.must_equal instance_id
+    backup.project_id.must_equal spanner.project
+    backup.expire_time.to_i.must_equal expire_time.to_i
+    backup.create_time.must_be_kind_of Time
+    backup.size_in_bytes.must_be :>, 0
+
+    # Update
+    backup.expire_time = expire_time + 3600
+    backup = instance.backup backup_id
+    backup.expire_time.to_i.must_equal((expire_time + 3600).to_i)
+
+    proc {
+      backup.expire_time = Time.now - 36000
+    }.must_raise Google::Cloud::Error
+    backup.expire_time.to_i.must_equal((expire_time + 3600 ).to_i)
+
+    # Restore
+    restore_database_id = "restore-#{database_id}"
+    backup = instance.backup backup_id
+    job = backup.restore restore_database_id
+    job.wont_be :done?
+
+    job.wait_until_done!
+
+    job.must_be :done?
+    job.wont_be :error?
+
+    database = job.database
+    database.must_be_kind_of Google::Cloud::Spanner::Database
+    database.database_id.must_equal restore_database_id
+    database.instance_id.must_equal instance_id
+    database.project_id.must_equal spanner.project
+
+    restore_info = database.restore_info
+    restore_info.must_be_kind_of Google::Cloud::Spanner::Database::RestoreInfo
+    restore_info.source_type.must_equal :BACKUP
+    restore_info.must_be :source_backup?
+
+    backup_info = restore_info.backup_info
+    backup_info.must_be_kind_of Google::Cloud::Spanner::Database::BackupInfo
+    backup_info.project_id.must_equal spanner.project
+    backup_info.instance_id.must_equal instance_id
+    backup_info.backup_id.must_equal backup_id
+    backup_info.source_database_project_id.must_equal spanner.project
+    backup_info.source_database_instance_id.must_equal instance_id
+    backup_info.source_database_id.must_equal database_id
+    backup_info.create_time.must_be_kind_of Time
+
+    # Delete
+    backup.delete
+    instance.backup(backup_id).must_be :nil?
+  end
+
+  it "cancel create backup operation" do
+    backup_id = "#{$spanner_database_id}-cancel"
+    database = spanner.database instance_id, database_id
+
+    job = database.create_backup backup_id, expire_time
+    job.wont_be :done?
+
+    job.cancel
+
+    job.reload!
+    job.must_be :done?
+    job.error.wont_be :nil?
+    job.error.code.must_equal 1
+    job.error.description.must_equal "CANCELLED"
+  end
+
+  it "lists and gets database backups" do
+    backup_id = "#{$spanner_database_id}-list"
+    database = spanner.database instance_id, database_id
+    database.wont_be :nil?
+
+    job = database.create_backup backup_id, expire_time
+    job.wait_until_done!
+    backup = job.backup
+
+    instance = spanner.instance instance_id
+
+    # List all
+    all_backups = instance.backups.all.to_a
+    all_backups.wont_be :empty?
+    all_backups.each do |backup|
+      backup.must_be_kind_of Google::Cloud::Spanner::Backup
+    end
+
+    # Filter by backup name
+    backups = instance.backups(filter: "name:#{backup_id}").to_a
+    backups.length.must_equal 1
+    backups.first.backup_id.must_equal backup_id
+
+    # Filter by database name
+    backups = instance.backups(filter: "database:#{database_id}").to_a
+    backups.wont_be :empty?
+    backups.first.database_id.must_equal database_id
+
+    backup.delete
+  end
+end

google-cloud-spanner/acceptance/spanner/database_operations_test.rb

@@ -0,0 +1,54 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "spanner_helper"
+
+describe "Spanner Database Operations", :spanner do
+  let(:instance_id) { $spanner_instance_id }
+  let(:database_id) { $spanner_database_id }
+
+  it "list database operations" do
+    instance = spanner.instance instance_id
+    instance.wont_be :nil?
+
+    # All
+    jobs = instance.database_operations.all.to_a
+    jobs.wont_be :empty?
+
+    jobs.each do |job|
+      job.must_be_kind_of Google::Cloud::Spanner::Database::Job
+
+      if job.database
+        job.database.must_be_kind_of Google::Cloud::Spanner::Database
+      end
+    end
+
+    # Filter completed jobs
+    filter = "done:true"
+    jobs = instance.database_operations(filter: filter).all.to_a
+    jobs.wont_be :empty?
+    jobs.each do |job|
+      job.must_be :done?
+    end
+
+    # Filter by metdata type
+    filter = "metadata.@type:CreateDatabaseMetadata"
+    jobs = instance.database_operations(filter: filter).all.to_a
+    jobs.wont_be :empty?
+    jobs.each do |job|
+      job.grpc.metadata.must_be_kind_of Google::Spanner::Admin::Database::V1::CreateDatabaseMetadata
+    end
+  end
+end

google-cloud-spanner/acceptance/spanner/database_test.rb

@@ -44,8 +44,7 @@
     job2.wait_until_done!
 
     job2.must_be :done?
-    job2.database.wont_be :nil?
-    job2.database.must_be_kind_of Google::Cloud::Spanner::Database
+    job2.database.must_be :nil?
 
     database.drop
     spanner.database(instance_id, database_id).must_be :nil?

google-cloud-spanner/acceptance/spanner_helper.rb

@@ -18,6 +18,7 @@
 require "minitest/autorun"
 require "minitest/focus"
 require "minitest/rg"
+
 require "google/cloud/spanner"
 
 # define SecureRandom.int64
@@ -256,9 +257,9 @@ def default_item_rows
 fixture = Object.new
 fixture.extend Acceptance::SpannerTest::Fixtures
 
-instance = $spanner.instance "google-cloud-ruby-tests"
+instance = $spanner.instance $spanner_instance_id
 instance ||= begin
-  inst_job = $spanner.create_instance "google-cloud-ruby-tests", name: "google-cloud-ruby-tests", config: "regional-us-central1", nodes: 1
+  inst_job = $spanner.create_instance $spanner_instance_id, name: "google-cloud-ruby-tests", config: "regional-us-central1", nodes: 1
   inst_job.wait_until_done!
   fail GRPC::BadStatus.new(inst_job.error.code, inst_job.error.message) if inst_job.error?
   inst_job.instance
@@ -274,8 +275,21 @@ def default_item_rows
 def clean_up_spanner_objects
   puts "Cleaning up instances and databases after spanner tests."
   $spanner.instance($spanner_instance_id).database($spanner_database_id).drop
+
   puts "Closing the Spanner Client."
   $spanner_client.close
+
+  puts "Cleaning up instances databases and backups after spanner tests."
+  instance = $spanner.instance($spanner_instance_id)
+
+  # Delete test database backups.
+  instance.backups(filter: "name:#{$spanner_database_id}").all.each do |backup|
+    backup.delete
+  end
+
+  # Delete test restored database.
+  restored_db = instance.database("restore-#{$spanner_database_id}")
+  restored_db.drop if restored_db
 rescue => e
   puts "Error while cleaning up instances and databases after spanner tests.\n\n#{e}"
 end

google-cloud-spanner/lib/google/cloud/spanner/admin/database/v1/database_admin_client_config.json

@@ -66,7 +66,7 @@
           "retry_params_name": "default"
         },
         "CreateBackup": {
-          "timeout_millis": 30000,
+          "timeout_millis": 3600000,
           "retry_codes_name": "non_idempotent",
           "retry_params_name": "default"
         },
@@ -76,12 +76,12 @@
           "retry_params_name": "default"
         },
         "UpdateBackup": {
-          "timeout_millis": 30000,
+          "timeout_millis": 3600000,
           "retry_codes_name": "non_idempotent",
           "retry_params_name": "default"
         },
         "DeleteBackup": {
-          "timeout_millis": 30000,
+          "timeout_millis": 3600000,
           "retry_codes_name": "idempotent",
           "retry_params_name": "default"
         },
@@ -91,7 +91,7 @@
           "retry_params_name": "default"
         },
         "RestoreDatabase": {
-          "timeout_millis": 30000,
+          "timeout_millis": 3600000,
           "retry_codes_name": "non_idempotent",
           "retry_params_name": "default"
         },