Offer to clear the data directory for new standby servers

This will allow seamless reintegration of failed primary
servers after a failover.

When this happens the user will be given the option to clear
the existing database and re-clone the new primary into this server
and then continue to set up a standby as before.

https://bugzilla.redhat.com/show_bug.cgi?id=1426718
https://bugzilla.redhat.com/show_bug.cgi?id=1426769
https://bugzilla.redhat.com/show_bug.cgi?id=1442911

lib/gems/pending/appliance_console/database_replication_standby.rb

@@ -13,7 +13,7 @@ class DatabaseReplicationStandby < DatabaseReplication
     REGISTER_CMD    = 'repmgr standby register'.freeze
     REPMGRD_SERVICE = 'rh-postgresql95-repmgr'.freeze
 
-    attr_accessor :disk, :standby_host, :run_repmgrd_configuration
+    attr_accessor :disk, :standby_host, :run_repmgrd_configuration, :resync_data
 
     def initialize
       self.cluster_name      = nil
@@ -23,11 +23,13 @@ def initialize
       self.database_password = nil
       self.primary_host      = nil
       self.standby_host      = LinuxAdmin::NetworkInterface.new(NETWORK_INTERFACE).address
+      self.resync_data       = false
     end
 
     def ask_questions
       clear_screen
       say("Establish Replication Standby Server\n")
+      return false if !data_dir_empty? && !confirm_data_resync
       self.disk = ask_for_disk("Standby database disk")
       ask_for_unique_cluster_node_number
       ask_for_database_credentials
@@ -60,9 +62,8 @@ def ask_for_repmgrd_configuration
     def activate
       say("Configuring Replication Standby Server...")
       initialize_postgresql_disk if disk
-      PostgresAdmin.prep_data_directory
-      data_dir_empty? &&
-        generate_cluster_name &&
+      PostgresAdmin.prep_data_directory if disk || resync_data
+      generate_cluster_name &&
         create_config_file(standby_host) &&
         clone_standby_server &&
         start_postgres &&
@@ -72,14 +73,17 @@ def activate
     end
 
     def data_dir_empty?
-      return true if Dir[PostgresAdmin.data_directory.join("*")].empty?
+      Dir[PostgresAdmin.data_directory.join("*")].empty?
+    end
+
+    def confirm_data_resync
       Logging.logger.info("Appliance database found under: #{PostgresAdmin.data_directory}")
       say("")
       say("Appliance database found under: #{PostgresAdmin.data_directory}")
       say("Replication standby server can not be configured if the database already exists")
-      say("Remove the existing database before configuring as a standby server")
-      say("")
-      false
+      say("Would you like to remove the existing database before configuring as a standby server?")
+      say("  WARNING: This is destructive. This will remove all previous data from this server")
+      self.resync_data = ask_yn?("Continue")
     end
 
     def clone_standby_server

spec/appliance_console/database_replication_standby_spec.rb

@@ -21,51 +21,89 @@
   end
 
   context "#ask_questions" do
-    before do
-      expect(subject).to receive(:ask_for_unique_cluster_node_number)
-      expect(subject).to receive(:ask_for_database_credentials)
-      expect(subject).to receive(:ask_for_standby_host)
-      expect(subject).to receive(:ask_for_repmgrd_configuration)
-      expect(subject).to receive(:ask_for_disk).and_return("/dev/sdd")
+    it "returns false immediatly when the data directory is not empty and data resync is not confirmed" do
+      with_non_empty_data_directory do
+        allow(subject).to receive(:say)
+        expect(subject).to receive(:ask_yn?).and_return(false)
+        expect(subject.ask_questions).to be false
+      end
     end
 
-    it "sets the disk and returns true when input is confirmed" do
-      expect(subject).to receive(:repmgr_configured?).and_return(false)
-      expect(subject).to_not receive(:confirm_reconfiguration)
-      expect(subject).to receive(:confirm).and_return(true)
-      expect(subject.ask_questions).to be true
+    context "asks the default questions and" do
+      before do
+        expect(subject).to receive(:ask_for_unique_cluster_node_number)
+        expect(subject).to receive(:ask_for_database_credentials)
+        expect(subject).to receive(:ask_for_standby_host)
+        expect(subject).to receive(:ask_for_repmgrd_configuration)
+        expect(subject).to receive(:ask_for_disk).and_return("/dev/sdd")
+      end
 
-      expect(subject.disk).to eq("/dev/sdd")
-    end
+      context "with empty data directory" do
+        it "returns true when repmgr is not already configured" do
+          with_empty_data_directory do
+            expect(subject).to receive(:repmgr_configured?).and_return(false)
+            expect(subject).to_not receive(:confirm_reconfiguration)
+            expect(subject).to receive(:confirm).and_return(true)
+            expect(subject.ask_questions).to be true
+          end
+        end
 
-    it "returns true when confirm_reconfigure and input is confirmed" do
-      expect(subject).to receive(:repmgr_configured?).and_return(true)
-      expect(subject).to receive(:confirm_reconfiguration).and_return(true)
-      expect(subject).to receive(:confirm).and_return(true)
-      expect(subject.ask_questions).to be true
-    end
+        it "sets the disk and returns true when input is confirmed" do
+          with_empty_data_directory do
+            expect(subject).to receive(:repmgr_configured?).and_return(false)
+            expect(subject).to_not receive(:confirm_reconfiguration)
+            expect(subject).to receive(:confirm).and_return(true)
+            expect(subject.ask_questions).to be true
 
-    it "returns false when confirm_reconfigure is canceled" do
-      expect(subject).to receive(:repmgr_configured?).and_return(true)
-      expect(subject).to receive(:confirm_reconfiguration).and_return(false)
-      expect(subject).to_not receive(:confirm)
-      expect(subject.ask_questions).to be false
-    end
+            expect(subject.disk).to eq("/dev/sdd")
+          end
+        end
+
+        it "returns true when confirm_reconfigure and input is confirmed" do
+          with_empty_data_directory do
+            expect(subject).to receive(:repmgr_configured?).and_return(true)
+            expect(subject).to receive(:confirm_reconfiguration).and_return(true)
+            expect(subject).to receive(:confirm).and_return(true)
+            expect(subject.ask_questions).to be true
+          end
+        end
 
-    it "returns false when input is not confirmed" do
-      expect(subject).to receive(:repmgr_configured?).and_return(false)
-      expect(subject).to_not receive(:confirm_reconfiguration)
-      expect(subject).to receive(:confirm).and_return(false)
-      expect(subject.ask_questions).to be false
+        it "returns false when confirm_reconfigure is canceled" do
+          with_empty_data_directory do
+            expect(subject).to receive(:repmgr_configured?).and_return(true)
+            expect(subject).to receive(:confirm_reconfiguration).and_return(false)
+            expect(subject).to_not receive(:confirm)
+            expect(subject.ask_questions).to be false
+          end
+        end
+
+        it "returns false when input is not confirmed" do
+          with_empty_data_directory do
+            expect(subject).to receive(:repmgr_configured?).and_return(false)
+            expect(subject).to_not receive(:confirm_reconfiguration)
+            expect(subject).to receive(:confirm).and_return(false)
+            expect(subject.ask_questions).to be false
+          end
+        end
+      end
+
+      context "with non-empty data directory" do
+        it "returns true when resync is confirmed" do
+          with_non_empty_data_directory do
+            allow(subject).to receive(:say)
+            expect(subject).to receive(:ask_yn?).and_return(true)
+            expect(subject).to receive(:repmgr_configured?).and_return(false)
+            expect(subject).to_not receive(:confirm_reconfiguration)
+            expect(subject).to receive(:confirm).and_return(true)
+            expect(subject.ask_questions).to be true
+          end
+        end
+      end
     end
   end
 
   context "#activate" do
     before do
-      subject.run_repmgrd_configuration = false
-      expect(PostgresAdmin).to receive(:prep_data_directory)
-
-      expect(subject).to receive(:data_dir_empty?).and_return(true)
       expect(subject).to receive(:generate_cluster_name).and_return(true)
       expect(subject).to receive(:create_config_file).and_return(true)
       expect(subject).to receive(:clone_standby_server).and_return(true)
@@ -74,7 +112,15 @@
       expect(subject).to receive(:write_pgpass_file).and_return(true)
     end
 
+    it "cleans the data directory if resync_data is set" do
+      subject.run_repmgrd_configuration = false
+      subject.resync_data = true
+      expect(PostgresAdmin).to receive(:prep_data_directory)
+      expect(subject.activate).to be true
+    end
+
     it "returns true when configure succeeds" do
+      subject.run_repmgrd_configuration = false
       expect(subject.activate).to be true
     end
 
@@ -91,6 +137,7 @@
       expect(ApplianceConsole::LogicalVolumeManagement).to receive(:new)
         .with(hash_including(:disk => "/dev/sdd")).and_return(lvm)
       expect(lvm).to receive(:setup)
+      expect(PostgresAdmin).to receive(:prep_data_directory)
       expect(subject.activate).to be true
     end
 
@@ -153,19 +200,14 @@
   end
 
   context "#data_dir_empty?" do
-    it "should log a message and return false when not empty" do
-      Dir.mktmpdir do |dir|
-        open("#{dir}/this_directory_is_not_empty", "w")
-        expect(subject).to receive(:say).with(/^Appliance/i)
-        expect(PostgresAdmin).to receive(:data_directory).exactly(3).times.and_return(Pathname.new(dir))
+    it "should return false when not empty" do
+      with_non_empty_data_directory do
         expect(subject.data_dir_empty?).to be_falsey
       end
     end
 
-    it "should quietly return true when empty" do
-      Dir.mktmpdir do |dir|
-        expect(subject).to_not receive(:say)
-        expect(PostgresAdmin).to receive(:data_directory).once.and_return(Pathname.new(dir))
+    it "should return true when empty" do
+      with_empty_data_directory do
         expect(subject.data_dir_empty?).to be_truthy
       end
     end
@@ -199,4 +241,19 @@
       expect(subject.start_repmgrd).to be false
     end
   end
+
+  def with_empty_data_directory
+    Dir.mktmpdir do |dir|
+      allow(PostgresAdmin).to receive(:data_directory).and_return(Pathname.new(dir))
+      yield
+    end
+  end
+
+  def with_non_empty_data_directory
+    Dir.mktmpdir do |dir|
+      open("#{dir}/this_directory_is_not_empty", "w")
+      allow(PostgresAdmin).to receive(:data_directory).and_return(Pathname.new(dir))
+      yield
+    end
+  end
 end