[1LP][RFR] Sort out DB HA related tests

cfme/tests/cli/test_appliance_cli.py

@@ -2,7 +2,8 @@
 from wait_for import wait_for
 
 from cfme import test_requirements
-from cfme.utils.appliance.console import waiting_for_ha_monitor_started
+from cfme.utils.appliance.console import check_db_ha_failover
+from cfme.utils.blockers import GH
 from cfme.utils.log_validator import LogValidator
 
 pytestmark = [
@@ -236,6 +237,7 @@ def test_appliance_console_cli_configure_dedicated_db(unconfigured_appliance, ap
 
 @test_requirements.ha_proxy
 @pytest.mark.tier(2)
+@pytest.mark.meta(blockers=[GH('ManageIQ/manageiq:20455')])
 def test_appliance_console_cli_ha_crud(unconfigured_appliances, app_creds):
     """Tests the configuration of HA with three appliances including failover to standby node
 
@@ -270,17 +272,4 @@ def test_appliance_console_cli_ha_crud(unconfigured_appliances, app_creds):
         'vmdb_production', apps[1].unpartitioned_disks[0]
     )
 
-    with waiting_for_ha_monitor_started(apps[2], app1_ip, timeout=300):
-        # Configure automatic failover on EVM appliance
-        command_set = ('ap', '', '10', '1', '')
-        apps[2].appliance_console.run_commands(command_set)
-
-    with LogValidator(evm_log,
-                      matched_patterns=['Starting to execute failover'],
-                      hostname=apps[2].hostname).waiting(timeout=450):
-        # Cause failover to occur
-        result = apps[0].ssh_client.run_command('systemctl stop $APPLIANCE_PG_SERVICE', timeout=15)
-        assert result.success, f"Failed to stop APPLIANCE_PG_SERVICE: {result.output}"
-
-    apps[2].evmserverd.wait_for_running()
-    apps[2].wait_for_miq_ready()
+    check_db_ha_failover(apps[0], apps[2])

cfme/tests/cli/test_appliance_console.py

@@ -12,7 +12,6 @@
 from cfme.exceptions import SSHExpectTimeoutError
 from cfme.tests.cli import app_con_menu
 from cfme.utils import conf
-from cfme.utils.appliance.console import waiting_for_ha_monitor_started
 from cfme.utils.appliance.implementations.ui import navigate_to
 from cfme.utils.blockers import BZ
 from cfme.utils.conf import credentials
@@ -269,92 +268,6 @@ def test_appliance_console_dedicated_db(unconfigured_appliance, app_creds):
     wait_for(lambda: unconfigured_appliance.db.is_dedicated_active, timeout=900)
 
 
-@test_requirements.appliance
-@test_requirements.ha_proxy
-@pytest.mark.tier(2)
-def test_appliance_console_ha_crud(unconfigured_appliances, app_creds):
-    """Testing HA configuration with 3 appliances.
-
-    Appliance one configuring dedicated database, 'ap' launch appliance_console,
-    '' clear info screen, '5' setup db, '1' Creates v2_key, '1' selects internal db,
-    '1' use partition, 'y' create dedicated db, 'pwd' db password, 'pwd' confirm db password + wait
-    360 secs and '' finish.
-
-    Appliance two creating region in dedicated database, 'ap' launch appliance_console, '' clear
-    info screen, '5' setup db, '2' fetch v2_key, 'app0_ip' appliance ip address, '' default user,
-    'pwd' appliance password, '' default v2_key location, '2' create region in external db, '0' db
-    region number, 'y' confirm create region in external db 'app0_ip', '' ip and default port for
-    dedicated db, '' use default db name, '' default username, 'pwd' db password, 'pwd' confirm db
-    password + wait 360 seconds and '' finish.
-
-    Appliance one configuring primary node for replication, 'ap' launch appliance_console, '' clear
-    info screen, '6' configure db replication, '1' configure node as primary, '1' cluster node
-    number set to 1, '' default dbname, '' default user, 'pwd' password, 'pwd' confirm password,
-    'app0_ip' primary appliance ip, confirm settings and wait 360 seconds to configure, '' finish.
-
-
-    Appliance three configuring standby node for replication, 'ap' launch appliance_console, ''
-    clear info screen, '6' configure db replication, '1' configure node as primary, '1' cluster node
-    number set to 1, '' default dbname, '' default user, 'pwd' password, 'pwd' confirm password,
-    'app0_ip' primary appliance ip, confirm settings and wait 360 seconds to configure, '' finish.
-
-
-    Appliance two configuring automatic failover of database nodes, 'ap' launch appliance_console,
-    '' clear info screen '9' configure application database failover monitor, '1' start failover
-    monitor. wait 30 seconds for service to start '' finish.
-
-    Appliance one, stop APPLIANCE_PG_SERVICE and check that the standby node takes over correctly
-    and evm starts up again pointing at the new primary database.
-
-
-    Polarion:
-        assignee: jhenner
-        caseimportance: high
-        casecomponent: HAProxy
-        initialEstimate: 1h
-        testtype: structural
-    """
-    apps = unconfigured_appliances
-    app0_ip = apps[0].hostname
-    app1_ip = apps[1].hostname
-    pwd = app_creds['password']
-    # Configure first appliance as dedicated database
-    command_set = ('ap', RETURN, '7', '1', '1', '2', 'y', pwd, TimedCommand(pwd, 360), RETURN)
-    apps[0].appliance_console.run_commands(command_set)
-    wait_for(lambda: apps[0].db.is_dedicated_active)
-    # Configure EVM webui appliance with create region in dedicated database
-    command_set = ('ap', RETURN, '7', '2', app0_ip, RETURN, pwd, RETURN, '2', '0', 'y', app0_ip,
-                   RETURN, RETURN, RETURN, pwd, TimedCommand(pwd, 360), RETURN)
-    apps[2].appliance_console.run_commands(command_set)
-    apps[2].evmserverd.wait_for_running()
-    apps[2].wait_for_miq_ready()
-    # Configure primary replication node
-    command_set = ('ap', RETURN, '8', '1', '1', RETURN, RETURN, pwd, pwd, app0_ip, 'y',
-                   TimedCommand('y', 60), RETURN)
-    apps[0].appliance_console.run_commands(command_set)
-    # Configure secondary replication node
-    command_set = ('ap', RETURN, '8', '2', '2', app0_ip, RETURN, pwd, RETURN, '2', '2', RETURN,
-                   RETURN, pwd, pwd, app0_ip, app1_ip, 'y', TimedCommand('y', 360), RETURN)
-
-    apps[1].appliance_console.run_commands(command_set)
-
-    with waiting_for_ha_monitor_started(apps[2], app1_ip, timeout=300):
-        # Configure automatic failover on EVM appliance
-        command_set = ('ap', RETURN, '10', TimedCommand('1', 30), RETURN)
-        apps[2].appliance_console.run_commands(command_set)
-
-    with LogValidator(evm_log,
-                      matched_patterns=['Starting to execute failover'],
-                      hostname=apps[2].hostname).waiting(timeout=450):
-        # Cause failover to occur
-        result = apps[0].ssh_client.run_command(
-            'systemctl stop $APPLIANCE_PG_SERVICE', timeout=15)
-        assert result.success, f"Failed to stop APPLIANCE_PG_SERVICE: {result.output}"
-
-    apps[2].evmserverd.wait_for_running()
-    apps[2].wait_for_miq_ready()
-
-
 @pytest.mark.tier(2)
 def test_appliance_console_external_db(temp_appliance_unconfig_funcscope, app_creds, appliance):
     """ Commands:
@@ -789,37 +702,6 @@ def test_appliance_console_restart(temp_appliance_preconfig_funcscope):
     navigate_to(appliance.server, "LoggedIn")
 
 
-@test_requirements.ha_proxy
-@pytest.mark.manual
-@pytest.mark.tier(2)
-def test_appliance_console_ha_dc_re_establish():
-    """
-    Test that upon re-connection of networks the repmgr process continues
-    to replicate correctly to the disconnected node within DC2.
-    To setup HA follow:
-    https://access.redhat.com/documentation/en-us/red_hat_cloudforms/4.7/h
-    tml/high_availability_guide//installation
-
-    Polarion:
-        assignee: jhenner
-        casecomponent: HAProxy
-        initialEstimate: 1/2h
-        setup: Restore network connectivity between DC1 and DC2
-        startsin: 5.8
-        testSteps:
-            1. Setup HA
-            2. Disconnect DC2.
-            3. Check repmgr process replicating to DC1.
-            4. Restore network connectivity between DC1 and DC2
-        expectedResults:
-            1.
-            2.
-            3.
-            4. Confirm replication is working correctly
-    """
-    pass
-
-
 @pytest.mark.tier(2)
 def test_appliance_console_evm_stop(temp_appliance_preconfig_funcscope):
     """

cfme/tests/cli/test_appliance_console_db_restore.py

@@ -10,7 +10,6 @@
 from cfme.fixtures.cli import provider_app_crud
 from cfme.infrastructure.provider.virtualcenter import VMwareProvider
 from cfme.utils.appliance.console import configure_appliances_ha
-from cfme.utils.appliance.console import waiting_for_ha_monitor_started
 from cfme.utils.appliance.implementations.ui import navigate_to
 from cfme.utils.browser import manager
 from cfme.utils.conf import cfme_data
@@ -104,76 +103,6 @@ def get_ext_appliances_with_providers(temp_appliances_unconfig_funcscope_rhevm,
     return temp_appliances_unconfig_funcscope_rhevm
 
 
-@pytest.fixture
-def get_ha_appliances_with_providers(unconfigured_appliances, app_creds):
-    """Configure HA environment
-
-    Appliance one configuring dedicated database, 'ap' launch appliance_console,
-    '' clear info screen, '7' setup db, '1' Creates v2_key, '1' selects internal db,
-    '2' use partition, 'y' create dedicated db, 'pwd' db password, 'pwd' confirm db password + wait
-    360 secs and '' finish.
-
-    Appliance two creating region in dedicated database, 'ap' launch appliance_console, '' clear
-    info screen, '7' setup db, '2' fetch v2_key, 'app0_ip' appliance ip address, '' default user,
-    'pwd' appliance password, '' default v2_key location, '2' create region in external db, '0' db
-    region number, 'y' confirm create region in external db 'app0_ip', '' ip and default port for
-    dedicated db, '' use default db name, '' default username, 'pwd' db password, 'pwd' confirm db
-    password + wait 360 seconds and '' finish.
-
-    Appliance one configuring primary node for replication, 'ap' launch appliance_console, '' clear
-    info screen, '8' configure db replication, '1' configure node as primary, '1' cluster node
-    number set to 1, '' default dbname, '' default user, 'pwd' password, 'pwd' confirm password,
-    'app0_ip' primary appliance ip, confirm settings and wait 360 seconds to configure, '' finish.
-
-
-    Appliance three configuring standby node for replication, 'ap' launch appliance_console, ''
-    clear info screen, '8' configure db replication, '1' configure node as primary, '1' cluster node
-    number set to 1, '' default dbname, '' default user, 'pwd' password, 'pwd' confirm password,
-    'app0_ip' primary appliance ip, confirm settings and wait 360 seconds to configure, '' finish.
-
-
-    Appliance two configuring automatic failover of database nodes, 'ap' launch appliance_console,
-    '' clear info screen '10' configure application database failover monitor, '1' start failover
-    monitor. wait 30 seconds for service to start '' finish.
-
-    """
-    appl1, appl2, appl3 = unconfigured_appliances
-    app0_ip = appl1.hostname
-    app1_ip = appl2.hostname
-    pwd = app_creds['password']
-    # Configure first appliance as dedicated database
-    command_set = ('ap', '', '7', '1', '1', '2', 'y', pwd, TimedCommand(pwd, 360), '')
-    appl1.appliance_console.run_commands(command_set)
-    wait_for(lambda: appl1.db.is_dedicated_active)
-    # Configure EVM webui appliance with create region in dedicated database
-    command_set = ('ap', '', '7', '2', app0_ip, '', pwd, '', '2', '0', 'y', app0_ip, '', '', '',
-        TimedCommand(pwd, 360), '')
-    appl3.appliance_console.run_commands(command_set)
-    appl3.evmserverd.wait_for_running()
-    appl3.wait_for_miq_ready()
-    # Configure primary replication node
-    command_set = ('ap', '', '8', '1', '1', '', '', pwd, pwd, app0_ip, TimedCommand('y', 60), '')
-    appl1.appliance_console.run_commands(command_set)
-
-    # Configure secondary replication node
-    command_set = ('ap', '', '8', '2', '2', app0_ip, '', pwd, '', '2', '2', '', '', pwd, pwd,
-                   app0_ip, app1_ip, 'y', TimedCommand('y', 60), '')
-    appl2.appliance_console.run_commands(command_set)
-    #
-    # Configure automatic failover on EVM appliance
-    with waiting_for_ha_monitor_started(appl3, app1_ip, timeout=300):
-        # Configure automatic failover on EVM appliance
-        command_set = ('ap', '', '10', TimedCommand('1', 30), '')
-        appl3.appliance_console.run_commands(command_set)
-
-    # Add infra/cloud providers and create db backup
-    provider_app_crud(VMwareProvider, appl3).setup()
-    provider_app_crud(OpenStackProvider, appl3).setup()
-    appl1.db.backup()
-
-    return unconfigured_appliances
-
-
 def fetch_v2key(appl1, appl2):
     # Fetch v2_key and database.yml from the first appliance
     rand_v2_filename = f"/tmp/v2_key_{fauxfactory.gen_alphanumeric()}"

cfme/tests/cli/test_appliance_update.py

@@ -4,6 +4,7 @@
 import fauxfactory
 import pytest
 
+from cfme import test_requirements
 from cfme.fixtures.cli import do_appliance_versions_match
 from cfme.fixtures.cli import provider_app_crud
 from cfme.fixtures.cli import provision_vm
@@ -12,10 +13,10 @@
 from cfme.test_framework.sprout.client import AuthException
 from cfme.test_framework.sprout.client import SproutClient
 from cfme.utils.appliance import find_appliance
+from cfme.utils.appliance.console import check_db_ha_failover
 from cfme.utils.appliance.implementations.ui import navigate_to
 from cfme.utils.conf import cfme_data
 from cfme.utils.log import logger
-from cfme.utils.log_validator import LogValidator
 from cfme.utils.version import Version
 from cfme.utils.wait import wait_for
 from cfme.utils.wait import wait_for_decorator
@@ -312,6 +313,9 @@ def test_update_replicated_webui(replicated_appliances_preupdate_with_providers,
     soft_assert(vm2.provider.mgmt.does_vm_exist(vm2.name), "vm not provisioned")
 
 
+@test_requirements.appliance
+@test_requirements.ha_proxy
+@test_requirements.update
 @pytest.mark.ignore_stream("upstream")
 @pytest.mark.meta(automates=[1704835])
 @pytest.mark.parametrize("update_strategy", [update_appliance, do_yum_update], ids=["webui", "yum"])
@@ -328,26 +332,13 @@ def test_update_ha(ha_appliances_with_providers, appliance, update_strategy, req
     Bugzilla:
         1704835
     """
-    evm_log = '/var/www/miq/vmdb/log/evm.log'
     update_strategy(ha_appliances_with_providers[2])
     wait_for(do_appliance_versions_match, func_args=(appliance, ha_appliances_with_providers[2]),
              num_sec=900, delay=20, handle_exception=True,
              message='Waiting for appliance to update')
 
-    with LogValidator(evm_log,
-                      matched_patterns=['Starting database failover monitor'],
-                      hostname=ha_appliances_with_providers[2].hostname).waiting(wait=60):
-        ha_appliances_with_providers[2].evm_failover_monitor.restart()
-        assert ha_appliances_with_providers[2].evm_failover_monitor.running
+    check_db_ha_failover(ha_appliances_with_providers[0], ha_appliances_with_providers[2])
 
-    with LogValidator(evm_log,
-                      matched_patterns=['Starting to execute failover'],
-                      hostname=ha_appliances_with_providers[2].hostname).waiting(wait=450):
-        # Cause failover to occur
-        ha_appliances_with_providers[0].db_service.stop()
-
-    ha_appliances_with_providers[2].evmserverd.wait_for_running()
-    ha_appliances_with_providers[2].wait_for_miq_ready()
     # Verify that existing provider can detect new VMs
     virtual_crud = provider_app_crud(VMwareProvider, ha_appliances_with_providers[2])
     vm = provision_vm(request, virtual_crud)

cfme/utils/appliance/console.py

@@ -1,5 +1,6 @@
 import re
 import socket
+import typing
 from contextlib import contextmanager
 
 import lxml
@@ -15,6 +16,9 @@
 from cfme.utils.version import VersionPicker
 from cfme.utils.wait import wait_for
 
+# Workaround problems with circular imports
+if typing.TYPE_CHECKING:
+    from cfme.utils.appliance import IPAppliance
 
 AP_WELCOME_SCREEN_TIMEOUT = 30
 
@@ -281,6 +285,25 @@ def waiting_for_ha_monitor_started(appl, standby_server_ip, timeout):
         wait_for(lambda: appl.evm_failover_monitor.running, timeout=300)
 
 
+def check_db_ha_failover(appl_to_fail: 'IPAppliance', appl_to_takeover: 'IPAppliance'):
+    evm_log = '/var/www/miq/vmdb/log/evm.log'
+
+    with LogValidator(evm_log,
+                      matched_patterns=['Starting database failover monitor'],
+                      hostname=appl_to_takeover.hostname).waiting(wait=60):
+        appl_to_takeover.evm_failover_monitor.restart()
+        assert appl_to_takeover.evm_failover_monitor.running
+
+    with LogValidator(evm_log,
+                      matched_patterns=['Starting to execute failover'],
+                      hostname=appl_to_takeover.hostname).waiting(wait=450):
+        # Cause failover to occur
+        appl_to_fail.db_service.stop()
+
+    appl_to_takeover.evmserverd.wait_for_running()
+    appl_to_takeover.wait_for_miq_ready()
+
+
 def configure_appliances_ha(appliances, pwd):
     """Configure HA environment