From 0e1a1f4e3e09619e5ebb8851a11dbc696b1f609c Mon Sep 17 00:00:00 2001 From: Ravi Agrawal Date: Fri, 16 Nov 2018 15:35:49 +0530 Subject: [PATCH] PTL doesn't revert pbs.conf in setUp --- test/fw/ptl/utils/pbs_testsuite.py | 326 ++++++++++++++++++++++++ test/tests/pbs_smoketest.py | 5 +- test/tests/selftest/pbs_pbstestsuite.py | 260 +++++++++++++++++++ 3 files changed, 588 insertions(+), 3 deletions(-) create mode 100644 test/tests/selftest/pbs_pbstestsuite.py diff --git a/test/fw/ptl/utils/pbs_testsuite.py b/test/fw/ptl/utils/pbs_testsuite.py index 9c16b1f45a5..1795db4a6f1 100644 --- a/test/fw/ptl/utils/pbs_testsuite.py +++ b/test/fw/ptl/utils/pbs_testsuite.py @@ -440,6 +440,7 @@ def setUp(self): return self.log_enter_setup() self.init_proc_mon() + self.revert_pbsconf() self.revert_servers() self.revert_comms() self.revert_schedulers() @@ -831,6 +832,331 @@ def init_proc_mon(self): frequency=freq) self._process_monitoring = True + def _get_dflt_pbsconfval(self, conf, primary_host, hosttype, hostobj): + """ + Helper function to revert_pbsconf, tries to determine and return + default value for the pbs.conf variable given + + :param conf: the pbs.conf variable + :type conf: str + :param primary_host: hostname of the primary server host + :type primary_host: str + :param hosttype: type of host being reverted + :type hosttype: str + :param hostobj: PTL object associated with the host + :type hostobj: PBSService + + :return default value of the pbs.conf variable if it can be determined + as a string, otherwise None + """ + if conf == "PBS_SERVER": + return primary_host + elif conf == "PBS_START_SCHED": + if hosttype == "server": + return "1" + else: + return "0" + elif conf == "PBS_START_COMM": + if hosttype == "comm": + return "1" + else: + return "0" + elif conf == "PBS_START_SERVER": + if hosttype == "server": + return "1" + else: + return "0" + elif conf == "PBS_START_MOM": + if hosttype == "mom": + return "1" + else: + return "0" + elif conf == "PBS_CORE_LIMIT": + return "unlimited" + elif conf == "PBS_SCP": + scppath = self.du.which(hostobj.hostname, "scp") + if scppath != "scp": + return scppath + + return None + + def _revert_pbsconf_comm(self, primary_server, vals_to_set): + """ + Helper function to revert_pbsconf to revert all comm daemons' pbs.conf + + :param primary_server: object of the primary PBS server + :type primary_server: PBSService + :param vals_to_set: dict of pbs.conf values to set + :type vals_to_set: dict + """ + for comm in self.comms.values(): + if comm.hostname == primary_server.hostname: + continue + + new_pbsconf = dict(vals_to_set) + restart_comm = False + pbs_conf_val = self.du.parse_pbs_config(comm.hostname) + if not pbs_conf_val: + raise ValueError("Could not parse pbs.conf on host %s" % + (comm.hostname)) + + # to start with, set all keys in new_pbsconf with values from the + # existing pbs.conf + keys_to_delete = [] + for conf in new_pbsconf: + if conf in pbs_conf_val: + new_pbsconf[conf] = pbs_conf_val[conf] + else: + # existing pbs.conf doesn't have a default variable set + # Try to determine the default + val = self._get_dflt_pbsconfval(conf, + primary_server.hostname, + "comm", comm) + if val is None: + self.logger.error("Couldn't revert %s in pbs.conf" + " to its default value" % + (conf)) + keys_to_delete.append(conf) + else: + new_pbsconf[conf] = val + + for key in keys_to_delete: + del(new_pbsconf[key]) + + # Set the comm start bit to 1 + if new_pbsconf["PBS_START_COMM"] != "1": + new_pbsconf["PBS_START_COMM"] = "1" + restart_comm = True + + # Set PBS_CORE_LIMIT, PBS_SCP and PBS_SERVER + if new_pbsconf["PBS_CORE_LIMIT"] != "unlimited": + new_pbsconf["PBS_CORE_LIMIT"] = "unlimited" + restart_comm = True + if new_pbsconf["PBS_SERVER"] != primary_server.hostname: + new_pbsconf["PBS_SERVER"] = primary_server.hostname + restart_comm = True + if "PBS_SCP" not in new_pbsconf: + scppath = self.du.which(comm.hostname, "scp") + if scppath != "scp": + new_pbsconf["PBS_SCP"] = scppath + restart_comm = True + + # Check if existing pbs.conf has more/less entries than the + # default list + if len(pbs_conf_val) != len(new_pbsconf): + restart_comm = True + + if restart_comm: + self.du.set_pbs_config(comm.hostname, confs=new_pbsconf) + comm.pbs_conf = new_pbsconf + comm.pi.initd(comm.hostname, "restart", daemon="comm") + + def _revert_pbsconf_mom(self, primary_server, vals_to_set): + """ + Helper function to revert_pbsconf to revert all mom daemons' pbs.conf + + :param primary_server: object of the primary PBS server + :type primary_server: PBSService + :param vals_to_set: dict of pbs.conf values to set + :type vals_to_set: dict + """ + for mom in self.moms.values(): + if mom.hostname == primary_server.hostname: + continue + + new_pbsconf = dict(vals_to_set) + restart_mom = False + pbs_conf_val = self.du.parse_pbs_config(mom.hostname) + if not pbs_conf_val: + raise ValueError("Could not parse pbs.conf on host %s" % + (mom.hostname)) + + # to start with, set all keys in new_pbsconf with values from the + # existing pbs.conf + keys_to_delete = [] + for conf in new_pbsconf: + if conf in pbs_conf_val: + new_pbsconf[conf] = pbs_conf_val[conf] + else: + # existing pbs.conf doesn't have a default variable set + # Try to determine the default + val = self._get_dflt_pbsconfval(conf, + primary_server.hostname, + "mom", mom) + if val is None: + self.logger.error("Couldn't revert %s in pbs.conf" + " to its default value" % + (conf)) + keys_to_delete.append(conf) + else: + new_pbsconf[conf] = val + + for key in keys_to_delete: + del(new_pbsconf[key]) + + # Set the mom start bit to 1 + if (new_pbsconf["PBS_START_MOM"] != "1"): + new_pbsconf["PBS_START_MOM"] = "1" + restart_mom = True + + # Set PBS_CORE_LIMIT, PBS_SCP and PBS_SERVER + if new_pbsconf["PBS_CORE_LIMIT"] != "unlimited": + new_pbsconf["PBS_CORE_LIMIT"] = "unlimited" + restart_mom = True + if new_pbsconf["PBS_SERVER"] != primary_server.hostname: + new_pbsconf["PBS_SERVER"] = primary_server.hostname + restart_mom = True + if "PBS_SCP" not in new_pbsconf: + scppath = self.du.which(mom.hostname, "scp") + if scppath != "scp": + new_pbsconf["PBS_SCP"] = scppath + restart_mom = True + + # Check if existing pbs.conf has more/less entries than the + # default list + if len(pbs_conf_val) != len(new_pbsconf): + restart_mom = True + + if restart_mom: + self.du.set_pbs_config(mom.hostname, confs=new_pbsconf, + append=False) + mom.pbs_conf = new_pbsconf + mom.pi.initd(mom.hostname, "restart", daemon="mom") + + def _revert_pbsconf_server(self, primary_server, vals_to_set): + """ + Helper function to revert_pbsconf to revert all servers' pbs.conf + + :param primary_server: object of the primary PBS server + :type primary_server: PBSService + :param vals_to_set: dict of pbs.conf values to set + :type vals_to_set: dict + """ + for server in self.servers.values(): + new_pbsconf = dict(vals_to_set) + cmds_to_exec = [] + dmns_to_restart = 0 + restart_pbs = False + pbs_conf_val = self.du.parse_pbs_config(server.hostname) + if not pbs_conf_val: + raise ValueError("Could not parse pbs.conf on host %s" % + (server.hostname)) + + # to start with, set all keys in new_pbsconf with values from the + # existing pbs.conf + keys_to_delete = [] + for conf in new_pbsconf: + if conf in pbs_conf_val: + new_pbsconf[conf] = pbs_conf_val[conf] + else: + # existing pbs.conf doesn't have a default variable set + # Try to determine the default + val = self._get_dflt_pbsconfval(conf, + primary_server.hostname, + "server", server) + if val is None: + self.logger.error("Couldn't revert %s in pbs.conf" + " to its default value" % + (conf)) + keys_to_delete.append(conf) + else: + new_pbsconf[conf] = val + + for key in keys_to_delete: + del(new_pbsconf[key]) + + # Set all start bits + if (new_pbsconf["PBS_START_SERVER"] != "1"): + new_pbsconf["PBS_START_SERVER"] = "1" + dmns_to_restart += 1 + cmds_to_exec.append(["server", "start"]) + if (new_pbsconf["PBS_START_SCHED"] != "1"): + new_pbsconf["PBS_START_SCHED"] = "1" + cmds_to_exec.append(["sched", "start"]) + dmns_to_restart += 1 + if self.moms and server.hostname not in self.moms: + if new_pbsconf["PBS_START_MOM"] != "0": + new_pbsconf["PBS_START_MOM"] = "0" + cmds_to_exec.append(["mom", "stop"]) + dmns_to_restart += 1 + else: + if (new_pbsconf["PBS_START_MOM"] != "1"): + new_pbsconf["PBS_START_MOM"] = "1" + cmds_to_exec.append(["mom", "start"]) + dmns_to_restart += 1 + if self.comms and server.hostname not in self.comms: + if new_pbsconf["PBS_START_COMM"] != "0": + new_pbsconf["PBS_START_COMM"] = "0" + cmds_to_exec.append(["comm", "stop"]) + else: + if (new_pbsconf["PBS_START_COMM"] != "1"): + new_pbsconf["PBS_START_COMM"] = "1" + cmds_to_exec.append(["comm", "start"]) + dmns_to_restart += 1 + + if dmns_to_restart == 4: + # If all daemons need to be started again, just restart PBS + # instead of making PTL start each of them one at a time + restart_pbs = True + + # Set PBS_CORE_LIMIT, PBS_SCP and PBS_SERVER + if new_pbsconf["PBS_CORE_LIMIT"] != "unlimited": + new_pbsconf["PBS_CORE_LIMIT"] = "unlimited" + restart_pbs = True + if new_pbsconf["PBS_SERVER"] != primary_server.hostname: + new_pbsconf["PBS_SERVER"] = primary_server.hostname + restart_pbs = True + if "PBS_SCP" not in new_pbsconf: + scppath = self.du.which(server.hostname, "scp") + if scppath != "scp": + new_pbsconf["PBS_SCP"] = scppath + restart_pbs = True + + # Check if existing pbs.conf has more/less entries than the + # default list + if len(pbs_conf_val) != len(new_pbsconf): + restart_pbs = True + + if restart_pbs or dmns_to_restart > 0: + # Write out the new pbs.conf file + self.du.set_pbs_config(server.hostname, confs=new_pbsconf, + append=False) + server.pbs_conf = new_pbsconf + + if restart_pbs: + # Restart all + server.pi.restart(server.hostname) + else: + for initcmd in cmds_to_exec: + # start/stop the particular daemon + server.pi.initd(server.hostname, initcmd[1], + daemon=initcmd[0]) + + def revert_pbsconf(self): + """ + Revert contents of the pbs.conf file + Also start/stop the appropriate daemons + """ + primary_server = self.server + + vals_to_set = { + "PBS_HOME": None, + "PBS_EXEC": None, + "PBS_SERVER": None, + "PBS_START_SCHED": None, + "PBS_START_COMM": None, + "PBS_START_SERVER": None, + "PBS_START_MOM": None, + "PBS_CORE_LIMIT": None, + "PBS_SCP": None + } + + self._revert_pbsconf_server(primary_server, vals_to_set) + + self._revert_pbsconf_mom(primary_server, vals_to_set) + + self._revert_pbsconf_comm(primary_server, vals_to_set) + def revert_servers(self, force=False): """ Revert the values set for servers diff --git a/test/tests/pbs_smoketest.py b/test/tests/pbs_smoketest.py index 3a77b38a873..32f7acd9a11 100644 --- a/test/tests/pbs_smoketest.py +++ b/test/tests/pbs_smoketest.py @@ -484,9 +484,8 @@ def test_mom_hook(self): max_attempts=100, interval=5) j = Job(TEST_USER) jid = self.server.submit(j) - self.server.expect(JOB, {ATTR_state: 'H'}, id=jid) - self.mom.log_match("my custom message", max_attempts=10, - starttime=self.server.ctime) + self.mom.log_match("my custom message", starttime=self.server.ctime, + interval=1) @skipOnCpuSet def test_shrink_to_fit(self): diff --git a/test/tests/selftest/pbs_pbstestsuite.py b/test/tests/selftest/pbs_pbstestsuite.py new file mode 100644 index 00000000000..f4c2dd4a70f --- /dev/null +++ b/test/tests/selftest/pbs_pbstestsuite.py @@ -0,0 +1,260 @@ +# coding: utf-8 + +# Copyright (C) 1994-2018 Altair Engineering, Inc. +# For more information, contact Altair at www.altair.com. +# +# This file is part of the PBS Professional ("PBS Pro") software. +# +# Open Source License Information: +# +# PBS Pro is free software. You can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# Commercial License Information: +# +# For a copy of the commercial license terms and conditions, +# go to: (http://www.pbspro.com/UserArea/agreement.html) +# or contact the Altair Legal Department. +# +# Altair’s dual-license business model allows companies, individuals, and +# organizations to create proprietary derivative works of PBS Pro and +# distribute them - whether embedded or bundled with other software - +# under a commercial license agreement. +# +# Use of Altair’s trademarks, including but not limited to "PBS™", +# "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's +# trademark licensing policies. + +from tests.selftest import * +from ptl.utils.pbs_snaputils import * + + +class TestPBSTestSuite(TestSelf): + """ + Contains tests for pbs_testsuite module's functionality + """ + + def test_revert_pbsconf_onehost(self): + """ + Test the functionality of PBSTestSuite.revert_pbsconf() + for a single host type 1 installation + """ + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertTrue(pbs_conf_val and len(pbs_conf_val) >= 1, + "Could not parse pbs.conf on host %s" % + (self.server.hostname)) + + # Since the setUp already ran, check that the start bits are turned on + self.assertEqual(pbs_conf_val["PBS_START_MOM"], "1") + self.assertEqual(pbs_conf_val["PBS_START_SERVER"], "1") + self.assertEqual(pbs_conf_val["PBS_START_SCHED"], "1") + self.assertEqual(pbs_conf_val["PBS_START_COMM"], "1") + + self.server.pi.stop() + + # Now, change pbs.conf to turn the sched off + pbs_conf_val["PBS_START_SCHED"] = "0" + self.du.set_pbs_config(confs=pbs_conf_val) + + # Start PBS again + self.server.pi.start() + + # Verify that the scheduler didn't come up + self.assertFalse(self.scheduler.isUp()) + + # Now call revert_pbsconf() + self.revert_pbsconf() + + # Verify that the scheduler came up and start bit is 1 + self.assertTrue(self.scheduler.isUp()) + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertEqual(pbs_conf_val["PBS_START_SCHED"], "1") + + def test_revert_pbsconf_remotemom(self): + """ + Test the functionality of PBSTestSuite.revert_pbsconf() + with a remote mom setup + """ + remotemom = None + for mom in self.moms.values(): + if not self.du.is_localhost(mom.hostname): + remotemom = mom + break + if remotemom is None: + self.skip_test("Test needs at least one remote Mom host," + " use -p moms=") + + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertTrue(pbs_conf_val and len(pbs_conf_val) >= 1, + "Could not parse pbs.conf on host %s" % + (self.server.hostname)) + + # Check that the start bits on server host are set correctly + self.assertEqual(pbs_conf_val["PBS_START_SERVER"], "1") + self.assertEqual(pbs_conf_val["PBS_START_SCHED"], "1") + self.assertEqual(pbs_conf_val["PBS_START_COMM"], "1") + if self.server.hostname in self.moms: + self.assertEqual(pbs_conf_val["PBS_START_MOM"], "1") + else: + self.assertEqual(pbs_conf_val["PBS_START_MOM"], "0") + + # Check that the remote mom's pbs.conf has mom start bit on + pbs_conf_val = self.du.parse_pbs_config(remotemom.hostname) + self.assertEqual(pbs_conf_val["PBS_START_MOM"], "1") + + # Now set it to 0 and restart the mom + remotemom.pi.stop(remotemom.hostname) + pbs_conf_val["PBS_START_MOM"] = "0" + self.du.set_pbs_config(remotemom.hostname, confs=pbs_conf_val) + remotemom.pi.start(remotemom.hostname) + + # Confirm that the mom is down + self.assertFalse(remotemom.isUp()) + + # Now call revert_pbsconf() + self.revert_pbsconf() + + # Confirm that the mom came up and start bit is 1 + self.assertTrue(remotemom.isUp()) + pbs_conf_val = self.du.parse_pbs_config(remotemom.hostname) + self.assertEqual(pbs_conf_val["PBS_START_MOM"], "1") + + def test_revert_pbsconf_corelimit(self): + """ + Test the functionality of PBSTestSuite.revert_pbsconf() when + PBS_CORE_LIMIT is set to a value other than the default + """ + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertTrue(pbs_conf_val and len(pbs_conf_val) >= 1, + "Could not parse pbs.conf on host %s" % + (self.server.hostname)) + + # Since the setUp already ran, check that PBS_CORE_LIMIT is set to + # unlimited + self.assertEqual(pbs_conf_val["PBS_CORE_LIMIT"], "unlimited") + + # Now, set the core limit to 0 and restart PBS + self.server.pi.stop() + pbs_conf_val["PBS_CORE_LIMIT"] = "0" + self.du.set_pbs_config(confs=pbs_conf_val) + self.server.pi.start() + + # First, check that there's no existing core file in mom_priv + mom_priv_path = os.path.join(self.server.pbs_conf["PBS_HOME"], + "mom_priv") + mom_priv_filenames = self.du.listdir(self.server.hostname, + mom_priv_path, sudo=True, + fullpath=False) + for filename in mom_priv_filenames: + if filename.startswith("core"): + # Found a core file, delete it + corepath = os.path.join(mom_priv_path, filename) + self.du.rm(self.server.hostname, corepath, sudo=True, + force=True) + + # Send SIGSEGV to pbs_mom + self.assertTrue(self.mom.isUp()) + self.mom.signal("-SEGV") + self.assertFalse(self.mom.isUp()) + + # Confirm that no core file was generated + mom_priv_filenames = self.du.listdir(self.server.hostname, + mom_priv_path, sudo=True, + fullpath=False) + corefound = False + for filename in mom_priv_filenames: + if filename.startswith("core"): + corefound = True + break + self.assertFalse(corefound, "mom unexpectedly dumped core") + + # Now, call self.revert_pbsconf() + self.revert_pbsconf() + + # Confirm that PBS_CORE_LIMIT was reverted + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertEqual(pbs_conf_val["PBS_CORE_LIMIT"], "unlimited") + + # Send another SIGSEGV to pbs_mom + self.assertTrue(self.mom.isUp()) + self.mom.signal("-SEGV") + self.assertFalse(self.mom.isUp()) + + # Confirm that a core file was generated this time + mom_priv_filenames = self.du.listdir(self.server.hostname, + mom_priv_path, sudo=True, + fullpath=False) + corefound = False + for filename in mom_priv_filenames: + if filename.startswith("core"): + corefound = True + break + self.assertTrue(corefound, + "mom was expected to dump core but it didn't") + + def test_revert_pbsconf_extra_vars(self): + """ + Test the functionality of PBSTestSuite.revert_pbsconf() when + there are extra pbs.conf variables than the default + """ + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertTrue(pbs_conf_val and len(pbs_conf_val) >= 1, + "Could not parse pbs.conf on host %s" % + (self.server.hostname)) + + # Set a non-default pbs.conf variables, let's say + # PBS_LOG_HIGHRES_TIMESTAMP, and restart PBS + self.server.pi.stop() + pbs_conf_val["PBS_LOG_HIGHRES_TIMESTAMP"] = "1" + self.du.set_pbs_config(confs=pbs_conf_val) + self.server.pi.start() + + # Confirm that the pbs.conf variable is set + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertEqual(pbs_conf_val["PBS_LOG_HIGHRES_TIMESTAMP"], "1") + + # Now, call self.revert_pbsconf() + self.revert_pbsconf() + + # Confirm that the value gets removed from the list as it is not + # a default setting + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertFalse("PBS_LOG_HIGHRES_TIMESTAMP" in pbs_conf_val) + + def test_revert_pbsconf_fewer_vars(self): + """ + Test the functionality of PBSTestSuite.revert_pbsconf() when + there are fewer pbs.conf variables than the default + """ + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertTrue(pbs_conf_val and len(pbs_conf_val) >= 1, + "Could not parse pbs.conf on host %s" % + (self.server.hostname)) + + # Remove a default pbs.conf variable, say PBS_CORE_LIMIT, + # and restart PBS + self.server.pi.stop() + del pbs_conf_val["PBS_CORE_LIMIT"] + self.du.set_pbs_config(confs=pbs_conf_val, append=False) + self.server.pi.start() + + # Confirm that the pbs.conf variable is gone + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertNotIn("PBS_CORE_LIMIT", pbs_conf_val) + + # Now, call self.revert_pbsconf() + self.revert_pbsconf() + + # Confirm that the variable was set again + pbs_conf_val = self.du.parse_pbs_config(self.server.hostname) + self.assertIn("PBS_CORE_LIMIT", pbs_conf_val)