PP-736: update log_match in PTL to report failure

test/fw/doc/conf.py

@@ -17,10 +17,10 @@
 
 HAS_RTD = False
 try:
-        import sphinx_rtd_theme
-        HAS_RTD = True
+    import sphinx_rtd_theme
+    HAS_RTD = True
 except:
-        HAS_RTD = False
+    HAS_RTD = False
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -111,10 +111,10 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 if HAS_RTD:
-        html_theme = 'sphinx_rtd_theme'
-        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 else:
-        html_theme = 'sphinxdoc'
+    html_theme = 'sphinxdoc'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -240,7 +240,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [('index', 'pbstestlab', u'PbsTestLab Documentation',
-             [u'Copyright (C) 1994-2017 Altair Engineering, Inc'], 1)]
+              [u'Copyright (C) 1994-2017 Altair Engineering, Inc'], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False

test/fw/ptl/lib/pbs_testlib.py

@@ -469,6 +469,10 @@ class PbsQtermError(PtlException):
     pass
 
 
+class PtlLogMatchError(PtlFailureException):
+    pass
+
+
 class PbsTypeSize(str):
 
     """
@@ -4627,7 +4631,7 @@ def log_lines(self, logtype, id=None, n=50, tail=True, day=None,
     def _log_match(self, logtype, msg, id=None, n=50, tail=True,
                    allmatch=False, regexp=False, day=None, max_attempts=1,
                    interval=1, starttime=None, endtime=None,
-                   level=logging.INFO):
+                   level=logging.INFO, existence=True):
         """
         If ``'msg'`` found in the ``'n'`` lines of the log file,
         returns a ``tupe (x,y)`` where x is the matching line
@@ -4653,30 +4657,38 @@ def _log_match(self, logtype, msg, id=None, n=50, tail=True,
                        Defaults to False
         :type regex: bool
         :param day: Optional day in YYYMMDD format.
+        :type day: str or None
         :param max_attempts: the number of attempts to make to find
                              a matching entry
         :type max_attemps: int
         :param interval: the interval between attempts
         :type interval: int
         :param starttime: If set ignore matches that occur before
                           specified time
+        :type starttime: str or None
         :param endtime: If set ignore matches that occur after
                         specified time
+        :type endtime: str or None
+        :param existence: If True (default), check for existence of
+                        given msg, else check for non-existence of
+                        given msg.
+        :type existence: bool
 
         .. note:: The matching line number is relative to the record
                   number, not the absolute line number in the file.
         """
         try:
             from ptl.utils.pbs_logutils import PBSLogUtils
         except:
-            self.logger.error('error loading ptl.utils.pbs_logutils')
-            return None
+            _msg = 'error loading ptl.utils.pbs_logutils'
+            raise PtlLogMatchError(rc=1, rv=False, msg=_msg)
 
         if self.logutils is None:
             self.logutils = PBSLogUtils()
 
         rv = (None, None)
         attempt = 1
+        lines = None
         name = self._instance_to_servicename(logtype)
         infomsg = (name + ' ' + self.shortname +
                    ' log match: searching for "' + msg + '"')
@@ -4716,12 +4728,15 @@ def _log_match(self, logtype, msg, id=None, n=50, tail=True,
             lines.close()
         except:
             pass
+        if (rv is None and existence) or (rv is not None and not existence):
+            _msg = infomsg + attemptmsg
+            raise PtlLogMatchError(rc=1, rv=False, msg=_msg)
         return rv
 
     def accounting_match(self, msg, id=None, n=50, tail=True,
                          allmatch=False, regexp=False, day=None,
                          max_attempts=1, interval=1, starttime=None,
-                         endtime=None):
+                         endtime=None, level=logging.INFO, existence=True):
         """
         Find msg in accounting log.
 
@@ -4762,16 +4777,19 @@ def accounting_match(self, msg, id=None, n=50, tail=True,
         """
         return self._log_match('accounting', msg, id, n, tail, allmatch,
                                regexp, day, max_attempts, interval, starttime,
-                               endtime)
+                               endtime, level, existence)
 
-    def tracejob_match(self, msg, id=None, n=50, tail=True, allmatch=False,
-                       regexp=False, **kwargs):
+    def tracejob_match(self, msg, id=None, n=50, tail=True,
+                       allmatch=False, regexp=False, day=None,
+                       max_attempts=1, interval=1, starttime=None,
+                       endtime=None, level=logging.INFO, existence=True):
         """
         Find msg in tracejob log. See _log_match for details
 
         """
         return self._log_match('tracejob', msg, id, n, tail, allmatch,
-                               regexp, kwargs)
+                               regexp, day, max_attempts, interval, starttime,
+                               endtime, level, existence)
 
     def _save_config_file(self, dict_conf, fname):
         ret = self.du.cat(self.hostname, fname, sudo=True)
@@ -4982,13 +5000,14 @@ def restart(self):
 
     def log_match(self, msg=None, id=None, n=50, tail=True, allmatch=False,
                   regexp=False, day=None, max_attempts=1, interval=1,
-                  starttime=None, endtime=None, level=logging.INFO):
+                  starttime=None, endtime=None, level=logging.INFO,
+                  existence=True):
         """
         Match the comm logs
         """
         return self._log_match(self, msg, id, n, tail, allmatch, regexp,
                                day, max_attempts, interval, starttime, endtime,
-                               level=level)
+                               level=level, existence=existence)
 
 
 class Server(PBSService):
@@ -5453,13 +5472,14 @@ def restart(self):
 
     def log_match(self, msg=None, id=None, n=50, tail=True, allmatch=False,
                   regexp=False, day=None, max_attempts=1, interval=1,
-                  starttime=None, endtime=None, level=logging.INFO):
+                  starttime=None, endtime=None, level=logging.INFO,
+                  existence=True):
         """
         Match the PBS server logs
         """
         return self._log_match(self, msg, id, n, tail, allmatch, regexp,
                                day, max_attempts, interval, starttime, endtime,
-                               level=level)
+                               level=level, existence=existence)
 
     def revert_to_defaults(self, reverthooks=True, revertqueues=True,
                            revertresources=True, delhooks=True,
@@ -10799,13 +10819,14 @@ def restart(self):
 
     def log_match(self, msg=None, id=None, n=50, tail=True, allmatch=False,
                   regexp=False, day=None, max_attempts=1, interval=1,
-                  starttime=None, endtime=None, level=logging.INFO):
+                  starttime=None, endtime=None, level=logging.INFO,
+                  existence=True):
         """
         Match the scheduler logs
         """
         return self._log_match(self, msg, id, n, tail, allmatch, regexp, day,
                                max_attempts, interval, starttime, endtime,
-                               level=level)
+                               level=level, existence=existence)
 
     def pbs_version(self):
         """
@@ -11020,14 +11041,13 @@ def apply_config(self, config=None, validate=True, path=None):
 
         if validate:
             self.signal('-HUP')
-
-            m = self.log_match("Error reading line", n=10,
-                               starttime=reconfig_time)
-            if m is None:
-                # no match, successful config
-                return True
-            raise PbsSchedConfigError(rc=1, rv=False, msg=str(m))
-
+            try:
+                self.log_match("Error reading line", n=10,
+                               starttime=reconfig_time, existence=False)
+            except PtlLogMatchError:
+                _msg = 'Error in validating sched_config changes'
+                raise PbsSchedConfigError(rc=1, rv=False,
+                                          msg=_msg)
         return True
 
     def set_sched_config(self, confs={}, apply=True, validate=True):
@@ -12673,12 +12693,14 @@ def restart(self):
 
     def log_match(self, msg=None, id=None, n=50, tail=True, allmatch=False,
                   regexp=False, day=None, max_attempts=1, interval=1,
-                  starttime=None, endtime=None):
+                  starttime=None, endtime=None, level=logging.INFO,
+                  existence=True):
         """
         Match the PBS mom logs
         """
         return self._log_match(self, msg, id, n, tail, allmatch, regexp, day,
-                               max_attempts, interval, starttime, endtime)
+                               max_attempts, interval, starttime, endtime,
+                               level, existence)
 
     def pbs_version(self):
         """
@@ -12810,10 +12832,12 @@ def is_cray(self):
         """
         Returns True if the version of PBS used was built for Cray platforms
         """
-        rv = self.log_match("alps_client", tail=False, max_attempts=10)
-        if rv:
+        try:
+            self.log_match("alps_client", tail=False, max_attempts=1)
+        except PtlLogMatchError:
+            return False
+        else:
             return True
-        return False
 
     def is_cpuset_mom(self):
         """

test/fw/ptl/utils/pbs_dshutils.py

@@ -63,6 +63,7 @@ class PbsConfigError(Exception):
     """
     Initialize PBS configuration error
     """
+
     def __init__(self, message=None, rv=None, rc=None, msg=None):
         self.message = message
         self.rv = rv
@@ -82,6 +83,7 @@ class PtlUtilError(Exception):
     """
     Initialize PTL Util error
     """
+
     def __init__(self, message=None, rv=None, rc=None, msg=None):
         self.message = message
         self.rv = rv

test/fw/ptl/utils/pbs_logutils.py

@@ -1049,6 +1049,7 @@ class JobEstimatedStartTimeInfo(object):
     """
     Information regarding Job estimated start time
     """
+
     def __init__(self, jobid):
         self.jobid = jobid
         self.started_at = None