DM-10608: optimise Config history

python/lsst/pex/config/callStack.py

@@ -0,0 +1,182 @@
+#
+# LSST Data Management System
+# Copyright 2017 AURA/LSST.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <https://www.lsstcorp.org/LegalNotices/>.
+#
+
+from __future__ import print_function, division, absolute_import
+
+__all__ = ['getCallerFrame', 'getStackFrame', 'StackFrame', 'getCallStack']
+
+from builtins import object
+
+import re
+import os
+import inspect
+import linecache
+
+
+def getCallerFrame(relative=0):
+    """Retrieve the frame for the caller
+
+    By "caller", we mean our user's caller.
+
+    Parameters
+    ----------
+    relative : `int`, non-negative
+        Number of frames above the caller to retrieve.
+
+    Returns
+    -------
+    frame : `__builtin__.Frame`
+        Frame for the caller.
+    """
+    frame = inspect.currentframe().f_back.f_back  # Our caller's caller
+    for ii in range(relative):
+        frame = frame.f_back
+    return frame
+
+
+def getStackFrame(relative=0):
+    """Retrieve the stack frame for the caller
+
+    By "caller", we mean our user's caller.
+
+    Parameters
+    ----------
+    relative : `int`, non-negative
+        Number of frames above the caller to retrieve.
+
+    Returns
+    -------
+    frame : `StackFrame`
+        Stack frame for the caller.
+    """
+    frame = getCallerFrame(relative + 1)
+    return StackFrame.fromFrame(frame)
+
+
+class StackFrame(object):
+    """A single element of the stack trace
+
+    This differs slightly from the standard system mechanisms for
+    getting a stack trace by the fact that it does not look up the
+    source code until it is absolutely necessary, reducing the I/O.
+
+    Parameters
+    ----------
+    filename : `str`
+        Name of file containing the code being executed.
+    lineno : `int`
+        Line number of file being executed.
+    function : `str`
+        Function name being executed.
+    content : `str` or `None`
+        The actual content being executed. If not provided, it will be
+        loaded from the file.
+    """
+    _STRIP = "/python/lsst/"  # String to strip from the filename
+
+    def __init__(self, filename, lineno, function, content=None):
+        loc = filename.rfind(self._STRIP)
+        if loc > 0:
+            filename = filename[loc + len(self._STRIP):]
+        self.filename = filename
+        self.lineno = lineno
+        self.function = function
+        self._content = content
+
+    @property
+    def content(self):
+        """Getter for content being executed
+
+        Load from file on demand.
+        """
+        if self._content is None:
+            self._content = linecache.getline(self.filename, self.lineno).strip()
+        return self._content
+
+    @classmethod
+    def fromFrame(cls, frame):
+        """Construct from a Frame object
+
+        inspect.currentframe() provides a Frame object. This is
+        a convenience constructor to interpret that Frame object.
+
+        Parameters
+        ----------
+        frame : `Frame`
+            Frame object to interpret.
+
+        Returns
+        -------
+        output : `StackFrame`
+            Constructed object.
+        """
+        filename = frame.f_code.co_filename
+        lineno = frame.f_lineno
+        function = frame.f_code.co_name
+        return cls(filename, lineno, function)
+
+    def __repr__(self):
+        return "%s(%s, %s, %s)" % (self.__class__.__name__ , self.filename, self.lineno, self.function)
+
+    def format(self, full=False):
+        """Format for printing
+
+        Parameters
+        ----------
+        full : `bool`
+            Print full details, including content being executed?
+
+        Returns
+        -------
+        result : `str`
+            Formatted string.
+        """
+        result = "  File %s:%s (%s)" % (self.filename, self.lineno, self.function)
+        if full:
+            result += "\n    %s" % (self.content,)
+        return result
+
+
+def getCallStack(skip=0):
+    """Retrieve the call stack for the caller
+
+    By "caller", we mean our user's caller - we don't include ourselves
+    or our caller.
+
+    The result is ordered with the most recent frame last.
+
+    Parameters
+    ----------
+    skip : `int`, non-negative
+        Number of stack frames above caller to skip.
+
+    Returns
+    -------
+    output : `list` of `StackFrame`
+        The call stack.
+    """
+    frame = getCallerFrame(skip + 1)
+    stack = []
+    while frame:
+        stack.append(StackFrame.fromFrame(frame))
+        frame = frame.f_back
+    return list(reversed(stack))

python/lsst/pex/config/choiceField.py

@@ -21,9 +21,9 @@
 #
 oldStringType = str  # Need to keep hold of original str type
 from builtins import str
-import traceback
 
 from .config import Field, _typeStr
+from .callStack import getStackFrame
 
 __all__ = ["ChoiceField"]
 
@@ -56,7 +56,7 @@ def __init__(self, doc, dtype, allowed, default=None, optional=True):
 
         Field.__init__(self, doc=doc, dtype=dtype, default=default,
                        check=None, optional=optional)
-        self.source = traceback.extract_stack(limit=2)[0]
+        self.source = getStackFrame()
 
     def _validateValue(self, value):
         Field._validateValue(self, value)

python/lsst/pex/config/config.py

@@ -28,14 +28,14 @@
 
 import os
 import io
-import traceback
 import sys
 import math
 import copy
 import tempfile
 import shutil
 
 from .comparison import getComparisonName, compareScalars, compareConfigs
+from .callStack import getStackFrame, getCallStack
 from future.utils import with_metaclass
 
 __all__ = ("Config", "Field", "FieldValidationError")
@@ -100,7 +100,7 @@ class ConfigMeta(type):
     def __init__(self, name, bases, dict_):
         type.__init__(self, name, bases, dict_)
         self._fields = {}
-        self._source = traceback.extract_stack(limit=2)[0]
+        self._source = getStackFrame()
 
         def getFields(classtype):
             fields = {}
@@ -146,8 +146,7 @@ def __init__(self, field, config, msg):
                 "For more information read the Field definition at:\n%s"\
                 "And the Config definition at:\n%s" % \
             (self.fieldType.__name__, self.fullname, msg,
-             traceback.format_list([self.fieldSource])[0],
-             traceback.format_list([self.configSource])[0])
+             self.fieldSource.format(), self.configSource.format())
         ValueError.__init__(self, error)
 
 
@@ -183,7 +182,7 @@ def __init__(self, doc, dtype, default=None, check=None, optional=False):
         if dtype == str:
             dtype = oldStringType
 
-        source = traceback.extract_stack(limit=2)[0]
+        source = getStackFrame()
         self._setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
 
     def _setup(self, doc, dtype, default, check, optional, source):
@@ -339,7 +338,7 @@ def __set__(self, instance, value, at=None, label='assignment'):
 
         instance._storage[self.name] = value
         if at is None:
-            at = traceback.extract_stack()[:-1]
+            at = getCallStack()
         history.append((value, at, label))
 
     def __delete__(self, instance, at=None, label='deletion'):
@@ -349,7 +348,7 @@ def __delete__(self, instance, at=None, label='deletion'):
         directly
         """
         if at is None:
-            at = traceback.extract_stack()[:-1]
+            at = getCallStack()
         self.__set__(instance, None, at=at, label=label)
 
     def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
@@ -483,7 +482,7 @@ def __new__(cls, *args, **kw):
         should call the base Config.__init__
         """
         name = kw.pop("__name", None)
-        at = kw.pop("__at", traceback.extract_stack()[:-1])
+        at = kw.pop("__at", getCallStack())
         # remove __label and ignore it
         kw.pop("__label", "default")
 
@@ -496,7 +495,7 @@ def __new__(cls, *args, **kw):
         # load up defaults
         for field in instance._fields.values():
             instance._history[field.name] = []
-            field.__set__(instance, field.default, at=at+[field.source], label="default")
+            field.__set__(instance, field.default, at=at + [field.source], label="default")
         # set custom default-overides
         instance.setDefaults()
         # set constructor overides
@@ -531,7 +530,7 @@ def update(self, **kw):
         history tracebacks of the config. Modifying these keywords allows users
         to lie about a Config's history. Please do not do so!
         """
-        at = kw.pop("__at", traceback.extract_stack()[:-1])
+        at = kw.pop("__at", getCallStack())
         label = kw.pop("__label", "update")
 
         for name, value in kw.items():
@@ -718,7 +717,7 @@ def __setattr__(self, attr, value, at=None, label="assignment"):
         """
         if attr in self._fields:
             if at is None:
-                at = traceback.extract_stack()[:-1]
+                at = getCallStack()
             # This allows Field descriptors to work.
             self._fields[attr].__set__(self, value, at=at, label=label)
         elif hasattr(getattr(self.__class__, attr, None), '__set__'):
@@ -734,7 +733,7 @@ def __setattr__(self, attr, value, at=None, label="assignment"):
     def __delattr__(self, attr, at=None, label="deletion"):
         if attr in self._fields:
             if at is None:
-                at = traceback.extract_stack()[:-1]
+                at = getCallStack()
             self._fields[attr].__delete__(self, at=at, label=label)
         else:
             object.__delattr__(self, attr)