yaml_parse.py

"""Support code for YAML parsing of experiment descriptions."""
import yaml
from pylearn2.utils import serial
from pylearn2.utils.exc import reraise_as
from pylearn2.utils.string_utils import preprocess
from pylearn2.utils.call_check import checked_call
from pylearn2.utils.string_utils import match
from collections import namedtuple
import logging
import warnings
import re

from theano.compat import six

SCIENTIFIC_NOTATION_REGEXP = r'^[\-\+]?(\d+\.?\d*|\d*\.?\d+)?[eE][\-\+]?\d+$'

is_initialized = False
additional_environ = None
logger = logging.getLogger(__name__)

# Lightweight container for initial YAML evaluation.
#
# This is intended as a robust, forward-compatible intermediate representation
# for either internal consumption or external consumption by another tool e.g.
# hyperopt.
#
# We've included a slot for positionals just in case, though they are
# unsupported by the instantiation mechanism as yet.
BaseProxy = namedtuple('BaseProxy', ['callable', 'positionals',
                                     'keywords', 'yaml_src'])


class Proxy(BaseProxy):
    """
    An intermediate representation between initial YAML parse and object
    instantiation.

    Parameters
    ----------
    callable : callable
        The function/class to call to instantiate this node.
    positionals : iterable
        Placeholder for future support for positional arguments (`*args`).
    keywords : dict-like
        A mapping from keywords to arguments (`**kwargs`), which may be
        `Proxy`s or `Proxy`s nested inside `dict` or `list` instances.
        Keys must be strings that are valid Python variable names.
    yaml_src : str
        The YAML source that created this node, if available.

    Notes
    -----
    This is intended as a robust, forward-compatible intermediate
    representation for either internal consumption or external consumption
    by another tool e.g. hyperopt.

    This particular class mainly exists to  override `BaseProxy`'s `__hash__`
    (to avoid hashing unhashable namedtuple elements).
    """
    __slots__ = []

    def __hash__(self):
        """
        Return a hash based on the object ID (to avoid hashing unhashable
        namedtuple elements).
        """
        return hash(id(self))


def do_not_recurse(value):
    """
    Function symbol used for wrapping an unpickled object (which should
    not be recursively expanded). This is recognized and respected by the
    instantiation parser. Implementationally, no-op (returns the value
    passed in as an argument).

    Parameters
    ----------
    value : object
        The value to be returned.

    Returns
    -------
    value : object
        The same object passed in as an argument.
    """
    return value


def _instantiate_proxy_tuple(proxy, bindings=None):
    """
    Helper function for `_instantiate` that handles objects of the `Proxy`
    class.

    Parameters
    ----------
    proxy : Proxy object
        A `Proxy` object that.
    bindings : dict, opitonal
        A dictionary mapping previously instantiated `Proxy` objects
        to their instantiated values.

    Returns
    -------
    obj : object
        The result object from recursively instantiating the object DAG.
    """
    if proxy in bindings:
        return bindings[proxy]
    else:
        # Respect do_not_recurse by just un-packing it (same as calling).
        if proxy.callable == do_not_recurse:
            obj = proxy.keywords['value']
        else:
            # TODO: add (requested) support for positionals (needs to be added
            # to checked_call also).
            if len(proxy.positionals) > 0:
                raise NotImplementedError('positional arguments not yet '
                                          'supported in proxy instantiation')
            kwargs = dict((k, _instantiate(v, bindings))
                          for k, v in six.iteritems(proxy.keywords))
            obj = checked_call(proxy.callable, kwargs)
        try:
            obj.yaml_src = proxy.yaml_src
        except AttributeError:  # Some classes won't allow this.
            pass
        bindings[proxy] = obj
        return bindings[proxy]


def _instantiate(proxy, bindings=None):
    """
    Instantiate a (hierarchy of) Proxy object(s).

    Parameters
    ----------
    proxy : object
        A `Proxy` object or list/dict/literal. Strings are run through
        `preprocess`.
    bindings : dict, opitonal
        A dictionary mapping previously instantiated `Proxy` objects
        to their instantiated values.

    Returns
    -------
    obj : object
        The result object from recursively instantiating the object DAG.

    Notes
    -----
    This should not be considered part of the stable, public API.
    """
    if bindings is None:
        bindings = {}
    if isinstance(proxy, Proxy):
        return _instantiate_proxy_tuple(proxy, bindings)
    elif isinstance(proxy, dict):
        # Recurse on the keys too, for backward compatibility.
        # Is the key instantiation feature ever actually used, by anyone?
        return dict((_instantiate(k, bindings), _instantiate(v, bindings))
                    for k, v in six.iteritems(proxy))
    elif isinstance(proxy, list):
        return [_instantiate(v, bindings) for v in proxy]
    # In the future it might be good to consider a dict argument that provides
    # a type->callable mapping for arbitrary transformations like this.
    elif isinstance(proxy, six.string_types):
        return preprocess(proxy)
    else:
        return proxy


def load(stream, environ=None, instantiate=True, **kwargs):
    """
    Loads a YAML configuration from a string or file-like object.

    Parameters
    ----------
    stream : str or object
        Either a string containing valid YAML or a file-like object
        supporting the .read() interface.
    environ : dict, optional
        A dictionary used for ${FOO} substitutions in addition to
        environment variables. If a key appears both in `os.environ`
        and this dictionary, the value in this dictionary is used.
    instantiate : bool, optional
        If `False`, do not actually instantiate the objects but instead
        produce a nested hierarchy of `Proxy` objects.

    Returns
    -------
    graph : dict or object
        The dictionary or object (if the top-level element specified
        a Python object to instantiate), or a nested hierarchy of
        `Proxy` objects.

    Notes
    -----
    Other keyword arguments are passed on to `yaml.load`.
    """
    global is_initialized
    global additional_environ
    if not is_initialized:
        initialize()
    additional_environ = environ

    if isinstance(stream, six.string_types):
        string = stream
    else:
        string = stream.read()

    proxy_graph = yaml.load(string, **kwargs)
    if instantiate:
        return _instantiate(proxy_graph)
    else:
        return proxy_graph


def load_path(path, environ=None, instantiate=True, **kwargs):
    """
    Convenience function for loading a YAML configuration from a file.

    Parameters
    ----------
    path : str
        The path to the file to load on disk.
    environ : dict, optional
        A dictionary used for ${FOO} substitutions in addition to
        environment variables. If a key appears both in `os.environ`
        and this dictionary, the value in this dictionary is used.
    instantiate : bool, optional
        If `False`, do not actually instantiate the objects but instead
        produce a nested hierarchy of `Proxy` objects.

    Returns
    -------
    graph : dict or object
        The dictionary or object (if the top-level element specified
        a Python object to instantiate), or a nested hierarchy of
        `Proxy` objects.

    Notes
    -----
    Other keyword arguments are passed on to `yaml.load`.
    """
    with open(path, 'r') as f:
        content = ''.join(f.readlines())

    # This is apparently here to avoid the odd instance where a file gets
    # loaded as Unicode instead (see 03f238c6d). It's rare instance where
    # basestring is not the right call.
    if not isinstance(content, str):
        raise AssertionError("Expected content to be of type str, got " +
                             str(type(content)))

    return load(content, instantiate=instantiate, environ=environ, **kwargs)


def try_to_import(tag_suffix):
    """
    .. todo::

        WRITEME
    """
    components = tag_suffix.split('.')
    modulename = '.'.join(components[:-1])
    try:
        exec('import %s' % modulename)
    except ImportError as e:
        # We know it's an ImportError, but is it an ImportError related to
        # this path,
        # or did the module we're importing have an unrelated ImportError?
        # and yes, this test can still have false positives, feel free to
        # improve it
        pieces = modulename.split('.')
        str_e = str(e)
        found = True in [piece.find(str(e)) != -1 for piece in pieces]

        if found:
            # The yaml file is probably to blame.
            # Report the problem with the full module path from the YAML
            # file
            reraise_as(ImportError("Could not import %s; ImportError was %s" %
                                   (modulename, str_e)))
        else:

            pcomponents = components[:-1]
            assert len(pcomponents) >= 1
            j = 1
            while j <= len(pcomponents):
                modulename = '.'.join(pcomponents[:j])
                try:
                    exec('import %s' % modulename)
                except Exception:
                    base_msg = 'Could not import %s' % modulename
                    if j > 1:
                        modulename = '.'.join(pcomponents[:j - 1])
                        base_msg += ' but could import %s' % modulename
                    reraise_as(ImportError(base_msg + '. Original exception: '
                                           + str(e)))
                j += 1
    try:
        obj = eval(tag_suffix)
    except AttributeError as e:
        try:
            # Try to figure out what the wrong field name was
            # If we fail to do it, just fall back to giving the usual
            # attribute error
            pieces = tag_suffix.split('.')
            module = '.'.join(pieces[:-1])
            field = pieces[-1]
            candidates = dir(eval(module))

            msg = ('Could not evaluate %s. ' % tag_suffix +
                   'Did you mean ' + match(field, candidates) + '? ' +
                   'Original error was ' + str(e))

        except Exception:
            warnings.warn("Attempt to decipher AttributeError failed")
            reraise_as(AttributeError('Could not evaluate %s. ' % tag_suffix +
                                      'Original error was ' + str(e)))
        reraise_as(AttributeError(msg))
    return obj


def initialize():
    """
    Initialize the configuration system by installing YAML handlers.
    Automatically done on first call to load() specified in this file.
    """
    global is_initialized

    # Add the custom multi-constructor
    yaml.add_multi_constructor('!obj:', multi_constructor_obj)
    yaml.add_multi_constructor('!pkl:', multi_constructor_pkl)
    yaml.add_multi_constructor('!import:', multi_constructor_import)

    yaml.add_constructor('!import', constructor_import)
    yaml.add_constructor("!float", constructor_float)

    pattern = re.compile(SCIENTIFIC_NOTATION_REGEXP)
    yaml.add_implicit_resolver('!float', pattern)

    is_initialized = True


###############################################################################
# Callbacks used by PyYAML


def multi_constructor_obj(loader, tag_suffix, node):
    """
    Callback used by PyYAML when a "!obj:" tag is encountered.

    See PyYAML documentation for details on the call signature.
    """
    yaml_src = yaml.serialize(node)
    construct_mapping(node)
    mapping = loader.construct_mapping(node)

    assert hasattr(mapping, 'keys')
    assert hasattr(mapping, 'values')

    for key in mapping.keys():
        if not isinstance(key, six.string_types):
            message = "Received non string object (%s) as " \
                      "key in mapping." % str(key)
            raise TypeError(message)
    if '.' not in tag_suffix:
        # TODO: I'm not sure how this was ever working without eval().
        callable = eval(tag_suffix)
    else:
        callable = try_to_import(tag_suffix)
    rval = Proxy(callable=callable, yaml_src=yaml_src, positionals=(),
                 keywords=mapping)
    return rval


def multi_constructor_pkl(loader, tag_suffix, node):
    """
    Callback used by PyYAML when a "!pkl:" tag is encountered.
    """
    global additional_environ
    if tag_suffix != "" and tag_suffix != u"":
        raise AssertionError('Expected tag_suffix to be "" but it is "'
                             + tag_suffix +
                             '": Put space between !pkl: and the filename.')

    mapping = loader.construct_yaml_str(node)
    obj = serial.load(preprocess(mapping, additional_environ))
    proxy = Proxy(callable=do_not_recurse, positionals=(),
                  keywords={'value': obj}, yaml_src=yaml.serialize(node))
    return proxy


def multi_constructor_import(loader, tag_suffix, node):
    """
    Callback used by PyYAML when a "!import:" tag is encountered.
    """
    if '.' not in tag_suffix:
        raise yaml.YAMLError("!import: tag suffix contains no '.'")
    return try_to_import(tag_suffix)


def constructor_import(loader, node):
    """
    Callback used by PyYAML when a "!import <str>" tag is encountered.
    This tag exects a (quoted) string as argument.
    """
    value = loader.construct_scalar(node)
    if '.' not in value:
        raise yaml.YAMLError("import tag suffix contains no '.'")
    return try_to_import(value)


def constructor_float(loader, node):
    """
    Callback used by PyYAML when a "!float <str>" tag is encountered.
    This tag exects a (quoted) string as argument.
    """
    value = loader.construct_scalar(node)
    return float(value)


def construct_mapping(node, deep=False):
    # This is a modified version of yaml.BaseConstructor.construct_mapping
    # in which a repeated key raises a ConstructorError
    if not isinstance(node, yaml.nodes.MappingNode):
        const = yaml.constructor
        message = "expected a mapping node, but found"
        raise const.ConstructorError(None, None,
                                     "%s %s " % (message, node.id),
                                     node.start_mark)
    mapping = {}
    constructor = yaml.constructor.BaseConstructor()
    for key_node, value_node in node.value:
        key = constructor.construct_object(key_node, deep=False)
        try:
            hash(key)
        except TypeError as exc:
            const = yaml.constructor
            reraise_as(const.ConstructorError("while constructing a mapping",
                                              node.start_mark,
                                              "found unacceptable key (%s)" %
                                              (exc, key_node.start_mark)))
        if key in mapping:
            const = yaml.constructor
            raise const.ConstructorError("while constructing a mapping",
                                         node.start_mark,
                                         "found duplicate key (%s)" % key)
        value = constructor.construct_object(value_node, deep=False)
        mapping[key] = value
    return mapping


if __name__ == "__main__":
    initialize()
    # Demonstration of how to specify objects, reference them
    # later in the configuration, etc.
    yamlfile = """{
        "corruptor" : !obj:pylearn2.corruption.GaussianCorruptor &corr {
            "corruption_level" : 0.9
        },
        "dae" : !obj:pylearn2.models.autoencoder.DenoisingAutoencoder {
            "nhid" : 20,
            "nvis" : 30,
            "act_enc" : null,
            "act_dec" : null,
            "tied_weights" : true,
            # we could have also just put the corruptor definition here
            "corruptor" : *corr
        }
    }"""
    # yaml.load can take a string or a file object
    loaded = yaml.load(yamlfile)
    logger.info(loaded)
    # These two things should be the same object
    assert loaded['corruptor'] is loaded['dae'].corruptor