Maximum recursion depth crash with pyreverse -S option

[glefebvr]

Steps to reproduce

Given this filen named test.py:

import numpy as np

class ExempleClass :
  typ = np.uint32
  
  def __init__(self,D):
      self.X = np.zeros( D, dtype=self.typ)

I run the command:
Pyreverse -o gv -A -my -S test.py

Current behavior

An error is raised. The error message is very very long. I paste here only the end of the message.

  File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\diadefslib.py", line 121, in extract_classes
    self.extract_classes(node, anc_level, association_level - 1)
  [Previous line repeated 920 more times]
  File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\diadefslib.py", line 115, in extract_classes
    self.add_class(klass_node)
  File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\diadefslib.py", line 85, in add_class
    self.linker.visit(node)
  File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\utils.py", line 217, in visit
    methods[0](node)
  File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\inspector.py", line 170, in visit_classdef
    self.handle_assignattr_type(assignattr, node)
  File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\inspector.py", line 235, in handle_assignattr_type
    values = set(node.infer())
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 132, in raise_if_nothing_inferred
    yield next(generator)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 96, in wrapped
    res = next(generator)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\bases.py", line 136, in _infer_stmts
    for inferred in stmt.infer(context=context):
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\util.py", line 160, in limit_inference
    yield from islice(iterator, size)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\context.py", line 113, in cache_generator
    for result in generator:
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 132, in raise_if_nothing_inferred
    yield next(generator)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 96, in wrapped
    res = next(generator)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\inference.py", line 227, in infer_call
    for callee in self.func.infer(context):
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\node_classes.py", line 357, in infer
    return self._explicit_inference(self, context, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\__init__.py", line 95, in _inference_tip_cached
    result = func(*args, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\brain\brain_numpy_ndarray.py", line 141, in infer_numpy_ndarray
    node = astroid.extract_node(ndarray)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 421, in extract_node
    tree = parse(code, module_name=module_name)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 278, in parse
    return builder.string_build(code, modname=module_name, path=path)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 144, in string_build
    return self._post_build(module, "utf-8")
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 158, in _post_build
    self.delayed_assattr(delayed)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 225, in delayed_assattr
    for inferred in node.expr.infer():
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 132, in raise_if_nothing_inferred
    yield next(generator)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 96, in wrapped
    res = next(generator)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\bases.py", line 136, in _infer_stmts
    for inferred in stmt.infer(context=context):
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\util.py", line 160, in limit_inference
    yield from islice(iterator, size)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\context.py", line 113, in cache_generator
    for result in generator:
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 132, in raise_if_nothing_inferred
    yield next(generator)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 93, in wrapped
    generator = _func(node, context, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\inference.py", line 850, in infer_assign
    stmts = list(self.assigned_stmts(context=context))
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\protocols.py", line 323, in _arguments_infer_argname
    functype = self.parent.type
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 72, in __get__
    val = self.wrapped(inst)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\scoped_nodes.py", line 1461, in type
    for decorator in self.extra_decorators:
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 72, in __get__
    val = self.wrapped(inst)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\scoped_nodes.py", line 1424, in extra_decorators
    for assign in frame._get_assign_nodes():
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 34, in cached
    cache[func] = result = func(*args, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\scoped_nodes.py", line 2927, in _get_assign_nodes
    return list(itertools.chain.from_iterable(children_assign_nodes))
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\scoped_nodes.py", line 2925, in <genexpr>
    child_node._get_assign_nodes() for child_node in self.body
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 34, in cached
    cache[func] = result = func(*args, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\mixins.py", line 153, in _get_assign_nodes
    return list(itertools.chain.from_iterable(children_assign_nodes))
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\mixins.py", line 151, in <genexpr>
    for child_node in block
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 34, in cached
    cache[func] = result = func(*args, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\node_classes.py", line 1953, in _get_assign_nodes
    return [self] + list(self.value._get_assign_nodes())
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 28, in cached
    cache = getattr(instance, "__cache", None)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\node_classes.py", line 2575, in __getattr__
    return super().__getattr__(name)
  File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\bases.py", line 113, in __getattr__
    return getattr(self._proxied, name)
RecursionError: maximum recursion depth exceeded while calling a Python object

Expected behavior

A generation of a file classes.gv

pylint --version output

pylint 2.5.0
astroid 2.4.0
Python 3.7.3 (default, Apr 24 2019, 15:29:51) [MSC v.1915 64 bit (AMD64)]

[Pierre-Sassoulas]

Look like it's the first issue opened for this recursion bug and the following one are:
#3986, #3985, #3982, #3977, #3952, #3913, #3836

A possible temporary solution to the problem was proposed here: #3836 (comment)

turns out this is a genuinely large recursion problem, and can be gotten around by putting the following in your pylintrc file:
init-hook='import sys; sys.setrecursionlimit(3 * sys.getrecursionlimit())'

[Pierre-Sassoulas]

I just checked with pylint 2.7 astroid 2.5, it's not crashing but it's taking really fucking long (more than 10mn and counting) for parsing 5 line of code.

Edit: OOM killed 15mn in. [1] 3619 killed pyreverse -o gv -A -my -S a.py

[semajson]

I have created a .pylintrc file and added init-hook='import sys; sys.setrecursionlimit(3 * sys.getrecursionlimit())', however my pyreverse command still fails at the same point.

In fact, when I set it to init-hook='assert False', my pyreverse command still failed the same way. I suspect that the .pylintrc file isn't being used for pyreverse commands. Perhaps that could be a fix here, allowing pylintrc files to be to be used for pyreverse commands (perhaps by having adding a --rcfile argument to the pyreverse command)?

[DudeNr33]

Could you please share the versions of pylint and astroid you are using?
Ideally with a minimum working example to reproduce the issue. Thank you!

[Pierre-Sassoulas]

On the latest main (2.14.0-b1) with the following a.py file:

import numpy as np

class ExempleClass :
  typ = np.uint32
  
  def __init__(self,D):
      self.X = np.zeros( D, dtype=self.typ)

running pyreverse -o gv -A -my -S a.py ends up in OOM kill.

[DudeNr33]

The offending option in this case is -S ("show recursively all associated off all associated classes").
The problem is with self.X = np.zeros(D, dtype=self.typ).
It is possible to limit the levels of association by using "-s2" (lower case -s, "show <association_level> levels of associated classes not in "). With this you end up with this diagram:

classDiagram
  class ExempleClass {
    X
    typ
  }
  class ndarray {
    T
    base : NoneType
    ctypes : NoneType
    data : NoneType
    dtype : NoneType
    flags : NoneType
    flat
    imag
    itemsize : NoneType
    nbytes : NoneType
    ndim : NoneType
    real
    shape
    size : NoneType
    strides : NoneType
    all(axis, out, keepdims)
    any(axis, out, keepdims)
    argmax(axis, out)
    argmin(axis, out)
    argpartition(kth, axis, kind, order)
    argsort(axis, kind, order)
    astype(dtype, order, casting, subok, copy)
    byteswap(inplace)
    choose(choices, out, mode)
    clip(min, max, out)
    compress(condition, axis, out)
    conj()
    conjugate()
    copy(order)
    cumprod(axis, dtype, out)
    cumsum(axis, dtype, out)
    diagonal(offset, axis1, axis2)
    dot(b, out)
    dump(file)
    dumps()
    fill(value)
    flatten(order)
    getfield(dtype, offset)
    item()
    itemset()
    max(axis, out)
    mean(axis, dtype, out, keepdims)
    min(axis, out, keepdims)
    newbyteorder(new_order)
    nonzero()
    partition(kth, axis, kind, order)
    prod(axis, dtype, out, keepdims)
    ptp(axis, out)
    put(indices, values, mode)
    ravel(order)
    repeat(repeats, axis)
    reshape(shape, order)
    resize(new_shape, refcheck)
    round(decimals, out)
    searchsorted(v, side, sorter)
    setfield(val, dtype, offset)
    setflags(write, align, uic)
    sort(axis, kind, order)
    squeeze(axis)
    std(axis, dtype, out, ddof, keepdims)
    sum(axis, dtype, out, keepdims)
    swapaxes(axis1, axis2)
    take(indices, axis, out, mode)
    tobytes(order)
    tofile(fid, sep, format)
    tolist()
    tostring(order)
    trace(offset, axis1, axis2, dtype, out)
    transpose()
    var(axis, dtype, out, ddof, keepdims)
    view(dtype, type)
  }
  class ndarray {
    T : ndarray
    base : NoneType
    ctypes : NoneType
    data : NoneType
    dtype : NoneType
    flags : NoneType
    flat : ndarray
    imag : ndarray
    itemsize : NoneType
    nbytes : NoneType
    ndim : NoneType
    real : ndarray
    shape : ndarray
    size : NoneType
    strides : NoneType
    all(axis, out, keepdims)
    any(axis, out, keepdims)
    argmax(axis, out)
    argmin(axis, out)
    argpartition(kth, axis, kind, order)
    argsort(axis, kind, order)
    astype(dtype, order, casting, subok, copy)
    byteswap(inplace)
    choose(choices, out, mode)
    clip(min, max, out)
    compress(condition, axis, out)
    conj()
    conjugate()
    copy(order)
    cumprod(axis, dtype, out)
    cumsum(axis, dtype, out)
    diagonal(offset, axis1, axis2)
    dot(b, out)
    dump(file)
    dumps()
    fill(value)
    flatten(order)
    getfield(dtype, offset)
    item()
    itemset()
    max(axis, out)
    mean(axis, dtype, out, keepdims)
    min(axis, out, keepdims)
    newbyteorder(new_order)
    nonzero()
    partition(kth, axis, kind, order)
    prod(axis, dtype, out, keepdims)
    ptp(axis, out)
    put(indices, values, mode)
    ravel(order)
    repeat(repeats, axis)
    reshape(shape, order)
    resize(new_shape, refcheck)
    round(decimals, out)
    searchsorted(v, side, sorter)
    setfield(val, dtype, offset)
    setflags(write, align, uic)
    sort(axis, kind, order)
    squeeze(axis)
    std(axis, dtype, out, ddof, keepdims)
    sum(axis, dtype, out, keepdims)
    swapaxes(axis1, axis2)
    take(indices, axis, out, mode)
    tobytes(order)
    tofile(fid, sep, format)
    tolist()
    tostring(order)
    trace(offset, axis1, axis2, dtype, out)
    transpose()
    var(axis, dtype, out, ddof, keepdims)
    view(dtype, type)
  }
  class ndarray {
    T : ndarray
    base : NoneType
    ctypes : NoneType
    data : NoneType
    dtype : NoneType
    flags : NoneType
    flat : ndarray
    imag : ndarray
    itemsize : NoneType
    nbytes : NoneType
    ndim : NoneType
    real : ndarray
    shape : ndarray
    size : NoneType
    strides : NoneType
    all(axis, out, keepdims)
    any(axis, out, keepdims)
    argmax(axis, out)
    argmin(axis, out)
    argpartition(kth, axis, kind, order)
    argsort(axis, kind, order)
    astype(dtype, order, casting, subok, copy)
    byteswap(inplace)
    choose(choices, out, mode)
    clip(min, max, out)
    compress(condition, axis, out)
    conj()
    conjugate()
    copy(order)
    cumprod(axis, dtype, out)
    cumsum(axis, dtype, out)
    diagonal(offset, axis1, axis2)
    dot(b, out)
    dump(file)
    dumps()
    fill(value)
    flatten(order)
    getfield(dtype, offset)
    item()
    itemset()
    max(axis, out)
    mean(axis, dtype, out, keepdims)
    min(axis, out, keepdims)
    newbyteorder(new_order)
    nonzero()
    partition(kth, axis, kind, order)
    prod(axis, dtype, out, keepdims)
    ptp(axis, out)
    put(indices, values, mode)
    ravel(order)
    repeat(repeats, axis)
    reshape(shape, order)
    resize(new_shape, refcheck)
    round(decimals, out)
    searchsorted(v, side, sorter)
    setfield(val, dtype, offset)
    setflags(write, align, uic)
    sort(axis, kind, order)
    squeeze(axis)
    std(axis, dtype, out, ddof, keepdims)
    sum(axis, dtype, out, keepdims)
    swapaxes(axis1, axis2)
    take(indices, axis, out, mode)
    tobytes(order)
    tofile(fid, sep, format)
    tolist()
    tostring(order)
    trace(offset, axis1, axis2, dtype, out)
    transpose()
    var(axis, dtype, out, ddof, keepdims)
    view(dtype, type)
  }
  class ndarray {
    T : ndarray
    base : NoneType
    ctypes : NoneType
    data : NoneType
    dtype : NoneType
    flags : NoneType
    flat : ndarray
    imag : ndarray
    itemsize : NoneType
    nbytes : NoneType
    ndim : NoneType
    real : ndarray
    shape : ndarray
    size : NoneType
    strides : NoneType
    all(axis, out, keepdims)
    any(axis, out, keepdims)
    argmax(axis, out)
    argmin(axis, out)
    argpartition(kth, axis, kind, order)
    argsort(axis, kind, order)
    astype(dtype, order, casting, subok, copy)
    byteswap(inplace)
    choose(choices, out, mode)
    clip(min, max, out)
    compress(condition, axis, out)
    conj()
    conjugate()
    copy(order)
    cumprod(axis, dtype, out)
    cumsum(axis, dtype, out)
    diagonal(offset, axis1, axis2)
    dot(b, out)
    dump(file)
    dumps()
    fill(value)
    flatten(order)
    getfield(dtype, offset)
    item()
    itemset()
    max(axis, out)
    mean(axis, dtype, out, keepdims)
    min(axis, out, keepdims)
    newbyteorder(new_order)
    nonzero()
    partition(kth, axis, kind, order)
    prod(axis, dtype, out, keepdims)
    ptp(axis, out)
    put(indices, values, mode)
    ravel(order)
    repeat(repeats, axis)
    reshape(shape, order)
    resize(new_shape, refcheck)
    round(decimals, out)
    searchsorted(v, side, sorter)
    setfield(val, dtype, offset)
    setflags(write, align, uic)
    sort(axis, kind, order)
    squeeze(axis)
    std(axis, dtype, out, ddof, keepdims)
    sum(axis, dtype, out, keepdims)
    swapaxes(axis1, axis2)
    take(indices, axis, out, mode)
    tobytes(order)
    tofile(fid, sep, format)
    tolist()
    tostring(order)
    trace(offset, axis1, axis2, dtype, out)
    transpose()
    var(axis, dtype, out, ddof, keepdims)
    view(dtype, type)
  }
  class ndarray {
    T : ndarray
    base : NoneType
    ctypes : NoneType
    data : NoneType
    dtype : NoneType
    flags : NoneType
    flat : ndarray
    imag : ndarray
    itemsize : NoneType
    nbytes : NoneType
    ndim : NoneType
    real : ndarray
    shape : ndarray
    size : NoneType
    strides : NoneType
    all(axis, out, keepdims)
    any(axis, out, keepdims)
    argmax(axis, out)
    argmin(axis, out)
    argpartition(kth, axis, kind, order)
    argsort(axis, kind, order)
    astype(dtype, order, casting, subok, copy)
    byteswap(inplace)
    choose(choices, out, mode)
    clip(min, max, out)
    compress(condition, axis, out)
    conj()
    conjugate()
    copy(order)
    cumprod(axis, dtype, out)
    cumsum(axis, dtype, out)
    diagonal(offset, axis1, axis2)
    dot(b, out)
    dump(file)
    dumps()
    fill(value)
    flatten(order)
    getfield(dtype, offset)
    item()
    itemset()
    max(axis, out)
    mean(axis, dtype, out, keepdims)
    min(axis, out, keepdims)
    newbyteorder(new_order)
    nonzero()
    partition(kth, axis, kind, order)
    prod(axis, dtype, out, keepdims)
    ptp(axis, out)
    put(indices, values, mode)
    ravel(order)
    repeat(repeats, axis)
    reshape(shape, order)
    resize(new_shape, refcheck)
    round(decimals, out)
    searchsorted(v, side, sorter)
    setfield(val, dtype, offset)
    setflags(write, align, uic)
    sort(axis, kind, order)
    squeeze(axis)
    std(axis, dtype, out, ddof, keepdims)
    sum(axis, dtype, out, keepdims)
    swapaxes(axis1, axis2)
    take(indices, axis, out, mode)
    tobytes(order)
    tofile(fid, sep, format)
    tolist()
    tostring(order)
    trace(offset, axis1, axis2, dtype, out)
    transpose()
    var(axis, dtype, out, ddof, keepdims)
    view(dtype, type)
  }
  class ndarray {
    T : ndarray
    base : NoneType
    ctypes : NoneType
    data : NoneType
    dtype : NoneType
    flags : NoneType
    flat : ndarray
    imag : ndarray
    itemsize : NoneType
    nbytes : NoneType
    ndim : NoneType
    real : ndarray
    shape : ndarray
    size : NoneType
    strides : NoneType
    all(axis, out, keepdims)
    any(axis, out, keepdims)
    argmax(axis, out)
    argmin(axis, out)
    argpartition(kth, axis, kind, order)
    argsort(axis, kind, order)
    astype(dtype, order, casting, subok, copy)
    byteswap(inplace)
    choose(choices, out, mode)
    clip(min, max, out)
    compress(condition, axis, out)
    conj()
    conjugate()
    copy(order)
    cumprod(axis, dtype, out)
    cumsum(axis, dtype, out)
    diagonal(offset, axis1, axis2)
    dot(b, out)
    dump(file)
    dumps()
    fill(value)
    flatten(order)
    getfield(dtype, offset)
    item()
    itemset()
    max(axis, out)
    mean(axis, dtype, out, keepdims)
    min(axis, out, keepdims)
    newbyteorder(new_order)
    nonzero()
    partition(kth, axis, kind, order)
    prod(axis, dtype, out, keepdims)
    ptp(axis, out)
    put(indices, values, mode)
    ravel(order)
    repeat(repeats, axis)
    reshape(shape, order)
    resize(new_shape, refcheck)
    round(decimals, out)
    searchsorted(v, side, sorter)
    setfield(val, dtype, offset)
    setflags(write, align, uic)
    sort(axis, kind, order)
    squeeze(axis)
    std(axis, dtype, out, ddof, keepdims)
    sum(axis, dtype, out, keepdims)
    swapaxes(axis1, axis2)
    take(indices, axis, out, mode)
    tobytes(order)
    tofile(fid, sep, format)
    tolist()
    tostring(order)
    trace(offset, axis1, axis2, dtype, out)
    transpose()
    var(axis, dtype, out, ddof, keepdims)
    view(dtype, type)
  }
  class uint32 {
  }
  ndarray --* ndarray : T
  ndarray --* ndarray : flat
  ndarray --* ndarray : imag
  ndarray --* ndarray : real
  ndarray --* ndarray : shape
  ndarray --* ExempleClass : X
  uint32 --* ExempleClass : typ

[Pierre-Sassoulas]

Sounds like the issue could be closed if we put a lower default value when nothing is set ? It's easy to shoot oneself in the foot right now.

[DudeNr33]

Well, -S explicitly says recursively all associated classes.
It is not turned on by default (*), so I don't see a need to change this behaviour (the recursion error itself should be fixed of course).

(*) this is not entirely true, as using the --class option to generate a diagram of a specific class sets -ASmy as default.
But for normal invocations, i.e. pyreverse a.py, no associated classes outside the own project will be shown.