RecursionError when run pyWGCNA_ct.preprocess()

[yizhang-zoey]

There is a RecursionError when I run pyWGCNA_ct.preprocess():

RecursionError: maximum recursion depth exceeded while calling a Python object

The detailed error message:

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/PyWGCNA/wgcna.py:236, in WGCNA.preprocess(self)
    233 sampleTree = WGCNA.hclust(pdist(self.datExpr.to_df()), method="average")
    235 plt.figure(figsize=(max(25, round(self.datExpr.X.shape[0] / 20)), 10), facecolor='white')
--> 236 dendrogram(sampleTree, color_threshold=self.cut, labels=self.datExpr.to_df().index, leaf_rotation=90,
    237            leaf_font_size=8)
    238 plt.axhline(y=self.cut, c='grey', lw=1, linestyle='dashed')
    239 plt.title('Sample clustering to detect outliers')

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/scipy/cluster/hierarchy.py:3352, in dendrogram(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, no_plot, no_labels, leaf_font_size, leaf_rotation, leaf_label_func, show_contracted, link_color_func, ax, above_threshold_color)
   3349 # Empty list will be filled in _dendrogram_calculate_info
   3350 contraction_marks = [] if show_contracted else None
-> 3352 _dendrogram_calculate_info(
   3353     Z=Z, p=p,
   3354     truncate_mode=truncate_mode,
   3355     color_threshold=color_threshold,
   3356     get_leaves=get_leaves,
   3357     orientation=orientation,
   3358     labels=labels,
   3359     count_sort=count_sort,
   3360     distance_sort=distance_sort,
   3361     show_leaf_counts=show_leaf_counts,
   3362     i=2*n - 2,
   3363     iv=0.0,
   3364     ivl=ivl,
   3365     n=n,
   3366     icoord_list=icoord_list,
   3367     dcoord_list=dcoord_list,
   3368     lvs=lvs,
   3369     current_color=current_color,
   3370     color_list=color_list,
   3371     currently_below_threshold=currently_below_threshold,
   3372     leaf_label_func=leaf_label_func,
   3373     contraction_marks=contraction_marks,
   3374     link_color_func=link_color_func,
   3375     above_threshold_color=above_threshold_color)
   3377 if not no_plot:
   3378     mh = max(Z[:, 2])

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/scipy/cluster/hierarchy.py:3657, in _dendrogram_calculate_info(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)
   3653     currently_below_threshold[0] = True
   3654     c = _link_line_colors[current_color[0]]
   3656 (uivb, uwb, ubh, ubmd) = \
-> 3657     _dendrogram_calculate_info(
   3658         Z=Z, p=p,
   3659         truncate_mode=truncate_mode,
   3660         color_threshold=color_threshold,
   3661         get_leaves=get_leaves,
   3662         orientation=orientation,
   3663         labels=labels,
   3664         count_sort=count_sort,
   3665         distance_sort=distance_sort,
   3666         show_leaf_counts=show_leaf_counts,
   3667         i=ub, iv=iv + uwa, ivl=ivl, n=n,
   3668         icoord_list=icoord_list,
   3669         dcoord_list=dcoord_list, lvs=lvs,
   3670         current_color=current_color,
   3671         color_list=color_list,
   3672         currently_below_threshold=currently_below_threshold,
   3673         leaf_label_func=leaf_label_func,
   3674         level=level + 1, contraction_marks=contraction_marks,
   3675         link_color_func=link_color_func,
   3676         above_threshold_color=above_threshold_color)
   3678 max_dist = max(uamd, ubmd, h)
   3680 icoord_list.append([uiva, uiva, uivb, uivb])

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/scipy/cluster/hierarchy.py:3657, in _dendrogram_calculate_info(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)
   3653     currently_below_threshold[0] = True
   3654     c = _link_line_colors[current_color[0]]
   3656 (uivb, uwb, ubh, ubmd) = \
-> 3657     _dendrogram_calculate_info(
   3658         Z=Z, p=p,
   3659         truncate_mode=truncate_mode,
   3660         color_threshold=color_threshold,
   3661         get_leaves=get_leaves,
   3662         orientation=orientation,
   3663         labels=labels,
   3664         count_sort=count_sort,
   3665         distance_sort=distance_sort,
   3666         show_leaf_counts=show_leaf_counts,
   3667         i=ub, iv=iv + uwa, ivl=ivl, n=n,
   3668         icoord_list=icoord_list,
   3669         dcoord_list=dcoord_list, lvs=lvs,
   3670         current_color=current_color,
   3671         color_list=color_list,
   3672         currently_below_threshold=currently_below_threshold,
   3673         leaf_label_func=leaf_label_func,
   3674         level=level + 1, contraction_marks=contraction_marks,
   3675         link_color_func=link_color_func,
   3676         above_threshold_color=above_threshold_color)
   3678 max_dist = max(uamd, ubmd, h)
   3680 icoord_list.append([uiva, uiva, uivb, uivb])

    [... skipping similar frames: _dendrogram_calculate_info at line 3657 (2963 times)]

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/scipy/cluster/hierarchy.py:3657, in _dendrogram_calculate_info(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)
   3653     currently_below_threshold[0] = True
   3654     c = _link_line_colors[current_color[0]]
   3656 (uivb, uwb, ubh, ubmd) = \
-> 3657     _dendrogram_calculate_info(
   3658         Z=Z, p=p,
   3659         truncate_mode=truncate_mode,
   3660         color_threshold=color_threshold,
   3661         get_leaves=get_leaves,
   3662         orientation=orientation,
   3663         labels=labels,
   3664         count_sort=count_sort,
   3665         distance_sort=distance_sort,
   3666         show_leaf_counts=show_leaf_counts,
   3667         i=ub, iv=iv + uwa, ivl=ivl, n=n,
   3668         icoord_list=icoord_list,
   3669         dcoord_list=dcoord_list, lvs=lvs,
   3670         current_color=current_color,
   3671         color_list=color_list,
   3672         currently_below_threshold=currently_below_threshold,
   3673         leaf_label_func=leaf_label_func,
   3674         level=level + 1, contraction_marks=contraction_marks,
   3675         link_color_func=link_color_func,
   3676         above_threshold_color=above_threshold_color)
   3678 max_dist = max(uamd, ubmd, h)
   3680 icoord_list.append([uiva, uiva, uivb, uivb])

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/scipy/cluster/hierarchy.py:3624, in _dendrogram_calculate_info(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)
   3620     ub = ab
   3622 # Updated iv variable and the amount of space used.
   3623 (uiva, uwa, uah, uamd) = \
-> 3624     _dendrogram_calculate_info(
   3625         Z=Z, p=p,
   3626         truncate_mode=truncate_mode,
   3627         color_threshold=color_threshold,
   3628         get_leaves=get_leaves,
   3629         orientation=orientation,
   3630         labels=labels,
   3631         count_sort=count_sort,
   3632         distance_sort=distance_sort,
   3633         show_leaf_counts=show_leaf_counts,
   3634         i=ua, iv=iv, ivl=ivl, n=n,
   3635         icoord_list=icoord_list,
   3636         dcoord_list=dcoord_list, lvs=lvs,
   3637         current_color=current_color,
   3638         color_list=color_list,
   3639         currently_below_threshold=currently_below_threshold,
   3640         leaf_label_func=leaf_label_func,
   3641         level=level + 1, contraction_marks=contraction_marks,
   3642         link_color_func=link_color_func,
   3643         above_threshold_color=above_threshold_color)
   3645 h = Z[i - n, 2]
   3646 if h >= color_threshold or color_threshold <= 0:

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/scipy/cluster/hierarchy.py:3624, in _dendrogram_calculate_info(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)
   3620     ub = ab
   3622 # Updated iv variable and the amount of space used.
   3623 (uiva, uwa, uah, uamd) = \
-> 3624     _dendrogram_calculate_info(
   3625         Z=Z, p=p,
   3626         truncate_mode=truncate_mode,
   3627         color_threshold=color_threshold,
   3628         get_leaves=get_leaves,
   3629         orientation=orientation,
   3630         labels=labels,
   3631         count_sort=count_sort,
   3632         distance_sort=distance_sort,
   3633         show_leaf_counts=show_leaf_counts,
   3634         i=ua, iv=iv, ivl=ivl, n=n,
   3635         icoord_list=icoord_list,
   3636         dcoord_list=dcoord_list, lvs=lvs,
   3637         current_color=current_color,
   3638         color_list=color_list,
   3639         currently_below_threshold=currently_below_threshold,
   3640         leaf_label_func=leaf_label_func,
   3641         level=level + 1, contraction_marks=contraction_marks,
   3642         link_color_func=link_color_func,
   3643         above_threshold_color=above_threshold_color)
   3645 h = Z[i - n, 2]
   3646 if h >= color_threshold or color_threshold <= 0:

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/scipy/cluster/hierarchy.py:3554, in _dendrogram_calculate_info(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)
   3550 # Otherwise, only truncate if we have a leaf node.
   3551 #
   3552 # Only place leaves if they correspond to original observations.
   3553 if i < n:
-> 3554     _append_singleton_leaf_node(Z, p, n, level, lvs, ivl,
   3555                                 leaf_label_func, i, labels)
   3556     return (iv + 5.0, 10.0, 0.0, 0.0)
   3558 # !!! Otherwise, we don't have a leaf node, so work on plotting a
   3559 # non-leaf node.
   3560 # Actual indices of a and b

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/scipy/cluster/hierarchy.py:3429, in _append_singleton_leaf_node(Z, p, n, level, lvs, ivl, leaf_label_func, i, labels)
   3425 else:
   3426     # Otherwise, if the dendrogram caller has passed a labels list
   3427     # for the leaf nodes, use it.
   3428     if labels is not None:
-> 3429         ivl.append(labels[int(i - n)])
   3430     else:
   3431         # Otherwise, use the id as the label for the leaf.x
   3432         ivl.append(str(int(i)))

File ~/app/miniconda3/envs/sc/lib/python3.9/site-packages/pandas/core/indexes/base.py:5363, in Index.__getitem__(self, key)
   5351 """
   5352 Override numpy.ndarray's __getitem__ method to work as desired.
   5353 
   (...)
   5359 
   5360 """
   5361 getitem = self._data.__getitem__
-> 5363 if is_integer(key) or is_float(key):
   5364     # GH#44051 exclude bool, which would return a 2d ndarray
   5365     key = com.cast_scalar_indexer(key)
   5366     return getitem(key)

RecursionError: maximum recursion depth exceeded while calling a Python object

<Figure size 28300x1000 with 0 Axes>

[nargesr]

Hi @yizhang-zoey ,

I think the error you got is related to the previous issue you opened a few days ago (issue #84). I hope suppressing the drawing will fix your issue.

Unfortunately, this is more like a scipy.cluster.hierarchy.dendrogram limitation that can not plot a huge dendrogram and right now I don't have any solution beyond not plotting the dendrogram