HTML representation of nwbfile results in RecursionError

[weiglszonja]

@rly I tried out the fix in #1006 to link a device (which is not named "device") in OptogeneticStimulusSite, the read/write is working fine but I get a weird error when trying to show the contents of such a file in a Jupyter notebook (posted here)

Is there something I missed?

The full traceback:

---------------------------------------------------------------------------
RecursionError                            Traceback (most recent call last)
File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/IPython/core/formatters.py:344, in BaseFormatter.__call__(self, obj)
    342     method = get_real_method(obj, self.print_method)
    343     if method is not None:
--> 344         return method()
    345     return None
    346 else:

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:617, in Container._repr_html_(self)
    615 html_repr += "<div class='container-wrap'>"
    616 html_repr += f"<div class='container-header'><div class='xr-obj-type'><h3>{header_text}</h3></div></div>"
--> 617 html_repr += self._generate_html_repr(self.fields, is_field=True)
    618 html_repr += "</div>"
    619 return html_repr

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:628, in Container._generate_html_repr(self, fields, level, access_code, is_field)
    626     for key, value in fields.items():
    627         current_access_code = f"{access_code}.{key}" if is_field else f"{access_code}['{key}']"
--> 628         html_repr += self._generate_field_html(key, value, level, current_access_code)
    629 elif isinstance(fields, list):
    630     for index, item in enumerate(fields):

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:656, in Container._generate_field_html(self, key, value, level, access_code)
    654     html_content = self._generate_html_repr(value.fields, level + 1, access_code, is_field=True)
    655 elif isinstance(value, (list, dict, np.ndarray)):
--> 656     html_content = self._generate_html_repr(value, level + 1, access_code, is_field=False)
    657 else:
    658     html_content = f'<span class="field-key">{value}</span>'

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:628, in Container._generate_html_repr(self, fields, level, access_code, is_field)
    626     for key, value in fields.items():
    627         current_access_code = f"{access_code}.{key}" if is_field else f"{access_code}['{key}']"
--> 628         html_repr += self._generate_field_html(key, value, level, current_access_code)
    629 elif isinstance(fields, list):
    630     for index, item in enumerate(fields):

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:654, in Container._generate_field_html(self, key, value, level, access_code)
    651     html_content = value.__repr_html__()
    653 elif hasattr(value, "fields"):
--> 654     html_content = self._generate_html_repr(value.fields, level + 1, access_code, is_field=True)
    655 elif isinstance(value, (list, dict, np.ndarray)):
    656     html_content = self._generate_html_repr(value, level + 1, access_code, is_field=False)

    [... skipping similar frames: Container._generate_html_repr at line 628 (1 times)]

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:656, in Container._generate_field_html(self, key, value, level, access_code)
    654     html_content = self._generate_html_repr(value.fields, level + 1, access_code, is_field=True)
    655 elif isinstance(value, (list, dict, np.ndarray)):
--> 656     html_content = self._generate_html_repr(value, level + 1, access_code, is_field=False)
    657 else:
    658     html_content = f'<span class="field-key">{value}</span>'

    [... skipping similar frames: Container._generate_html_repr at line 628 (1 times)]

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:654, in Container._generate_field_html(self, key, value, level, access_code)
    651     html_content = value.__repr_html__()
    653 elif hasattr(value, "fields"):
--> 654     html_content = self._generate_html_repr(value.fields, level + 1, access_code, is_field=True)
    655 elif isinstance(value, (list, dict, np.ndarray)):
    656     html_content = self._generate_html_repr(value, level + 1, access_code, is_field=False)

    [... skipping similar frames: Container._generate_html_repr at line 628 (3 times), Container._generate_field_html at line 656 (2 times), Container._generate_field_html at line 654 (1 times)]

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:632, in Container._generate_html_repr(self, fields, level, access_code, is_field)
    630     for index, item in enumerate(fields):
    631         access_code += f'[{index}]'
--> 632         html_repr += self._generate_field_html(index, item, level, access_code)
    633 elif isinstance(fields, np.ndarray):
    634     html_repr += self._generate_array_html(fields, level)

    [... skipping similar frames: Container._generate_field_html at line 654 (2 times), Container._generate_html_repr at line 628 (2 times), Container._generate_field_html at line 656 (1 times)]

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:632, in Container._generate_html_repr(self, fields, level, access_code, is_field)
    630     for index, item in enumerate(fields):
    631         access_code += f'[{index}]'
--> 632         html_repr += self._generate_field_html(index, item, level, access_code)
    633 elif isinstance(fields, np.ndarray):
    634     html_repr += self._generate_array_html(fields, level)

    [... skipping similar frames: Container._generate_field_html at line 654 (980 times), Container._generate_html_repr at line 628 (980 times), Container._generate_field_html at line 656 (489 times), Container._generate_html_repr at line 632 (489 times)]

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:656, in Container._generate_field_html(self, key, value, level, access_code)
    654     html_content = self._generate_html_repr(value.fields, level + 1, access_code, is_field=True)
    655 elif isinstance(value, (list, dict, np.ndarray)):
--> 656     html_content = self._generate_html_repr(value, level + 1, access_code, is_field=False)
    657 else:
    658     html_content = f'<span class="field-key">{value}</span>'

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:632, in Container._generate_html_repr(self, fields, level, access_code, is_field)
    630     for index, item in enumerate(fields):
    631         access_code += f'[{index}]'
--> 632         html_repr += self._generate_field_html(index, item, level, access_code)
    633 elif isinstance(fields, np.ndarray):
    634     html_repr += self._generate_array_html(fields, level)

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:654, in Container._generate_field_html(self, key, value, level, access_code)
    651     html_content = value.__repr_html__()
    653 elif hasattr(value, "fields"):
--> 654     html_content = self._generate_html_repr(value.fields, level + 1, access_code, is_field=True)
    655 elif isinstance(value, (list, dict, np.ndarray)):
    656     html_content = self._generate_html_repr(value, level + 1, access_code, is_field=False)

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:628, in Container._generate_html_repr(self, fields, level, access_code, is_field)
    626     for key, value in fields.items():
    627         current_access_code = f"{access_code}.{key}" if is_field else f"{access_code}['{key}']"
--> 628         html_repr += self._generate_field_html(key, value, level, current_access_code)
    629 elif isinstance(fields, list):
    630     for index, item in enumerate(fields):

File ~/anaconda3/envs/pinto311ndx/lib/python3.11/site-packages/hdmf/container.py:643, in Container._generate_field_html(self, key, value, level, access_code)
    640 def _generate_field_html(self, key, value, level, access_code):
    641     """Generates HTML for a single field."""
--> 643     if isinstance(value, (int, float, str, bool)):
    644         return f'<div style="margin-left: {level * 20}px;" class="container-fields"><span class="field-key"' \
    645                f' title="{access_code}">{key}: </span><span class="field-value">{value}</span></div>'
    647     if hasattr(value, "generate_html_repr"):

RecursionError: maximum recursion depth exceeded in __instancecheck__

[rly]

Do you have a cyclic structure in your file?

[weiglszonja]

@rly Sorry, it's not because of linked Device, I managed to reproduce the error by linking timestamps across multiple SpatialSeries objects (which is what I also have in this dataset)

You should be able to reproduce the above error if you run this from a Jupyter notebook:

from pynwb import NWBHDF5IO
from pynwb.testing.mock.file import mock_NWBFile
from pynwb.testing.mock.behavior import mock_SpatialSeries, mock_Position
import numpy as np

nwbfile = mock_NWBFile()

test_timestamps = np.zeros((1000,))
test_data = np.random.rand(*(1000, 3))
spatial_series1 = mock_SpatialSeries(name="test1", rate=None, data=test_data, timestamps=test_timestamps)
spatial_series2 = mock_SpatialSeries(name="test2", rate=None, data=test_data, timestamps=spatial_series1)
spatial_series3 = mock_SpatialSeries(name="test3", rate=None, data=test_data, timestamps=spatial_series1)
spatial_series4 = mock_SpatialSeries(name="test4", rate=None, data=test_data, timestamps=spatial_series1)
position = mock_Position(spatial_series=[spatial_series1, spatial_series2, spatial_series3, spatial_series4])

nwbfile.create_processing_module("behavior", description="contains processed behavior data")
nwbfile.processing["behavior"].add(position)

with NWBHDF5IO("test.nwb", "w") as io:
    io.write(nwbfile)

read_io = NWBHDF5IO("test.nwb", "r")
nwbfile_in = read_io.read()

nwbfile_in

In the real dataset, it looks like this:

SpatialSeries pynwb.behavior.SpatialSeries at 0x6297437200
 Fields:
   comments: no comments
   conversion: 0.01
   data: <HDF5 dataset "data": shape (77524, 3), type "<f8">
   description: The x, y, z position of the animal by ViRMEN iteration.
   interval: 1
   offset: 0.0
   reference_frame: (0,0) is the start of the 'sample' region (or 'cue' region) which varies by maze and task.
   resolution: -1.0
   timestamp_link: (
     SensorDots <class 'pynwb.base.TimeSeries'>,
     Velocity <class 'pynwb.base.TimeSeries'>,
     VelocityGain <class 'pynwb.base.TimeSeries'>,
     VelocityViewAngle <class 'pynwb.behavior.SpatialSeries'>
   )
   timestamps: <HDF5 dataset "timestamps": shape (77524,), type "<f8">
   timestamps_unit: seconds
   unit: meters,