Error: date dataframe columns fail to serialize

[antonioalegria]

Describe the bug
Running code in a sandbox:

import pandas as pd
from datetime import datetime, date

# Create sample data
data = {
    'datetime_col': [
        pd.Timestamp('2024-01-29 14:30:00'),
        pd.Timestamp('2024-01-28 09:15:30'),
        pd.Timestamp('2024-01-27 18:45:15')
    ],
    'date_col': [
        date(2024, 1, 29),
        date(2024, 1, 28),
        date(2024, 1, 27)
    ],
    'timestamp_col': [
        pd.Timestamp('2024-01-29 14:30:00').timestamp(),
        pd.Timestamp('2024-01-28 09:15:30').timestamp(),
        pd.Timestamp('2024-01-27 18:45:15').timestamp()
    ]
}

df = pd.DataFrame(data)
display(df)

Yields this error: ValueError: Can't clean for JSON: datetime.date(2024, 1, 29)

---------------------------------------------------------------------------TypeError                                 Traceback (most recent call last)File /usr/local/lib/python3.10/site-packages/jupyter_client/session.py:95, in json_packer(obj)
     94 try:
---> 95     return json.dumps(
     96         obj,
     97         default=json_default,
     98         ensure_ascii=False,
     99         allow_nan=False,
    100     ).encode("utf8", errors="surrogateescape")
    101 except (TypeError, ValueError) as e:
    102     # Fallback to trying to clean the json before serializing
File /usr/local/lib/python3.10/json/__init__.py:238, in dumps(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)
    233     cls = JSONEncoder
    234 return cls(
    235     skipkeys=skipkeys, ensure_ascii=ensure_ascii,
    236     check_circular=check_circular, allow_nan=allow_nan, indent=indent,
    237     separators=separators, default=default, sort_keys=sort_keys,
--> 238     **kw).encode(obj)
File /usr/local/lib/python3.10/json/encoder.py:199, in JSONEncoder.encode(self, o)
    196 # This doesn't pass the iterator directly to ''.join() because the
    197 # exceptions aren't as detailed.  The list call should be roughly
    198 # equivalent to the PySequence_Fast that ''.join() would do.
--> 199 chunks = self.iterencode(o, _one_shot=True)
    200 if not isinstance(chunks, (list, tuple)):
File /usr/local/lib/python3.10/json/encoder.py:257, in JSONEncoder.iterencode(self, o, _one_shot)
    253     _iterencode = _make_iterencode(
    254         markers, self.default, _encoder, self.indent, floatstr,
    255         self.key_separator, self.item_separator, self.sort_keys,
    256         self.skipkeys, _one_shot)
--> 257 return _iterencode(o, 0)
File /usr/local/lib/python3.10/site-packages/jupyter_client/jsonutil.py:125, in json_default(obj)
    123     return float(obj)
--> 125 raise TypeError("%r is not JSON serializable" % obj)
TypeError: datetime.date(2024, 1, 29) is not JSON serializable
During handling of the above exception, another exception occurred:
ValueError                                Traceback (most recent call last)Cell In[2], line 25
      6 data = {
      7     'datetime_col': [
      8         pd.Timestamp('2024-01-29 14:30:00'),
   (...)
     21     ]
     22 }
     24 df = pd.DataFrame(data)
---> 25 display(df)
File /usr/local/lib/python3.10/site-packages/IPython/core/display_functions.py:305, in display(include, exclude, metadata, transient, display_id, raw, clear, *objs, **kwargs)
    302         if metadata:
    303             # kwarg-specified metadata gets precedence
    304             _merge(md_dict, metadata)
--> 305         publish_display_data(data=format_dict, metadata=md_dict, **kwargs)
    306 if display_id:
    307     return DisplayHandle(display_id)
File /usr/local/lib/python3.10/site-packages/IPython/core/display_functions.py:93, in publish_display_data(data, metadata, source, transient, **kwargs)
     90 if transient:
     91     kwargs['transient'] = transient
---> 93 display_pub.publish(
     94     data=data,
     95     metadata=metadata,
     96     **kwargs
     97 )
File /usr/local/lib/python3.10/site-packages/ipykernel/zmqshell.py:130, in ZMQDisplayPublisher.publish(self, data, metadata, transient, update)
    127     if msg is None:
    128         return  # type:ignore[unreachable]
--> 130 self.session.send(
    131     self.pub_socket,
    132     msg,
    133     ident=self.topic,
    134 )
File /usr/local/lib/python3.10/site-packages/jupyter_client/session.py:852, in Session.send(self, stream, msg_or_type, content, parent, ident, buffers, track, header, metadata)
    850 if self.adapt_version:
    851     msg = adapt(msg, self.adapt_version)
--> 852 to_send = self.serialize(msg, ident)
    853 to_send.extend(buffers)
    854 longest = max([len(s) for s in to_send])
File /usr/local/lib/python3.10/site-packages/jupyter_client/session.py:721, in Session.serialize(self, msg, ident)
    719     content = self.none
    720 elif isinstance(content, dict):
--> 721     content = self.pack(content)
    722 elif isinstance(content, bytes):
    723     # content is already packed, as in a relayed message
    724     pass
File /usr/local/lib/python3.10/site-packages/jupyter_client/session.py:104, in json_packer(obj)
     95     return json.dumps(
     96         obj,
     97         default=json_default,
     98         ensure_ascii=False,
     99         allow_nan=False,
    100     ).encode("utf8", errors="surrogateescape")
    101 except (TypeError, ValueError) as e:
    102     # Fallback to trying to clean the json before serializing
    103     packed = json.dumps(
--> 104         json_clean(obj),
    105         default=json_default,
    106         ensure_ascii=False,
    107         allow_nan=False,
    108     ).encode("utf8", errors="surrogateescape")
    110     warnings.warn(
    111         f"Message serialization failed with:\n{e}\n"
    112         "Supporting this message is deprecated in jupyter-client 7, please make "
    113         "sure your message is JSON-compliant",
    114         stacklevel=2,
    115     )
    117     return packed
File /usr/local/lib/python3.10/site-packages/jupyter_client/jsonutil.py:185, in json_clean(obj)
    183     out = {}
    184     for k, v in obj.items():
--> 185         out[str(k)] = json_clean(v)
    186     return out
    188 if isinstance(obj, datetime):
File /usr/local/lib/python3.10/site-packages/jupyter_client/jsonutil.py:185, in json_clean(obj)
    183     out = {}
    184     for k, v in obj.items():
--> 185         out[str(k)] = json_clean(v)
    186     return out
    188 if isinstance(obj, datetime):
File /usr/local/lib/python3.10/site-packages/jupyter_client/jsonutil.py:185, in json_clean(obj)
    183     out = {}
    184     for k, v in obj.items():
--> 185         out[str(k)] = json_clean(v)
    186     return out
    188 if isinstance(obj, datetime):
File /usr/local/lib/python3.10/site-packages/jupyter_client/jsonutil.py:168, in json_clean(obj)
    165     obj = list(obj)
    167 if isinstance(obj, list):
--> 168     return [json_clean(x) for x in obj]
    170 if isinstance(obj, dict):
    171     # First, validate that the dict won't lose data in conversion due to
    172     # key collisions after stringification.  This can happen with keys like
    173     # True and 'true' or 1 and '1', which collide in JSON.
    174     nkeys = len(obj)
File /usr/local/lib/python3.10/site-packages/jupyter_client/jsonutil.py:168, in <listcomp>(.0)
    165     obj = list(obj)
    167 if isinstance(obj, list):
--> 168     return [json_clean(x) for x in obj]
    170 if isinstance(obj, dict):
    171     # First, validate that the dict won't lose data in conversion due to
    172     # key collisions after stringification.  This can happen with keys like
    173     # True and 'true' or 1 and '1', which collide in JSON.
    174     nkeys = len(obj)
File /usr/local/lib/python3.10/site-packages/jupyter_client/jsonutil.py:192, in json_clean(obj)
    189     return obj.strftime(ISO8601)
    191 # we don't understand it, it's probably an unserializable object
--> 192 raise ValueError("Can't clean for JSON: %r" % obj)
ValueError: Can't clean for JSON: datetime.date(2024, 1, 29)"

[linear]

E2B-1505 date dataframe columns fail to serialize