-
Notifications
You must be signed in to change notification settings - Fork 0
/
data_utils.py
187 lines (151 loc) · 6.16 KB
/
data_utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/utility/data_utils.ipynb.
# %% auto 0
__all__ = ['f_data_class_to_dict', 'f_to_dict', 'f_get_hash', 'f_from_dict', 'f_data_class_from_dict', 'f_dict_to_obj']
# %% ../../nbs/utility/data_utils.ipynb 1
import json
import hashlib
import dataclasses
import datetime
import numpy as np
from typing import Dict, Any, Type, T
import importlib
# %% ../../nbs/utility/data_utils.ipynb 3
def f_data_class_to_dict(thing: object) -> Dict[str, Any]:
"""
Convert a dataclass object to a dictionary.
Parameters:
- thing: the object to convert to a dictionary
Returns:
- a dictionary representation of the object
"""
# Extract the fields of the object
fields = dataclasses.fields(thing)
if isinstance(thing, type):
# Raise an error if the function is called on a class, rather than an instance
raise TypeError(f'got {type}, expected instance')
# Initialize an empty dictionary to store the field names and values
return_dict = {}
# Check for the presence of the _hash_exclude_ attribute
exclude = getattr(thing, '_hash_exclude_', ())
# Add the fields to the dictionary, skipping any that are in the exclude list
for field in fields:
if field.name in exclude:
continue
return dataclasses.asdict(thing)
# %% ../../nbs/utility/data_utils.ipynb 4
def f_to_dict(item):
"""
Convert the given item to a dictionary.
Parameters:
- item: the object to convert to a dictionary
Returns:
- a dictionary representation of the object
"""
# Try to convert the item to a dictionary using the f_data_class_to_dict function
try:
return f_data_class_to_dict(item)
except TypeError:
pass
# Handle values based on their type
if isinstance(item, bytes):
# Convert bytes to a string
return item.decode("utf-8").strip()
elif isinstance(item, (int, float, str, bool, dict, list, tuple, type(None))):
# These types are JSON serializable, so return them as-is
return item
elif isinstance(item, np.ndarray):
# Convert NumPy arrays to a list of lists
return item.tolist()
elif isinstance(item, datetime.datetime):
# Convert datetime objects to a string representation
return item.isoformat()
else:
# Raise a TypeError if the object is not JSON serializable
raise TypeError(f"Object of type {type(item).__name__} is not JSON serializable")
# %% ../../nbs/utility/data_utils.ipynb 5
def f_get_hash(item: object, prefix: str = 'h1') -> bytes:
"""
Generate a hash of the given item.
Parameters:
- item: the object to generate a hash for
- prefix: a prefix to include in the generated hash (default 'h1_')
Returns:
- a bytes object containing the generated hash
originally inspired by https://death.andgravity.com/stable-hashing
"""
# Convert the item to a JSON string
item_as_json = json.dumps(item, default=f_to_dict, ensure_ascii=False,
sort_keys=True, indent=None, separators=(',', ':'))
# Generate a hash of the JSON string
item_as_hash = hashlib.md5(item_as_json.encode('utf-8')).hexdigest()
# Return the prefix and hash as a bytes object
return f'{prefix}_{item_as_hash}'
# %% ../../nbs/utility/data_utils.ipynb 6
def f_from_dict(item: Dict[str, Any]) -> object:
"""
Convert a dictionary to an object.
Parameters:
- item: the dictionary to convert to an object
Returns:
- an object representation of the dictionary
"""
# Check if the dictionary contains a special key that indicates it should be converted to a specific type
if '__type__' in item:
# Extract the type information
type_name = item.pop('__type__')
module_name = item.pop('__module__')
# Retrieve the type from the specified module
module = importlib.import_module(module_name)
type_ = getattr(module, type_name)
# Check if the type is a dataclass
if dataclasses.is_dataclass(type_):
# Convert the dictionary to a dataclass object
return f_data_class_from_dict(type_, item)
else:
# Convert the dictionary to the specified type using the from_dict method
return type_.from_dict(item)
elif '__ndarray__' in item:
# Convert a list of lists back to a NumPy array
return np.array(item['__ndarray__'])
elif '__datetime__' in obj:
# Convert a string representation back to a datetime object
return datetime.fromisoformat(obj['__datetime__'])
else:
# If the dictionary doesn't contain type information, just return it as-is
return item
# %% ../../nbs/utility/data_utils.ipynb 7
def f_data_class_from_dict(type_: type, data: Dict[str, Any]) -> object:
"""
Convert a dictionary to a dataclass object.
Parameters:
- type_: the type of object to create
- data: the dictionary to convert to an object
Returns:
- an object of the specified type, initialized with the data from the dictionary
"""
# Extract the fields of the object
fields = dataclasses.fields(type_)
# Initialize an empty dictionary to store the field names and values
field_values = {}
# Add the fields to the dictionary
for field in fields:
# Check if the field is present in the data
if field.name in data:
# If it is, add it to the dictionary
field_values[field.name] = data[field.name]
else:
# If it's not present, use the default value for the field
field_values[field.name] = field.default
# Create a new object of the specified type and initialize it with the field values
return type_(**field_values)
# %% ../../nbs/utility/data_utils.ipynb 8
def f_dict_to_obj(d: Dict[str, Any], cls: Type[T]) -> T:
"""
Convert a dictionary to an object of the given class.
Parameters:
- d: the dictionary to convert to an object
- cls: the class to use for creating the object
Returns:
- an object of the given class, initialized with the values from the dictionary
"""
return cls(**d)