-
Notifications
You must be signed in to change notification settings - Fork 121
/
fingerprint_utils.py
145 lines (129 loc) · 5.88 KB
/
fingerprint_utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
from typing import Any, Dict, List, Union
import dill
import xxhash
from datasets.fingerprint import (_CACHING_ENABLED,
format_kwargs_for_fingerprint,
format_transform_for_fingerprint,
generate_random_fingerprint,
validate_fingerprint)
from loguru import logger
class Hasher:
"""Hasher that accepts python objets as inputs."""
dispatch: Dict = {}
def __init__(self):
self.m = xxhash.xxh64()
@classmethod
def hash_bytes(cls, value: Union[bytes, List[bytes]]) -> str:
value = [value] if isinstance(value, bytes) else value
m = xxhash.xxh64()
for x in value:
m.update(x)
return m.hexdigest()
@classmethod
def hash_default(cls, value: Any) -> str:
"""
Use dill to serialize objects to avoid serialization failures.
"""
return cls.hash_bytes(dill.dumps(value))
@classmethod
def hash(cls, value: Any) -> str:
if type(value) in cls.dispatch:
return cls.dispatch[type(value)](cls, value)
else:
return cls.hash_default(value)
def update(self, value: Any) -> None:
header_for_update = f'=={type(value)}=='
value_for_update = self.hash(value)
self.m.update(header_for_update.encode('utf8'))
self.m.update(value_for_update.encode('utf-8'))
def hexdigest(self) -> str:
return self.m.hexdigest()
def update_fingerprint(fingerprint, transform, transform_args):
"""
Combining various objects to update the fingerprint.
"""
global fingerprint_warnings
hasher = Hasher()
hasher.update(fingerprint)
try:
hasher.update(transform)
except: # noqa various errors might raise here from pickle or dill
if _CACHING_ENABLED:
if not fingerprint_warnings.get(
'update_fingerprint_transform_hash_failed', False):
logger.warning(
f"Transform {transform} couldn't be hashed properly, \
a random hash was used instead. Make sure your \
transforms and parameters are serializable with \
pickle or dill for the dataset fingerprinting and \
caching to work. If you reuse this transform, the \
caching mechanism will consider it to be different \
from the previous calls and recompute everything. \
This warning is only showed once. Subsequent hashing \
failures won't be showed.")
fingerprint_warnings[
'update_fingerprint_transform_hash_failed'] = True
else:
logger.info(
f"Transform {transform} couldn't be hashed properly, \
a random hash was used instead.")
else:
logger.info(
f"Transform {transform} couldn't be hashed properly, a \
random hash was used instead. This doesn't affect caching \
since it's disabled.")
return generate_random_fingerprint()
for key in sorted(transform_args):
hasher.update(key)
try:
hasher.update(transform_args[key])
except: # noqa various errors might raise here from pickle or dill
if _CACHING_ENABLED:
if not fingerprint_warnings.get(
'update_fingerprint_transform_hash_failed', False):
logger.warning(
f"Parameter '{key}'={transform_args[key]} of the \
transform {transform} couldn't be hashed properly, \
a random hash was used instead. Make sure your \
transforms and parameters are serializable with \
pickle or dill for the dataset fingerprinting and \
caching to work. If you reuse this transform, the \
caching mechanism will consider it to be different \
from the previous calls and recompute everything. \
This warning is only showed once. Subsequent hashing \
failures won't be showed.")
fingerprint_warnings[
'update_fingerprint_transform_hash_failed'] = True
else:
logger.info(
f"Parameter '{key}'={transform_args[key]} of the \
transform {transform} couldn't be hashed properly, \
a random hash was used instead.")
else:
logger.info(
f"Parameter '{key}'={transform_args[key]} of the transform \
{transform} couldn't be hashed properly, a random hash \
was used instead. This doesn't affect caching since it's \
disabled.")
return generate_random_fingerprint()
return hasher.hexdigest()
def generate_fingerprint(ds, *args, **kwargs):
"""
Generate new fingerprints by using various kwargs of the dataset.
"""
if args:
args = list(args)
dataset_kwargs = {'shard': ds, 'function': args[0]}
else:
dataset_kwargs = {'shard': ds}
dataset_kwargs.update(kwargs)
# we create a unique hash from the function,
# current dataset file and the mapping args
transform = format_transform_for_fingerprint(ds._map_single)
kwargs_for_fingerprint = format_kwargs_for_fingerprint(
ds._map_single, (), dataset_kwargs)
kwargs_for_fingerprint['fingerprint_name'] = 'new_fingerprint'
new_fingerprint = update_fingerprint(ds._fingerprint, transform,
kwargs_for_fingerprint)
validate_fingerprint(new_fingerprint)
return new_fingerprint