This repository has been archived by the owner on Dec 18, 2023. It is now read-only.
/
utils.py
135 lines (108 loc) · 3.84 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
import functools
import inspect
from typing import Callable, Union
import credoai.evaluators
from credoai.utils.common import dict_hash
def log_command(fun: Callable):
"""
Decorator loggin the full isgnature of a function call.
Parameters
----------
fun : Callable
A generic function, specifically used for Lens.add, Lens.delete, Lens.run
"""
@functools.wraps(fun)
def wrapper(self, *args, **kwargs):
tmp = fun(self, *args, **kwargs)
name = fun.__name__
self.command_list.append(get_command_string(name, args, kwargs))
return tmp
return wrapper
def get_command_string(name: str, arg: dict, kwarg: dict) -> str:
"""
Combines name and function arguments into a signature string.
Parameters
----------
name : str
Function's name.
arg : dict
Function's positional arguments.
kwarg : dict
Function's keyword argumwents
Returns
-------
str
Full function signature,e.g., fun_name(fun_arg1, fun_arg1..., fun_kwarg1...)
"""
arg_parse = [get_arg_info(arg) for arg in arg]
kwarg_parse = [f"{k}={get_arg_info(v)}" for k, v in kwarg.items()]
all_arguments = arg_parse + kwarg_parse
return f"{name}({','.join([x for x in all_arguments if x is not None])})"
def get_arg_info(arg: Union[Callable, str, int]) -> str:
"""
Takes a function's arguments and converts them into strings.
Parameters
----------
arg : Union[Callable, str, int]
This is quite custom made for usage in Lens(). The positional arguments
can be a call to a class, or int/str. This handles all cases.
Returns
-------
Union[str,int]
Either a string representing the function signature, or str/int
depending on the argument.
"""
if callable(arg):
# Get only initialization arguments
init_args = {
k: v
for k, v in arg.__dict__.items()
if k in list(inspect.signature(arg.__init__).parameters.keys())
}
return f"{type(arg).__name__}({get_string_of_arguments_from_kwargs(init_args)})"
elif isinstance(arg, int):
return str(arg)
elif isinstance(arg, str):
return f'"{arg}"'
def get_string_of_arguments_from_kwargs(keyarg: dict) -> str:
"""
Transform positional arguments in string.
Parameters
----------
keyarg : dict
Function's positional arguments.
Returns
-------
str
String representing the positional arguments
"""
return ",".join([f"{x[0]}={check_int_str(x[1])}" for x in keyarg.items()])
def check_int_str(x: Union[int, float, str]) -> Union[int, str, float]:
"""
Check what type is the argument and reformats in case it is a string.
"""
if isinstance(x, (int, float)):
return x
elif isinstance(x, str):
return f'"{x}"'
def add_metric_keys(prepared_results):
"""Adds metric keys to prepared results
Metric keys are used to associated charts, html blobs, and other assets with
specific metrics. They are a hash of most of the metric's attributes, except the value.
So if a metric changes value, the key will stay the same.
Metric keys should be defined after all pertinent information is appended to a metric.
Lens normally handles key association, because it may add additional metadata to a metric
beyond what the assessment creates (e.g., dataset name, model name, etc.)
Parameters
----------
prepared_results : DataFrame
output of CredoAssessment.prepare_results()
"""
if prepared_results is None:
return
ignored = ["value", "metadata"]
keys = [
dict_hash({k: v for k, v in metric_dict.items() if k not in ignored})
for metric_dict in prepared_results.reset_index().to_dict("records")
]
prepared_results["metric_key"] = keys