This repository has been archived by the owner on Dec 18, 2023. It is now read-only.
/
evaluator.py
181 lines (150 loc) · 5.96 KB
/
evaluator.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
from abc import ABC, abstractmethod
from connect.evidence import EvidenceContainer
from credoai.utils import global_logger
from credoai.utils.common import NotRunError, ValidationError
class Evaluator(ABC):
"""
Base abstract class for all lens evaluators.
Defines basic functions required from any evaluator object.
This class leverages the special method `__call__` to make artifacts
available in the class enclosure.
.. automethod:: __call__
.. automethod:: _init_artifacts
.. automethod:: _validate_arguments
.. automethod:: _setup
"""
def __init__(self):
self._results = None
self.artifact_keys = []
self.logger = global_logger
self.metadata = {}
@property
def name(self):
"""The name associated to the Evaluator, equals the class name."""
return self.__class__.__name__
@property
def results(self):
"""
Container for all results.
It is expected to be a list of EvidenceContainers. This is enforced in
the associated setter method.
Raises
------
NotRunError
It indicates that results are missing, the evaluator was not run.
"""
if self._results is not None:
return self._results
else:
raise NotRunError(
"No results available, please call the method: 'evaluate'."
)
@results.setter
def results(self, results):
"""Requires the results to be list of Evidence Containers"""
if not isinstance(results, list):
raise ValidationError("Results must be a list")
for result in results:
if not isinstance(result, EvidenceContainer):
raise ValidationError("All results must be EvidenceContainers")
self._results = results
@property
@abstractmethod
def required_artifacts(self):
"""
The required artifacts necessary for the functioning of the evaluator
This set contains the :ref:`artifacts<credoai.artifacts>` that Lens can feed to
an evaluator, the accepted values are ``{"model", "assessment_data", "training_data", "data"}``.
The string "data" means that the evaluator can be run on assessment and/or training data
(DataProfiler is an example). Lens will iterate over all the available artifacts internally.
The set can also include the string "sensitive_feature". This is to indicate
that the evaluator depends on sensitive features. Lens will iterate over the available sensitive
features internally.
"""
pass
def __call__(self, **kwargs):
"""
This method is used to pass the model, assessment_data and training_data
artifacts to instantiated evaluator.
The method is called internally by the Lens instance, which only passes the
artifacts specified in the property :meth:`required_artifacts<Evaluator.required_artifacts>`.
After the artifacts are passed, it performs arguments validation and calls :meth:`_setup<Evaluator._setup>`
At the end of these operation, the validated artifacts are available in the evaluator enclosure.
"""
self._init_artifacts(kwargs)
self._validate_arguments()
self._setup()
return self
@abstractmethod
def evaluate(self):
"""
Execute any data/model processing required for the evaluator.
Populates the self.results object.
"""
return self
def get_container_info(self, labels: dict = None, metadata: dict = None):
"""
Expands the base labels and metadata used to populate evidences.
Parameters
----------
labels : dict, optional
The default labels can be expanded by the user when defining a new evaluator.
A label is in general any information necessary to identify evidences in the Credo AI Platform,
therefore, by default None.
metadata : dict, optional
Any extra info the user wants to associate to the evidences. Compared
to labels these are not necessary for evidence identification, by default None.
"""
info = self._base_container_info()
if labels:
info["labels"].update(labels)
if metadata:
info["metadata"].update(metadata)
return info
def _base_container_info(self):
"""Extract basic info to populate labels and metadata."""
meta = {**self.metadata, **self._get_artifacts()}
labels = {"evaluator": self.name}
if "dataset_type" in meta:
labels["dataset_type"] = meta["dataset_type"]
return {"labels": labels, "metadata": meta}
def _get_artifacts(self):
"""
Extract artifacts that will be used by the evaluator.
The method also extract name info from the available artifacts.
"""
artifacts = {}
save_keys = {
"model": "model_name",
"data": "dataset_name",
"assessment_data": "assessment_dataset_name",
"training_data": "training_dataset_name",
}
for k in self.artifact_keys:
save_key = save_keys.get(k, k)
try:
artifacts[save_key] = self.__dict__[k].name
except AttributeError:
pass
return artifacts
def _init_artifacts(self, artifacts):
"""Adds artifacts to evaluator object
Parameters
----------
artifacts : dict
Dictionary of artifacts, e.g. {'model': Model}
"""
self.artifact_keys = list(artifacts.keys())
self.__dict__.update(artifacts)
@abstractmethod
def _setup(self):
"""
Contains any extra steps necessary to initialize the evaluator
"""
pass
@abstractmethod
def _validate_arguments(self):
"""
Check that basic requirements for the run of an evaluator are met.
"""
pass