-
Notifications
You must be signed in to change notification settings - Fork 403
/
test_disparities.py
129 lines (93 loc) · 4.76 KB
/
test_disparities.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
# Copyright (c) Microsoft Corporation and Fairlearn contributors.
# Licensed under the MIT License.
import pytest
from fairlearn.metrics import MetricFrame
from fairlearn.metrics import selection_rate, true_positive_rate, false_positive_rate
from fairlearn.metrics import (
demographic_parity_difference,
demographic_parity_ratio,
equalized_odds_difference,
equalized_odds_ratio
)
from .data_for_test import y_t, y_p, s_w, g_1
_aggregate_methods = ['between_groups', 'to_overall']
@pytest.mark.parametrize("agg_method", _aggregate_methods)
def test_demographic_parity_difference(agg_method):
actual = demographic_parity_difference(y_t, y_p, sensitive_features=g_1, method=agg_method)
gm = MetricFrame(metrics=selection_rate, y_true=y_t, y_pred=y_p, sensitive_features=g_1)
assert actual == gm.difference(method=agg_method)
@pytest.mark.parametrize("agg_method", _aggregate_methods)
def test_demographic_parity_difference_weighted(agg_method):
actual = demographic_parity_difference(y_t, y_p,
sensitive_features=g_1,
sample_weight=s_w,
method=agg_method)
gm = MetricFrame(metrics=selection_rate, y_true=y_t, y_pred=y_p,
sensitive_features=g_1,
sample_params={'sample_weight': s_w})
assert actual == gm.difference(method=agg_method)
@pytest.mark.parametrize("agg_method", _aggregate_methods)
def test_demographic_parity_ratio(agg_method):
actual = demographic_parity_ratio(y_t, y_p,
sensitive_features=g_1,
method=agg_method)
gm = MetricFrame(metrics=selection_rate, y_true=y_t, y_pred=y_p, sensitive_features=g_1)
assert actual == gm.ratio(method=agg_method)
@pytest.mark.parametrize("agg_method", _aggregate_methods)
def test_demographic_parity_ratio_weighted(agg_method):
actual = demographic_parity_ratio(y_t, y_p,
sensitive_features=g_1,
sample_weight=s_w,
method=agg_method)
gm = MetricFrame(metrics=selection_rate, y_true=y_t, y_pred=y_p,
sensitive_features=g_1,
sample_params={'sample_weight': s_w})
assert actual == gm.ratio(method=agg_method)
@pytest.mark.parametrize("agg_method", _aggregate_methods)
def test_equalized_odds_difference(agg_method):
actual = equalized_odds_difference(y_t, y_p,
sensitive_features=g_1,
method=agg_method)
metrics = {'tpr': true_positive_rate, 'fpr': false_positive_rate}
gm = MetricFrame(metrics=metrics, y_true=y_t, y_pred=y_p, sensitive_features=g_1)
diffs = gm.difference(method=agg_method)
assert actual == diffs.max()
@pytest.mark.parametrize("agg_method", _aggregate_methods)
def test_equalized_odds_difference_weighted(agg_method):
actual = equalized_odds_difference(y_t, y_p,
sensitive_features=g_1,
method=agg_method,
sample_weight=s_w)
metrics = {'tpr': true_positive_rate, 'fpr': false_positive_rate}
sw = {'sample_weight': s_w}
sp = {'tpr': sw, 'fpr': sw}
gm = MetricFrame(metrics=metrics,
y_true=y_t,
y_pred=y_p,
sensitive_features=g_1, sample_params=sp)
diffs = gm.difference(method=agg_method)
assert actual == diffs.max()
@pytest.mark.parametrize("agg_method", _aggregate_methods)
def test_equalized_odds_ratio(agg_method):
actual = equalized_odds_ratio(y_t, y_p,
method=agg_method,
sensitive_features=g_1)
metrics = {'tpr': true_positive_rate, 'fpr': false_positive_rate}
gm = MetricFrame(metrics=metrics, y_true=y_t, y_pred=y_p, sensitive_features=g_1)
ratios = gm.ratio(method=agg_method)
assert actual == ratios.min()
@pytest.mark.parametrize("agg_method", _aggregate_methods)
def test_equalized_odds_ratio_weighted(agg_method):
actual = equalized_odds_ratio(y_t, y_p,
method=agg_method,
sensitive_features=g_1,
sample_weight=s_w)
metrics = {'tpr': true_positive_rate, 'fpr': false_positive_rate}
sw = {'sample_weight': s_w}
sp = {'tpr': sw, 'fpr': sw}
gm = MetricFrame(metrics=metrics,
y_true=y_t,
y_pred=y_p,
sensitive_features=g_1, sample_params=sp)
ratios = gm.ratio(method=agg_method)
assert actual == ratios.min()