-
Notifications
You must be signed in to change notification settings - Fork 1k
/
accuracy.py
143 lines (102 loc) · 3.98 KB
/
accuracy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
"""
The :mod:`surprise.accuracy` module provides with tools for computing accuracy
metrics on a set of predictions.
Available accuracy metrics:
.. autosummary::
:nosignatures:
rmse
mae
fcp
"""
from __future__ import (absolute_import, division, print_function,
unicode_literals)
from collections import defaultdict
import numpy as np
from six import iteritems
def rmse(predictions, verbose=True):
"""Compute RMSE (Root Mean Squared Error).
.. math::
\\text{RMSE} = \\sqrt{\\frac{1}{|\\hat{R}|} \\sum_{\\hat{r}_{ui} \in
\\hat{R}}(r_{ui} - \\hat{r}_{ui})^2}.
Args:
predictions (:obj:`list` of :obj:`Prediction\
<surprise.prediction_algorithms.predictions.Prediction>`):
A list of predictions, as returned by the :meth:`test()
<surprise.prediction_algorithms.algo_base.AlgoBase.test>` method.
verbose: If True, will print computed value. Default is ``True``.
Returns:
The Root Mean Squared Error of predictions.
Raises:
ValueError: When ``predictions`` is empty.
"""
if not predictions:
raise ValueError('Prediction list is empty.')
mse = np.mean([float((true_r - est)**2)
for (_, _, true_r, est, _) in predictions])
rmse_ = np.sqrt(mse)
if verbose:
print('RMSE: {0:1.4f}'.format(rmse_))
return rmse_
def mae(predictions, verbose=True):
"""Compute MAE (Mean Absolute Error).
.. math::
\\text{MAE} = \\frac{1}{|\\hat{R}|} \\sum_{\\hat{r}_{ui} \in
\\hat{R}}|r_{ui} - \\hat{r}_{ui}|
Args:
predictions (:obj:`list` of :obj:`Prediction\
<surprise.prediction_algorithms.predictions.Prediction>`):
A list of predictions, as returned by the :meth:`test()
<surprise.prediction_algorithms.algo_base.AlgoBase.test>` method.
verbose: If True, will print computed value. Default is ``True``.
Returns:
The Mean Absolute Error of predictions.
Raises:
ValueError: When ``predictions`` is empty.
"""
if not predictions:
raise ValueError('Prediction list is empty.')
mae_ = np.mean([float(abs(true_r - est))
for (_, _, true_r, est, _) in predictions])
if verbose:
print('MAE: {0:1.4f}'.format(mae_))
return mae_
def fcp(predictions, verbose=True):
"""Compute FCP (Fraction of Concordant Pairs).
Computed as described in paper `Collaborative Filtering on Ordinal User
Feedback <http://www.ijcai.org/Proceedings/13/Papers/449.pdf>`_ by Koren
and Sill, section 5.2.
Args:
predictions (:obj:`list` of :obj:`Prediction\
<surprise.prediction_algorithms.predictions.Prediction>`):
A list of predictions, as returned by the :meth:`test()
<surprise.prediction_algorithms.algo_base.AlgoBase.test>` method.
verbose: If True, will print computed value. Default is ``True``.
Returns:
The Fraction of Concordant Pairs.
Raises:
ValueError: When ``predictions`` is empty.
"""
if not predictions:
raise ValueError('Prediction list is empty.')
predictions_u = defaultdict(list)
nc_u = defaultdict(int)
nd_u = defaultdict(int)
for u0, _, r0, est, _ in predictions:
predictions_u[u0].append((r0, est))
for u0, preds in iteritems(predictions_u):
for r0i, esti in preds:
for r0j, estj in preds:
if esti > estj and r0i > r0j:
nc_u[u0] += 1
if esti >= estj and r0i < r0j:
nd_u[u0] += 1
nc = np.mean(list(nc_u.values())) if nc_u else 0
nd = np.mean(list(nd_u.values())) if nd_u else 0
try:
fcp = nc / (nc + nd)
except ZeroDivisionError:
raise ValueError('cannot compute fcp on this list of prediction. ' +
'Does every user have at least two predictions?')
if verbose:
print('FCP: {0:1.4f}'.format(fcp))
return fcp