-
Notifications
You must be signed in to change notification settings - Fork 91
/
read_file.py
330 lines (280 loc) · 13.8 KB
/
read_file.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
# coding=utf-8
"""
© 2016. Case Recommender All Rights Reserved (License GPL3)
This file is responsible for read external files.
The accepted data format is:
user\titem\tinformation\n
* user and item must be integer!
* To change the spacing in the file, change the space_type var (default = \t).
Methods:
- main_information and main_information_item_recommendation: returns a set of information about the dataset:
* list with all users
* list with all items
* number of interactions
* users interactions (dictionary with seen items and feedback for each user |
number of interaction for each user)
* items interactions (dictionary with users and feedback for each item |
number of interaction for each item)
- cross_fold_validation: return triples [user, item, feedback] and number of interactions
- split_dataset: return triples [user, item, feedback], number of interactions and
users interactions (dictionary with seen items and feedback for each user |
number of interaction for each user) for each feedback type.
- rating_prediction: returns a set of specifics attributes from dataset in a dictionary:
* dictionary with all interactions
* list with all users
* list with all items
* dictionary of all users interaction
* dictionary of all items interaction
* mean of rates
- read_rankings: return a dictionary and a list about one ranking
- read_matrix: returns a data matrix
"""
import sys
import numpy as np
from caserec.utils.extra_functions import check_error_file
__author__ = 'Arthur Fortes'
class ReadFile(object):
def __init__(self, file_read, space_type='\t'):
self.file_read = file_read
self.space_type = space_type
self.list_users = set()
self.list_items = set()
self.number_interactions = 0
self.dict_users = dict()
self.dict_items = dict()
self.num_user_interactions = dict()
self.num_items_interactions = dict()
self.triple_dataset = list()
self.individual_interaction = list()
self.average_scores = dict()
self.mean_feedback = 0
def return_information(self, implicit=False):
check_error_file(self.file_read)
dict_file = dict()
d_feedback = dict()
list_feedback = list()
not_seen = dict()
map_user = dict()
map_index_user = dict()
du_feed = dict()
with open(self.file_read) as infile:
for line in infile:
if line.strip():
inline = line.split(self.space_type)
if len(inline) == 1:
print("Error: Space type is invalid!")
sys.exit()
try:
user, item, feedback = int(inline[0]), int(inline[1]), float(inline[2])
d_feedback.setdefault(user, {}).update({item: feedback})
self.triple_dataset.append((user, item, feedback))
self.dict_users.setdefault(user, set()).add(item)
self.dict_items.setdefault(item, set()).add(user)
du_feed.setdefault(user, list()).append(item)
self.list_users.add(user)
self.list_items.add(item)
self.mean_feedback += feedback
list_feedback.append(feedback)
self.number_interactions += 1
except ValueError:
pass
self.triple_dataset = sorted(self.triple_dataset)
self.mean_feedback /= float(self.number_interactions)
self.list_users = set(sorted(list(self.list_users)))
self.list_items = set(sorted(list(self.list_items)))
for user in self.list_users:
not_seen[user] = list(set(self.list_items) - set(self.dict_users[user]))
for u, user in enumerate(self.list_users):
map_user[user] = u
map_index_user[u] = user
map_item = dict()
map_index_item = dict()
self.list_items = set(sorted(list(self.list_items)))
for i, item in enumerate(self.list_items):
map_item[item] = i
map_index_item[i] = item
matrix = np.zeros((len(self.list_users), len(self.list_items)))
for user in self.list_users:
for item in self.dict_users[user]:
if implicit:
matrix[map_user[user]][map_item[item]] = 1
else:
matrix[map_user[user]][map_item[item]] = d_feedback[user][item]
# self.dict_items.setdefault(map_item[item], set()).add(map_user[user])
sparsity = (1 - (self.number_interactions / float(len(self.list_users) * len(self.list_items)))) * 100
dict_file.update({'feedback': d_feedback, 'users': self.list_users, 'items': self.list_items,
'du': self.dict_users, 'di': self.dict_items, 'mean_rates': self.mean_feedback,
'list_feedback': self.triple_dataset, 'ni': self.number_interactions,
'max': max(list_feedback), 'min': min(list_feedback), 'sparsity': sparsity,
'not_seen': not_seen, 'matrix': matrix, 'map_user': map_index_user,
'map_item': map_index_item, 'mu': map_user, 'du_order': du_feed})
return dict_file
def triple_information(self):
check_error_file(self.file_read)
with open(self.file_read) as infile:
for line in infile:
if line.strip():
inline = line.split(self.space_type)
if len(inline) == 1:
print("Error: Space type is invalid!")
sys.exit()
try:
user, item, feedback = int(inline[0]), int(inline[1]), inline[2].replace("\n", "")
self.triple_dataset.append([user, item, feedback])
self.number_interactions += 1
except ValueError:
pass
def split_dataset(self):
for i, feedback in enumerate(self.file_read):
self.dict_users = dict()
check_error_file(feedback)
with open(feedback) as infile:
for line in infile:
if line.strip():
inline = line.split(self.space_type)
if len(inline) == 1:
print("Error: Space type is invalid!")
sys.exit()
self.number_interactions += 1
try:
user, item, feedback = int(inline[0]), int(inline[1]), float(inline[2])
self.triple_dataset.append((user, item))
self.dict_users.setdefault(user, {}).update({item: feedback})
except ValueError:
pass
self.individual_interaction.append(self.dict_users)
def read_rankings(self):
list_feedback = list()
check_error_file(self.file_read)
with open(self.file_read) as infile:
for line in infile:
if line.strip():
inline = line.split(self.space_type)
if len(inline) == 1:
print("Error: Space type is invalid!")
sys.exit()
try:
user, item, feedback = int(inline[0]), int(inline[1]), float(inline[2])
self.dict_users.setdefault(user, {}).update({item: feedback})
list_feedback.append(feedback)
self.average_scores[user] = self.average_scores.get(user, 0) + feedback
self.num_user_interactions[user] = self.num_user_interactions.get(user, 0) + 1
except ValueError:
pass
return self.dict_users, list_feedback
def read_matrix(self):
matrix = list()
check_error_file(self.file_read)
with open(self.file_read) as infile:
for line in infile:
if line.strip():
inline = line.split(self.space_type)
inline = np.array(inline)
inline = np.delete(inline, len(inline)-1)
matrix.append(inline.astype(float))
return np.array(matrix)
def ensemble(self):
dict_info = dict()
for r, rank_file in enumerate(self.file_read):
self.list_users = set()
self.dict_users = dict()
with open(rank_file) as infile:
for line in infile:
if line.strip():
inline = line.split(self.space_type)
if len(inline) == 1:
print("Error: Space type is invalid!")
sys.exit()
try:
user, item, score = int(inline[0]), int(inline[1]), float(inline[2])
self.list_users.add(user)
self.dict_users.setdefault(user, list()).append([user, item, score])
except ValueError:
pass
self.list_users = set(sorted(self.list_users))
for user in self.list_users:
n_rank = len(self.dict_users[user])
for i, triple in enumerate(self.dict_users[user]):
self.dict_users[user][i][2] = n_rank - i
dict_info.setdefault(r, dict()).update({'rank': self.dict_users, 'users': self.list_users})
return dict_info
def ensemble_test(self):
user_info = dict()
rank_info = dict()
for r, rank_file in enumerate(self.file_read):
r_dict = dict()
with open(rank_file) as infile:
for line in infile:
if line.strip():
inline = line.split(self.space_type)
if len(inline) == 1:
print("Error: Space type is invalid!")
sys.exit()
try:
user, item, score = int(inline[0]), int(inline[1]), float(inline[2])
self.number_interactions += 1
self.list_users.add(user)
self.list_items.add(item)
self.dict_users.setdefault(user, {}).update({item: score})
r_dict.setdefault(user, {}).update({item: score})
except ValueError:
pass
rank_info[r] = r_dict
self.list_users = set(sorted(self.list_users))
self.list_items = set(sorted(self.list_items))
dict_non_seen = dict()
for user in self.dict_users:
dict_non_seen[user] = list(set(self.list_items) - set(self.dict_users[user]))
user_info[user] = {'j': dict_non_seen[user], 'i': self.dict_users[user].keys()}
return user_info, self.list_users, self.list_items, self.number_interactions, rank_info
def read_metadata(self, l_items):
dict_file = dict()
d_feedback = dict()
list_feedback = list()
map_user = dict()
map_index_user = dict()
map_item = dict()
map_index_item = dict()
check_error_file(self.file_read)
with open(self.file_read) as infile:
for line in infile:
if line.strip():
inline = line.split(self.space_type)
if len(inline) == 1:
print("Error: Space type is invalid in metadata file!")
print(inline, self.space_type)
sys.exit()
try:
user, item, feedback = int(inline[0]), int(inline[1]), float(inline[2])
d_feedback.setdefault(user, {}).update({item: feedback})
self.triple_dataset.append((user, item, feedback))
self.dict_users.setdefault(user, set()).add(item)
self.dict_items.setdefault(item, set()).add(user)
self.list_items.add(item)
self.mean_feedback += feedback
list_feedback.append(feedback)
self.number_interactions += 1
except ValueError:
pass
self.triple_dataset = sorted(self.triple_dataset)
self.mean_feedback /= float(self.number_interactions)
self.list_users = set(sorted(list(l_items)))
self.list_items = set(sorted(list(self.list_items)))
for u, user in enumerate(self.list_users):
map_user[user] = u
map_index_user[u] = user
for i, item in enumerate(self.list_items):
map_item[item] = i
map_index_item[i] = item
matrix = np.zeros((len(self.list_users), len(self.list_items)))
for user in self.list_users:
try:
for item in d_feedback[user]:
matrix[map_user[user]][map_item[item]] = d_feedback[user][item]
except KeyError:
pass
dict_file.update({'feedback': d_feedback, 'items': self.list_users, 'metadata': self.list_items,
'di': self.dict_users, 'dm': self.dict_items, 'mean_rates': self.mean_feedback,
'list_feedback': self.triple_dataset, 'ni': self.number_interactions,
'max': max(list_feedback), 'min': min(list_feedback), 'matrix': matrix})
return dict_file