-
Notifications
You must be signed in to change notification settings - Fork 0
/
dataset.py
101 lines (83 loc) · 3.95 KB
/
dataset.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
import os
import numpy as np
import pandas as pd
from scipy import sparse
import torch
from torch.utils.data import Dataset
from rectorch.data import DataProcessing, DatasetManager
class TrainDataset(Dataset):
def __init__(self, train_data: sparse.csr_matrix):
super().__init__()
# train_data = pd.read_csv(train_path, sep='\\s+', header=None, names=['uid', 'sid'], engine='python')
# train_data['uid'] = train_data['uid'] - 1
# train_data['sid'] = train_data['sid'] - 1
# rows_tr, cols_tr = train_data['uid'], train_data['sid']
# data_tr = sparse.csr_matrix((np.ones_like(rows_tr), (rows_tr, cols_tr)),
# dtype='float64',
# shape=(n, m))
self.train_dataset = torch.FloatTensor(train_data.toarray())
# test_data = pd.read_csv(test_path, sep='\\s+', header=None, names=['uid', 'sid'], engine='python')
# test_data['uid'] = test_data['uid'] - 1
# test_data['sid'] = test_data['sid'] - 1
# rows_te, cols_te = test_data['uid'], test_data['sid']
# data_te = sparse.csr_matrix((np.ones_like(rows_te), (rows_te, cols_te)),
# dtype='float64',
# shape=(n, m))
def __len__(self):
return len(self.train_dataset)
def __getitem__(self, idx):
train_record = self.train_dataset[idx]
return train_record
class TestDataset(Dataset):
def __init__(self, train_data: sparse.csr_matrix, test_data: sparse.csr_matrix):
super().__init__()
# train_data = pd.read_csv(train_path, sep='\\s+', header=None, names=['uid', 'sid'], engine='python')
# train_data['uid'] = train_data['uid'] - 1
# train_data['sid'] = train_data['sid'] - 1
# rows_tr, cols_tr = train_data['uid'], train_data['sid']
# data_tr = sparse.csr_matrix((np.ones_like(rows_tr), (rows_tr, cols_tr)),
# dtype='float64',
# shape=(n, m))
self.train_dataset = torch.FloatTensor(train_data.toarray())
# test_data = pd.read_csv(test_path, sep='\\s+', header=None, names=['uid', 'sid'], engine='python')
# test_data['uid'] = test_data['uid'] - 1
# test_data['sid'] = test_data['sid'] - 1
# rows_te, cols_te = test_data['uid'], test_data['sid']
# data_te = sparse.csr_matrix((np.ones_like(rows_te), (rows_te, cols_te)),
# dtype='float64',
# shape=(n, m))
self.test_dataset = torch.FloatTensor(test_data.toarray())
def __len__(self):
return len(self.train_dataset)
def __getitem__(self, idx):
train_record = self.train_dataset[idx]
test_record = self.test_dataset[idx]
return train_record, test_record
class ClientsSampler(Dataset):
def __init__(self, n):
super().__init__()
self.users_seq = np.arange(n)
def __len__(self):
return len(self.users_seq)
def __getitem__(self, idx):
return self.users_seq[idx]
class ClientsDataset:
def __init__(self, data: sparse.csr_matrix):
# data = pd.read_csv(data_path, sep='\\s+', header=None, names=['uid', 'sid'], engine='python')
# data['uid'] = data['uid'] - 1
# data['sid'] = data['sid'] - 1
# rows, cols = data['uid'], data['sid']
# clients_data = sparse.csr_matrix((np.ones_like(rows), (rows, cols)),
# dtype='float64',
# shape=(n, m))
self.clients_data = torch.FloatTensor(data.toarray())
def __len__(self):
return len(self.clients_data)
def __getitem__(self, idx):
return self.clients_data[idx]
def load_data(path):
dproc = DataProcessing(path)
if not os.path.exists(dproc.cfg.proc_path):
dproc.process()
dataset = DatasetManager(dproc.cfg)
return dataset