-
Notifications
You must be signed in to change notification settings - Fork 16
/
preprocess.py
141 lines (130 loc) · 4.01 KB
/
preprocess.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
import sys
from srs.utils import argparse
from pathlib import Path
parser = argparse.ArgumentParser()
parser.add_argument(
'--dataset',
choices=['gowalla', 'delicious', 'foursquare'],
required=True,
help='the dataset name',
)
parser.add_argument(
'--input-dir',
type=Path,
default='datasets',
help='the directory containing the raw data files',
)
parser.add_argument(
'--output-dir',
type=Path,
default='datasets',
help='the directory to store the preprocessed dataset',
)
parser.add_argument(
'--train-split', type=float, default=0.6, help='the ratio of the training set'
)
parser.add_argument(
'--max-len', type=int, default=50, help='the maximum session length'
)
args = parser.parse_args()
FILENAMES = {
'gowalla': ['loc-gowalla_totalCheckins.txt', 'loc-gowalla_edges.txt'],
'delicious': [
'user_taggedbookmarks-timestamps.dat',
'user_contacts-timestamps.dat',
],
'foursquare': [
'dataset_WWW_Checkins_anonymized.txt',
'dataset_WWW_friendship_new.txt',
'raw_POIs.txt',
],
}
filenames = FILENAMES[args.dataset]
for filename in filenames:
if not (args.input_dir / filename).exists():
print(f'File {filename} not found in {args.input_dir}', file=sys.stderr)
sys.exit(1)
clicks = args.input_dir / filenames[0]
edges = args.input_dir / filenames[1]
import numpy as np
import pandas as pd
from srs.utils.data.preprocess import preprocess, update_id
print('reading dataset...')
if args.dataset == 'gowalla':
args.interval = pd.Timedelta(days=1)
args.max_items = 50000
df = pd.read_csv(
clicks,
sep='\t',
header=None,
names=['userId', 'timestamp', 'latitude', 'longitude', 'itemId'],
parse_dates=['timestamp'],
infer_datetime_format=True,
)
df_clicks = df[['userId', 'timestamp', 'itemId']]
df_loc = df.groupby('itemId').agg({
'latitude': lambda col: col.iloc[0],
'longitude': lambda col: col.iloc[0],
}).reset_index()
df_edges = pd.read_csv(edges, sep='\t', header=None, names=['follower', 'followee'])
elif args.dataset == 'delicious':
df_clicks = pd.read_csv(
clicks,
sep='\t',
skiprows=1,
header=None,
names=['userId', 'sessionId', 'itemId', 'timestamp'],
)
df_clicks['timestamp'] = pd.to_datetime(df_clicks.timestamp, unit='ms')
df_loc = None
df_edges = pd.read_csv(
edges,
sep='\t',
skiprows=1,
header=None,
usecols=[0, 1],
names=['follower', 'followee'],
)
elif args.dataset == 'foursquare':
args.interval = pd.Timedelta(days=1)
args.max_users = 50000
args.max_items = 50000
df_loc = pd.read_csv(
args.input_dir / 'raw_POIs.txt',
sep='\t',
header=None,
usecols=[0, 1, 2],
names=['itemId', 'latitude', 'longitude']
)
df_clicks = pd.read_csv(
clicks,
sep='\t',
header=None,
usecols=[0, 1, 2],
names=['userId', 'itemId', 'timestamp'],
)
df_clicks['timestamp'] = pd.to_datetime(
df_clicks.timestamp, format='%a %b %d %H:%M:%S %z %Y', errors='coerce'
)
df_edges = pd.read_csv(edges, sep='\t', header=None, names=['follower', 'followee'])
df_edges_rev = pd.DataFrame({
'followee': df_edges.follower,
'follower': df_edges.followee
})
df_edges = df_edges.append(df_edges_rev, ignore_index=True)
else:
print(f'Unsupported dataset {args.dataset}', file=sys.stderr)
sys.exit(1)
df_edges = df_edges[df_edges.follower != df_edges.followee]
df_clicks = df_clicks.dropna()
print('converting IDs to integers...')
df_clicks, df_edges = update_id(
df_clicks, df_edges, colnames=['userId', 'followee', 'follower']
)
if df_loc is None:
df_clicks = update_id(df_clicks, colnames='itemId')
else:
df_clicks, df_loc = update_id(df_clicks, df_loc, colnames='itemId')
df_clicks = df_clicks.sort_values(['userId', 'timestamp'])
np.random.seed(123456)
preprocess(df_clicks, df_edges, df_loc, args)