-
Notifications
You must be signed in to change notification settings - Fork 2
/
create_scale_n_dataset.py
98 lines (75 loc) · 2.59 KB
/
create_scale_n_dataset.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 27 13:52:36 2017
@author: jens
"""
import os
import scipy.io as sio
import extract_features
import numpy as np
import matplotlib.pyplot as plt
import csv
import pickle
def load_csv():
N = '/home/jens/Documents/stanford/overview_file_cohorts.csv'
trainL = []
label = []
ID = []
with open(N) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
trainL += [int(row['Used for narco training'])]
ID += [row['ID']]
label += [int(row['Label'])]
return ID, label, trainL
if __name__ == '__main__':
ID, labelT, trainLT = load_csv()
P = '/home/jens/Documents/stanford/scored_data/'
D = os.listdir(P)
D.sort()
for d in D:
F = os.listdir(P+d)
F.sort()
count = 0
labels = np.zeros(len(F))
trainL = np.zeros(len(F))
featStack = []
for f in F:
name = f[30:-7]
try:
index = ID.index(name)
except:
print(name + ' not found')
continue
print(str(count) + '/' + str(len(F)))
contents = sio.loadmat(P+d+'/'+f)
pred = contents['predictions']
if len(pred)==0:
continue
labels[count] = labelT[index]
trainL[count] = trainLT[index]
feat = extract_features.extract(pred)
feat = np.expand_dims(feat,axis=1)
if len(featStack)==0:
featStack = feat
else:
featStack = np.concatenate([featStack,feat],axis=1)
count += 1
featStack = np.transpose(featStack)
labels = labels[:count]
trainL = trainL[:count]
m = np.mean(featStack,axis=1)+1e-10
v = np.percentile(featStack,85,axis=1) - np.percentile(featStack,15,axis=1)+1e-10
m = np.expand_dims(m,axis=1)
featStackScaled = featStack/np.expand_dims(v,axis=1)
featStackScaled[10<featStackScaled] = 10
featStackScaled[-10>featStackScaled] = -10
data = {'features': featStackScaled,
'labels': labels,
'trainL': trainL,
}
scale = {'mean': m,
'range': v}
output = open('narco_features/' + d +'_narcodata.pkl', 'wb')
pickle.dump(data, output, -1)
output.close()