-
Notifications
You must be signed in to change notification settings - Fork 0
/
model.py
73 lines (62 loc) · 2.37 KB
/
model.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
import nltk
import os
import os.path
import time
from preprocess import preprocess
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import svm
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from sklearn.externals import joblib
def buildModel():
traindata_dir = "aclImdb/train/"
testdata_dir = "aclImdb/test/"
classes = ['pos', 'neg']
# Read the data
train_data = []
train_labels = []
test_data = []
test_labels = []
for curr_class in classes:
dirname = os.path.join(traindata_dir, curr_class)
fnamelist = os.listdir(dirname)
for fname in fnamelist:
with open(os.path.join(dirname, fname), 'r', encoding="utf8") as f:
content = f.read()
train_data.append(content)
train_labels.append(curr_class)
for curr_class in classes:
dirname = os.path.join(testdata_dir, curr_class)
fnamelist = os.listdir(dirname)
for fname in fnamelist:
with open(os.path.join(dirname, fname), 'r', encoding="utf8") as f:
content = f.read()
test_data.append(content)
test_labels.append(curr_class)
# Create feature vectors
vectorizer = TfidfVectorizer(tokenizer=preprocess, sublinear_tf=True)
train_vectors = vectorizer.fit_transform(train_data)
test_vectors = vectorizer.transform(test_data)
joblib.dump(vectorizer, 'vectorizer.pkl')
# Perform classification with SVM, kernel=linear
classifier = svm.LinearSVC()
t0 = time.time()
classifier.fit(train_vectors, train_labels)
t1 = time.time()
joblib.dump(classifier, 'model.pkl')
prediction = classifier.predict(test_vectors)
t2 = time.time()
time_linear_train = t1-t0
time_linear_predict = t2-t1
# Print results in a nice table
print("Results for SVC(kernel=linear)")
print("Training time: %fs; Prediction time: %fs" % (time_linear_train, time_linear_predict))
print(classification_report(test_labels, prediction))
print(accuracy_score(test_labels, prediction))
return vectorizer, classifier
def loadModel():
vectorizer = joblib.load('vectorizer.pkl')
model = joblib.load('model.pkl')
return vectorizer, model
def isModelExists():
return os.path.isfile('model.pkl') and os.path.isfile('vectorizer.pkl')