-
Notifications
You must be signed in to change notification settings - Fork 0
/
task4.py
142 lines (108 loc) · 3.97 KB
/
task4.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import numpy as np
import pandas as pd
import sklearn.model_selection as skl_ms
import matplotlib.pyplot as plt
import sklearn.metrics as skl_m
import sklearn.feature_extraction.text as skl_text
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import f1_score, make_scorer
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
def prepare(data):
data.pop("severe_toxic")
data.pop("obscene")
data.pop("threat")
data.pop("insult")
data.pop("identity_hate")
teach_data, test_data = skl_ms.train_test_split(data, test_size=0.2, random_state=666)
vectorizer = skl_text.CountVectorizer(lowercase=True, ngram_range=(1, 1), strip_accents='unicode',
stop_words={'english'}, analyzer='word')
vectorizer.fit(data['comment_text'])
X = vectorizer.transform(teach_data["comment_text"])
y = teach_data['toxic']
X_test = vectorizer.transform(test_data["comment_text"])
y_test = test_data['toxic']
return X, y, X_test, y_test
class SGDLogisticRegression:
def __init__(self, epsilon=0.001):
self.w = None
self.epsilon = epsilon
pass
def sigmoid(self, x, w):
return 1 / (1 + np.exp(-1 * np.dot(x, w)))
def fit(self, X, y, iterations=500):
X = np.concatenate(
(np.ones((X.shape[0], 1)), X),
axis=1
)
self.w = X[0].copy()
self.w.fill(0.28147)
for i in range(iterations):
sigma = self.sigmoid(X, self.w)
self.w -= self.epsilon * np.dot(X.T, (sigma - y)) / y.shape[0]
def predict(self, X):
X = np.concatenate(
(np.ones((X.shape[0], 1)), X),
axis=1
)
return [i >= 0.5 for i in self.sigmoid(X, self.w)]
def get_params(self, deep):
return dict(epsilon=self.epsilon)
def set_params(self, **kwargs):
for key in kwargs:
setattr(self, key, kwargs[key])
return self
def start():
print("Read data from csv")
data = pd.read_csv("train.csv")
data = data.iloc[0:1000]
print("Prepare data")
X, y, X_test, y_test = prepare(data)
print("-" * 20, "\nStart grid search")
f1_scorer = make_scorer(f1_score, average='macro')
pipeline_scaler_regression = make_pipeline(SGDLogisticRegression())
pipeline = GridSearchCV(
pipeline_scaler_regression,
dict(sgdlogisticregression__epsilon=np.geomspace(0.0001, 1, num=13)),
# dict(sgdlogisticregression__epsilon=[1]),
scoring=f1_scorer,
error_score='raise'
)
pipeline.fit(X.toarray(), y.values)
print("Start predict")
y_predict = pipeline.predict(X_test.toarray())
f1 = skl_m.f1_score(y_test.values, y_predict, average='macro')
print("f1:", f1)
print("Create graphic")
epsilons = []
results = []
for eps in [0.0001, 0.001, 0.01, 0.1, 1]:
pipeline_scaler_regression = make_pipeline(StandardScaler(), SGDLogisticRegression())
pipeline = GridSearchCV(
pipeline_scaler_regression,
dict(sgdlinearregression__epsilon=[eps]),
scoring=f1_scorer
)
pipeline.fit(X, y)
epsilons.append(eps)
results.append(
skl_m.f1_score(y_test, y_predict, average='macro')
)
plt.plot(epsilons, results)
plt.show()
pipeline_sqdclass = SGDClassifier(loss='log',
penalty='l1',
epsilon=53,
random_state=42,
tol=None
)
pipeline = GridSearchCV(
pipeline_scaler_regression,
dict(sgdlinearregression__epsilon=[eps]),
scoring=f1_scorer
)
pipeline.fit(X, y)
print("SGD classifier:\t", skl_m.f1_score(y_test, y_predict, average='macro'))
if __name__ == '__main__':
start()