-
Notifications
You must be signed in to change notification settings - Fork 38
/
feature_process_helper.py
132 lines (114 loc) · 5.64 KB
/
feature_process_helper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
def names(train, test):
for i in [train, test]:
i['Name_Len'] = i['Name'].apply(lambda x: len(x))
i['Name_Title'] = i['Name'].apply(lambda x: x.split(',')[1]).apply(lambda x: x.split()[0])
del i['Name']
return train, test
def age_impute(train, test):
for i in [train, test]:
i['Age_Null_Flag'] = i['Age'].apply(lambda x: 1 if pd.isnull(x) else 0)
train['mean'] = train.groupby(['Name_Title', 'Pclass'])['Age'].transform('mean')
train['Age'] = train['Age'].fillna(train['mean'])
z = test.merge(train, on=['Name_Title', 'Pclass'], how='left').drop_duplicates(['PassengerId_x'])
test['Age'] = np.where(test['Age'].isnull(), z['mean'], test['Age'])
test['Age'] = test['Age'].fillna(test['Age'].mean())
del train['mean']
return train, test
def age_impute2(train, test):
for i in [train, test]:
i['Age_Null_Flag'] = i['Age'].apply(lambda x: 1 if pd.isnull(x) else 0)
train['mean'] = train.groupby(['Name_Title', 'Pclass'])['Age'].transform('mean')
train['Age'] = train['Age'].fillna(train['mean'])
z = test.merge(train, on=['Name_Title', 'Pclass'], how='left').drop_duplicates(['PassengerId_x'])
test['Age'] = np.where(test['Age'].isnull(), z['mean'], test['Age'])
test['Age'] = test['Age'].fillna(test['Age'].mean())
del train['mean']
return train, test
def cabin(train, test):
for i in [train, test]:
i['Cabin_Letter'] = i['Cabin'].apply(lambda x: str(x)[0])
del i['Cabin']
return train, test
def embarked_impute(train, test):
for i in [train, test]:
i['Embarked'] = i['Embarked'].fillna('S')
return train, test
def dummies(train, test, columns = ['Pclass', 'Sex', 'Embarked', 'Cabin_Letter', 'Name_Title']):
for column in columns:
train[column] = train[column].apply(lambda x: str(x))
test[column] = test[column].apply(lambda x: str(x))
good_cols = [column+'_'+i for i in train[column].unique() if i in test[column].unique()]
train = pd.concat((train, pd.get_dummies(train[column], prefix = column)[good_cols]), axis = 1)
test = pd.concat((test, pd.get_dummies(test[column], prefix = column)[good_cols]), axis = 1)
del train[column]
del test[column]
return train, test
def drop(train, test, bye = ['Ticket', 'SibSp', 'Parch']):
for i in [train, test]:
for z in bye:
del i[z]
return train, test
def fam_size(train, test):
for i in [train, test]:
i['Fam_Size'] = np.where((i['SibSp']+i['Parch']) == 0 , 'Solo',
np.where((i['SibSp']+i['Parch']) <= 3,'Nuclear', 'Big'))
del i['SibSp']
del i['Parch']
return train, test
def lda(X_train, X_test, y_train, cols=['Age', 'Fare']):
sc = StandardScaler()
X_train_std = sc.fit_transform(X_train[cols])
X_test_std = sc.transform(X_test[cols])
lda = LDA(n_components=None)
X_train_lda = lda.fit_transform(X_train_std, y_train.values.ravel())
X_test_lda = lda.transform(X_test_std)
X_train = pd.concat((X_train, pd.DataFrame(X_train_lda)), axis=1)
X_test = pd.concat((X_test, pd.DataFrame(X_test_lda)), axis=1)
for i in cols:
del X_train[i]
del X_test[i]
return X_train, X_test
def titles_grouped(train, test):
for i in [train, test]:
i['Name_Title'] = np.where((i['Name_Title']).isin(['Mr.', 'Mrs.', 'Miss.', 'Master.', 'Dr.']),
i['Name_Title'], 'other')
return train, test
def ticket_grouped(train, test):
for i in [train, test]:
i['Ticket_Lett'] = i['Ticket'].apply(lambda x: str(x)[0])
i['Ticket_Lett'] = i['Ticket_Lett'].apply(lambda x: str(x))
i['Ticket_Lett'] = np.where((i['Ticket_Lett']).isin(['1', '2', '3', 'S', 'P', 'C', 'A']), i['Ticket_Lett'],
np.where((i['Ticket_Lett']).isin(['W', '4', '7', '6', 'L', '5', '8']),
'Low_ticket', 'Other_ticket'))
i['Ticket_Len'] = i['Ticket'].apply(lambda x: len(x))
del i['Ticket']
return train, test
def ticket_grouped2(train, test):
for i in [train, test]:
i['Ticket_Lett'] = i['Ticket'].apply(lambda x: str(x)[0])
i['Ticket_Lett'] = i['Ticket_Lett'].apply(lambda x: str(x))
i['Ticket_Lett'] = np.where((i['Ticket_Lett']).isin(['1', '2', '3', 'S', 'P', 'C', 'A']), i['Ticket_Lett'],
np.where((i['Ticket_Lett']).isin(['W', '4', '7', '6', 'L', '5', '8']),
'Low_ticket', 'Other_ticket'))
i['Ticket_Len'] = i['Ticket'].apply(lambda x: len(x))
del i['Ticket']
return train, test
def cabin_num(train, test):
for i in [train, test]:
i['Cabin_num1'] = i['Cabin'].apply(lambda x: str(x).split(' ')[-1][1:])
i['Cabin_num1'].replace('an', np.NaN, inplace = True)
i['Cabin_num1'] = i['Cabin_num1'].apply(lambda x: int(x) if not pd.isnull(x) and x <> '' else np.NaN)
train['Cabin_num'], bins = pd.qcut(train['Cabin_num1'],3, retbins=True)
test['Cabin_num'] = pd.cut(test['Cabin_num1'], bins=bins, include_lowest=True)
#i['Cabin_num'] = i['Cabin_num'].isnull().apply(lambda x: float(x))
train = pd.concat((train, pd.get_dummies(train['Cabin_num'], prefix = 'Cabin_num')), axis = 1)
test = pd.concat((test, pd.get_dummies(test['Cabin_num'], prefix = 'Cabin_num')), axis = 1)
del train['Cabin_num']
del test['Cabin_num']
del train['Cabin_num1']
del test['Cabin_num1']
return train, test