-
Notifications
You must be signed in to change notification settings - Fork 0
/
neural_network5_analysis_old.py
99 lines (81 loc) · 3.38 KB
/
neural_network5_analysis_old.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import pickle
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
idx = pd.IndexSlice
def run_analysis():
with open('./nn_output/neural_network5_nn_results_dict.pkl', 'rb') as f:
results = pickle.load(f)
#print(results)
raw_data = results['two_features']
# get all curves into a list of np.arrays
#curves_list = []
curves_df = pd.DataFrame.from_dict(
{(i, j, k): raw_data[i][j][k]['curve']
for i in raw_data.keys()
for j in raw_data[i].keys()
for k in raw_data[i][j].keys()
}, orient='index'
)
curves_df.index = pd.MultiIndex.from_tuples(curves_df.index,
names=('algorithm', 'hyperparameters', 'run'))
avg_curves = curves_df.mean(level=0)
fig, ax = plt.subplots(figsize=(6,4))
#print(avg_curves)
avg_curves.T.plot(ax=ax, color=['C0', 'C1', 'C2', 'C3'])
curves_df.T.plot(ax=ax, color=['C0']*3 + ['C1']*3 + ['C2']*3 + ['C3']*3, style=':', legend=False)
ax.set_ylim(-8, 0)
ax.set_xlabel('iteration')
ax.set_ylabel('Negative Log Loss')
plt.title('Fitness curves for means and individual runs per Algorithm')
plt.savefig('./nn_plots/nn_total_fitness_curve.png')
fig, ax = plt.subplots(figsize=(6,4))
#print(avg_curves)
avg_curves.T.plot(ax=ax, color=['C0', 'C1', 'C2', 'C3'])
curves_df.T.plot(ax=ax, color=['C0']*3 + ['C1']*3 + ['C2']*3 + ['C3']*3, style=':', legend=False)
ax.set_ylim(-3, 0)
ax.set_xlim(0, 2000)
ax.set_xlabel('iteration')
ax.set_ylabel('Negative Log Loss')
plt.title('Zoom In: Fitness curves for means and individual runs per Algorithm')
plt.savefig('./nn_plots/nn_zoom_fitness_curve.png')
fit_time_df = pd.DataFrame
ba_df = pd.DataFrame.from_dict(
{(i, j, k): raw_data[i][j][k]['balanced_accuracy']
for i in raw_data.keys()
for j in raw_data[i].keys()
for k in raw_data[i][j].keys()
}, orient='index'
)
ba_df.index = pd.MultiIndex.from_tuples(curves_df.index,
names=('algorithm', 'hyperparameters', 'run'))
ft_df = pd.DataFrame.from_dict(
{(i, j, k): raw_data[i][j][k]['fit_time']
for i in raw_data.keys()
for j in raw_data[i].keys()
for k in raw_data[i][j].keys()
}, orient='index'
)
ft_df.index = pd.MultiIndex.from_tuples(curves_df.index,
names=('algorithm', 'hyperparameters', 'run'))
combined = pd.merge(left=ba_df,
right=ft_df,
how='left',
left_index=True,
right_index=True
)
combined.columns = ['balanced_accuracy', 'fit_time']
fig, ax = plt.subplots(figsize=(6, 4))
plt.subplots_adjust(bottom=.15)
for algo in raw_data.keys():
df = combined.loc[idx[algo], :]
ax.scatter(df['fit_time'], df['balanced_accuracy'], label=algo)
ax.set_xscale('log')
plt.xlabel('time (seconds)')
plt.ylabel('Balanced Accuracy')
plt.suptitle('Fitness vs Time for each algorithm with Neural Networks')
plt.legend()
plt.plot()
plt.show()
if __name__=="__main__":
run_analysis()