add baseline results

hannanabdul55 · Nov 26, 2020 · 3d603d1 · 3d603d1
1 parent 96672d5
commit 3d603d1
Show file tree

Hide file tree

Showing 38 changed files with 44 additions and 0 deletions.
diff --git a/logs/res8769568.err b/logs/res8769568.err
@@ -0,0 +1,2 @@
+2020-11-25 16:05:37,416	INFO services.py:1164 -- View the Ray dashboard at [1m[32mhttp://127.0.0.1:8265[39m[22m
+slurmstepd: error: *** JOB 8769568 ON swarm011 CANCELLED AT 2020-11-25T16:18:00 DUE TO TIME LIMIT ***
diff --git a/logs/res8769569.err b/logs/res8769569.err
@@ -0,0 +1,2 @@
+2020-11-25 16:05:51,739	INFO services.py:1164 -- View the Ray dashboard at [1m[32mhttp://127.0.0.1:8265[39m[22m
+slurmstepd: error: *** JOB 8769569 ON swarm012 CANCELLED AT 2020-11-25T16:17:59 DUE TO TIME LIMIT ***
diff --git a/logs/res8769570.err b/logs/res8769570.err
@@ -0,0 +1 @@
+2020-11-25 16:32:39,657	INFO services.py:1164 -- View the Ray dashboard at [1m[32mhttp://127.0.0.1:8265[39m[22m
diff --git a/logs/res8769571.err b/logs/res8769571.err
@@ -0,0 +1 @@
+2020-11-25 16:33:01,576	INFO services.py:1164 -- View the Ray dashboard at [1m[32mhttp://127.0.0.1:8265[39m[22m
diff --git a/logs/res_8769568.txt b/logs/res_8769568.txt
diff --git a/logs/res_8769569.txt b/logs/res_8769569.txt
diff --git a/logs/res_8769570.txt b/logs/res_8769570.txt
@@ -0,0 +1,19 @@
+Running experiment with checkpoint: results-1606339959.1371334
+Config used: {'name': 'cmaes_double_delta_hf', 'N': [200, 600, 1000, 4000, 25000, 78000, 100000], 'trials': 30, 'methods': 'hoeffdings', 'D': 5, 'tprs': [0.3, 0.9], 'test_size': 0.4, 'stratify': True, 'opt': 'CMAES'}
+False
+[2m[36m(pid=32807)[0m Results for N=200: 
+[2m[36m(pid=32807)[0m {'N': 200, 'opt': 'CMAES', 'n': 200, 'trials': 29, 'failure_rate': 0.2, 'failure_rate_std': 0.4, 'sol_found_rate': 0.6, 'sol_found_rate_std': 0.48989794855663565, 'accuracy': 0.4881260555555556, 'ghat': -0.101941680986588, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.7814151666666664, 'uc_ghat': 0.5714554530221138, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=32846)[0m Results for N=600: 
+[2m[36m(pid=32846)[0m {'N': 600, 'opt': 'CMAES', 'n': 600, 'trials': 29, 'failure_rate': 0.03333333333333333, 'failure_rate_std': 0.17950549357115012, 'sol_found_rate': 0.7333333333333333, 'sol_found_rate_std': 0.4422166387140533, 'accuracy': 0.48576175, 'ghat': -0.10057866190134732, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8010258333333335, 'uc_ghat': 0.598194123245181, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=32803)[0m Results for N=1000: 
+[2m[36m(pid=32803)[0m {'N': 1000, 'opt': 'CMAES', 'n': 1000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 0.9666666666666667, 'sol_found_rate_std': 0.1795054935711501, 'accuracy': 0.47643737777777784, 'ghat': -0.13007322136130045, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001033333333332, 'uc_ghat': 0.6000391004904616, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=32802)[0m Results for N=4000: 
+[2m[36m(pid=32802)[0m {'N': 4000, 'opt': 'CMAES', 'n': 4000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 0.9666666666666667, 'sol_found_rate_std': 0.17950549357115014, 'accuracy': 0.4866342000000001, 'ghat': -0.10647052550241022, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=32835)[0m Results for N=25000: 
+[2m[36m(pid=32835)[0m {'N': 25000, 'opt': 'CMAES', 'n': 25000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 0.9666666666666667, 'sol_found_rate_std': 0.17950549357115012, 'accuracy': 0.5097756111111111, 'ghat': -0.05666382107601553, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=32804)[0m Results for N=78000: 
+[2m[36m(pid=32804)[0m {'N': 78000, 'opt': 'CMAES', 'n': 78000, 'trials': 29, 'failure_rate': 0.03333333333333333, 'failure_rate_std': 0.17950549357115012, 'sol_found_rate': 0.9666666666666667, 'sol_found_rate_std': 0.17950549357115012, 'accuracy': 0.5004099888888889, 'ghat': -0.07818310870278515, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
+saving results
+Time run: 7548 seconds
+[2m[36m(pid=32808)[0m Results for N=100000: 
+[2m[36m(pid=32808)[0m {'N': 100000, 'opt': 'CMAES', 'n': 100000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 1.0, 'sol_found_rate_std': 0.0, 'accuracy': 0.4997039222222222, 'ghat': -0.08206338989291982, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
diff --git a/logs/res_8769571.txt b/logs/res_8769571.txt
@@ -0,0 +1,19 @@
+Running experiment with checkpoint: results-1606339980.8639772
+Config used: {'name': 'cmaes_double_delta_baseline', 'N': [200, 600, 1000, 4000, 25000, 78000, 100000], 'trials': 30, 'methods': 'ttest', 'D': 5, 'tprs': [0.3, 0.9], 'test_size': 0.4, 'stratify': True, 'opt': 'CMAES'}
+False
+[2m[36m(pid=34748)[0m Results for N=600: 
+[2m[36m(pid=34748)[0m {'N': 600, 'opt': 'CMAES', 'n': 600, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 0.8333333333333334, 'sol_found_rate_std': 0.37267799624996495, 'accuracy': 0.47488491111111114, 'ghat': -0.1175313406757112, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=34752)[0m Results for N=200: 
+[2m[36m(pid=34752)[0m {'N': 200, 'opt': 'CMAES', 'n': 200, 'trials': 29, 'failure_rate': 0.03333333333333333, 'failure_rate_std': 0.17950549357115012, 'sol_found_rate': 0.8, 'sol_found_rate_std': 0.4, 'accuracy': 0.4559311777777778, 'ghat': -0.18022088939096925, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.7931096666666666, 'uc_ghat': 0.5883002452696756, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=34776)[0m Results for N=1000: 
+[2m[36m(pid=34776)[0m {'N': 1000, 'opt': 'CMAES', 'n': 1000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 0.8666666666666667, 'sol_found_rate_std': 0.33993463423951903, 'accuracy': 0.4765560833333333, 'ghat': -0.12611868615490632, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001038333333336, 'uc_ghat': 0.6000381005027945, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=34759)[0m Results for N=4000: 
+[2m[36m(pid=34759)[0m {'N': 4000, 'opt': 'CMAES', 'n': 4000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 1.0, 'sol_found_rate_std': 0.0, 'accuracy': 0.49323309444444446, 'ghat': -0.09430158527510733, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=34756)[0m Results for N=25000: 
+[2m[36m(pid=34756)[0m {'N': 25000, 'opt': 'CMAES', 'n': 25000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 1.0, 'sol_found_rate_std': 0.0, 'accuracy': 0.5037596611111111, 'ghat': -0.07331094861281072, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
+[2m[36m(pid=34762)[0m Results for N=78000: 
+[2m[36m(pid=34762)[0m {'N': 78000, 'opt': 'CMAES', 'n': 78000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 1.0, 'sol_found_rate_std': 0.0, 'accuracy': 0.5058756222222223, 'ghat': -0.06666271106232537, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
+saving results
+Time run: 7329 seconds
+[2m[36m(pid=34774)[0m Results for N=100000: 
+[2m[36m(pid=34774)[0m {'N': 100000, 'opt': 'CMAES', 'n': 100000, 'trials': 29, 'failure_rate': 0.0, 'failure_rate_std': 0.0, 'sol_found_rate': 1.0, 'sol_found_rate_std': 0.0, 'accuracy': 0.5015003666666666, 'ghat': -0.07570666704889364, 'uc_failure_rate': 1.0, 'uc_failure_rate_std': 0.0, 'uc_accuracy': 0.8001031666666666, 'uc_ghat': 0.6000394338196837, 'method': 'ttest', 'device': device(type='cpu')}
diff --git a/result/result_cmaes_double_delta_baseline/final_res_results-1606339980.8639772.p b/result/result_cmaes_double_delta_baseline/final_res_results-1606339980.8639772.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606338336.756615_1000.p b/result/result_cmaes_double_delta_baseline/results-1606338336.756615_1000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606338336.756615_100000.p b/result/result_cmaes_double_delta_baseline/results-1606338336.756615_100000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606338336.756615_200.p b/result/result_cmaes_double_delta_baseline/results-1606338336.756615_200.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606338336.756615_25000.p b/result/result_cmaes_double_delta_baseline/results-1606338336.756615_25000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606338336.756615_4000.p b/result/result_cmaes_double_delta_baseline/results-1606338336.756615_4000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606338336.756615_600.p b/result/result_cmaes_double_delta_baseline/results-1606338336.756615_600.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606338336.756615_78000.p b/result/result_cmaes_double_delta_baseline/results-1606338336.756615_78000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_1000.p b/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_1000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_100000.p b/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_100000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_200.p b/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_200.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_25000.p b/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_25000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_4000.p b/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_4000.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_600.p b/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_600.p
diff --git a/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_78000.p b/result/result_cmaes_double_delta_baseline/results-1606339980.8639772_78000.p
diff --git a/result/result_cmaes_double_delta_hf/final_res_results-1606339959.1371334.p b/result/result_cmaes_double_delta_hf/final_res_results-1606339959.1371334.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606338350.864698_1000.p b/result/result_cmaes_double_delta_hf/results-1606338350.864698_1000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606338350.864698_100000.p b/result/result_cmaes_double_delta_hf/results-1606338350.864698_100000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606338350.864698_200.p b/result/result_cmaes_double_delta_hf/results-1606338350.864698_200.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606338350.864698_25000.p b/result/result_cmaes_double_delta_hf/results-1606338350.864698_25000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606338350.864698_4000.p b/result/result_cmaes_double_delta_hf/results-1606338350.864698_4000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606338350.864698_600.p b/result/result_cmaes_double_delta_hf/results-1606338350.864698_600.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606338350.864698_78000.p b/result/result_cmaes_double_delta_hf/results-1606338350.864698_78000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606339959.1371334_1000.p b/result/result_cmaes_double_delta_hf/results-1606339959.1371334_1000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606339959.1371334_100000.p b/result/result_cmaes_double_delta_hf/results-1606339959.1371334_100000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606339959.1371334_200.p b/result/result_cmaes_double_delta_hf/results-1606339959.1371334_200.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606339959.1371334_25000.p b/result/result_cmaes_double_delta_hf/results-1606339959.1371334_25000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606339959.1371334_4000.p b/result/result_cmaes_double_delta_hf/results-1606339959.1371334_4000.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606339959.1371334_600.p b/result/result_cmaes_double_delta_hf/results-1606339959.1371334_600.p
diff --git a/result/result_cmaes_double_delta_hf/results-1606339959.1371334_78000.p b/result/result_cmaes_double_delta_hf/results-1606339959.1371334_78000.p