Fix TSEMO (#32)

Complete overhaul of the TSEMO strategy (not called TSEMO2 anymore). Also add multiobjective test functions DTLZ2 and VLMOP2.
sustainable-processes · Jul 17, 2020 · f33590c · f33590c
1 parent 6dcd243
commit f33590c
Show file tree

Hide file tree

Showing 252 changed files with 25,910 additions and 1,168 deletions.
diff --git a/.gitignore b/.gitignore
@@ -111,3 +111,8 @@ venv.bak/
 ~*.xlsx
 *.DS_STORE
 .vscode/
+
+# TSEMO
+.TSEMO_DATA
+tmp_files
+Pytest*
diff --git a/case_studies/web/web.py b/case_studies/web/web.py
@@ -1,7 +1,7 @@
 from summit.initial_design import LatinDesigner
 from summit.data import DataSet
 from summit.domain import ContinuousVariable, Constraint, Domain
-from summit.strategies import TSEMO2
+from summit.strategies import TSEMO
 from summit.models import GPyModel
 
 import pandas as pd
@@ -93,7 +93,7 @@ def get_suggestions(project, experiments, num_experiments):
         logging.info(f"Running optimization for project {project_id}")
         # Run the optimization
         try:
-            tsemo = TSEMO2(domain=domain, models=models)
+            tsemo = TSEMOs(domain=domain, models=models)
             results = tsemo.generate_experiments(
                 previous_results=data, num_experiments=num_experiments
             )

diff --git a/docs/TSEMO_DTLZ2.png b/docs/TSEMO_DTLZ2.png
diff --git a/docs/dtlz2_y_matlab.csv b/docs/dtlz2_y_matlab.csv
@@ -0,0 +1,130 @@
+1.4442,0.22432
+1.4533,0.53214
+0.45625,1.2313
+1.0269,1.2743
+0.63816,1.0093
+1.009,1.1832
+1.2078,0.7444
+1.1929,0.61732
+1.0596,0.87457
+0.090487,1.3285
+0.21613,1.2802
+1.2037,1.1661
+0.89772,0.90311
+0.5458,1.3357
+1.3279,0.3355
+1.1478,0.1849
+0.018182,1.5369
+0.40613,1.3041
+1.2372,0.055481
+0.66423,1.337
+1.1926,0.93439
+1.1685,0.788
+1.5741,0.67902
+0.32477,1.476
+0.52918,1.017
+0.20911,1.3909
+1.1333,0.54573
+0.82998,1.2537
+1.4758,0.11945
+1.383,0.41736
+0.1228,1.1079
+1.3807,0.047972
+1.0469,0.84804
+0.35095,1.095
+1.3212,0.049186
+1.4457,0.066752
+1.1787,0
+1.3153,0.24169
+0.24355,1.2215
+1.3496,0.10212
+1.5305,0.10687
+1.0599,0.019637
+0.11157,1.0469
+0.54901,0.93802
+1.3386,0.7536
+1.4766,0.36087
+0.1902,1.2242
+0.27021,1.0619
+1.5265,0.021097
+1.4092,0.0074445
+1.3606,0.0011906
+0.90683,0.74933
+1.0302,0.12365
+1.0333,0.12916
+1.3424,0.11521
+1.0703,0.52541
+0.12172,1.0677
+1.2282,0.033581
+1.1152,0.007508
+1.3533,0.20025
+0.94415,0.3911
+0.13634,1.3162
+0.13804,1.0729
+0.086997,1.221
+1.15,0.25471
+0.67567,0.97718
+0.8505,0.62903
+0.14219,1.3247
+0.11277,1.0665
+0.32813,0.98071
+0.3664,1.1825
+0.99659,0.34934
+6.8222e-17,1.1142
+0.093814,1.0044
+0.10887,1.2202
+0.068832,1.1018
+0.097343,1.2403
+0.031594,1.0549
+0.009302,1.0272
+0.78323,0.6585
+0.75363,0.72306
+6.5452e-17,1.0689
+0.037734,1.0623
+6.3659e-17,1.0396
+6.9688e-17,1.1381
+1.0542,0
+0.45135,1.0437
+1.0678,0
+6.839e-17,1.1169
+6.2972e-17,1.0284
+1.0373,0.017992
+1.1672,0.068738
+0.66986,0.92676
+0.0055809,1.0268
+1.0523,0.046351
+0.44203,0.92618
+0.047159,1.0647
+1.0355,0.19555
+1.0343,0.20912
+0.007092,1.2329
+0.5454,0.86933
+1.0595,0.022365
+1.0161,0.055607
+1.0267,0.091026
+0.08821,1.0292
+1.0046,0.15263
+6.4649e-17,1.0558
+1.0376,0
+0.024267,1.0225
+0.71996,0.72518
+6.1772e-17,1.0088
+1.0101,0
+1.0075,0.083267
+6.3202e-17,1.0322
+6.2054e-17,1.0134
+1.0563,0
+0.87087,0.54394
+6.1415e-17,1.003
+1.0482,0
+0.28818,0.97122
+6.4e-17,1.0452
+0.23693,1.0168
+1.0324,0.032295
+0.97589,0.39199
+1.036,0.019428
+0.96258,0.28787
+6.4772e-17,1.0578
+0.10451,1.006
+1.0101,0.17254
+1.109,0
diff --git a/docs/dtlz2_y_python.csv b/docs/dtlz2_y_python.csv
diff --git a/docs/multiobjective.ipynb b/docs/multiobjective.ipynb
diff --git a/docs/snar_benchmark.ipynb b/docs/snar_benchmark.ipynb
diff --git a/docs/transforms.ipynb b/docs/transforms.ipynb
@@ -140,6 +140,13 @@
     "ax.tick_params(direction='in')\n",
     "plt.show(fig)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

diff --git a/experiments/tsemo/.TSEMO_DATA/models.h5 b/experiments/tsemo/.TSEMO_DATA/models.h5
diff --git a/experiments/tsemo/2020-07-11_train_gp_matlab_data_boxplot_100_restarts_sampling.csv b/experiments/tsemo/2020-07-11_train_gp_matlab_data_boxplot_100_restarts_sampling.csv
@@ -0,0 +1,11 @@
+,rmse_train_y0,rmse_train_y1,rmse_test_y0,rmse_test_y1,rmse_train_spectral_y0,rmse_train_spectral_y1,rmse_test_spectral_y0,rmse_test_spectral_y1,objective_y0,objective_y1
+0,0.0023967260359035163,0.0028009133327277507,0.17820776536965363,0.2602862995317092,0.3570494437342601,1.0217466303421214,0.359071953453995,1.1844212274590702,46.10712802210574,44.30702338811433
+1,0.0023967368059518166,0.002800937593490912,0.1782075997441569,0.2602864712984501,0.25798946153679614,0.9843567602101819,0.34358040226003955,1.0848299793099314,46.10712802253512,44.307023390441316
+2,0.00239674618771439,0.002800917037054622,0.1782074797374011,0.2602862678876559,0.21196249151725927,0.7467274339665004,0.28502263149409396,0.786891846022866,46.10712802657551,44.30702338816976
+3,0.0023967282876490525,0.0028008975582849535,0.178207773716562,0.2602841005922434,0.36765607254963434,0.9982183467407343,0.41639818207384677,1.1031814143179162,46.10712802206446,44.30702340025243
+4,0.0023967271789552733,0.0028009091828776127,0.17820757028452122,0.260286362336545,0.3925690016294729,0.8732664861284314,0.4334202129631925,1.0628361571212281,46.10712802245874,44.30702338821708
+5,0.0023967280791820015,0.0028009430299798447,0.17820776090032775,0.26027979412824204,0.21826875405506574,0.7849330497357729,0.32255958677867863,0.7589694591671052,46.107128022184824,44.30702352199966
+6,0.0023967295755134943,0.0028009145606611003,0.1782078332927343,0.26028640244026,0.26917237706257485,0.8929242587618657,0.4884721478430218,0.9911809466484311,46.107128022180206,44.307023388148565
+7,0.002396730319389694,0.0028009127589555564,0.1782075925386086,0.26028636461799537,0.20100418917477558,0.9261201928485173,0.25710950628775775,0.9610207161585145,46.10712802291616,44.30702338813383
+8,0.0023967341215180775,0.002800913743189143,0.17820807728550164,0.26028639042102797,0.3489658913071267,1.01504009108453,0.43803643887249066,1.0994786415278852,46.10712802403817,44.307023388120925
+9,0.002396740858522284,0.0028009160023145367,0.17820799910328228,0.26028626418258705,0.19656821036918073,0.9478944477070335,0.32466111342908155,1.1311372592875606,46.10712802587247,44.307023388325334
diff --git a/...riments/tsemo/2020-07-11_train_gp_matlab_data_boxplot_100_restarts_sampling.png b/...riments/tsemo/2020-07-11_train_gp_matlab_data_boxplot_100_restarts_sampling.png
diff --git a/experiments/tsemo/2020-07-11_train_gp_matlab_data_boxplot_1_restarts_sampling.csv b/experiments/tsemo/2020-07-11_train_gp_matlab_data_boxplot_1_restarts_sampling.csv
@@ -0,0 +1,6 @@
+,rmse_train_y0,rmse_train_y1,rmse_test_y0,rmse_test_y1,rmse_train_spectral_y0,rmse_train_spectral_y1,rmse_test_spectral_y0,rmse_test_spectral_y1,objective_y0,objective_y1
+0,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,0.4272685552633198,0.6518887060190612,0.5545578415418995,0.6920221612143477,63.84467842061074,63.64682031068847
+1,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,0.49226793144122,0.5764306501647667,0.5739421142113373,0.6895448393982515,63.84467842061074,63.64682031068847
+2,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,0.2852157280032307,0.6657746921220241,0.32017332165867474,0.7119504043655513,63.84467842061074,63.64682031068847
+3,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,0.3817000093518604,0.6791740090534725,0.49997315031944695,0.7076970643096143,63.84467842061074,63.64682031068847
+4,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,0.434117982609039,0.622896309112811,0.4606261473071154,0.8377025671108096,63.84467842061074,63.64682031068847
diff --git a/experiments/tsemo/2020-07-11_train_gp_matlab_data_boxplot_1_restarts_sampling.png b/experiments/tsemo/2020-07-11_train_gp_matlab_data_boxplot_1_restarts_sampling.png
diff --git a/experiments/tsemo/2020-07-11_train_gp_matlab_data_pyrff_100_restarts_sampling.csv b/experiments/tsemo/2020-07-11_train_gp_matlab_data_pyrff_100_restarts_sampling.csv
@@ -0,0 +1,11 @@
+,rmse_train_y0,rmse_train_y1,rmse_test_y0,rmse_test_y1,rmse_train_spectral_y0,rmse_train_spectral_y1,rmse_test_spectral_y0,rmse_test_spectral_y1,objective_y0,objective_y1
+0,0.00239672857159928,0.0028009153363530727,0.1782077390822616,0.2602860313953371,0.10930358394261555,0.8618436586423488,0.22066520171080475,0.8997640966366522,46.10712802235808,44.30702338825369
+1,0.0025541225174027745,0.0028008883141937614,0.1762222837677833,0.2602864946732744,0.09406442310670152,0.9056209002964065,0.2672015412826641,1.0216131959751684,53.28155838698292,44.30702339495526
+2,0.002396729151818394,0.0028009136191755684,0.1782078335462268,0.2602864131822919,0.09600694304932705,0.8823595467407418,0.22519446237411941,1.0348657351346067,46.107128022222156,44.30702338817725
+3,0.0023967320684916524,0.0028009051347779148,0.1782077365576733,0.26028616422956946,0.0971167457404649,0.8653936478975549,0.2805612306195307,1.0593883603616228,46.107128022116655,44.307023388276995
+4,0.0023967356015194836,0.00280092784549182,0.1782077725402637,0.2602881428958813,0.07771483045862543,0.8745783567509324,0.3029973781649687,1.0737681553629812,46.10712802211286,44.30702339865161
+5,0.002396732700487983,0.002800914649584779,0.17820771504900146,0.2602863260964604,0.11665780923364917,0.8652842304904743,0.27290286046581397,0.9643583566294193,46.1071280222082,44.307023388429904
+6,0.002396674049862903,0.0028009096374788504,0.1782083219695028,0.26028817841506197,0.08747039245670289,0.8680482539104791,0.24823670644719184,0.9619631946470065,46.10712803338987,44.30702340362053
+7,0.002396710082865535,0.003023403702970029,0.17820819584946473,0.2538600721759,0.1023171918244976,0.8863236237044916,0.22892250622543947,0.9541500143048917,46.107128027570866,51.53731957986594
+8,0.0023967282889315695,0.0028008576400962355,0.17820760315621972,0.26028647824814327,0.0815822639659009,0.8696342608259777,0.2339394363545513,0.9362873548565661,46.107128025675024,44.30702339691576
+9,0.002396730071732103,0.0028009170021378567,0.17820774482586293,0.2602863095313672,0.09488231045695139,0.8354875032631268,0.2877056847036122,1.002753729201382,46.10712802384664,44.30702338821837
diff --git a/...s/tsemo/2020-07-11_train_gp_matlab_data_pyrff_boxplot_100_restarts_sampling.png b/...s/tsemo/2020-07-11_train_gp_matlab_data_pyrff_boxplot_100_restarts_sampling.png
diff --git a/experiments/tsemo/20200604_matlab_tsemo_comparison.png b/experiments/tsemo/20200604_matlab_tsemo_comparison.png
diff --git a/experiments/tsemo/20200709_train_gp_matlab_data_no_sample.png b/experiments/tsemo/20200709_train_gp_matlab_data_no_sample.png
diff --git a/experiments/tsemo/20200709_train_gp_matlab_data_sample.png b/experiments/tsemo/20200709_train_gp_matlab_data_sample.png
diff --git a/experiments/tsemo/20200710_train_gp_matlab_data_boxplot_100_restarts.png b/experiments/tsemo/20200710_train_gp_matlab_data_boxplot_100_restarts.png
diff --git a/...710_train_gp_matlab_data_boxplot_100_restarts_sampling_1500_spectral_sample.png b/...710_train_gp_matlab_data_boxplot_100_restarts_sampling_1500_spectral_sample.png
diff --git a/experiments/tsemo/20200710_train_gp_matlab_data_boxplot_10_restarts.png b/experiments/tsemo/20200710_train_gp_matlab_data_boxplot_10_restarts.png
diff --git a/experiments/tsemo/20200710_train_gp_matlab_data_boxplot_no_restarts.png b/experiments/tsemo/20200710_train_gp_matlab_data_boxplot_no_restarts.png
diff --git a/experiments/tsemo/20200710_train_gp_matlab_data_comparison_data_100_restarts.csv b/experiments/tsemo/20200710_train_gp_matlab_data_comparison_data_100_restarts.csv
@@ -0,0 +1,21 @@
+,rmse_train_y0,rmse_train_y1,rmse_test_y0,rmse_test_y1,objective_y0,objective_y1
+0,0.0023967444079334758,0.002800890639875926,0.17820767018958855,0.2602865247405828,46.107128023018866,44.30702338914011
+1,0.0023967272925905082,0.0028009323310313737,0.17820779574636209,0.26028698354928315,46.10712802208177,44.30702339297203
+2,0.0023966316256719437,0.0028009202691540976,0.17820959830942995,0.26028690793512194,46.10712807148278,44.30702338886053
+3,0.0023967254039335703,0.002800959505333087,0.17820778340090182,0.2602868448823239,46.10712802215031,44.3070234029716
+4,0.0023967425998196153,0.0028009342284789234,0.17820735038454688,0.2602864024829602,46.10712802702614,44.30702338875906
+5,0.002396721107326527,0.002800924663711571,0.17820790575159234,0.260286415984117,46.107128022563046,44.30702338826791
+6,0.0023967128507002347,0.00280088712234274,0.1782078223099631,0.2602865184055487,46.10712802284476,44.307023392118644
+7,0.0023967278761933375,0.002800885550119041,0.17820779572902712,0.2602861476579643,46.10712802209062,44.30702339035584
+8,0.002396581461319782,0.00280091554025392,0.17820799169522755,0.2602859096101463,46.10712839621159,44.307023392508356
+9,0.0023967453154439947,0.002800913423011518,0.17820787767794463,0.2602863038615794,46.10712802307106,44.30702338821296
+10,0.0023967285009043075,0.002800926919085807,0.17820785393134156,0.2602854136007956,46.107128022499744,44.307023389842435
+11,0.002396738739449592,0.002800918466817923,0.17820784211750246,0.2602860826125411,46.10712802241618,44.30702339038997
+12,0.002396734458334592,0.0028008959142370366,0.17820719865901627,0.26028645339513423,46.10712803683225,44.307023410859884
+13,0.0023967607024571918,0.0028008930560401305,0.17820820172432048,0.2602867168812863,46.10712803070278,44.30702341808191
+14,0.0023967299823361952,0.002800924492568037,0.17820773345201205,0.26028627949665667,46.10712802222007,44.3070233945769
+15,0.002396726982702712,0.002800913521849701,0.1782078377982063,0.2602863245712512,46.10712802288874,44.307023388105186
+16,0.0023967304207168035,0.0028009208643660906,0.1782075641272776,0.2602863369816836,46.10712802278948,44.30702338913193
+17,0.0023967345232216246,0.0028008989578546593,0.17820768791723057,0.2602863909379048,46.10712802220209,44.30702338863301
+18,0.002396727962492575,0.002800914334626474,0.17820763993088506,0.2602863046116862,46.107128022743055,44.307023388117805
+19,0.002396714713599493,0.0028009078848452735,0.17820752307645077,0.2602880374034777,46.107128036426815,44.30702340129854
diff --git a/experiments/tsemo/20200710_train_gp_matlab_data_comparison_data_100_restarts_sampling.csv b/experiments/tsemo/20200710_train_gp_matlab_data_comparison_data_100_restarts_sampling.csv
@@ -0,0 +1,6 @@
+,rmse_train_y0,rmse_train_y1,rmse_test_y0,rmse_test_y1,rmse_train_spectral_y0,rmse_train_spectral_y1,rmse_test_spectral_y0,rmse_test_spectral_y1,objective_y0,objective_y1
+0,0.002554208599061451,0.0028008910075870337,0.17622115468341074,0.2602880882420602,0.19542901452517533,0.9809018659727423,0.23638623857576846,1.1316168203893924,52.826849512873004,44.30702339616958
+1,0.0025541564053824925,0.0028006938079722502,0.17622322521792846,0.26028440809376985,0.2780230540369667,0.9200200201838712,0.2841531963224192,1.0756881477974214,52.081719058659125,44.307023492523314
+2,0.00263157969630347,0.0030664868396832246,0.1755748918034649,0.27379321886938274,0.23786197720690627,1.0846981862930292,0.3261404803367641,1.1447295001168314,54.051389513709076,55.2762561301051
+3,0.002554114874673168,0.003023460329809938,0.17622121125851758,0.25386018305815183,0.2907804450745195,0.9252267528772365,0.33397101002187823,1.091838342724364,53.6619811771968,56.512231039259355
+4,0.0023967290500535895,0.00280096313018135,0.17820777564791127,0.2602865422325946,0.2496164980110655,1.033184123674759,0.2861814991831469,1.1202254337713393,46.10712802206486,44.307023405778104
diff --git a/experiments/tsemo/20200710_train_gp_matlab_data_comparison_data_10_restarts.csv b/experiments/tsemo/20200710_train_gp_matlab_data_comparison_data_10_restarts.csv
@@ -0,0 +1,21 @@
+,rmse_train_y0,rmse_train_y1,rmse_test_y0,rmse_test_y1,objective_y0,objective_y1
+0,0.21579344154527952,0.002801094717753581,0.3222074047162828,0.2602899724763069,63.84467842061074,44.30702358638203
+1,0.0029099982117576253,0.22406602653334173,0.18964413103975467,0.35831143597104775,63.68856928346478,63.64682031068847
+2,0.21579344154527952,0.0028008127293703907,0.3222074047162828,0.2602873310812293,63.84467842061074,44.307023420001414
+3,0.21579344154527952,0.003066526622516523,0.3222074047162828,0.27379361593605644,63.84467842061074,51.690893879502724
+4,0.00254835676366701,0.003093879366342448,0.1740623473487139,0.25600034139226163,56.016785879271296,52.66735089913
+5,0.2157938054082276,0.002800939461788531,0.32220802746398647,0.2602852168720173,63.84465656664746,44.30702339512317
+6,0.21579344154527952,0.00319464534172263,0.3222074047162828,0.26782399966181863,63.84467842061074,52.91402620658131
+7,0.002631597521750959,0.0028008812245441197,0.17557393263811324,0.26028529785309923,53.425071727842386,44.30702356675823
+8,0.21579344154527952,0.0028009350137389497,0.3222074047162828,0.26028272538767294,63.84467842061074,44.307023429901896
+9,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+10,0.002548332719680543,0.0028008126535818876,0.17406242495598928,0.26028448726846554,53.0242574084611,44.30702390139444
+11,0.21579344154527952,0.0028008881275804875,0.3222074047162828,0.2602856895438654,63.84467842061074,44.307023395378835
+12,0.00239668822654135,0.22406602653334173,0.17820920490520906,0.35831143597104775,46.10712805544631,63.64682031068847
+13,0.21579344154527952,0.0028008886046314687,0.3222074047162828,0.2602837840791877,63.84467842061074,44.30702347617111
+14,0.21579344154527952,0.0030037750697595954,0.3222074047162828,0.2507873604381538,63.84467842061074,54.48630126251105
+15,0.21579344154527952,0.0028008228700890946,0.3222074047162828,0.26028666583482224,63.84467842061074,44.3070234022044
+16,0.002396730195649604,0.0028009486030555774,0.17820785120741003,0.26028597928041086,46.10712802218016,44.30702340011041
+17,0.21579344154527952,0.002800922051957487,0.3222074047162828,0.2602863427930061,63.84467842061074,44.307023684314494
+18,0.0025483425274623944,0.0030002491360249517,0.1740626669213917,0.25078431305110604,53.48757239346427,53.00766194196815
+19,0.002396732752177754,0.0028008864806561497,0.17820773164748985,0.2602851123050184,46.10712802217166,44.30702342785626
diff --git a/experiments/tsemo/20200710_train_gp_matlab_data_comparison_data_no_restarts.csv b/experiments/tsemo/20200710_train_gp_matlab_data_comparison_data_no_restarts.csv
@@ -0,0 +1,21 @@
+,rmse_train_y0,rmse_train_y1,rmse_test_y0,rmse_test_y1,objective_y0,objective_y1
+0,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+1,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+2,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+3,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+4,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+5,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+6,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+7,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+8,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+9,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+10,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+11,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+12,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+13,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+14,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+15,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+16,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+17,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+18,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
+19,0.21579344154527952,0.22406602653334173,0.3222074047162828,0.35831143597104775,63.84467842061074,63.64682031068847
diff --git a/experiments/tsemo/20200710_train_gp_matlab_data_no_sample.png b/experiments/tsemo/20200710_train_gp_matlab_data_no_sample.png
diff --git a/experiments/tsemo/3x3_lhs_matlab.csv b/experiments/tsemo/3x3_lhs_matlab.csv
@@ -0,0 +1,15 @@
+0.56735,0.083249,0.71
+0.63696,0.31633,0.05356
+0.090246,0.83294,0.38326
+0.68604,0.68673,0.15893
+0.41635,0.94061,0.23511
+0.048265,0.40271,0.90989
+0.22135,0.030186,0.61128
+0.35633,0.79076,0.42765
+0.85583,0.92338,0.96335
+0.9254,0.51617,0.072187
+0.16678,0.61062,0.58094
+0.93602,0.58305,0.48285
+0.77731,0.21238,0.74975
+0.29432,0.38377,0.8413
+0.51841,0.13805,0.29548
diff --git a/experiments/tsemo/acc_time_res_study.npy b/experiments/tsemo/acc_time_res_study.npy
diff --git a/experiments/tsemo/acc_time_tradeoff_spectral_sampling.png b/experiments/tsemo/acc_time_tradeoff_spectral_sampling.png