diff --git a/csrank/tuning.py b/csrank/tuning.py
index 3e269d67..a4fca2a4 100644
--- a/csrank/tuning.py
+++ b/csrank/tuning.py
@@ -179,13 +179,13 @@ def splitter_dict(itr_dict):
))
if "use_early_stopping" in self._ranker_params:
self._ranker_class._use_early_stopping = self._ranker_params["use_early_stopping"]
+ param_ranges = self._ranker_class.set_tunable_parameter_ranges(parameters_ranges)
if (optimizer is not None):
opt = optimizer
self.logger.debug('Setting the provided optimizer')
self.log_best_params(opt)
else:
- param_ranges = self._ranker_class.set_tunable_parameter_ranges(parameters_ranges)
transformed = []
for param in param_ranges:
transformed.append(check_dimension(param))
diff --git a/experiments/experiment_cv.py b/experiments/experiment_cv.py
index 0b9e6ee1..109c87d3 100644
--- a/experiments/experiment_cv.py
+++ b/experiments/experiment_cv.py
@@ -40,7 +40,7 @@
get_loss_for_array)
from experiments.util import get_ranker_and_dataset_functions, get_ranker_parameters, ERROR_OUTPUT_STRING, \
lp_metric_dict, get_duration_microsecond, get_applicable_ranker_dataset, get_dataset_str, \
- log_test_train_data
+ log_test_train_data, get_optimizer
DIR_PATH = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
@@ -97,32 +97,8 @@
optimizer_path = os.path.join(DIR_PATH, OPTIMIZER_FOLDER,
(FILE_FORMAT).format(dataset_str, ranker_name, cluster_index))
create_dir_recursively(optimizer_path, True)
- logger.info('Retrieving model stored at: {}'.format(optimizer_path))
- try:
- optimizer = load(optimizer_path)
- logger.info('Loading model stored at: {}'.format(optimizer_path))
-
- except KeyError:
- logger.error('Cannot open the file {}'.format(optimizer_path))
- optimizer = None
-
- except ValueError:
- logger.error('Cannot open the file {}'.format(optimizer_path))
- optimizer = None
- except FileNotFoundError:
- logger.error('No such file or directory: {}'.format(optimizer_path))
- optimizer = None
- if optimizer is not None:
- finished_iterations = np.array(optimizer.yi).shape[0]
- if finished_iterations == 0:
- optimizer = None
- logger.info('Optimizer did not finish any iterations so setting optimizer to null')
- else:
- n_iter = n_iter - finished_iterations
- if n_iter < 0:
- n_iter = 0
- logger.info(
- 'Iterations already done: {} and running iterations {}'.format(finished_iterations, n_iter))
+
+ optimizer, n_iter = get_optimizer(logger, optimizer_path, n_iter)
optimizer_fit_params = {'n_iter': n_iter, 'cv_iter': inner_cv, 'optimizer': optimizer,
"parameters_ranges": parameter_ranges, 'acq_func': 'EIps'}
diff --git a/experiments/experiment_script.py b/experiments/experiment_script.py
index e0e9140b..71c3d687 100644
--- a/experiments/experiment_script.py
+++ b/experiments/experiment_script.py
@@ -28,14 +28,14 @@
import pandas as pd
from docopt import docopt
from sklearn.model_selection import ShuffleSplit
-from skopt import load
from csrank.tuning import ParameterOptimizer
from csrank.util import create_dir_recursively, configure_logging_numpy_keras, \
duration_tillnow, microsec_to_time, get_mean_loss_for_dictionary, \
get_loss_for_array
from experiments.util import get_ranker_and_dataset_functions, get_ranker_parameters, ERROR_OUTPUT_STRING, \
- lp_metric_dict, get_duration_microsecond, get_applicable_ranker_dataset, get_dataset_str, log_test_train_data
+ lp_metric_dict, get_duration_microsecond, get_applicable_ranker_dataset, get_dataset_str, log_test_train_data, \
+ get_optimizer
DIR_PATH = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
@@ -96,31 +96,7 @@
create_dir_recursively(optimizer_path, True)
create_dir_recursively(pred_file, is_file_path=True)
- logger.info('Retrieving model stored at: {}'.format(optimizer_path))
- try:
- optimizer = load(optimizer_path)
- logger.info('Loading model stored at: {}'.format(optimizer_path))
-
- except KeyError:
- logger.error('Cannot open the file {}'.format(optimizer_path))
- optimizer = None
-
- except ValueError:
- logger.error('Cannot open the file {}'.format(optimizer_path))
- optimizer = None
- except FileNotFoundError:
- logger.error('No such file or directory: {}'.format(optimizer_path))
- optimizer = None
- if optimizer is not None:
- finished_iterations = np.array(optimizer.yi).shape[0]
- if finished_iterations == 0:
- optimizer = None
- logger.info('Optimizer did not finish any iterations so setting optimizer to null')
- else:
- n_iter = n_iter - finished_iterations
- if n_iter < 0:
- n_iter = 0
- logger.info('Iterations already done: {} and running iterations {}'.format(finished_iterations, n_iter))
+ optimizer, n_iter = get_optimizer(logger, optimizer_path, n_iter)
if not (n_iter == 0 and os.path.isfile(pred_file)):
optimizer_fit_params = {'n_iter': n_iter, 'cv_iter': cv, 'optimizer': optimizer,
"parameters_ranges": parameter_ranges, 'acq_func': 'EIps'}
diff --git a/experiments/util.py b/experiments/util.py
index 73316b8f..e52aca7f 100644
--- a/experiments/util.py
+++ b/experiments/util.py
@@ -1,9 +1,11 @@
import re
from collections import OrderedDict
+import numpy as np
from keras.losses import categorical_crossentropy
from keras.metrics import categorical_accuracy
from keras.optimizers import SGD
+from skopt import load
from csrank.callbacks import DebugOutput, LRScheduler
from csrank.constants import OBJECT_RANKING, LABEL_RANKING, DYAD_RANKING, DISCRETE_CHOICE, BATCH_SIZE, LEARNING_RATE, \
@@ -185,3 +187,32 @@ def log_test_train_data(X_train, X_test, logger):
n_instances, n_objects, n_features = X_train.shape
logger.info("Train Set instances {} objects {} features {}".format(n_instances, n_objects, n_features))
return n_features, n_objects
+
+
+def get_optimizer(logger, optimizer_path, n_iter):
+ logger.info('Retrieving model stored at: {}'.format(optimizer_path))
+ try:
+ optimizer = load(optimizer_path)
+ logger.info('Loading model stored at: {}'.format(optimizer_path))
+
+ except KeyError:
+ logger.error('Cannot open the file {}'.format(optimizer_path))
+ optimizer = None
+
+ except ValueError:
+ logger.error('Cannot open the file {}'.format(optimizer_path))
+ optimizer = None
+ except FileNotFoundError:
+ logger.error('No such file or directory: {}'.format(optimizer_path))
+ optimizer = None
+ if optimizer is not None:
+ finished_iterations = np.array(optimizer.yi).shape[0]
+ if finished_iterations == 0:
+ optimizer = None
+ logger.info('Optimizer did not finish any iterations so setting optimizer to null')
+ else:
+ n_iter = n_iter - finished_iterations
+ if n_iter < 0:
+ n_iter = 0
+ logger.info('Iterations already done: {} and running iterations {}'.format(finished_iterations, n_iter))
+ return optimizer, n_iter
\ No newline at end of file
diff --git a/notebooks/GeneralizationOfExperiments.ipynb b/notebooks/GeneralizationOfExperiments.ipynb
index 6057f1e1..1b8ccdaa 100644
--- a/notebooks/GeneralizationOfExperiments.ipynb
+++ b/notebooks/GeneralizationOfExperiments.ipynb
@@ -3,17 +3,15 @@
{
"cell_type": "code",
"execution_count": 1,
- "metadata": {},
+ "metadata": {
+ "collapsed": false
+ },
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/home/prithag/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
- " from ._conv import register_converters as _register_converters\n",
- "Using TensorFlow backend.\n",
- "/home/prithag/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
- " return f(*args, **kwds)\n"
+ "Using TensorFlow backend.\n"
]
}
],
@@ -29,7 +27,8 @@
"from csrank.callbacks import DebugOutput\n",
"from csrank.metrics import zero_one_rank_loss_for_scores\n",
"from csrank.util import rename_file_if_exist, configure_logging_numpy_keras, get_tensor_value\n",
- "from csrank.dataset_reader import SyntheticDatasetGenerator"
+ "from csrank.dataset_reader import SyntheticDatasetGenerator\n",
+ "from keras.callbacks import History"
]
},
{
@@ -48,7 +47,10 @@
"outputs": [],
"source": [
"MODEL = \"aModel\"\n",
- "ERROR_OUTPUT_STRING = 'Out of sample error {} : {} for n_objects {}'"
+ "ERROR_OUTPUT_STRING = 'Out of sample error {} : {} for n_objects {}'\n",
+ "his = History()\n",
+ "his.__name__ = \"History\"\n",
+ "objects = \"Objects\""
]
},
{
@@ -67,8 +69,10 @@
"outputs": [],
"source": [
"def generate_dataset(n_objects=5, random_state=42):\n",
- " parameters = {\"n_features\": 2, \"n_objects\": n_objects, \"n_train_instances\": 10000, \"n_test_instances\": 100000,\n",
- " \"dataset_type\": \"medoid\",\"random_state\":random_state}\n",
+ " parameters = {\"n_features\": 2, \"n_objects\": n_objects, \n",
+ " \"n_train_instances\": 10000, \"n_test_instances\": 100000,\n",
+ " \"dataset_type\": \"medoid\",\n",
+ " \"random_state\":random_state}\n",
" generator = SyntheticDatasetGenerator(**parameters)\n",
" return generator.get_single_train_test_split()"
]
@@ -89,9 +93,9 @@
"outputs": [],
"source": [
"def get_evaluation_result(gor, X_train, Y_train, epochs):\n",
- " gor.fit(X_train, Y_train, log_callbacks=[DebugOutput(delta=10)], verbose=False, epochs=epochs)\n",
+ " gor.fit(X_train, Y_train, log_callbacks=[DebugOutput(delta=10), his], verbose=False, epochs=epochs)\n",
" eval_results = {}\n",
- " for n_objects in np.arange(3, 15):\n",
+ " for n_objects in np.arange(3, 20):\n",
" _, _, X_test, Y_test = generate_dataset(n_objects=n_objects, random_state=seed + n_objects * 5)\n",
" y_pred_scores = gor.predict_scores(X_test, batch_size=X_test.shape[0])\n",
" metric_loss = get_tensor_value(zero_one_rank_loss_for_scores(Y_test, y_pred_scores))\n",
@@ -144,7 +148,7 @@
"X_train, Y_train, _, _ = generate_dataset(n_objects=n_objects, random_state=seed)\n",
"n_instances, n_objects, n_features = X_train.shape\n",
"\n",
- "epochs = 50\n",
+ "epochs = 700\n",
"params = {\"n_objects\": n_objects, \n",
" \"n_features\": n_features, \n",
" \"n_object_features\": n_features, \n",
@@ -161,19 +165,16 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
"logger.info(\"############################# With Best Parameters FETA ##############################\")\n",
- "best_point = [1, 16, 4.2054947998521569e-05, 2.6263496065703243e-10, 777]\n",
"gor = FETANetwork(**params)\n",
- "gor.set_tunable_parameter_ranges({})\n",
- "gor.set_tunable_parameters(best_point)\n",
"result = get_evaluation_result(gor, X_train, Y_train, epochs)\n",
- "result[MODEL] = \"FETARanker\"\n",
+ "result[MODEL] = \"FETARankerDefault\"\n",
"rows_list.append(result)"
]
},
@@ -188,73 +189,1265 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
"from csrank.losses import smooth_rank_loss\n",
"logger.info(\"############################# With Best Parameters FATE ##############################\")\n",
- "best_point = [1003, 0.0002908115170179143, 16, 132, 6, 247, 3.4195015492773324e-05]\n",
"gor = FATEObjectRanker(**params)\n",
- "gor.set_tunable_parameter_ranges({})\n",
- "gor.set_tunable_parameters(best_point)\n",
"result = get_evaluation_result(gor, X_train, Y_train, epochs)\n",
- "result[MODEL] = \"FATERanker\"\n",
+ "result[MODEL] = \"FATERankerDefault\"\n",
"rows_list.append(result)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Save the results into a dataframe"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 70,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " aModel | \n",
+ " FATERankerDefault | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 3 | \n",
+ " 0.218561 | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " 0.947871 | \n",
+ "
\n",
+ " \n",
+ " 5 | \n",
+ " 0.692061 | \n",
+ "
\n",
+ " \n",
+ " 6 | \n",
+ " 0.669744 | \n",
+ "
\n",
+ " \n",
+ " 7 | \n",
+ " 0.932732 | \n",
+ "
\n",
+ " \n",
+ " 8 | \n",
+ " 0.804914 | \n",
+ "
\n",
+ " \n",
+ " 9 | \n",
+ " 0.941135 | \n",
+ "
\n",
+ " \n",
+ " 10 | \n",
+ " 0.964858 | \n",
+ "
\n",
+ " \n",
+ " 11 | \n",
+ " 0.970023 | \n",
+ "
\n",
+ " \n",
+ " 12 | \n",
+ " 0.090049 | \n",
+ "
\n",
+ " \n",
+ " 13 | \n",
+ " 0.503294 | \n",
+ "
\n",
+ " \n",
+ " 14 | \n",
+ " 0.228598 | \n",
+ "
\n",
+ " \n",
+ " 15 | \n",
+ " 0.287196 | \n",
+ "
\n",
+ " \n",
+ " 16 | \n",
+ " 0.000157 | \n",
+ "
\n",
+ " \n",
+ " 17 | \n",
+ " 0.060977 | \n",
+ "
\n",
+ " \n",
+ " 18 | \n",
+ " 0.381702 | \n",
+ "
\n",
+ " \n",
+ " 19 | \n",
+ " 0.617856 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ "aModel FATERankerDefault\n",
+ "3 0.218561\n",
+ "4 0.947871\n",
+ "5 0.692061\n",
+ "6 0.669744\n",
+ "7 0.932732\n",
+ "8 0.804914\n",
+ "9 0.941135\n",
+ "10 0.964858\n",
+ "11 0.970023\n",
+ "12 0.090049\n",
+ "13 0.503294\n",
+ "14 0.228598\n",
+ "15 0.287196\n",
+ "16 0.000157\n",
+ "17 0.060977\n",
+ "18 0.381702\n",
+ "19 0.617856"
+ ]
+ },
+ "execution_count": 70,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"df = pd.DataFrame(rows_list)\n",
+ "df = df.set_index(MODEL).T\n",
+ "cols = list(df.columns.values)\n",
"df"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If the there is an eisting csv file saved then load it and add the new columns containing the results to it"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 18,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Objects | \n",
+ " FETARankerBestParams | \n",
+ " FETARanker | \n",
+ " FATERanker | \n",
+ " FATERanker10SetLayers | \n",
+ " FATERanker8SetLayers | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 | \n",
+ " 3 | \n",
+ " 0.198587 | \n",
+ " 0.194951 | \n",
+ " 0.082433 | \n",
+ " 0.065285 | \n",
+ " 0.094951 | \n",
+ "
\n",
+ " \n",
+ " 1 | \n",
+ " 4 | \n",
+ " 0.226475 | \n",
+ " 0.228402 | \n",
+ " 0.141888 | \n",
+ " 0.120779 | \n",
+ " 0.125498 | \n",
+ "
\n",
+ " \n",
+ " 2 | \n",
+ " 5 | \n",
+ " 0.218070 | \n",
+ " 0.224131 | \n",
+ " 0.109125 | \n",
+ " 0.103804 | \n",
+ " 0.107595 | \n",
+ "
\n",
+ " \n",
+ " 3 | \n",
+ " 6 | \n",
+ " 0.215545 | \n",
+ " 0.227914 | \n",
+ " 0.133907 | \n",
+ " 0.109337 | \n",
+ " 0.107254 | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " 7 | \n",
+ " 0.211587 | \n",
+ " 0.228970 | \n",
+ " 0.134549 | \n",
+ " 0.110699 | \n",
+ " 0.107393 | \n",
+ "
\n",
+ " \n",
+ " 5 | \n",
+ " 8 | \n",
+ " 0.207721 | \n",
+ " 0.230498 | \n",
+ " 0.142830 | \n",
+ " 0.112166 | \n",
+ " 0.107801 | \n",
+ "
\n",
+ " \n",
+ " 6 | \n",
+ " 9 | \n",
+ " 0.203656 | \n",
+ " 0.230520 | \n",
+ " 0.147908 | \n",
+ " 0.113533 | \n",
+ " 0.110187 | \n",
+ "
\n",
+ " \n",
+ " 7 | \n",
+ " 10 | \n",
+ " 0.200119 | \n",
+ " 0.230697 | \n",
+ " 0.153753 | \n",
+ " 0.115446 | \n",
+ " 0.112828 | \n",
+ "
\n",
+ " \n",
+ " 8 | \n",
+ " 11 | \n",
+ " 0.196479 | \n",
+ " 0.230296 | \n",
+ " 0.157704 | \n",
+ " 0.116400 | \n",
+ " 0.115010 | \n",
+ "
\n",
+ " \n",
+ " 9 | \n",
+ " 12 | \n",
+ " 0.193027 | \n",
+ " 0.230299 | \n",
+ " 0.161388 | \n",
+ " 0.116445 | \n",
+ " 0.116744 | \n",
+ "
\n",
+ " \n",
+ " 10 | \n",
+ " 13 | \n",
+ " 0.189883 | \n",
+ " 0.229609 | \n",
+ " 0.164661 | \n",
+ " 0.117337 | \n",
+ " 0.119210 | \n",
+ "
\n",
+ " \n",
+ " 11 | \n",
+ " 14 | \n",
+ " 0.186184 | \n",
+ " 0.229495 | \n",
+ " 0.167531 | \n",
+ " 0.117598 | \n",
+ " 0.121214 | \n",
+ "
\n",
+ " \n",
+ " 12 | \n",
+ " 15 | \n",
+ " 0.182840 | \n",
+ " 0.227923 | \n",
+ " 0.170489 | \n",
+ " 0.117623 | \n",
+ " 0.123238 | \n",
+ "
\n",
+ " \n",
+ " 13 | \n",
+ " 16 | \n",
+ " 0.179923 | \n",
+ " 0.227556 | \n",
+ " 0.172129 | \n",
+ " 0.117890 | \n",
+ " 0.125474 | \n",
+ "
\n",
+ " \n",
+ " 14 | \n",
+ " 17 | \n",
+ " 0.177795 | \n",
+ " 0.226943 | \n",
+ " 0.174690 | \n",
+ " 0.118051 | \n",
+ " 0.127573 | \n",
+ "
\n",
+ " \n",
+ " 15 | \n",
+ " 18 | \n",
+ " 0.174808 | \n",
+ " 0.226251 | \n",
+ " 0.176415 | \n",
+ " 0.117521 | \n",
+ " 0.129286 | \n",
+ "
\n",
+ " \n",
+ " 16 | \n",
+ " 19 | \n",
+ " 0.173107 | \n",
+ " 0.225330 | \n",
+ " 0.178538 | \n",
+ " 0.118120 | \n",
+ " 0.131830 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " Objects FETARankerBestParams FETARanker FATERanker \\\n",
+ "0 3 0.198587 0.194951 0.082433 \n",
+ "1 4 0.226475 0.228402 0.141888 \n",
+ "2 5 0.218070 0.224131 0.109125 \n",
+ "3 6 0.215545 0.227914 0.133907 \n",
+ "4 7 0.211587 0.228970 0.134549 \n",
+ "5 8 0.207721 0.230498 0.142830 \n",
+ "6 9 0.203656 0.230520 0.147908 \n",
+ "7 10 0.200119 0.230697 0.153753 \n",
+ "8 11 0.196479 0.230296 0.157704 \n",
+ "9 12 0.193027 0.230299 0.161388 \n",
+ "10 13 0.189883 0.229609 0.164661 \n",
+ "11 14 0.186184 0.229495 0.167531 \n",
+ "12 15 0.182840 0.227923 0.170489 \n",
+ "13 16 0.179923 0.227556 0.172129 \n",
+ "14 17 0.177795 0.226943 0.174690 \n",
+ "15 18 0.174808 0.226251 0.176415 \n",
+ "16 19 0.173107 0.225330 0.178538 \n",
+ "\n",
+ " FATERanker10SetLayers FATERanker8SetLayers \n",
+ "0 0.065285 0.094951 \n",
+ "1 0.120779 0.125498 \n",
+ "2 0.103804 0.107595 \n",
+ "3 0.109337 0.107254 \n",
+ "4 0.110699 0.107393 \n",
+ "5 0.112166 0.107801 \n",
+ "6 0.113533 0.110187 \n",
+ "7 0.115446 0.112828 \n",
+ "8 0.116400 0.115010 \n",
+ "9 0.116445 0.116744 \n",
+ "10 0.117337 0.119210 \n",
+ "11 0.117598 0.121214 \n",
+ "12 0.117623 0.123238 \n",
+ "13 0.117890 0.125474 \n",
+ "14 0.118051 0.127573 \n",
+ "15 0.117521 0.129286 \n",
+ "16 0.118120 0.131830 "
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "cols = list(df.columns.values)\n",
- "cols = cols[-7:] + cols[:-7]\n",
- "MODEL = \"aModel\"\n",
- "for x in ['Unnamed: 0', 'aModel']:\n",
- " if x in cols:\n",
- " cols.remove(x)\n",
- " cols.insert(0, x)\n",
- "df = df[cols]\n",
- "#del df['Unnamed: 0']\n",
- "df = df.set_index(MODEL).T"
+ "if not os.path.isfile(df_path):\n",
+ " dataFrame = df\n",
+ "else:\n",
+ " dataFrame = pd.read_csv(df_path, index_col=0)\n",
+ " dataFrame = dataFrame.append(df, ignore_index=True)\n",
+ "dataFrame"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Save the dataframe to given file path"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 71,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
- "df.to_csv(df_path)"
+ "dataFrame.to_csv(df_path, index=objects)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Plot the results of the zero one rank accuracy"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 17,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "/* Put everything inside the global mpl namespace */\n",
+ "window.mpl = {};\n",
+ "\n",
+ "\n",
+ "mpl.get_websocket_type = function() {\n",
+ " if (typeof(WebSocket) !== 'undefined') {\n",
+ " return WebSocket;\n",
+ " } else if (typeof(MozWebSocket) !== 'undefined') {\n",
+ " return MozWebSocket;\n",
+ " } else {\n",
+ " alert('Your browser does not have WebSocket support.' +\n",
+ " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
+ " 'Firefox 4 and 5 are also supported but you ' +\n",
+ " 'have to enable WebSockets in about:config.');\n",
+ " };\n",
+ "}\n",
+ "\n",
+ "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n",
+ " this.id = figure_id;\n",
+ "\n",
+ " this.ws = websocket;\n",
+ "\n",
+ " this.supports_binary = (this.ws.binaryType != undefined);\n",
+ "\n",
+ " if (!this.supports_binary) {\n",
+ " var warnings = document.getElementById(\"mpl-warnings\");\n",
+ " if (warnings) {\n",
+ " warnings.style.display = 'block';\n",
+ " warnings.textContent = (\n",
+ " \"This browser does not support binary websocket messages. \" +\n",
+ " \"Performance may be slow.\");\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " this.imageObj = new Image();\n",
+ "\n",
+ " this.context = undefined;\n",
+ " this.message = undefined;\n",
+ " this.canvas = undefined;\n",
+ " this.rubberband_canvas = undefined;\n",
+ " this.rubberband_context = undefined;\n",
+ " this.format_dropdown = undefined;\n",
+ "\n",
+ " this.image_mode = 'full';\n",
+ "\n",
+ " this.root = $('');\n",
+ " this._root_extra_style(this.root)\n",
+ " this.root.attr('style', 'display: inline-block');\n",
+ "\n",
+ " $(parent_element).append(this.root);\n",
+ "\n",
+ " this._init_header(this);\n",
+ " this._init_canvas(this);\n",
+ " this._init_toolbar(this);\n",
+ "\n",
+ " var fig = this;\n",
+ "\n",
+ " this.waiting = false;\n",
+ "\n",
+ " this.ws.onopen = function () {\n",
+ " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n",
+ " fig.send_message(\"send_image_mode\", {});\n",
+ " if (mpl.ratio != 1) {\n",
+ " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n",
+ " }\n",
+ " fig.send_message(\"refresh\", {});\n",
+ " }\n",
+ "\n",
+ " this.imageObj.onload = function() {\n",
+ " if (fig.image_mode == 'full') {\n",
+ " // Full images could contain transparency (where diff images\n",
+ " // almost always do), so we need to clear the canvas so that\n",
+ " // there is no ghosting.\n",
+ " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
+ " }\n",
+ " fig.context.drawImage(fig.imageObj, 0, 0);\n",
+ " };\n",
+ "\n",
+ " this.imageObj.onunload = function() {\n",
+ " fig.ws.close();\n",
+ " }\n",
+ "\n",
+ " this.ws.onmessage = this._make_on_message_function(this);\n",
+ "\n",
+ " this.ondownload = ondownload;\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype._init_header = function() {\n",
+ " var titlebar = $(\n",
+ " '');\n",
+ " var titletext = $(\n",
+ " '');\n",
+ " titlebar.append(titletext)\n",
+ " this.root.append(titlebar);\n",
+ " this.header = titletext[0];\n",
+ "}\n",
+ "\n",
+ "\n",
+ "\n",
+ "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n",
+ "\n",
+ "}\n",
+ "\n",
+ "\n",
+ "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n",
+ "\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype._init_canvas = function() {\n",
+ " var fig = this;\n",
+ "\n",
+ " var canvas_div = $('');\n",
+ "\n",
+ " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n",
+ "\n",
+ " function canvas_keyboard_event(event) {\n",
+ " return fig.key_event(event, event['data']);\n",
+ " }\n",
+ "\n",
+ " canvas_div.keydown('key_press', canvas_keyboard_event);\n",
+ " canvas_div.keyup('key_release', canvas_keyboard_event);\n",
+ " this.canvas_div = canvas_div\n",
+ " this._canvas_extra_style(canvas_div)\n",
+ " this.root.append(canvas_div);\n",
+ "\n",
+ " var canvas = $('');\n",
+ " canvas.addClass('mpl-canvas');\n",
+ " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n",
+ "\n",
+ " this.canvas = canvas[0];\n",
+ " this.context = canvas[0].getContext(\"2d\");\n",
+ "\n",
+ " var backingStore = this.context.backingStorePixelRatio ||\n",
+ "\tthis.context.webkitBackingStorePixelRatio ||\n",
+ "\tthis.context.mozBackingStorePixelRatio ||\n",
+ "\tthis.context.msBackingStorePixelRatio ||\n",
+ "\tthis.context.oBackingStorePixelRatio ||\n",
+ "\tthis.context.backingStorePixelRatio || 1;\n",
+ "\n",
+ " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
+ "\n",
+ " var rubberband = $('');\n",
+ " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n",
+ "\n",
+ " var pass_mouse_events = true;\n",
+ "\n",
+ " canvas_div.resizable({\n",
+ " start: function(event, ui) {\n",
+ " pass_mouse_events = false;\n",
+ " },\n",
+ " resize: function(event, ui) {\n",
+ " fig.request_resize(ui.size.width, ui.size.height);\n",
+ " },\n",
+ " stop: function(event, ui) {\n",
+ " pass_mouse_events = true;\n",
+ " fig.request_resize(ui.size.width, ui.size.height);\n",
+ " },\n",
+ " });\n",
+ "\n",
+ " function mouse_event_fn(event) {\n",
+ " if (pass_mouse_events)\n",
+ " return fig.mouse_event(event, event['data']);\n",
+ " }\n",
+ "\n",
+ " rubberband.mousedown('button_press', mouse_event_fn);\n",
+ " rubberband.mouseup('button_release', mouse_event_fn);\n",
+ " // Throttle sequential mouse events to 1 every 20ms.\n",
+ " rubberband.mousemove('motion_notify', mouse_event_fn);\n",
+ "\n",
+ " rubberband.mouseenter('figure_enter', mouse_event_fn);\n",
+ " rubberband.mouseleave('figure_leave', mouse_event_fn);\n",
+ "\n",
+ " canvas_div.on(\"wheel\", function (event) {\n",
+ " event = event.originalEvent;\n",
+ " event['data'] = 'scroll'\n",
+ " if (event.deltaY < 0) {\n",
+ " event.step = 1;\n",
+ " } else {\n",
+ " event.step = -1;\n",
+ " }\n",
+ " mouse_event_fn(event);\n",
+ " });\n",
+ "\n",
+ " canvas_div.append(canvas);\n",
+ " canvas_div.append(rubberband);\n",
+ "\n",
+ " this.rubberband = rubberband;\n",
+ " this.rubberband_canvas = rubberband[0];\n",
+ " this.rubberband_context = rubberband[0].getContext(\"2d\");\n",
+ " this.rubberband_context.strokeStyle = \"#000000\";\n",
+ "\n",
+ " this._resize_canvas = function(width, height) {\n",
+ " // Keep the size of the canvas, canvas container, and rubber band\n",
+ " // canvas in synch.\n",
+ " canvas_div.css('width', width)\n",
+ " canvas_div.css('height', height)\n",
+ "\n",
+ " canvas.attr('width', width * mpl.ratio);\n",
+ " canvas.attr('height', height * mpl.ratio);\n",
+ " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n",
+ "\n",
+ " rubberband.attr('width', width);\n",
+ " rubberband.attr('height', height);\n",
+ " }\n",
+ "\n",
+ " // Set the figure to an initial 600x600px, this will subsequently be updated\n",
+ " // upon first draw.\n",
+ " this._resize_canvas(600, 600);\n",
+ "\n",
+ " // Disable right mouse context menu.\n",
+ " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n",
+ " return false;\n",
+ " });\n",
+ "\n",
+ " function set_focus () {\n",
+ " canvas.focus();\n",
+ " canvas_div.focus();\n",
+ " }\n",
+ "\n",
+ " window.setTimeout(set_focus, 100);\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype._init_toolbar = function() {\n",
+ " var fig = this;\n",
+ "\n",
+ " var nav_element = $('')\n",
+ " nav_element.attr('style', 'width: 100%');\n",
+ " this.root.append(nav_element);\n",
+ "\n",
+ " // Define a callback function for later on.\n",
+ " function toolbar_event(event) {\n",
+ " return fig.toolbar_button_onclick(event['data']);\n",
+ " }\n",
+ " function toolbar_mouse_event(event) {\n",
+ " return fig.toolbar_button_onmouseover(event['data']);\n",
+ " }\n",
+ "\n",
+ " for(var toolbar_ind in mpl.toolbar_items) {\n",
+ " var name = mpl.toolbar_items[toolbar_ind][0];\n",
+ " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
+ " var image = mpl.toolbar_items[toolbar_ind][2];\n",
+ " var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
+ "\n",
+ " if (!name) {\n",
+ " // put a spacer in here.\n",
+ " continue;\n",
+ " }\n",
+ " var button = $('');\n",
+ " button.addClass('ui-button ui-widget ui-state-default ui-corner-all ' +\n",
+ " 'ui-button-icon-only');\n",
+ " button.attr('role', 'button');\n",
+ " button.attr('aria-disabled', 'false');\n",
+ " button.click(method_name, toolbar_event);\n",
+ " button.mouseover(tooltip, toolbar_mouse_event);\n",
+ "\n",
+ " var icon_img = $('');\n",
+ " icon_img.addClass('ui-button-icon-primary ui-icon');\n",
+ " icon_img.addClass(image);\n",
+ " icon_img.addClass('ui-corner-all');\n",
+ "\n",
+ " var tooltip_span = $('');\n",
+ " tooltip_span.addClass('ui-button-text');\n",
+ " tooltip_span.html(tooltip);\n",
+ "\n",
+ " button.append(icon_img);\n",
+ " button.append(tooltip_span);\n",
+ "\n",
+ " nav_element.append(button);\n",
+ " }\n",
+ "\n",
+ " var fmt_picker_span = $('');\n",
+ "\n",
+ " var fmt_picker = $('');\n",
+ " fmt_picker.addClass('mpl-toolbar-option ui-widget ui-widget-content');\n",
+ " fmt_picker_span.append(fmt_picker);\n",
+ " nav_element.append(fmt_picker_span);\n",
+ " this.format_dropdown = fmt_picker[0];\n",
+ "\n",
+ " for (var ind in mpl.extensions) {\n",
+ " var fmt = mpl.extensions[ind];\n",
+ " var option = $(\n",
+ " '', {selected: fmt === mpl.default_extension}).html(fmt);\n",
+ " fmt_picker.append(option)\n",
+ " }\n",
+ "\n",
+ " // Add hover states to the ui-buttons\n",
+ " $( \".ui-button\" ).hover(\n",
+ " function() { $(this).addClass(\"ui-state-hover\");},\n",
+ " function() { $(this).removeClass(\"ui-state-hover\");}\n",
+ " );\n",
+ "\n",
+ " var status_bar = $('');\n",
+ " nav_element.append(status_bar);\n",
+ " this.message = status_bar[0];\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.request_resize = function(x_pixels, y_pixels) {\n",
+ " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
+ " // which will in turn request a refresh of the image.\n",
+ " this.send_message('resize', {'width': x_pixels, 'height': y_pixels});\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.send_message = function(type, properties) {\n",
+ " properties['type'] = type;\n",
+ " properties['figure_id'] = this.id;\n",
+ " this.ws.send(JSON.stringify(properties));\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.send_draw_message = function() {\n",
+ " if (!this.waiting) {\n",
+ " this.waiting = true;\n",
+ " this.ws.send(JSON.stringify({type: \"draw\", figure_id: this.id}));\n",
+ " }\n",
+ "}\n",
+ "\n",
+ "\n",
+ "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
+ " var format_dropdown = fig.format_dropdown;\n",
+ " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
+ " fig.ondownload(fig, format);\n",
+ "}\n",
+ "\n",
+ "\n",
+ "mpl.figure.prototype.handle_resize = function(fig, msg) {\n",
+ " var size = msg['size'];\n",
+ " if (size[0] != fig.canvas.width || size[1] != fig.canvas.height) {\n",
+ " fig._resize_canvas(size[0], size[1]);\n",
+ " fig.send_message(\"refresh\", {});\n",
+ " };\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.handle_rubberband = function(fig, msg) {\n",
+ " var x0 = msg['x0'] / mpl.ratio;\n",
+ " var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
+ " var x1 = msg['x1'] / mpl.ratio;\n",
+ " var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
+ " x0 = Math.floor(x0) + 0.5;\n",
+ " y0 = Math.floor(y0) + 0.5;\n",
+ " x1 = Math.floor(x1) + 0.5;\n",
+ " y1 = Math.floor(y1) + 0.5;\n",
+ " var min_x = Math.min(x0, x1);\n",
+ " var min_y = Math.min(y0, y1);\n",
+ " var width = Math.abs(x1 - x0);\n",
+ " var height = Math.abs(y1 - y0);\n",
+ "\n",
+ " fig.rubberband_context.clearRect(\n",
+ " 0, 0, fig.canvas.width, fig.canvas.height);\n",
+ "\n",
+ " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.handle_figure_label = function(fig, msg) {\n",
+ " // Updates the figure title.\n",
+ " fig.header.textContent = msg['label'];\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.handle_cursor = function(fig, msg) {\n",
+ " var cursor = msg['cursor'];\n",
+ " switch(cursor)\n",
+ " {\n",
+ " case 0:\n",
+ " cursor = 'pointer';\n",
+ " break;\n",
+ " case 1:\n",
+ " cursor = 'default';\n",
+ " break;\n",
+ " case 2:\n",
+ " cursor = 'crosshair';\n",
+ " break;\n",
+ " case 3:\n",
+ " cursor = 'move';\n",
+ " break;\n",
+ " }\n",
+ " fig.rubberband_canvas.style.cursor = cursor;\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.handle_message = function(fig, msg) {\n",
+ " fig.message.textContent = msg['message'];\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.handle_draw = function(fig, msg) {\n",
+ " // Request the server to send over a new figure.\n",
+ " fig.send_draw_message();\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.handle_image_mode = function(fig, msg) {\n",
+ " fig.image_mode = msg['mode'];\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.updated_canvas_event = function() {\n",
+ " // Called whenever the canvas gets updated.\n",
+ " this.send_message(\"ack\", {});\n",
+ "}\n",
+ "\n",
+ "// A function to construct a web socket function for onmessage handling.\n",
+ "// Called in the figure constructor.\n",
+ "mpl.figure.prototype._make_on_message_function = function(fig) {\n",
+ " return function socket_on_message(evt) {\n",
+ " if (evt.data instanceof Blob) {\n",
+ " /* FIXME: We get \"Resource interpreted as Image but\n",
+ " * transferred with MIME type text/plain:\" errors on\n",
+ " * Chrome. But how to set the MIME type? It doesn't seem\n",
+ " * to be part of the websocket stream */\n",
+ " evt.data.type = \"image/png\";\n",
+ "\n",
+ " /* Free the memory for the previous frames */\n",
+ " if (fig.imageObj.src) {\n",
+ " (window.URL || window.webkitURL).revokeObjectURL(\n",
+ " fig.imageObj.src);\n",
+ " }\n",
+ "\n",
+ " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
+ " evt.data);\n",
+ " fig.updated_canvas_event();\n",
+ " fig.waiting = false;\n",
+ " return;\n",
+ " }\n",
+ " else if (typeof evt.data === 'string' && evt.data.slice(0, 21) == \"data:image/png;base64\") {\n",
+ " fig.imageObj.src = evt.data;\n",
+ " fig.updated_canvas_event();\n",
+ " fig.waiting = false;\n",
+ " return;\n",
+ " }\n",
+ "\n",
+ " var msg = JSON.parse(evt.data);\n",
+ " var msg_type = msg['type'];\n",
+ "\n",
+ " // Call the \"handle_{type}\" callback, which takes\n",
+ " // the figure and JSON message as its only arguments.\n",
+ " try {\n",
+ " var callback = fig[\"handle_\" + msg_type];\n",
+ " } catch (e) {\n",
+ " console.log(\"No handler for the '\" + msg_type + \"' message type: \", msg);\n",
+ " return;\n",
+ " }\n",
+ "\n",
+ " if (callback) {\n",
+ " try {\n",
+ " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
+ " callback(fig, msg);\n",
+ " } catch (e) {\n",
+ " console.log(\"Exception inside the 'handler_\" + msg_type + \"' callback:\", e, e.stack, msg);\n",
+ " }\n",
+ " }\n",
+ " };\n",
+ "}\n",
+ "\n",
+ "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
+ "mpl.findpos = function(e) {\n",
+ " //this section is from http://www.quirksmode.org/js/events_properties.html\n",
+ " var targ;\n",
+ " if (!e)\n",
+ " e = window.event;\n",
+ " if (e.target)\n",
+ " targ = e.target;\n",
+ " else if (e.srcElement)\n",
+ " targ = e.srcElement;\n",
+ " if (targ.nodeType == 3) // defeat Safari bug\n",
+ " targ = targ.parentNode;\n",
+ "\n",
+ " // jQuery normalizes the pageX and pageY\n",
+ " // pageX,Y are the mouse positions relative to the document\n",
+ " // offset() returns the position of the element relative to the document\n",
+ " var x = e.pageX - $(targ).offset().left;\n",
+ " var y = e.pageY - $(targ).offset().top;\n",
+ "\n",
+ " return {\"x\": x, \"y\": y};\n",
+ "};\n",
+ "\n",
+ "/*\n",
+ " * return a copy of an object with only non-object keys\n",
+ " * we need this to avoid circular references\n",
+ " * http://stackoverflow.com/a/24161582/3208463\n",
+ " */\n",
+ "function simpleKeys (original) {\n",
+ " return Object.keys(original).reduce(function (obj, key) {\n",
+ " if (typeof original[key] !== 'object')\n",
+ " obj[key] = original[key]\n",
+ " return obj;\n",
+ " }, {});\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.mouse_event = function(event, name) {\n",
+ " var canvas_pos = mpl.findpos(event)\n",
+ "\n",
+ " if (name === 'button_press')\n",
+ " {\n",
+ " this.canvas.focus();\n",
+ " this.canvas_div.focus();\n",
+ " }\n",
+ "\n",
+ " var x = canvas_pos.x * mpl.ratio;\n",
+ " var y = canvas_pos.y * mpl.ratio;\n",
+ "\n",
+ " this.send_message(name, {x: x, y: y, button: event.button,\n",
+ " step: event.step,\n",
+ " guiEvent: simpleKeys(event)});\n",
+ "\n",
+ " /* This prevents the web browser from automatically changing to\n",
+ " * the text insertion cursor when the button is pressed. We want\n",
+ " * to control all of the cursor setting manually through the\n",
+ " * 'cursor' event from matplotlib */\n",
+ " event.preventDefault();\n",
+ " return false;\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
+ " // Handle any extra behaviour associated with a key event\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.key_event = function(event, name) {\n",
+ "\n",
+ " // Prevent repeat events\n",
+ " if (name == 'key_press')\n",
+ " {\n",
+ " if (event.which === this._key)\n",
+ " return;\n",
+ " else\n",
+ " this._key = event.which;\n",
+ " }\n",
+ " if (name == 'key_release')\n",
+ " this._key = null;\n",
+ "\n",
+ " var value = '';\n",
+ " if (event.ctrlKey && event.which != 17)\n",
+ " value += \"ctrl+\";\n",
+ " if (event.altKey && event.which != 18)\n",
+ " value += \"alt+\";\n",
+ " if (event.shiftKey && event.which != 16)\n",
+ " value += \"shift+\";\n",
+ "\n",
+ " value += 'k';\n",
+ " value += event.which.toString();\n",
+ "\n",
+ " this._key_event_extra(event, name);\n",
+ "\n",
+ " this.send_message(name, {key: value,\n",
+ " guiEvent: simpleKeys(event)});\n",
+ " return false;\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.toolbar_button_onclick = function(name) {\n",
+ " if (name == 'download') {\n",
+ " this.handle_save(this, null);\n",
+ " } else {\n",
+ " this.send_message(\"toolbar_button\", {name: name});\n",
+ " }\n",
+ "};\n",
+ "\n",
+ "mpl.figure.prototype.toolbar_button_onmouseover = function(tooltip) {\n",
+ " this.message.textContent = tooltip;\n",
+ "};\n",
+ "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
+ "\n",
+ "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
+ "\n",
+ "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n",
+ " // Create a \"websocket\"-like object which calls the given IPython comm\n",
+ " // object with the appropriate methods. Currently this is a non binary\n",
+ " // socket, so there is still some room for performance tuning.\n",
+ " var ws = {};\n",
+ "\n",
+ " ws.close = function() {\n",
+ " comm.close()\n",
+ " };\n",
+ " ws.send = function(m) {\n",
+ " //console.log('sending', m);\n",
+ " comm.send(m);\n",
+ " };\n",
+ " // Register the callback with on_msg.\n",
+ " comm.on_msg(function(msg) {\n",
+ " //console.log('receiving', msg['content']['data'], msg);\n",
+ " // Pass the mpl event to the overriden (by mpl) onmessage function.\n",
+ " ws.onmessage(msg['content']['data'])\n",
+ " });\n",
+ " return ws;\n",
+ "}\n",
+ "\n",
+ "mpl.mpl_figure_comm = function(comm, msg) {\n",
+ " // This is the function which gets called when the mpl process\n",
+ " // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
+ "\n",
+ " var id = msg.content.data.id;\n",
+ " // Get hold of the div created by the display call when the Comm\n",
+ " // socket was opened in Python.\n",
+ " var element = $(\"#\" + id);\n",
+ " var ws_proxy = comm_websocket_adapter(comm)\n",
+ "\n",
+ " function ondownload(figure, format) {\n",
+ " window.open(figure.imageObj.src);\n",
+ " }\n",
+ "\n",
+ " var fig = new mpl.figure(id, ws_proxy,\n",
+ " ondownload,\n",
+ " element.get(0));\n",
+ "\n",
+ " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
+ " // web socket which is closed, not our websocket->open comm proxy.\n",
+ " ws_proxy.onopen();\n",
+ "\n",
+ " fig.parent_element = element.get(0);\n",
+ " fig.cell_info = mpl.find_output_cell(\"\");\n",
+ " if (!fig.cell_info) {\n",
+ " console.error(\"Failed to find cell for figure\", id, fig);\n",
+ " return;\n",
+ " }\n",
+ "\n",
+ " var output_index = fig.cell_info[2]\n",
+ " var cell = fig.cell_info[0];\n",
+ "\n",
+ "};\n",
+ "\n",
+ "mpl.figure.prototype.handle_close = function(fig, msg) {\n",
+ " var width = fig.canvas.width/mpl.ratio\n",
+ " fig.root.unbind('remove')\n",
+ "\n",
+ " // Update the output cell to use the data from the current canvas.\n",
+ " fig.push_to_output();\n",
+ " var dataURL = fig.canvas.toDataURL();\n",
+ " // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
+ " // the notebook keyboard shortcuts fail.\n",
+ " IPython.keyboard_manager.enable()\n",
+ " $(fig.parent_element).html('');\n",
+ " fig.close_ws(fig, msg);\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.close_ws = function(fig, msg){\n",
+ " fig.send_message('closing', msg);\n",
+ " // fig.ws.close()\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.push_to_output = function(remove_interactive) {\n",
+ " // Turn the data on the canvas into data in the output cell.\n",
+ " var width = this.canvas.width/mpl.ratio\n",
+ " var dataURL = this.canvas.toDataURL();\n",
+ " this.cell_info[1]['text/html'] = '';\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.updated_canvas_event = function() {\n",
+ " // Tell IPython that the notebook contents must change.\n",
+ " IPython.notebook.set_dirty(true);\n",
+ " this.send_message(\"ack\", {});\n",
+ " var fig = this;\n",
+ " // Wait a second, then push the new image to the DOM so\n",
+ " // that it is saved nicely (might be nice to debounce this).\n",
+ " setTimeout(function () { fig.push_to_output() }, 1000);\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype._init_toolbar = function() {\n",
+ " var fig = this;\n",
+ "\n",
+ " var nav_element = $('')\n",
+ " nav_element.attr('style', 'width: 100%');\n",
+ " this.root.append(nav_element);\n",
+ "\n",
+ " // Define a callback function for later on.\n",
+ " function toolbar_event(event) {\n",
+ " return fig.toolbar_button_onclick(event['data']);\n",
+ " }\n",
+ " function toolbar_mouse_event(event) {\n",
+ " return fig.toolbar_button_onmouseover(event['data']);\n",
+ " }\n",
+ "\n",
+ " for(var toolbar_ind in mpl.toolbar_items){\n",
+ " var name = mpl.toolbar_items[toolbar_ind][0];\n",
+ " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
+ " var image = mpl.toolbar_items[toolbar_ind][2];\n",
+ " var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
+ "\n",
+ " if (!name) { continue; };\n",
+ "\n",
+ " var button = $('');\n",
+ " button.click(method_name, toolbar_event);\n",
+ " button.mouseover(tooltip, toolbar_mouse_event);\n",
+ " nav_element.append(button);\n",
+ " }\n",
+ "\n",
+ " // Add the status bar.\n",
+ " var status_bar = $('');\n",
+ " nav_element.append(status_bar);\n",
+ " this.message = status_bar[0];\n",
+ "\n",
+ " // Add the close button to the window.\n",
+ " var buttongrp = $('');\n",
+ " var button = $('');\n",
+ " button.click(function (evt) { fig.handle_close(fig, {}); } );\n",
+ " button.mouseover('Stop Interaction', toolbar_mouse_event);\n",
+ " buttongrp.append(button);\n",
+ " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n",
+ " titlebar.prepend(buttongrp);\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype._root_extra_style = function(el){\n",
+ " var fig = this\n",
+ " el.on(\"remove\", function(){\n",
+ "\tfig.close_ws(fig, {});\n",
+ " });\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype._canvas_extra_style = function(el){\n",
+ " // this is important to make the div 'focusable\n",
+ " el.attr('tabindex', 0)\n",
+ " // reach out to IPython and tell the keyboard manager to turn it's self\n",
+ " // off when our div gets focus\n",
+ "\n",
+ " // location in version 3\n",
+ " if (IPython.notebook.keyboard_manager) {\n",
+ " IPython.notebook.keyboard_manager.register_events(el);\n",
+ " }\n",
+ " else {\n",
+ " // location in version 2\n",
+ " IPython.keyboard_manager.register_events(el);\n",
+ " }\n",
+ "\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
+ " var manager = IPython.notebook.keyboard_manager;\n",
+ " if (!manager)\n",
+ " manager = IPython.keyboard_manager;\n",
+ "\n",
+ " // Check for shift+enter\n",
+ " if (event.shiftKey && event.which == 13) {\n",
+ " this.canvas_div.blur();\n",
+ " event.shiftKey = false;\n",
+ " // Send a \"J\" for go to next cell\n",
+ " event.which = 74;\n",
+ " event.keyCode = 74;\n",
+ " manager.command_mode();\n",
+ " manager.handle_keydown(event);\n",
+ " }\n",
+ "}\n",
+ "\n",
+ "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
+ " fig.ondownload(fig, null);\n",
+ "}\n",
+ "\n",
+ "\n",
+ "mpl.find_output_cell = function(html_output) {\n",
+ " // Return the cell and output element which can be found *uniquely* in the notebook.\n",
+ " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
+ " // IPython event is triggered only after the cells have been serialised, which for\n",
+ " // our purposes (turning an active figure into a static one), is too late.\n",
+ " var cells = IPython.notebook.get_cells();\n",
+ " var ncells = cells.length;\n",
+ " for (var i=0; i= 3 moved mimebundle to data attribute of output\n",
+ " data = data.data;\n",
+ " }\n",
+ " if (data['text/html'] == html_output) {\n",
+ " return [cell, data, j];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ "}\n",
+ "\n",
+ "// Register the function which deals with the matplotlib target/channel.\n",
+ "// The kernel may be null if the page has been refreshed.\n",
+ "if (IPython.notebook.kernel != null) {\n",
+ " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n",
+ "}\n"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
- "df"
+ "import matplotlib.pyplot as plt\n",
+ "%matplotlib notebook\n",
+ "\n",
+ "import seaborn as sns; \n",
+ "sns.set(color_codes=True)\n",
+ "plt.style.use('default')\n",
+ "sns.set_style(\"dark\")\n",
+ "\n",
+ "y_label=\"ZeroOneRankAccuracy\"\n",
+ "x_label=\"Number of Objects\"\n",
+ "fig, ax = plt.subplots()\n",
+ "for col in dataFrame.columns.values[1:]:\n",
+ " plt.plot(dataFrame[dataFrame.columns.values[0]], 1 - dataFrame[col], marker=\"o\",label=col)\n",
+ "ax.xaxis.set_ticks(np.arange(3, 25))\n",
+ "ax.set_ylabel(y_label)\n",
+ "ax.set_xlabel(x_label)\n",
+ "plt.xticks(np.arange(5, 20, 5))\n",
+ "plt.yticks(np.arange(0.75,1.01,0.05))\n",
+ "plt.legend(loc=\"best\")\n",
+ "plot_path =os.path.join(os.getcwd(), \"logs\" ,\"generalizing.png\")\n",
+ "plt.savefig(plot_path, facecolor='w', edgecolor='w',transparent=False, dpi=800,format='png')\n",
+ "plt.show()"
]
},
{
@@ -283,7 +1476,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.3"
+ "version": "3.6.0"
}
},
"nbformat": 4,
diff --git a/notebooks/ParameterOptimizer-quickstart.ipynb b/notebooks/ParameterOptimizer-quickstart.ipynb
index afcc71fb..01bada93 100644
--- a/notebooks/ParameterOptimizer-quickstart.ipynb
+++ b/notebooks/ParameterOptimizer-quickstart.ipynb
@@ -2,9 +2,19 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [],
+ "execution_count": 1,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using TensorFlow backend.\n"
+ ]
+ }
+ ],
"source": [
"from csrank.dataset_reader import SyntheticDatasetGenerator\n",
"from sklearn.utils import check_random_state\n",
@@ -16,7 +26,45 @@
"from csrank.util import configure_logging_numpy_keras\n",
"import numpy as np\n",
"from keras import backend as K\n",
- "import os"
+ "import os\n",
+ "from skopt import load"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "def get_optimizer(logger, optimizer_path, n_iter):\n",
+ " logger.info('Retrieving model stored at: {}'.format(optimizer_path))\n",
+ " try:\n",
+ " optimizer = load(optimizer_path)\n",
+ " logger.info('Loading model stored at: {}'.format(optimizer_path))\n",
+ "\n",
+ " except KeyError:\n",
+ " logger.error('Cannot open the file {}'.format(optimizer_path))\n",
+ " optimizer = None\n",
+ "\n",
+ " except ValueError:\n",
+ " logger.error('Cannot open the file {}'.format(optimizer_path))\n",
+ " optimizer = None\n",
+ " except FileNotFoundError:\n",
+ " logger.error('No such file or directory: {}'.format(optimizer_path))\n",
+ " optimizer = None\n",
+ " if optimizer is not None:\n",
+ " finished_iterations = np.array(optimizer.yi).shape[0]\n",
+ " if finished_iterations == 0:\n",
+ " optimizer = None\n",
+ " logger.info('Optimizer did not finish any iterations so setting optimizer to null')\n",
+ " else:\n",
+ " n_iter = n_iter - finished_iterations\n",
+ " if n_iter < 0:\n",
+ " n_iter = 0\n",
+ " logger.info('Iterations already done: {} and running iterations {}'.format(finished_iterations, n_iter))\n",
+ " return optimizer, n_iter"
]
},
{
@@ -28,7 +76,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 3,
"metadata": {
"collapsed": true
},
@@ -51,8 +99,10 @@
},
{
"cell_type": "code",
- "execution_count": 17,
- "metadata": {},
+ "execution_count": 4,
+ "metadata": {
+ "collapsed": false
+ },
"outputs": [],
"source": [
"logger = configure_logging_numpy_keras(log_path=os.path.join(os.getcwd(), 'logs' ,\"test_models.log\"), name='Experiment')\n",
@@ -61,50 +111,83 @@
"n_objects = 5\n",
"random_state = check_random_state(42)\n",
"skf = ShuffleSplit(n_splits=2, test_size=0.5, random_state=random_state)\n",
- "epochs = 100"
+ "epochs = 100\n",
+ "optimizer_path = os.path.join(os.getcwd(), 'logs',\"optimizer\")\n",
+ "n_iter = 5\n",
+ "optimizer, n_iter = get_optimizer(logger, optimizer_path, n_iter)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Defining Ranker init and fitting parameters.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "ranker_params = {'n_objects': n_objects,\n",
+ " 'n_features':n_features, \n",
+ " 'n_object_features':n_features}\n",
+ "\n",
+ "fit_params = {'epochs': epochs,\n",
+ " 'log_callbacks':[DebugOutput()]}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "Defining Ranker init and fitting parameters.\n",
"Defining the Parameter optimizer init and fitting parameters"
]
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
- "ranker_params = {'n_objects': n_objects,'n_features':n_features, 'n_object_features':n_features}\n",
- "fit_params = {'epochs': epochs,'log_callbacks':[DebugOutput()]}\n",
- "medoids_params = {'dataset_type':\"medoid\",'n_test_instances': n_instances, 'n_train_instances': n_instances,'n_features': n_features, 'n_objects': n_objects,\n",
- " 'random_state': random_state}\n",
- "optimizer_fit_params = {'n_iter': 1, 'cv_iter': skf, 'optimizer': None, \"total_duration\":get_duration_microsecond(\"10h\")}\n",
+ "optimizer_fit_params = {'n_iter': n_iter, \n",
+ " 'cv_iter': skf, \n",
+ " 'optimizer': optimizer, \n",
+ " \"total_duration\":get_duration_microsecond(\"10h\")}\n",
"\n",
- "optimizer_params = {'ranker_class': FATEObjectRanker, 'fit_params': fit_params, 'ranker_params': ranker_params,\n",
- " 'random_state': random_state, \"optimizer_path\":os.path.join(os.getcwd(), 'logs',\"optimizer\")}\n"
+ "optimizer_params = {'ranker_class': FATEObjectRanker, \n",
+ " 'fit_params': fit_params,\n",
+ " 'ranker_params': ranker_params,\n",
+ " 'random_state': random_state, \n",
+ " \"optimizer_path\":optimizer_path}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "Generating the dataset"
+ "Generating the medoid test and train dataset with defined parameters"
]
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
+ "medoids_params = {'dataset_type':\"medoid\",\n",
+ " 'n_test_instances': n_instances,\n",
+ " 'n_train_instances': n_instances,\n",
+ " 'n_features': n_features,\n",
+ " 'n_objects': n_objects,\n",
+ " 'random_state': random_state}\n",
"dr = SyntheticDatasetGenerator(**medoids_params)\n",
"X,Y,X_test,Y_test = dr.get_single_train_test_split()"
]
@@ -118,9 +201,9 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
@@ -137,21 +220,11 @@
},
{
"cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "ename": "NameError",
- "evalue": "name 'np' is not defined",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
- "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m optimizer_model.model = optimizer_model._ranker_class(random_state=optimizer_model.random_state,\n\u001b[1;32m 3\u001b[0m **optimizer_model._ranker_params)\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mbest_point\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mXi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0myi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0moptimizer_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_tunable_parameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_point\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"
- ]
- }
- ],
+ "execution_count": 11,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
"source": [
"if optimizer_model.model is None:\n",
" optimizer_model.model = optimizer_model._ranker_class(random_state=optimizer_model.random_state,\n",
@@ -159,18 +232,44 @@
" best_point = optimizer_model.optimizer.Xi[np.argmin(optimizer_model.optimizer.yi)]\n",
" optimizer_model.model.set_tunable_parameters(best_point)\n",
" logger.info(optimizer_model.model.__dict__)\n",
- " optimizer_model.model.fit(X_train, Y_train, **optimizer_model._fit_params)\n",
- "predicted = optimizer_model.predict_scores(test)"
+ " optimizer_model.model.fit(X, Y, **optimizer_model._fit_params)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Predict Scores for the best model"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 13,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
- "outputs": [],
- "source": []
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([[-0.60119748, -3.00899553, -1.15418839, -1.89158905, 3.21277809],\n",
+ " [-2.29909468, -2.76785064, -3.73014832, -0.86105549, -4.87870646],\n",
+ " [-1.33258545, -3.09852505, -3.92024136, -0.89974248, -3.95369244],\n",
+ " ..., \n",
+ " [-3.43628454, -0.66553152, -2.91205454, -1.99111378, -1.79274499],\n",
+ " [ 2.20938802, 1.50373054, -0.01428777, -1.63791025, 3.09338284],\n",
+ " [-2.47337866, -4.68289852, -0.53654063, -1.74233592, -6.30659676]], dtype=float32)"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "predicted = optimizer_model.predict_scores(X_test)\n",
+ "predicted"
+ ]
}
],
"metadata": {
@@ -190,7 +289,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.3"
+ "version": "3.6.0"
}
},
"nbformat": 4,
diff --git a/notebooks/logs/generalizing_mean_5.csv b/notebooks/logs/generalizing_mean_5.csv
new file mode 100644
index 00000000..6fd13117
--- /dev/null
+++ b/notebooks/logs/generalizing_mean_5.csv
@@ -0,0 +1,18 @@
+,Objects,FETARankerBestParams,FETARanker,FATERanker,FATERanker10SetLayers,FATERanker8SetLayers
+0,3,0.1985868365,0.1949512064,0.0824327916,0.0652848184,0.094950527
+1,4,0.2264747024,0.2284020334,0.1418879628,0.1207793728,0.1254982352
+2,5,0.2180698067,0.2241305709,0.1091250777,0.1038039848,0.1075947434
+3,6,0.2155445069,0.2279135138,0.1339071542,0.1093370989,0.1072543934
+4,7,0.2115869969,0.2289696932,0.13454872369999998,0.1106991246,0.1073934957
+5,8,0.2077214718,0.2304981053,0.1428300738,0.1121656373,0.1078011245
+6,9,0.2036555409,0.2305198312,0.1479084641,0.1135326102,0.1101869941
+7,10,0.2001185417,0.230696559,0.1537525654,0.1154463217,0.1128275767
+8,11,0.1964794695,0.2302957177,0.1577038765,0.1163998023,0.1150102317
+9,12,0.1930269897,0.2302988023,0.1613877863,0.1164451987,0.116744414
+10,13,0.1898829341,0.2296092361,0.16466102,0.1173365936,0.1192103326
+11,14,0.186183989,0.229495272,0.167530641,0.117598176,0.1212140322
+12,15,0.1828398108,0.2279229164,0.17048923670000002,0.1176226065,0.123238273
+13,16,0.1799229831,0.2275559753,0.17212864760000002,0.1178899184,0.1254735589
+14,17,0.1777946502,0.2269429266,0.1746897846,0.1180509478,0.1275726408
+15,18,0.17480814460000002,0.2262506783,0.1764152795,0.1175213084,0.129285872
+16,19,0.17310652140000002,0.225329712,0.1785377413,0.1181196868,0.1318298876
diff --git a/notebooks/logs/optimizer b/notebooks/logs/optimizer
new file mode 100644
index 00000000..49bc7e47
Binary files /dev/null and b/notebooks/logs/optimizer differ