From be9dd3d4fd699844ce3e04b624c682ead758e4ea Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Mar 2017 20:10:02 +0100 Subject: [PATCH 01/13] added tag mechanisms for runs #214 --- openml/runs/functions.py | 52 +++++++++++++++------------ openml/runs/run.py | 10 ++---- tests/test_runs/test_run_functions.py | 8 ++++- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index ab84063dc..c683e694f 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -15,7 +15,7 @@ from ..util import URLError from ..tasks.functions import _create_task_from_xml from .._api_calls import _perform_api_call -from .run import OpenMLRun +from .run import OpenMLRun, _get_version_information # _get_version_info, _get_dict and _create_setup_string are in run.py to avoid @@ -66,8 +66,10 @@ def run_task(task, model): raise ValueError('The task has no class labels. This method currently ' 'only works for tasks with class labels.') + run_environment = _get_version_information() + tags = ['openml-python', run_environment[1]] # execute the run - run = OpenMLRun(task_id=task.task_id, flow_id=flow_id, dataset_id=dataset.dataset_id, model=model) + run = OpenMLRun(task_id=task.task_id, flow_id=flow_id, dataset_id=dataset.dataset_id, model=model, tags=tags) try: run.data_content, run.trace_content, run.trace_attributes = _run_task_get_arffcontent(model, task, class_labels) @@ -337,27 +339,31 @@ def _create_run_from_xml(xml): evaluations = dict() detailed_evaluations = defaultdict(lambda: defaultdict(dict)) evaluation_flows = dict() - for evaluation_dict in run['oml:output_data']['oml:evaluation']: - key = evaluation_dict['oml:name'] - if 'oml:value' in evaluation_dict: - value = float(evaluation_dict['oml:value']) - elif 'oml:array_data' in evaluation_dict: - value = evaluation_dict['oml:array_data'] - else: - raise ValueError('Could not find keys "value" or "array_data" ' - 'in %s' % str(evaluation_dict.keys())) - - if '@repeat' in evaluation_dict and '@fold' in evaluation_dict: - repeat = int(evaluation_dict['@repeat']) - fold = int(evaluation_dict['@fold']) - repeat_dict = detailed_evaluations[key] - fold_dict = repeat_dict[repeat] - fold_dict[fold] = value - else: - evaluations[key] = value - evaluation_flows[key] = flow_id + if 'oml:output_data' in run and 'oml:evaluation' in run['oml:output_data']: + for evaluation_dict in run['oml:output_data']['oml:evaluation']: + key = evaluation_dict['oml:name'] + if 'oml:value' in evaluation_dict: + value = float(evaluation_dict['oml:value']) + elif 'oml:array_data' in evaluation_dict: + value = evaluation_dict['oml:array_data'] + else: + raise ValueError('Could not find keys "value" or "array_data" ' + 'in %s' % str(evaluation_dict.keys())) + + if '@repeat' in evaluation_dict and '@fold' in evaluation_dict: + repeat = int(evaluation_dict['@repeat']) + fold = int(evaluation_dict['@fold']) + repeat_dict = detailed_evaluations[key] + fold_dict = repeat_dict[repeat] + fold_dict[fold] = value + else: + evaluations[key] = value + evaluation_flows[key] = flow_id - evaluation_flows[key] = flow_id + evaluation_flows[key] = flow_id + tags = None + if 'oml:tag' in run: + tags = run['oml:tag'] return OpenMLRun(run_id=run_id, uploader=uploader, uploader_name=uploader_name, task_id=task_id, @@ -368,7 +374,7 @@ def _create_run_from_xml(xml): parameter_settings=parameters, dataset_id=dataset_id, predictions_url=predictions_url, evaluations=evaluations, - detailed_evaluations=detailed_evaluations) + detailed_evaluations=detailed_evaluations, tags=tags) def _get_cached_run(run_id): diff --git a/openml/runs/run.py b/openml/runs/run.py index 8a21a5ff3..825fb05c6 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -48,6 +48,7 @@ def __init__(self, task_id, flow_id, dataset_id, setup_string=None, self.flow = flow self.run_id = run_id self.model = model + self.tags = tags def _generate_arff_dict(self): """Generates the arff dictionary for uploading predictions to the server. @@ -142,24 +143,17 @@ def _create_description_xml(self): xml_string : string XML description of run. """ - run_environment = _get_version_information() # TODO: don't we have flow object in data structure? Use this one downloaded_flow = openml.flows.get_flow(self.flow_id) openml_param_settings = OpenMLRun._parse_parameters(self.model, downloaded_flow) - # as a tag, it must be of the form ([a-zA-Z0-9_\-\.])+ - # so we format time from 'mm/dd/yy hh:mm:ss' to 'mm-dd-yy_hh.mm.ss' - well_formatted_time = time.strftime("%c").replace( - ' ', '_').replace('/', '-').replace(':', '.') - tags = run_environment + [well_formatted_time] + ['run_task'] + \ - [self.model.__module__ + "." + self.model.__class__.__name__] description = _to_dict(taskid=self.task_id, flow_id=self.flow_id, setup_string=_create_setup_string(self.model), parameter_settings=openml_param_settings, error_message=self.error_message, - tags=tags) + tags=self.tags) description_xml = xmltodict.unparse(description, pretty=True) return description_xml diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index eeb012d15..fb4f13971 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -57,7 +57,10 @@ def test_run_iris(self): num_instances = 150 clf = LogisticRegression() - self._perform_run(task_id,num_instances, clf) + res = self._perform_run(task_id,num_instances, clf) + + downloaded = openml.runs.get_run(res.run_id) + assert('openml-python' in downloaded.tags) def test_run_optimize_randomforest_iris(self): task_id = 10107 @@ -141,6 +144,7 @@ def test__run_task_get_arffcontent(self): 'Iris-virginica']) def test_get_run(self): + openml.config.server = self.production_server run = openml.runs.get_run(473350) self.assertEqual(run.dataset_id, 1167) self.assertEqual(run.evaluations['f_measure'], 0.624668) @@ -155,6 +159,8 @@ def test_get_run(self): (8, 0.56759), (9, 0.64621)]: self.assertEqual(run.detailed_evaluations['f_measure'][0][i], value) + assert('weka' in run.tags) + assert('stacking' in run.tags) def _check_run(self, run): self.assertIsInstance(run, dict) From b4e052d4933cd12b475d5960a8766959d1e20902 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Mar 2017 20:33:18 +0100 Subject: [PATCH 02/13] added unit testcase for flow tagging #214 --- tests/test_flows/test_flow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 3f52bbb23..0d3d0b087 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -198,6 +198,7 @@ def test_sklearn_to_upload_to_flow(self): estimator=model, param_distributions=parameter_grid, cv=cv) rs.fit(X, y) flow = openml.flows.sklearn_to_flow(rs) + flow.tags.extend(['openml-python', 'unittest']) # Add the sentinel to all name strings in all subflows. Adds it to # name to make it easier in the web gui to see that the flow is only @@ -253,5 +254,6 @@ def test_sklearn_to_upload_to_flow(self): % sentinel self.assertEqual(new_flow.name, fixture_name) - + self.assertTrue('openml-python' in new_flow.tags) + self.assertTrue('unittest' in new_flow.tags) new_flow.model.fit(X, y) From 30840fdec441028b4bd479b0a89d9a7cb74739cb Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Mar 2017 20:40:51 +0100 Subject: [PATCH 03/13] fix for bug created when refactoring for issue #210 --- openml/runs/functions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index c683e694f..8b4f2dfed 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -180,10 +180,11 @@ def _run_task_get_arffcontent(model, task, class_labels): fold_no = fold_no + 1 rep_no = rep_no + 1 - - if get_traceble_model(model): + + traceable_model = get_traceble_model(model) + if traceable_model: # arff_tracecontent is already set - arff_trace_attributes = _extract_arfftrace_attributes(model_fold) + arff_trace_attributes = _extract_arfftrace_attributes(traceable_model) else: arff_tracecontent = None arff_trace_attributes = None From afc367414441219e7a4388f77689d1f0c654878c Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Mar 2017 20:44:12 +0100 Subject: [PATCH 04/13] same as previous --- openml/runs/functions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 8b4f2dfed..6675195b0 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -160,8 +160,9 @@ def _run_task_get_arffcontent(model, task, class_labels): try: model_fold.fit(trainX, trainY) - if get_traceble_model(model_fold): - arff_tracecontent.extend(_extract_arfftrace(model_fold, rep_no, fold_no)) + traceable_model = get_traceble_model(model) + if traceable_model: + arff_tracecontent.extend(_extract_arfftrace(traceable_model, rep_no, fold_no)) model_classes = model_fold.best_estimator_.classes_ else: model_classes = model_fold.classes_ @@ -180,7 +181,7 @@ def _run_task_get_arffcontent(model, task, class_labels): fold_no = fold_no + 1 rep_no = rep_no + 1 - + traceable_model = get_traceble_model(model) if traceable_model: # arff_tracecontent is already set From 869d078637bf655493d0ed80f8220fc04cfa4ef3 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Mar 2017 21:00:30 +0100 Subject: [PATCH 05/13] bugfixes belonging to #210 --- openml/flows/sklearn_converter.py | 2 +- openml/runs/functions.py | 29 +++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index f421d76fb..e0319851e 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -561,7 +561,7 @@ def get_traceble_model(model): Tree, maybe later) ''' if isinstance(model, sklearn.model_selection._search.BaseSearchCV): - return True + return model count = 0 returnValue = None if isinstance(model, sklearn.pipeline.Pipeline): diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 6675195b0..017ff54f0 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -159,17 +159,18 @@ def _run_task_get_arffcontent(model, task, class_labels): try: model_fold.fit(trainX, trainY) - - traceable_model = get_traceble_model(model) - if traceable_model: - arff_tracecontent.extend(_extract_arfftrace(traceable_model, rep_no, fold_no)) - model_classes = model_fold.best_estimator_.classes_ - else: - model_classes = model_fold.classes_ except AttributeError as e: # typically happens when training a regressor on classification task raise PyOpenMLError(str(e)) + # extract trace + traceable_model = get_traceble_model(model_fold) + if traceable_model: + arff_tracecontent.extend(_extract_arfftrace(traceable_model, rep_no, fold_no)) + model_classes = model_fold.best_estimator_.classes_ + else: + model_classes = model_fold.classes_ + ProbaY = model_fold.predict_proba(testX) PredY = model_fold.predict(testX) if ProbaY.shape[1] != len(class_labels): @@ -182,7 +183,7 @@ def _run_task_get_arffcontent(model, task, class_labels): fold_no = fold_no + 1 rep_no = rep_no + 1 - traceable_model = get_traceble_model(model) + traceable_model = get_traceble_model(model_fold) if traceable_model: # arff_tracecontent is already set arff_trace_attributes = _extract_arfftrace_attributes(traceable_model) @@ -194,6 +195,12 @@ def _run_task_get_arffcontent(model, task, class_labels): def _extract_arfftrace(model, rep_no, fold_no): + if not isinstance(model, sklearn.model_selection._search.BaseSearchCV): + raise ValueError('model should be instance of'\ + ' sklearn.model_selection._search.BaseSearchCV') + if not hasattr(model, 'cv_results_'): + raise ValueError('model should contain `cv_results_`') + arff_tracecontent = [] for itt_no in range(0, len(model.cv_results_['mean_test_score'])): # we use the string values for True and False, as it is defined in this way by the OpenML server @@ -209,6 +216,12 @@ def _extract_arfftrace(model, rep_no, fold_no): return arff_tracecontent def _extract_arfftrace_attributes(model): + if not isinstance(model, sklearn.model_selection._search.BaseSearchCV): + raise ValueError('model should be instance of'\ + ' sklearn.model_selection._search.BaseSearchCV') + if not hasattr(model, 'cv_results_'): + raise ValueError('model should contain `cv_results_`') + # attributes that will be in trace arff, regardless of the model trace_attributes = [('repeat', 'NUMERIC'), ('fold', 'NUMERIC'), From 588ffbf777eaab1d733250d482591addbfd5defd Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Mar 2017 21:05:47 +0100 Subject: [PATCH 06/13] hack for traceable model and obtaining classes --- openml/runs/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 017ff54f0..f2d2a7e9e 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -167,7 +167,7 @@ def _run_task_get_arffcontent(model, task, class_labels): traceable_model = get_traceble_model(model_fold) if traceable_model: arff_tracecontent.extend(_extract_arfftrace(traceable_model, rep_no, fold_no)) - model_classes = model_fold.best_estimator_.classes_ + model_classes = traceable_model.best_estimator_.classes_ else: model_classes = model_fold.classes_ From 83c0c102adb6ac8b462e4f4ee5fc46ef387bc505 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Tue, 21 Mar 2017 21:16:25 +0100 Subject: [PATCH 07/13] addedfunction to tag flows on run time #214 --- openml/runs/functions.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index f2d2a7e9e..b39ba8ee5 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -23,7 +23,7 @@ -def run_task(task, model): +def run_task(task, model, flow_tags=[]): """Performs a CV run on the dataset of the given task, using the split. Parameters @@ -34,17 +34,22 @@ def run_task(task, model): a model which has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model [1] [1](http://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html) - + flow_tags : list(str) + a list of tags that the flow should have at creation Returns ------- run : OpenMLRun Result of the run. """ + if not isinstance(flow_tags, list): + raise ValueError("flow_tags should be list") # TODO move this into its onwn module. While it somehow belongs here, it # adds quite a lot of functionality which is better suited in other places! # TODO why doesn't this accept a flow as input? - this would make this more flexible! flow = sklearn_to_flow(model) + flow.tags = flow_tags + # TODO: also tag the flow if it already exists flow_id = flow._ensure_flow_exists() if flow_id < 0: print("No flow") From 03d7807fd5db41edbf3d500b247a17690f430b0f Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Thu, 20 Apr 2017 17:41:43 +0200 Subject: [PATCH 08/13] small update to run functions related to tags --- openml/runs/functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 37cc1e852..dbfd8dbc5 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -22,7 +22,7 @@ -def run_task(task, model, avoid_duplicate_runs=True): +def run_task(task, model, avoid_duplicate_runs=True, flow_tags=None): """Performs a CV run on the dataset of the given task, using the split. Parameters @@ -70,7 +70,7 @@ def run_task(task, model, avoid_duplicate_runs=True): run_environment = _get_version_information() tags = ['openml-python', run_environment[1]] # execute the run - run = OpenMLRun(task_id=task.task_id, flow_id=None, dataset_id=dataset.dataset_id, model=model) + run = OpenMLRun(task_id=task.task_id, flow_id=None, dataset_id=dataset.dataset_id, model=model, tags=tags) run.data_content, run.trace_content, run.trace_attributes = _run_task_get_arffcontent(model, task, class_labels) From 094772cf31c6e293f48f75fb5e4d171228d704c2 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Thu, 20 Apr 2017 17:53:21 +0200 Subject: [PATCH 09/13] fixed errors that occured by solving merge conflict --- openml/runs/functions.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index dbfd8dbc5..dfe920eb0 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -41,7 +41,7 @@ def run_task(task, model, avoid_duplicate_runs=True, flow_tags=None): run : OpenMLRun Result of the run. """ - if not isinstance(flow_tags, list): + if flow_tags is not None and not isinstance(flow_tags, list): raise ValueError("flow_tags should be list") # TODO move this into its onwn module. While it somehow belongs here, it # adds quite a lot of functionality which is better suited in other places! @@ -183,10 +183,9 @@ def _run_task_get_arffcontent(model, task, class_labels): raise PyOpenMLError(str(e)) # extract trace - traceable_model = get_traceble_model(model_fold) - if traceable_model: - arff_tracecontent.extend(_extract_arfftrace(traceable_model, rep_no, fold_no)) - model_classes = traceable_model.best_estimator_.classes_ + if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): + arff_tracecontent.extend(_extract_arfftrace(model_fold, rep_no, fold_no)) + model_classes = model_fold.best_estimator_.classes_ else: model_classes = model_fold.classes_ @@ -204,7 +203,7 @@ def _run_task_get_arffcontent(model, task, class_labels): if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): # arff_tracecontent is already set - arff_trace_attributes = _extract_arfftrace_attributes(traceable_model) + arff_trace_attributes = _extract_arfftrace_attributes(model_fold) else: arff_tracecontent = None arff_trace_attributes = None @@ -401,6 +400,15 @@ def _create_run_from_xml(xml): evaluation_flows[key] = flow_id evaluation_flows[key] = flow_id + tags = None + if 'oml:tag' in run: + if isinstance(run['oml:tag'], str): + tags = [run['oml:tag']] + elif isinstance(run['oml:tag'], list): + tags = run['oml:tag'] + else: + raise ValueError('Received not string and non list as tag item') + return OpenMLRun(run_id=run_id, uploader=uploader, uploader_name=uploader_name, task_id=task_id, From 265b034ce6b8123efa465abecf8a01301bed5e57 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Thu, 20 Apr 2017 18:16:09 +0200 Subject: [PATCH 10/13] merge conflict bug resolved --- openml/runs/functions.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index dfe920eb0..806e14989 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -73,7 +73,6 @@ def run_task(task, model, avoid_duplicate_runs=True, flow_tags=None): run = OpenMLRun(task_id=task.task_id, flow_id=None, dataset_id=dataset.dataset_id, model=model, tags=tags) run.data_content, run.trace_content, run.trace_attributes = _run_task_get_arffcontent(model, task, class_labels) - if flow_id == False: # means the flow did not exists. As we could run it, publish it now flow = flow.publish() @@ -172,12 +171,6 @@ def _run_task_get_arffcontent(model, task, class_labels): try: model_fold.fit(trainX, trainY) - - if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): - arff_tracecontent.extend(_extract_arfftrace(model_fold, rep_no, fold_no)) - model_classes = model_fold.best_estimator_.classes_ - else: - model_classes = model_fold.classes_ except AttributeError as e: # typically happens when training a regressor on classification task raise PyOpenMLError(str(e)) From 3aea218b21fae892e4d6987a2bd8c57f2cabdcea Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Thu, 20 Apr 2017 18:33:05 +0200 Subject: [PATCH 11/13] different error catched --- tests/test_runs/test_run_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index aae2a9a2e..203109893 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -41,7 +41,7 @@ def test_run_regression_on_classif_task(self): clf = LinearRegression() task = openml.tasks.get_task(task_id) - self.assertRaises(openml.exceptions.PyOpenMLError, openml.runs.run_task, + self.assertRaises(openml.exceptions.AttributeError, openml.runs.run_task, task=task, model=clf, avoid_duplicate_runs=False) @mock.patch('openml.flows.sklearn_to_flow') @@ -83,6 +83,7 @@ def test_run_optimize_randomforest_iris(self): n_iter=num_iterations) run = self._perform_run(task_id, num_instances, random_search) + print(run.trace_content) self.assertEqual(len(run.trace_content), num_iterations * num_folds) def test_run_optimize_bagging_iris(self): From 9f6d8f101bdc3b0c41997e78a896046d4598ad7d Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Thu, 20 Apr 2017 18:34:32 +0200 Subject: [PATCH 12/13] sorry, once again --- tests/test_runs/test_run_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 203109893..e4926491a 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -41,7 +41,7 @@ def test_run_regression_on_classif_task(self): clf = LinearRegression() task = openml.tasks.get_task(task_id) - self.assertRaises(openml.exceptions.AttributeError, openml.runs.run_task, + self.assertRaises(AttributeError, openml.runs.run_task, task=task, model=clf, avoid_duplicate_runs=False) @mock.patch('openml.flows.sklearn_to_flow') From 456f91967a8550ca6138db13977d0993ce073ad1 Mon Sep 17 00:00:00 2001 From: Jan van Rijn Date: Fri, 21 Apr 2017 14:07:31 +0200 Subject: [PATCH 13/13] merge conflict bug fix --- openml/runs/functions.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index f5a8bfd7c..9cb08d7e5 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -181,12 +181,6 @@ def _run_task_get_arffcontent(model, task, class_labels): if version_complies(3, 3): modelfit_duration = (time.process_time() - modelfit_starttime) * 1000 user_defined_measures['usercpu_time_millis_training'][rep_no][fold_no] = modelfit_duration - - if isinstance(model_fold, sklearn.model_selection._search.BaseSearchCV): - arff_tracecontent.extend(_extract_arfftrace(model_fold, rep_no, fold_no)) - model_classes = model_fold.best_estimator_.classes_ - else: - model_classes = model_fold.classes_ except AttributeError as e: # typically happens when training a regressor on classification task raise PyOpenMLError(str(e))