From 4abc05b147dbe5f678f06eea341096e272e9b868 Mon Sep 17 00:00:00 2001 From: Jinu Date: Fri, 20 Apr 2018 13:25:22 +0200 Subject: [PATCH 1/7] Add example.rst. --- doc/Makefile | 1 + doc/conf.py | 10 +++++++++- doc/examples.rst | 1 + example/README.txt | 0 example/example_crossvalidation.py | 7 ++++++- example/example_eips.py | 6 ++++++ example/example_feature_types.py | 6 ++++++ example/example_holdout.py | 6 ++++++ example/example_metrics.py | 6 ++++++ example/example_parallel.py | 6 ++++++ example/example_random_search.py | 6 ++++++ example/example_regression.py | 6 ++++++ example/example_sequential.py | 6 ++++++ 13 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 doc/examples.rst create mode 100644 example/README.txt diff --git a/doc/Makefile b/doc/Makefile index f4b7bf3ce2..8dca341ddd 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -51,6 +51,7 @@ help: clean: rm -rf $(BUILDDIR)/* rm -rf generated + rm -rf auto_examples/ html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/doc/conf.py b/doc/conf.py index 6b93d98691..f1594e728e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -41,7 +41,15 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', - 'numpydoc'] + 'sphinx_gallery.gen_gallery', 'numpydoc'] + +# Sphinx-gallery configuration. +sphinx_gallery_conf = { + # path to the examples + 'examples_dirs': '../example', + # path where to save gallery generated examples + 'gallery_dirs': 'auto_examples' +} # Configure the extensions numpydoc_show_class_members = False diff --git a/doc/examples.rst b/doc/examples.rst new file mode 100644 index 0000000000..07b21861e6 --- /dev/null +++ b/doc/examples.rst @@ -0,0 +1 @@ +.. _examples: \ No newline at end of file diff --git a/example/README.txt b/example/README.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/example/example_crossvalidation.py b/example/example_crossvalidation.py index 06299b48db..048f2d5cab 100644 --- a/example/example_crossvalidation.py +++ b/example/example_crossvalidation.py @@ -1,11 +1,16 @@ # -*- encoding: utf-8 -*- +""" +================ +Cross Validation +================ +""" + import sklearn.model_selection import sklearn.datasets import sklearn.metrics import autosklearn.classification - def main(): X, y = sklearn.datasets.load_digits(return_X_y=True) X_train, X_test, y_train, y_test = \ diff --git a/example/example_eips.py b/example/example_eips.py index 3e0019c4c2..dc53840c2c 100644 --- a/example/example_eips.py +++ b/example/example_eips.py @@ -1,3 +1,9 @@ +""" +==== +EIPS +==== +""" + import sklearn.model_selection import sklearn.datasets import sklearn.metrics diff --git a/example/example_feature_types.py b/example/example_feature_types.py index 53b0b70b07..076520aeac 100644 --- a/example/example_feature_types.py +++ b/example/example_feature_types.py @@ -1,4 +1,10 @@ # -*- encoding: utf-8 -*- +""" +============= +Feature Types +============= +""" + import sklearn.model_selection import sklearn.datasets import sklearn.metrics diff --git a/example/example_holdout.py b/example/example_holdout.py index a17707aac1..9a5df02cc0 100644 --- a/example/example_holdout.py +++ b/example/example_holdout.py @@ -1,3 +1,9 @@ +""" +======= +Holdout +======= +""" + import sklearn.model_selection import sklearn.datasets import sklearn.metrics diff --git a/example/example_metrics.py b/example/example_metrics.py index 603b82f946..6b154e24b0 100644 --- a/example/example_metrics.py +++ b/example/example_metrics.py @@ -1,4 +1,10 @@ # -*- encoding: utf-8 -*- +""" +======= +Metrics +======= +""" + import numpy as np import sklearn.model_selection diff --git a/example/example_parallel.py b/example/example_parallel.py index 4aacbaf0fe..fac02299c0 100644 --- a/example/example_parallel.py +++ b/example/example_parallel.py @@ -1,4 +1,10 @@ # -*- encoding: utf-8 -*- +""" +==================== +Parallel Computation +==================== +""" + import multiprocessing import shutil diff --git a/example/example_random_search.py b/example/example_random_search.py index 1c62a190dd..1a6b37dac5 100644 --- a/example/example_random_search.py +++ b/example/example_random_search.py @@ -1,3 +1,9 @@ +""" +============= +Random Search +============= +""" + import sklearn.model_selection import sklearn.datasets import sklearn.metrics diff --git a/example/example_regression.py b/example/example_regression.py index 9420d88f83..2ee85287f2 100644 --- a/example/example_regression.py +++ b/example/example_regression.py @@ -1,4 +1,10 @@ # -*- encoding: utf-8 -*- + +""" +========== +Regression +========== +""" import sklearn.model_selection import sklearn.datasets import sklearn.metrics diff --git a/example/example_sequential.py b/example/example_sequential.py index 019ad3be9c..c28b125d97 100644 --- a/example/example_sequential.py +++ b/example/example_sequential.py @@ -1,3 +1,9 @@ + +""" +================ +Sequential Usage +================ +""" import sklearn.model_selection import sklearn.datasets import sklearn.metrics From d7c7e326f3f04fa8099334bb13060775892cbe97 Mon Sep 17 00:00:00 2001 From: Jinu Date: Mon, 23 Apr 2018 17:43:31 +0200 Subject: [PATCH 2/7] . --- doc/conf.py | 15 ++++++- doc/examples.rst | 5 ++- example/example_crossvalidation.py | 2 + example/example_eips.py | 2 + example/example_feature_types.py | 7 ++++ example/example_holdout.py | 9 +++++ example/example_metrics.py | 65 ++++++++++++++++++++++++++++++ example/example_parallel.py | 2 + example/example_random_search.py | 2 + example/example_regression.py | 2 + example/example_sequential.py | 2 + 11 files changed, 111 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index f1594e728e..ce5407e8ea 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -19,6 +19,7 @@ import os import sys import sphinx_bootstrap_theme +from sphinx_gallery.sorting import FileNameSortKey import autosklearn # Add the parent directory of this file to the PYTHONPATH import os @@ -48,7 +49,18 @@ # path to the examples 'examples_dirs': '../example', # path where to save gallery generated examples - 'gallery_dirs': 'auto_examples' + 'gallery_dirs': 'examples', + #'subsection_order': ExplicitOrder(['../example/example_holdout', + # '../example/example_crossvalidation', + # '../example/example_parallel', + # '../example/example_sequential', + # '../example/example_regression', + # '../example/example_feature_types', + # '../example/example_metrics', + # '../example/example_random_search', + # '../example/example_eips']), + 'within_subsection_order': FileNameSortKey, + 'thumbnail_size': (10, 10) } # Configure the extensions @@ -142,6 +154,7 @@ ('Releases', 'releases'), ('Installation', 'installation'), ('Manual', 'manual'), + ('Examples', 'examples/index'), ('API', 'api'), ('Extending', 'extending'), ], diff --git a/doc/examples.rst b/doc/examples.rst index 07b21861e6..6b6db9d8e7 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -1 +1,4 @@ -.. _examples: \ No newline at end of file +:orphan: + +.. _examples: + diff --git a/example/example_crossvalidation.py b/example/example_crossvalidation.py index 048f2d5cab..8b6fc5c9ff 100644 --- a/example/example_crossvalidation.py +++ b/example/example_crossvalidation.py @@ -3,6 +3,8 @@ ================ Cross Validation ================ + +The following example shows how to use cross-validation in *auto-sklearn*. """ import sklearn.model_selection diff --git a/example/example_eips.py b/example/example_eips.py index dc53840c2c..5223b1db04 100644 --- a/example/example_eips.py +++ b/example/example_eips.py @@ -2,6 +2,8 @@ ==== EIPS ==== + +Example description goes here. """ import sklearn.model_selection diff --git a/example/example_feature_types.py b/example/example_feature_types.py index 076520aeac..9dd6cdf557 100644 --- a/example/example_feature_types.py +++ b/example/example_feature_types.py @@ -3,6 +3,13 @@ ============= Feature Types ============= + +Example description goes here. +In *auto-sklearn* it is possible to specify the feature types of a dataset when +calling the `fit() `_ method by specifying the argument ``feat_type``. +The following example demonstrates a way it can be done. + +.. _fit: ../../../build/html/api.html#autosklearn.classification.AutoSklearnClassifier.fit """ import sklearn.model_selection diff --git a/example/example_holdout.py b/example/example_holdout.py index 9a5df02cc0..4c50c57da7 100644 --- a/example/example_holdout.py +++ b/example/example_holdout.py @@ -2,6 +2,13 @@ ======= Holdout ======= + + +In *auto-sklearn* it is possible to use different resampling strategies +by specifying the parameters ``resampling_strategy`` and +``resampling_strategy_arguments``. The following example shows how to use the +holdout method as well as set the train-test split ratio when instantiating +``AutoSklearnClassifier``. """ import sklearn.model_selection @@ -22,6 +29,8 @@ def main(): tmp_folder='/tmp/autosklearn_holdout_example_tmp', output_folder='/tmp/autosklearn_holdout_example_out', disable_evaluator_output=False, + resampling_strategy='holdout', + resampling_strategy_arguments={'train_size': 0.67} ) automl.fit(X_train, y_train, dataset_name='digits') diff --git a/example/example_metrics.py b/example/example_metrics.py index 6b154e24b0..67ce6ea815 100644 --- a/example/example_metrics.py +++ b/example/example_metrics.py @@ -3,6 +3,13 @@ ======= Metrics ======= + +*Auto-sklearn* supports various built-in metrics, which can be found `here `_. +However, it is also possible to define your own metric and use it to fit your model. +The following examples show how to use built-in as well as self-defined metrics +for a classification problem. + +.. _metric: ../../../build/html/api.html#built-in-metrics """ import numpy as np @@ -20,12 +27,21 @@ def accuracy(solution, prediction): # function defining accuracy return np.mean(solution == prediction) +def error(solution, prediction): + # function defining error + return np.mean(solution != prediction) + def accuracy_wk(solution, prediction, dummy): # function defining accuracy and accepting an additional argument assert dummy is None return np.mean(solution == prediction) +def error_wk(solution, prediction, dummy): + # function defining error and accepting an additional argument + assert dummy is None + return np.mean(solution == prediction) + def main(): @@ -61,6 +77,7 @@ def main(): accuracy_scorer = autosklearn.metrics.make_scorer( name="accu", score_func=accuracy, + optimum=1, greater_is_better=True, needs_proba=False, needs_threshold=False, @@ -77,12 +94,35 @@ def main(): format(sklearn.metrics.accuracy_score(y_test, predictions), cls._automl._metric.name)) + print("#"*80) + print("Use self defined error metric") + error_rate = autosklearn.metrics.make_scorer( + name='error', + score_func=error, + optimum=0, + greater_is_better=False, + needs_proba=False, + needs_threshold=False + ) + cls = autosklearn.classification.AutoSklearnClassifier( + time_left_for_this_task=60, + per_run_time_limit=30, + seed=1 + ) + cls.fit(X_train, y_train, metric=error_rate) + + cls.predictions = cls.predict(X_test) + print("Error rate {:g} using {:s}". + format(error_rate(y_test, predictions), + cls._automl._metric.name)) + # Third example: Use own accuracy metric with additional argument print("#"*80) print("Use self defined accuracy with additional argument") accuracy_scorer = autosklearn.metrics.make_scorer( name="accu_add", score_func=accuracy_wk, + optimum=1, greater_is_better=True, needs_proba=False, needs_threshold=False, @@ -103,6 +143,31 @@ def main(): ) ) + print("#"*80) + print("Use self defined error with additional argument") + error_rate = autosklearn.metrics.make_scorer( + name="error_add", + score_func=error_wk, + optimum=0, + greater_is_better=True, + needs_proba=False, + needs_threshold=False, + dummy=None, + ) + cls = autosklearn.classification.AutoSklearnClassifier( + time_left_for_this_task=60, + per_run_time_limit=30, + seed=1, + ) + cls.fit(X_train, y_train, metric=error_rate) + + predictions = cls.predict(X_test) + print( + "Error rate {:g} using {:s}".format( + error_rate(y_test, predictions), + cls._automl._metric.name + ) + ) if __name__ == "__main__": main() diff --git a/example/example_parallel.py b/example/example_parallel.py index fac02299c0..3e1b01c851 100644 --- a/example/example_parallel.py +++ b/example/example_parallel.py @@ -3,6 +3,8 @@ ==================== Parallel Computation ==================== + +Example description goes here. """ import multiprocessing diff --git a/example/example_random_search.py b/example/example_random_search.py index 1a6b37dac5..bb36ce87dc 100644 --- a/example/example_random_search.py +++ b/example/example_random_search.py @@ -2,6 +2,8 @@ ============= Random Search ============= + +Example description goes here. """ import sklearn.model_selection diff --git a/example/example_regression.py b/example/example_regression.py index 2ee85287f2..801cbf2517 100644 --- a/example/example_regression.py +++ b/example/example_regression.py @@ -4,6 +4,8 @@ ========== Regression ========== + +Example description here. """ import sklearn.model_selection import sklearn.datasets diff --git a/example/example_sequential.py b/example/example_sequential.py index c28b125d97..1d73eb2fb2 100644 --- a/example/example_sequential.py +++ b/example/example_sequential.py @@ -3,6 +3,8 @@ ================ Sequential Usage ================ + +Example description goes here. """ import sklearn.model_selection import sklearn.datasets From a432a28f0b966fcade93b6ff41ce9ee57c4bbac2 Mon Sep 17 00:00:00 2001 From: Jinu Date: Tue, 24 Apr 2018 22:17:53 +0200 Subject: [PATCH 3/7] add documentation to examples; minor changes in conf.py --- doc/conf.py | 12 ++---------- example/example_crossvalidation.py | 8 ++++++-- example/example_feature_types.py | 1 - example/example_holdout.py | 1 - example/example_metrics.py | 16 ++++++++-------- example/example_parallel.py | 15 +++++++++++---- example/example_random_search.py | 14 +++++++------- example/example_regression.py | 3 ++- example/example_sequential.py | 7 ++++++- 9 files changed, 42 insertions(+), 35 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index ce5407e8ea..a20d3ec735 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -50,17 +50,9 @@ 'examples_dirs': '../example', # path where to save gallery generated examples 'gallery_dirs': 'examples', - #'subsection_order': ExplicitOrder(['../example/example_holdout', - # '../example/example_crossvalidation', - # '../example/example_parallel', - # '../example/example_sequential', - # '../example/example_regression', - # '../example/example_feature_types', - # '../example/example_metrics', - # '../example/example_random_search', - # '../example/example_eips']), 'within_subsection_order': FileNameSortKey, - 'thumbnail_size': (10, 10) + #TODO: It seems that sphinx-gallery does not support removing the thumbnails. + #'thumbnail_size': (600, 600) } # Configure the extensions diff --git a/example/example_crossvalidation.py b/example/example_crossvalidation.py index 8b6fc5c9ff..28c8b72c35 100644 --- a/example/example_crossvalidation.py +++ b/example/example_crossvalidation.py @@ -1,10 +1,14 @@ # -*- encoding: utf-8 -*- """ ================ -Cross Validation +Cross-Validation ================ -The following example shows how to use cross-validation in *auto-sklearn*. +In *auto-sklearn* it is possible to use different resampling strategies +by specifying the parameters ``resampling_strategy`` and +``resampling_strategy_arguments``. The following example shows how to use the +cross-validation method as well as set its folds when instantiating +``AutoSklearnClassifier``. """ import sklearn.model_selection diff --git a/example/example_feature_types.py b/example/example_feature_types.py index 9dd6cdf557..a5526152e5 100644 --- a/example/example_feature_types.py +++ b/example/example_feature_types.py @@ -4,7 +4,6 @@ Feature Types ============= -Example description goes here. In *auto-sklearn* it is possible to specify the feature types of a dataset when calling the `fit() `_ method by specifying the argument ``feat_type``. The following example demonstrates a way it can be done. diff --git a/example/example_holdout.py b/example/example_holdout.py index 4c50c57da7..84e98a73a2 100644 --- a/example/example_holdout.py +++ b/example/example_holdout.py @@ -3,7 +3,6 @@ Holdout ======= - In *auto-sklearn* it is possible to use different resampling strategies by specifying the parameters ``resampling_strategy`` and ``resampling_strategy_arguments``. The following example shows how to use the diff --git a/example/example_metrics.py b/example/example_metrics.py index 67ce6ea815..055fc2dade 100644 --- a/example/example_metrics.py +++ b/example/example_metrics.py @@ -5,9 +5,9 @@ ======= *Auto-sklearn* supports various built-in metrics, which can be found `here `_. -However, it is also possible to define your own metric and use it to fit your model. -The following examples show how to use built-in as well as self-defined metrics -for a classification problem. +However, it is also possible to define your own metric and use it to fit and +evaluate your model. The following examples show how to use built-in as well +as self-defined metrics for a sample classification problem. .. _metric: ../../../build/html/api.html#built-in-metrics """ @@ -24,23 +24,23 @@ def accuracy(solution, prediction): - # function defining accuracy + # custom function defining accuracy return np.mean(solution == prediction) def error(solution, prediction): - # function defining error + # custom function defining error return np.mean(solution != prediction) def accuracy_wk(solution, prediction, dummy): - # function defining accuracy and accepting an additional argument + # custom function defining accuracy and accepting an additional argument assert dummy is None return np.mean(solution == prediction) def error_wk(solution, prediction, dummy): - # function defining error and accepting an additional argument + # custom function defining error and accepting an additional argument assert dummy is None - return np.mean(solution == prediction) + return np.mean(solution != prediction) def main(): diff --git a/example/example_parallel.py b/example/example_parallel.py index 3e1b01c851..517bdb5dbf 100644 --- a/example/example_parallel.py +++ b/example/example_parallel.py @@ -1,11 +1,14 @@ # -*- encoding: utf-8 -*- """ ==================== -Parallel Computation +Parallel Usage ==================== -Example description goes here. +*Auto-sklearn* allows users to determine whether to run multiple instances of it +either in parallel, using several cores, or sequentially, using just one core, +depending on their need. This example shows how to execute *auto-sklearn* in parallel. """ +#TODO: add link to Psmac? Anyhting more specific? import multiprocessing import shutil @@ -70,8 +73,8 @@ def spawn_classifier(seed, dataset_name): return spawn_classifier -if __name__ == '__main__': - +def main(): + X, y = sklearn.datasets.load_digits(return_X_y=True) X_train, X_test, y_train, y_test = \ sklearn.model_selection.train_test_split(X, y, random_state=1) @@ -114,3 +117,7 @@ def spawn_classifier(seed, dataset_name): predictions = automl.predict(X_test) print(automl.show_models()) print("Accuracy score", sklearn.metrics.accuracy_score(y_test, predictions)) + + +if __name__ == '__main__': + main() diff --git a/example/example_random_search.py b/example/example_random_search.py index bb36ce87dc..fe5e6a1f9a 100644 --- a/example/example_random_search.py +++ b/example/example_random_search.py @@ -3,7 +3,11 @@ Random Search ============= -Example description goes here. +A crucial feature of *auto-sklearn* is automatically optimizing the hyperparameters +through SMAC, introduced `here `_. +Additionally, it is possible to use `random search `_ +instead of SMAC, as demonstrated in the example below. Since this is an +advanced feature, we recommend first getting familiar with SMAC before using it. """ import sklearn.model_selection @@ -25,9 +29,7 @@ def get_roar_object_callback( runhistory, run_id, ): - """Random online adaptive racing. - - http://ml.informatik.uni-freiburg.de/papers/11-LION5-SMAC.pdf""" + """Random online adaptive racing.""" scenario_dict['input_psmac_dirs'] = backend.get_smac_output_glob() scenario = Scenario(scenario_dict) return ROAR( @@ -48,9 +50,7 @@ def get_random_search_object_callback( runhistory, run_id, ): - """Random search. - - http://www.jmlr.org/papers/v13/bergstra12a.html""" + """Random search.""" scenario_dict['input_psmac_dirs'] = backend.get_smac_output_glob() scenario_dict['minR'] = len(scenario_dict['instances']) scenario_dict['initial_incumbent'] = 'RANDOM' diff --git a/example/example_regression.py b/example/example_regression.py index 801cbf2517..34c4411551 100644 --- a/example/example_regression.py +++ b/example/example_regression.py @@ -5,7 +5,8 @@ Regression ========== -Example description here. +The following example shows how to fit a simple regression model with +*auto-sklearn*. """ import sklearn.model_selection import sklearn.datasets diff --git a/example/example_sequential.py b/example/example_sequential.py index 1d73eb2fb2..93e3552266 100644 --- a/example/example_sequential.py +++ b/example/example_sequential.py @@ -4,8 +4,13 @@ Sequential Usage ================ -Example description goes here. +*Auto-sklearn* allows users to determine whether to run multiple instances of it +either in parallel, using several cores, or sequentially, using just one core, +depending on their need. This example demonstrates how to appropriately set +the arguments to run *auto-sklearn* sequentially, and build the ensemble +afterwards. """ + import sklearn.model_selection import sklearn.datasets import sklearn.metrics From 1e76a2f2dbf2155b424b7222d938a58d03f1aa97 Mon Sep 17 00:00:00 2001 From: Jinu Date: Tue, 24 Apr 2018 22:27:41 +0200 Subject: [PATCH 4/7] . --- doc/examples.rst | 4 ---- example/example_crossvalidation.py | 1 + example/example_holdout.py | 3 +++ example/example_regression.py | 1 - example/example_sequential.py | 1 - 5 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 doc/examples.rst diff --git a/doc/examples.rst b/doc/examples.rst deleted file mode 100644 index 6b6db9d8e7..0000000000 --- a/doc/examples.rst +++ /dev/null @@ -1,4 +0,0 @@ -:orphan: - -.. _examples: - diff --git a/example/example_crossvalidation.py b/example/example_crossvalidation.py index 28c8b72c35..26418ab132 100644 --- a/example/example_crossvalidation.py +++ b/example/example_crossvalidation.py @@ -17,6 +17,7 @@ import autosklearn.classification + def main(): X, y = sklearn.datasets.load_digits(return_X_y=True) X_train, X_test, y_train, y_test = \ diff --git a/example/example_holdout.py b/example/example_holdout.py index 84e98a73a2..175af7684e 100644 --- a/example/example_holdout.py +++ b/example/example_holdout.py @@ -28,6 +28,9 @@ def main(): tmp_folder='/tmp/autosklearn_holdout_example_tmp', output_folder='/tmp/autosklearn_holdout_example_out', disable_evaluator_output=False, + # 'holdout' with 'train_size'=0.67 is the default argument setting + # for AutoSklearnClassifier. It is explicitly specified in this example + # for demonstrational purpose. resampling_strategy='holdout', resampling_strategy_arguments={'train_size': 0.67} ) diff --git a/example/example_regression.py b/example/example_regression.py index 34c4411551..0ce57b9472 100644 --- a/example/example_regression.py +++ b/example/example_regression.py @@ -1,5 +1,4 @@ # -*- encoding: utf-8 -*- - """ ========== Regression diff --git a/example/example_sequential.py b/example/example_sequential.py index 93e3552266..a887b414df 100644 --- a/example/example_sequential.py +++ b/example/example_sequential.py @@ -1,4 +1,3 @@ - """ ================ Sequential Usage From b498cb0d89cb4dc66a9c232e84b061bdbd4f0a41 Mon Sep 17 00:00:00 2001 From: Jinu Date: Wed, 25 Apr 2018 13:43:04 +0200 Subject: [PATCH 5/7] . --- doc/Makefile | 2 +- doc/conf.py | 10 ++++++---- doc/manual.rst | 16 +++++++++------- {example => examples}/README.txt | 0 {example => examples}/example_crossvalidation.py | 0 {example => examples}/example_eips.py | 0 {example => examples}/example_feature_types.py | 0 {example => examples}/example_holdout.py | 0 {example => examples}/example_metrics.py | 0 {example => examples}/example_parallel.py | 0 {example => examples}/example_random_search.py | 0 {example => examples}/example_regression.py | 0 {example => examples}/example_sequential.py | 0 13 files changed, 16 insertions(+), 12 deletions(-) rename {example => examples}/README.txt (100%) rename {example => examples}/example_crossvalidation.py (100%) rename {example => examples}/example_eips.py (100%) rename {example => examples}/example_feature_types.py (100%) rename {example => examples}/example_holdout.py (100%) rename {example => examples}/example_metrics.py (100%) rename {example => examples}/example_parallel.py (100%) rename {example => examples}/example_random_search.py (100%) rename {example => examples}/example_regression.py (100%) rename {example => examples}/example_sequential.py (100%) diff --git a/doc/Makefile b/doc/Makefile index 8dca341ddd..d96822320d 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -51,7 +51,7 @@ help: clean: rm -rf $(BUILDDIR)/* rm -rf generated - rm -rf auto_examples/ + rm -rf examples/ html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/doc/conf.py b/doc/conf.py index a20d3ec735..623ad554d9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -19,7 +19,6 @@ import os import sys import sphinx_bootstrap_theme -from sphinx_gallery.sorting import FileNameSortKey import autosklearn # Add the parent directory of this file to the PYTHONPATH import os @@ -44,15 +43,18 @@ 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', 'sphinx_gallery.gen_gallery', 'numpydoc'] +from sphinx_gallery.sorting import ExplicitOrder, FileNameSortKey + # Sphinx-gallery configuration. sphinx_gallery_conf = { # path to the examples - 'examples_dirs': '../example', + 'examples_dirs': '../examples', # path where to save gallery generated examples 'gallery_dirs': 'examples', - 'within_subsection_order': FileNameSortKey, - #TODO: It seems that sphinx-gallery does not support removing the thumbnails. + # It seems that sphinx-gallery does not support removing the thumbnails. Also, + # the example files cannot be sorted in custom order #'thumbnail_size': (600, 600) + #TODO: add sphinx-gallery to the list of libs to be imported } # Configure the extensions diff --git a/doc/manual.rst b/doc/manual.rst index c9f581def7..256452077a 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -15,13 +15,15 @@ Examples *auto-sklearn* comes with the following examples which demonstrate several aspects of its usage: -* `Holdout `_ -* `Cross-validation `_ -* `Parallel usage `_ -* `Sequential usage `_ -* `Regression `_ -* `Continuous and categorical data `_ -* `Using custom metrics `_ +* `Holdout `_ +* `Cross-validation `_ +* `Parallel usage `_ +* `Sequential usage `_ +* `Regression `_ +* `Continuous and categorical data `_ +* `Using custom metrics `_ +* `Random search `_ +* `EIPS `_ Time and memory limits diff --git a/example/README.txt b/examples/README.txt similarity index 100% rename from example/README.txt rename to examples/README.txt diff --git a/example/example_crossvalidation.py b/examples/example_crossvalidation.py similarity index 100% rename from example/example_crossvalidation.py rename to examples/example_crossvalidation.py diff --git a/example/example_eips.py b/examples/example_eips.py similarity index 100% rename from example/example_eips.py rename to examples/example_eips.py diff --git a/example/example_feature_types.py b/examples/example_feature_types.py similarity index 100% rename from example/example_feature_types.py rename to examples/example_feature_types.py diff --git a/example/example_holdout.py b/examples/example_holdout.py similarity index 100% rename from example/example_holdout.py rename to examples/example_holdout.py diff --git a/example/example_metrics.py b/examples/example_metrics.py similarity index 100% rename from example/example_metrics.py rename to examples/example_metrics.py diff --git a/example/example_parallel.py b/examples/example_parallel.py similarity index 100% rename from example/example_parallel.py rename to examples/example_parallel.py diff --git a/example/example_random_search.py b/examples/example_random_search.py similarity index 100% rename from example/example_random_search.py rename to examples/example_random_search.py diff --git a/example/example_regression.py b/examples/example_regression.py similarity index 100% rename from example/example_regression.py rename to examples/example_regression.py diff --git a/example/example_sequential.py b/examples/example_sequential.py similarity index 100% rename from example/example_sequential.py rename to examples/example_sequential.py From 273f269d3949a1560e136ad38bca94e166f4d86f Mon Sep 17 00:00:00 2001 From: Jinu Date: Mon, 30 Apr 2018 20:10:30 +0200 Subject: [PATCH 6/7] . --- autosklearn/estimators.py | 4 ++-- doc/Makefile | 1 + doc/api.rst | 2 ++ doc/conf.py | 29 ++++++++++++++++++++--------- doc/index.rst | 2 +- examples/README.txt | 9 +++++++++ examples/example_crossvalidation.py | 8 +++++--- examples/example_feature_types.py | 4 +--- examples/example_holdout.py | 2 +- examples/example_metrics.py | 11 +++++------ examples/example_parallel.py | 11 +++++++---- examples/example_random_search.py | 3 +-- examples/example_regression.py | 3 ++- examples/example_sequential.py | 9 ++++----- 14 files changed, 61 insertions(+), 37 deletions(-) diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py index 2b6febb30b..4a251b7115 100644 --- a/autosklearn/estimators.py +++ b/autosklearn/estimators.py @@ -408,7 +408,7 @@ def fit(self, X, y, """Fit *auto-sklearn* to given training set (X, y). Fit both optimizes the machine learning models and builds an ensemble - out of them. To disable ensembling, set ``ensemble_size==1``. + out of them. To disable ensembling, set ``ensemble_size==0``. Parameters ---------- @@ -512,7 +512,7 @@ def fit(self, X, y, """Fit *Auto-sklearn* to given training set (X, y). Fit both optimizes the machine learning models and builds an ensemble - out of them. To disable ensembling, set ``ensemble_size==1``. + out of them. To disable ensembling, set ``ensemble_size==0``. Parameters ---------- diff --git a/doc/Makefile b/doc/Makefile index d96822320d..9355370597 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -52,6 +52,7 @@ clean: rm -rf $(BUILDDIR)/* rm -rf generated rm -rf examples/ + rm -rf gen_modules/ html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/doc/api.rst b/doc/api.rst index 1ade6bcd8a..34107e78a7 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -103,3 +103,5 @@ Extension Interfaces .. autoclass:: autosklearn.pipeline.components.base.AutoSklearnPreprocessingAlgorithm :members: + + diff --git a/doc/conf.py b/doc/conf.py index 623ad554d9..0e650b4f69 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -41,26 +41,37 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', - 'sphinx_gallery.gen_gallery', 'numpydoc'] + 'sphinx_gallery.gen_gallery', 'sphinx.ext.autosectionlabel', + # sphinx.ext.autosexctionlabel raises duplicate label warnings + # because same section headers are used multiple times throughout + # the documentation. + 'numpydoc'] + from sphinx_gallery.sorting import ExplicitOrder, FileNameSortKey +# Configure the extensions +numpydoc_show_class_members = False +autosummary_generate = True + +# prefix each section label with the name of the document it is in, in order to avoid +# ambiguity when there are multiple same section labels in different documents. +autosectionlabel_prefix_document = True + # Sphinx-gallery configuration. sphinx_gallery_conf = { # path to the examples 'examples_dirs': '../examples', # path where to save gallery generated examples 'gallery_dirs': 'examples', - # It seems that sphinx-gallery does not support removing the thumbnails. Also, - # the example files cannot be sorted in custom order - #'thumbnail_size': (600, 600) - #TODO: add sphinx-gallery to the list of libs to be imported + #TODO: fix back/forward references for the examples. + #'doc_module': ('autosklearn'), + #'reference_url': { + # 'autosklearn': None + #}, + #'backreferences_dir': 'gen_modules/backreferences' } -# Configure the extensions -numpydoc_show_class_members = False -autosummary_generate = True - # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/doc/index.rst b/doc/index.rst index bb4826625d..4af101979b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -47,7 +47,7 @@ Manual * :ref:`installation` * :ref:`manual` -* :ref:`API` +* :ref:`api` * :ref:`extending` diff --git a/examples/README.txt b/examples/README.txt index e69de29bb2..2878fe2917 100644 --- a/examples/README.txt +++ b/examples/README.txt @@ -0,0 +1,9 @@ +:orphan: + +.. _examples: + +======== +Examples +======== + +General introductory examples for *auto-sklearn* can be found here. diff --git a/examples/example_crossvalidation.py b/examples/example_crossvalidation.py index 26418ab132..85530b591b 100644 --- a/examples/example_crossvalidation.py +++ b/examples/example_crossvalidation.py @@ -1,13 +1,15 @@ # -*- encoding: utf-8 -*- + + """ ================ Cross-Validation ================ In *auto-sklearn* it is possible to use different resampling strategies -by specifying the parameters ``resampling_strategy`` and -``resampling_strategy_arguments``. The following example shows how to use the -cross-validation method as well as set its folds when instantiating +by specifying the arguments ``resampling_strategy`` and +``resampling_strategy_arguments``. The following example shows how to use +cross-validation and how to set the folds when instantiating ``AutoSklearnClassifier``. """ diff --git a/examples/example_feature_types.py b/examples/example_feature_types.py index a5526152e5..c33f77c195 100644 --- a/examples/example_feature_types.py +++ b/examples/example_feature_types.py @@ -5,10 +5,8 @@ ============= In *auto-sklearn* it is possible to specify the feature types of a dataset when -calling the `fit() `_ method by specifying the argument ``feat_type``. +calling the method :meth:`fit() ` by specifying the argument ``feat_type``. The following example demonstrates a way it can be done. - -.. _fit: ../../../build/html/api.html#autosklearn.classification.AutoSklearnClassifier.fit """ import sklearn.model_selection diff --git a/examples/example_holdout.py b/examples/example_holdout.py index 175af7684e..fe1ff1c7a7 100644 --- a/examples/example_holdout.py +++ b/examples/example_holdout.py @@ -4,7 +4,7 @@ ======= In *auto-sklearn* it is possible to use different resampling strategies -by specifying the parameters ``resampling_strategy`` and +by specifying the arguments ``resampling_strategy`` and ``resampling_strategy_arguments``. The following example shows how to use the holdout method as well as set the train-test split ratio when instantiating ``AutoSklearnClassifier``. diff --git a/examples/example_metrics.py b/examples/example_metrics.py index 055fc2dade..ebd86ec0e0 100644 --- a/examples/example_metrics.py +++ b/examples/example_metrics.py @@ -4,12 +4,11 @@ Metrics ======= -*Auto-sklearn* supports various built-in metrics, which can be found `here `_. -However, it is also possible to define your own metric and use it to fit and -evaluate your model. The following examples show how to use built-in as well -as self-defined metrics for a sample classification problem. - -.. _metric: ../../../build/html/api.html#built-in-metrics +*Auto-sklearn* supports various built-in metrics, which can be found in the +:ref:`metrics section in the API `. However, it is also +possible to define your own metric and use it to fit and evaluate your model. +The following examples show how to use built-in and self-defined metrics for a +classification problem. """ import numpy as np diff --git a/examples/example_parallel.py b/examples/example_parallel.py index 517bdb5dbf..f5572ab97d 100644 --- a/examples/example_parallel.py +++ b/examples/example_parallel.py @@ -4,11 +4,14 @@ Parallel Usage ==================== -*Auto-sklearn* allows users to determine whether to run multiple instances of it -either in parallel, using several cores, or sequentially, using just one core, -depending on their need. This example shows how to execute *auto-sklearn* in parallel. +*Auto-sklearn* uses *SMAC* to automatically optimize the hyperparameters of +the training models. A variant of *SMAC*, called *pSMAC* (parallel SMAC), +provides a means of running several instances of *auto-sklearn* in a parallel +mode using several computational resources (detailed information of +*pSMAC* can be found `here `_). +This example shows the necessary steps to configure *auto-sklearn* in +parallel mode. """ -#TODO: add link to Psmac? Anyhting more specific? import multiprocessing import shutil diff --git a/examples/example_random_search.py b/examples/example_random_search.py index fe5e6a1f9a..5759d9af0e 100644 --- a/examples/example_random_search.py +++ b/examples/example_random_search.py @@ -6,8 +6,7 @@ A crucial feature of *auto-sklearn* is automatically optimizing the hyperparameters through SMAC, introduced `here `_. Additionally, it is possible to use `random search `_ -instead of SMAC, as demonstrated in the example below. Since this is an -advanced feature, we recommend first getting familiar with SMAC before using it. +instead of SMAC, as demonstrated in the example below. """ import sklearn.model_selection diff --git a/examples/example_regression.py b/examples/example_regression.py index 0ce57b9472..738409df39 100644 --- a/examples/example_regression.py +++ b/examples/example_regression.py @@ -11,8 +11,9 @@ import sklearn.datasets import sklearn.metrics -import autosklearn.regression +from autosklearn.regression import AutoSklearnRegressor +import autosklearn.regression def main(): X, y = sklearn.datasets.load_boston(return_X_y=True) diff --git a/examples/example_sequential.py b/examples/example_sequential.py index a887b414df..06820e7ebe 100644 --- a/examples/example_sequential.py +++ b/examples/example_sequential.py @@ -3,11 +3,10 @@ Sequential Usage ================ -*Auto-sklearn* allows users to determine whether to run multiple instances of it -either in parallel, using several cores, or sequentially, using just one core, -depending on their need. This example demonstrates how to appropriately set -the arguments to run *auto-sklearn* sequentially, and build the ensemble -afterwards. +By default, *auto-sklearn* fits the machine learning models and build their +ensembles in parallel. However, it is also possible to run the two processes +sequentially. The example below shows how to first fit the models and build the +ensembles afterwards. """ import sklearn.model_selection From 522d57f196035e7150156918be9be1837a3c7a51 Mon Sep 17 00:00:00 2001 From: Jinu Date: Mon, 30 Apr 2018 20:15:32 +0200 Subject: [PATCH 7/7] . --- examples/example_regression.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/example_regression.py b/examples/example_regression.py index 738409df39..79b1a18f08 100644 --- a/examples/example_regression.py +++ b/examples/example_regression.py @@ -11,8 +11,6 @@ import sklearn.datasets import sklearn.metrics -from autosklearn.regression import AutoSklearnRegressor - import autosklearn.regression def main():