Paper: Better and faster hyper-parameter optimization with Dask

papers/scott_sievert/refs.bib

@@ -0,0 +1,347 @@
+@article{bergstra2012random,
+    author = {Bergstra, James and Bengio, Yoshua},
+    title = {Random search for hyper-parameter optimization},
+    journal = {Journal of Machine Learning Research},
+    volume = {13},
+    number = {Feb},
+    pages = {281–281},
+    year = {2012},
+    url = {http://jmlr.csail.mit.edu/papers/v13/bergstra12a.html},
+    abstract = {},
+    location = {}}
+
+@article{pedregosa2011,
+    author = {Pedregosa, Fabian and Varoquaux, Gaël and Gramfort, Alexandre and Michel, Vincent and Thirion, Bertrand and Grisel, Olivier and Blondel, Mathieu and Prettenhofer, Peter and Weiss, Ron and Dubourg, Vincent},
+    title = {Scikit-learn: Machine learning in Python},
+    journal = {Journal of machine learning research},
+    volume = {12},
+    number = {Oct},
+    pages = {2825–2830},
+    year = {2011},
+    url = {http://jmlr.csail.mit.edu/papers/v12/pedregosa11a.html},
+}
+
+@article{li2016hyperband,
+  author  = {Lisha Li and Kevin Jamieson and Giulia DeSalvo and Afshin Rostamizadeh and Ameet Talwalkar},
+  title   = {Hyperband: A Novel Bandit-Based Approach to Hyperparameter Optimization},
+  journal = {Journal of Machine Learning Research},
+  year    = {2018},
+  volume  = {18},
+  number  = {185},
+  pages   = {1-52},
+  url     = {http://jmlr.org/papers/v18/16-558.html}
+}
+
+@proceedings{hutter2011,
+author = {Hutter, Frank and Hoos, Holger H and Leyton-Brown, Kevin},
+editor = {},
+title = {Sequential model-based optimization for general algorithm configuration},
+booktitle = {Sequential model-based optimization for general algorithm configuration},
+volume = {International Conference on Learning and Intelligent Optimization},
+publisher = {Springer},
+address = {},
+pages = {507-523},
+year = {2011},
+doi = {10.1007/978-3-642-25566-3_40},
+}
+
+
+@proceedings{bergstra2011,
+title = {Algorithms for Hyper-Parameter Optimization},
+author = {James S. Bergstra and Bardenet, R\'{e}mi and Bengio, Yoshua and Bal\'{a}zs K\'{e}gl},
+booktitle = {Advances in Neural Information Processing Systems 24},
+editor = {J. Shawe-Taylor and R. S. Zemel and P. L. Bartlett and F. Pereira and K. Q. Weinberger},
+pages = {2546--2554},
+year = {2011},
+publisher = {Curran Associates, Inc.},
+url = {http://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf}
+}
+
+@proceedings{snoek2012,
+title = {Practical Bayesian Optimization of Machine Learning Algorithms},
+author = {Snoek, Jasper and Larochelle, Hugo and Adams, Ryan P},
+booktitle = {Advances in Neural Information Processing Systems 25},
+editor = {F. Pereira and C. J. C. Burges and L. Bottou and K. Q. Weinberger},
+pages = {2951--2959},
+year = {2012},
+publisher = {Curran Associates, Inc.},
+url = {http://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf}
+}
+
+
+
+@inproceedings{kleinbayesopt17,
+author    = {A. Klein and S. Falkner and N. Mansur and F. Hutter},
+title     = {RoBO: A Flexible and Robust Bayesian Optimization Framework in Python},
+booktitle = {NIPS 2017 Bayesian Optimization Workshop},
+year      = {2017},
+month     = dec,
+url = {https://github.com/automl/RoBO},
+}
+
+@Article{falkner2018,
+  title = 	 {{BOHB}: Robust and Efficient Hyperparameter Optimization at Scale},
+  author = 	 {Falkner, Stefan and Klein, Aaron and Hutter, Frank},
+  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
+  pages = 	 {1437--1446},
+  year = 	 {2018},
+  editor = 	 {Dy, Jennifer and Krause, Andreas},
+  volume = 	 {80},
+  series = 	 {Proceedings of Machine Learning Research},
+  address = 	 {Stockholmsmässan, Stockholm Sweden},
+  month = 	 {10--15 Jul},
+  publisher = 	 {PMLR},
+  pdf = 	 {http://proceedings.mlr.press/v80/falkner18a/falkner18a.pdf},
+  url = 	 {http://proceedings.mlr.press/v80/falkner18a.html},
+}
+
+@Article{klein2016,
+author = {Klein, Aaron and Falkner, Stefan and Bartels, Simon and Hennig, Philipp and Hutter, Frank},
+title = {Fast bayesian optimization of machine learning hyperparameters on large datasets},
+journal = {arXiv preprint arXiv:1605.07079},
+volume = {},
+number = {},
+pages = {},
+url = {https://arxiv.org/abs/1605.07079},
+year = {2016}}
+
+
+@Article{tibshirani1996,
+author = {Tibshirani, Robert},
+title = {Regression shrinkage and selection via the lasso},
+journal = {Journal of the Royal Statistical Society: Series B (Methodological)},
+volume = {58},
+number = {1},
+pages = {267–288},
+doi = {10.1111/j.2517-6161.1996.tb02080.x},
+year = {1996}}
+
+@Article{marquardt1975,
+author = { Donald W.   Marquardt  and  Ronald D.   Snee },
+title = {Ridge Regression in Practice},
+journal = {The American Statistician},
+volume = {29},
+number = {1},
+pages = {3-20},
+year  = {1975},
+publisher = {Taylor & Francis},
+doi = {10.1080/00031305.1975.10479105}}
+
+@Article{wattenberg2016,
+  author = {Wattenberg, Martin and Viégas, Fernanda and Johnson, Ian},
+  title = {How to Use t-SNE Effectively},
+  journal = {Distill},
+  year = {2016},
+  url = {http://distill.pub/2016/misread-tsne},
+  doi = {10.23915/distill.00002}
+  }
+
+
+@Article{kaufmann2015complexity,
+  author  = {Emilie Kaufmann and Olivier Capp{{\'e}} and Aur{{\'e}}lien Garivier},
+  title   = {On the Complexity of Best-Arm Identification in Multi-Armed Bandit Models},
+  journal = {Journal of Machine Learning Research},
+  year    = {2016},
+  volume  = {17},
+  number  = {1},
+  pages   = {1-42},
+  url     = {http://jmlr.org/papers/v17/kaufman16a.html}
+}
+
+@Incollection{bottou2012stochastic,
+  author = {Bottou, L\'{e}on},
+  title = {Stochastic Gradient Tricks},
+  booktitle = {Neural Networks, Tricks of the Trade, Reloaded},
+  pages = {430--445},
+  editor = {Montavon, Gr\'{e}goire and Orr, Genevieve B. and M\"{u}ller, Klaus-Robert},
+  series = {Lecture Notes in Computer Science (LNCS 7700)},
+  publisher = {Springer},
+  year = {2012},
+  url = {http://leon.bottou.org/papers/bottou-tricks-2012},
+  }
+
+@InProceedings{shamir2013,
+  title = 	 {Stochastic Gradient Descent for Non-smooth Optimization: Convergence Results and Optimal Averaging Schemes},
+  author = 	 {Ohad Shamir and Tong Zhang},
+  booktitle = 	 {Proceedings of the 30th International Conference on Machine Learning},
+  pages = 	 {71--79},
+  year = 	 {2013},
+  editor = 	 {Sanjoy Dasgupta and David McAllester},
+  volume = 	 {28},
+  number =       {1},
+  series = 	 {Proceedings of Machine Learning Research},
+  address = 	 {Atlanta, Georgia, USA},
+  month = 	 {17--19 Jun},
+  publisher = 	 {PMLR},
+  pdf = 	 {http://proceedings.mlr.press/v28/shamir13.pdf},
+  url = 	 {http://proceedings.mlr.press/v28/shamir13.html},
+}
+
+
+@article{prechelt1998automatic,
+  title={Automatic early stopping using cross validation: quantifying the criteria},
+  author={Prechelt, Lutz},
+  journal={Neural Networks},
+  volume={11},
+  number={4},
+  pages={761--767},
+  year={1998},
+  publisher={Elsevier},
+  doi = {10.1016/S0893-6080(98)00010-0},
+}
+
+@Incollection{bottou2010large,
+  author = {Bottou, L\'{e}on},
+  title = {Large-Scale Machine Learning with Stochastic Gradient Descent},
+  year = {2010},
+  booktitle = {Proceedings of the 19th International Conference on Computational Statistics (COMPSTAT'2010)},
+  editor = {Lechevallier, Yves and Saporta, Gilbert},
+  address = {Paris, France},
+  month = {August},
+  publisher = {Springer},
+  pages = {177--187},
+  url = {http://leon.bottou.org/papers/bottou-2010},
+  }
+
+@inproceedings{paszke2017automatic,
+  title={Automatic differentiation in PyTorch},
+  author={Paszke, Adam and Gross, Sam and Chintala, Soumith and Chanan, Gregory and Yang, Edward and DeVito, Zachary and Lin, Zeming and Desmaison, Alban and Antiga, Luca and Lerer, Adam},
+  booktitle={NIPS-W},
+  year={2017},
+  url = {https://openreview.net/pdf?id=BJJsrmfCZ},
+}
+
+@article{maaten2008visualizing,
+  title={Visualizing data using t-SNE},
+  author={Maaten, Laurens van der and Hinton, Geoffrey},
+  journal={Journal of machine learning research},
+  volume={9},
+  number={Nov},
+  pages={2579--2605},
+  year={2008},
+  url={http://jmlr.csail.mit.edu/papers/v9/vandermaaten08a.html},
+}
+
+@Manual{dask,
+  title = {Dask: Library for dynamic task scheduling},
+  author = {{Dask Development Team}},
+  year = {2016},
+  url = {https://dask.org},
+}
+
+@article{gilbert1992global,
+  title={Global convergence properties of conjugate gradient methods for optimization},
+  author={Gilbert, Jean Charles and Nocedal, Jorge},
+  journal={SIAM Journal on optimization},
+  volume={2},
+  number={1},
+  pages={21--42},
+  year={1992},
+  publisher={SIAM},
+  doi={10.1137/0802003},
+}
+
+@incollection{maren2015prob,
+title = {Probabilistic Line Searches for Stochastic Optimization},
+author = {Mahsereci, Maren and Hennig, Philipp},
+booktitle = {Advances in Neural Information Processing Systems 28},
+editor = {C. Cortes and N. D. Lawrence and D. D. Lee and M. Sugiyama and R. Garnett},
+pages = {181--189},
+year = {2015},
+publisher = {Curran Associates, Inc.},
+url = {http://papers.nips.cc/paper/5753-probabilistic-line-searches-for-stochastic-optimization.pdf}
+}
+
+@inproceedings{leaky-relu,
+  title={Rectifier nonlinearities improve neural network acoustic models},
+  author={Maas, Andrew L and Hannun, Awni Y and Ng, Andrew Y},
+  booktitle={Proc. icml},
+  volume={30},
+  number={1},
+  pages={3},
+  year={2013}
+}
+
+@inproceedings{relu,
+  title={Rectified linear units improve restricted boltzmann machines},
+  author={Nair, Vinod and Hinton, Geoffrey E},
+  booktitle={Proceedings of the 27th international conference on machine learning (ICML-10)},
+  pages={807--814},
+  year={2010}
+}
+
+@inproceedings{prelu,
+  title={Delving deep into rectifiers: Surpassing human-level performance on imagenet classification},
+  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
+  booktitle={Proceedings of the IEEE international conference on computer vision},
+  pages={1026--1034},
+  year={2015}
+}
+
+
+@article{elu,
+  title={Fast and accurate deep network learning by exponential linear units (elus)},
+  author={Clevert, Djork-Arn{\'e} and Unterthiner, Thomas and Hochreiter, Sepp},
+  journal={arXiv preprint arXiv:1511.07289},
+  year={2015}
+}
+
+@inproceedings{xavier,
+  title={Understanding the difficulty of training deep feedforward neural networks},
+  author={Glorot, Xavier and Bengio, Yoshua},
+  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
+  pages={249--256},
+  year={2010}
+}
+
+@inproceedings{kaiming,
+  title={Delving deep into rectifiers: Surpassing human-level performance on imagenet classification},
+  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
+  booktitle={Proceedings of the IEEE international conference on computer vision},
+  pages={1026--1034},
+  year={2015}
+}
+
+@article{adam,
+  title={Adam: A method for stochastic optimization},
+  author={Kingma, Diederik P and Ba, Jimmy},
+  journal={arXiv preprint arXiv:1412.6980},
+  year={2014}
+}
+
+@Book{nesterov2013a,
+author = {Nesterov, Yurii},
+title = {Introductory lectures on convex optimization: A basic course},
+volume = {87},
+pages = {},
+editor = {},
+publisher = {Springer Science \& Business Media},
+address = {},
+year = {2013},
+doi = {10.1007/978-1-4419-8853-9},
+keywords = {}}
+
+@Article{bubeck2015convex,
+author = {Bubeck, Sébastien and others},
+title = {Convex optimization: Algorithms and complexity},
+journal = {Foundations and Trends® in Machine Learning},
+volume = {8},
+number = {3-4},
+pages = {231–231},
+year = {2015},
+abstract = {},
+location = {},
+}
+
+@Article{wilson2017b,
+author = {Wilson, Ashia C and Roelofs, Rebecca and Stern, Mitchell and Srebro, Nathan and Recht, Benjamin},
+title = {The Marginal Value of Adaptive Gradient Methods in Machine Learning},
+journal = {arXiv preprint arXiv:1705.08292},
+volume = {},
+number = {},
+pages = {},
+year = {2017},
+location = {}}
+
+
+