Merge pull request #149 from postgresml/montana/lightgbm

add support for lightgbm

Author: montanalow

pgml-docs/docs/user_guides/setup/gpu_support.md

@@ -1,12 +1,33 @@
 # GPU Support
 
-PostgresML is capable of leveraging GPUs when the underlying libraries and hardware are properly configured on the database. 
+PostgresML is capable of leveraging GPUs when the underlying libraries and hardware are properly configured on the database server. 
+
+!!! tip
+    Models trained on GPU will also require GPU support to make predictions.
 
 ## XGBoost 
-XGBoost is currently the only integrated library that provides GPU accellaration. GPU setup for this library is covered in the [xgboost documentation](https://xgboost.readthedocs.io/en/stable/gpu/index.html). Additionally, you'll need to pass `pgml.train('GPU project', hyperparams => '{tree_method: "gpu_hist"}')` to take advantage during training.
+GPU setup for XGBoost is covered in the [xgboost documentation](https://xgboost.readthedocs.io/en/stable/gpu/index.html).
+
+!!! example 
+    ```sql linenums="1"
+        pgml.train(
+            'GPU project', 
+            algorithm => 'xgboost', 
+            hyperparams => '{"tree_method" : "gpu_hist"}'
+        );
+    ```
+
+## LightGBM
+GPU setup for LightGBM is covered in the [lightgbm documentation](https://lightgbm.readthedocs.io/en/latest/GPU-Tutorial.html). 
 
-!!! warning
-    XGBoost models trained on GPU will also require GPU support to make predictions.
+!!! example 
+    ```sql linenums="1"
+        pgml.train(
+            'GPU project', 
+            algorithm => 'lightgbm', 
+            hyperparams => '{"device" : "gpu"}'
+        );
+    ```
 
 ## Scikit-learn
 None of the scikit-learn algorithms natively support GPU devices. There are a few projects to improve scikit performance with additional parralellism, although we currently have not integrated these with PostgresML:

pgml-docs/docs/user_guides/training/algorithm_selection.md

@@ -10,11 +10,12 @@ The PostgresML dashboard makes it easy to compare various algorithms on your dat
 ![Model Selection](/images/dashboard/models.png)
 
 
-## XGBoost
+## Gradient Boosting
 Algorithm | Regression | Classification
 --- | --- | ---
 `xgboost` | [XGBRegressor](https://xgboost.readthedocs.io/en/stable/python/python_api.html#xgboost.XGBRegressor) | [XGBClassifier](https://xgboost.readthedocs.io/en/stable/python/python_api.html#xgboost.XGBClassifier)
 `xgboost_random_forest` | [XGBRFRegressor](https://xgboost.readthedocs.io/en/stable/python/python_api.html#xgboost.XGBRFRegressor) | [XGBRFClassifier](https://xgboost.readthedocs.io/en/stable/python/python_api.html#xgboost.XGBRFClassifier)
+`lightgbm` | [LGBMRegressor](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html#lightgbm.LGBMRegressor) | [LGBMClassifier](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMClassifier.html#lightgbm.LGBMClassifier)
 
 ## Scikit Ensembles
 Algorithm | Regression | Classification

pgml-extension/examples/binary_classification.sql

@@ -48,8 +48,11 @@ SELECT malignant, pgml.predict(
 FROM pgml.breast_cancer 
 LIMIT 10;
 
+--
 -- After a project has been trained, ommited parameters will be reused from previous training runs
 -- In these examples we'll reuse the training data snapshots from the initial call.
+--
+
 -- linear models
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'ridge');
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'stochastic_gradient_descent');
@@ -60,20 +63,24 @@ SELECT * FROM pgml.train('Breast Cancer', algorithm => 'passive_aggressive');
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'svm');
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'nu_svm');
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'linear_svm');
+
 -- ensembles
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'ada_boost');
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'bagging');
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'extra_trees', hyperparams => '{"n_estimators": 10}');
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'gradient_boosting_trees', hyperparams => '{"n_estimators": 10}');
--- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
--- SELECT * FROM pgml.train('Breast Cancer', algorithim => 'hist_gradient_boosting', hyperparams => '{"max_iter": 2}');
 SELECT * FROM pgml.train('Breast Cancer', algorithm => 'random_forest', hyperparams => '{"n_estimators": 10}');
+
 -- other
 -- Gaussian Process is too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml.train('Breast Cancer', algorithm => 'gaussian_process', hyperparams => '{"max_iter_predict": 100, "warm_start": true}');
--- XGBoost
-SELECT * FROM pgml.train('Breast Cancer', algorithm => 'xgboost');
-SELECT * FROM pgml.train('Breast Cancer', algorithm => 'xgboost_random_forest');
+
+-- Gradient Boosting
+SELECT * FROM pgml.train('Breast Cancer', algorithm => 'xgboost', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train('Breast Cancer', algorithm => 'xgboost_random_forest', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train('Breast Cancer', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 1}');
+-- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml.train('Breast Cancer', algorithim => 'hist_gradient_boosting', hyperparams => '{"max_iter": 2}');
 
 
 -- check out all that hard work

pgml-extension/examples/image_classification.sql

@@ -25,30 +25,39 @@ SELECT target, pgml.predict('Handwritten Digit Image Classifier', image) AS pred
 FROM pgml.digits 
 LIMIT 10;
 
+--
 -- After a project has been trained, ommited parameters will be reused from previous training runs
 -- In these examples we'll reuse the training data snapshots from the initial call.
+--
+
 -- linear models
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'ridge');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'stochastic_gradient_descent');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'perceptron');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'passive_aggressive');
+
 -- support vector machines
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'svm');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'nu_svm');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'linear_svm');
+
 -- ensembles
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'ada_boost');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'bagging');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'extra_trees', hyperparams => '{"n_estimators": 10}');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'gradient_boosting_trees', hyperparams => '{"n_estimators": 10}');
--- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
--- SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'hist_gradient_boosting', hyperparams => '{"max_iter": 2}');
 SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'random_forest', hyperparams => '{"n_estimators": 10}');
+
 -- other
 -- Gaussian Process is too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'gaussian_process', hyperparams => '{"max_iter_predict": 100, "warm_start": true}');
-SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'xgboost');
-SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'xgboost_random_forest');
+
+-- gradient boosting
+SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'xgboost', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'xgboost_random_forest', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 1}');
+-- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml.train('Handwritten Digit Image Classifier', algorithm => 'hist_gradient_boosting', hyperparams => '{"max_iter": 2}');
 
 
 -- check out all that hard work

pgml-extension/examples/joint_regression.sql

@@ -18,7 +18,7 @@ SELECT weight, waste, pulse, pgml.predict_joint('Exercise vs Physiology', ARRAY[
 FROM pgml.linnerud 
 LIMIT 10;
 
--- -- linear models
+-- linear models
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'ridge');
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'lasso');
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'elastic_net');
@@ -34,25 +34,32 @@ SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'theil_sen
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'huber');
 -- Quantile Regression too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'quantile');
---- support vector machines
+
+-- support vector machines
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'svm', hyperparams => '{"max_iter": 100}');
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'nu_svm', hyperparams => '{"max_iter": 10}');
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'linear_svm', hyperparams => '{"max_iter": 100}');
--- -- ensembles
+
+-- ensembles
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'ada_boost', hyperparams => '{"n_estimators": 5}');
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'bagging', hyperparams => '{"n_estimators": 5}');
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'extra_trees', hyperparams => '{"n_estimators": 5}');
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'gradient_boosting_trees', hyperparams => '{"n_estimators": 5}');
--- -- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
--- SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'hist_gradient_boosting', hyperparams => '{"max_iter": 10}');
 SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'random_forest', hyperparams => '{"n_estimators": 5}');
+
 -- other
---SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'kernel_ridge');
-SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'xgboost');
-SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'xgboost_random_forest');
+-- Kernel Ridge is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'kernel_ridge');
 -- Gaussian Process is too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'gaussian_process');
 
+-- gradient boosting
+SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'xgboost', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'xgboost_random_forest', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 1}');
+-- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml.train_joint('Exercise vs Physiology', algorithm => 'hist_gradient_boosting', hyperparams => '{"max_iter": 10}');
+
 -- check out all that hard work
 SELECT trained_models.* FROM pgml.trained_models 
 JOIN pgml.models on models.id = trained_models.id

pgml-extension/examples/multi_classification.sql

@@ -18,31 +18,39 @@ SELECT target, pgml.predict('Iris Classifier', ARRAY[sepal_length, sepal_width,
 FROM iris_view 
 LIMIT 10;
 
+--
 -- After a project has been trained, ommited parameters will be reused from previous training runs
 -- In these examples we'll reuse the training data snapshots from the initial call.
+--
+
 -- linear models
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'ridge');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'stochastic_gradient_descent');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'perceptron');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'passive_aggressive');
+
 -- support vector machines
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'svm');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'nu_svm');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'linear_svm');
+
 -- ensembles
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'ada_boost');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'bagging');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'extra_trees', hyperparams => '{"n_estimators": 10}');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'gradient_boosting_trees', hyperparams => '{"n_estimators": 10}');
--- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
--- SELECT * FROM pgml.train('Iris Classifier', algorithim => 'hist_gradient_boosting', hyperparams => '{"max_iter": 2}');
 SELECT * FROM pgml.train('Iris Classifier', algorithm => 'random_forest', hyperparams => '{"n_estimators": 10}');
+
 -- other
 -- Gaussian Process is too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml.train('Iris Classifier', algorithm => 'gaussian_process', hyperparams => '{"max_iter_predict": 100, "warm_start": true}');
--- XGBoost
-SELECT * FROM pgml.train('Iris Classifier', algorithm => 'xgboost');
-SELECT * FROM pgml.train('Iris Classifier', algorithm => 'xgboost_random_forest');
+
+-- gradient boosting
+SELECT * FROM pgml.train('Iris Classifier', algorithm => 'xgboost', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train('Iris Classifier', algorithm => 'xgboost_random_forest', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train('Iris Classifier', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 1}');
+-- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml.train('Iris Classifier', algorithim => 'hist_gradient_boosting', hyperparams => '{"max_iter": 2}');
 
 
 -- check out all that hard work

pgml-extension/examples/regression.sql

@@ -37,6 +37,11 @@ CROSS JOIN LATERAL (
 ) models
 LIMIT 10;
 
+--
+-- After a project has been trained, ommited parameters will be reused from previous training runs
+-- In these examples we'll reuse the training data snapshots from the initial call.
+--
+
 -- linear models
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'ridge');
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'lasso');
@@ -53,25 +58,33 @@ SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'theil_sen', hyper
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'huber');
 -- Quantile Regression too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'quantile');
---- support vector machines
+
+-- support vector machines
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'svm', hyperparams => '{"max_iter": 100}');
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'nu_svm', hyperparams => '{"max_iter": 10}');
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'linear_svm', hyperparams => '{"max_iter": 100}');
+
 -- ensembles
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'ada_boost', hyperparams => '{"n_estimators": 5}');
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'bagging', hyperparams => '{"n_estimators": 5}');
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'extra_trees', hyperparams => '{"n_estimators": 5}');
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'gradient_boosting_trees', hyperparams => '{"n_estimators": 5}');
--- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
--- SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'hist_gradient_boosting', hyperparams => '{"max_iter": 10}');
 SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'random_forest', hyperparams => '{"n_estimators": 5}');
+
 -- other
---SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'kernel_ridge');
-SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'xgboost');
-SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'xgboost_random_forest');
+-- Kernel Ridge is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'kernel_ridge');
 -- Gaussian Process is too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'gaussian_process');
 
+-- gradient boosting
+SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'xgboost', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'xgboost_random_forest', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 1}');
+-- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml.train('Diabetes Progression', algorithm => 'hist_gradient_boosting', hyperparams => '{"max_iter": 10}');
+
+
 -- check out all that hard work
 SELECT trained_models.* FROM pgml.trained_models 
 JOIN pgml.models on models.id = trained_models.id

pgml-extension/pgml_extension/model.py

@@ -10,6 +10,7 @@
 import numpy
 import xgboost as xgb
 import diptest
+import lightgbm
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import (
     mean_squared_error,
@@ -445,6 +446,8 @@ def algorithm_from_name_and_objective(cls, name: str, objective: str):
             "xgboost_classification": xgb.XGBClassifier,
             "xgboost_random_forest_regression": xgb.XGBRFRegressor,
             "xgboost_random_forest_classification": xgb.XGBRFClassifier,
+            "lightgbm_regression": lightgbm.LGBMRegressor,
+            "lightgbm_classification": lightgbm.LGBMClassifier,
         }[name + "_" + objective]
 
     @classmethod
@@ -659,6 +662,7 @@ def algorithm(self):
                         "linear_svm",
                         "ada_boost",
                         "gradient_boosting_trees",
+                        "lightgbm",
                     ]:
                         self._algorithm = sklearn.multioutput.MultiOutputRegressor(self._algorithm)
 

pgml-extension/setup.py

@@ -84,9 +84,10 @@ def install_sql(filename, database_url):
         'install': InstallCommand,
     },
     install_requires=[
+        "diptest",
         "sklearn",
         "xgboost",
-        "diptest",
+        "lightgbm",
     ],
     extras_require={"dev": "pytest"},
     packages=setuptools.find_packages(exclude=("tests",)),