Tests and checks (#341)

Author: levkk

pgml-extension/pgml_rust/sql/schema.sql

@@ -1,3 +1,8 @@
+---
+--- Validate we have the necessary Python dependencies.
+---
+SELECT pgml_rust.validate_python_dependencies();
+
 --- 
 --- Track of updates to data
 ---

pgml-extension/pgml_rust/src/engines/sklearn.rs

@@ -19,6 +19,23 @@ use crate::orm::task::Task;
 
 use pgx::*;
 
+#[pg_extern]
+pub fn validate_python_dependencies() {
+    Python::with_gil(|py| {
+        for module in ["xgboost", "lightgbm", "numpy", "sklearn"] {
+            match py.import(module) {
+                Ok(_) => (),
+                Err(_) => {
+                    panic!(
+                        "The {} package is missing. Install it with `sudo pip3 install {}`",
+                        module, module
+                    );
+                }
+            }
+        }
+    });
+}
+
 #[pg_extern]
 pub fn sklearn_version() -> String {
     let mut version = String::new();
@@ -64,6 +81,7 @@ fn sklearn_algorithm_name(task: Task, algorithm: Algorithm) -> &'static str {
             Algorithm::least_angle => "least_angle_regression",
             Algorithm::lasso_least_angle => "lasso_least_angle_regression",
             Algorithm::linear_svm => "linear_svm_regression",
+            Algorithm::lightgbm => "lightgbm_regression",
             _ => panic!("{:?} does not support regression", algorithm),
         },
 
@@ -85,6 +103,7 @@ fn sklearn_algorithm_name(task: Task, algorithm: Algorithm) -> &'static str {
             Algorithm::gradient_boosting_trees => "gradient_boosting_trees_classification",
             Algorithm::hist_gradient_boosting => "hist_gradient_boosting_classification",
             Algorithm::linear_svm => "linear_svm_classification",
+            Algorithm::lightgbm => "lightgbm_classification",
             _ => panic!("{:?} does not support classification", algorithm),
         },
     }

pgml-extension/pgml_rust/src/engines/wrappers.py

@@ -11,6 +11,7 @@
 import sklearn.gaussian_process
 import sklearn.model_selection
 import xgboost as xgb
+import lightgbm
 import numpy as np
 import pickle
 import json
@@ -61,6 +62,8 @@
     "xgboost_classification": xgb.XGBClassifier,
     "xgboost_random_forest_regression": xgb.XGBRFRegressor,
     "xgboost_random_forest_classification": xgb.XGBRFClassifier,
+    "lightgbm_regression": lightgbm.LGBMRegressor,
+    "lightgbm_classification": lightgbm.LGBMClassifier,
 }
 
 

pgml-extension/pgml_rust/tests/binary_classification.sql

@@ -78,7 +78,7 @@ SELECT * FROM pgml_rust.train('Breast Cancer Detection', algorithm => 'random_fo
 -- Gradient Boosting
 SELECT * FROM pgml_rust.train('Breast Cancer Detection', algorithm => 'xgboost', hyperparams => '{"n_estimators": 10}');
 SELECT * FROM pgml_rust.train('Breast Cancer Detection', algorithm => 'xgboost_random_forest', hyperparams => '{"n_estimators": 10}');
--- SELECT * FROM pgml_rust.train('Breast Cancer Detection', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 1}');
+SELECT * FROM pgml_rust.train('Breast Cancer Detection', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 100}');
 -- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml_rust.train('Breast Cancer Detection', algorithim => 'hist_gradient_boosting', hyperparams => '{"max_iter": 2}');
 

pgml-extension/pgml_rust/tests/image_classification.sql

@@ -0,0 +1,96 @@
+-- This example trains models on the sklean digits dataset
+-- which is a copy of the test set of the UCI ML hand-written digits datasets
+-- https://archive.ics.uci.edu/ml/datasets/Optical+Recognition+of+Handwritten+Digits
+--
+-- This demonstrates using a table with a single array feature column
+-- for classification.
+--
+-- The final result after a few seconds of training is not terrible. Maybe not perfect
+-- enough for mission critical applications, but it's telling how quickly "off the shelf" 
+-- solutions can solve problems these days.
+
+-- Exit on error (psql)
+\set ON_ERROR_STOP true
+
+SELECT pgml_rust.load_dataset('digits');
+
+-- view the dataset
+SELECT left(image::text, 40) || ',...}', target FROM pgml_rust.digits LIMIT 10;
+
+-- train a simple model to classify the data
+SELECT * FROM pgml_rust.train('Handwritten Digits', 'classification', 'pgml_rust.digits', 'target');
+
+-- check out the predictions
+SELECT target, pgml_rust.predict('Handwritten Digits', image) AS prediction
+FROM pgml_rust.digits 
+LIMIT 10;
+
+--
+-- After a project has been trained, ommited parameters will be reused from previous training runs
+-- In these examples we'll reuse the training data snapshots from the initial call.
+--
+
+-- linear models
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'ridge');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'stochastic_gradient_descent');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'perceptron');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'passive_aggressive');
+
+-- support vector machines
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'svm');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'nu_svm');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'linear_svm');
+
+-- ensembles
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'ada_boost');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'bagging');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'extra_trees', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'gradient_boosting_trees', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'random_forest', hyperparams => '{"n_estimators": 10}');
+
+-- other
+-- Gaussian Process is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'gaussian_process', hyperparams => '{"max_iter_predict": 100, "warm_start": true}');
+
+-- gradient boosting
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'xgboost', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'xgboost_random_forest', hyperparams => '{"n_estimators": 10}');
+SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 100}');
+-- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
+-- SELECT * FROM pgml_rust.train('Handwritten Digits', algorithm => 'hist_gradient_boosting', hyperparams => '{"max_iter": 2}');
+
+
+-- check out all that hard work
+SELECT trained_models.* FROM pgml_rust.trained_models 
+JOIN pgml_rust.models on models.id = trained_models.id
+ORDER BY models.metrics->>'f1' DESC LIMIT 5;
+
+-- deploy the random_forest model for prediction use
+SELECT * FROM pgml_rust.deploy('Handwritten Digits', 'most_recent', 'random_forest');
+-- check out that throughput
+SELECT * FROM pgml_rust.deployed_models ORDER BY deployed_at DESC LIMIT 5;
+
+-- do a hyperparam search on your favorite algorithm
+SELECT pgml_rust.train(
+    'Handwritten Digits', 
+    algorithm => 'svm', 
+    hyperparams => '{"random_state": 0}',
+    search => 'grid', 
+    search_params => '{
+        "kernel": ["linear", "poly", "sigmoid"], 
+        "shrinking": [true, false]
+    }'
+);
+
+-- TODO SELECT pgml_rust.hypertune(100, 'Handwritten Digits', 'classification', 'pgml_rust.digits', 'target', 'gradient_boosted_trees');
+
+-- deploy the "best" model for prediction use
+SELECT * FROM pgml_rust.deploy('Handwritten Digits', 'best_score');
+SELECT * FROM pgml_rust.deploy('Handwritten Digits', 'most_recent');
+SELECT * FROM pgml_rust.deploy('Handwritten Digits', 'rollback');
+SELECT * FROM pgml_rust.deploy('Handwritten Digits', 'best_score', 'svm');
+
+-- check out the improved predictions
+SELECT target, pgml_rust.predict('Handwritten Digits', image) AS prediction
+FROM pgml_rust.digits 
+LIMIT 10;

pgml-extension/pgml_rust/tests/regression.sql

@@ -81,7 +81,7 @@ SELECT * FROM pgml_rust.train('Diabetes Progression', algorithm => 'random_fores
 -- gradient boosting
 SELECT * FROM pgml_rust.train('Diabetes Progression', algorithm => 'xgboost', hyperparams => '{"n_estimators": 10}');
 SELECT * FROM pgml_rust.train('Diabetes Progression', algorithm => 'xgboost_random_forest', hyperparams => '{"n_estimators": 10}');
--- SELECT * FROM pgml_rust.train('Diabetes Progression', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 1}');
+SELECT * FROM pgml_rust.train('Diabetes Progression', algorithm => 'lightgbm', hyperparams => '{"n_estimators": 100}');
 -- Histogram Gradient Boosting is too expensive for normal tests on even a toy dataset
 -- SELECT * FROM pgml_rust.train('Diabetes Progression', algorithm => 'hist_gradient_boosting', hyperparams => '{"max_iter": 10}');