From bb016b27fbeec0fa1fe91aaadb15e8068f94e479 Mon Sep 17 00:00:00 2001 From: Sanuj Date: Mon, 30 May 2016 11:26:32 +0530 Subject: [PATCH] add cookbook page for svr --- .../regression/support_vector_regression.rst | 48 +++++++++++++++++ doc/cookbook/source/references.bib | 26 ++++++++++ .../regression/support_vector_regression.sg | 39 ++++++++++++++ .../regression_libsvr_modular.cs | 35 ------------- .../regression_libsvr_modular.java | 38 -------------- .../regression_libsvr_modular.py | 51 ------------------- .../ruby_modular/regression_libsvr_modular.rb | 39 -------------- 7 files changed, 113 insertions(+), 163 deletions(-) create mode 100644 doc/cookbook/source/examples/regression/support_vector_regression.rst create mode 100644 examples/meta/src/regression/support_vector_regression.sg delete mode 100644 examples/undocumented/csharp_modular/regression_libsvr_modular.cs delete mode 100644 examples/undocumented/java_modular/regression_libsvr_modular.java delete mode 100644 examples/undocumented/python_modular/regression_libsvr_modular.py delete mode 100644 examples/undocumented/ruby_modular/regression_libsvr_modular.rb diff --git a/doc/cookbook/source/examples/regression/support_vector_regression.rst b/doc/cookbook/source/examples/regression/support_vector_regression.rst new file mode 100644 index 00000000000..aa01c98c9ad --- /dev/null +++ b/doc/cookbook/source/examples/regression/support_vector_regression.rst @@ -0,0 +1,48 @@ +========================= +Support Vector Regression +========================= + +Support vector regression is a regression model inspired from support vector machines. The solution can be written as: + +.. math:: + f({\bf x})=\sum_{i=1}^{N} \alpha_i k({\bf x}, {\bf x}_i)+b + +where :math:`{\bf x}` is the new data point, :math:`{\bf x}_i` is a training sample, :math:`N` denotes number of training samples, :math:`k` is a kernel function, :math:`\alpha` and :math:`b` are determined in training. + +See :cite:`scholkopf2002learning` for a more detailed introduction. :sgclass:`LibSVR` performs support vector regression using LibSVM :cite:`chang2011libsvm`. + +------- +Example +------- + +Imagine we have files with training and test data. We create `CDenseFeatures` (here 64 bit floats aka RealFeatures) and :sgclass:`CRegressionLabels` as + +.. sgexample:: support_vector_regression.sg:create_features + +Choose an appropriate :sgclass:`CKernel` and instantiate it. Here we use a :sgclass:`CGaussianKernel`. + +.. sgexample:: support_vector_regression.sg:create_appropriate_kernel + +We create an instance of :sgclass:`CLibSVR` classifier by passing it the kernel, labels, solver type and some more parameters. More solver types are available in :sgclass:`CLibSVR`. See :cite:`chang2002training` for more details. + +.. sgexample:: support_vector_regression.sg:create_instance + +Then we train the regression model and apply it to test data to get the predicted :sgclass:`CRegressionLabels`. + +.. sgexample:: support_vector_regression.sg:train_and_apply + +After training, we can extract :math:`\alpha`. + +.. sgexample:: support_vector_regression.sg:extract_alpha + +Finally, we can evaluate the :sgclass:`CMeanSquaredError`. + +.. sgexample:: support_vector_regression.sg:evaluate_error + +---------- +References +---------- +:wiki:`Support_vector_machine` + +.. bibliography:: ../../references.bib + :filter: docname in docnames \ No newline at end of file diff --git a/doc/cookbook/source/references.bib b/doc/cookbook/source/references.bib index 4316c5cf309..8a071b909b4 100644 --- a/doc/cookbook/source/references.bib +++ b/doc/cookbook/source/references.bib @@ -33,4 +33,30 @@ @article{ueda2000smem pages={2109--2128}, year={2000}, publisher={MIT Press} +} +@article{chang2011libsvm, + title={LIBSVM: a library for support vector machines}, + author={C.C. Chang and C.J. Lin}, + journal={ACM Transactions on Intelligent Systems and Technology (TIST)}, + volume={2}, + number={3}, + pages={27}, + year={2011}, + publisher={ACM} +} +@article{chang2002training, + title={Training v-support vector regression: theory and algorithms}, + author={C.C. Chang and C.B. Lin}, + journal={Neural Computation}, + volume={14}, + number={8}, + pages={1959--1977}, + year={2002}, + publisher={MIT Press} +} +@book{scholkopf2002learning, + title={Learning with kernels: support vector machines, regularization, optimization, and beyond}, + author={B. Sch{\"o}lkopf and A.J. Smola}, + year={2002}, + publisher={MIT press} } \ No newline at end of file diff --git a/examples/meta/src/regression/support_vector_regression.sg b/examples/meta/src/regression/support_vector_regression.sg new file mode 100644 index 00000000000..beb0a601312 --- /dev/null +++ b/examples/meta/src/regression/support_vector_regression.sg @@ -0,0 +1,39 @@ +CSVFile f_feats_train("../../data/regression_1d_sinc_features_train.dat") +CSVFile f_feats_test("../../data/regression_1d_sinc_features_test.dat") +CSVFile f_labels_train("../../data/regression_1d_sinc_labels_train.dat") +CSVFile f_labels_test("../../data/regression_1d_sinc_labels_test.dat") + +#![create_features] +RealFeatures features_train(f_feats_train) +RealFeatures features_test(f_feats_test) +RegressionLabels labels_train(f_labels_train) +RegressionLabels labels_test(f_labels_test) +#![create_features] + +#![create_appropriate_kernel] +real width = 1 +GaussianKernel kernel(width) +#![create_appropriate_kernel] + +#![create_instance] +real svm_c = 1 +real svr_param = 0.1 +LibSVR svr(svm_c, svr_param, kernel, labels_train, enum LIBSVR_SOLVER_TYPE.LIBSVR_EPSILON_SVR) +#![create_instance] + +#![train_and_apply] +svr.train(features_train) +RegressionLabels labels_predict = svr.apply_regression(features_test) +#![train_and_apply] + +#![extract_alpha] +RealVector alpha = svr.get_alphas() +#![extract_alpha] + +#![evaluate_error] +MeanSquaredError eval() +real mse = eval.evaluate(labels_predict, labels_test) +#![evaluate_error] + +# integration testing variables +RealVector output = labels_test.get_labels() diff --git a/examples/undocumented/csharp_modular/regression_libsvr_modular.cs b/examples/undocumented/csharp_modular/regression_libsvr_modular.cs deleted file mode 100644 index 6ad03d9aaa5..00000000000 --- a/examples/undocumented/csharp_modular/regression_libsvr_modular.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System; - -public class regression_libsvr_modular { - public static void Main() { - - modshogun.init_shogun_with_defaults(); - double width = 0.8; - int C = 1; - double epsilon = 1e-5; - double tube_epsilon = 1e-2; - - double[,] traindata_real = Load.load_numbers("../data/fm_train_real.dat"); - double[,] testdata_real = Load.load_numbers("../data/fm_test_real.dat"); - - double[] trainlab = Load.load_labels("../data/label_train_twoclass.dat"); - - RealFeatures feats_train = new RealFeatures(traindata_real); - RealFeatures feats_test = new RealFeatures(testdata_real); - GaussianKernel kernel= new GaussianKernel(feats_train, feats_train, width); - - RegressionLabels labels = new RegressionLabels(trainlab); - - LibSVR svr = new LibSVR(C, epsilon, kernel, labels); - svr.set_tube_epsilon(tube_epsilon); - svr.train(); - - kernel.init(feats_train, feats_test); - double[] out_labels = LabelsFactory.to_regression(svr.apply()).get_labels(); - - foreach (double item in out_labels) - Console.Write(out_labels); - - - } -} diff --git a/examples/undocumented/java_modular/regression_libsvr_modular.java b/examples/undocumented/java_modular/regression_libsvr_modular.java deleted file mode 100644 index ac3c29aa5ac..00000000000 --- a/examples/undocumented/java_modular/regression_libsvr_modular.java +++ /dev/null @@ -1,38 +0,0 @@ -import org.shogun.*; -import org.jblas.*; - -import static org.shogun.LabelsFactory.to_regression; - -public class regression_libsvr_modular { - static { - System.loadLibrary("modshogun"); - } - - public static void main(String argv[]) { - modshogun.init_shogun_with_defaults(); - double width = 0.8; - int C = 1; - double epsilon = 1e-5; - double tube_epsilon = 1e-2; - - DoubleMatrix traindata_real = Load.load_numbers("../data/fm_train_real.dat"); - DoubleMatrix testdata_real = Load.load_numbers("../data/fm_test_real.dat"); - - DoubleMatrix trainlab = Load.load_labels("../data/label_train_twoclass.dat"); - - RealFeatures feats_train = new RealFeatures(traindata_real); - RealFeatures feats_test = new RealFeatures(testdata_real); - GaussianKernel kernel= new GaussianKernel(feats_train, feats_train, width); - - RegressionLabels labels = new RegressionLabels(trainlab); - - LibSVR svr = new LibSVR(C, tube_epsilon, kernel, labels); - svr.set_epsilon(epsilon); - svr.train(); - - kernel.init(feats_train, feats_test); - DoubleMatrix out_labels = to_regression(svr.apply()).get_labels(); - System.out.println(out_labels.toString()); - - } -} diff --git a/examples/undocumented/python_modular/regression_libsvr_modular.py b/examples/undocumented/python_modular/regression_libsvr_modular.py deleted file mode 100644 index 4687923c1de..00000000000 --- a/examples/undocumented/python_modular/regression_libsvr_modular.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -from numpy import * - -parameter_list=[[1, 0.1,20,100,6,10,0.5,1, 1], [2,0.2,20,100,6,10,0.5,1, 2]] - -def regression_libsvr_modular (svm_c=1, svr_param=0.1, n=100,n_test=100, \ - x_range=6,x_range_test=10,noise_var=0.5,width=1, seed=1): - - from modshogun import RegressionLabels, RealFeatures - from modshogun import GaussianKernel - from modshogun import LibSVR, LIBSVR_NU_SVR, LIBSVR_EPSILON_SVR - - # reproducable results - random.seed(seed) - - # easy regression data: one dimensional noisy sine wave - n=15 - n_test=100 - x_range_test=10 - noise_var=0.5; - X=random.rand(1,n)*x_range - - X_test=array([[float(i)/n_test*x_range_test for i in range(n_test)]]) - Y_test=sin(X_test) - Y=sin(X)+random.randn(n)*noise_var - - # shogun representation - labels=RegressionLabels(Y[0]) - feats_train=RealFeatures(X) - feats_test=RealFeatures(X_test) - - kernel=GaussianKernel(feats_train, feats_train, width) - - # two svr models: epsilon and nu - svr_epsilon=LibSVR(svm_c, svr_param, kernel, labels, LIBSVR_EPSILON_SVR) - svr_epsilon.train() - svr_nu=LibSVR(svm_c, svr_param, kernel, labels, LIBSVR_NU_SVR) - svr_nu.train() - - # predictions - kernel.init(feats_train, feats_test) - out1_epsilon=svr_epsilon.apply().get_labels() - out2_epsilon=svr_epsilon.apply(feats_test).get_labels() - out1_nu=svr_epsilon.apply().get_labels() - out2_nu=svr_epsilon.apply(feats_test).get_labels() - - return out1_epsilon,out2_epsilon,out1_nu,out2_nu ,kernel - -if __name__=='__main__': - print('LibSVR') - regression_libsvr_modular(*parameter_list[0]) diff --git a/examples/undocumented/ruby_modular/regression_libsvr_modular.rb b/examples/undocumented/ruby_modular/regression_libsvr_modular.rb deleted file mode 100644 index 0f47a1dbf74..00000000000 --- a/examples/undocumented/ruby_modular/regression_libsvr_modular.rb +++ /dev/null @@ -1,39 +0,0 @@ -require 'modshogun' -require 'pp' -require 'load' - -traindat = LoadMatrix.load_numbers('../data/fm_train_real.dat') -testdat = LoadMatrix.load_numbers('../data/fm_test_real.dat') -label_traindat = LoadMatrix.load_labels('../data/label_train_twoclass.dat') - - -parameter_list = [[traindat,testdat,label_traindat,2.1,1,1e-5,1e-2], [traindat,testdat,label_traindat,2.1,1,1e-5,1e-2]] - - -def regression_libsvr_modular(fm_train=traindat,fm_test=testdat,label_train=label_traindat,width=2.1,c=1,epsilon=1e-5,tube_epsilon=1e-2) - - - feats_train=Modshogun::RealFeatures.new - feats_train.set_feature_matrix(fm_train) - feats_test=Modshogun::RealFeatures.new - feats_test.set_feature_matrix(fm_test) - - kernel=Modshogun::GaussianKernel.new(feats_train, feats_train, width) - labels=Modshogun::RegressionLabels.new(label_train) - - svr=Modshogun::LibSVR.new(c, tube_epsilon, kernel, labels) - svr.set_epsilon(epsilon) - svr.train() - - kernel.init(feats_train, feats_test) - out1=svr.apply().get_labels() - out2=svr.apply(feats_test).get_labels() - - return out1,out2,kernel - -end - -if __FILE__ == $0 - puts 'LibSVR' - pp regression_libsvr_modular(*parameter_list[0]) -end