diff --git a/Cargo.toml b/Cargo.toml index 64ee502..cd4d8b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,8 @@ maintenance = { status = "experimental" } [dependencies] -simd_aligned = "0.1.1" +# simd_aligned = "0.1.1" +simd_aligned = { path = "../simd_aligned" } packed_simd = "0.1" rand = "0.5" pest = "1.0" diff --git a/README.md b/README.md index 757071f..4a479f0 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ You trained a SVM using [libSVM](https://github.com/cjlin1/libsvm), now you want * free of `unsafe` code ;) -# Principal Usage +# Usage Train with [libSVM](https://github.com/cjlin1/libsvm) (e.g., using the tool `svm-train`), then classify with `ffsvm-rust`. @@ -42,7 +42,7 @@ features[3] = -0.221184; svm.predict_value(&mut problem)?; -assert_eq!(problem.result(), Outcome::Label(42)); +assert_eq!(problem.solution(), Solution::Label(42)); ``` From C / FFI: diff --git a/benches/svm_dense.rs b/benches/svm_dense.rs index f70f509..583e922 100644 --- a/benches/svm_dense.rs +++ b/benches/svm_dense.rs @@ -4,7 +4,7 @@ extern crate ffsvm; extern crate test; mod svm_dense { - use ffsvm::{DenseSVM, Linear, ModelFile, Poly, Predict, Problem, Rbf, SVMType, Sigmoid}; + use ffsvm::{DenseSVM, ModelFile, Predict, Problem}; use std::convert::TryFrom; use test::Bencher; @@ -12,7 +12,7 @@ mod svm_dense { #[allow(dead_code)] fn produce_testcase(svm_type: &str, kernel_type: &str, total_sv: u32, num_attributes: u32) -> impl FnMut() { let raw_model = ModelFile::random_dense(svm_type, kernel_type, total_sv, num_attributes); - let mut svm = DenseSVM::try_from(&raw_model).unwrap(); + let svm = DenseSVM::try_from(&raw_model).unwrap(); let mut problem = Problem::from(&svm); let problem_mut = problem.features().as_slice_mut(); @@ -20,7 +20,7 @@ mod svm_dense { problem_mut[i as usize] = i as f32; } - move || (&mut svm).predict_value(&mut problem).expect("This should work") + move || (&svm).predict_value(&mut problem).expect("This should work") } // RBF diff --git a/benches/svm_sparse.rs b/benches/svm_sparse.rs index 11abc6b..452eb3a 100644 --- a/benches/svm_sparse.rs +++ b/benches/svm_sparse.rs @@ -4,7 +4,7 @@ extern crate ffsvm; extern crate test; mod svm_sparse { - use ffsvm::{Linear, ModelFile, Poly, Predict, Problem, Rbf, SVMType, Sigmoid, SparseSVM}; + use ffsvm::{ModelFile, Predict, Problem, SparseSVM}; use std::convert::TryFrom; use test::Bencher; @@ -20,7 +20,7 @@ mod svm_sparse { problem_mut[i as usize] = i as f32; } - move || (&mut svm).predict_value(&mut problem).expect("This should work") + move || (&svm).predict_value(&mut problem).expect("This should work") } // RBF diff --git a/docs/performance.md b/docs/performance.md index 06b2eb7..51a5790 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -2,7 +2,7 @@ # Performance vs. LibSVM -Benchmarks are a tricky thing, but for classifying dense RBF-C-SVMs `ffsvm` should be between 2.5x and 14x faster than `libSVM` on reasonably modern x86 CPUs (supporting AVX2). +Benchmarks are a tricky thing, but for classifying dense SVMs `ffsvm` should be between 2.5x and 14x faster than `libSVM` on reasonably modern x86 CPUs (supporting AVX2). ![performance](performance_absolute.v3.png) @@ -14,7 +14,7 @@ There are 3 major factors contributing to this: * no allocation during classification * cache-friendly memory layout -* usage of SIMD / AVX +* usage of SIMD In addition, ffsvm mixes seamlessly with Rayon for _batch classification_, providing even higher speed ups if you classify more than one problem at a time. diff --git a/examples/basic.rs b/examples/basic.rs index ce8664e..e647f71 100644 --- a/examples/basic.rs +++ b/examples/basic.rs @@ -7,7 +7,7 @@ fn main() -> Result<(), Error> { let svm = DenseSVM::try_from(SAMPLE_MODEL)?; let mut problem = Problem::from(&svm); - let mut features = problem.features(); + let features = problem.features(); features[0] = 0.55838; features[1] = -0.157895; @@ -16,7 +16,7 @@ fn main() -> Result<(), Error> { svm.predict_value(&mut problem)?; - assert_eq!(problem.result(), Outcome::Label(12)); + assert_eq!(problem.solution(), Solution::Label(12)); Ok(()) } diff --git a/src/errors.rs b/src/errors.rs index 6d31c86..13a44c5 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -40,7 +40,7 @@ pub enum Error { /// If the model does not have a `degree` set this error may be raised. NoDegree, - /// Wrapper for [ModelError] when unifiying error handling. + /// Wrapper for internal parsing error when unifiying error handling. ParsingError(String), } diff --git a/src/lib.rs b/src/lib.rs index a1e6d65..e3ee9b6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,7 +60,7 @@ //! //! svm.predict_value(&mut problem)?; //! -//! assert_eq!(problem.result(), Outcome::Label(42)); +//! assert_eq!(problem.solution(), Solution::Label(42)); //! //! Ok(()) //! } @@ -86,9 +86,9 @@ pub use crate::{ parser::ModelFile, svm::{ core::SVMCore, - kernel::{KernelDense, Linear, Poly, Rbf, Sigmoid}, + kernel::{KernelDense, KernelSparse, Linear, Poly, Rbf, Sigmoid}, predict::Predict, - problem::{Outcome, Problem}, + problem::{DenseProblem, Problem, Solution, SparseProblem}, DenseSVM, SVMType, SparseSVM, }, }; diff --git a/src/sparse.rs b/src/sparse.rs index 7913979..c2f872a 100644 --- a/src/sparse.rs +++ b/src/sparse.rs @@ -1,28 +1,25 @@ -use std::{ - collections::{btree_map::Iter, BTreeMap}, - ops::{Index, IndexMut}, -}; +use std::ops::{Index, IndexMut}; #[derive(Clone, Debug)] struct Entry where - T: Copy, + T: Copy + Clone + Default, { index: u32, value: T, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct SparseVector where - T: Clone + Copy, + T: Clone + Copy + Default, { entries: Vec>, } impl SparseVector where - T: Clone + Copy, + T: Clone + Copy + Default, { pub fn new() -> Self { SparseVector { entries: Vec::new() } } @@ -35,7 +32,7 @@ where #[derive(Clone, Debug)] pub struct SparseVectorIter<'a, T: 'a> where - T: Clone + Copy, + T: Clone + Copy + Default, { /// Reference to the matrix we iterate over. crate vector: &'a SparseVector, @@ -46,7 +43,7 @@ where impl<'a, T> Iterator for SparseVectorIter<'a, T> where - T: Clone + Copy, + T: Clone + Copy + Default, { type Item = (u32, T); @@ -64,14 +61,14 @@ where impl Index for SparseVector where - T: Copy + Sized, + T: Copy + Sized + Default, { type Output = T; fn index(&self, index: usize) -> &T { // TODO: Beautify me - for (i, e) in self.entries.iter().enumerate() { + for e in self.entries.iter() { if e.index == index as u32 { return &e.value; } @@ -111,14 +108,14 @@ where #[derive(Clone, Debug)] pub struct SparseMatrix where - T: Clone + Copy, + T: Clone + Copy + Default, { vectors: Vec>, } impl SparseMatrix where - T: Clone + Copy, + T: Clone + Copy + Default, { pub fn with(rows: usize) -> Self { SparseMatrix { @@ -134,7 +131,7 @@ where impl Index<(usize, usize)> for SparseMatrix where - T: Copy + Sized, + T: Copy + Sized + Default, { type Output = T; @@ -152,7 +149,7 @@ where #[derive(Clone, Debug)] pub struct SparseMatrixIter<'a, T: 'a> where - T: Clone + Copy, + T: Clone + Copy + Default, { /// Reference to the matrix we iterate over. crate matrix: &'a SparseMatrix, @@ -163,7 +160,7 @@ where impl<'a, T> Iterator for SparseMatrixIter<'a, T> where - T: Clone + Copy, + T: Clone + Copy + Default, { type Item = &'a SparseVector; diff --git a/src/svm/class.rs b/src/svm/class.rs index 8ce72a7..3bb4080 100644 --- a/src/svm/class.rs +++ b/src/svm/class.rs @@ -32,7 +32,7 @@ impl Class> { impl Class> { /// Creates a new class with the given parameters. - pub fn with_parameters(classes: usize, support_vectors: usize, attributes: usize, label: u32) -> Class> { + pub fn with_parameters(classes: usize, support_vectors: usize, _attributes: usize, label: u32) -> Class> { Class { label, num_support_vectors: support_vectors, diff --git a/src/svm/core/dense.rs b/src/svm/core/dense.rs index 038cc49..2991d3a 100644 --- a/src/svm/core/dense.rs +++ b/src/svm/core/dense.rs @@ -9,341 +9,39 @@ use crate::{ core::SVMCore, kernel::{KernelDense, Linear, Poly, Rbf, Sigmoid}, predict::Predict, - problem::{Problem, Outcome}, - Probabilities, SVMType, + problem::{Problem, Solution}, + DenseSVM, Probabilities, SVMType, }, util::{find_max_index, set_all, sigmoid_predict}, vectors::Triangular, }; -impl SVMCore, SimdVector, SimdVector> { - /// Computes the kernel values for this problem - crate fn compute_kernel_values(&self, problem: &mut Problem>) { - // Get current problem and decision values array - let features = &problem.features; - let kernel_values = &mut problem.kernel_values; - - // Compute kernel values per class - for (i, class) in self.classes.iter().enumerate() { - let kvalues = kernel_values.row_as_flat_mut(i); - - self.kernel.compute(&class.support_vectors, features.as_raw(), kvalues); - } - } - - // This is pretty much copy-paste of `multiclass_probability` from libSVM which we need - // to be compatibly for predicting probability for multiclass SVMs. The method is in turn - // based on Method 2 from the paper "Probability Estimates for Multi-class - // Classification by Pairwise Coupling", Journal of Machine Learning Research 5 (2004) 975-1005, - // by Ting-Fan Wu, Chih-Jen Lin and Ruby C. Weng. - crate fn compute_multiclass_probabilities(&self, problem: &mut Problem>) -> Result<(), Error> { - let num_classes = self.classes.len(); - let max_iter = 100.max(num_classes); - let mut q = problem.q.flat_mut(); - let qp = &mut problem.qp; - let eps = 0.005 / num_classes as f64; // Magic number .005 comes from libSVM. - let pairwise = problem.pairwise.flat(); - let probabilities = problem.probabilities.flat_mut(); - - // We first build up matrix Q as defined in (14) in the paper above. Q should have - // the property of being a transition matrix for a Markov Chain. - for t in 0 .. num_classes { - probabilities[t] = 1.0 / num_classes as f64; - - q[(t, t)] = 0.0; - - for j in 0 .. t { - q[(t, t)] += pairwise[(j, t)] * pairwise[(j, t)]; - q[(t, j)] = q[(j, t)]; - } - - for j in t + 1 .. num_classes { - q[(t, t)] += pairwise[(j, t)] * pairwise[(j, t)]; - q[(t, j)] = -pairwise[(j, t)] * pairwise[(t, j)]; - } - } - - // We now try to satisfy (21), (23) and (24) in the paper above. - for i in 0 ..= max_iter { - let mut pqp = 0.0; - - for t in 0 .. num_classes { - qp[t] = 0.0; - - for j in 0 .. num_classes { - qp[t] += q[(t, j)] * probabilities[j]; - } - - pqp += probabilities[t] * qp[t]; - } - - // Check if we fulfilled our abort criteria, which seems to be related - // to (21). - let mut max_error = 0.0; - - for item in qp.iter() { - let error = (*item - pqp).abs(); - - if error > max_error { - max_error = error; - } - } - - if max_error < eps { - break; - } - - // In case we are on the last iteration round past the threshold - // we know something went wrong. Signal we exceeded the threshold. - if i == max_iter { - return Err(Error::IterationsExceeded); - } - - // This seems to be the main function performing (23) and (24). - for t in 0 .. num_classes { - let diff = (-qp[t] + pqp) / q[(t, t)]; - - probabilities[t] += diff; - pqp = (pqp + diff * (diff * q[(t, t)] + 2.0 * qp[t])) / (1.0 + diff) / (1.0 + diff); - - for j in 0 .. num_classes { - qp[j] = (qp[j] + diff * q[(t, j)]) / (1.0 + diff); - probabilities[j] /= 1.0 + diff; - } - } - } - - Ok(()) - } - - /// Based on kernel values, computes the decision values for this problem. - crate fn compute_classification_values(&self, problem: &mut Problem>) { - // Reset all votes - set_all(&mut problem.vote, 0); - - // Since classification is symmetric, if we have N classes, we only need to go through - // (N * N - 1) - 1 cases. For example for 4 classes we do: - // - // j ---> - // 0 1 2 3 - // i 0 x x x - // | 1 x x - // v 2 x - // 3 - // - // For each valid combination (e.g., i:1, j:2), we then need to compute - // the decision values, which consists of two parts: - // - // a) The coefficients of class(1) related to class(2) and - // b) The coefficients of class(2) related to class(1). - // - // Both a) and b) are multiplied with the computed kernel values and summed, - // and eventually used to compute on which side we are. - - for i in 0 .. self.classes.len() { - for j in (i + 1) .. self.classes.len() { - let sv_coef0 = self.classes[i].coefficients.row(j - 1); - let sv_coef1 = self.classes[j].coefficients.row(i); - - let kvalues0 = problem.kernel_values.row(i); - let kvalues1 = problem.kernel_values.row(j); - - let sum0 = sv_coef0.iter().zip(kvalues0).map(|(a, b)| (*a * *b).sum()).sum::(); - let sum1 = sv_coef1.iter().zip(kvalues1).map(|(a, b)| (*a * *b).sum()).sum::(); - - let sum = sum0 + sum1 - self.rho[(i, j)]; - let index_to_vote = if sum > 0.0 { i } else { j }; - - problem.decision_values[(i, j)] = sum; - problem.vote[index_to_vote] += 1; - } - } - } - - /// Based on kernel values, computes the decision values for this problem. - crate fn compute_regression_values(&self, problem: &mut Problem>) { - let class = &self.classes[0]; - let coef = class.coefficients.row(0); - let kvalues = problem.kernel_values.row(0); - - let mut sum = coef.iter().zip(kvalues).map(|(a, b)| (*a * *b).sum()).sum::(); - - sum -= self.rho[0]; - - problem.result = Outcome::Value(sum as f32); - } +impl DenseSVM { + impl_common_svm!(SimdVector); } -impl Predict, SimdVector> for SVMCore, SimdVector, SimdVector> { - fn predict_probability(&self, problem: &mut Problem>) -> Result<(), Error> { - match self.svm_type { - SVMType::CSvc | SVMType::NuSvc => { - const MIN_PROB: f64 = 1e-7; - - // Ensure we have probabilities set. If not, somebody used us the wrong way - if self.probabilities.is_none() { - return Err(Error::NoProbabilities); - } - - let num_classes = self.classes.len(); - let probabilities = self.probabilities.as_ref().unwrap(); - - // First we need to predict the problem for our decision values - self.predict_value(problem)?; - - let mut pairwise = problem.pairwise.flat_mut(); - - // Now compute probability values - for i in 0 .. num_classes { - for j in i + 1 .. num_classes { - let decision_value = problem.decision_values[(i, j)]; - let a = probabilities.a[(i, j)]; - let b = probabilities.b[(i, j)]; - - let sigmoid = sigmoid_predict(decision_value, a, b).max(MIN_PROB).min(1f64 - MIN_PROB); - - pairwise[(i, j)] = sigmoid; - pairwise[(j, i)] = 1f64 - sigmoid; - } - } - - let problem_probabilities = problem.probabilities.flat_mut(); - - if num_classes == 2 { - problem_probabilities[0] = pairwise[(0, 1)]; - problem_probabilities[1] = pairwise[(1, 0)]; - } else { - self.compute_multiclass_probabilities(problem)?; - } - - let max_index = find_max_index(problem.probabilities.flat()); - problem.result = Outcome::Label(self.classes[max_index].label); - - Ok(()) - } - // This fallback behavior is mandated by `libSVM`. - SVMType::ESvr | SVMType::NuSvr => self.predict_value(problem), - } - } - - // Predict the value for one problem. - fn predict_value(&self, problem: &mut Problem>) -> Result<(), Error> { - match self.svm_type { - SVMType::CSvc | SVMType::NuSvc => { - // Compute kernel, decision values and eventually the label - self.compute_kernel_values(problem); - self.compute_classification_values(problem); - - // Compute highest vote - let highest_vote = find_max_index(&problem.vote); - problem.result = Outcome::Label(self.classes[highest_vote].label); - - Ok(()) - } - SVMType::ESvr | SVMType::NuSvr => { - self.compute_kernel_values(problem); - self.compute_regression_values(problem); - Ok(()) - } - } - } +impl Predict, SimdVector> for DenseSVM { + impl_common_predict!(SimdVector); } -impl<'a, 'b> TryFrom<&'a str> for SVMCore, SimdVector, SimdVector> { +impl<'a, 'b> TryFrom<&'a str> for DenseSVM { type Error = Error; - fn try_from(input: &'a str) -> Result, SimdVector, SimdVector>, Error> { + fn try_from(input: &'a str) -> Result { let raw_model = ModelFile::try_from(input)?; Self::try_from(&raw_model) } } -impl<'a, 'b> TryFrom<&'a ModelFile<'b>> for SVMCore, SimdVector, SimdVector> { +impl<'a, 'b> TryFrom<&'a ModelFile<'b>> for DenseSVM { type Error = Error; - fn try_from(raw_model: &'a ModelFile) -> Result, SimdVector, SimdVector>, Error> { - // To quickly check what broke again during parsing ... - // println!("{:?}", raw_model); + fn try_from(raw_model: &'a ModelFile<'_>) -> Result { + let (mut svm, nr_sv) = prepare_svm!(raw_model, dyn KernelDense, SimdMatrix); - let header = &raw_model.header; let vectors = &raw_model.vectors; - // Get basic info - let num_attributes = vectors[0].features.len(); - let num_total_sv = header.total_sv as usize; - - let svm_type = match raw_model.header.svm_type { - "c_svc" => SVMType::CSvc, - "nu_svc" => SVMType::NuSvc, - "epsilon_svr" => SVMType::ESvr, - "nu_svr" => SVMType::NuSvr, - _ => unimplemented!(), - }; - - let kernel: Box = match raw_model.header.kernel_type { - "rbf" => Box::new(Rbf::try_from(raw_model)?), - "linear" => Box::new(Linear::from(raw_model)), - "polynomial" => Box::new(Poly::try_from(raw_model)?), - "sigmoid" => Box::new(Sigmoid::try_from(raw_model)?), - _ => unimplemented!(), - }; - - let num_classes = match svm_type { - SVMType::CSvc | SVMType::NuSvc => header.nr_class as usize, - // For SVRs we set number of classes to 1, since that resonates better - // with our internal handling - SVMType::ESvr | SVMType::NuSvr => 1, - }; - - let nr_sv = match svm_type { - SVMType::CSvc | SVMType::NuSvc => header.nr_sv.clone(), - // For SVRs we set number of classes to 1, since that resonates better - // with our internal handling - SVMType::ESvr | SVMType::NuSvr => vec![num_total_sv as u32], - }; - - // Construct vector of classes - let classes = match svm_type { - // TODO: CLEAN THIS UP ... We can probably unify the logic - SVMType::CSvc | SVMType::NuSvc => (0 .. num_classes) - .map(|c| { - let label = header.label[c]; - let num_sv = nr_sv[c] as usize; - Class::>::with_parameters(num_classes, num_sv, num_attributes, label) - }).collect::>>>(), - SVMType::ESvr | SVMType::NuSvr => vec![Class::>::with_parameters(2, num_total_sv, num_attributes, 0)], - }; - - let probabilities = match (&raw_model.header.prob_a, &raw_model.header.prob_b) { - // Regular case for classification with probabilities - (&Some(ref a), &Some(ref b)) => Some(Probabilities { - a: Triangular::from(a), - b: Triangular::from(b), - }), - // For SVRs only one probability array is given - (&Some(ref a), None) => Some(Probabilities { - a: Triangular::from(a), - b: Triangular::with_dimension(0, 0.0), - }), - // Regular case for classification w/o probabilities - (_, _) => None, - }; - - // Allocate model - let mut svm = SVMCore { - num_total_sv, - num_attributes, - probabilities, - kernel, - svm_type, - rho: Triangular::from(&header.rho), - classes, - phantomV32: PhantomData, - phantomV64: PhantomData, - }; - // Things down here are a bit ugly as the file format is a bit ugly ... - // Now read all vectors and decode stored information let mut start_offset = 0; @@ -390,3 +88,20 @@ impl<'a, 'b> TryFrom<&'a ModelFile<'b>> for SVMCore Result<(), Error> { + let svm = DenseSVM::try_from(SAMPLE_MODEL)?; + + assert_eq!(None, svm.class_index_for_label(0)); + assert_eq!(Some(1), svm.class_index_for_label(42)); + + Ok(()) + } + +} diff --git a/src/svm/core/mod.rs b/src/svm/core/mod.rs index 5119afc..5f41a50 100644 --- a/src/svm/core/mod.rs +++ b/src/svm/core/mod.rs @@ -1,35 +1,26 @@ -mod dense; -mod sparse; - -use simd_aligned::{f32s, f64s, RowOptimized, SimdMatrix, SimdVector}; -use std::{convert::TryFrom, marker::PhantomData}; +use std::marker::PhantomData; use crate::{ - errors::Error, - parser::ModelFile, - svm::{ - class::Class, - kernel::{KernelDense, Linear, Poly, Rbf, Sigmoid}, - predict::Predict, - problem::{Outcome, Problem}, - Probabilities, SVMType, - }, - util::{find_max_index, set_all, sigmoid_predict}, + svm::{class::Class, Probabilities, SVMType}, vectors::Triangular, }; /// Generic support vector machine core, used by [DenseSVM] and [SparseSVM]. /// /// The SVM holds a kernel, class information and all other numerical data read from -/// the [ModelFile]. It implements [Predict] to predict [Problem] instances. +/// the model. It implements [Predict] to predict [Problem] instances. /// /// # Creating a SVM /// -/// The only SVM currently implemented is the [RbfSVM]. It can be constructed from a -/// [ModelFile] like this: +/// Models can be constructed like this: /// -/// ```ignore -/// let svm = RbfSVM::try_from(&model)!; +/// ``` +/// #![feature(try_from)] +/// +/// use ffsvm::*; +/// use std::convert::TryFrom; +/// +/// let svm = SparseSVM::try_from("..."); /// ``` /// pub struct SVMCore @@ -54,61 +45,399 @@ where /// All classes crate classes: Vec>, - phantomV32: PhantomData, + phantom_v32: PhantomData, + + phantom_v64: PhantomData, +} + +impl SVMCore +where + K: ?Sized, +{ + /// Returns number of attributes, reflecting the libSVM model. + pub fn attributes(&self) -> usize { self.num_attributes } - phantomV64: PhantomData, + /// Returns number of classes, reflecting the libSVM model. + pub fn classes(&self) -> usize { self.classes.len() } } -impl SVMCore { - /// Finds the class index for a given label. - /// - /// # Description - /// - /// This method takes a `label` as defined in the libSVM training model - /// and returns the internal `index` where this label resides. The index - /// equals the [Problem]'s `.probabilities` index where that label's - /// probability can be found. - /// - /// # Returns - /// - /// If the label was found its index returned in the [Option]. Otherwise `None` - /// is returned. - /// - pub fn class_index_for_label(&self, label: u32) -> Option { - for (i, class) in self.classes.iter().enumerate() { - if class.label != label { - continue; +macro_rules! impl_common_svm { + ($v32:ty) => { + /// Finds the class index for a given label. + /// + /// # Description + /// + /// This method takes a `label` as defined in the libSVM training model + /// and returns the internal `index` where this label resides. The index + /// equals [Problem::probabilities] index where that label's + /// probability can be found. + /// + /// # Returns + /// + /// If the label was found its index returned in the [Option]. Otherwise `None` + /// is returned. + /// + pub fn class_index_for_label(&self, label: u32) -> Option { + for (i, class) in self.classes.iter().enumerate() { + if class.label != label { + continue; + } + + return Some(i); } - return Some(i); + None } - None - } + /// Returns the class label for a given index. + /// + /// # Description + /// + /// The inverse of [SVMCore::class_index_for_label], this function returns the class label + /// associated with a certain internal index. The index equals the [Problem]'s + /// `.probabilities` index where a label's probability can be found. + /// + /// # Returns + /// + /// If the index was found it is returned in the [Option]. Otherwise `None` + /// is returned. + pub fn class_label_for_index(&self, index: usize) -> Option { + if index >= self.classes.len() { + None + } else { + Some(self.classes[index].label) + } + } - /// Returns the class label for a given index. - /// - /// # Description - /// - /// The inverse of [SVM::class_index_for_label], this function returns the class label - /// associated with a certain internal index. The index equals the [Problem]'s - /// `.probabilities` index where a label's probability can be found. - /// - /// # Returns - /// - /// If the index was found it is returned in the [Option]. Otherwise `None` - /// is returned. - pub fn class_label_for_index(&self, index: usize) -> Option { - if index >= self.classes.len() { - None - } else { - Some(self.classes[index].label) + /// Computes the kernel values for this problem + crate fn compute_kernel_values(&self, problem: &mut Problem<$v32>) { + // Get current problem and decision values array + let features = &problem.features; + let kernel_values = &mut problem.kernel_values; + + // Compute kernel values per class + for (i, class) in self.classes.iter().enumerate() { + let kvalues = kernel_values.row_as_flat_mut(i); + + self.kernel.compute(&class.support_vectors, features.as_raw(), kvalues); + } + } + + + // This is pretty much copy-paste of `multiclass_probability` from libSVM which we need + // to be compatibly for predicting probability for multiclass SVMs. The method is in turn + // based on Method 2 from the paper "Probability Estimates for Multi-class + // Classification by Pairwise Coupling", Journal of Machine Learning Research 5 (2004) 975-1005, + // by Ting-Fan Wu, Chih-Jen Lin and Ruby C. Weng. + crate fn compute_multiclass_probabilities(&self, problem: &mut Problem<$v32>) -> Result<(), Error> { + let num_classes = self.classes.len(); + let max_iter = 100.max(num_classes); + let mut q = problem.q.flat_mut(); + let qp = &mut problem.qp; + let eps = 0.005 / num_classes as f64; // Magic number .005 comes from libSVM. + let pairwise = problem.pairwise.flat(); + let probabilities = problem.probabilities.flat_mut(); + + // We first build up matrix Q as defined in (14) in the paper above. Q should have + // the property of being a transition matrix for a Markov Chain. + for t in 0 .. num_classes { + probabilities[t] = 1.0 / num_classes as f64; + + q[(t, t)] = 0.0; + + for j in 0 .. t { + q[(t, t)] += pairwise[(j, t)] * pairwise[(j, t)]; + q[(t, j)] = q[(j, t)]; + } + + for j in t + 1 .. num_classes { + q[(t, t)] += pairwise[(j, t)] * pairwise[(j, t)]; + q[(t, j)] = -pairwise[(j, t)] * pairwise[(t, j)]; + } + } + + // We now try to satisfy (21), (23) and (24) in the paper above. + for i in 0 ..= max_iter { + let mut pqp = 0.0; + + for t in 0 .. num_classes { + qp[t] = 0.0; + + for j in 0 .. num_classes { + qp[t] += q[(t, j)] * probabilities[j]; + } + + pqp += probabilities[t] * qp[t]; + } + + // Check if we fulfilled our abort criteria, which seems to be related + // to (21). + let mut max_error = 0.0; + + for item in qp.iter() { + let error = (*item - pqp).abs(); + + if error > max_error { + max_error = error; + } + } + + if max_error < eps { + break; + } + + // In case we are on the last iteration round past the threshold + // we know something went wrong. Signal we exceeded the threshold. + if i == max_iter { + return Err(Error::IterationsExceeded); + } + + // This seems to be the main function performing (23) and (24). + for t in 0 .. num_classes { + let diff = (-qp[t] + pqp) / q[(t, t)]; + + probabilities[t] += diff; + pqp = (pqp + diff * (diff * q[(t, t)] + 2.0 * qp[t])) / (1.0 + diff) / (1.0 + diff); + + for j in 0 .. num_classes { + qp[j] = (qp[j] + diff * q[(t, j)]) / (1.0 + diff); + probabilities[j] /= 1.0 + diff; + } + } + } + + Ok(()) + } + + /// Based on kernel values, computes the decision values for this problem. + crate fn compute_classification_values(&self, problem: &mut Problem<$v32>) { + // Reset all votes + set_all(&mut problem.vote, 0); + + // Since classification is symmetric, if we have N classes, we only need to go through + // (N * N - 1) - 1 cases. For example for 4 classes we do: + // + // j ---> + // 0 1 2 3 + // i 0 x x x + // | 1 x x + // v 2 x + // 3 + // + // For each valid combination (e.g., i:1, j:2), we then need to compute + // the decision values, which consists of two parts: + // + // a) The coefficients of class(1) related to class(2) and + // b) The coefficients of class(2) related to class(1). + // + // Both a) and b) are multiplied with the computed kernel values and summed, + // and eventually used to compute on which side we are. + + for i in 0 .. self.classes.len() { + for j in (i + 1) .. self.classes.len() { + let sv_coef0 = self.classes[i].coefficients.row(j - 1); + let sv_coef1 = self.classes[j].coefficients.row(i); + + let kvalues0 = problem.kernel_values.row(i); + let kvalues1 = problem.kernel_values.row(j); + + let sum0 = sv_coef0.iter().zip(kvalues0).map(|(a, b)| (*a * *b).sum()).sum::(); + let sum1 = sv_coef1.iter().zip(kvalues1).map(|(a, b)| (*a * *b).sum()).sum::(); + + let sum = sum0 + sum1 - self.rho[(i, j)]; + let index_to_vote = if sum > 0.0 { i } else { j }; + + problem.decision_values[(i, j)] = sum; + problem.vote[index_to_vote] += 1; + } + } } + + /// Based on kernel values, computes the decision values for this problem. + crate fn compute_regression_values(&self, problem: &mut Problem<$v32>) { + let class = &self.classes[0]; + let coef = class.coefficients.row(0); + let kvalues = problem.kernel_values.row(0); + + let mut sum = coef.iter().zip(kvalues).map(|(a, b)| (*a * *b).sum()).sum::(); + + sum -= self.rho[0]; + + problem.result = Solution::Value(sum as f32); + } + + + }; +} + +macro_rules! impl_common_predict { + ($v32:ty) => { + + fn predict_probability(&self, problem: &mut Problem<$v32>) -> Result<(), Error> { + match self.svm_type { + SVMType::CSvc | SVMType::NuSvc => { + const MIN_PROB: f64 = 1e-7; + + // Ensure we have probabilities set. If not, somebody used us the wrong way + if self.probabilities.is_none() { + return Err(Error::NoProbabilities); + } + + let num_classes = self.classes.len(); + let probabilities = self.probabilities.as_ref().unwrap(); + + // First we need to predict the problem for our decision values + self.predict_value(problem)?; + + let mut pairwise = problem.pairwise.flat_mut(); + + // Now compute probability values + for i in 0 .. num_classes { + for j in i + 1 .. num_classes { + let decision_value = problem.decision_values[(i, j)]; + let a = probabilities.a[(i, j)]; + let b = probabilities.b[(i, j)]; + + let sigmoid = sigmoid_predict(decision_value, a, b).max(MIN_PROB).min(1f64 - MIN_PROB); + + pairwise[(i, j)] = sigmoid; + pairwise[(j, i)] = 1f64 - sigmoid; + } + } + + let problem_probabilities = problem.probabilities.flat_mut(); + + if num_classes == 2 { + problem_probabilities[0] = pairwise[(0, 1)]; + problem_probabilities[1] = pairwise[(1, 0)]; + } else { + self.compute_multiclass_probabilities(problem)?; + } + + let max_index = find_max_index(problem.probabilities.flat()); + problem.result = Solution::Label(self.classes[max_index].label); + + Ok(()) + } + // This fallback behavior is mandated by `libSVM`. + SVMType::ESvr | SVMType::NuSvr => self.predict_value(problem), + } + } + + + // Predict the value for one problem. + fn predict_value(&self, problem: &mut Problem<$v32>) -> Result<(), Error> { + match self.svm_type { + SVMType::CSvc | SVMType::NuSvc => { + // Compute kernel, decision values and eventually the label + self.compute_kernel_values(problem); + self.compute_classification_values(problem); + + // Compute highest vote + let highest_vote = find_max_index(&problem.vote); + problem.result = Solution::Label(self.classes[highest_vote].label); + + Ok(()) + } + SVMType::ESvr | SVMType::NuSvr => { + self.compute_kernel_values(problem); + self.compute_regression_values(problem); + Ok(()) + } + } + } + } +} - /// Returns number of attributes, reflecting the libSVM model. - pub fn attributes(&self) -> usize { self.num_attributes } +macro_rules! prepare_svm { + ($raw_model:expr, $k:ty, $m32:ty) => { + // To quickly check what broke again during parsing ... + // println!("{:?}", raw_model); + { + let header = &$raw_model.header; + let vectors = &$raw_model.vectors; - /// Returns number of classes, reflecting the libSVM model. - pub fn classes(&self) -> usize { self.classes.len() } + // Get basic info + let num_attributes = vectors[0].features.len(); + let num_total_sv = header.total_sv as usize; + + let svm_type = match $raw_model.header.svm_type { + "c_svc" => SVMType::CSvc, + "nu_svc" => SVMType::NuSvc, + "epsilon_svr" => SVMType::ESvr, + "nu_svr" => SVMType::NuSvr, + _ => unimplemented!(), + }; + + let kernel: Box<$k> = match $raw_model.header.kernel_type { + "rbf" => Box::new(Rbf::try_from($raw_model)?), + "linear" => Box::new(Linear::from($raw_model)), + "polynomial" => Box::new(Poly::try_from($raw_model)?), + "sigmoid" => Box::new(Sigmoid::try_from($raw_model)?), + _ => unimplemented!(), + }; + + let num_classes = match svm_type { + SVMType::CSvc | SVMType::NuSvc => header.nr_class as usize, + // For SVRs we set number of classes to 1, since that resonates better + // with our internal handling + SVMType::ESvr | SVMType::NuSvr => 1, + }; + + let nr_sv = match svm_type { + SVMType::CSvc | SVMType::NuSvc => header.nr_sv.clone(), + // For SVRs we set number of classes to 1, since that resonates better + // with our internal handling + SVMType::ESvr | SVMType::NuSvr => vec![num_total_sv as u32], + }; + + // Construct vector of classes + let classes = match svm_type { + // TODO: CLEAN THIS UP ... We can probably unify the logic + SVMType::CSvc | SVMType::NuSvc => (0 .. num_classes) + .map(|c| { + let label = header.label[c]; + let num_sv = nr_sv[c] as usize; + Class::<$m32>::with_parameters(num_classes, num_sv, num_attributes, label) + }).collect::>>(), + SVMType::ESvr | SVMType::NuSvr => vec![Class::<$m32>::with_parameters(2, num_total_sv, num_attributes, 0)], + }; + + let probabilities = match (&$raw_model.header.prob_a, &$raw_model.header.prob_b) { + // Regular case for classification with probabilities + (&Some(ref a), &Some(ref b)) => Some(Probabilities { + a: Triangular::from(a), + b: Triangular::from(b), + }), + // For SVRs only one probability array is given + (&Some(ref a), None) => Some(Probabilities { + a: Triangular::from(a), + b: Triangular::with_dimension(0, 0.0), + }), + // Regular case for classification w/o probabilities + (_, _) => None, + }; + + // Allocate model + ( + SVMCore { + num_total_sv, + num_attributes, + probabilities, + kernel, + svm_type, + rho: Triangular::from(&header.rho), + classes, + phantom_v32: PhantomData, + phantom_v64: PhantomData, + }, + nr_sv, + ) + } + }; } + +// We do late include here to capture our macros above ... +mod dense; +mod sparse; diff --git a/src/svm/core/sparse.rs b/src/svm/core/sparse.rs index 2f13b93..6263d00 100644 --- a/src/svm/core/sparse.rs +++ b/src/svm/core/sparse.rs @@ -10,341 +10,39 @@ use crate::{ core::SVMCore, kernel::{KernelSparse, Linear, Poly, Rbf, Sigmoid}, predict::Predict, - problem::{Outcome, Problem}, - Probabilities, SVMType, + problem::{Problem, Solution}, + Probabilities, SVMType, SparseSVM, }, util::{find_max_index, set_all, sigmoid_predict}, vectors::Triangular, }; -impl SVMCore, SparseVector, SparseVector> { - /// Computes the kernel values for this problem - crate fn compute_kernel_values(&self, problem: &mut Problem>) { - // Get current problem and decision values array - let features = &problem.features; - let kernel_values = &mut problem.kernel_values; - - // Compute kernel values per class - for (i, class) in self.classes.iter().enumerate() { - let kvalues = kernel_values.row_as_flat_mut(i); - - self.kernel.compute(&class.support_vectors, features.as_raw(), kvalues); - } - } - - // This is pretty much copy-paste of `multiclass_probability` from libSVM which we need - // to be compatibly for predicting probability for multiclass SVMs. The method is in turn - // based on Method 2 from the paper "Probability Estimates for Multi-class - // Classification by Pairwise Coupling", Journal of Machine Learning Research 5 (2004) 975-1005, - // by Ting-Fan Wu, Chih-Jen Lin and Ruby C. Weng. - crate fn compute_multiclass_probabilities(&self, problem: &mut Problem>) -> Result<(), Error> { - let num_classes = self.classes.len(); - let max_iter = 100.max(num_classes); - let mut q = problem.q.flat_mut(); - let qp = &mut problem.qp; - let eps = 0.005 / num_classes as f64; // Magic number .005 comes from libSVM. - let pairwise = problem.pairwise.flat(); - let probabilities = problem.probabilities.flat_mut(); - - // We first build up matrix Q as defined in (14) in the paper above. Q should have - // the property of being a transition matrix for a Markov Chain. - for t in 0 .. num_classes { - probabilities[t] = 1.0 / num_classes as f64; - - q[(t, t)] = 0.0; - - for j in 0 .. t { - q[(t, t)] += pairwise[(j, t)] * pairwise[(j, t)]; - q[(t, j)] = q[(j, t)]; - } - - for j in t + 1 .. num_classes { - q[(t, t)] += pairwise[(j, t)] * pairwise[(j, t)]; - q[(t, j)] = -pairwise[(j, t)] * pairwise[(t, j)]; - } - } - - // We now try to satisfy (21), (23) and (24) in the paper above. - for i in 0 ..= max_iter { - let mut pqp = 0.0; - - for t in 0 .. num_classes { - qp[t] = 0.0; - - for j in 0 .. num_classes { - qp[t] += q[(t, j)] * probabilities[j]; - } - - pqp += probabilities[t] * qp[t]; - } - - // Check if we fulfilled our abort criteria, which seems to be related - // to (21). - let mut max_error = 0.0; - - for item in qp.iter() { - let error = (*item - pqp).abs(); - - if error > max_error { - max_error = error; - } - } - - if max_error < eps { - break; - } - - // In case we are on the last iteration round past the threshold - // we know something went wrong. Signal we exceeded the threshold. - if i == max_iter { - return Err(Error::IterationsExceeded); - } - - // This seems to be the main function performing (23) and (24). - for t in 0 .. num_classes { - let diff = (-qp[t] + pqp) / q[(t, t)]; - - probabilities[t] += diff; - pqp = (pqp + diff * (diff * q[(t, t)] + 2.0 * qp[t])) / (1.0 + diff) / (1.0 + diff); - - for j in 0 .. num_classes { - qp[j] = (qp[j] + diff * q[(t, j)]) / (1.0 + diff); - probabilities[j] /= 1.0 + diff; - } - } - } - - Ok(()) - } - - /// Based on kernel values, computes the decision values for this problem. - crate fn compute_classification_values(&self, problem: &mut Problem>) { - // Reset all votes - set_all(&mut problem.vote, 0); - - // Since classification is symmetric, if we have N classes, we only need to go through - // (N * N - 1) - 1 cases. For example for 4 classes we do: - // - // j ---> - // 0 1 2 3 - // i 0 x x x - // | 1 x x - // v 2 x - // 3 - // - // For each valid combination (e.g., i:1, j:2), we then need to compute - // the decision values, which consists of two parts: - // - // a) The coefficients of class(1) related to class(2) and - // b) The coefficients of class(2) related to class(1). - // - // Both a) and b) are multiplied with the computed kernel values and summed, - // and eventually used to compute on which side we are. - - for i in 0 .. self.classes.len() { - for j in (i + 1) .. self.classes.len() { - let sv_coef0 = self.classes[i].coefficients.row(j - 1); - let sv_coef1 = self.classes[j].coefficients.row(i); - - let kvalues0 = problem.kernel_values.row(i); - let kvalues1 = problem.kernel_values.row(j); - - let sum0 = sv_coef0.iter().zip(kvalues0).map(|(a, b)| (*a * *b).sum()).sum::(); - let sum1 = sv_coef1.iter().zip(kvalues1).map(|(a, b)| (*a * *b).sum()).sum::(); - - let sum = sum0 + sum1 - self.rho[(i, j)]; - let index_to_vote = if sum > 0.0 { i } else { j }; - - problem.decision_values[(i, j)] = sum; - problem.vote[index_to_vote] += 1; - } - } - } - - /// Based on kernel values, computes the decision values for this problem. - crate fn compute_regression_values(&self, problem: &mut Problem>) { - let class = &self.classes[0]; - let coef = class.coefficients.row(0); - let kvalues = problem.kernel_values.row(0); - - let mut sum = coef.iter().zip(kvalues).map(|(a, b)| (*a * *b).sum()).sum::(); - - sum -= self.rho[0]; - - problem.result = Outcome::Value(sum as f32); - } +impl SparseSVM { + impl_common_svm!(SparseVector); } -impl Predict, SparseVector> for SVMCore, SparseVector, SparseVector> { - fn predict_probability(&self, problem: &mut Problem>) -> Result<(), Error> { - match self.svm_type { - SVMType::CSvc | SVMType::NuSvc => { - const MIN_PROB: f64 = 1e-7; - - // Ensure we have probabilities set. If not, somebody used us the wrong way - if self.probabilities.is_none() { - return Err(Error::NoProbabilities); - } - - let num_classes = self.classes.len(); - let probabilities = self.probabilities.as_ref().unwrap(); - - // First we need to predict the problem for our decision values - self.predict_value(problem)?; - - let mut pairwise = problem.pairwise.flat_mut(); - - // Now compute probability values - for i in 0 .. num_classes { - for j in i + 1 .. num_classes { - let decision_value = problem.decision_values[(i, j)]; - let a = probabilities.a[(i, j)]; - let b = probabilities.b[(i, j)]; - - let sigmoid = sigmoid_predict(decision_value, a, b).max(MIN_PROB).min(1f64 - MIN_PROB); - - pairwise[(i, j)] = sigmoid; - pairwise[(j, i)] = 1f64 - sigmoid; - } - } - - let problem_probabilities = problem.probabilities.flat_mut(); - - if num_classes == 2 { - problem_probabilities[0] = pairwise[(0, 1)]; - problem_probabilities[1] = pairwise[(1, 0)]; - } else { - self.compute_multiclass_probabilities(problem)?; - } - - let max_index = find_max_index(problem.probabilities.flat()); - problem.result = Outcome::Label(self.classes[max_index].label); - - Ok(()) - } - // This fallback behavior is mandated by `libSVM`. - SVMType::ESvr | SVMType::NuSvr => self.predict_value(problem), - } - } - - // Predict the value for one problem. - fn predict_value(&self, problem: &mut Problem>) -> Result<(), Error> { - match self.svm_type { - SVMType::CSvc | SVMType::NuSvc => { - // Compute kernel, decision values and eventually the label - self.compute_kernel_values(problem); - self.compute_classification_values(problem); - - // Compute highest vote - let highest_vote = find_max_index(&problem.vote); - problem.result = Outcome::Label(self.classes[highest_vote].label); - - Ok(()) - } - SVMType::ESvr | SVMType::NuSvr => { - self.compute_kernel_values(problem); - self.compute_regression_values(problem); - Ok(()) - } - } - } +impl Predict, SparseVector> for SparseSVM { + impl_common_predict!(SparseVector); } -impl<'a, 'b> TryFrom<&'a str> for SVMCore, SparseVector, SparseVector> { +impl<'a, 'b> TryFrom<&'a str> for SparseSVM { type Error = Error; - fn try_from(input: &'a str) -> Result, SparseVector, SparseVector>, Error> { + fn try_from(input: &'a str) -> Result { let raw_model = ModelFile::try_from(input)?; Self::try_from(&raw_model) } } -impl<'a, 'b> TryFrom<&'a ModelFile<'b>> for SVMCore, SparseVector, SparseVector> { +impl<'a, 'b> TryFrom<&'a ModelFile<'b>> for SparseSVM { type Error = Error; - fn try_from(raw_model: &'a ModelFile) -> Result, SparseVector, SparseVector>, Error> { - // To quickly check what broke again during parsing ... - // println!("{:?}", raw_model); + fn try_from(raw_model: &'a ModelFile<'_>) -> Result { + let (mut svm, nr_sv) = prepare_svm!(raw_model, dyn KernelSparse, SparseMatrix); - let header = &raw_model.header; let vectors = &raw_model.vectors; - // Get basic info - let num_attributes = vectors[0].features.len(); - let num_total_sv = header.total_sv as usize; - - let svm_type = match raw_model.header.svm_type { - "c_svc" => SVMType::CSvc, - "nu_svc" => SVMType::NuSvc, - "epsilon_svr" => SVMType::ESvr, - "nu_svr" => SVMType::NuSvr, - _ => unimplemented!(), - }; - - let kernel: Box = match raw_model.header.kernel_type { - "rbf" => Box::new(Rbf::try_from(raw_model)?), - "linear" => Box::new(Linear::from(raw_model)), - "polynomial" => Box::new(Poly::try_from(raw_model)?), - "sigmoid" => Box::new(Sigmoid::try_from(raw_model)?), - _ => unimplemented!(), - }; - - let num_classes = match svm_type { - SVMType::CSvc | SVMType::NuSvc => header.nr_class as usize, - // For SVRs we set number of classes to 1, since that resonates better - // with our internal handling - SVMType::ESvr | SVMType::NuSvr => 1, - }; - - let nr_sv = match svm_type { - SVMType::CSvc | SVMType::NuSvc => header.nr_sv.clone(), - // For SVRs we set number of classes to 1, since that resonates better - // with our internal handling - SVMType::ESvr | SVMType::NuSvr => vec![num_total_sv as u32], - }; - - // Construct vector of classes - let classes = match svm_type { - // TODO: CLEAN THIS UP ... We can probably unify the logic - SVMType::CSvc | SVMType::NuSvc => (0 .. num_classes) - .map(|c| { - let label = header.label[c]; - let num_sv = nr_sv[c] as usize; - Class::>::with_parameters(num_classes, num_sv, num_attributes, label) - }).collect::>>>(), - SVMType::ESvr | SVMType::NuSvr => vec![Class::>::with_parameters(2, num_total_sv, num_attributes, 0)], - }; - - let probabilities = match (&raw_model.header.prob_a, &raw_model.header.prob_b) { - // Regular case for classification with probabilities - (&Some(ref a), &Some(ref b)) => Some(Probabilities { - a: Triangular::from(a), - b: Triangular::from(b), - }), - // For SVRs only one probability array is given - (&Some(ref a), None) => Some(Probabilities { - a: Triangular::from(a), - b: Triangular::with_dimension(0, 0.0), - }), - // Regular case for classification w/o probabilities - (_, _) => None, - }; - - // Allocate model - let mut svm = SVMCore { - num_total_sv, - num_attributes, - probabilities, - kernel, - svm_type, - rho: Triangular::from(&header.rho), - classes, - phantomV32: PhantomData, - phantomV64: PhantomData, - }; - // Things down here are a bit ugly as the file format is a bit ugly ... - // Now read all vectors and decode stored information let mut start_offset = 0; @@ -354,14 +52,10 @@ impl<'a, 'b> TryFrom<&'a ModelFile<'b>> for SVMCore a = a_iter.next(), (Some((i_a, _)), Some((i_b, _))) if i_a > i_b => b = b_iter.next(), - _ => break sum as f64, + _ => break f64::from(sum), } } } diff --git a/src/svm/kernel/poly.rs b/src/svm/kernel/poly.rs index e27e160..8235917 100644 --- a/src/svm/kernel/poly.rs +++ b/src/svm/kernel/poly.rs @@ -7,7 +7,6 @@ use crate::{ sparse::{SparseMatrix, SparseVector}, }; -use rand::random; use simd_aligned::{f32s, RowOptimized, SimdMatrix, SimdVector}; #[derive(Copy, Clone, Debug, Default)] diff --git a/src/svm/kernel/rbf.rs b/src/svm/kernel/rbf.rs index 27dae85..66e879d 100644 --- a/src/svm/kernel/rbf.rs +++ b/src/svm/kernel/rbf.rs @@ -7,7 +7,6 @@ use crate::{ sparse::{SparseMatrix, SparseVector}, }; -use rand::random; use simd_aligned::{f32s, RowOptimized, SimdMatrix, SimdVector}; #[derive(Copy, Clone, Debug, Default)] diff --git a/src/svm/kernel/sigmoid.rs b/src/svm/kernel/sigmoid.rs index a333034..aaac910 100644 --- a/src/svm/kernel/sigmoid.rs +++ b/src/svm/kernel/sigmoid.rs @@ -7,7 +7,6 @@ use crate::{ sparse::{SparseMatrix, SparseVector}, }; -use rand::random; use simd_aligned::{f32s, RowOptimized, SimdMatrix, SimdVector}; #[derive(Copy, Clone, Debug, Default)] diff --git a/src/svm/predict.rs b/src/svm/predict.rs index b3b9851..decb5cd 100644 --- a/src/svm/predict.rs +++ b/src/svm/predict.rs @@ -1,14 +1,19 @@ use crate::{errors::Error, svm::problem::Problem}; -/// Implemented by [SVM]s to predict a [Problem]. +/// Implemented by [DenseSVM] and [SparseSVM] to predict a [Problem]. /// /// # Predicting a label /// /// To predict a label, first make sure the [Problem] has all features set. Then calling -/// ```ignore -/// svm.predict_value(&mut problem)! /// ``` -/// will update the `.label` field to correspond to the class label with the highest likelihood. +/// use ffsvm::*; +/// +/// fn set_features(svm: &DenseSVM, problem: &mut DenseProblem) { +/// // Predicts the value. +/// svm.predict_value(problem); +/// } +/// ``` +/// will update the [Problem::solution] to correspond to the class label with the highest likelihood. /// /// # Predicting a label and obtaining probability estimates. /// @@ -18,13 +23,18 @@ use crate::{errors::Error, svm::problem::Problem}; /// /// Probabilities are estimated like this: /// -/// ```ignore -/// svm.predict_probability(&mut problem)! +/// ``` +/// use ffsvm::*; +/// +/// fn set_features(svm: &DenseSVM, problem: &mut DenseProblem) { +/// // Predicts the value. +/// svm.predict_probability(problem); +/// } /// ``` /// -/// Predicting probabilities automatically predicts the best label. In addition `.probabilities` -/// will be updated accordingly. The class labels for each `.probabilities` entry can be obtained -/// by [SVM]'s `class_label_for_index` and `class_index_for_label` methods. +/// Predicting probabilities automatically predicts the best label. In addition [Problem::probabilities] +/// will be updated accordingly. The class labels for each probablity entry can be obtained +/// by the [SVMCore::class_label_for_index] and [SVMCore::class_index_for_label] methods. /// pub trait Predict where @@ -32,14 +42,14 @@ where { /// Predict a single value for a [Problem]. /// - /// The problem needs to have all `.features` set. Once this method returns, - /// the [Problem]'s field `.label` will be set. - fn predict_value(&self, _: &mut Problem) -> Result<(), Error>; + /// The problem needs to have all features set. Once this method returns, + /// the [Problem::solution] will be set. + fn predict_value(&self, problem: &mut Problem) -> Result<(), Error>; /// Predict a probability value for a problem. /// - /// The problem needs to have all `.features` set. Once this method returns, - /// both the [Problem]'s field `.label` will be set, and all `.probabilities` will - /// be set accordingly. - fn predict_probability(&self, _: &mut Problem) -> Result<(), Error>; + /// The problem needs to have all features set. Once this method returns, + /// both [Problem::solution] will be set, and all [Problem::probabilities] will + /// be available accordingly. + fn predict_probability(&self, problem: &mut Problem) -> Result<(), Error>; } diff --git a/src/svm/problem.rs b/src/svm/problem.rs index a186ca8..34146b7 100644 --- a/src/svm/problem.rs +++ b/src/svm/problem.rs @@ -1,22 +1,26 @@ -use std::{ - marker::PhantomData, - ops::{Index, IndexMut}, -}; +use std::ops::{Index, IndexMut}; use crate::{ - sparse::{SparseMatrix, SparseVector}, - svm::{ - core::SVMCore, - kernel::{KernelDense, KernelSparse}, - }, + sparse::SparseVector, + svm::{DenseSVM, SparseSVM}, vectors::Triangular, }; use simd_aligned::{f32s, f64s, RowOptimized, SimdMatrix, SimdVector}; +/// Problems produced for [DenseSVM]s. +/// +/// Also see [Problem] for more methods for this type. +pub type DenseProblem = Problem>; + +/// Problems produced for [SparseSVM]s. +/// +/// Also see [Problem] for more methods for this type. +pub type SparseProblem = Problem>; + /// The result of a classification #[derive(Copy, Debug, Clone, PartialEq)] -pub enum Outcome { +pub enum Solution { /// If classified this will hold the label. Label(u32), @@ -34,24 +38,41 @@ pub struct Features { /// A single problem a [DenseSVM] or [SparseSVM] should classify. /// -/// # Creating a problem +/// # Creating a `Problem` +/// +/// Problems are created via the `Problem::from` method and match the SVM type they were created for: +/// +/// ```rust +/// #![feature(try_from)] +/// +/// use ffsvm::*; +/// use std::convert::TryFrom; /// -/// Problems are created via the `Problem::from` method: +/// fn main() -> Result<(), Error> { +/// let svm = DenseSVM::try_from(SAMPLE_MODEL)?; /// -/// ```ignore -/// let mut problem = Problem::from(&svm); +/// let mut problem = Problem::from(&svm); +/// +/// Ok(()) +/// } /// ``` /// -/// # Classifying a problem +/// # Setting Features /// -/// A problem is an instance of the SVM's problem domain. To be classified, all `features` need +/// A `Problem` is an instance of the SVM's problem domain. Before it can be classified, all `features` need /// to be set, for example by: /// -/// ```ignore -/// problem.features = vec![-0.55838, -0.157895, 0.581292, -0.221184, 0.135713, -0.874396, -0.563197, -1.0, -1.0]; +/// ``` +/// use ffsvm::*; +/// +/// fn set_features(problem: &mut DenseProblem) { +/// let features = problem.features(); +/// features[0] = -0.221184; +/// features[3] = 0.135713; +/// } /// ``` /// -/// It can then be handed over to the [SVM] (via the [Predict] trait). +/// It can then be classified via the [Predict] trait. /// #[derive(Debug, Clone)] pub struct Problem { @@ -77,24 +98,25 @@ pub struct Problem { crate qp: Vec, /// Probability estimates that will be updated after this problem was processed - /// by `predict_probability` in [Predict] if the model supports it. + /// by `predict_probability`. crate probabilities: SimdVector, - /// Computed label that will be updated after this problem was processed by [Predict]. - crate result: Outcome, + /// Computed label that will be updated after this problem was processed. + crate result: Solution, } impl Problem { - pub fn result(&self) -> Outcome { self.result } + /// After a [Problem] has been classified, this will hold the SVMs solution. + pub fn solution(&self) -> Solution { self.result } + /// Returns the probability estimates. Only really useful if the model was trained with probability estimates and you classified with them. pub fn probabilities(&self) -> &[f64] { self.probabilities.flat() } - pub fn probabilities_mut(&mut self) -> &mut [f64] { self.probabilities.flat_mut() } - + /// Returns the features. You must set them first and classifiy the problem before you can get a solution. pub fn features(&mut self) -> &mut Features { &mut self.features } } -impl Problem> { +impl DenseProblem { /// Creates a new problem with the given parameters. crate fn with_dimension(total_sv: usize, num_classes: usize, num_attributes: usize) -> Problem> { Problem { @@ -108,14 +130,17 @@ impl Problem> { decision_values: Triangular::with_dimension(num_classes, Default::default()), vote: vec![Default::default(); num_classes], probabilities: SimdVector::with(0.0, num_classes), - result: Outcome::None, + result: Solution::None, } } } -impl Problem> { +impl SparseProblem { + /// Clears the [Problem] when reusing it between calls. Only needed for [SparseSVM] problems. + pub fn clear(&mut self) { self.features.data.clear(); } + /// Creates a new problem with the given parameters. - crate fn with_dimension(total_sv: usize, num_classes: usize, num_attributes: usize) -> Problem> { + crate fn with_dimension(total_sv: usize, num_classes: usize, _num_attributes: usize) -> Problem> { Problem { features: Features { data: SparseVector::new() }, kernel_values: SimdMatrix::with_dimension(num_classes, total_sv), @@ -125,21 +150,17 @@ impl Problem> { decision_values: Triangular::with_dimension(num_classes, Default::default()), vote: vec![Default::default(); num_classes], probabilities: SimdVector::with(0.0, num_classes), - result: Outcome::None, + result: Solution::None, } } } -impl<'a> From<&'a SVMCore, SimdVector, SimdVector>> for Problem> { - fn from(svm: &SVMCore, SimdVector, SimdVector>) -> Self { - Problem::>::with_dimension(svm.num_total_sv, svm.classes.len(), svm.num_attributes) - } +impl<'a> From<&'a DenseSVM> for DenseProblem { + fn from(svm: &DenseSVM) -> Self { Problem::>::with_dimension(svm.num_total_sv, svm.classes.len(), svm.num_attributes) } } -impl<'a> From<&'a SVMCore, SparseVector, SparseVector>> for Problem> { - fn from(svm: &SVMCore, SparseVector, SparseVector>) -> Self { - Problem::>::with_dimension(svm.num_total_sv, svm.classes.len(), svm.num_attributes) - } +impl<'a> From<&'a SparseSVM> for SparseProblem { + fn from(svm: &SparseSVM) -> Self { Problem::>::with_dimension(svm.num_total_sv, svm.classes.len(), svm.num_attributes) } } impl Features { diff --git a/tests/svm_dense_class.rs b/tests/svm_dense_class.rs index 307a48a..09d4ac4 100644 --- a/tests/svm_dense_class.rs +++ b/tests/svm_dense_class.rs @@ -40,15 +40,15 @@ macro_rules! test_model { svm.predict_value(&mut problem_0)?; svm.predict_value(&mut problem_7)?; - assert_eq!(problem_0.result(), Outcome::Label($libsvm[0]), "predict_value(problem_0)"); - assert_eq!(problem_7.result(), Outcome::Label($libsvm[1]), "predict_value(problem_7)"); + assert_eq!(problem_0.solution(), Solution::Label($libsvm[0]), "predict_value(problem_0)"); + assert_eq!(problem_7.solution(), Solution::Label($libsvm[1]), "predict_value(problem_7)"); if $prob { svm.predict_probability(&mut problem_0)?; svm.predict_probability(&mut problem_7)?; - assert_eq!(problem_0.result(), Outcome::Label($libsvm_prob[0]), "predict_probability(problem_0)"); - assert_eq!(problem_7.result(), Outcome::Label($libsvm_prob[1]), "predict_probability(problem_7)"); + assert_eq!(problem_0.solution(), Solution::Label($libsvm_prob[0]), "predict_probability(problem_0)"); + assert_eq!(problem_7.solution(), Solution::Label($libsvm_prob[1]), "predict_probability(problem_7)"); } Ok(()) @@ -58,7 +58,7 @@ macro_rules! test_model { #[cfg(test)] mod svm_dense_class { - use ffsvm::{DenseSVM, Error, Outcome, Predict, Problem}; + use ffsvm::{DenseSVM, Error, Predict, Problem, Solution}; use std::convert::TryFrom; // CSVM diff --git a/tests/svm_dense_regression.rs b/tests/svm_dense_regression.rs index e392959..c416789 100644 --- a/tests/svm_dense_regression.rs +++ b/tests/svm_dense_regression.rs @@ -1,11 +1,11 @@ #![feature(test)] #![feature(try_from)] -use ffsvm::Outcome; +use ffsvm::Solution; -fn similar(a: Outcome, b: Outcome) -> bool { +fn similar(a: Solution, b: Solution) -> bool { match (a, b) { - (Outcome::Value(a), Outcome::Value(b)) => (a - b).abs() < 0.001 * ((a + b) / 2.0), + (Solution::Value(a), Solution::Value(b)) => (a - b).abs() < 0.001 * ((a + b) / 2.0), _ => false, } } @@ -46,15 +46,15 @@ macro_rules! test_model { svm.predict_value(&mut problem_0)?; svm.predict_value(&mut problem_7)?; - assert!(similar(problem_0.result(), Outcome::Value($libsvm[0]))); - assert!(similar(problem_7.result(), Outcome::Value($libsvm[1]))); + assert!(similar(problem_0.solution(), Solution::Value($libsvm[0]))); + assert!(similar(problem_7.solution(), Solution::Value($libsvm[1]))); if $prob { svm.predict_probability(&mut problem_0)?; svm.predict_probability(&mut problem_7)?; - assert!(similar(problem_0.result(), Outcome::Value($libsvm_prob[0]))); - assert!(similar(problem_7.result(), Outcome::Value($libsvm_prob[1]))); + assert!(similar(problem_0.solution(), Solution::Value($libsvm_prob[0]))); + assert!(similar(problem_7.solution(), Solution::Value($libsvm_prob[1]))); } Ok(()) @@ -65,7 +65,7 @@ macro_rules! test_model { #[cfg(test)] mod svm_dense_regression { use super::similar; - use ffsvm::{DenseSVM, Predict, Problem, Error, Outcome}; + use ffsvm::{DenseSVM, Error, Predict, Problem, Solution}; use std::convert::TryFrom; // E-SVR diff --git a/tests/svm_sparse_class.rs b/tests/svm_sparse_class.rs index db4c4c5..8686ba3 100644 --- a/tests/svm_sparse_class.rs +++ b/tests/svm_sparse_class.rs @@ -43,15 +43,15 @@ macro_rules! test_model { svm.predict_value(&mut problem_0)?; svm.predict_value(&mut problem_7)?; - assert_eq!(problem_0.result(), Outcome::Label($libsvm[0]), "predict_value(problem_0)"); - assert_eq!(problem_7.result(), Outcome::Label($libsvm[1]), "predict_value(problem_7)"); + assert_eq!(problem_0.solution(), Solution::Label($libsvm[0]), "predict_value(problem_0)"); + assert_eq!(problem_7.solution(), Solution::Label($libsvm[1]), "predict_value(problem_7)"); if $prob { svm.predict_probability(&mut problem_0)?; svm.predict_probability(&mut problem_7)?; - assert_eq!(problem_0.result(), Outcome::Label($libsvm_prob[0]), "predict_probability(problem_0)"); - assert_eq!(problem_7.result(), Outcome::Label($libsvm_prob[1]), "predict_probability(problem_7)"); + assert_eq!(problem_0.solution(), Solution::Label($libsvm_prob[0]), "predict_probability(problem_0)"); + assert_eq!(problem_7.solution(), Solution::Label($libsvm_prob[1]), "predict_probability(problem_7)"); } Ok(()) @@ -61,7 +61,7 @@ macro_rules! test_model { #[cfg(test)] mod svm_sparse_class { - use ffsvm::{Error, Outcome, Predict, Problem, SparseSVM}; + use ffsvm::{Error, Predict, Problem, Solution, SparseSVM}; use std::convert::TryFrom; // CSVM