multiclass heuristics for probabilistic outputs estimations

examples/undocumented/libshogun/Makefile

@@ -108,6 +108,7 @@ TARGETS = basic_minimal \
   		  classifier_mkl_svmlight_modelselection_bug \
   		  base_migration_new_buggy \
   		  regression_libsvr \
+  		  classifier_multiclass_prob_heuristics \
 
 
 all: $(TARGETS)

examples/undocumented/libshogun/classifier_multiclass_prob_heuristics.cpp

@@ -0,0 +1,93 @@
+#include <shogun/io/AsciiFile.h>
+#include <shogun/labels/MulticlassLabels.h>
+#include <shogun/io/SGIO.h>
+#include <shogun/features/DenseFeatures.h>
+#include <shogun/multiclass/MulticlassStrategy.h>
+#include <shogun/multiclass/MulticlassOneVsOneStrategy.h>
+#include <shogun/multiclass/MulticlassOneVsRestStrategy.h>
+#include <shogun/machine/LinearMulticlassMachine.h>
+#include <shogun/classifier/svm/LibLinear.h>
+#include <shogun/base/init.h>
+
+#define  EPSILON  1e-5
+
+using namespace shogun;
+
+/* file data */
+const char fname_feats[]="../data/fm_train_real.dat";
+const char fname_labels[]="../data/label_train_multiclass.dat";
+
+void test()
+{
+	/* dense features from matrix */
+	CAsciiFile* feature_file = new CAsciiFile(fname_feats);
+	SGMatrix<float64_t> mat=SGMatrix<float64_t>();
+	mat.load(feature_file);
+	SG_UNREF(feature_file);
+
+	CDenseFeatures<float64_t>* features=new CDenseFeatures<float64_t>(mat);
+	SG_REF(features);
+
+	/* labels from vector */
+	CAsciiFile* label_file = new CAsciiFile(fname_labels);
+	SGVector<float64_t> label_vec;
+	label_vec.load(label_file);
+	SG_UNREF(label_file);
+
+	CMulticlassLabels* labels=new CMulticlassLabels(label_vec);
+	SG_REF(labels);
+
+	// Create liblinear svm classifier with L2-regularized L2-loss
+	CLibLinear* svm = new CLibLinear(L2R_L2LOSS_SVC);
+	SG_REF(svm);
+
+	// Add some configuration to the svm
+	svm->set_epsilon(EPSILON);
+	svm->set_bias_enabled(true);
+
+	// Create a multiclass svm classifier that consists of several of the previous one
+	// There are several heuristics are implemented: 
+	// OVA_NORM, OVA_SOFTMAX
+	// OVO_PRICE, OVO_HASTIE, OVO_HAMAMURA
+	CLinearMulticlassMachine* mc_svm = new CLinearMulticlassMachine(
+			new CMulticlassOneVsOneStrategy(OVO_HASTIE), (CDotFeatures*) features, svm, labels);
+	SG_REF(mc_svm);
+
+	// Train the multiclass machine using the data passed in the constructor
+	mc_svm->train();
+
+	// Classify the training examples and show the results
+	CMulticlassLabels* output = CMulticlassLabels::obtain_from_generic(mc_svm->apply());
+
+	SGVector< int32_t > out_labels = output->get_int_labels();
+	SGVector<int32_t>::display_vector(out_labels.vector, out_labels.vlen);
+
+	for (int32_t i=0; i<output->get_num_labels(); i++)
+	{
+		SG_SPRINT("out_values[%d] = ", i);
+		SGVector<float64_t> out_values = output->get_multiclass_confidences(i);
+		SGVector<float64_t>::display_vector(out_values.vector, out_values.vlen);
+		SG_SPRINT("\n");
+	}
+
+	//Free resources
+	SG_UNREF(mc_svm);
+	SG_UNREF(svm);
+	SG_UNREF(output);
+	SG_UNREF(features);
+	SG_UNREF(labels);
+}
+
+int main(int argc, char** argv)
+{
+	init_shogun_with_defaults();
+
+	//sg_io->set_loglevel(MSG_DEBUG);
+
+	test();
+
+	exit_shogun();
+
+	return 0;
+}
+

src/shogun/lib/SGMatrixList.cpp

@@ -78,7 +78,7 @@ void SGMatrixList<T>::cleanup_matrices()
 		{
 			// Explicit call to the destructor required
 			// due to the use of in-place constructors
-			matrix_list[i].~SGMatrix();
+			matrix_list[i].~SGMatrix<T>();
 		}
 	}
 }

src/shogun/machine/MulticlassMachine.cpp

@@ -6,6 +6,7 @@
  *
  * Written (W) 1999-2011 Soeren Sonnenburg
  * Written (W) 2012 Fernando José Iglesias García and Sergey Lisitsyn
+ * Written (W) 2013 Shell Hu and Heiko Strathmann
  * Copyright (C) 2012 Sergey Lisitsyn, Fernando José Iglesias Garcia
  */
 
@@ -16,6 +17,7 @@
 #include <shogun/base/Parameter.h>
 #include <shogun/labels/MulticlassLabels.h>
 #include <shogun/labels/RegressionLabels.h>
+#include <shogun/mathematics/Statistics.h>
 
 using namespace shogun;
 
@@ -112,21 +114,67 @@ CMulticlassLabels* CMulticlassMachine::apply_multiclass(CFeatures* data)
 			SG_ERROR("num_machines = %d, did you train your machine?", num_machines)
 
 		CMulticlassLabels* result=new CMulticlassLabels(num_vectors);
-		result->allocate_confidences_for(num_machines);
+
+		// if outputs are prob, only one confidence for each class
+		int32_t num_classes=m_multiclass_strategy->get_num_classes();
+		EProbHeuristicType heuris = get_prob_heuris();
+
+		if (heuris!=PROB_HEURIS_NONE)
+			result->allocate_confidences_for(num_classes);
+		else
+			result->allocate_confidences_for(num_machines);
 
 		CBinaryLabels** outputs=SG_MALLOC(CBinaryLabels*, num_machines);
+		SGVector<float64_t> As(num_machines);
+		SGVector<float64_t> Bs(num_machines);
 
-		for (int32_t i=0; i < num_machines; ++i)
+		for (int32_t i=0; i<num_machines; ++i)
+		{
 			outputs[i] = (CBinaryLabels*) get_submachine_outputs(i);
 
+			if (heuris==OVA_SOFTMAX)
+			{
+				CStatistics::SigmoidParamters params = CStatistics::fit_sigmoid(outputs[i]->get_values());
+				As[i] = params.a;
+				Bs[i] = params.b;
+			}
+
+			if (heuris!=PROB_HEURIS_NONE && heuris!=OVA_SOFTMAX)
+				outputs[i]->scores_to_probabilities(0,0);
+		}
+
 		SGVector<float64_t> output_for_i(num_machines);
+		SGVector<float64_t> r_output_for_i(num_machines);
+		if (heuris!=PROB_HEURIS_NONE)
+			r_output_for_i.resize_vector(num_classes);
+
 		for (int32_t i=0; i<num_vectors; i++)
 		{
 			for (int32_t j=0; j<num_machines; j++)
 				output_for_i[j] = outputs[j]->get_value(i);
 
-			result->set_label(i, m_multiclass_strategy->decide_label(output_for_i));
-			result->set_multiclass_confidences(i, output_for_i.clone());
+			if (heuris==PROB_HEURIS_NONE)
+			{
+				r_output_for_i = output_for_i;
+			}
+			else
+			{
+				if (heuris==OVA_SOFTMAX)
+					m_multiclass_strategy->rescale_outputs(output_for_i,As,Bs);
+				else
+					m_multiclass_strategy->rescale_outputs(output_for_i);
+
+				// only first num_classes are returned
+				for (int32_t r=0; r<num_classes; r++)
+					r_output_for_i[r] = output_for_i[r];
+
+				SG_DEBUG("%s::apply_multiclass(): sum(r_output_for_i) = %f\n",
+					get_name(), SGVector<float64_t>::sum(r_output_for_i.vector,num_classes));
+			}
+
+			// use rescaled outputs for label decision
+			result->set_label(i, m_multiclass_strategy->decide_label(r_output_for_i));
+			result->set_multiclass_confidences(i, r_output_for_i);
 		}
 
 		for (int32_t i=0; i < num_machines; ++i)

src/shogun/machine/MulticlassMachine.h

@@ -6,6 +6,7 @@
  *
  * Written (W) 1999-2011 Soeren Sonnenburg
  * Written (W) 2012 Fernando José Iglesias García and Sergey Lisitsyn
+ * Written (W) 2013 Shell Hu and Heiko Strathmann
  * Copyright (C) 2012 Sergey Lisitsyn, Fernando José Iglesias Garcia
  */
 
@@ -138,6 +139,20 @@ class CMulticlassMachine : public CBaseMulticlassMachine
 			return "MulticlassMachine";
 		}
 
+		/** get prob output heuristic of multiclass strategy */
+		inline EProbHeuristicType get_prob_heuris()
+		{
+			return m_multiclass_strategy->get_prob_heuris_type();
+		}
+
+		/** set prob output heuristic of multiclass strategy 
+		 * @param prob_heuris type of probability heuristic
+		 */
+		inline void set_prob_heuris(EProbHeuristicType prob_heuris)
+		{
+			m_multiclass_strategy->set_prob_heuris_type(prob_heuris);
+		}
+
 	protected:
 		/** init strategy */
 		void init_strategy();