Skip to content

Commit

Permalink
Merge pull request #1813 from tklein23/libbmrm_memory_corruption
Browse files Browse the repository at this point in the history
Fixing integer overflows and memory corruption in libbmrm
  • Loading branch information
tklein23 committed Feb 4, 2014
2 parents 25c1e4f + d52d373 commit a5761ea
Show file tree
Hide file tree
Showing 4 changed files with 181 additions and 28 deletions.
2 changes: 1 addition & 1 deletion src/shogun/structure/DualLibQPBMSOSVM.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ bool CDualLibQPBMSOSVM::train_machine(CFeatures* data)
SG_ERROR("CDualLibQPBMSOSVM: m_solver=%d is not supported", m_solver);
}

if (m_result.exitflag==1)
if (m_result.exitflag>0)
return true;
else
return false;
Expand Down
78 changes: 52 additions & 26 deletions src/shogun/structure/libbmrm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
#include <shogun/lib/Time.h>
#include <shogun/io/SGIO.h>

#include <climits>
#include <limits>

namespace shogun
{
static const uint32_t QPSolverMaxIter=0xFFFFFFFF;
Expand All @@ -32,7 +35,8 @@ void add_cutting_plane(
float64_t* cp_data,
uint32_t dim)
{
ASSERT(map[free_idx])
REQUIRE(map[free_idx],
"add_cutting_plane: CP index %u is not free\n", free_idx)

LIBBMRM_MEMCPY(A+free_idx*dim, cp_data, dim*sizeof(float64_t));
map[free_idx]=false;
Expand Down Expand Up @@ -166,6 +170,7 @@ void clean_icp(ICP_stats* icp_stats,
icp_stats->H_buff[LIBBMRM_INDEX(i, j, icp_stats->maxCPs)];

bmrm.nCP=nCP_new;
ASSERT(bmrm.nCP<BufSize);
}
}

Expand Down Expand Up @@ -214,6 +219,7 @@ BmrmStatistics svm_bmrm_solver(
BufSize=_BufSize;
QPSolverTolRel=1e-9;

uint32_t histSize = BufSize;
H=NULL;
b=NULL;
beta=NULL;
Expand All @@ -223,6 +229,7 @@ BmrmStatistics svm_bmrm_solver(
I=NULL;
prevW=NULL;


H= (float64_t*) LIBBMRM_CALLOC(BufSize*BufSize, float64_t);

if (H==NULL)
Expand All @@ -231,7 +238,15 @@ BmrmStatistics svm_bmrm_solver(
goto cleanup;
}

A= (float64_t*) LIBBMRM_CALLOC(nDim*BufSize, float64_t);
ASSERT(nDim > 0);
ASSERT(BufSize > 0);
REQUIRE(BufSize < (std::numeric_limits<size_t>::max() / nDim),
"overflow: %u * %u > %u -- biggest possible BufSize=%u or nDim=%u\n",
BufSize, nDim, std::numeric_limits<size_t>::max(),
(std::numeric_limits<size_t>::max() / nDim),
(std::numeric_limits<size_t>::max() / BufSize));

A= (float64_t*) LIBBMRM_CALLOC(size_t(nDim)*size_t(BufSize), float64_t);

if (A==NULL)
{
Expand Down Expand Up @@ -337,9 +352,9 @@ BmrmStatistics svm_bmrm_solver(
goto cleanup;
}

bmrm.hist_Fp = SGVector< float64_t >(BufSize);
bmrm.hist_Fd = SGVector< float64_t >(BufSize);
bmrm.hist_wdist = SGVector< float64_t >(BufSize);
bmrm.hist_Fp = SGVector< float64_t >(histSize);
bmrm.hist_Fd = SGVector< float64_t >(histSize);
bmrm.hist_wdist = SGVector< float64_t >(histSize);

/* Iinitial solution */
R=machine->risk(subgrad, W);
Expand Down Expand Up @@ -372,7 +387,7 @@ BmrmStatistics svm_bmrm_solver(
/* Verbose output */

if (verbose)
SG_SDEBUG("%4d: tim=%.3lf, Fp=%lf, Fd=%lf, R=%lf\n",
SG_SPRINT("%4d: tim=%.3lf, Fp=%lf, Fd=%lf, R=%lf\n",
bmrm.nIter, tstop-tstart, bmrm.Fp, bmrm.Fd, R);

/* store Fp, Fd and wdist history */
Expand All @@ -384,7 +399,7 @@ BmrmStatistics svm_bmrm_solver(
helper = machine->get_helper();

/* main loop */

ASSERT(bmrm.nCP<BufSize);
while (bmrm.exitflag==0)
{
tstart=ttime.cur_time_diff(false);
Expand Down Expand Up @@ -417,8 +432,9 @@ BmrmStatistics svm_bmrm_solver(
diag_H[bmrm.nCP]=H[LIBBMRM_INDEX(bmrm.nCP, bmrm.nCP, BufSize)];
I[bmrm.nCP]=1;

bmrm.nCP++;
beta[bmrm.nCP]=0.0; // [beta; 0]
bmrm.nCP++;
ASSERT(bmrm.nCP<BufSize);

#if 0
/* TODO: scaling...*/
Expand Down Expand Up @@ -488,55 +504,64 @@ BmrmStatistics svm_bmrm_solver(
wdist=CMath::sqrt(sq_norm_Wdiff);

/* Stopping conditions */

if (bmrm.Fp - bmrm.Fd <= TolRel*LIBBMRM_ABS(bmrm.Fp))
bmrm.exitflag=1;

if (bmrm.Fp - bmrm.Fd <= TolAbs)
bmrm.exitflag=2;

if (bmrm.nCP >= BufSize)
bmrm.exitflag=-1;

tstop=ttime.cur_time_diff(false);

/* Verbose output */

if (verbose)
SG_SDEBUG("%4d: tim=%.3lf, Fp=%lf, Fd=%lf, (Fp-Fd)=%lf, (Fp-Fd)/Fp=%lf, R=%lf, nCP=%d, nzA=%d, QPexitflag=%d\n",
SG_SPRINT("%4d: tim=%.3lf, Fp=%lf, Fd=%lf, (Fp-Fd)=%lf, (Fp-Fd)/Fp=%lf, R=%lf, nCP=%d, nzA=%d, QPexitflag=%d\n",
bmrm.nIter, tstop-tstart, bmrm.Fp, bmrm.Fd, bmrm.Fp-bmrm.Fd,
(bmrm.Fp-bmrm.Fd)/bmrm.Fp, R, bmrm.nCP, bmrm.nzA, qp_exitflag.exitflag);

// iteration exceeds histSize
if (bmrm.nIter >= histSize)
{
histSize += BufSize;
bmrm.hist_Fp.resize_vector(histSize);
bmrm.hist_Fd.resize_vector(histSize);
bmrm.hist_wdist.resize_vector(histSize);
}

/* Keep Fp, Fd and w_dist history */
ASSERT(bmrm.nIter < histSize);
bmrm.hist_Fp[bmrm.nIter]=bmrm.Fp;
bmrm.hist_Fd[bmrm.nIter]=bmrm.Fd;
bmrm.hist_wdist[bmrm.nIter]=wdist;

/* Check size of Buffer */

if (bmrm.nCP>=BufSize)
{
bmrm.exitflag=-2;
SG_SERROR("Buffer exceeded.\n")
}

/* keep W (for wdist history track) */
LIBBMRM_MEMCPY(prevW, W, nDim*sizeof(float64_t));

/* Inactive Cutting Planes (ICP) removal */
if (cleanICP)
{
clean_icp(&icp_stats, bmrm, &CPList_head, &CPList_tail, H, diag_H, beta, map, cleanAfter, b, I);
ASSERT(bmrm.nCP<BufSize);
}

// next CP would exceed BufSize
if (bmrm.nCP+1 >= BufSize)
bmrm.exitflag=-1;

/* Debug: compute objective and training error */
if (verbose)
if (verbose && SG_UNLIKELY(sg_io->loglevel_above(MSG_DEBUG)))
{
float64_t debug_tstart=ttime.cur_time_diff(false);

SGVector<float64_t> w_debug(W, nDim, false);
float64_t primal = CSOSVMHelper::primal_objective(w_debug, model, _lambda);
float64_t train_error = CSOSVMHelper::average_loss(w_debug, model);
helper->add_debug_info(primal, bmrm.nIter, train_error);

float64_t debug_tstop=ttime.cur_time_diff(false);
SG_SPRINT("%4d: add_debug_info: tim=%.3lf, primal=%.3lf, train_error=%lf\n",
bmrm.nIter, debug_tstop-debug_tstart, primal, train_error);
}

} /* end of main loop */

if (verbose)
Expand All @@ -545,9 +570,10 @@ BmrmStatistics svm_bmrm_solver(
SG_UNREF(helper);
}

bmrm.hist_Fp.resize_vector(bmrm.nIter);
bmrm.hist_Fd.resize_vector(bmrm.nIter);
bmrm.hist_wdist.resize_vector(bmrm.nIter);
ASSERT(bmrm.nIter+1 <= histSize);
bmrm.hist_Fp.resize_vector(bmrm.nIter+1);
bmrm.hist_Fd.resize_vector(bmrm.nIter+1);
bmrm.hist_wdist.resize_vector(bmrm.nIter+1);

cp_ptr=CPList_head;

Expand Down
6 changes: 5 additions & 1 deletion src/shogun/structure/libbmrm.h
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,11 @@ inline float64_t * get_cutting_plane(bmrm_ll *ptr) { return ptr->address; }
* @return Index of unoccupied memory field in CP physical memory
*/
inline uint32_t find_free_idx(bool *map, uint32_t size)
{ for(uint32_t i=0; i<size; ++i) if (map[i]) return i; return size+1; }
{
for (uint32_t i=0; i<size; ++i) if (map[i]) return i;
SG_SERROR("No free index available in CP buffer of size %d.\n", size);
return size-1;
}

/** Standard BMRM Solver for Structured Output Learning
*
Expand Down
123 changes: 123 additions & 0 deletions tests/unit/structure/DualLibQPBMSOSVM_unittest.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* Written (W) 2014 Thoralf Klein
*/

#include <shogun/lib/config.h>
#include <gtest/gtest.h>

#include <shogun/lib/SGVector.h>
#include <shogun/lib/SGSparseVector.h>
#include <shogun/lib/SGSparseMatrix.h>

#include <shogun/features/SparseFeatures.h>
#include <shogun/structure/MulticlassSOLabels.h>
#include <shogun/labels/StructuredLabels.h>

#include <shogun/structure/MulticlassModel.h>
#include <shogun/structure/DualLibQPBMSOSVM.h>

using namespace shogun;

TEST(DualLibQPBMSOSVM,train_bmrm_small_buffer)
{
// toy data
int32_t N = 100;
int32_t feat_dim = 5;
int32_t num_feat = 4;

SGVector<float64_t> labs(N);
SGSparseMatrix<float64_t> feats(feat_dim, N);

for (int32_t i=0; i<N; i++)
{
feats.sparse_matrix[i] = SGSparseVector<float64_t>(num_feat);
int32_t f = 0;

ASSERT(f < num_feat);
feats.sparse_matrix[i].features[f].feat_index = 0;
feats.sparse_matrix[i].features[f].entry = i;
f++;

ASSERT(f < num_feat);
feats.sparse_matrix[i].features[f].feat_index = 1;
feats.sparse_matrix[i].features[f].entry = i%2 - 0.5;
f++;

ASSERT(f < num_feat);
feats.sparse_matrix[i].features[f].feat_index = 2;
feats.sparse_matrix[i].features[f].entry = i%2;
f++;

ASSERT(f < num_feat);
feats.sparse_matrix[i].features[f].feat_index = 3;
feats.sparse_matrix[i].features[f].entry = i%3;
f++;

labs[i] = float64_t(i/3);
}

// initialization
float64_t lambda=1e3, eps=0.01;
bool icp=1;
uint32_t cp_models=1;
ESolver solver=BMRM;

// Create train labels
CMulticlassSOLabels* labels = new CMulticlassSOLabels(labs);

// Create train features
CSparseFeatures< float64_t >* features = new CSparseFeatures< float64_t >(feats);

// Create structured model
CMulticlassModel* model = new CMulticlassModel(features, labels);

// Create SO-SVM, train
CDualLibQPBMSOSVM* sosvm = new CDualLibQPBMSOSVM(model, labels, lambda);
SG_REF(sosvm);

sosvm->set_cleanAfter(10);
sosvm->set_cleanICP(icp);
sosvm->set_TolRel(eps);
sosvm->set_cp_models(cp_models);
sosvm->set_solver(solver);

// sosvm->set_verbose(true);
sosvm->set_BufSize(2);

sosvm->train();

BmrmStatistics res = sosvm->get_result();
//SG_SPRINT("result = { Fp=%lf, Fd=%lf, nIter=%d, nCP=%d, nzA=%d, exitflag=%d }\n",
// res.Fp, res.Fd, res.nIter, res.nCP, res.nzA, res.exitflag);

ASSERT_LE(res.nCP, 2);
ASSERT_LE(res.nzA, 2);
ASSERT_LE(res.exitflag, 0);

CStructuredLabels* out = CLabelsFactory::to_structured(sosvm->apply());
SG_REF(out);

SG_SPRINT("\n");

// Compute error
//-------------------------------------------------------------------------
float64_t error=0.0;

for (int32_t i=0; i<num_feat; ++i)
{
CRealNumber* rn = CRealNumber::obtain_from_generic( out->get_label(i) );
error+=(rn->value==labs.get_element(i)) ? 0.0 : 1.0;
SG_UNREF(rn);
}

// SG_SPRINT("Error = %lf %% \n", error/num_feat*100);

// Free memory
SG_UNREF(sosvm);
SG_UNREF(out);
}

0 comments on commit a5761ea

Please sign in to comment.