Skip to content

Commit

Permalink
Merge pull request #602 from Atollogy/memory-fix
Browse files Browse the repository at this point in the history 
Track external memory usage for matrices
  • Loading branch information
@peterbraden
peterbraden committed Apr 4, 2018
2 parents 48a4fc3 + e1c0d8e commit 41730f8
Show file tree
Hide file tree
Showing 14 changed files with 1,419 additions and 319 deletions.
35 changes: 14 additions & 21 deletions src/BackgroundSubtractor.cc
View file
Expand Up @@ -255,10 +255,6 @@ NAN_METHOD(BackgroundSubtractorWrap::ApplyMOG) {


try {
Local<Object> fgMask =
Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *img = Nan::ObjectWrap::Unwrap<Matrix>(fgMask);

cv::Mat mat;
if (Buffer::HasInstance(info[0])) {
uint8_t *buf = (uint8_t *) Buffer::Data(info[0]->ToObject());
Expand Down Expand Up @@ -287,7 +283,7 @@ NAN_METHOD(BackgroundSubtractorWrap::ApplyMOG) {
#endif
}

img->mat = _fgMask;
Local<Object> fgMask = Matrix::CreateWrappedFromMat(_fgMask.clone());
mat.release();

argv[0] = Nan::Null();
Expand All @@ -313,15 +309,15 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker {
AsyncBackgroundSubtractorWorker(
Nan::Callback *callback,
BackgroundSubtractorWrap *bg,
cv::Mat &img_mat):
Matrix *matrix_in):
Nan::AsyncWorker(callback),
bg(bg),
img_mat(img_mat) { // note: this makes a new cv::Mat, and so increments the ref count for the data without copying it
matrix(new Matrix(matrix_in)) {

}

~AsyncBackgroundSubtractorWorker() {
// upon destroy, img_mat will reduce refcount on data by one

}

// Executed inside the worker-thread.
Expand All @@ -332,9 +328,9 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker {
// wait here if already in apply - auto-release on scope exit
BGAutoMutex(bg->applymutex);
#if CV_MAJOR_VERSION >= 3
bg->subtractor->apply(this->img_mat, _fgMask);
bg->subtractor->apply(matrix->mat, _fgMask);
#else
bg->subtractor->operator()(this->img_mat, _fgMask);
bg->subtractor->operator()(matrix->mat, _fgMask);
#endif
}

Expand All @@ -344,9 +340,10 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker {
void HandleOKCallback() {
Nan::HandleScope scope;

Local<Object> im_to_return= Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *imgout = Nan::ObjectWrap::Unwrap<Matrix>(im_to_return);
imgout->mat = _fgMask;
delete matrix;
matrix = NULL;

Local<Object> im_to_return = Matrix::CreateWrappedFromMat(_fgMask.clone());

Local<Value> argv[] = {
Nan::Null()
Expand All @@ -362,7 +359,7 @@ class AsyncBackgroundSubtractorWorker: public Nan::AsyncWorker {

private:
BackgroundSubtractorWrap *bg;
cv::Mat img_mat;
Matrix *matrix;
cv::Mat _fgMask;
};

Expand All @@ -372,7 +369,6 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) {
SETUP_FUNCTION(BackgroundSubtractorWrap);
int callback_arg = -1;
int numargs = info.Length();
int success = 1;

Local<Function> cb;

Expand Down Expand Up @@ -402,15 +398,12 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) {

Nan::Callback *callback = new Nan::Callback(cb.As<Function>());
Matrix *_img = Nan::ObjectWrap::Unwrap<Matrix>(info[0]->ToObject());
Nan::AsyncQueueWorker(new AsyncBackgroundSubtractorWorker( callback, self, _img->mat));
Nan::AsyncQueueWorker(new AsyncBackgroundSubtractorWorker( callback, self, _img));
return;
} else { //synchronous - return the image

try {
Local<Object> fgMask =
Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *img = Nan::ObjectWrap::Unwrap<Matrix>(fgMask);

Local<Object> fgMask;
cv::Mat mat;
if (Buffer::HasInstance(info[0])) {
uint8_t *buf = (uint8_t *) Buffer::Data(info[0]->ToObject());
Expand All @@ -436,7 +429,7 @@ NAN_METHOD(BackgroundSubtractorWrap::Apply) {
#else
self->subtractor->operator()(mat, _fgMask);
#endif
img->mat = _fgMask;
fgMask = Matrix::CreateWrappedFromMat(_fgMask.clone());
}

mat.release();
Expand Down
47 changes: 19 additions & 28 deletions src/Calib3D.cc
View file
Expand Up @@ -3,15 +3,6 @@

#ifdef HAVE_OPENCV_CALIB3D

inline Local<Object> matrixFromMat(cv::Mat &input) {
Local<Object> matrixWrap =
Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *matrix = Nan::ObjectWrap::Unwrap<Matrix>(matrixWrap);
matrix->mat = input;

return matrixWrap;
}

inline cv::Mat matFromMatrix(Local<Value> matrix) {
Matrix* m = Nan::ObjectWrap::Unwrap<Matrix>(matrix->ToObject());
return m->mat;
Expand Down Expand Up @@ -237,11 +228,11 @@ NAN_METHOD(Calib3D::CalibrateCamera) {
ret->Set(Nan::New<String>("reprojectionError").ToLocalChecked(), Nan::New<Number>(error));

// K
Local<Object> KMatrixWrap = matrixFromMat(K);
Local<Object> KMatrixWrap = Matrix::CreateWrappedFromMat(K);
ret->Set(Nan::New<String>("K").ToLocalChecked(), KMatrixWrap);

// dist
Local<Object> distMatrixWrap = matrixFromMat(dist);
Local<Object> distMatrixWrap = Matrix::CreateWrappedFromMat(dist);
ret->Set(Nan::New<String>("distortion").ToLocalChecked(), distMatrixWrap);

// Per frame R and t, skiping for now
Expand Down Expand Up @@ -287,11 +278,11 @@ NAN_METHOD(Calib3D::SolvePnP) {
Local<Object> ret = Nan::New<Object>();

// rvec
Local<Object> rMatrixWrap = matrixFromMat(rvec);
Local<Object> rMatrixWrap = Matrix::CreateWrappedFromMat(rvec);
ret->Set(Nan::New<String>("rvec").ToLocalChecked(), rMatrixWrap);

// tvec
Local<Object> tMatrixWrap = matrixFromMat(tvec);
Local<Object> tMatrixWrap = Matrix::CreateWrappedFromMat(tvec);
ret->Set(Nan::New<String>("tvec").ToLocalChecked(), tMatrixWrap);

// Return
Expand Down Expand Up @@ -334,7 +325,7 @@ NAN_METHOD(Calib3D::GetOptimalNewCameraMatrix) {
newImageSize);

// Wrap the output K
Local<Object> KMatrixWrap = matrixFromMat(Kout);
Local<Object> KMatrixWrap = Matrix::CreateWrappedFromMat(Kout);

// Return the new K matrix
info.GetReturnValue().Set(KMatrixWrap);
Expand Down Expand Up @@ -394,28 +385,28 @@ NAN_METHOD(Calib3D::StereoCalibrate) {
// Make the output arguments

// k1
Local<Object> K1MatrixWrap = matrixFromMat(k1);
Local<Object> K1MatrixWrap = Matrix::CreateWrappedFromMat(k1);

// d1
Local<Object> d1MatrixWrap = matrixFromMat(d1);
Local<Object> d1MatrixWrap = Matrix::CreateWrappedFromMat(d1);

// k2
Local<Object> K2MatrixWrap = matrixFromMat(k2);
Local<Object> K2MatrixWrap = Matrix::CreateWrappedFromMat(k2);

// d2
Local<Object> d2MatrixWrap = matrixFromMat(d2);
Local<Object> d2MatrixWrap = Matrix::CreateWrappedFromMat(d2);

// R
Local<Object> RMatrixWrap = matrixFromMat(R);
Local<Object> RMatrixWrap = Matrix::CreateWrappedFromMat(R);

// t
Local<Object> tMatrixWrap = matrixFromMat(t);
Local<Object> tMatrixWrap = Matrix::CreateWrappedFromMat(t);

// E
Local<Object> EMatrixWrap = matrixFromMat(E);
Local<Object> EMatrixWrap = Matrix::CreateWrappedFromMat(E);

// F
Local<Object> FMatrixWrap = matrixFromMat(F);
Local<Object> FMatrixWrap = Matrix::CreateWrappedFromMat(F);

// Add to return object
ret->Set(Nan::New<String>("K1").ToLocalChecked(), K1MatrixWrap);
Expand Down Expand Up @@ -479,11 +470,11 @@ NAN_METHOD(Calib3D::StereoRectify) {
// Make the return object
Local<Object> ret = Nan::New<Object>();

ret->Set(Nan::New<String>("R1").ToLocalChecked(), matrixFromMat(R1));
ret->Set(Nan::New<String>("R2").ToLocalChecked(), matrixFromMat(R2));
ret->Set(Nan::New<String>("P1").ToLocalChecked(), matrixFromMat(P1));
ret->Set(Nan::New<String>("P2").ToLocalChecked(), matrixFromMat(P2));
ret->Set(Nan::New<String>("Q").ToLocalChecked(), matrixFromMat(Q));
ret->Set(Nan::New<String>("R1").ToLocalChecked(), Matrix::CreateWrappedFromMat(R1));
ret->Set(Nan::New<String>("R2").ToLocalChecked(), Matrix::CreateWrappedFromMat(R2));
ret->Set(Nan::New<String>("P1").ToLocalChecked(), Matrix::CreateWrappedFromMat(P1));
ret->Set(Nan::New<String>("P2").ToLocalChecked(), Matrix::CreateWrappedFromMat(P2));
ret->Set(Nan::New<String>("Q").ToLocalChecked(), Matrix::CreateWrappedFromMat(Q));

// Return the rectification parameters
info.GetReturnValue().Set(ret);
Expand Down Expand Up @@ -557,7 +548,7 @@ NAN_METHOD(Calib3D::ReprojectImageTo3D) {
cv::reprojectImageTo3D(disparity, depthImage, Q);

// Wrap the depth image
Local<Object> depthImageMatrix = matrixFromMat(depthImage);
Local<Object> depthImageMatrix = Matrix::CreateWrappedFromMat(depthImage);

info.GetReturnValue().Set(depthImageMatrix);
} catch (cv::Exception &e) {
Expand Down
6 changes: 4 additions & 2 deletions src/CascadeClassifierWrap.cc
View file
Expand Up @@ -50,7 +50,7 @@ class AsyncDetectMultiScale: public Nan::AsyncWorker {
Matrix* im, double scale, int neighbors, int minw, int minh) :
Nan::AsyncWorker(callback),
cc(cc),
im(im),
im(new Matrix(im)), //copy the matrix so we aren't affected if the original is released
scale(scale),
neighbors(neighbors),
minw(minw),
Expand Down Expand Up @@ -82,7 +82,9 @@ class AsyncDetectMultiScale: public Nan::AsyncWorker {

void HandleOKCallback() {
Nan::HandleScope scope;
// this->matrix->Unref();

delete im;
im = NULL;

Local < Value > argv[2];
v8::Local < v8::Array > arr = Nan::New < v8::Array > (this->res.size());
Expand Down
49 changes: 36 additions & 13 deletions src/FaceRecognizer.cc
View file
Expand Up @@ -27,6 +27,8 @@ namespace cv {
#define FISHER 2

// Todo, move somewhere useful
// Note: References to the returned object here should not be retained past the end of the calling function.
// Otherwise, node might not keep track of external memory usage correctly.
cv::Mat fromMatrixOrFilename(Local<Value> v) {
cv::Mat im;
if (v->IsString()) {
Expand All @@ -40,6 +42,20 @@ cv::Mat fromMatrixOrFilename(Local<Value> v) {
return im;
}

// Note: Use this function when you might need to retain the returned object past the end of the calling function,
// such as in asynchronous methods
Matrix *CreateFromMatrixOrFilename(Local<Value> v) {
if (v->IsString()) {
Matrix *im = new Matrix();
std::string filename = std::string(*Nan::Utf8String(v->ToString()));
im->setMat(cv::imread(filename));
return im;
// std::cout<< im.size();
} else {
return new Matrix(Nan::ObjectWrap::Unwrap<Matrix>(v->ToObject()));
}
}

Nan::Persistent<FunctionTemplate> FaceRecognizerWrap::constructor;

void FaceRecognizerWrap::Init(Local<Object> target) {
Expand Down Expand Up @@ -188,7 +204,7 @@ Local<Value> UnwrapTrainingData(Nan::NAN_METHOD_ARGS_TYPE info,
}

int label = valarr->Get(0)->Uint32Value();
cv::Mat im = fromMatrixOrFilename(valarr->Get(1));
cv::Mat im = fromMatrixOrFilename(valarr->Get(1)); //this is ok because we clone the image
im = im.clone();
if (im.channels() == 3) {
cv::cvtColor(im, im, CV_RGB2GRAY);
Expand Down Expand Up @@ -295,7 +311,9 @@ NAN_METHOD(FaceRecognizerWrap::PredictSync) {

cv::Mat im = fromMatrixOrFilename(info[0]); // TODO CHECK!
if (im.channels() == 3) {
cv::cvtColor(im, im, CV_RGB2GRAY);
cv::Mat previous = im;
im = cv::Mat();
cv::cvtColor(previous, im, CV_RGB2GRAY);
}

int predictedLabel = -1;
Expand All @@ -322,10 +340,10 @@ NAN_METHOD(FaceRecognizerWrap::PredictSync) {

class PredictASyncWorker: public Nan::AsyncWorker {
public:
PredictASyncWorker(Nan::Callback *callback, cv::Ptr<cv::FaceRecognizer> rec, cv::Mat im) :
PredictASyncWorker(Nan::Callback *callback, cv::Ptr<cv::FaceRecognizer> rec, Matrix *matrix_in) :
Nan::AsyncWorker(callback),
rec(rec),
im(im) {
matrix(new Matrix(matrix_in)) {
predictedLabel = -1;
confidence = 0.0;
}
Expand All @@ -334,7 +352,7 @@ class PredictASyncWorker: public Nan::AsyncWorker {
}

void Execute() {
this->rec->predict(this->im, this->predictedLabel, this->confidence);
rec->predict(matrix->mat, predictedLabel, confidence);
#if CV_MAJOR_VERSION >= 3
// Older versions of OpenCV3 incorrectly returned label=0 at
// confidence=DBL_MAX instead of label=-1 on failure. This can be removed
Expand All @@ -350,6 +368,9 @@ class PredictASyncWorker: public Nan::AsyncWorker {
void HandleOKCallback() {
Nan::HandleScope scope;

delete matrix;
matrix = NULL;

v8::Local<v8::Object> res = Nan::New<Object>();
res->Set(Nan::New("id").ToLocalChecked(), Nan::New<Number>(predictedLabel));
res->Set(Nan::New("confidence").ToLocalChecked(), Nan::New<Number>(confidence));
Expand All @@ -367,7 +388,7 @@ class PredictASyncWorker: public Nan::AsyncWorker {

private:
cv::Ptr<cv::FaceRecognizer> rec;
cv::Mat im;
Matrix *matrix;
int predictedLabel;
double confidence;
};
Expand All @@ -381,13 +402,17 @@ NAN_METHOD(FaceRecognizerWrap::Predict) {

REQ_FUN_ARG(1, cb);

cv::Mat im = fromMatrixOrFilename(info[0]);
if (im.channels() == 3) {
cv::cvtColor(im, im, CV_RGB2GRAY);
Matrix *m = CreateFromMatrixOrFilename(info[0]);
if (m->mat.channels() == 3) {
cv::Mat grayMat;
cv::cvtColor(m->mat, grayMat, CV_RGB2GRAY);
m->setMat(grayMat);
}

Nan::Callback *callback = new Nan::Callback(cb.As<Function>());
Nan::AsyncQueueWorker(new PredictASyncWorker(callback, self->rec, im));
Nan::AsyncQueueWorker(new PredictASyncWorker(callback, self->rec, m));

delete m;

return;
}
Expand Down Expand Up @@ -448,9 +473,7 @@ NAN_METHOD(FaceRecognizerWrap::GetMat) {
m = self->rec->getMat(key);
#endif

Local<Object> im = Nan::NewInstance(Nan::GetFunction(Nan::New(Matrix::constructor)).ToLocalChecked()).ToLocalChecked();
Matrix *img = Nan::ObjectWrap::Unwrap<Matrix>(im);
img->mat = m;
Local<Object> im = Matrix::CreateWrappedFromMat(m);

info.GetReturnValue().Set(im);
}
Expand Down

0 comments on commit 41730f8

Please sign in to comment.