Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Branch: master
Fetching contributors…

Cannot retrieve contributors at this time

147 lines (121 sloc) 6.391 kB
This is Gary Bradski's 10/19/11 multi-modal perception/version of linemod. This is just a basic guide to use, it won't compile.
Below, I use the a color image to compute gradient linemod features and the same color image to compute the color features for filtering.
//MAIN FUNCTION (EXAMPLE WITH LOTS OF DEBUG IN IT)
My Main file is in MyMMod.cpp
//CLASSES ARE:
mmod_objects -- holds recognition results, it contains a map of modalities (color, depth, gradients ...) in
mmod_mode -- holds a features for a modality, it contains a map of features and offsets which are in
mmod_features -- holds a list of object views (vector<vector<uchar> >) of 8 bit binarized features and their offsets ... these are the model templates
mmod_general -- Almost all the learning and matching computation and utility functions are here
mmod_color -- Shouldn't be named "color", should be named mmod_calc_feature -- these classes, one for each feature take a modality as input
(depth image, color image) and creates a feature image of 8 bit values. These take a mask (training) or not (test), see below.
//////////////////A WALK THROUGH OF HOW TO CALL THESE FUNCTIONS//////////////////
//INCLUDES
#include <opencv2/opencv.hpp>
#include <iostream>
#include <stdio.h>
#include <valgrind/callgrind.h>
#include "mmod_general.h"
#include "mmod_objects.h"
#include "mmod_mode.h"
#include "mmod_features.h"
#include "mmod_color.h"
//For serialization
#include <fstream>
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
using namespace cv;
using namespace std;
//INSTANTIATE STUFF:
mmod_objects Objs; //Object train and test.
mmod_general g; //General utilities
mmod_filters filt("Color"); //For post recognition filter object check, I'm using color, you can use any (one) mode
colorhls calcHLS; //Color feature processing
gradients calcGrad; //Gradient feature processing
// depthgrad calcDepth; //Depth feature processing
Mat colorfeat, depthfeat, gradfeat; //To hold feature outputs. These will be CV_8UC1 images
Mat ColorRaw0,Mask0,ColorRaw,Mask,noMask; //Will hold raw images and masks
Mat Ivis,Gvis; //feature images, CV_8UC1
vector<Mat> ColorPyr,MaskPyr; //Image pyramid
vector<Mat> FeatModes; //List of images
vector<string> modesCD; //Names of modes (color and depth)
string SessionID, ObjectName; //Fill these in.
int framenum = 0;
float Score;
//SET UP:
//Set up our modes (right now we have color gradients and depth. Below, I just use gradients
#define PYRLEVELS 2 //How many levels of pyramid reduction
#define SKIPAMT 8 //SKIP Amount (skipX, skipY)
// modesCD.push_back("Color");
modesCD.push_back("Grad");
// modesCD.push_back("Depth");
float learn_thresh = 0.97; //Just a guess
float match_threshold = 0.97; //Total guess
float frac_overlap = 0.5; //the fraction of overlap between 2 above threshold feature's bounding box rectangles that constitutes "overlap"
float cthresh = 0.91; //Color thresh
. . .
//GET OUR IMAGES IN, AND REDUCE IF YOU WANT A SPEED UP:
... Mat::ColorRaw0 ... //BGR image of type CV_8UC3
... Mat::DepthRaw0 ... //Depth image of type CV_16UC1
... Mat::Mask0 ... //Object mask of type CV_8UC1 or CV_8UC3
buildPyramid(ColorRaw0, ColorPyr, PYRLEVELS); //This is optional, you can just use the raw images. I had 1Kx1K images, so did this for speed.
buildPyramid(Mask0,MaskPyr,PYRLEVELS);
ColorRaw = ColorPyr[PYRLEVELS];
Mask = MaskPyr[PYRLEVELS];
//PROCESS TO GET FEATURES
calcHLS.computeColorHLS(ColorRaw,colorfeat,Mask,"train");
calcGrad.computeGradients(ColorRaw,gradfeat,Mask,"train");
// calcDepth.computeDepthGradients(DepthRaw,depthfeat,Mask,"train");
FeatModes.clear(); //Stack the features ... here I only use gradients
FeatModes.push_back(gradfeat);
// FeatModes.push_back(colorfeat);
// FeatModes.push_back(depthfeat);
//LEARN A TEMPLATE (for now, it will slow down with each view learned).
int num_templ = Objs.learn_a_template(FeatModes,modesCD, Mask,
SessionID, ObjectName, framenum, learn_thresh, &Score);
//LEARN A FILTER TO CONFIRM RECOGNIZED OBJECTS IN TEST MODE
int num_fs = filt.learn_a_template(colorfeat,Mask,"Tea",framenum);
. . .
//EXAMPLE OF I/O -- I USE A FILTER IN THIS EXAMPLE, YOU CAN USE OBJECTS IN A SIMILAR WAY
{ //Serialize out
cout << "Writing models filt.txt out" << endl;
std::ofstream ofs("filt.txt");
boost::archive::text_oarchive oa(ofs);
oa << filt;
}
// ----
mmod_filters filt2("foo");
{ //Serialize in
cout << "Reading models filt.txt in" << endl;
std::ifstream ifs("filt.txt");
boost::archive::text_iarchive ia(ifs);
// read class state from archive
ia >> filt2;
}
. . .
//TEST MODE
int skipX = SKIPAMT, skipY = SKIPAMT; //These control sparse testing of the feature images, here SKIPAMT 8
. . . ColorRaw0 is read in . . .
buildPyramid(ColorRaw0, ColorPyr, PYRLEVELS); //PYRAMID DOWN IF YOU WANT (FOR SPEED)
ColorRaw = ColorPyr[PYRLEVELS]; //OR, could just use the full scale image
//Calculate features
calcHLS.computeColorHLS(ColorRaw,colorfeat,noMask,"test");
calcGrad.computeGradients(ColorRaw,gradfeat,noMask,"test");
// calcDepth.computeDepthGradients(DepthRaw,depthfeat,noMask,"test");
FeatModes.clear();
FeatModes.push_back(gradfeat); //I'm just using gradients here, you can push back as many modalities as you like
// FeatModes.push_back(colorfeat); //Color will instead be used for filtering
// FeatModes.push_back(depthfeat);
//RECOGNIZE
int num_matches = Objs.match_all_objects(FeatModes,modesCD,noMask,
match_threshold,frac_overlap,skipX,skipY,&numrawmatches);
. . . Optionally, check the recognitions with a filter (here my trained color filter)
filt.filter_object_recognitions(colorfeat,Objs,cthresh);
//USE THE RESULTS
Objs.rv contains a vector of rectanglular bounding boxes of recognized objects
Objs.ids contains the corrsponsing list of their names
Objs.scores contains their matching scores
Objs.frame_nums contains the frame number of the model, so that we can relate it to the view learned.
//UTILITIES
g.visualize_binary_image(gradfeat,Gvis); //DISPLAY COLOR CODED BINARY IMAGE
Objs.draw_matches(ColorRaw); //DISPLAYS RECOGNITION RESULTS ONTO THE RAW IMAGE
Jump to Line
Something went wrong with that request. Please try again.