Skip to content

Commit

Permalink
refactoring, moving towards a server classifier..
Browse files Browse the repository at this point in the history
  • Loading branch information
Roy Shilkrot committed Aug 23, 2011
1 parent c02154f commit 0fdf7fa
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 209 deletions.
6 changes: 6 additions & 0 deletions FoodcamClassifier.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
/* Begin PBXBuildFile section */
8DD76F650486A84900D96B5E /* main.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 08FB7796FE84155DC02AAC07 /* main.cpp */; settings = {ATTRIBUTES = (); }; };
8DD76F6A0486A84900D96B5E /* FoodcamClassifier.1 in CopyFiles */ = {isa = PBXBuildFile; fileRef = C6859E8B029090EE04C91782 /* FoodcamClassifier.1 */; };
D79318151404316D00C2209B /* predict_common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D79318141404316D00C2209B /* predict_common.cpp */; };
D7BC00A413FEE61A004D115F /* libopencv_ml.2.3.1.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D7BC00A313FEE61A004D115F /* libopencv_ml.2.3.1.dylib */; };
D7BC013713FF0035004D115F /* libopencv_core.2.3.1.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D7BCFF1213FE37FD004D115F /* libopencv_core.2.3.1.dylib */; };
D7BC013813FF0035004D115F /* libopencv_features2d.2.3.1.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D7BCFF1713FE3812004D115F /* libopencv_features2d.2.3.1.dylib */; };
Expand Down Expand Up @@ -135,6 +136,8 @@
08FB7796FE84155DC02AAC07 /* main.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = main.cpp; sourceTree = "<group>"; };
8DD76F6C0486A84900D96B5E /* FoodcamClassifier */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = FoodcamClassifier; sourceTree = BUILT_PRODUCTS_DIR; };
C6859E8B029090EE04C91782 /* FoodcamClassifier.1 */ = {isa = PBXFileReference; lastKnownFileType = text.man; path = FoodcamClassifier.1; sourceTree = "<group>"; };
D79318131404316D00C2209B /* predict_common.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = predict_common.h; sourceTree = "<group>"; };
D79318141404316D00C2209B /* predict_common.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = predict_common.cpp; sourceTree = "<group>"; };
D7BC00A313FEE61A004D115F /* libopencv_ml.2.3.1.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopencv_ml.2.3.1.dylib; path = usr/local/lib/libopencv_ml.2.3.1.dylib; sourceTree = SDKROOT; };
D7BC014013FF0035004D115F /* FoodcamClassifier */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = FoodcamClassifier; sourceTree = BUILT_PRODUCTS_DIR; };
D7BC01BD13FF3F9F004D115F /* train_bovw.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = train_bovw.h; sourceTree = "<group>"; };
Expand Down Expand Up @@ -280,6 +283,8 @@
D7BC04491401E5FF004D115F /* make_test_background_image.h */,
D7BC044A1401E5FF004D115F /* make_test_background_image.cpp */,
D7BC060D1402DB5C004D115F /* training_common.cpp */,
D79318131404316D00C2209B /* predict_common.h */,
D79318141404316D00C2209B /* predict_common.cpp */,
);
name = Source;
sourceTree = "<group>";
Expand Down Expand Up @@ -507,6 +512,7 @@
buildActionMask = 2147483647;
files = (
D7BC028914001E3D004D115F /* test_classifiers.cpp in Sources */,
D79318151404316D00C2209B /* predict_common.cpp in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
229 changes: 20 additions & 209 deletions test_classifiers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,85 +9,16 @@

#include "test_classifiers.h"

#include <stdio.h>
#include <stdlib.h>

#include <opencv2/opencv.hpp>
#include <fstream>
#include <iostream>
#include <string>
#include <set>

#include <dirent.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>

#include <omp.h>

using namespace cv;
using namespace std;

int main(int argc, char** argv) {
string dir, filepath;
DIR *dp;
struct dirent *dirp;
struct stat filestat;

cout << "load SVM classifiers" << endl;
dir = ".";
dp = opendir( dir.c_str() );

map<string,CvSVM> classes_classifiers;
int count = 0;
while ((dirp = readdir( dp )))
{
filepath = dir + "/" + dirp->d_name;

// If the file is a directory (or is in some way invalid) we'll skip it
if (stat( filepath.c_str(), &filestat )) continue;
if (S_ISDIR( filestat.st_mode )) continue;
if (filepath.find("SVM_classifier_with_color") != string::npos)
{
string class_ = filepath.substr(filepath.rfind('_')+1,filepath.rfind('.')-filepath.rfind('_')-1);
cout << "load " << filepath << ", class: " << class_ << endl;
classes_classifiers.insert(pair<string,CvSVM>(class_,CvSVM()));
classes_classifiers[class_].load(filepath.c_str());
}
}

cout << "read vocabulary form file"<<endl;
Mat vocabulary;
FileStorage fs("vocabulary_color_1000.yml", FileStorage::READ);
fs["vocabulary"] >> vocabulary;
fs.release();

Ptr<FeatureDetector > detector(new SurfFeatureDetector()); //detector
//Ptr<DescriptorExtractor > extractor(new SurfDescriptorExtractor());// extractor;
Ptr<DescriptorExtractor > extractor(
new OpponentColorDescriptorExtractor(
Ptr<DescriptorExtractor>(new SurfDescriptorExtractor())
)
);
Ptr<DescriptorMatcher > matcher(new BruteForceMatcher<L2<float> >());
BOWImgDescriptorExtractor bowide(extractor,matcher);
bowide.setVocabulary(vocabulary);
string filepath;

cout << "------- test ---------\n";

//evaluate
dir = "foodcamimages/TEST";
dp = opendir( dir.c_str() );
count = 0;
// Mat response_hist;


map<string,Scalar> classes_colors;
int ccount = 0;
for (map<string,CvSVM>::iterator it = classes_classifiers.begin(); it != classes_classifiers.end(); ++it) {
classes_colors[(*it).first] = Scalar((float)(ccount++)/(float)(classes_classifiers.size())*180.0f,255,255);
cout << "class " << (*it).first << " color " << classes_colors[(*it).first].val[0] << endl;
}
// dir = "foodcamimages/TEST";
// dp = opendir( dir.c_str() );
// count = 0;
//// Mat response_hist;

ifstream ifs("test.txt",ifstream::in);
char buf[255];
Expand All @@ -99,6 +30,9 @@ int main(int argc, char** argv) {
ifs.close();
cout << "total " << lines.size() << " samples to scan" <<endl;

FoodcamPredictor predictor;

map<string,CvSVM>& classes_classifiers = predictor.getClassesClassifiers();
map<string,map<string,int> > confusion_matrix;
for (map<string,CvSVM>::iterator it = classes_classifiers.begin(); it != classes_classifiers.end(); ++it) {
for (map<string,CvSVM>::iterator it1 = classes_classifiers.begin(); it1 != classes_classifiers.end(); ++it1) {
Expand All @@ -108,7 +42,6 @@ int main(int argc, char** argv) {
}
}

Mat background = imread("background.png");
// while ((dirp = readdir( dp )))
for (int i = 0; i < lines.size(); i++) {
// count++;
Expand Down Expand Up @@ -140,153 +73,32 @@ int main(int argc, char** argv) {

Mat __img = imread(filepath),_img;
if(__img.size() != Size(640,480)) continue;

Mat diff = (__img - background), diff_8UC1;

cvtColor(diff, diff_8UC1, CV_BGR2GRAY);
// imshow("img no back", diff_8UC1);
Mat fg_mask = (diff_8UC1 > 5);
GaussianBlur(fg_mask, fg_mask, Size(11,11), 5.0);
fg_mask = fg_mask > 50;

// {
// Mat _out; __img.copyTo(_out, fg_mask);
// imshow("foregroung", _out);
// imshow("to scan",__img);
// waitKey(0);
// }

Rect crop_rect(0,100,640,480-100);
__img = __img(crop_rect); //crop off top section
fg_mask = fg_mask(crop_rect);

//_img.create(__img.size(), __img.type());
// cvtColor(__img, _img, CV_BGR2GRAY);
// equalizeHist(__img, _img);
Mat copy; cvtColor(__img, copy, CV_BGR2HSV);

vector<Point> check_points;
//Sliding window approach.. (creating a vector here to ease the OMP parallel for-loop)
int winsize = 200;
map<string,pair<int,float> > found_classes;
for (int x=0; x<__img.cols; x+=winsize/4) {
for (int y=0; y<__img.rows; y+=winsize/4) {
if (fg_mask.at<uchar>(y,x) == 0) {
continue;
}
check_points.push_back(Point(x,y));
}
}

cout << "to check: " << check_points.size() << " points"<<endl;

Mat seg = Mat::zeros(copy.size(),CV_8UC3);

#pragma omp parallel for
for (int i = 0; i < check_points.size(); i++) {
int x = check_points[i].x;
int y = check_points[i].y;
// cout << omp_get_thread_num() << " scan " << check_points[i] << endl;
Mat img,response_hist;
__img(Rect(x-winsize/2,y-winsize/2,winsize,winsize)&Rect(0,0,__img.cols,__img.rows)).copyTo(img);

vector<KeyPoint> keypoints;
detector->detect(img,keypoints);
// vector<vector<int> > pointIdxsOfClusters;
bowide.compute(img, keypoints, response_hist); //, &pointIdxsOfClusters);
if (response_hist.cols == 0 || response_hist.rows == 0) {
continue;
}

// drawKeypoints(img, keypoints, img, Scalar(0,0,255));
// for (int i = 0; i < pointIdxsOfClusters.size(); i++) {
// if(pointIdxsOfClusters[i].size()>0) {
// Scalar clr(i/1000.0*255.0,0,0);
// for (int j = 0; j < pointIdxsOfClusters[i].size(); j++) {
// circle(img, keypoints[pointIdxsOfClusters[i][j]].pt, 1, clr, 2);
// }
// }
// }
// imshow("pic",img);

//test vs. SVMs
try {
float minf = FLT_MAX; string minclass;
for (map<string,CvSVM>::iterator it = classes_classifiers.begin(); it != classes_classifiers.end(); ++it) {
float res = (*it).second.predict(response_hist,true);
if ((*it).first == "misc" && res > 0.9) {
continue;
}
if(res > 1.0) continue;
if (res < minf) {
minf = res;
minclass = (*it).first;
}
}
// cout << "best class: " << minclass << " ("<<minf<<")"<<endl;
cout << "."; cout.flush();
//circle(copy, Point(x,y), 5, classes_colors[minclass], CV_FILLED);
float dim = 1.0f - MAX(MIN(minf - 0.8f,0.3f),0.0f) / 0.3f; //dimming the color: [0.8,1.1] -> [0.0,1.0]
Scalar color_(classes_colors[minclass].val[0], classes_colors[minclass].val[1], classes_colors[minclass].val[2] * dim);

#pragma omp critical
{
putText(copy, minclass.substr(0, 4), Point(x-35,y+10), CV_FONT_HERSHEY_PLAIN, 2.0, Scalar(0,0,255), 2);
circle(seg, check_points[i], winsize/5, color_, CV_FILLED);
found_classes[minclass].first++;
found_classes[minclass].second += minf;
}
}
catch (cv::Exception) {
continue;
}
}

cout << endl << "found classes: ";
float max_class_f = 0; string max_class;
for (map<string,pair<int,float> >::iterator it=found_classes.begin(); it != found_classes.end(); ++it) {
float score = (*it).second.first * (*it).second.second;
cout << (*it).first << "(" << score << "),"; //<< (*it).second.first << "," << (*it).second.second / (float)(*it).second.first << "), ";
if(score > max_class_f) {
max_class_f = score;
max_class = (*it).first;
}
}
cout << endl;
if(max_class.compare("cake")==0) max_class = "cookies";
if(max_class.compare("fruit")==0) max_class = "fruit_veggie";

cout << "chosen class: " << max_class << endl;
vector<string> max_class;
predictor.evaluateOneImage(__img,max_class);
cout << "manual class: "; for(int j_=0;j_<classes_.size();j_++) cout << classes_[j_] << ",";
cout << endl;

int j_=0;
for(;j_<classes_.size();j_++) {
if(classes_[j_].compare(max_class)==0) //got a hit
if(classes_[j_].compare(max_class[0])==0) //got a hit
{
confusion_matrix[max_class][classes_[j_]]++;
confusion_matrix[max_class[0]][classes_[j_]]++;
break;
}
}
if(j_==classes_.size()) //no hit was found, just use any class
confusion_matrix[max_class][classes_[0]]++;
confusion_matrix[max_class[0]][classes_[0]]++;

cvtColor(copy, copy, CV_HSV2BGR);
cvtColor(seg, seg, CV_HSV2BGR);
addWeighted(seg, 0.2, copy, 0.8, 1.0, seg);
//imshow("seg", seg);
// cvtColor(copy, copy, CV_HSV2BGR);
// cvtColor(seg, seg, CV_HSV2BGR);
// addWeighted(seg, 0.2, copy, 0.8, 1.0, seg);
// imshow("seg", seg);

Mat out; __img.copyTo(out);
stringstream ss; ss << max_class << "!";
putText(out, ss.str(), Point(out.cols/2-100,out.rows/2-30), CV_FONT_HERSHEY_PLAIN, 3.0, Scalar(255), 2);
//imshow("out",out);
//waitKey(0);
putText(out, max_class[0] + "!", Point(out.cols/2-100,out.rows/2-30), CV_FONT_HERSHEY_PLAIN, 3.0, Scalar(255), 2);
imshow("out",out);
waitKey(0);
imwrite("output/"+filepath, out);


//TODO: sliding window on the image to crop smaller secions and classify each section (segmentation? booya)

// cout << ".";
}

for(map<string,map<string,int> >::iterator it = confusion_matrix.begin(); it != confusion_matrix.end(); ++it) {
Expand All @@ -298,6 +110,5 @@ int main(int argc, char** argv) {
}

cout << endl;
closedir( dp );

}
1 change: 1 addition & 0 deletions test_classifiers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@
*
*/

#include "predict_common.h"

0 comments on commit 0fdf7fa

Please sign in to comment.