diff --git a/cells/CMakeLists.txt b/cells/CMakeLists.txt new file mode 100644 index 0000000..2358c6f --- /dev/null +++ b/cells/CMakeLists.txt @@ -0,0 +1,26 @@ +find_package(ecto) +if(NOT ecto_FOUND) + return() +endif() + +ecto_python_env_gen(${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) + +include_directories(../../object_recognition/include) +include_directories(../src) + +ectomodule(mmod + module.cpp + MModModelInserter.cpp + MModModelLoader.cpp + MModTester.cpp + MModTrainer.cpp + MModPersister.cpp + Pyramid.cpp +) + +link_ecto(mmod + mmod + object_recognition_db + ${OpenCV_LIBS} +) + diff --git a/cells/MModModelInserter.cpp b/cells/MModModelInserter.cpp new file mode 100644 index 0000000..da9b1b6 --- /dev/null +++ b/cells/MModModelInserter.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "object_recognition/db/db.h" +#include "object_recognition/db/opencv.h" + +#include "mmod_features.h" +#include "mmod_objects.h" + +using object_recognition::db_future::CollectionName; +using object_recognition::db_future::DocumentId; + +namespace mmod +{ + /** Class inserting the TOD models in the db + */ + struct ModelInserter + { + static void + declare_params(ecto::tendrils& params) + { + params.declare ("collection_models", + "std::string The collection in which to store the models on the db", "models").required( + true); + params.declare ("db_params", "The DB parameters").required(true); + params.declare ("object_id", "The object id, to associate this frame with.").required(true); + params.declare ("model_json_params", "The parameters used for the model, as JSON.").required(true); + } + + static void + declare_io(const ecto::tendrils& params, ecto::tendrils& inputs, ecto::tendrils& outputs) + { + inputs.declare ("objects", "The objects."); + inputs.declare ("filters", "The filters."); + } + + void + configure(const ecto::tendrils& params, const ecto::tendrils& inputs, const ecto::tendrils& outputs) + { + object_id_ = params["object_id"]; + db_.set_params(params.get ("db_params")); + collection_models_ = params.get ("collection_models"); + params_ = params.get ("model_json_params"); + + objects_ = inputs["objects"]; + filters_ = inputs["filters"]; + } + + int + process(const ecto::tendrils& inputs, const ecto::tendrils& outputs) + { + object_recognition::db_future::Document doc(db_, collection_models_); + + { + std::stringstream objects_stream; + boost::archive::binary_oarchive objects_archive(objects_stream); + objects_archive << *objects_; + doc.set_attachment_stream("objects", objects_stream); + } + { + std::stringstream filters_stream; + boost::archive::binary_oarchive filters_archive(filters_stream); + filters_archive << *filters_; + doc.set_attachment_stream("filters", filters_stream); + } + doc.set_value("object_id", *object_id_); + + // Convert the parameters to a property tree and insert them + boost::property_tree::ptree params; + std::stringstream ssparams; + ssparams << params_; + boost::property_tree::read_json(ssparams, params); + if (params.find("type") != params.not_found()) + params.erase("type"); + doc.set_values("parameters", params); + + doc.set_value("Type", "Model"); + doc.set_value("ModelType", "LINEMOD"); + std::cout << "Persisting" << std::endl; + doc.Persist(); + + return ecto::OK; + } + private: + object_recognition::db_future::ObjectDb db_; + ecto::spore object_id_; + CollectionName collection_models_; + /** The JSON parameters used to compute the model */ + std::string params_; + ecto::spore objects_; + ecto::spore filters_; + }; +} + +ECTO_CELL(mmod, mmod::ModelInserter, "ModelInserter","???") +; diff --git a/cells/MModModelLoader.cpp b/cells/MModModelLoader.cpp new file mode 100644 index 0000000..4ccfd1c --- /dev/null +++ b/cells/MModModelLoader.cpp @@ -0,0 +1,179 @@ +/* + * Software License Agreement (BSD License) + * + * Copyright (c) 2009, Willow Garage, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Willow Garage, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +//#include + +#include "object_recognition/common/types.h" +#include "object_recognition/db/db.h" +#include "mmod_objects.h" //For train and test (includes mmod_mode.h, mmod_features.h, mmod_general.h +#include "mmod_color.h" //For depth and color processing (yes, I should change the name) +namespace mmod +{ + struct TemplateLoader + { + static void + declare_params(ecto::tendrils& p) + { + p.declare ("collection_models", "The collection where the models are stored.").required(true); + p.declare ("db_params", "The DB parameters").required(true); + p.declare ("model_ids", "The list of model ids we should consider.\n").required(); + p.declare ("object_ids", "The list of model ids we should consider.\n").required(true); + p.declare ("feature_descriptor_params", "JSON string describing the template parameters").required( + true); + p.declare(&TemplateLoader::do_update_, "do_update", "Update the matcher from the database, expensive.", false); + } + + static void + declare_io(const ecto::tendrils& params, ecto::tendrils& inputs, ecto::tendrils& outputs) + { + outputs.declare > ("templates", "The templates"); + outputs.declare > ("filters", "The filters"); + outputs.declare > ("ids", "The matching object ids"); + outputs.declare ("do_update", "If true, that means new templates have been loaded"); + } + + void + configure(const ecto::tendrils& params, const ecto::tendrils& inputs, const ecto::tendrils& outputs) + { + // Load the list of Models to study + { + const boost::python::object & python_model_ids = params.get ("model_ids"); + boost::python::stl_input_iterator begin(python_model_ids), end; + std::copy(begin, end, std::back_inserter(model_ids_)); + } + + // Load the list of Object to study + { + const boost::python::object & python_object_ids = params.get ("object_ids"); + boost::python::stl_input_iterator begin(python_object_ids), end; + std::copy(begin, end, std::back_inserter(object_ids_)); + } + + if ((model_ids_.size() != object_ids_.size()) || (model_ids_.empty())) + { + std::stringstream ss; + ss << object_ids_.size() << " object ids given and " << model_ids_.size() << " model ids given." << std::endl; + throw std::runtime_error(ss.str()); + } + + // load the descriptors from the DB + db_params_ = params["db_params"]; + collection_models_ = params["collection_models"]; + + templates_ = outputs["templates"]; + filters_ = outputs["filters"]; + do_update_out_ = outputs["do_update"]; + ids_ = outputs["ids"]; + do_update_.set_callback(boost::bind(&TemplateLoader::on_do_update, this, _1)); + *do_update_ = true; + do_update_.dirty(true); + do_update_.notify(); + } + void + on_do_update(bool on_do_update) + { + if (!on_do_update) + return; + + std::cout << "Loading models. This may take some time..." << std::endl; + + *do_update_out_ = true; + object_recognition::db_future::ObjectDb db(*db_params_); + std::vector::const_iterator model_id = model_ids_.begin(), model_id_end = model_ids_.end(); + std::vector::const_iterator object_id = object_ids_.begin(); + templates_->reserve(model_ids_.size()); + filters_->reserve(model_ids_.size()); + ids_->reserve(model_ids_.size()); + for (; model_id != model_id_end; ++model_id, ++object_id) + { + std::cout << "Loading model for object id: " << *object_id << std::endl; + object_recognition::db_future::Document doc(db, *collection_models_, *model_id); + mmod_objects templates; + doc.get_attachment ("templates", templates); + templates_->push_back(templates); + + // Store the id conversion + ids_->push_back(*object_id); + + // Store the 3d positions + mmod_filters filters; + doc.get_attachment ("filters", filters); + filters_->push_back(filters); + } + } + + int + process(const ecto::tendrils& inputs, const ecto::tendrils& outputs) + { + *do_update_out_ = *do_update_; + *do_update_ = false; + return ecto::OK; + } + + /** The collection where the models are stored */ + ecto::spore collection_models_; + /** The objects ids to use */ + std::vector object_ids_; + /** The matching model ids to use */ + std::vector model_ids_; + /** the DB JSON parameters */ + ecto::spore db_params_; + + /** The filters */ + ecto::spore > filters_; + /** The templates */ + ecto::spore > templates_; + /** If True, that means we got new data */ + ecto::spore do_update_out_; + /** If True, load from the db */ + ecto::spore do_update_; + /** Matching between an OpenCV integer ID and the ids found in the JSON */ + ecto::spore > ids_; + }; +} + +ECTO_CELL(mmod, mmod::TemplateLoader, "TemplateLoader", + "Loads templates from the DB.") diff --git a/cells/MModPersister.cpp b/cells/MModPersister.cpp new file mode 100644 index 0000000..bc30a91 --- /dev/null +++ b/cells/MModPersister.cpp @@ -0,0 +1,55 @@ +#include +#include +#include + +//mmod inclues +#include "mmod_general.h" +#include "mmod_objects.h" +#include "mmod_mode.h" +#include "mmod_features.h" +#include "mmod_color.h" + +//For serialization +#include +#include + +using ecto::tendrils; +using ecto::spore; +namespace mmod +{ + struct MModPersister + { + static void + declare_params(tendrils& params) + { + params.declare(&MModPersister::filename_objects,"filename_objects"); + params.declare(&MModPersister::filename_filter,"filename_filter"); + } + static void + declare_io(const tendrils& params, tendrils& in, tendrils& out) + { + in.declare(&MModPersister::objects_in,"objects"); + in.declare(&MModPersister::filters_in,"filters"); + } + int process(const tendrils& /*in*/, const tendrils& /*out*/) + { + { + std::ofstream filter_out(filename_filter->c_str()); + boost::archive::binary_oarchive oa(filter_out); + oa << *filters_in; + } + { + std::ofstream objects_out(filename_objects->c_str()); + boost::archive::binary_oarchive oa(objects_out); + oa << *objects_in; + } + return ecto::OK; + } + spore filename_filter,filename_objects; + spore objects_in; + spore filters_in; + }; +} + +ECTO_CELL(mmod, mmod::MModPersister, "MModPersister","???"); + diff --git a/cells/MModTester.cpp b/cells/MModTester.cpp new file mode 100644 index 0000000..ca404d4 --- /dev/null +++ b/cells/MModTester.cpp @@ -0,0 +1,153 @@ +#include +#include +#include +#include + +#include "object_recognition/common/types.h" + +#include "mmod_objects.h" //For train and test (includes mmod_mode.h, mmod_features.h, mmod_general.h +#include "mmod_color.h" //For depth and color processing (yes, I should change the name) +using namespace std; +using namespace cv; +namespace mmod +{ + using ecto::tendrils; + using ecto::spore; + struct MModTester + { + static void + declare_params(tendrils& p) + { + p.declare ("filename", "Output file name to save training to."); + + p.declare ("thresh_match", "The threshold for declaring an object detected", 0.95); + p.declare ( + "frac_overlap", + "the fraction of overlap between 2 above threshold feature's bounding box rectangles that constitutes 'overlap'", + 0.6); + p.declare ("color_filter_thresh", "The color filter threshold to confirm a match", 0.91); + p.declare ("skip_x", "Control sparse testing of the feature images", 8); + p.declare ("skip_y", "Control sparse testing of the feature images", 8); + } + + static void + declare_io(const tendrils& p, tendrils& i, tendrils& o) + { + i.declare > ("templates", "The templates"); + i.declare > ("filters", "The filters"); + i.declare > ("ids", "The matching object ids"); + i.declare ("do_update", "If true, that means new templates have been loaded"); + + i.declare ("image", "An image. BGR image of type CV_8UC3").required(true); + i.declare ("depth", "Depth image of type CV_16UC1").required(true); + // i.declare ("mask", "Object mask of type CV_8UC1 or CV_8UC3").required(false); + o.declare ("debug_image", "Debug image."); + } + + void + configure(const tendrils& p, const tendrils& i, const tendrils& o) + { + std::string filename; + //parameters + p["filename"] >> filename; //?? THIS WORKS FOR FILES ON DISK, WHAT ABOUT DB? + thresh_match_ = p["thresh_match"]; + frac_overlap_ = p["frac_overlap"]; + color_filter_thresh_ = p["color_filter_thresh"]; + skip_x_ = p["skip_x"]; + skip_y_ = p["skip_y"]; + modesCD.push_back("Grad"); + // modesCD.push_back("Color"); + // modesCD.push_back("Depth"); + //deserialize from file. + std::ifstream file(filename.c_str()); + boost::archive::text_iarchive ia(file); + ia >> templates_; //?? HOW TO GET FILTERS IN? + ia >> filters_; //??THIS IS TOTALLY WRONG ... JUST FOR COMPILE + + // inputs + image_ = i["image"]; + // mask_ = i["mask"]; + depth_ = i["depth"]; + + //outputs + debug_image_ = o["debug_image"]; + } + + int + process(const tendrils& i, const tendrils& o) + { + //iputs spores are like smart pointers, dereference to get at under + //lying data type. + cv::Mat image = *image_, depth = *depth_;// , mask = *mask_; //We don't need mask for rec + + //run detections. + //TEST (note that you can also match_all_objects_at_a_point(...): + calcHLS.computeColorHLS(image, colorfeat, noMask); + calcGrad.computeGradients(image, gradfeat, noMask); + + FeatModes.clear(); + FeatModes.push_back(gradfeat); + // FeatModes.push_back(colorfeat); + // FeatModes.push_back(depthfeat); + for (unsigned int i = 0; i < templates_->size(); ++i) + { + mmod_objects & mmod_object = (*templates_)[i]; + mmod_filters & mmod_filter = (*filters_)[i]; + + int numrawmatches = 0; //Number of matches before non-max suppression + cout << "num_matches = " << numrawmatches << endl; + int num_matches = mmod_object.match_all_objects(FeatModes, modesCD, noMask, *thresh_match_, *frac_overlap_, + *skip_x_, *skip_y_, &numrawmatches); + cout << "num_matches = " << num_matches << ", selected from # of raw matches = " << numrawmatches << endl; + // vector scs = templates_.scores; //Copy the scores over + + //FILTER RECOGNITIONS BY COLOR + mmod_filter.filter_object_recognitions(colorfeat, mmod_object, *color_filter_thresh_); + + } + + //TO DISPLAY MATCHES (NON-MAX SUPPRESSED) + cv::Mat debug_image; + image.copyTo(debug_image); + for (unsigned int i = 0; i < templates_->size(); ++i) + { + mmod_objects & mmod_object = (*templates_)[i]; + + mmod_object.draw_matches(debug_image); //draw results... + mmod_object.cout_matches(); + } + + *debug_image_ = debug_image; + return ecto::OK; + } + + colorhls calcHLS; //Color feature processing + // depthgrad calcDepth; //Depth feature processing + gradients calcGrad; //Gradient feature processing + std::vector FeatModes; //List of images + std::vector modesCD; //Names of modes (color and depth) + cv::Mat gradfeat, colorfeat, depthfeat; //To hold feature outputs. These will be CV_8UC1 images + cv::Mat noMask; //This is simply an empty image which means to search the whole test image + + //params + spore thresh_match_, frac_overlap_, color_filter_thresh_; + spore skip_x_, skip_y_; + + //inputs + spore image_, mask_, depth_; + + //outputs + spore debug_image_; + + /** The filters */ + ecto::spore > filters_; + /** The templates */ + ecto::spore > templates_; + /** If True, load from the db */ + ecto::spore do_update_; + /** Matching between an OpenCV integer ID and the ids found in the JSON */ + ecto::spore > ids_; + }; +} +ECTO_CELL(mmod, mmod::MModTester, "MModTester", "An mmod template detector.") +; diff --git a/cells/MModTrainer.cpp b/cells/MModTrainer.cpp new file mode 100644 index 0000000..607af15 --- /dev/null +++ b/cells/MModTrainer.cpp @@ -0,0 +1,143 @@ +#include +#include +#include +#include +#include +#include + +//mmod inclues +#include "mmod_general.h" +#include "mmod_objects.h" +#include "mmod_mode.h" +#include "mmod_features.h" +#include "mmod_color.h" + + +namespace mmod +{ +//#define output_data 1 + + using ecto::tendrils; + using ecto::spore; + using namespace std; + struct MModTrainer + { +#ifdef output_data + MModTrainer() + : + color_filter("Color"), framenum(0), objID("foo") + { + modesCD.push_back("Grad"); + } +#else + MModTrainer() + : + color_filter("Color") + { + modesCD.push_back("Grad"); + } +#endif + + static void + declare_params(tendrils& params) + { + params.declare(&MModTrainer::thresh_learn,"thresh_learn","The threshold" + "for learning a new template",0.0); //Zero thresh_learn => learn every view + params.declare(&MModTrainer::object_id,"object_id", + "The object id, to learn.") + .required(true); + + } + static void + declare_io(const tendrils& params, tendrils& in, tendrils& out) + { + in.declare(&MModTrainer::image_in,"image", "The input image.").required(true); + in.declare(&MModTrainer::depth_in,"depth", "The depth image.").required(true); + in.declare(&MModTrainer::mask_in,"mask", "The mask.").required(false); + in.declare(&MModTrainer::frame_number_in, "frame_number", + " The frame number").required(true); + + out.declare(&MModTrainer::grad_vis,"grad_vis", + "A visualization of the gradient feature."); + out.declare(&MModTrainer::filter_vis,"filter_vis", + "A visualization of the gradient feature."); + out.declare(&MModTrainer::objects_out,"objects"); + out.declare(&MModTrainer::filters_out,"filters"); + } +#ifdef output_data + string Int2string(int number) + { + stringstream ss;//create a stringstream + ss << number;//add number to the stream + return ss.str();//return a string with the contents of the stream + } +#endif + + int process(const tendrils& /*in*/, const tendrils& /*out*/) + { + std::cout <<" training on image:" << *frame_number_in + << " for object id:" << * object_id << std::endl; +#ifdef output_data + if(objID.compare(*object_id)){ framenum = 0; objID= *object_id; cout << "Object ID changed from " << objID << " to " << *object_id << endl;} + string fn = Int2string(framenum++); + string jp = ".jpg"; + string png = ".png"; + string Im = "Image_"+*object_id+"_"+fn+jp, De = "Depth_"+*object_id+"_"+fn+png, Ma = "Mask_"+*object_id+"_"+fn+jp; + cv::imwrite(Im,*image_in); + cv::imwrite(Ma,*mask_in); + cv::imwrite(De,*depth_in); +#endif + + //reset our outputs, so that we can be thread safe. + *filter_vis = cv::Mat(); + *grad_vis = cv::Mat(); + if(!cv::countNonZero(*mask_in)) return ecto::OK; + + //PROCESS TO GET FEATURES + cv::Mat colorfeat, gradfeat; + calcHLS.computeColorHLS(*image_in,colorfeat,*mask_in,"train"); + calcGrad.computeGradients(*image_in,gradfeat,*mask_in,"train"); + g.visualize_binary_image(gradfeat, *grad_vis); + FeatModes.clear(); + FeatModes.push_back(gradfeat); + float Score; + int num_templ = Objs.learn_a_template(FeatModes,modesCD, *mask_in, + *object_id, *object_id, *frame_number_in, *thresh_learn, &Score); + std::cout << "#"<<*frame_number_in + <<": Number of templates learned = " << num_templ + <<", Score = "< +#include +#include +using ecto::tendrils; +namespace mmod +{ + struct Pyramid + { + static void + declare_params(tendrils& p) + { + p.declare(&Pyramid::n_levels,"n_levels","The number of pyramid levels", 2); + } + + static void + declare_io(const tendrils& p, tendrils& i, tendrils& o) + { + int n_levels; + p["n_levels"] >> n_levels; + for(int iter = 0; iter < n_levels; iter++) + { + std::string name = boost::str( boost::format("level_%d")%iter ); + o.declare(name,"A level from the pyramid"); + } + i.declare(&Pyramid::image,"image"); + } + + int process(const tendrils& i,const tendrils& o) + { + pyramid_levels.clear(); //for multithreaded apps. + cv::buildPyramid(*image,pyramid_levels,*n_levels); + for(int iter = 0, end = *n_levels; iter != end; iter++) + { + std::string name = boost::str( boost::format("level_%d")%iter ); + o[name] << pyramid_levels[iter]; + } + return ecto::OK; + } + ecto::spore n_levels; + ecto::spore image; + std::vector pyramid_levels; + }; + +} + +ECTO_CELL(mmod, mmod::Pyramid, "Pyramid","???"); diff --git a/cells/module.cpp b/cells/module.cpp new file mode 100644 index 0000000..a63c025 --- /dev/null +++ b/cells/module.cpp @@ -0,0 +1,4 @@ +#include + +ECTO_DEFINE_MODULE(mmod){ +} diff --git a/cells/scripts/testing.py b/cells/scripts/testing.py new file mode 100644 index 0000000..a9ece87 --- /dev/null +++ b/cells/scripts/testing.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +import ecto +import sys +import couchdb +from ecto_opencv.highgui import imshow +from ecto_object_recognition import capture +from object_recognition import models, dbtools +import mmod +from ecto_object_recognition.object_recognition_db import ObjectDbParameters + +# FROM AN OLDER FILE ... +import argparse +import time +import tempfile +import os +import math +import subprocess + +#import object_recognition +#from object_recognition import dbtools, models, capture, observations + +from mmod import MModTester + +def parse_args(): + parser = argparse.ArgumentParser(description='Test mmod features on a sequence of inputs.') + parser.add_argument('-i', '--training', metavar='TRAINING_FILE', dest='training', type=str, default='', + help='The training file') + parser.add_argument('--use_kinect', dest='use_kinect', action='store_true', + default=False, help='Use a highres kinect') + parser.add_argument('--use_db', dest='use_db', action='store_true', + default=True, help='Use the db to test') + dbtools.add_db_options(parser) + args = parser.parse_args() + if args.training == '': + parser.print_usage() + print 'You must supply a training file.' + sys.exit(1) + return args + +def kinect_highres(device_n=0): + from ecto_openni import Capture, ResolutionMode, Device + return Capture('ni device', rgb_resolution=ResolutionMode.SXGA_RES, + depth_resolution=ResolutionMode.VGA_RES, + rgb_fps=15, depth_fps=30, + device_number=device_n, + registration=True, + synchronize=False, + device=Device.KINECT + ) +def hookup_kinect(plasm): + ''' + returns a kinect based source of data. + ''' + kinect_raw = kinect_highres() + kinect_cv = highgui.NiConverter('Kinect CV') + rescale_depth = capture.RescaledRegisteredDepth() #this is for SXGA mode scale handling. + #connect up the kinect as input + plasm.connect( + kinect_raw[:] >> kinect_cv[:], + kinect_cv['image'] >> rescale_depth['image'], + kinect_cv['depth'] >> rescale_depth['depth'], + ) + + return (kinect_cv['image'], rescale_depth['depth']) + +def hookup_db(plasm, db_root): + couch = couchdb.Server(db_root) + dbs = dbtools.init_object_databases(couch) + models.sync_models(dbs) + sessions = dbs['sessions'] + observations = dbs['observations'] + obs_ids = models.find_all_observation_ids(sessions, observations) + db_reader = capture.ObservationReader('db_reader', db_url=db_root, collection='observations') + observation_dealer = ecto.Dealer(typer=db_reader.inputs.at('observation'), iterable=obs_ids) + db_reader = capture.ObservationReader('db_reader', db_url=db_root, collection='observations') + rescale_depth = capture.RescaledRegisteredDepth() #this is for SXGA mode scale handling. + #connect some initial filters + plasm.connect( + observation_dealer[:] >> db_reader['observation'], + db_reader['image'] >> rescale_depth['image'], + db_reader['depth'] >> rescale_depth['depth'], + ) + return (db_reader['image'], db_reader['depth']) + +def test_mmod(args): + ''' + Run mmod testing + ''' + plasm = ecto.Plasm() + + image , depth = None, None + if args.use_kinect: + # a source of data, image, and depth, same size. + image, depth = hookup_kinect(plasm) + elif args.use_db: + image, depth = hookup_db(plasm, args.db_root) + else: + raise RuntimeError("No source given --use_kinect or --use_db") + + #hook up the tester + mmod_tester = MModTester(filename=args.training,thresh_match=0.95,skip_x=8,skip_y=8) + + plasm.connect( + image >> mmod_tester['image'], + depth >> mmod_tester['depth'], + ) + #visualize raw data + fps = highgui.FPSDrawer() + plasm.connect( + image >> fps[:], + fps[:] >> highgui.imshow('image', name='image')[:], + depth >> highgui.imshow('depth', name='depth')[:], + mmod_tester['debug_image'] >> highgui.imshow('mmod debug', name='mmod depth')[:], + ) + sched = ecto.schedulers.Singlethreaded(plasm) + sched.execute() + +if "__main__" == __name__: + args = parse_args() + test_mmod(args) diff --git a/cells/scripts/training.py b/cells/scripts/training.py new file mode 100755 index 0000000..cfdfe6c --- /dev/null +++ b/cells/scripts/training.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +import ecto +import sys +import couchdb +from ecto_opencv.highgui import imshow +from ecto_object_recognition import capture +from object_recognition import models, dbtools +import mmod +from ecto_object_recognition.object_recognition_db import ObjectDbParameters + + +def parse_args(): + import argparse + parser = argparse.ArgumentParser(description='Train MyAlgorithm on views from the database.') + parser.add_argument('objects', metavar='OBJECT', type=str, nargs='*', + help='Object ids to train.') + parser.add_argument('--all', dest='compute_all', action='store_const', + const=True, default=False, + help='Compute meshes for all possible sessions.') + dbtools.add_db_options(parser) + args = parser.parse_args() + return args + +args = parse_args() +db = dbtools.init_object_databases(couchdb.Server(args.db_root)) + +if args.compute_all: + results = models.Session.all(db) + for session in results: + args.objects.append(session.object_id) + +for object_id in args.objects: + #get a list of observation ids for a particular object id. + obs_ids = models.find_all_observations_for_object(db, object_id) + + if not obs_ids: + print 'No observations found for object %s.' % object_id + continue + + plasm = ecto.Plasm() + #the db_reader transforms observation id into a set of image,depth,mask,K,R,T + db_reader = capture.ObservationReader("db_reader", db_url=args.db_root) + #this iterates over all of the observation ids. + observation_dealer = ecto.Dealer(tendril=db_reader.inputs.at('observation'), + iterable=obs_ids) + + plasm.connect(observation_dealer[:] >> db_reader['observation']) + + #allocate mmod trainer each time to keep it simple. + mmod_trainer = mmod.MModTrainer(thresh_learn=0.97,object_id=str(object_id)) + pyr_depth = mmod.Pyramid(n_levels=3) + pyr_image = mmod.Pyramid(n_levels=3) + pyr_mask = mmod.Pyramid(n_levels=3) + mmod_persistance_ = mmod.MModPersister(filename_filter='filter_%s.txt'%str(object_id), + filename_objects='objects_%s.txt'%str(object_id) + ) + mmod_model_inserter_ = mmod.ModelInserter(collection_models='object_recognition',object_id=str(object_id), + model_json_params='{"none":"none"}', db_params=ObjectDbParameters({'type':'CouchDB', 'root':args.db_root, + 'collection':'object_recognition'}) + ) + + mmod_persistance = ecto.If('Persistance', + cell = mmod_persistance_ + ) + mmod_model_inserter = ecto.If('ModelInserter', + cell = mmod_model_inserter_ + ) + + #connect trainer + plasm.connect(db_reader['frame_number'] >> mmod_trainer['frame_number'], + db_reader['depth'] >> pyr_depth['image'], + db_reader['image'] >> pyr_image['image'], + db_reader['mask'] >> pyr_mask['image'], + pyr_depth['level_2'] >> mmod_trainer['depth'], + pyr_image['level_2'] >> mmod_trainer['image'], + pyr_mask['level_2'] >> mmod_trainer['mask'], + ) + + #connect training debug visualization + plasm.connect(mmod_trainer['filter_vis'] >> imshow(name='filter')['image'], + mmod_trainer['grad_vis'] >> imshow(name='grad')['image'] + ) + + #connect persistance + plasm.connect(mmod_trainer['filters','objects'] >> mmod_persistance['filters','objects']) + plasm.connect(mmod_trainer['filters','objects'] >> mmod_model_inserter['filters','objects']) + + #visualization + plasm.connect(db_reader['image'] >> imshow(name='image')['image'], + db_reader['mask'] >> imshow(name='mask')['image'], + db_reader['depth'] >> imshow(name='depth')['image'] + ) + + sched = ecto.schedulers.Singlethreaded(plasm) + sched.execute() + + #thunk one final process on the persistance to save to disk + mmod_persistance_.process() + mmod_model_inserter_.process() + + #After done execution upload the resulting model to the db.... +