diff --git a/TinTin/touchless_tracking/CMakeLists.txt b/TinTin/touchless_tracking/CMakeLists.txt index 65965fa..5ff0ec2 100644 --- a/TinTin/touchless_tracking/CMakeLists.txt +++ b/TinTin/touchless_tracking/CMakeLists.txt @@ -8,7 +8,7 @@ add_definitions(-msse2 -pthread -std=c++11 -fPIC -ffast-math) set(VOXEL_INCLUDE_DIRS . /usr/include/voxel /usr/include/voxel/pcl /usr/include/voxel/ti3dtof /usr/include/voxel/Filter ) set(VOXEL_LIBRARIES /usr/lib/libti3dtof.so /usr/lib/libvoxel.so /usr/lib/libvoxelpcl.so) -add_executable(touchless touchless.cpp Jive.cpp TOFApp.cpp) +add_executable(touchless touchless.cpp Jive.cpp TOFApp.cpp FakeMouse.cpp) target_include_directories(touchless PUBLIC ${VOXEL_INCLUDE_DIRS} ${PCL_INCLUDE_DIRS} ${EIGEN_INCLUDE_DIRS}) target_link_libraries(touchless voxelpcl X11 ${OpenCV_LIBS} ${VOXEL_LIBRARIES} ${PCL_COMMON_LIBRARIES} ${PCL_IO_LIBRARIES} ${PCL_VISUALIZATION_LIBRARIES}) diff --git a/TinTin/touchless_tracking/FakeMouse.cpp b/TinTin/touchless_tracking/FakeMouse.cpp new file mode 100644 index 0000000..05ffa1a --- /dev/null +++ b/TinTin/touchless_tracking/FakeMouse.cpp @@ -0,0 +1,236 @@ +/*! + * ========================================================================================== + * + * @addtogroup FakeMouse + * @{ + * + * @file FakeMouse.cpp + * @version 1.0 + * @date 1/7/2016 + * + * @note FakeMouse class + * + * Copyright(c) 20015-2016 Texas Instruments Corporation, All Rights Reserved.q + * TI makes NO WARRANTY as to software products, which are supplied "AS-IS" + * + * ========================================================================================== + */ +#define __FAKEMOUSE_CPP__ +#include "FakeMouse.h" + + +/*! + *=========================================================================================== + * @brief Constructors + *=========================================================================================== + */ +FakeMouse::FakeMouse() +{ + setDisplay(XOpenDisplay(0)); +} + +FakeMouse::FakeMouse(Display *disp) +{ + setDisplay(disp); +} + +/*! + *=========================================================================================== + * @brief Initialize X display info + *=========================================================================================== + */ +void FakeMouse::setDisplay(Display *disp) +{ + display = disp; + root = DefaultRootWindow(display); + screen = DefaultScreenOfDisplay(display); + width = screen->width; + height = screen->height; +} + + +/*! + *=========================================================================================== + * @brief Destructor + *=========================================================================================== + */ +FakeMouse::~FakeMouse() +{ +} + + +/*! + *=========================================================================================== + * @brief Generate a button event + *=========================================================================================== + */ +bool FakeMouse::buttonEvent(int button, int event_type) +{ + XEvent event; + + memset(&event, 0, sizeof(event)); + event.type = event_type; + event.xbutton.button = button; + event.xbutton.same_screen = True; + XQueryPointer(display, RootWindow(display, DefaultScreen(display)), + &event.xbutton.root, &event.xbutton.window, + &event.xbutton.x_root, &event.xbutton.y_root, + &event.xbutton.x, &event.xbutton.y, + &event.xbutton.state); + event.xbutton.subwindow = event.xbutton.window; + while (event.xbutton.subwindow) { + event.xbutton.window = event.xbutton.subwindow; + + XQueryPointer(display, event.xbutton.window, + &event.xbutton.root, &event.xbutton.subwindow, + &event.xbutton.x_root, &event.xbutton.y_root, + &event.xbutton.x, &event.xbutton.y, + &event.xbutton.state); + } + if (event_type == ButtonRelease) + event.xbutton.state = 1 << (button+7); + + if (XSendEvent(display, PointerWindow, True, 0xfff, &event) == 0) + return false; + XFlush(display); + return true; +} + + +/*! + *=========================================================================================== + * @brief Generate a button down event + *=========================================================================================== + */ +void FakeMouse::buttonDown(int button) +{ + buttonEvent(button, ButtonPress); +} + + +/*! + *=========================================================================================== + * @brief Generate a button up event + *=========================================================================================== + */ +void FakeMouse::buttonUp(int button) +{ + buttonEvent(button, ButtonRelease); +} + + +/*! + *=========================================================================================== + * @brief Generate a click event + *=========================================================================================== + */ +void FakeMouse::click(int button) +{ + buttonDown(button); + usleep(100000); + buttonUp(button); +} + + +/*! + *=========================================================================================== + * @brief Generate a double click event + *=========================================================================================== + */ +void FakeMouse::doubleClick(int button) +{ + click(button); + usleep(100000); + click(button); +} + + +/*! + *=========================================================================================== + * @brief Move mouse to absolute position (x, y) + *=========================================================================================== + */ +void FakeMouse::moveTo(int x, int y) +{ + XWarpPointer(display, None, root, 0, 0, 0, 0, x, y); + XFlush(display); +} + + +/*! + *=========================================================================================== + * @brief Move mouse position incrementally by (dx, dy) + *=========================================================================================== + */ +void FakeMouse::moveBy(int dx, int dy) +{ + int x, y; + + getPos(x, y); + XWarpPointer(display, None, root, 0, 0, 0, 0, x+dx, y+dy); + XFlush(display); +} + + +/*! + *=========================================================================================== + * @brief Get current mouse absolute position + *=========================================================================================== + */ +void FakeMouse::getPos(int &x, int &y) +{ + XEvent event; + XQueryPointer(display, RootWindow(display, DefaultScreen(display)), + &event.xbutton.root, &event.xbutton.window, + &event.xbutton.x_root, &event.xbutton.y_root, + &event.xbutton.x, &event.xbutton.y, + &event.xbutton.state); + x = event.xbutton.x; + y = event.xbutton.y; +} + + +/*! + *=========================================================================================== + * @brief Get display dimension + *=========================================================================================== + */ +void FakeMouse::getDim(int &x, int &y) +{ + x = width; + y = height; +} + + +/*! + *=========================================================================================== + * @brief Unit Test + *=========================================================================================== + */ +#ifdef __UNIT_TEST__ +int main(int argc,char * argv[]) +{ + int i=0; + int x , y; + x=atoi(argv[1]); + y=atoi(argv[2]); + Display *display = XOpenDisplay(0); + FakeMouse mouse(display); + Screen *screen = DefaultScreenOfDisplay(display); + mouse.moveTo(x, y); + mouse.click(Button1); + + mouse.moveTo(0,0); + for (int i=0; i < 200; i++) { + mouse.moveBy(screen->width/200, screen->height/200); + usleep(10000); + } + + XCloseDisplay(display); + return 0; +} +#undef __UNIT_TEST__ +#endif // UNIT_TEST + +#undef __FAKEMOUSE_CPP__ + diff --git a/TinTin/touchless_tracking/FakeMouse.h b/TinTin/touchless_tracking/FakeMouse.h new file mode 100644 index 0000000..281cdcb --- /dev/null +++ b/TinTin/touchless_tracking/FakeMouse.h @@ -0,0 +1,38 @@ +/* + * FakeMouse - simulated X windows mouse actions + * + * Copyright (c) 2015-2016 Texas Instruments Inc. + */ +#include +#include +#include +#include +#include +#include + +#ifndef __FAKEMOUSE_H__ +#define __FAKEMOUSE_H__ + +class FakeMouse { +public: + FakeMouse(); + FakeMouse(Display *disp); + ~FakeMouse(); + void setDisplay(Display *disp); + void buttonDown(int button); + void buttonUp(int button); + void click(int button); + void doubleClick(int button); + void moveTo(int x, int y); + void moveBy(int dx, int dy); + void getPos(int &x, int &y); + void getDim(int &xmax, int &ymax); + +private: + bool buttonEvent(int btn, int e); + Display *display; + Window root; + Screen *screen; + int width, height; +}; +#endif diff --git a/TinTin/touchless_tracking/Jive.cpp b/TinTin/touchless_tracking/Jive.cpp index 04baa6a..4515835 100644 --- a/TinTin/touchless_tracking/Jive.cpp +++ b/TinTin/touchless_tracking/Jive.cpp @@ -1,5 +1,5 @@ /*! - * ============================================================================ + * ========================================================================================== * * @addtogroup Jive * @{ @@ -13,7 +13,7 @@ * Copyright(c) 20015-2016 Texas Instruments Corporation, All Rights Reserved.q * TI makes NO WARRANTY as to software products, which are supplied "AS-IS" * - * ============================================================================ + * ========================================================================================== */ #define __JIVE_CPP__ #include "Jive.h" @@ -37,12 +37,6 @@ Jive::Jive(int w, int h) : TOFApp(w, h) { _zMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); _aMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); - _zDiffMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); - _aDiffMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); - _zHeatMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); - _aHeatMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); - _zPrevMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); - _aPrevMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); _zBkgMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); _aBkgMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); _zFgMap = Mat::zeros(getDim().height, getDim().width, CV_32FC1); @@ -51,61 +45,41 @@ Jive::Jive(int w, int h) : TOFApp(w, h) _drawing = Mat::zeros(getDim().height, getDim().width, CV_8UC3 ); _bkgUpdated = false; - _aHeatMapThresh = 0; - _zHeatMapThresh = 0; - _aDiffMapThresh = 0; - _zDiffMapThresh = 0; - _heatMapCoef = 0.5; - _zTrigger = 0.05; - _zLowThresh = 0.02; + _zTrigger = 0.03; + _zLowThresh = 0.01; _zHighThresh = 1; _aGain = 10; _minContourSize = 100; - _minConvDefDepth = 10; - _movementCount = 0; - _separation = 5; _Xmin = 66; _Xmax = 279; _Ymin = 51; _Ymax = 171; - _maxAngle = 60.0; + _Xcur = 0; + _Ycur = 0; // Setup parameter map - //_param["aHeatMapThresh"] = std::make_tuple(&_aHeatMapThresh, 1, 4095); - //_param["zHeatMapThresh"] = std::make_tuple(&_zHeatMapThresh, 1000, 2); - //_param["aDiffMapThresh"] = std::make_tuple(&_aDiffMapThresh, 1, 4095); - //_param["zDiffMapThresh"] = std::make_tuple(&_zDiffMapThresh, 1000, 2); - //_param["heatMapCoef"] = std::make_tuple(&_heatMapCoef, 100, 1); - _param["zTrigger"] = std::make_tuple(&_zLowThresh, 1000, 2); + _param["zTrigger"] = std::make_tuple(&_zTrigger, 1000, 2); _param["zLowThresh"] = std::make_tuple(&_zLowThresh, 1000, 2); _param["zHighThresh"] = std::make_tuple(&_zHighThresh, 1000, 2); _param["aGain"] = std::make_tuple(&_aGain, 10, 200); _param["minContourSize"] = std::make_tuple(&_minContourSize, 1, 10000); - _param["minConvDefDepth"] = std::make_tuple(&_minConvDefDepth, 1, 300); - _param["maxAngle"] = std::make_tuple(&_maxAngle, 1, 90); - _param["seperation"] = std::make_tuple(&_separation, 1, 40); _param["Xmin"] = std::make_tuple(&_Xmin, 1, 319); _param["Xmax"] = std::make_tuple(&_Xmax, 1, 319); _param["Ymin"] = std::make_tuple(&_Ymin, 1, 239); _param["Ymax"] = std::make_tuple(&_Ymax, 1, 239); + _param["Xcur"] = std::make_tuple(&_Xcur, 1, 300); + _param["Ycur"] = std::make_tuple(&_Ycur, 1, 300); // Setup image map _images["aMap"] = &_aMap; _images["aBkgMap"] = &_aBkgMap; _images["aFgMap"] = &_aFgMap; - _images["aHeatMap"] = &_aHeatMap; - _images["aDiffMap"] = &_aDiffMap; - _images["aPrevMap"] = &_aPrevMap; _images["zMap"] = &_zMap; _images["zBkgMap"] = &_zBkgMap; _images["zFgMap"] = &_zFgMap; - _images["zHeatMap"] = &_zHeatMap; - _images["zDiffMap"] = &_zDiffMap; - _images["zPrevMap"] = &_zPrevMap; _images["bMap"] = &_bMap; _images["drawing"] = &_drawing; - } @@ -165,10 +139,46 @@ void Jive::initDisplays() */ void Jive::displayMaps() { + // Draw crop lines + cv::line(_drawing, cv::Point(_Xmin, _Ymin), cv::Point(_Xmin, _Ymax), Scalar(0,0,255), 1); + cv::line(_drawing, cv::Point(_Xmin, _Ymax), cv::Point(_Xmax, _Ymax), Scalar(0,0,255), 1); + cv::line(_drawing, cv::Point(_Xmax, _Ymax), cv::Point(_Xmax, _Ymin), Scalar(0,0,255), 1); + cv::line(_drawing, cv::Point(_Xmax, _Ymin), cv::Point(_Xmin, _Ymin), Scalar(0,0,255), 1); + + // Draw hand features + if (_bkgUpdated && _numHands > 0) + { + for (int i=0; i < _numHands; i++) + { + // Draw tip of hand + Scalar color = Scalar(0,255,255); + int sz = 4; + if (_zBkgMap.at(_handTip[i]) - _zMap.at(_handTip[i]) < _zTrigger) + { + sz = 10; + } + cv::circle(_drawing, _handTip[i], sz, color, 1); + + // Draw palm + cv::circle(_drawing, _palmCenter[i], (int)_palmRadius[i], Scalar(0,255,0), 1); + + // Draw hand + cv::drawContours(_drawing, _contours, _handContour[i], Scalar(0, 0, 255), 0, 1, vector(), 0, cv::Point() ); + + // Draw text + if (_numHands == 2) + { + std::string s = (i == _leftHand) ? "LEFT" : "RIGHT"; + cv::putText(_drawing, s, _handTip[i], FONT_HERSHEY_COMPLEX_SMALL, 0.4, Scalar(255,255,255)); + } + } // for (i) + + } + + // Display all registered maps for (int i=0; i < _mapsToDisplay.size(); i++) { - if (_mapsToDisplay[i] == "aMap" || _mapsToDisplay[i] == "aHeatMap" - || _mapsToDisplay[i] == "aDiffMap" || _mapsToDisplay[i] == "aBkgMap" + if (_mapsToDisplay[i] == "aMap" || _mapsToDisplay[i] == "aBkgMap" || _mapsToDisplay[i] == "aFgMap" || _mapsToDisplay[i] == "drawing" ) imshow(_mapsToDisplay[i], *_images[_mapsToDisplay[i]]*_aGain ); else @@ -213,30 +223,6 @@ map< std::string, cv::Mat* > &Jive::getImageMap() return _images; } - -/*! - *=========================================================================================== - * @brief Check for movement. - *=========================================================================================== - */ -bool Jive::noMovement(int t) -{ - bool rc=false; - - if (cv::sum(abs(_zDiffMap))[0] > _zDiffMapThresh || cv::sum(abs(_aDiffMap))[0] > _aDiffMapThresh) - _movementCount = 0; - - if (_movementCount++ >= t) - { - _movementCount = t; - rc = true; - } - - return rc; -} - - - /*! *=========================================================================================== * @brief Initialize control window based on Jive parameters @@ -306,14 +292,6 @@ void Jive::updateMaps(Frame *frame) int idx = i*getDim().width+j; _zMap.at(i,j) = frm->points[idx].z; _aMap.at(i,j) = frm->points[idx].i; - //_zDiffMap.at(i,j) = abs(_zMap.at(i,j)-_zPrevMap.at(i,j)); - //_aDiffMap.at(i,j) = abs(_aMap.at(i,j)-_aPrevMap.at(i,j)); - //_zHeatMap.at(i,j) = (1-_heatMapCoef)*_zHeatMap.at(i,j) - // + _heatMapCoef*_zDiffMap.at(i,j); - //_aHeatMap.at(i,j) = (1-_heatMapCoef)*_aHeatMap.at(i,j) - // + _heatMapCoef*_aDiffMap.at(i,j); - // _zPrevMap.at(i,j) = _zMap.at(i,j); - //_aPrevMap.at(i,j) = _aMap.at(i,j); } } } @@ -338,7 +316,7 @@ void Jive::morphClean(Mat &in, Mat &out) * @brief Find palm center *=========================================================================================== */ -bool Jive::findPalmCenter(vector &contour, cv::Point ¢er, float &radius) +bool Jive::findContourCenter(vector &contour, cv::Point ¢er, float &radius) { bool rc = false; float m10, m01, m00; @@ -373,254 +351,83 @@ bool Jive::findPalmCenter(vector &contour, cv::Point ¢er, float & } -/*! - *=========================================================================================== - * @brief Find point closest to center of cluster - *=========================================================================================== - */ -int Jive::findCenterPoint(vector &contour, vector &cluster) -{ - float minDist; - int minIndex = -1; - cv::Point centroid = cv::Point(0,0); - - if (cluster.size() > 0) - { - // Find centroid - for (int i=0; i < cluster.size(); i++) - { - centroid.x += contour[cluster[i]].x; - centroid.y += contour[cluster[i]].y; - } - centroid.x /= cluster.size(); - centroid.y /= cluster.size(); - - // Find closest point to centroid - minDist = 1e10; - for (int i=0; i < cluster.size(); i++) - { - cv::Point p = contour[cluster[i]]; - float dist = (float)(p.x-centroid.x)*(p.x-centroid.x) + (p.y-centroid.y)*(p.y-centroid.y); - if (dist < minDist) - { - minDist = dist; - minIndex = cluster[i]; - } - } - } - - return minIndex; -} - /*! *=========================================================================================== - * @brief Find fingertips from convex hull + * @brief Find hand tips of qualified contour *=========================================================================================== */ -bool Jive::findTips(vector &contour, vector &hulls, vector &tips, float maxDist) +bool Jive::findHandTips(vector< vector > &contours) { bool found = false; - int first, second; - cv::Point p1, p2; - vector cluster; - - tips.clear(); - if (hulls.size() > 0) - { - for (int h=0; h (double)maxDist) - { - failed = true; - break; - } - } - - if (!failed || cluster.size() == 0) - { - cluster.push_back(hulls[h]); - } - else - { - int k= findCenterPoint(contour, cluster); - if (k != -1) - { - tips.push_back(k); - } - cluster.clear(); - cluster.push_back(hulls[h]); - found = true; - } - } - if (cluster.size() > 0) - { - int k= findCenterPoint(contour, cluster); - if (k != -1) - { - tips.push_back(k); - } - cluster.clear(); - found = true; - } - } - - return found; -} - - -/*! - *=========================================================================================== - * @brief Find fingertips from k-Curvature - *=========================================================================================== - */ -void Jive::findKCurv(vector &contour, vector &hull, int kmin, int kmax, - double ang, vector &tips) -{ - int closest = 100000; - float closest_a = 10e10; - - tips.clear(); - for (int i=0; i < hull.size(); i++) { - int n = hull[i]; - cv::Point p = contour[n]; - bool done = false; - for (int k = 1; k < kmax && !done; k++) { - cv::Point p1 = (n-k < 0) ? contour[contour.size()+(n-k)] : contour[n-k]; - cv::Point p2 = (n+k >= contour.size()) ? contour[n+k-contour.size()] : contour[n+k]; - cv::Point v1 = cv::Point(p1.x-p.x, p1.y-p.y); - cv::Point v2 = cv::Point(p2.x-p.x, p2.y-p.y); - double dval = (float)(v1.x*v2.x + v1.y*v2.y) / (sqrt(v1.x*v1.x+v1.y*v1.y) * sqrt(v2.x*v2.x+v2.y*v2.y)); - double a = acos(dval)*180.0/3.1415926; - - if (p.y < closest) { - closest = n; - closest_a = a; - } - - if (a < ang && k >= kmin) - { - Scalar color = Scalar(0,255,00); - cv::line(_drawing, p, p1, color); - cv::line(_drawing, p, p2, color); - tips.push_back(n); - done = true; - } - - } - } -} + _leftHand = 0; + _rightHand = 1; - -/*! - *=========================================================================================== - * @brief Auto adjust pixel distance based on scaled image - *=========================================================================================== - */ -int Jive::adjPix(int pix) -{ - int out = pix*getDim().width/TOF_WIDTH; - return (out<=0)?1:out; -} - - -/*! - *=========================================================================================== - * @brief Try recognize gesture of qualified contour - *=========================================================================================== - */ -bool Jive::findGesture(vector< vector > &contours, enum Gesture &gesture, int *values) -{ - bool found = false; - int numHands = 0; - int handContour[2]; - - gesture = GESTURE_NULL; + _palmCenter[_leftHand] = cv::Point(0,0); + _palmCenter[_rightHand] = cv::Point(0,0); + _palmRadius[_leftHand] = 0; + _palmRadius[_rightHand] = 0; // Find number of qualified hands and remember their contour index + _numHands = 0; for (int i=0; i adjPix((int)_minContourSize)) + if (cv::contourArea(contours[i]) > (int)_minContourSize) { - if (numHands < 2) - handContour[numHands] = i; - numHands++; + if (_numHands < 2) + _handContour[_numHands] = i; + _numHands++; } } - - // Only process for 1 or 2 hands - if (numHands <= MAX_HANDS) + // Find palms + if (_numHands > 0 && _numHands <= 2) { - for (int i=0; i < numHands; i++) + for (int i=0; i < _numHands; i++) { // Find convex hull and defects vector hulls, defects; vector convDef; - vector contour = contours[handContour[i]]; - convexHull(Mat(contour), hulls, false); - convexityDefects(contour, hulls, convDef); - for (int k=1; k adjPix((int)_minConvDefDepth)*256) + vector contour = contours[_handContour[i]]; + + // Find contour center + cv::Point contourCenter; + findContourCenter(contour, contourCenter, _palmRadius[i]); + + // Find hand tip + int tip_k = 0; + int maxY = 0; + for (int k=0; k maxY) { - int ind = convDef[k][2]; - defects.push_back(ind); - } // if (convDef[k][3]) - } // for (k) - - // Remove 'border' hulls - vector newhulls; - for (int i=0; i< hulls.size(); i++) - { - cv::Point p = contour[hulls[i]]; - if (p.y > _Ymin+5 && p.y < _Ymax-5 && p.x < _Xmax-2 && p.x > _Xmin+2 ) - newhulls.push_back(hulls[i]); + maxY = contour[k].y; + tip_k = k; + } } - hulls = newhulls; - - // Find palms - findPalmCenter(contour, _palmCenter[i], _palmRadius[i]); - - // Find kCurv - vector tips_temp; - findKCurv(contour, hulls, adjPix(5), adjPix(8), _maxAngle, tips_temp); - - // Find tips - int rc = findTips(contour, tips_temp, _tips[i], (float)adjPix((int)_separation)); - - // Draw hand - cv::drawContours(_drawing, contours, handContour[i], Scalar(0, 0, 255), 0, 1, vector(), 0, cv::Point() ); + // Find palm Center + cv::Point tip = contour[tip_k]; + _palmCenter[i] = cv::Point((tip.x+contourCenter.x)/2, (tip.y+contourCenter.y)/2); + _handTip[i] = cv::Point((tip.x+_palmCenter[i].x)/2,(tip.y+_palmCenter[i].y)/2); - // Draw tips - for (int k=0; k<_tips[i].size(); k++) - cv::circle(_drawing, contour[_tips[i][k]], 4, Scalar(255,255,0), 1 ); - -#if 1 - // Draw defects - for (int k=0; k > &contours, enum Gesture &gest */ void Jive::update(Frame *frame) { - vector< vector > contours; vector hierarchy; RNG rng(12345); @@ -650,72 +456,26 @@ void Jive::update(Frame *frame) // Crop unused sections cropMaps(_bMap, (int)_Xmin, (int)_Xmax, (int)_Ymin, (int)_Ymax); - - cv::line(_drawing, cv::Point(_Xmin, _Ymin), cv::Point(_Xmin, _Ymax), Scalar(0,0,255), 1); - cv::line(_drawing, cv::Point(_Xmin, _Ymax), cv::Point(_Xmax, _Ymax), Scalar(0,0,255), 1); - cv::line(_drawing, cv::Point(_Xmax, _Ymax), cv::Point(_Xmax, _Ymin), Scalar(0,0,255), 1); - cv::line(_drawing, cv::Point(_Xmax, _Ymin), cv::Point(_Xmin, _Ymin), Scalar(0,0,255), 1); - Mat canny = _bMap.clone(); // Find all contours - findContours(canny, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, cv::Point(0,0)); - - // Find gestures - enum Gesture gesture; - int values[2]; - findGesture(contours, gesture, values); -#if 0 - // Dispatch gesture actions - switch (gesture) + findContours(canny, _contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, cv::Point(0,0)); + + // Find hand tips + findHandTips(_contours); + + // Move mouse + if (_numHands > 0) { - // One-hand - case GESTURE_1H1F: - cout << "1H1F" << " POS:" << value[0] << endl; - break; - - case GESTURE_1H2F: - cout << "1H2F" << " POS:" << value[0] << endl; - break; - - case GESTURE_1H3F: - cout << "1H3F" << " POS:" << value[0] << endl; - break; - - case GESTURE_1H4F: - cout << "1H4F" << " POS:" << value[0] << endl; - break; - - case GESTURE_1H5F: - cout << "1H5F" << " POS:" << value[0] << endl; - break; - - // Two-hands - case GESTURE_2H1F: - cout << "2H1F" << " POS:" << value[0] << " ROT:" << value[1] << endl; - break; - - case GESTURE_2H2F: - cout << "2H2F" << " POS:" << value[0] << " ROT:" << value[1] << endl; - break; - - case GESTURE_2H3F: - cout << "2H3F" << " POS:" << value[0] << " ROT:" << value[1] << endl; - break; - - case GESTURE_2H4F: - cout << "2H4F" << " POS:" << value[0] << " ROT:" << value[1] << endl; - break; - - case GESTURE_2H5F: - cout << "2H5F" << " POS:" << value[0] << " ROT:" << value[1] << endl; - break; - - // Default - default: - break; - } // switch() -#endif + int screen_width, screen_height; + int sensor_width, sensor_height; + int scaled_x, scaled_y; + + mouse.getDim(screen_width, screen_height); + scaled_x = screen_width - screen_width * (_handTip[0].x-(int)_Xmin) / (int)(_Xmax-_Xmin); + scaled_y = screen_height - screen_height * (_handTip[0].y-(int)_Ymin) / (int)(_Ymax-_Ymin); + mouse.moveTo(scaled_x-(int)_Xcur, scaled_y-(int)_Ycur); + } } // if (_bkgUpdated) diff --git a/TinTin/touchless_tracking/Jive.h b/TinTin/touchless_tracking/Jive.h index efc156e..621254d 100644 --- a/TinTin/touchless_tracking/Jive.h +++ b/TinTin/touchless_tracking/Jive.h @@ -20,6 +20,7 @@ #include #include #include +#include "FakeMouse.h" #ifndef __JIVE_H__ #define __JIVE_H__ @@ -28,22 +29,6 @@ class Jive : public TOFApp { -public: - enum Gesture - { - GESTURE_NULL = 0, - GESTURE_1H1F, - GESTURE_1H2F, - GESTURE_1H3F, - GESTURE_1H4F, - GESTURE_1H5F, - GESTURE_2H1F, - GESTURE_2H2F, - GESTURE_2H3F, - GESTURE_2H4F, - GESTURE_2H5F, - }; - public: Jive(int w, int h); void update(Frame *frm); @@ -55,17 +40,12 @@ class Jive : public TOFApp void sampleBackground(); private: - Mat _aMap, _aBkgMap, _aFgMap, _aHeatMap, _aDiffMap, _aPrevMap; - Mat _zMap, _zBkgMap, _zFgMap, _zHeatMap, _zDiffMap, _zPrevMap; + Mat _aMap, _aBkgMap, _aFgMap, _aPrevMap; + Mat _zMap, _zBkgMap, _zFgMap, _zPrevMap; Mat _bMap, _drawing; bool _bkgUpdated; XYZIPointCloudFrame _prevFrame; - float _aHeatMapThresh; - float _zHeatMapThresh; - float _aDiffMapThresh; - float _zDiffMapThresh; - float _heatMapCoef; float _zTrigger; float _zHighThresh; float _zLowThresh; @@ -74,13 +54,14 @@ class Jive : public TOFApp float _minConvDefDepth; float _Xmin, _Xmax; float _Ymin, _Ymax; - cv::Point _palmCenter[MAX_HANDS]; - float _palmRadius[MAX_HANDS]; - vector _tips[MAX_HANDS]; - float _maxAngle; - int _movementCount; - float _separation; - + float _Xcur, _Ycur; + cv::Point _palmCenter[2]; + float _palmRadius[2]; + cv::Point _handTip[2]; + int _leftHand, _rightHand; + int _numHands; + int _handContour[2]; + vector< vector > _contours; // Parameter map: map< std::string, std::tuple > _param; @@ -90,20 +71,19 @@ class Jive : public TOFApp // Display and controls vector _sliderPos; + // Mouse + FakeMouse mouse; + private: void findForeground(float zLowThr, float aHighThr, Mat &fgMap); void updateMaps(Frame *frame); void morphClean(Mat &in, Mat &out); void initControls(); void displayMaps(); - bool noMovement(int t); void cropMaps(Mat &m, int xmin, int xmax, int ymin, int ymax); - bool findGesture(vector< vector > &contours, enum Gesture &gesture, int *value); - bool findPalmCenter(vector &contour, cv::Point ¢er, float &radius); - void findKCurv(vector &contour, vector &hull, int kmin, int kmax, double ang, vector &tips); - bool findTips(vector &contour, vector &hulls, vector &tips, float maxDist); - int findCenterPoint(vector &contour, vector &cluster); - int adjPix(int pix); + bool findContourCenter(vector &contour, cv::Point ¢er, float &radius); + bool findHandTips(vector< vector > &contours); + }; #endif // __JIVE_H__ diff --git a/TinTin/touchless_tracking/TOFApp.cpp b/TinTin/touchless_tracking/TOFApp.cpp index 9baf921..0eb9eb8 100644 --- a/TinTin/touchless_tracking/TOFApp.cpp +++ b/TinTin/touchless_tracking/TOFApp.cpp @@ -19,9 +19,9 @@ #include "TOFApp.h" -#define FRAME_QUEUE_SZ 3 +#define FRAME_QUEUE_SZ 3 #define DEFAULT_ILLUM_POWER 100 -#define DEFAULT_EXPOSURE 20 +#define DEFAULT_EXPOSURE 20 //============================================================================= @@ -41,7 +41,7 @@ static void frameCallback(DepthCamera &dc, const Frame &frame, DepthCamera::Fram qFrame.push_back(nf); } else if (c == DepthCamera::FRAME_XYZI_POINT_CLOUD_FRAME) { - const Voxel::XYZIPointCloudFrame *f = dynamic_cast(&frame); + const Voxel::XYZIPointCloudFrame *f = dynamic_cast(&frame); Voxel::Frame *nf = dynamic_cast(new Voxel::XYZIPointCloudFrame(*f)); qFrame.push_back(nf); } @@ -212,7 +212,30 @@ bool TOFApp::connect(DepthCamera::FrameType frmType) } else return false; - + + // setup spatial and temportal median filters + FilterPtr p = _sys.createFilter("Voxel::MedianFilter", DepthCamera::FRAME_RAW_FRAME_PROCESSED); + if (!p) + { + logger(LOG_ERROR) << "Failed to get MedianFilter" << std::endl; + return -1; + } + + _depthCamera->addFilter(p, DepthCamera::FRAME_RAW_FRAME_PROCESSED); + +#if 1 + p = _sys.createFilter("Voxel::TemporalMedianFilter", DepthCamera::FRAME_RAW_FRAME_PROCESSED); + if (!p) + { + logger(LOG_ERROR) << "Failed to get TemporalMedianFilter" << std::endl; + return -1; + } + + p->set("deadband", 0.01f); + _depthCamera->addFilter(p, DepthCamera::FRAME_RAW_FRAME_PROCESSED); +#endif + + // Setup frame callback profile and settings _frameType = frmType; _depthCamera->registerCallback(_frameType, frameCallback); _depthCamera->setFrameSize(_dimen); @@ -220,28 +243,10 @@ bool TOFApp::connect(DepthCamera::FrameType frmType) cout << "Profile " << _profile << " found." << endl; else cout << "Profile " << _profile << "not found." << endl; - setIllumPower(DEFAULT_ILLUM_POWER); setExposure(DEFAULT_EXPOSURE); -#if 0 - FilterPtr p = _sys.createFilter("Voxel::MedianFilter", DepthCamera::FRAME_RAW_FRAME_PROCESSED); - if (!p) { - logger(LOG_ERROR) << "Failed to get MedianFilter" << std::endl; - return false; - } - //p->set("deadband", 0.1f); - _depthCamera->addFilter(p, DepthCamera::FRAME_RAW_FRAME_PROCESSED, 0); - - p = _sys.createFilter("Voxel::TemporalMedianFilter", DepthCamera::FRAME_RAW_FRAME_PROCESSED); - if (!p) { - logger(LOG_ERROR) << "Failed to get TemporalMedianFilter" << std::endl; - return false; - } - //p->set("deadband", 0.01f); - _depthCamera->addFilter(p, DepthCamera::FRAME_RAW_FRAME_PROCESSED, 0); -#endif - + // Start the camera _depthCamera->start(); _isConnected = true; diff --git a/TinTin/touchless_tracking/touchless.cpp b/TinTin/touchless_tracking/touchless.cpp index ad47ddb..ce328cc 100644 --- a/TinTin/touchless_tracking/touchless.cpp +++ b/TinTin/touchless_tracking/touchless.cpp @@ -74,11 +74,6 @@ int main(int argc, char *argv[]) return -1; } - // eye.addMapToDisplay("aMap"); - // eye.addMapToDisplay("zMap"); - // eye.addMapToDisplay("bMap"); - // eye.addMapToDisplay("zFgMap"); - // eye.addMapToDisplay("aBkgMap"); eye.addMapToDisplay("zBkgMap"); eye.addMapToDisplay("drawing");