Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Apriori Finite State Machine: Gesture Recognition

  • Loading branch information...
commit f0c51c58f0f2908f7a3aaf825f917d9f530887cb 1 parent 882975b
iasoule authored

Showing 1 changed file with 419 additions and 0 deletions. Show diff stats Hide diff stats

  1. +419 0 fsm.h
419 fsm.h
... ... @@ -0,0 +1,419 @@
  1 +/* Finite State Machine Implementation
  2 + Emitting mouse events to the windows event queue
  3 +
  4 + Idris Soule
  5 +*/
  6 +
  7 +#include <stdlib.h>
  8 +#include <assert.h>
  9 +#include <time.h>
  10 +#include <errno.h>
  11 +#include <windows.h>
  12 +#include <pthread.h>
  13 +
  14 +#pragma warning(disable:4716) //disable missing return from function error
  15 +
  16 +#define MAX_QUEUE_ENTRY 256
  17 +#define NUM_STATES 4
  18 +
  19 +extern POINT cursor;
  20 +
  21 +typedef enum {INITIAL = 0x0A, TIMING, ACCEPTING ,FINAL} stateType_t;
  22 +typedef enum {LEFT = 0x01, RIGHT, ZOOM, TRACK, QUIT, NOP} stateEvent_t ;
  23 +typedef enum {TIMER_EXPIRED, TIMER_ALIVE, TIMER_RESET} timerState_t;
  24 +
  25 +typedef struct FSMState_t {
  26 + stateType_t stateType;
  27 + stateEvent_t sEvent, prevEvent;
  28 + void * (*emit)(void *);
  29 + /* payload for mouse events */
  30 + INPUT mouseEvents[5];
  31 + int numEvents;
  32 +}FSMState_t;
  33 +
  34 +static FSMState_t currState;
  35 +static timerState_t clickTimer; //timer between successive clicks
  36 +
  37 +static pthread_mutex_t eventLock = PTHREAD_ERRORCHECK_MUTEX_INITIALIZER;
  38 +static pthread_mutex_t stateLock = PTHREAD_ERRORCHECK_MUTEX_INITIALIZER;
  39 +static pthread_mutex_t timerLock = PTHREAD_ERRORCHECK_MUTEX_INITIALIZER;
  40 +static pthread_mutex_t queueLock = PTHREAD_ERRORCHECK_MUTEX_INITIALIZER;
  41 +
  42 +static pthread_t fsmThread;
  43 +static pthread_cond_t timerCond = PTHREAD_COND_INITIALIZER;
  44 +
  45 +/* Bounded buffer for detector to emit posture */
  46 +static stateEvent_t fsm_event_queue[MAX_QUEUE_ENTRY];
  47 +static int *queueHeadr, *queueTail;
  48 +
  49 +
  50 +
  51 +/* timer_thread:
  52 + One-shot timer which allows the machine to distinguish between
  53 + a single-click and a double-click, 475 ms is the interval time to re-click
  54 +*/
  55 +static void * timer_thread(void *arg)
  56 +{
  57 + struct timespec tm;
  58 + tm.tv_sec = (long)time(0) + 2L;
  59 + tm.tv_nsec = 0; //475 ms
  60 +
  61 + /*
  62 + Single Click constitutes: Timer Expired => Timer beats user to the punch
  63 + Double Click constitutes: Timer Reset => User beats timer to the punch
  64 + */
  65 + pthread_mutex_lock(&timerLock);
  66 + clickTimer = TIMER_ALIVE;
  67 + int rt = 0;
  68 +
  69 + //TODO: prevent spurious wake ups (small amount of jitter)
  70 + rt = pthread_cond_timedwait(&timerCond, &timerLock, &tm);
  71 +
  72 + /* condition not signalled in correct amount of time */
  73 + if(rt == ETIMEDOUT) //mutex has been re-acquired
  74 + clickTimer = TIMER_EXPIRED;
  75 + else
  76 + clickTimer = TIMER_RESET ;//forced reset t < 475 ms
  77 +
  78 + pthread_mutex_unlock(&timerLock);
  79 + pthread_exit(NULL);
  80 +}
  81 +
  82 +/* functions for triggered events */
  83 +
  84 +static void * event_left_click(void *payload)
  85 +{
  86 + stateEvent_t eventPrev;
  87 + pthread_t timerThread;
  88 +
  89 + /* Three Cases
  90 + 1. Starting a timer (single click or double ?)
  91 + 2. Setting up payload for event_track to fire
  92 + 3. Drag event
  93 + */
  94 +
  95 + pthread_mutex_lock(&stateLock);
  96 + eventPrev = currState.prevEvent;
  97 + pthread_mutex_unlock(&stateLock);
  98 +
  99 + switch(eventPrev){
  100 + case TRACK:
  101 + /* assume single-click */
  102 + if(clickTimer == TIMER_RESET){ //start the timer thread
  103 + pthread_create(&timerThread, NULL, timer_thread, NULL);
  104 + }
  105 + else if(clickTimer == TIMER_ALIVE){ //set double click payload + kill timer
  106 + pthread_cond_signal(&timerCond);
  107 + pthread_mutex_lock(&stateLock);
  108 +
  109 + static DWORD mouseEvents[4] ={MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP,
  110 + MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP};
  111 + ZeroMemory(currState.mouseEvents, sizeof(currState.mouseEvents) * 5);
  112 + currState.numEvents = 4;
  113 + for(int i = 0; i < currState.numEvents; i++){
  114 + currState.mouseEvents[i].type = INPUT_MOUSE;
  115 + currState.mouseEvents[i].mi.dwFlags = mouseEvents[i];
  116 + }
  117 + pthread_mutex_unlock(&stateLock);
  118 + clickTimer = TIMER_RESET;
  119 + }
  120 + else { //clickTimer == TIMER_EXPIRED ... single click payload
  121 +
  122 + pthread_mutex_lock(&stateLock);
  123 + ZeroMemory(currState.mouseEvents, sizeof(currState.mouseEvents) * 5);
  124 + currState.mouseEvents[0].type = INPUT_MOUSE;
  125 + currState.mouseEvents[0].mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
  126 + currState.mouseEvents[1].type = INPUT_MOUSE;
  127 + currState.mouseEvents[1].mi.dwFlags = MOUSEEVENTF_LEFTUP;
  128 + currState.numEvents = 2;
  129 + pthread_mutex_unlock(&stateLock);
  130 + clickTimer = TIMER_RESET; //mutex not needed
  131 + }
  132 + break;
  133 +
  134 + case LEFT:
  135 + //Kalman Filter(..) drag event
  136 + break;
  137 +
  138 + }
  139 + pthread_mutex_lock(&stateLock);
  140 + currState.prevEvent = LEFT;
  141 + pthread_mutex_unlock(&stateLock);
  142 + pthread_exit(NULL);
  143 +}
  144 +
  145 +
  146 +static void * event_right_click(void *payload)
  147 +{
  148 + stateEvent_t eventPrev;
  149 +
  150 + pthread_mutex_lock(&stateLock);
  151 + eventPrev = currState.prevEvent;
  152 + pthread_mutex_unlock(&stateLock);
  153 +
  154 + /* only care about Track, Left previous state don't care about RIGHT*/
  155 + switch(eventPrev){
  156 + case TRACK:
  157 + /* setup RIGHT single-click payload */
  158 + pthread_mutex_lock(&stateLock);
  159 + ZeroMemory(currState.mouseEvents, sizeof(currState.mouseEvents) * 5);
  160 + currState.mouseEvents[0].type = INPUT_MOUSE;
  161 + currState.mouseEvents[0].mi.dwFlags = MOUSEEVENTF_RIGHTDOWN;
  162 + currState.mouseEvents[1].type = INPUT_MOUSE;
  163 + currState.mouseEvents[1].mi.dwFlags = MOUSEEVENTF_RIGHTUP;
  164 + currState.numEvents = 2;
  165 + pthread_mutex_unlock(&stateLock);
  166 + break;
  167 +
  168 + case LEFT:
  169 + /* Special case T L R T
  170 + Needs to set two payloads
  171 + */
  172 + pthread_mutex_lock(&stateLock);
  173 + static DWORD mouseEvents[4] ={MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP,
  174 + MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP};
  175 + ZeroMemory(currState.mouseEvents, sizeof(currState.mouseEvents) * 5);
  176 + currState.numEvents = 4;
  177 + for(int i = 0; i < currState.numEvents; i++){
  178 + currState.mouseEvents[i].type = INPUT_MOUSE;
  179 + currState.mouseEvents[i].mi.dwFlags = mouseEvents[i];
  180 + }
  181 + pthread_mutex_unlock(&stateLock);
  182 + break;
  183 +
  184 + }
  185 +
  186 + pthread_mutex_lock(&stateLock);
  187 + currState.prevEvent = RIGHT;
  188 + pthread_mutex_unlock(&stateLock);
  189 + pthread_exit(NULL);
  190 +}
  191 +
  192 +static void * event_zoom(void *payload)
  193 +{
  194 +
  195 +
  196 +
  197 +
  198 + pthread_mutex_lock(&stateLock);
  199 + currState.prevEvent = ZOOM;
  200 + pthread_mutex_unlock(&stateLock);
  201 + pthread_exit(NULL);
  202 +}
  203 +
  204 +/* event_track:
  205 + The initial position, final and tracking (x,y-coord) state
  206 + Takes care of executing other events based on the states from the queue
  207 + State information is read to determine previous state.
  208 + Based on state respective thread is fired and mouse event in that thread is fired.
  209 +*/
  210 +static void * event_track(void *arg)
  211 +{
  212 +
  213 + stateEvent_t eventPrev;
  214 + /* Two cases to worry about
  215 + 1. Moving the cursor
  216 + 2. Firing an event from the previous state (L, R, Z)
  217 + */
  218 +
  219 + pthread_mutex_lock(&stateLock);
  220 + eventPrev = currState.prevEvent;
  221 + pthread_mutex_unlock(&stateLock);
  222 +
  223 + switch(eventPrev){
  224 + case TRACK:
  225 + /* get current cursor position (kalman stuff) and setcursorpos */
  226 + SetCursorPos((cursor.x - 40 + 1)*8.95, (cursor.y - 63 + 1)*7.7);
  227 + break;
  228 +
  229 + case LEFT:
  230 + while(clickTimer == TIMER_ALIVE)
  231 + ; //wait at most 475ms
  232 + /* now either single/double click is ready */
  233 + pthread_mutex_lock(&stateLock);
  234 + SendInput(currState.numEvents, currState.mouseEvents, sizeof(INPUT));
  235 + pthread_mutex_unlock(&stateLock);
  236 + break;
  237 +
  238 + case RIGHT:
  239 + /* Fire right click or left-right click */
  240 + pthread_mutex_lock(&stateLock);
  241 + UINT sendresult;
  242 + sendresult = SendInput(currState.numEvents, currState.mouseEvents, sizeof(INPUT));
  243 + assert(sendresult == currState.numEvents);
  244 + pthread_mutex_unlock(&stateLock);
  245 + break;
  246 +
  247 + case ZOOM:
  248 + /* fire zoom payload */
  249 + break;
  250 + default: // internal error
  251 + break;
  252 + }
  253 +
  254 + pthread_mutex_lock(&stateLock);
  255 + currState.prevEvent = TRACK;
  256 + pthread_mutex_unlock(&stateLock);
  257 + pthread_exit(NULL);
  258 +}
  259 +
  260 +/*
  261 +fsm_queue_emit:
  262 + The given camera thread will call this function
  263 + to emit the detected posture. Function shall write to
  264 + the non-full queue.
  265 +
  266 + @id: one of the enumerated states {RIGHT, ZOOM ...}
  267 + @return: notification if the state could be written to the queue
  268 +
  269 + */
  270 +bool fsm_queue_emit(stateEvent_t id)
  271 +{
  272 + if(queueTail >= queueHeadr + MAX_QUEUE_ENTRY)
  273 + return false; //full queue
  274 +
  275 + pthread_mutex_lock(&queueLock);
  276 + *queueTail = id;
  277 + queueTail++; //one beyond the latest entry
  278 + pthread_mutex_unlock(&queueLock);
  279 + return true;
  280 +}
  281 +
  282 +/*
  283 +fsm_queue_consume:
  284 + Consumes the states from the non-empty buffer
  285 +
  286 + @return: event ( of posture)
  287 +*/
  288 +static stateEvent_t fsm_queue_consume(void)
  289 +{
  290 + static int i;
  291 + stateEvent_t state;
  292 + if(queueHeadr == queueTail){ //empty
  293 + i = 0;
  294 + return NOP; //triggers default in fsm_execute
  295 + }
  296 +
  297 + assert(queueHeadr == (int *)&fsm_event_queue);
  298 + state = (stateEvent_t)queueHeadr[i++];
  299 + if(queueHeadr + i == queueTail){
  300 + pthread_mutex_lock(&queueLock);
  301 + queueTail = queueHeadr;
  302 + i = 0;
  303 + pthread_mutex_unlock(&queueLock);
  304 + }
  305 + return state;
  306 +}
  307 +
  308 +/*
  309 +fsm_execute:
  310 + The State Machine
  311 + Once an event is consumed from the queue it is classified.
  312 + State information is attributed to the event and a thread is created to execute
  313 + its functionality
  314 +
  315 + Fine grain granularity is emplored with the locks to keep the event-loop as live as possible.
  316 +*/
  317 +
  318 +
  319 +static void fsm_execute(void)
  320 +{
  321 + stateEvent_t sid;
  322 + pthread_t *trackThread;
  323 + pthread_t eventThread[NUM_STATES];
  324 +
  325 + while(sid = fsm_queue_consume())
  326 + {
  327 + switch(sid){
  328 + case LEFT:
  329 + pthread_mutex_lock(&stateLock);
  330 + currState.emit = event_left_click;
  331 + currState.stateType = TIMING;
  332 + currState.sEvent = LEFT;
  333 + pthread_mutex_unlock(&stateLock);
  334 + trackThread = &eventThread[0];
  335 + break;
  336 +
  337 + case RIGHT:
  338 + pthread_mutex_lock(&stateLock);
  339 + currState.emit = event_right_click;
  340 + currState.stateType = ACCEPTING;
  341 + currState.sEvent = RIGHT;
  342 + pthread_mutex_unlock(&stateLock);
  343 + trackThread = &eventThread[1];
  344 + break;
  345 +
  346 + case ZOOM:
  347 + pthread_mutex_lock(&stateLock);
  348 + currState.emit = event_zoom;
  349 + currState.stateType = ACCEPTING;
  350 + currState.sEvent = ZOOM;
  351 + pthread_mutex_unlock(&stateLock);
  352 + trackThread = &eventThread[2];
  353 + break;
  354 +
  355 + case TRACK:
  356 + pthread_mutex_lock(&stateLock);
  357 + currState.emit = event_track;
  358 + currState.stateType = INITIAL;
  359 + currState.sEvent = TRACK;
  360 + pthread_mutex_unlock(&stateLock);
  361 + trackThread = &eventThread[3];
  362 + break;
  363 +
  364 + case QUIT:
  365 + pthread_cancel(*trackThread);
  366 + pthread_cond_destroy(&timerCond);
  367 + pthread_mutex_destroy(&eventLock);
  368 + pthread_mutex_destroy(&stateLock);
  369 + pthread_mutex_destroy(&timerLock);
  370 + pthread_mutex_destroy(&queueLock);
  371 + pthread_exit(NULL);
  372 + break;
  373 +
  374 + /* Illegal event/No event (empty buffer)*/
  375 + default:
  376 + continue; //ignore it
  377 + break;
  378 + }
  379 + /* Launch Track thread */
  380 + /* NOTE:: Possibility to pass blob analysis as parameter to thread */
  381 + if(pthread_create(trackThread, NULL, currState.emit, (void *)sid)){
  382 + perror("fsm_execute:: couldn't create thread!");
  383 + pthread_exit(NULL);
  384 + }
  385 + //pthread_join(*trackThread, NULL);
  386 + }
  387 +}
  388 +
  389 +/*
  390 +fsm_initialize:
  391 + Initializes the state machine.
  392 + Caller will put the machine in a given state
  393 + Once called, threads are created for fsm_execute
  394 +
  395 + @initial: initial state of FSM
  396 + @return: status of initialization
  397 +*/
  398 +
  399 +bool fsm_initialize(stateEvent_t initial)
  400 +{
  401 + currState.prevEvent = TRACK;
  402 + currState.sEvent = TRACK;
  403 + currState.stateType = INITIAL;
  404 + currState.emit = event_track;
  405 + queueHeadr = queueTail = (int *)&fsm_event_queue;
  406 +
  407 + clickTimer = TIMER_RESET;
  408 + SetDoubleClickTime(2000);
  409 +
  410 + const char *fsmErr = "FSM::%s: Couldn't create %s-thread!";
  411 + if(pthread_create(&fsmThread, NULL,
  412 + (void * (*)(void *))fsm_execute,
  413 + NULL)){
  414 + printf(fsmErr, __FUNCTION__, "fsm");
  415 + return false;
  416 + }
  417 +
  418 +return true;
  419 +}

0 comments on commit f0c51c5

Please sign in to comment.
Something went wrong with that request. Please try again.