diff --git a/.github/workflows/cmake_build.yml b/.github/workflows/cmake_build.yml index d1fe2a3..8426e96 100644 --- a/.github/workflows/cmake_build.yml +++ b/.github/workflows/cmake_build.yml @@ -2,9 +2,9 @@ name: Build and Test on: push: - branches: [ dev ] + branches: [ dev main docs-update] pull_request: - branches: [ dev ] + branches: [ dev main] env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index adf4491..a53743e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,9 +6,9 @@ name: Docs # events but only for the master branch on: push: - branches: [ dev ] + branches: [ dev docs-update] pull_request: - branches: [ dev ] + branches: [ dev main] diff --git a/dconfig b/dconfig index 2ec017d..e411765 100644 --- a/dconfig +++ b/dconfig @@ -963,7 +963,7 @@ RECURSIVE = NO # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = +EXCLUDE = src/MultiThreadedSchedulableLink.* src/BatchCNNProcessor.* src/NThreadedCNNProcessor.* # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded diff --git a/src/BlockingQueue.cpp b/src/BlockingQueue.cpp index 1639d03..a7c3f64 100644 --- a/src/BlockingQueue.cpp +++ b/src/BlockingQueue.cpp @@ -2,7 +2,11 @@ // Blocking queue implementation, currently only blocks when Get is attempted while there are no elements inside the queue. In the future and max size could be defined and blocking could be added when the queue is full // #include "BlockingQueue.h" - +/*! + * Pop method, uses mutex lock to sleep the current thread until data are available on the queue. This may be used to synchonise and schedule threads. + * @tparam T + * @return Element on deque back + */ template T BlockingQueue::Pop() { std::unique_lock lock(mutex); @@ -11,17 +15,31 @@ T BlockingQueue::Pop() { internalQueue.pop_back(); return ret; } - +/*! + * + * @tparam T + * @return true if internal queue empty, false otherwise + */ template bool BlockingQueue::IsEmpty(){ return internalQueue.empty(); } +/*! + * Gets the current length of elements in the queue. + * @tparam T + * @return size of internal deque + */ template int BlockingQueue::Size(){ return internalQueue.size(); } +/*! + * adds element to the queue. + * @tparam T + * @param toPush + */ template void BlockingQueue::Push(T toPush) { { diff --git a/src/BlockingQueue.h b/src/BlockingQueue.h index fdffd1e..d7005a6 100644 --- a/src/BlockingQueue.h +++ b/src/BlockingQueue.h @@ -15,6 +15,11 @@ template /*! */ + +/*! + * Class to wrap around std::deque and block thread execution when no data is available at the output. + * @tparam T Type of elements in the queue + */ class BlockingQueue { public: void Push(T toPush); diff --git a/src/CNNProcessor.cpp b/src/CNNProcessor.cpp index 72e3467..4bd770b 100644 --- a/src/CNNProcessor.cpp +++ b/src/CNNProcessor.cpp @@ -1,5 +1,9 @@ #include "CNNProcessor.h" +/*! + * Loads neural network from location on disk + * @param modelPath + */ void CNNProcessor::LoadModel(std::string modelPath){ net = cv::dnn::readNetFromTensorflow(modelPath); } @@ -9,6 +13,11 @@ CNNProcessor::CNNProcessor(CNNProcessorSettings* s) { LoadModel(settings->ModelPath); } +/*! + * Makes OpenCV image "blob" from the region of interest in the frame. Blob format is required for inference. + * @param scene + * @return matrix image blob, ready for inference + */ cv::Mat CNNProcessor::MakeBlob(Scene scene){ int x = scene.regionOfInterest.UpperLeft.x; int y = scene.regionOfInterest.UpperLeft.y; int width = scene.regionOfInterest.LowerRight.x - x; @@ -21,6 +30,11 @@ cv::Mat CNNProcessor::MakeBlob(Scene scene){ return blob; } +/*! + * Executed network inference on given scene, populates the scene with results. + * @param scene input scene + * @return output processed scene + */ Scene CNNProcessor::ProcessScene(Scene scene){ if(scene.frame.empty()) return scene; cv::Mat blob = MakeBlob(scene); @@ -34,11 +48,4 @@ Scene CNNProcessor::ProcessScene(Scene scene){ return scene; } -//void CNNProcessor::NextScene(Scene next) { -// Scene updatedFrame = ProcessScene(next); -// if(!sceneCallback) return; -// sceneCallback->NextScene(updatedFrame); -//} - - diff --git a/src/CNNProcessor.h b/src/CNNProcessor.h index 97bc25f..cc92c5a 100644 --- a/src/CNNProcessor.h +++ b/src/CNNProcessor.h @@ -14,10 +14,9 @@ #include "SignapseUtils.h" -//! CNNProcessor class. /*! - Class for interfacing with convolutional neural network -*/ + * Schedulable Pipeline Link which performs network inference on each Scene, populating the results field. + */ class CNNProcessor : public SchedulableLink{ public: Scene ProcessScene(Scene s); diff --git a/src/CNNProcessorSettings.h b/src/CNNProcessorSettings.h index 5466389..da56e32 100644 --- a/src/CNNProcessorSettings.h +++ b/src/CNNProcessorSettings.h @@ -5,6 +5,9 @@ #define MOBNET_V2_INPUT_DIM_Y 224 #define MOBNET_V2_PATH "models/asl-mobilenetv2.pb" +/*! + * A simple struct to store default configurations for CNNProcessor settings. + */ struct CNNProcessorSettings { CNNProcessorSettings(std::string network = "mobnetv2"){ if(network == "mobnetv2") { diff --git a/src/Camera.cpp b/src/Camera.cpp index dc6e877..0a65912 100644 --- a/src/Camera.cpp +++ b/src/Camera.cpp @@ -1,19 +1,32 @@ #include "Camera.h" +/*! + * Camera constructor, sets isOn to true. + */ Camera::Camera() { isOn=true; } +/*! + * Sets the isOn state of the camera + * @param state + */ void Camera::setOn(bool state){ isOn = state; } +/*! + * Loops while camera is on to add frames to the pipeline + */ void Camera::threadLoop(){ while(isOn){ postFrame(); } } +/*! + * Gets the next available frame and passes it on to the registered callback. Relies on the videoCapture.read() OpenCV method which is understood to wait for an intra-frame delay. + */ void Camera::postFrame(){ if(!sceneCallback) return; cv::Mat cap; @@ -28,17 +41,27 @@ void Camera::postFrame(){ sceneCallback->NextScene(s); } +/*! + * Starts the worker thread recording + */ void Camera::Start(){ videoCapture.open(deviceID, apiID); cameraThread = std::thread(&Camera::threadLoop, this); } +/*! + * Frees thread resources and stops recording, must be called prior to program exit. + */ void Camera::Stop(){ isOn=false; cameraThread.join(); } +/*! + * Returns the isON state of the camera + * @return true if on; false otherwise + */ bool Camera::getOn() { return isOn; } diff --git a/src/Camera.h b/src/Camera.h index cd2ec7f..bdb8579 100644 --- a/src/Camera.h +++ b/src/Camera.h @@ -8,46 +8,22 @@ #include "PipelineLink.h" -//! Camera class which inherits from Reel. /*! - Class for reading web cam data, and creating a Reel of scenes. -*/ + * Simple Pipeline element which integrates frame acquisition from the camera feed. + */ class Camera: public PipelineLink{ public: - //! Constructor. - /*! - Turns the camera object "ON" and configures the video capture. - */ Camera(); ~Camera(); bool getOn(); - void setOn(bool state); - - //! Member function for starting video capturing thread. - /*! - Starts "Stream" private member function as a thread. - */ void Start(); void Stop(); - - private: void postFrame(); - void threadLoop(); - - //! Private member object. - /*! - OpenCV object for reading web cam data. - */ cv::VideoCapture videoCapture; - //! Private member variable containing web cam device ID. - /*! - 0 = open default camera - - */ int deviceID = 0; //! Private member variable containing web cam API ID. /*! diff --git a/src/Gui.cpp b/src/Gui.cpp index ffd1873..eef57c1 100644 --- a/src/Gui.cpp +++ b/src/Gui.cpp @@ -3,6 +3,11 @@ #define UI_WDH 1000 #define UI_HGT 700 +/*! + * Constructor sets up GUI, makes signal connections + * @param win points to QMainWindow from main thread + * @param ui_win points to Ui_MainWindow from main thread + */ Gui::Gui(QMainWindow* win, Ui_MainWindow* ui_win) { widget = win; widget->setFixedSize(UI_WDH, UI_HGT); @@ -11,7 +16,10 @@ Gui::Gui(QMainWindow* win, Ui_MainWindow* ui_win) { SetTask("A"); makeConnections(); } - +/*! + * Handles the next scene from the video processing pipeline. If the result is empty, the viewing pane is updated, otherwise, the progress bar is updated. + * @param next The Scene to be processed + */ void Gui::NextScene(Scene next) { //flip frame if(next.result == "") { @@ -28,30 +36,36 @@ void Gui::NextScene(Scene next) { } } - +/*! + * Sets underlying widget visibility. + * @param visible + */ void Gui::SetVisible(bool visible) { widget->setVisible(visible); } - -void Gui::SetTargetImage(int target) { - std::string letter = SignapseUtils::getLetterFromDigit(target); - SetTargetImage(letter); - progressChanged(0); -} - +/*! + * Sets the target to match with user sign + * @param letter string represenation of the target sign + */ void Gui::SetTargetImage(std::string letter) { std::string impath = testFolder + letter + "_test.jpg"; cv::Mat img = cv::imread(impath); setDemoImage(img); setTaskText(letter); } +/*! + * Method handler for when the next task button is pressed, sets new task and resets the progress bar. + */ void Gui::ButtonPressed(){ - std::string new_task = SignapseUtils::getLetterFromDigit(SignapseUtils::makeTask()); + std::string new_task = SignapseUtils::makeTask(); SetTargetImage(new_task); currentTask = new_task; progress_bar.reset_progress(); } +/*! + * Makes signal connections for the GUI interrupts. + */ void Gui::makeConnections() { QObject::connect(ui->pushButton, &QPushButton::released, this, &Gui::ButtonPressed); QObject::connect(this, &Gui::progressChanged, ui->progressBar, &QProgressBar::setValue); diff --git a/src/Gui.h b/src/Gui.h index 7b88774..211666c 100644 --- a/src/Gui.h +++ b/src/Gui.h @@ -16,7 +16,8 @@ #define testFolder "test/asl_alphabet_test/" /* - * A class to wrap the QT generated header file and handle GUI functionality + * A class to wrap the QT generated header file and handle GUI functionality. + * Inherits from QWidget and SceneCallback */ class Gui : public QWidget, public SceneCallback{ Q_OBJECT diff --git a/src/LinkSplitter.cpp b/src/LinkSplitter.cpp index ba05913..c50ac3e 100644 --- a/src/LinkSplitter.cpp +++ b/src/LinkSplitter.cpp @@ -1,11 +1,17 @@ #include "LinkSplitter.h" - +/*! + * Override of NextScene to pass scene reference to a maximum of two registered callbacks + * @param s scene + */ void LinkSplitter::NextScene(Scene s) { if(sceneCallback) sceneCallback->NextScene(s); if(secondarySceneCallback) secondarySceneCallback->NextScene(s); } - +/*! + * Used to register a secondary callback. + * @param scb callback to register. + */ void LinkSplitter::RegisterSecondaryCallback(SceneCallback *scb) { secondarySceneCallback = scb; } \ No newline at end of file diff --git a/src/LinkSplitter.h b/src/LinkSplitter.h index 44d935f..070b6d2 100644 --- a/src/LinkSplitter.h +++ b/src/LinkSplitter.h @@ -3,6 +3,10 @@ #include "PipelineLink.h" +/*! + * A pipeline element which extends PipelineLink to add a secondary callback. The NextScene function is overridden to pass on the scene reference to both registered callbacks. + * N.B. this does not duplicate the scene, merely copies the reference to another element. No extra memory is allocated! + */ class LinkSplitter : public PipelineLink{ public: void NextScene(Scene s); diff --git a/src/PipelineLink.cpp b/src/PipelineLink.cpp index d56e3e6..7cecbb7 100644 --- a/src/PipelineLink.cpp +++ b/src/PipelineLink.cpp @@ -1,9 +1,16 @@ #include "PipelineLink.h" +/*! + * Used to register the next pipeline element. + * @param scb reference to the next pipeline element + */ void PipelineLink::RegisterCallback(SceneCallback *scb) { sceneCallback = scb; } - +/*! + * Default NextScene implementation, simply passes the scene on to the registered callback + * @param scene + */ void PipelineLink::NextScene(Scene scene) { if(!sceneCallback) return; sceneCallback->NextScene(scene); diff --git a/src/PipelineLink.h b/src/PipelineLink.h index 2b5ca23..f0ab32d 100644 --- a/src/PipelineLink.h +++ b/src/PipelineLink.h @@ -3,6 +3,10 @@ #include "SceneCallback.h" +/*! + * A simple class which extends SceneCallback to add a RegisterCallback method. Also implements a default NextScene function. + * This class forms the basis for pipeline elements in the Signapse video processing architecture. + */ class PipelineLink : public SceneCallback{ public: void RegisterCallback(SceneCallback* scb); diff --git a/src/PreProcessor.cpp b/src/PreProcessor.cpp index 4fabdde..06aca79 100644 --- a/src/PreProcessor.cpp +++ b/src/PreProcessor.cpp @@ -3,6 +3,10 @@ #include "PreProcessor.h" +/*! + * Handles processing the given scene and calling back to next pipeline element + * @param scene + */ void PreProcessor::NextScene(Scene scene){ cv::Size sz = scene.frame.size(); scene.regionOfInterest = BoundingBox((int)(sz.width * settings->relativeBoundingBox[0]), @@ -14,17 +18,31 @@ void PreProcessor::NextScene(Scene scene){ if(!sceneCallback) return; sceneCallback->NextScene(out); } - +/*! + * + * @param PreProcessorSettings for initialisation + */ PreProcessor::PreProcessor(PreProcessorSettings* s) { settings = s; } +/*! + * Processing method to switch the pixel format of the internal frame. Performed with standard OpenCV functions + * @param s scene + * @return scene with pixel format rearranged + */ Scene PreProcessor::switchRGB2BGR(Scene s) { cv::Mat temp = s.frame; cv::cvtColor(temp, s.frame, cv::COLOR_BGR2RGB); return s; } + +/*! + * Adds a green rectangle into the internal scene frame, highlighting the region of interest. This shows the user where to put their hand sign to interface with the neural network. + * @param s scene for processing + * @return the processed scene + */ Scene PreProcessor::drawBox(Scene s) { cv::Mat temp = s.frame; int x = s.regionOfInterest.UpperLeft.x; int y = s.regionOfInterest.UpperLeft.y; @@ -37,6 +55,13 @@ Scene PreProcessor::drawBox(Scene s) { return s; } +/*! + * Used to set the bounding box each scene is marked with. Set as float values in range [0-1] indicating fractional coordinates in the image frame. + * @param upperLeftX + * @param upperLeftY + * @param lowerRightX + * @param lowerRightY + */ void PreProcessor::SetBoundingBox(float upperLeftX, float upperLeftY, float lowerRightX, float lowerRightY) { settings->relativeBoundingBox[0] = upperLeftX; settings->relativeBoundingBox[1] = upperLeftY; diff --git a/src/PreProcessor.h b/src/PreProcessor.h index 61d52bd..e77898b 100644 --- a/src/PreProcessor.h +++ b/src/PreProcessor.h @@ -2,7 +2,9 @@ #include "Scene.h" #include "PipelineLink.h" #include "PreProcessorSettings.h" - +/*! + * A pipeline element to perform pre-processing required for Signapse. Adds region of interest, bounding box rectangle and switches pixel format for neural network. + */ class PreProcessor : public PipelineLink{ public: PreProcessor(PreProcessorSettings* settings); diff --git a/src/PreProcessorSettings.h b/src/PreProcessorSettings.h index fad3b68..c519e4b 100644 --- a/src/PreProcessorSettings.h +++ b/src/PreProcessorSettings.h @@ -3,7 +3,9 @@ #define Y0 0.25f #define X1 0.75f #define Y1 0.75f - +/*! + * A simple struct to add some default settings for PreProcessor + */ struct PreProcessorSettings{ PreProcessorSettings(){ relativeBoundingBox[0] = X0; diff --git a/src/QtGeneratedGui.h b/src/QtGeneratedGui.h index 69988fd..6efef52 100644 --- a/src/QtGeneratedGui.h +++ b/src/QtGeneratedGui.h @@ -26,7 +26,9 @@ #include QT_BEGIN_NAMESPACE - +/*! + * Auto-generated class, created from parsing our .ui file (gui/TaskMaster.ui) from QtDesigner. As auto-generated elements are not in best code-style, this is wrapped up within the Gui class + */ class Ui_MainWindow { public: diff --git a/src/SceneCallback.h b/src/SceneCallback.h index d1b366c..7cbc1d8 100644 --- a/src/SceneCallback.h +++ b/src/SceneCallback.h @@ -2,7 +2,9 @@ #define SIGNAPSE_SCENECALLBACK_H #include "Scene.h" - +/*! + * A simple callback interface for passing Scenes through the pipeline + */ class SceneCallback{ public: virtual void NextScene(Scene next) = 0; diff --git a/src/SchedulableLink.cpp b/src/SchedulableLink.cpp index 28c4645..a379838 100644 --- a/src/SchedulableLink.cpp +++ b/src/SchedulableLink.cpp @@ -2,7 +2,9 @@ #include "BlockingQueue.cpp" - +/*! + * Looping process to process scenes and call-back to next pipeline element. Worker thread is slept while no elements are available using BlockingQueue functionality + */ void SchedulableLink::Run(){ //waits for scenes to appear on the scheduleQueue, while(isOn){ @@ -12,24 +14,37 @@ void SchedulableLink::Run(){ sceneCallback->NextScene(out); } } - +/*! + * Adds a scene to the intenal scheduleQueue + * @param s + */ void SchedulableLink::Enqueue(Scene s) { scheduleQueue.Push(s); } - +/*! + * @return true if the scheduleQueue is empty; false otherwise. + */ bool SchedulableLink::Available() { return scheduleQueue.IsEmpty(); //singly threaded for now } +/*! + * Starts execution with the worker thread. + */ void SchedulableLink::Start() { scheduleWorker = std::thread(&SchedulableLink::Run, this); } - +/*! + * Turns off the link, kills worker thread. Must be called to release thread resources. + */ void SchedulableLink::Stop(){ isOn = false; scheduleWorker.join(); } - +/*! + * If space is available on the scheduleQueue, add the scene to the queue. Otherwise skip. + * @param scene + */ void SchedulableLink::NextScene(Scene scene) { //if space on schedule queue, add this scene; otherwise pass the scene through if(scheduleQueue.IsEmpty()) { //singly threaded for now @@ -37,7 +52,3 @@ void SchedulableLink::NextScene(Scene scene) { } } -SchedulableLink::~SchedulableLink() { - //Stop(); -} - diff --git a/src/SchedulableLink.h b/src/SchedulableLink.h index 6323f62..d651421 100644 --- a/src/SchedulableLink.h +++ b/src/SchedulableLink.h @@ -9,17 +9,20 @@ #include "BlockingQueue.h" - +/*! + * A class which extends PipelineLink to handle latency-bound stages. + * The NextScene function is inherited to add Scenes to an internal BlockingQueue. The pure virtual function ProcessScene must be implemented by derived classes to define Scene processing behaviour. The Run method is called in a separate thread and handles waking up the thread when scenes are available and calling back to the next pipeline element. + * BlockingQueue is used as a scheduling mechanism for future extension to multithreading. + */ class SchedulableLink : public PipelineLink{ public: void NextScene(Scene s); virtual Scene ProcessScene(Scene s) = 0; void Start(); void Stop(); - void Enqueue(Scene s); bool Available(); - ~SchedulableLink(); protected: + void Enqueue(Scene s); void Run(); BlockingQueue scheduleQueue; bool isOn = true; diff --git a/src/SignapseUtils.h b/src/SignapseUtils.h index 9f02474..a08bd77 100644 --- a/src/SignapseUtils.h +++ b/src/SignapseUtils.h @@ -9,42 +9,60 @@ #define MODEL_PATH "models/asl-mobilenetv2.pb" #define NR_TASKS 26 - - -class SignapseUtils { -public: - static std::string getLetterFromDigit(int digit){ - std::string results[] = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", - "W", "X", "Y", "Z", "del", "space", "nothing"}; - return results[digit]; - } - static void welcomeMessage(){ - printf(" _____ _ \n" - " / ____(_) \n" - " | (___ _ __ _ _ __ __ _ _ __ ___ ___ \n" - " \\___ \\| |/ _` | '_ \\ / _` | '_ \\/ __|/ _ \\\n" - " ____) | | (_| | | | | (_| | |_) \\__ \\ __/\n" - " |_____/|_|\\__, |_| |_|\\__,_| .__/|___/\\___|\n" - " __/ | | | \n" - " |___/ |_| \n \nWelcome to Signapse, the tool for helping everyday people learn sign language for free!\n"); - } - - - static void printProgress(float percentage, int prediction) { - std::string letter = getLetterFromDigit(prediction); - std::cout << "\r Prediction: " << letter << ", Progress: " << percentage << "% - keep signing! "; - fflush(stdout); - } - - static int makeTask(){ - srand( (unsigned)time( NULL ) ); - return rand() % NR_TASKS; - } - - static std::string getModelPath(){ - return MODEL_PATH; - } -}; - +namespace { +/*! + * Simple static class for some utilities needed by Signapse objects. + */ + class SignapseUtils { + public: + /*! + * Array of ordered results from expected at the output of the current CNN model in use. + */ + static std::vector results; + + /*! + * Prints welcome message and ASCII art logo + */ + static void welcomeMessage() { + printf(" _____ _ \n" + " / ____(_) \n" + " | (___ _ __ _ _ __ __ _ _ __ ___ ___ \n" + " \\___ \\| |/ _` | '_ \\ / _` | '_ \\/ __|/ _ \\\n" + " ____) | | (_| | | | | (_| | |_) \\__ \\ __/\n" + " |_____/|_|\\__, |_| |_|\\__,_| .__/|___/\\___|\n" + " __/ | | | \n" + " |___/ |_| \n \nWelcome to Signapse, the tool for helping everyday people learn sign language for free!\n"); + } + /*! + * Used to convert an integer index result into a string sign + * @param digit integer index + * @return std::string representing sign at given index + */ + static std::string getLetterFromDigit(int digit) { + return results[digit % results.size()]; + } + /*! + * Generates a random sign for use assigning tasks to users. + * + */ + static std::string makeTask() { + srand((unsigned) time(NULL)); + int task_int = rand() % NR_TASKS; + return getLetterFromDigit(task_int); + } + /*! + * @return Path of the currently used CNN network. Path given as std::string + */ + static std::string getModelPath() { + return MODEL_PATH; + } + }; + /*! + * Declares results array, this may be changed for different CNN output results. + */ + std::vector SignapseUtils::results = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", + "N", "O", "P", "Q", "R", "S", "T", "U", "V", + "W", "X", "Y", "Z", "del", "space", "nothing"}; +} #endif diff --git a/test/gtest_signapseutils.h b/test/gtest_signapseutils.h index d70a81b..6771f72 100644 --- a/test/gtest_signapseutils.h +++ b/test/gtest_signapseutils.h @@ -7,7 +7,7 @@ TEST(signapseutils_test, checkModelPath_ret){ } TEST(signapseutils_test, checkMakeTask_int){ - int task = SignapseUtils::makeTask(); - EXPECT_LE(task, NR_TASKS); + int size = SignapseUtils::results.size(); + EXPECT_LE(NR_TASKS, size); }