create_gesture_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.5"
    },
    "colab": {
      "name": "NodeConfEU - Create gesture model",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y82-GzNLIeRT",
        "colab_type": "text"
      },
      "source": [
        "<img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAc4AAABtCAMAAAAfx3F2AAABaFBMVEX///8sNYEqM4AoMX8eKXwjLX4hK30AEnXp6vIlL34AFXUcJ3v09PlES44ACnObn8F1eaWoqsQSIHnMzt7T1eOgo8JnbKHd3+uTl7oyO4X4+PtPVpKAOnhKUI9yvkTtKGtbYJjdMDFjyuXe4CCDh7C2udIUInkKG3fBxNlVW5M7Q4jc3elscaOIjLTJytnm5/C8vtThKygAAGuvssxSWHVMU5np6zONkbdkcKRGPoabU4+/X5NzSIroSoH1OnS/Vo7pPHfjUod4QIXdWY2qXJODQYVYRYpjWJR/dKRvOnKnX3yWTHKMVouANHRzZpvHUmG6UWeKSICLYZRdMXKyXHfjRkVcPXvKXm18VoWTVoEyUZBFTYNwttScn4zd31aTl3K0t4WdoWp/wNt0eI/k5kGytZWOkYN6f2VfhrNvoMXa3F1+g49iZ2+Dz+ilqGA8UH1ok3Z8v1lurF1TcnlIaXU8aGZxo21Yf3Mq2ZZtAAAUdElEQVR4nO2d/Z/btn3H+Sw+niiIeiBpOkyiAyRKOslWHMVJ1nV9SNbOTestWds4TZeu29JsWZZ0a//9kQRIAiTIO/vO0Z1f/PyQ+PgkkG8C+D4AoCD06tWrV69bLC84dQl63Zh2UyAeTl2IXjcje+gDX4NT79QF6XUD2jsALmZ7HygL+9Rl6XVNefEArbJ6GQwtBCZ9F3qXpbsWivbFHwsIkn3n8b1usexJ2mnOqQo5ji0wXZ+uQL2uoX2IkDtjNgUHEanbvgu9e/Ji01xhYzY4f8cPj7hWBnMLwckpC9br+aUvDOSc43/Pwsfvvvfe+yKBuFsAGC1PV7Rez6vgkABxTtpUO3z/b1577bUf/K1URBLGEYSb3clK1+v5FGyQuSktnuHjH76W6z217DRHioV6nndEQQzOq7/89zDN137oVFsDx+xx3hEFMRpXf539HcH5gx+Nqq3xoMd5R1TD+eMe590UcTJZnMlPisY2oezZEqfee6G3U7MpinJHk8V5/pi0tu/61MEEZzCHfersNkqfi0CC6mJXxylsHv/4B2nd/IlIb8Q49xFQFRj3qbPbpkOWBlvPNeRP6jgD1//Ruz/9wGdCBxnO2QaAcLxMgOX2Le5tkhebKMzq2G6DkLNncQrCeii5ezY3lh5yHKAkb2iHWp86u0Xabag02Dg2FQmwOIXAr7ensSoDeUgY5qmzc6HXLVA9DRaMHEOrZU1sv8b3wpcNOs7nxVY/+uQ26DxqpMF2kQQTxl6t4dy5UIa1RPbBR/3ok1Or7DRpBbHqAORQBBmc9tFEYjPIFwwVBEd9F3o67RYKcpp9XmbZjlfIqsLwFM7g4ANpbk85UaFd2oVG/eiTE8ke+VUajFbuqAT03grneAWs6awtyDcOobWZcXb0etmyYzDYcAf+EL9zt0DA0fGxBc6JCXCory1mezAA6G3cE0h3EOk0l27s7qlaWoYRZrKMoZU4t2CL/8Hi9I7xOxOywQXDl1jqXi3SnbP8/7NV8uHff+hElUFURYV8qY7TJBYvjTNw/Z/9/B9+5uNdI7PHeQIRnHr00ZOnv7j35KOk5PO8ODePf/n6gwev/0rO9/U4TyKCc+ukMO/d+8XTD6fFnufEOZZ/+eD1VA9+5Wc9bY/zJCI41X/MaKY8n5hF9xnEgIzaa8WpRxVO9+nrRP+U5bd7nCcRxun5P71H5BQpk8AFKrZ5W3DakwSqJc7w4weY5oOnR6HHeSIVOAua96rBBkHqbhhZ2JaPcxxi35Mo/HmB80mP82Qija32BDe2955AylexjwiJBx7O/Wxjmsyw6cUT0tZ+8s830dgG30fgd+ctx57+PfzQ9yaC0/3gKcb5IaiF2RFwfBHjLBNkWyuSkc+E53XX/zU2hT5OGqaQR1QL5c7w1noQY3aYJqJsyH60WDagzjxGsxemnv6KpGqKphoJ52dqsrHqoeiWzZfL5ivbtb7IRZcIb7nSJK/CUXE+fJoZth8lrsoGiZYhlOUUo70+zEV3MtbTwruqJh7ptzo4JOE70a8/efDgk4/9hqNiR6qRyarNanGtbKsyZTaONz5UJUkSJUnWLGdi884pJCWhe15nsZ3yFDPDKGaL8lfE9Gei+s+wmkUhFhsDC6b59ojc1tpdcORyBlHZccjTKru8OwCpLJrBw2zLwO0qYaEijOBFjz94/4PH/lLwYkRP/xOCkSaJ80WCTBMic2DFk1gzYga5F8JJICz83/z2t79R8LhNFqchZpINNmvjqtlWlcaZ2V+SWEmuT4TB55SSZBU4I+YIYcoQL4SooGMwNzT6V9Kfcbrm26w1Q85Ue/OCSM22wjn+cww0tSmwaF5PT/D1WBl+hnOhZbfl0893kBVR41yHc2WCUxD2i6l7yDkufci8Ur4kWQhFi9FoNNmIA2CIFh2QnW0e4qq6nrvuhFRaFqeMn5ohMr+N0RjUM1r6iliTbG70xjnsEdBnEjjT5iGpqMysHkGpvts465gQNzPI8QU5rADfl1Lg1BpXbcGgO7wjpRvFSWsiO1S0T5TTRq94poHnyqpS3bs9587d5eIUFaZMjdo5UWTObVrRrH5OAyh93ctwzpLiACmrFGT3oqMHLHGK2gW1+Ro4eff58nBmSUuXdBS2q0hsAs2bWuWAvvPE5zZTfJwSpKt1vXYeAO95pMAjvXZO40lYm4rGJThnkVqcJDph5MhWhsTtsmcqnEZCHXclnEo7TkNhZMnZa/tScKbMHDMvZrABzNDaXFvTyCF6q7Mh/znwcYpSQlkTNZxjwIWZ3UpcO6epIscjFDglCBg9LF6kEF9BgsloracO0XgIFMNnrIV61q/CydDh45QAonXGMWFwY2tMh3NGw6zSvCScaWfqZ13kBvkcz2wCU9tr5z5sHRfUglNUw+oYtrHVE14LhAWP7DkcodIgynFKiaezIsAWVn64QQ3bD46g6lqWLhqggeLSrSqFU0TUeTycctT2PCthnBZ34sBLwykIcymeWwnX4TnCaGh0DHxvw1nevFCvnW7DCqKkeMw5HElScUiOU0744QEPYZoOM1qivEVvpeSWtaQp06qO0jilanRqC87L3VDc2MLvG2c2wh3yRxUEsdI5LaUVp1jFKZja6RkdNEV1w5zDPSQmh3TixOfLSb09xbqQSjdJ0pKyoDROqn25Gzjp4kzApr6byENOV7l5OLEVaThFA83Uzk0rJ/wQPfocrhB5BF04Z9hF0fhepmfR750hFpcocOL/WkXLfw2cL6+xrY2ttI9woVd7B63tqWtW3sq5H7HmUrBFDZzyBht0ClvTcO20B500y7vpwFkc0oUzf1iixo+w2A7bQhgJ2UFwSiGmbZJ+9RbitEMozqs7D7K194A/IhVoiZzWM5egMEm9KYRQWVCvxXJlgaqwBKcy2cP8qZDAEYPzALtxygl9Dv8QYp524AxWGTAJ8uOfE6t2RUCcG4LT2m/yt0F2SAz7RXHaeTGskRAwyvddD2fqYVqgHOy+ngKwWm4gWuHWaIs6YiUQd0D20UgvMFSBXxy7c9NLUpZhgXNIzB1JxXYI3dhutW6cokyfwxdpbYlluxOaz2rm5x5CzL+hhmldGN0EJ1zufHwnuHY/F077fFI1dMPsxZFDNqqML3NNnGQpqKyt1BcDnCe5cJCZD5SNrYbLWSnOzfuRgsA2wLPO8lypPQfIYMKodnnbgZT/i1gTFM4gbvdSsJQldQ5fpPUificyKQ20vLKOlbyV47+jY7UeBpAAbqVKnMLezHfgJbK4OCU/plS26p4KrEEZ6/ByF1tmgsoQH3ttnGnbGCG4mU1UJBVBgUMClONsIYsdaZmt5VwsHQCK8NE4NkHsHXwAa6OwK5yp/Yqbrdzppxpb278MJ2HVhVPDNgovKiThtvcif4wtxvq+6SkRK7zCWbitIHsuLWEEQzZkGf/HKnDacta8VrFI1CyiemM4hWCYaKIiulX3Z299TdRkvyNxNLQMWTVWVf0NDpEmWnJjFDaFEzczxLSkaqct8uN7lchD68Kp4jvuwIl7R8hvcjjdN8RNAoUzwOaSEdpXCfIphRG8zN8jqTTqObd7gziFXaxaqk+/tV6iWpLUjVPSNJn+FXtqpF54oymjcQb4YRvZ46Vx+pfixFG8ztqJH8iL4jyvW0JpJWzgFMbk38er4LQK8x6/KpV9hvtwjdbNNbb2EQH/MEVmGeaZuWcoPE9Uv2OhmblizY8QyIUdZQ8R0CYuAknNq6NxCjruobQp29h2RPgIzuetnRakBMTLG9ul0sBBnDgaZ+qK5+9HepGWxpb6YbNgsM773CoCkbXrRswmurG9cW2cwchH+bSifYS0vL21jzKeIR9qHabQFKW3t94gGOZW7MEB1lbPutC0I2a8VQansMSefGpN0KbQqjMoJF6p7yQ/gHH6o3NGuVHQaQrZUsMUIpESBqdAvBV/xzeFEvpXqwgYkGVVKf7Ukdjmd26V6+Ecr8r5Xvny/RM79T0lHFt3UcfEIV/Mz9pHwFrsMt+T1O1ssqBIT9xmcQpu/jhSb2VB+Z1dmPLDtcstW5ID6/A7152OSqOV1kiahsWpi9g8j4Pw6iH4YBg5cQn3PGt7+UG+kXUdnMECIWpgxcw1kQ/MYkb8wWyL8aXdLVjhd9ceQmAYSKzI6wtEO54Ep0VumwRfjBg7Jxhnw4Wv4ySjCTv9TkywKyqEwwiAP1txXL92ETBjcQp70r4M+Thbwgj02MT8Lq6M07w6Tj0B7FW9eFDh3fmo1RaaozJrZS+Awo7DnE1RlTmp1U7Bw52MgbtLHEbAnUu7DPLSd+A04kujQrgpK4zI5l4m4yqVhkwNp7DFNjCJ5N5skO+Q46RHPcwQ8/QuuXIjBE8PU1ygYX03kS0a9CIXdZOJE4KvCjSiHxsJvVxiCxWvcgfO4pArhOBlfgg+2NCDiCovsY7TjulS3GxGZZm/cBYVh8lbZrG+CkXblbsSZMIQ+i3TqI/Q6VpesROnEFOGD8moHLrSnaJMZgx34JQ08hyvkCAzWhJkgasUJTOsajxjHWcRDXlxnO21c6bWe+C8yJJ1pRVhOnF6sRnBKXfXEspR12rw3Th1o6qLBGfgdNm2xeynDpxlabrT12f5wSrLs3pll5FpKZpimXSKqIFTGFFhnZvFSSIM1eR17AmU7/NlV27FiceOONyMp6elftfBt1pHp3bjpH28Is69bI6WLKWGxTNqxamVrx3B2dLpb8ngErVaXsXe0mnA2WS+mB9Y2g2cxFt5YZzZA9GO47qynUcSRjzm79hsiPvplpxe88otOO25GGbm1TqBzVFuSx9u82PUtvEll+AsnqpIDf06tubIJKusPm04qeaT+BvhilFIDLegGPqlhaOdbtu6N080ozMVz8OpJ2Vj8qI4Rbk2wlrLFyOd4fC1pPnhdLMiA48lqyN4zlyZj3Pp+HgMteAlgB3zLgQTFRKPbD1F/A+qXIbTDstOqsAZTNuSZErVLLXglKl4B8HJ5iuqaPjaITwNy3Li0IHZ4N4rDcykcRKT5UVxktENrEhOl9THtICGWo7X3l5yyfLKPJxe/HBYUtrJKnApoOcRUKqrj31Qm1SQ6zKcwmxASloNm7Ydrjkk0atm8HGqKlU+/jjbyjmhhk1LRjFsGnTx5OGsGpMXx1l/J0mKflHPt0gtBgzvygVOfXxxgRssfTFw6b7Dl0QLOvMLb7b29q4EVHZSw8iXsJdkjw+HMbkZzhwVi/WcRpZUwynYG07/aWi0ycDDKYMpbSdchlPQQ6s5qaErUc/FKazUGk4lq2LG1RvbNpzC3NKoAyS1Gpd6+ZUJzpHz6aefJkc7bUprA3/WsqSsImgiNREVE4mxbyjMpCT9mC8Ntn/07K23nj3CnSlnBhmsOcIbPDWInscTzI1aDlm2IqZ3buCUZJiwi8ZNVYkjxaV/RmTb9cbMJlZraGU6Y12/nYU3m+Rhj/0kU3gFnD4+syZYTuFZuwlUcCehQX9z9VUrC5zH5LOv3/j6szBeRirzdOyjKhmjQL+YTx3Dj9zDWvBC2YoZ4tmssxH43edvv/3573EvxpvfWfP1bI8zv9OLFSqxoVp+baoePSFQ1TTL0qJJzYTn42QjQWtXsYoXR9LSX+l0A+xJrmHNBV+SzeRRBLtZqt0VPnYQjCZcUb3WbjnfZNMNp9vl86yfRnDu/a/fyBWfHZm3a6SCtI/AN1tO13VVFaApU+6lc/Yvb7+Z6u0vpOwCLz77OrU0Uf7yQmRNz+uv+pEeubE4jvYv+qkI/bCRTAQQMpPtq7OCIMHpfIZpvvEHxjJaRgC6s6Q+md4Fh0OC1CNdcYb/+sc3sd7KOtZrTabX98Ph8LB86V9Ot71xIzp5t4Vx2lKB8w1qkdPdRgNZn9JYG8FFh9ReUoFDGSnh7wjNN3+fxTr7pS5OIoxTTwqab1STzicW0PLF3bk48zw1issK+ujfCpxfxEKP80Qija1V1k6zsEyClUWGw7fgzIJDZ2U/Pf33twnO/8gs2B7nSURwbr4kNP+kFXs6FnErcNKLuJ1HX2Ganz/LjupxnkQE59r/U07zM6MMEDwnTsH5z88zmn/8rzxk3+M8iQq/c4++/OzrP3xpVvGH58Wp+8+++Oqr/36Gu9Me50mkOwMy4mfrWLJLxR9KnIFYX554U6Qf2eWJJ48QCEntPrYOY+j1EqU7lkN89cBmAjAFzmWo1XG6iKwSVVs8PChWwJq5htl/6fwU8lYQcD9lg3HuNhAUH98ocepbBS/OxF8LPjiKQOpeR6vXS9PBR4NFM2aZ4QyOkJoNRn14Yz3NZ51xce59hDjX6/U9KRiqQG18eyOIrYUPALWd+SzOMkRwump++3ocAdR/uOq02m1UENW+/hesDEVhZoOxH60KJgmUQQ3cbqGAqP/kxsl1EQHILMamu6JcmztU/6TcbqHKdNg2reY+UoZ9p3kbdBDBYFuuvDc0oYRqHaDd/OCjbKGwZJx1mm7fad4S2XMZiDh1nU2+XkQmiyZYiqNaVxnDYdqF4k8LjmPYd5q3SrOpBsKl4E0BjL0gHjA4zx89++abhF21LbVsAzzrzF6oIGkkm3udVssEgVjNF7yo4ZxY//Pt/fvfvfWI5pk7KjMXoSgB8Ni3s7dPBwPhIXcszrHx3f1cf6bHCBK/0wsR2rxaaf1XRvqQLILE4Iz/imne/5aeeliGEQ5dA+F63QKxOB+Synn//v9SrknbBx973TrVcH5b4Pw/Krbe47wzCmKTGuF5xq2dYTPI1+t2KlhB51B6Hu6fCc3v5LLOzjbGWf9N5Lui9RSCcr7fWvwLqZzFaPLg6AOxZSX4XrdR5wkyi1TXufnXtPv8yzdkPQnh3Ad9RO+uaSKmXihZNnIjnVmPSMfZp8HupnauChySOtN13SYb+zTYXdU4BOzok2AoAq1Pg91ZnUtVF9qnwe6+gqEExAlepHDVp8HuvshkMn1h9GmwV0LLCIFpnwZ7dXQQ006zD+q9MtLnfRqsV69eva6q/wfWDUSHHWIuCQAAAABJRU5ErkJggg==\" alt=\"NodeConfEU 2019 logo\">"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ztLVE0dgsrVE",
        "colab_type": "text"
      },
      "source": [
        "# Creating machine learning model for gesture detection"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_ptMsFanrrE",
        "colab_type": "text"
      },
      "source": [
        "Hi, this is the Google Colab for the NodeConfEU 2019 Bangle.js workshop. This document will help you understand how the built-in machine learning model for detecting gestures works and also allow you to extend the model with your own gestures. No understanding of machine learning is required and if you get stuck we are here to help."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mUq9BRUonjgA",
        "colab_type": "text"
      },
      "source": [
        "## Introduction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a8Pz30nsj5ML",
        "colab_type": "text"
      },
      "source": [
        "The gesture recognition works by measure acceleration via the accelerometer that is built into the watch. This acceleration is relative to the watch and does therefore not provide an exact way to measure position, for example, relative to your head. However, it is still possible to recognize a few simple gestures.\n",
        "\n",
        "The watch comes with a set of pre-trained gesture:\n",
        "* swipeleft: an arc on the left side.\n",
        "* swiperight: an arc on the right side.\n",
        "* upup: hand moving up in two steps.\n",
        "* waggle: rotating the arm side to side.\n",
        "* clap2: two claps with the hands.\n",
        "\n",
        "The firmware also attempts to filter out steady motion, to save power. Additionally, there is a `random` gesture which is for undefined hand moments, such as scratching your head.\n",
        "\n",
        "All of these pre-trained gestures may not suit you, as they make assumptions about speed, movement, if the watch is on the right or left hand, etc.. However, using this document you will be able to understand how the model is created and create your own model that fits your movements and needs.\n",
        "\n",
        "_Note, if you are a bit familiar with machine learning we encourage you to develop your own model. But note that the memory and compute capabilities of the watch are very limited. The model's total size should not exceed 20kb. There are also many limitations to what \"TensorFlow 2.0 TFLite for Microprocessors\" can do. But more on that later._"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yWi7JUq6nvkn",
        "colab_type": "text"
      },
      "source": [
        "### How to run the code"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ghPH5LdKno8n",
        "colab_type": "text"
      },
      "source": [
        "Everything is ready, all you need to do is **press Runtime -> Run all**. You can also take it step-by-step, by clicking the ( ▶) button on the left of each code section. Note that set variables remain set, even if you execute previous code, which can sometimes cause issues. To reset the session press _Runtime -> Restart Runtime_."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9bae4TEcz_wq",
        "colab_type": "text"
      },
      "source": [
        "## Setup - that you should not spend time on"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "--u8D0o9L2Bp",
        "colab_type": "text"
      },
      "source": [
        "You shouldn't spend much time on this part. This is just to setup your Colab. What is does is:\n",
        "\n",
        "* Set the TensorFlow version to be 2.x, otherwise it will be 1.x.\n",
        "* Clone our GitHub repository, that you can also find at https://github.com/nearform/nodeconfeu-gesture-models.git\n",
        "* Install all dependencies, including the special `nodeconfeu_watch` module, which you will help you load the data, evaluate the model, and export the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sLRTYL8i1tpe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%tensorflow_version 2.x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mW5p4fBltLgZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!rm -rf sample_data nodeconfeu-gesture-models\n",
        "!git clone https://github.com/nearform/nodeconfeu-gesture-models.git\n",
        "!mkdir -p exports"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "edmOuI25usYh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cd nodeconfeu-gesture-models && pip install ."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eVm64DlNj3Ag",
        "colab_type": "text"
      },
      "source": [
        "# Introduction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5kw4yjf9Q58j",
        "colab_type": "text"
      },
      "source": [
        "## Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yrI1nI36TK7J",
        "colab_type": "text"
      },
      "source": [
        "The goal is to be able to recognize hand gestures by measuring the acceleration from the builtin watch accelerometer. This can be quite challenging as there a correlation between the acceleration and the gesture is not very obvious. To understand this challenge better, we start by looking at the acceleration for different gestures.\n",
        "\n",
        "The `AccelerationReader` loads the dataset, this will be discussed in more detail later."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OOFJ9_-L0ecc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nodeconfeu_watch.reader import AccelerationReader\n",
        "\n",
        "dataset = AccelerationReader({\n",
        "        \"james\": ['./nodeconfeu-gesture-models/data/james-v2'],\n",
        "        \"conor\": ['./nodeconfeu-gesture-models/data/conor-v2']\n",
        "    },\n",
        "    test_ratio=0.2, validation_ratio=0.2,\n",
        "    classnames=['swiperight', 'swipeleft', 'upup', 'waggle', 'clap2', 'random'])\n",
        "\n",
        "dataset_df = dataset.dataframe() # This variable is just for plotting"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "moyh79Id8ZIk",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title\n",
        "import altair as alt\n",
        "import numpy as np\n",
        "\n",
        "alt.Chart(\n",
        "    dataset_df[\n",
        "      np.logical_and(\n",
        "          dataset_df['subset'] == 'validation',\n",
        "          dataset_df['person'] == 'conor'\n",
        "      )\n",
        "    ],\n",
        "    width=200\n",
        ").mark_line(\n",
        "    opacity=0.4\n",
        ").encode(\n",
        "    x='time:Q',\n",
        "    y='acceleration:Q',\n",
        "    detail='id:N',\n",
        "    color='label:N',\n",
        "    column='dimension:N'\n",
        ").interactive(bind_y=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qmQJfG71-0XQ",
        "colab_type": "text"
      },
      "source": [
        "The above plot, clearly shows that there is some difference  between each gesture. However, it is not clear how to accurately quantify this difference into something meaningful. To solve this problem machine learning is used."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0bpv7G8_kJm",
        "colab_type": "text"
      },
      "source": [
        "### Understanding the data format"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UOSidSIg8Zol",
        "colab_type": "text"
      },
      "source": [
        "Each observation is a row in CSV file, for example:\n",
        "\n",
        "```\n",
        "swiperight(14), 2,-20,-22,0,-34,-25,0,-24,-27,2,-25,-26,0,-23,-23,0,-23,-23,-3,-23,-23,-11,-41,-14,-9,-40,8,3,-56,21,45,-29,1,14,15,-27,4,-15,-29,0,-26,-29\n",
        "```\n",
        "\n",
        "The observations are accelerometer data in `(x,y,z)` coordinates measured with 0.1 seconds interval. A typical gesture is between 1 and 2 seconds long. Meaning 10 to 15 samples.\n",
        "\n",
        "The `AccelerationReader` will read this format and change it into a 2d-array, with (x, y, z) as columns and samples as rows. This format is useful for the convolutional neural network which will be used later. An example of this is displayed below.\n",
        "\n",
        "_All observations are extended to have 50 samples, where missing samples are set to have acceleration `0`. You can set `max_sequence_length` in `AccelerationReader` to change this behavior._"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7mhE17QMLcbw",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title\n",
        "import pandas as pd\n",
        "display(pd.DataFrame(dataset.train.x[0, 0:20, 0, :], columns=['x', 'y', 'z']))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-7bPW0_lNV9y",
        "colab_type": "text"
      },
      "source": [
        "Plotting this data looks like the following:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N6sdEhTYoKgY",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title\n",
        "import altair as alt\n",
        "import pandas as pd\n",
        "\n",
        "alt.Chart(\n",
        "    pd.DataFrame(\n",
        "        dataset.train.x[0, :, 0, :],\n",
        "        columns=['x', 'y', 'z']\n",
        "    ).assign(\n",
        "        time = np.arange(0, 5, 0.1)\n",
        "    ),\n",
        "    width=600\n",
        ").transform_fold(\n",
        "    ['x', 'y', 'z'],\n",
        "    as_=['dimension', 'acceleration']\n",
        ").mark_line(\n",
        ").encode(\n",
        "    x='time:Q',\n",
        "    y='acceleration:Q',\n",
        "    color='dimension:N'\n",
        ").interactive(bind_y=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8hkQLuQfXgIG",
        "colab_type": "text"
      },
      "source": [
        "### Adding your own gesture"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WA6B5YIyXhMs",
        "colab_type": "text"
      },
      "source": [
        "To train with your own gestures you need to modify the `AccelerationReader`. For example, to add add the `swipeup` event from the `extra` dataset:\n",
        "\n",
        "```python\n",
        "dataset = AccelerationReader({\n",
        "        \"james\": ['./nodeconfeu-gesture-models/data/james-v2'],\n",
        "        \"conor\": ['./nodeconfeu-gesture-models/data/conor-v2'],\n",
        "        \"extra\": ['./nodeconfeu-gesture-models/data/extra-v2'] # added extra dataset\n",
        "    },\n",
        "    test_ratio=0.2, validation_ratio=0.2,\n",
        "    classnames=['swiperight', 'swipeleft', 'swipeup', 'upup', 'waggle', 'clap2', 'random']) # added 'swipeup'\n",
        "```\n",
        "\n",
        "_Note: The `extra` dataset, is just to get you started. You should record your own data and use that instead of `extra`._"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w06ENGMFDIYR",
        "colab_type": "text"
      },
      "source": [
        "### Remarks about cultural bias"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dTCDAcV-DNtk",
        "colab_type": "text"
      },
      "source": [
        "Describing a gesture isn't always straight forward. While we might think that something as simple as clapping is universally understood, it too is affected by our culture, environment, situation, etc.. Therefore while James and Conor both clapped with the watch on the left hand while sitting down, there is a significant difference in the data (see figure below).\n",
        "\n",
        "When trying out the model or when training it with your own data, it is therefore important to be aware of these differences. Either communicate the gestures precisely or even better try and collaborate with others to make a dataset that encompasses these differences."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aMcOGMv8M0LD",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title\n",
        "import altair as alt\n",
        "\n",
        "# plot the acceleration data\n",
        "alt.Chart(\n",
        "    dataset_df[\n",
        "      np.logical_and(\n",
        "          dataset_df['subset'] == 'validation',\n",
        "          dataset_df['label'] == 'clap2'\n",
        "      )\n",
        "    ],\n",
        "    width=200\n",
        ").mark_line(\n",
        "    opacity=0.4\n",
        ").encode(\n",
        "    x='time:Q',\n",
        "    y='acceleration:Q',\n",
        "    detail='id:N',\n",
        "    color='person:N',\n",
        "    column='dimension:N'\n",
        ").interactive(bind_y=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vBekzXg0NrTO",
        "colab_type": "text"
      },
      "source": [
        "### Splitting the dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g24qjINbUW72",
        "colab_type": "text"
      },
      "source": [
        "You should now have a reasonable understanding of the data format and the challenges creating a model that can correlate the acceleration data with a particular gesture. However, before diving in to TensorFlow and the model, it is critial to understand how data should be split. Otherwise you may end up with a false belief that your model is good.\n",
        "\n",
        "A dataset in machine learning is always split up into at least 3 parts:\n",
        "\n",
        "1. A training dataset, used for learning the model.\n",
        "2. A validation dataset, used for making decisions about the model, such as the number of parameters or how it is optimized.\n",
        "3. A test dataset, used when you really think the model is good and you want to evaluate how it would perform on data that have never been seen before, by the model or by you!\n",
        "\n",
        "The `AccelerationReader` constructor parses the CSV data and generates these three datasets. `test_ratio` and `validation_ratio` controls how much data is put aside for validation and testing. The datasets are exposed as `dataset.train`, `dataset.validation`, and `dataset.test`.\n",
        "\n",
        "_Note that this uses [statified sampling](https://en.wikipedia.org/wiki/Stratified_sampling), meaning `validation_ratio=0.2` takes 20% for each gesture type and not just from the entire dataset. This is relevant when there is not an equal amount of observations for each gesture._"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aEETykFqOZJg",
        "colab_type": "text"
      },
      "source": [
        "## Introduction to TensorFlow"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dZeJrGPPdB7C",
        "colab_type": "text"
      },
      "source": [
        "TensorFlow is a framework for operating with tensors, which are just fixed-sized multidimensional arrays. Such an operation could be adding each element together, for example `[1,2] + [3,4] = [4,6]` or summing all elements, such as `[1,2] = 3`. However, TensorFlow's capabilities extend far beyond these simple operations. In particular, it provides essential mathematical tools for optimizing models, such as computing gradients.\n",
        "\n",
        "For defining the model, the [keras framework](https://keras.io/) is used. This is a high-level framework on top of TensorFlow which makes it easy to define common neural networks, while still allowing the user to implement custom components. For this workshop, only predefined components are used.\n",
        "\n",
        "### TensorFlow Lite for Microcontrollers\n",
        "\n",
        "While there are popular alternatives to TensorFlow such as [PyTorch](https://pytorch.org/), TensorFlow provides capabilities useful for the industry that other frameworks have not yet prioritized. One such example is [\"TensorFlow Lite for Microcontrollers\"](https://www.tensorflow.org/lite/microcontrollers), which is what allows us to run models on Microcontrollers such as the smartwatch badge.\n",
        "\n",
        "Without going into too many details, TensorFlow Lite allows us to export the model loosely compressed (called quantization) and save it to a flatbuffer format. This file can then be used to evaluate the model in a microcontroller, using the [\"TensorFlow Lite for Microcontrollers\"](https://www.tensorflow.org/lite/microcontrollers) library. This is a small C++ library that doesn't depend on an operating system, thereby allowing us to use it on the smartwatch."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zyUjW97H0Nl9",
        "colab_type": "text"
      },
      "source": [
        "## Define model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JreF35EoN5fo",
        "colab_type": "text"
      },
      "source": [
        "When creating a model for an embedded device, it is important to keep things simple as this the total-size should not exceed 20kb, which is not a lot for a contemporary neural network.\n",
        "\n",
        "The model used here, is a simple [convolutional neural network](https://en.wikipedia.org/wiki/Convolutional_neural_network) \\([easy blog post](https://victorzhou.com/blog/intro-to-cnns-part-1/)). The central idea is to first learn micro-patterns, such as right movement, left movement, or sudden stop. Then learn macro-patterns such as a right movement, followed by a sudden stop, followed by a left movement, then a right movement again, and finally a sudden stop, may indicate a clap.\n",
        "\n",
        "Both the micro and macro patterns are looked for over time, and at each time step, it outputs some signal of this pattern being detected. To summarize this, the model then looks for the strongest signal (MaxPool) over time.\n",
        "\n",
        "This gives a 1 dimensional array of values, each one indicating either a gesture or a significant part of a gesture. To then scale and combine these values, such they can attain a probabilistic meaning, a final transformation (Dense) is applied."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54ZuYEpEHL_x",
        "colab_type": "code",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "e3f7eb30-e454-45f7-f4df-fe8829637068"
      },
      "source": [
        "#@title\n",
        "%%html\n",
        "<svg version=\"1.1\" viewBox=\"0 0 410.01 194.07\" width=\"350\" xmlns=\"http://www.w3.org/2000/svg\">\n",
        " <g transform=\"matrix(1.5284 0 0 1.5284 -814.5 -430.73)\">\n",
        "  <g transform=\"matrix(.61712 0 0 .61712 207.54 107.91)\">\n",
        "   <g stroke=\"#000\">\n",
        "    <g stroke-width=\"1.9\">\n",
        "     <rect class=\"rect\" x=\"543\" y=\"363\" width=\"1\" height=\"50\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"552\" y=\"372\" width=\"1\" height=\"50\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"561\" y=\"381\" width=\"1\" height=\"50\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"609\" y=\"313.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"618\" y=\"322.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"627\" y=\"331.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"636\" y=\"340.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"645\" y=\"349.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"654\" y=\"358.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"663\" y=\"367.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"672\" y=\"376.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"681\" y=\"385.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"690\" y=\"394.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"699\" y=\"403.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"708\" y=\"412.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"717\" y=\"421.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"726\" y=\"430.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"766\" y=\"354\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"775\" y=\"363\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"784\" y=\"372\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"793\" y=\"381\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
        "     <rect class=\"rect\" x=\"802\" y=\"390\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
        "    </g>\n",
        "    <g stroke-opacity=\".8\">\n",
        "     <rect class=\"conv\" x=\"561\" y=\"401.85\" width=\"1\" height=\"5\" fill-opacity=\"0\" stroke-width=\"1.9\"/>\n",
        "     <rect class=\"conv\" x=\"726\" y=\"436.74\" width=\"1\" height=\"9\" fill-opacity=\"0\" stroke-width=\"1.9\"/>\n",
        "     <line class=\"link\" x1=\"562\" x2=\"726.66\" y1=\"406.85\" y2=\"451.82\" stroke-width=\".95\"/>\n",
        "     <line class=\"link\" x1=\"562\" x2=\"726.66\" y1=\"401.85\" y2=\"451.82\" stroke-width=\".95\"/>\n",
        "    </g>\n",
        "    <polygon class=\"poly\" points=\"880.95 374.97 870.95 374.97 866 370.03 876 370.03\" fill=\"#e0e0e0\" opacity=\".8\" stroke-width=\"1.9\"/>\n",
        "    <polygon class=\"poly\" transform=\"translate(61.771)\" points=\"887.9 374.97 882.95 370.03 892.95 370.03 897.9 374.97\" fill=\"#e0e0e0\" opacity=\".8\" stroke-width=\"1.9\"/>\n",
        "    <line class=\"line\" x1=\"880.95\" x2=\"950\" y1=\"374.97\" y2=\"374.97\" stroke-opacity=\".8\" stroke-width=\".95\"/>\n",
        "    <line class=\"line\" x1=\"876.64\" x2=\"944.64\" y1=\"370.03\" y2=\"370.03\" stroke-opacity=\".8\" stroke-width=\".95\"/>\n",
        "   </g>\n",
        "   <g font-family=\"sans-serif\" font-size=\"16px\">\n",
        "    <text class=\"text\" x=\"600\" y=\"481\" dy=\"0.34999999em\">Conv2D</text>\n",
        "    <text class=\"text\" x=\"740.75\" y=\"481\" dy=\"0.34999999em\">Conv2D</text>\n",
        "    <text class=\"text\" x=\"823.11694\" y=\"481\" dy=\"5.5999999\">MaxPool</text>\n",
        "    <text class=\"info\" x=\"526.03632\" y=\"348\" dy=\"-4.8000002\">3@50x1</text>\n",
        "    <text class=\"info\" x=\"604.75903\" y=\"298.5\" dy=\"-4.8000002\">14@46x1</text>\n",
        "    <text class=\"info\" x=\"752.75903\" y=\"330\">5@46x1</text>\n",
        "    <text class=\"info\" x=\"863.87952\" y=\"355.02512\" dy=\"-4.8000002\">1x5</text>\n",
        "    <text class=\"info\" x=\"931.31934\" y=\"355.02512\" dy=\"-4.8000002\">1x5</text>\n",
        "    <text class=\"text\" x=\"903.69458\" y=\"486.6001\">Dense</text>\n",
        "   </g>\n",
        "   <g fill=\"none\" stroke=\"#000\" stroke-width=\"1.0602px\">\n",
        "    <path d=\"m803 390 67.95-15.025\" stroke-opacity=\".8\"/>\n",
        "    <path d=\"m767 354 99 16.025\" stroke-opacity=\".8\"/>\n",
        "    <path d=\"m727 445.74 76.144-46.615-76.144 37.615\" stroke-opacity=\".8\"/>\n",
        "   </g>\n",
        "  </g>\n",
        " </g>\n",
        "</svg>"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<svg version=\"1.1\" viewBox=\"0 0 410.01 194.07\" width=\"350\" xmlns=\"http://www.w3.org/2000/svg\">\n",
              " <g transform=\"matrix(1.5284 0 0 1.5284 -814.5 -430.73)\">\n",
              "  <g transform=\"matrix(.61712 0 0 .61712 207.54 107.91)\">\n",
              "   <g stroke=\"#000\">\n",
              "    <g stroke-width=\"1.9\">\n",
              "     <rect class=\"rect\" x=\"543\" y=\"363\" width=\"1\" height=\"50\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"552\" y=\"372\" width=\"1\" height=\"50\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"561\" y=\"381\" width=\"1\" height=\"50\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"609\" y=\"313.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"618\" y=\"322.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"627\" y=\"331.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"636\" y=\"340.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"645\" y=\"349.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"654\" y=\"358.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"663\" y=\"367.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"672\" y=\"376.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"681\" y=\"385.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"690\" y=\"394.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"699\" y=\"403.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"708\" y=\"412.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"717\" y=\"421.5\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"726\" y=\"430.5\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"766\" y=\"354\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"775\" y=\"363\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"784\" y=\"372\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"793\" y=\"381\" width=\"1\" height=\"46\" fill=\"#a0a0a0\" opacity=\".8\"/>\n",
              "     <rect class=\"rect\" x=\"802\" y=\"390\" width=\"1\" height=\"46\" fill=\"#e0e0e0\" opacity=\".8\"/>\n",
              "    </g>\n",
              "    <g stroke-opacity=\".8\">\n",
              "     <rect class=\"conv\" x=\"561\" y=\"401.85\" width=\"1\" height=\"5\" fill-opacity=\"0\" stroke-width=\"1.9\"/>\n",
              "     <rect class=\"conv\" x=\"726\" y=\"436.74\" width=\"1\" height=\"9\" fill-opacity=\"0\" stroke-width=\"1.9\"/>\n",
              "     <line class=\"link\" x1=\"562\" x2=\"726.66\" y1=\"406.85\" y2=\"451.82\" stroke-width=\".95\"/>\n",
              "     <line class=\"link\" x1=\"562\" x2=\"726.66\" y1=\"401.85\" y2=\"451.82\" stroke-width=\".95\"/>\n",
              "    </g>\n",
              "    <polygon class=\"poly\" points=\"880.95 374.97 870.95 374.97 866 370.03 876 370.03\" fill=\"#e0e0e0\" opacity=\".8\" stroke-width=\"1.9\"/>\n",
              "    <polygon class=\"poly\" transform=\"translate(61.771)\" points=\"887.9 374.97 882.95 370.03 892.95 370.03 897.9 374.97\" fill=\"#e0e0e0\" opacity=\".8\" stroke-width=\"1.9\"/>\n",
              "    <line class=\"line\" x1=\"880.95\" x2=\"950\" y1=\"374.97\" y2=\"374.97\" stroke-opacity=\".8\" stroke-width=\".95\"/>\n",
              "    <line class=\"line\" x1=\"876.64\" x2=\"944.64\" y1=\"370.03\" y2=\"370.03\" stroke-opacity=\".8\" stroke-width=\".95\"/>\n",
              "   </g>\n",
              "   <g font-family=\"sans-serif\" font-size=\"16px\">\n",
              "    <text class=\"text\" x=\"600\" y=\"481\" dy=\"0.34999999em\">Conv2D</text>\n",
              "    <text class=\"text\" x=\"740.75\" y=\"481\" dy=\"0.34999999em\">Conv2D</text>\n",
              "    <text class=\"text\" x=\"823.11694\" y=\"481\" dy=\"5.5999999\">MaxPool</text>\n",
              "    <text class=\"info\" x=\"526.03632\" y=\"348\" dy=\"-4.8000002\">3@50x1</text>\n",
              "    <text class=\"info\" x=\"604.75903\" y=\"298.5\" dy=\"-4.8000002\">14@46x1</text>\n",
              "    <text class=\"info\" x=\"752.75903\" y=\"330\">5@46x1</text>\n",
              "    <text class=\"info\" x=\"863.87952\" y=\"355.02512\" dy=\"-4.8000002\">1x5</text>\n",
              "    <text class=\"info\" x=\"931.31934\" y=\"355.02512\" dy=\"-4.8000002\">1x5</text>\n",
              "    <text class=\"text\" x=\"903.69458\" y=\"486.6001\">Dense</text>\n",
              "   </g>\n",
              "   <g fill=\"none\" stroke=\"#000\" stroke-width=\"1.0602px\">\n",
              "    <path d=\"m803 390 67.95-15.025\" stroke-opacity=\".8\"/>\n",
              "    <path d=\"m767 354 99 16.025\" stroke-opacity=\".8\"/>\n",
              "    <path d=\"m727 445.74 76.144-46.615-76.144 37.615\" stroke-opacity=\".8\"/>\n",
              "   </g>\n",
              "  </g>\n",
              " </g>\n",
              "</svg>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u1fh8bt02vI0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "\n",
        "tf.random.set_seed(1)\n",
        "\n",
        "model = keras.Sequential()\n",
        "# Define input\n",
        "model.add(keras.Input(shape=(50, 1, 3), name='acceleration'))\n",
        "# Define first convolution layer (detects micro patterns)\n",
        "# You can experiment with 14, 5, and activation here.\n",
        "model.add(keras.layers.Conv2D(14, (5, 1),\n",
        "                              padding='valid', activation='relu'))\n",
        "model.add(keras.layers.Dropout(0.2))\n",
        "# Define second convolution layer (detects macro patterns)\n",
        "# You can experiment with 5, 2, and activation here. `len(dataset.classnames)`\n",
        "#   can also be changed to someting larger than `len(dataset.classnames)`.\n",
        "model.add(keras.layers.Conv2D(len(dataset.classnames), (5, 1),\n",
        "                              padding='same', activation='relu', dilation_rate=2))\n",
        "model.add(keras.layers.Dropout(0.1))\n",
        "\n",
        "# Define the sequence summarization layer\n",
        "# The number 46 is meant to compass the entire sequence, which is now 46 long,\n",
        "#   this is computed as `dataset.max_sequence_length - (micro_filter_width - 1)`.\n",
        "#   Where micro_filter_width is the `5` in the first convolution layer.\n",
        "# Normally one should use `keras.layers.GlobalMaxPool2D()` which does the same,\n",
        "#   however that is not yet supported by TFLite Micro.\n",
        "model.add(keras.layers.MaxPool2D((46, 1)))\n",
        "model.add(keras.layers.Flatten())\n",
        "\n",
        "# Define the final layer that outputs classes.\n",
        "# The input needs to be len(dataset.classnames).\n",
        "model.add(keras.layers.Dense(len(dataset.classnames), use_bias=False))\n",
        "\n",
        "# Compile the model, this defines how the model is optimizied\n",
        "model.compile(optimizer=keras.optimizers.Adam(),\n",
        "              loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=[keras.metrics.SparseCategoricalAccuracy()])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7kYbldzb_ieg",
        "colab_type": "text"
      },
      "source": [
        "### Training model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z1xqblPlr4rl",
        "colab_type": "text"
      },
      "source": [
        "The model is then learned on the `dataset.train` dataset using `model.fit`. A validation dataset (`dataset.validation`) is also used. This dataset is not directly used to optimize the model, and therefore represents new data as you would get when deploying the model.\n",
        "\n",
        "Disgusting these datasets are essential, as a model can easily just memoize what exact input corresponds to a gesture, instead of learning meaningful patterns, this is called [overfitting](https://en.wikipedia.org/wiki/Overfitting). The behavior that we want to prevent is that accuracy on the training dataset keeps increasing, but the accuracy on the validation dataset starts to decrease.\n",
        "\n",
        "The model here is designed to prevent overfitting. However, if you change the model parameters you may encounter these issues. A simple solution is to set the `epochs` (how many times the model sees the same data) to where the model performed the best on the validation dataset.\n",
        "\n",
        "_Note, if you want to see more details set `verbose=1` and consider removing `callbacks=[KerasLivePlot()]`._"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sjEyVbaw_ZSe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nodeconfeu_watch.visual import KerasLivePlot\n",
        "\n",
        "history = model.fit(dataset.train.x, dataset.train.y,\n",
        "                    batch_size=64,\n",
        "                    epochs=200,\n",
        "                    verbose=0,\n",
        "                    validation_data=(dataset.validation.x, dataset.validation.y),\n",
        "                    callbacks=[KerasLivePlot()])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2rKNmfE6RbxU",
        "colab_type": "text"
      },
      "source": [
        "## Evaluate model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TQVqqSLquHV-",
        "colab_type": "text"
      },
      "source": [
        "There is not a single way to [measure the quality of a model](https://en.wikipedia.org/wiki/Precision_and_recall). Precision can get a high score by under-classifying, and recall can get a high score by over-classifying. The F1-score attempts to achieve the best of both worlds.\n",
        "\n",
        "If you are looking for a high-quality model, a high F1-score is a reasonable metric. The precision and recall may provide you additional information on why the F1-score is not high.\n",
        "\n",
        "The default model and dataset should give you an F1-score of approximately 96-99%."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g2aYZ_dy4ePf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nodeconfeu_watch.visual import classification_report\n",
        "\n",
        "print(classification_report(model, dataset, subset='validation'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PYAdzV7k6mr6",
        "colab_type": "text"
      },
      "source": [
        "## Export model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EPDrJK-eylz_",
        "colab_type": "text"
      },
      "source": [
        "To run the model on the watch, you need to export the model to the TFLite format, this is a [flatbuffer schema](https://google.github.io/flatbuffers/) defined in [`tensorflow/lite/schema/`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs). While this format [supports many operations](https://www.tensorflow.org/lite/guide/ops_compatibility), TFLite for Microcontrollers does not support that many operations. The `ExportModel` abstraction attempts to assert these restrictions for you, but you may encounter additional issues if you use alternative layers in the model definition."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FPtrSov85paZ",
        "colab_type": "text"
      },
      "source": [
        "### Not-quantized exporting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bvuAe3u80qA0",
        "colab_type": "text"
      },
      "source": [
        "The simplest approach to exporting the model is to just export the model as is. However, this does waste some space as models use `float32` weights. This normal and not a big issue when executing models in the cloud, however when executing models on a microprocessor size matters and floating-point operations typically takes many CPU-cycles to compute. To reduce the size and improve computation speed, a model is therefore typically quantized."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5lZa7_pT5g6O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nodeconfeu_watch.convert import ExportModel\n",
        "\n",
        "exporter_raw = ExportModel(model, dataset, quantize=False)\n",
        "print(classification_report(model, dataset, subset='test'))\n",
        "print(exporter_raw.size_report())\n",
        "exporter_raw.save('exports/model_not_quantized.tflite')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zw8b-PGm51wB",
        "colab_type": "text"
      },
      "source": [
        "### Quantized exporting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LcjY3mNz1hkz",
        "colab_type": "text"
      },
      "source": [
        "Quantizing converting floating-point weights to something more categorical in this case an `int8`. This conversion is more complicated than just casting the value, as the weights may be outside of the `int8` range; `[-127,127]`. This conversion can reduce accuracy, but the model usually remains quite accurate.\n",
        "\n",
        "You can read more about the process here, in the [tensorflow documentation](https://www.tensorflow.org/lite/performance/post_training_quantization#representation_for_quantized_tensors), as well as [the TensorFlow paper on quantization](https://arxiv.org/abs/1712.05877)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X8jT9zXe53f4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nodeconfeu_watch.convert import ExportModel\n",
        "\n",
        "exporter_quant = ExportModel(model, dataset, quantize=True)\n",
        "print(classification_report(model, dataset, subset='test'))\n",
        "print(exporter_quant.size_report())\n",
        "exporter_quant.save('exports/model_quantized.tflite')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HWuDATGT6roq",
        "colab_type": "text"
      },
      "source": [
        "### Transfer the model to your Bangle.js"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aoZGo9Rp_jm5",
        "colab_type": "text"
      },
      "source": [
        "Finally, you can upload the model to your watch. The easiest way is to run the code below and paste its output into the Left-Hand-Side REPL in the Espruino Wed IDE whilst its connected to the Bangle.js. Alternatively you can download it from the `exports/` directory."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hEcPqRap6ssG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(f'require(\"Storage\").write(\"tfmodel\",atob(\"{exporter_quant.base64()}\"));')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}