emotion_recognition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Connecting to '192.168.86.64' at port '7000'\n",
      "init values\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "/Users/paxysnowman/Desktop/Screen Shot 2021-07-23 at 09.56.42.png\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "{'region': {'x': 180, 'y': 67, 'w': 429, 'h': 576}, 'emotion': {'angry': 0.08201049547518763, 'disgust': 6.551556129396054e-10, 'fear': 0.1297090861959211, 'happy': 0.0001717298789612772, 'sad': 0.503359997845292, 'surprise': 0.0014882782102502467, 'neutral': 99.28326006431938}, 'dominant_emotion': 'neutral'}\n",
      "<class 'dict'>\n",
      "mood value is: 3\n",
      "dis: 0\n",
      "sucessfully ran\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "/Users/paxysnowman/Desktop/Screen Shot 2021-07-27 at 10.22.50.png\n",
      "empty\n",
      "ahhhhh\n",
      "WARNING:tensorflow:5 out of the last 17 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fb26fa7f170> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "empty\n",
      "ahhhhh\n",
      "{'region': {'x': 219, 'y': 118, 'w': 179, 'h': 253}, 'emotion': {'angry': 10.663244654326993, 'disgust': 0.05857035400998342, 'fear': 3.9051272905024637, 'happy': 0.399003411710669, 'sad': 64.25099542748873, 'surprise': 0.6287890770332905, 'neutral': 20.094273399624036}, 'dominant_emotion': 'sad'}\n",
      "<class 'dict'>\n",
      "mood value is: 5\n",
      "dis: 0\n",
      "sucessfully ran\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n",
      "empty\n",
      "ahhhhh\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "import socket\n",
    "from time import sleep\n",
    "import math\n",
    "from time import time\n",
    "from guizero import App, Text, TextBox, PushButton, Slider, Picture\n",
    "import cv2\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "##import tensorflow as tf\n",
    "##print(tf.__version__)\n",
    "from deepface import DeepFace\n",
    "import tkinter as tk\n",
    "from tkinter import filedialog\n",
    "import serial\n",
    "import threading\n",
    "from secret import IP\n",
    "\n",
    "\n",
    "\n",
    "ser = serial.Serial('/dev/cu.usbmodem143101', baudrate = 115200, timeout = 1)\n",
    "\n",
    "#! important: make a file called secret in the same directory and write your IP as a string.\n",
    "#! if you want to run the Pure Data and Python in the same machine, you may also write 'localhost' \n",
    "#! e.g. IP = \"192.168.1.1\" or IP = \"localhost\"\n",
    "#from secret import IP\n",
    "\n",
    "#IP = '192.168.86.64'\n",
    "port = 7000\n",
    "\n",
    "\n",
    "\n",
    "class PDSocket:\n",
    "    \"\"\"PD-PI Socket\"\"\"\n",
    "\n",
    "    def __init__(self, sock=None):\n",
    "        if sock is None:\n",
    "            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n",
    "        else:\n",
    "            self.sock = sock\n",
    "\n",
    "    def connect(self, host, port):\n",
    "        self.sock.connect((host, port))\n",
    "\n",
    "    def close(self):\n",
    "        self.sock.close()\n",
    "\n",
    "    def send(self, msg):\n",
    "        pdmsg = ';'.join([f'{k} {v}' for k, v in msg.items()])\n",
    "        sent = self.sock.sendall(bytes(pdmsg + ';', 'ascii'))\n",
    "        if sent == 0:\n",
    "            raise RuntimeError(\"socket connection broken\")\n",
    "\n",
    "    def receive(self):\n",
    "        msg = str(self.sock.recv(1024), 'ascii')\n",
    "        if msg == b'':\n",
    "            raise RuntimeError(\"socket connection broken\")\n",
    "        return msg[:-2].split(';\\n')\n",
    "\n",
    "\n",
    "def main(host=IP, port=7000):\n",
    "    \n",
    "    times = []\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "            \n",
    "    \n",
    "    print(f\"> Connecting to '{host}' at port '{port}'\")\n",
    "    socket = PDSocket()\n",
    "    socket.connect(host, port)\n",
    "    \n",
    "\n",
    "    # ?inputs \n",
    "    start = 1  # switch\n",
    "    emotion = 1  # mood\n",
    "    dis = 0  # dissonance\n",
    "    dspToggle = 0  # dsp\n",
    "\n",
    "    #! killswitch\n",
    "    #start = 0\n",
    "    #dsp = 0\n",
    "        \n",
    "    def update_face():\n",
    "        mood = 1\n",
    "        dis = 0\n",
    "        valid = True\n",
    "        while (valid):\n",
    "            root = tk.Tk()\n",
    "            root.withdraw()\n",
    "            file_path = filedialog.askopenfilename()\n",
    "            print(file_path)\n",
    "            try:\n",
    "                cures = cv2.imread(file_path)\n",
    "                plt.imshow(cures)\n",
    "                predictions = DeepFace.analyze(cures, actions = ['emotion'])\n",
    "                print(predictions)\n",
    "                print(type(predictions))\n",
    "                if predictions.get('dominant_emotion') == 'anger' or predictions.get('dominant_emotion') == 'disgust':\n",
    "                    dis = 1\n",
    "                    mood = 5\n",
    "                elif predictions.get('dominant_emotion') == 'surprise':\n",
    "                    dis = 1\n",
    "                    mood = 1\n",
    "                elif predictions.get('dominant_emotion') == 'fear':\n",
    "                    dis = 1\n",
    "                    mood = 2\n",
    "\n",
    "                elif predictions.get('dominant_emotion') == 'neutral':\n",
    "                    dis = 0\n",
    "                    mood = 3\n",
    "                elif predictions.get('dominant_emotion') == 'happy':\n",
    "                    dis = 0\n",
    "                    if predictions.get('emotion').get('neutral') > 30:\n",
    "                        mood = 2\n",
    "                    else:\n",
    "                        mood = 1\n",
    "                elif predictions.get('dominant_emotion') == 'sad':\n",
    "                    dis = 0\n",
    "                    if predictions.get('emotion').get('neutral') > 30:\n",
    "                        mood = 4\n",
    "                    else:\n",
    "                        mood = 5\n",
    "\n",
    "                print(\"mood value is: \"+ str(mood))\n",
    "                print(\"dis: \"+ str(dis))\n",
    "                print('sucessfully ran')\n",
    "\n",
    "                valid = False\n",
    "            except ValueError:\n",
    "                print('no face was found. try again')\n",
    "            except TypeError:\n",
    "                print('bad file type. only accept png or jpg')\n",
    "        socket.send({'emotion': mood, 'dis': dis})\n",
    "\n",
    "    socket.send({'tempo': 100})\n",
    "    \n",
    "    values = {'start': start, 'emotion': emotion, 'dis': dis, 'dspToggle': dspToggle}\n",
    "    \n",
    "    try:\n",
    "        print(\"init values\")\n",
    "        socket.send(values)\n",
    "\n",
    "        def arduinoCom():   \n",
    "            while True:\n",
    "                print(\"ahhhhh\")\n",
    "                arduinoData = ser.readline().decode('ascii').replace(\" \", \"\")\n",
    "\n",
    "                if arduinoData == '':\n",
    "                    print('empty')\n",
    "                else:\n",
    "                    print(arduinoData)\n",
    "                    socket.send({'tempo': int(arduinoData)})\n",
    "\n",
    "        \n",
    "        t1 = threading.Thread(target=arduinoCom, name='t1')\n",
    "        t1.start()\n",
    "        app = App(title=\"Yousic Home\")\n",
    "        welcome_message = Text(app, text=\"Mood Generation Input\", size=40, font=\"Times New Roman\", color=\"lightblue\")\n",
    "        update_face = PushButton(app, command=update_face, text=\"Upload image of your face\")\n",
    "        my_name = TextBox(app, width=50)\n",
    "        app.display()\n",
    "        \n",
    "        \n",
    "                    \n",
    "                    \n",
    "    except KeyboardInterrupt:\n",
    "        print(\"> Closing socket\")\n",
    "        socket.close()\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}