Made lock gesture optional

* Both for website and for python library
* Attempted to improve accuracy of ML model
* Added note on web socem page
* Added documentation on concessions and making good images
* Added support for website that checks a user's gestures and waits for thier approval before submitting
* Fixed bug in error by replacing if with try-catch
* Fixed bug in user exists logic
* Fixed bug in argdict name where creating a user will use the real file name and not the profile name
* Added support for checking if a profile face compares with a kinesis stream face
* Fixed bug in custom labels project waiter
* Prettyfied video and lock/unlock displays on website

Fixes: #40
Fixes: #79
Signed-off-by: Morgan Davies <morgan.davies@students.plymouth.ac.uk>

README.md

@@ -1,6 +1,6 @@
 # eye-of-horus
 
-A face and gesture recognition authentication system, the eye of horus is my Final Year University of Plymouth Project that will account for 40 credits of my grade this year. My supervisor for this project is [Professor Nathan Clark](https://www.plymouth.ac.uk/staff/nathan-clarke).
+A face and gesture recognition authentication system, the eye of horus is my [Final Year University of Plymouth Project](http://web.socem.plymouth.ac.uk/secam/public/cis.php) that will account for 40 credits of my grade this year. My supervisor for this project is [Professor Nathan Clark](https://www.plymouth.ac.uk/staff/nathan-clarke).
 
 <p align="center">
     <a href="https://aws.amazon.com/s3/"><img height="50px" width="70px" src="https://www.alooma.com/img/integrations/s3.png"/></a>
@@ -21,6 +21,7 @@ A face and gesture recognition authentication system, the eye of horus is my Fin
     - [Architecture](#Architecture)
     - [Method](#Method)
 - [Management](#Management)
+- [Concessions](#Concessions)
 
 ## Vision
 
@@ -103,3 +104,10 @@ Due to university regulations on final year projects, my non code backlog is rec
 https://outlook.office365.com/owa/eyeofhorusPlanner@group.plymouth.ac.uk/groupsubscription.ashx?action=join&source=MSExchange/LokiServer&guid=a322b80f-c38a-44dd-b956-e9b43f82ec87
 
 Code related tasks are recorded on the [GitHub Project Board](https://github.com/M-Davies/eye-of-horus/projects/1) to allow for greater automation and drawing of references between tasks. See active and past sprints (that are tied to the board and individual tasks) by [viewing the project milestones](https://github.com/M-Davies/eye-of-horus/milestones).
+
+## Concessions
+
+These concessions are known bugs/problems with either the website, python library, aws services or all or some of the above.
+
+- **Facial Recognition:** The python library employs counter presentation attack tactics to ensure a picture of a user's face cannot be used to authenticate without their permission. Inevitably, this may result in false positives where the system thinks your real face is a fake image being shown to it. Please ensure that when authenticating with the system, you position your face to the same position as your stored face as close as humanly possible. If that doesn't work, ensure that you are using the same device to authenticate as you did to take the stored photo and adjusting your background to be less cluttered.
+- **Gesture Recognition:** Gesture recognition is plagued by inaccuracies due to how similar each gesture type is to each other. If your gestures are being identified incorrectly (or not at all), try following the steps at the end of the [training notes document](testing/gesture/TrainingNotes.md).

src/components/middleware.js

@@ -43,4 +43,46 @@ export function uploadFiles(files) {
                 return "Server error in uploading files, please try again later"
             }
         })
-}
+}
+
+export function checkIfLock(username) {
+    let params = new FormData()
+    params.append("user", username)
+
+    return axios.post("http://localhost:3001/user/hasLock", params)
+        .then(res => {
+            if (res.status === 200) {
+                return res.data
+            } else {
+                return JSON.stringify(res.data)
+            }
+        })
+        .catch(function (error) {
+            try {
+                return error.response.data
+            } catch {
+                return "Server error in authenticating user, please try again later"
+            }
+        })
+}
+
+export function checkCombination(paths) {
+    let params = new FormData()
+    params.append("files", paths)
+
+    return axios.post("http://localhost:3001/types", params)
+        .then(res => {
+            if (res.status === 200) {
+                return Array.from(res.data)
+            } else {
+                return JSON.stringify(res.data)
+            }
+        })
+        .catch(function (error) {
+            try {
+                return error.response.data
+            } catch {
+                return "Server error in checking gesture combination types"
+            }
+        })
+}

src/components/views/authenticate.js

@@ -7,7 +7,7 @@ import PropTypes from 'prop-types'
 import Webcam from 'react-webcam'
 import axios from 'axios'
 
-import { uploadFiles, uploadEncoded } from '../middleware'
+import { uploadFiles, uploadEncoded, checkCombination } from '../middleware'
 
 import "../../styles/authenticate.css"
 
@@ -21,8 +21,8 @@ export default function AuthenticateComponent({
 }) {
     const [loading, setLoading] = useState(false)
     const [streaming, setStreaming] = useState(false)
-    const [lockFiles, setLockFiles] = useState()
-    const [unlockFiles, setUnlockFiles] = useState()
+    const [lockFiles, setLockFiles] = useState({})
+    const [unlockFiles, setUnlockFiles] = useState({})
     const [lockDisplay, setLockDisplay] = useState([
         <ListGroup.Item variant="secondary" key="lock-placeholder">No lock gestures added</ListGroup.Item>
     ])
@@ -46,21 +46,46 @@ export default function AuthenticateComponent({
             params.append("face", facePath)
         }
 
-        if (lockFiles === undefined || lockFiles === null) { return "No lock files were selected" }
-        const lockPaths = await uploadFiles(Array.from(lockFiles))
-        if (!lockPaths instanceof Array) { return "Failed to upload lock files" }
-        params.append("locks", lockPaths)
+        if (lockFiles) {
+            const lockPaths = await uploadFiles(Array.from(lockFiles))
+            if (!lockPaths instanceof Array) { return "Failed to upload lock files" }
 
-        if (unlockFiles === undefined || unlockFiles === null) { return "No unlock files were selected" }
+            // Verify combination is what the user expected
+            const identifiedGestures = await checkCombination(lockPaths)
+            if (identifiedGestures instanceof Array) {
+                if (identifiedGestures.includes("UNKNOWN")) {
+                    return `One or more gestures haven't been identified as a known gesture type, please ensure your image clearly shows the gesture being performed\n${identifiedGestures.join(' ')}`
+                } else {
+                    if (!window.confirm(`Identified your lock gesture combination as the below. Is this correct?\n${identifiedGestures.join(' ')}`)) {
+                        return "Please chose images for your gesture combination that clearly show the gesture type you wish to use"
+                    }
+                }
+            } else {
+                return "Failed to query server on gestures given, please try again later"
+            }
+
+            params.append("locks", lockPaths)
+        }
+
+        if (!unlockFiles) { return "No unlock files were selected" }
         const unlockPaths = await uploadFiles(Array.from(unlockFiles))
         if (!unlockPaths instanceof Array) { return "Failed to upload unlock files" }
-        params.append("unlocks", unlockPaths)
 
-        console.log("Requesting user create with params")
-        console.log(params.username)
-        console.log(params.face)
-        console.log(params.locks)
-        console.log(params.unlocks)
+        // Verify combination is what the user expected
+        const identifiedGestures = await checkCombination(unlockPaths)
+        if (identifiedGestures instanceof Array) {
+            if (identifiedGestures.includes("UNKNOWN")) {
+                return `One or more gestures haven't been identified as a known gesture type, please ensure your image clearly shows the gesture being performed\n${identifiedGestures.join(' ')}`
+            } else {
+                if (!window.confirm(`Identified your unlock gesture combination as the below\nIs this correct?\n${identifiedGestures.join(' ')}`)) {
+                    return "Please chose images for your gesture combination that clearly show the gesture type you wish to use"
+                }
+            }
+        } else {
+            return "Failed to query server on gestures given, please try again later"
+        }
+
+        params.append("unlocks", unlockPaths)
 
         // Create user profile
         return axios.post("http://localhost:3001/user/create", params)
@@ -72,9 +97,9 @@ export default function AuthenticateComponent({
                 }
             })
             .catch(function (error) {
-                if (error.response.data) {
+                try {
                     return error.response.data
-                } else {
+                } catch {
                     return "Server error in user creation, please try again later"
                 }
             })
@@ -93,9 +118,23 @@ export default function AuthenticateComponent({
             params.append("face", facePath)
         }
 
-        if (unlockFiles === undefined || unlockFiles === null) { return "No unlock files were selected" }
+        if (!unlockFiles) { return "No unlock files were selected" }
         const unlockPaths = await uploadFiles(Array.from(unlockFiles))
         if (!unlockPaths instanceof Array) { return "Failed to upload unlock files" }
+
+        const identifiedGestures = await checkCombination(unlockPaths)
+        if (identifiedGestures instanceof Array) {
+            if (identifiedGestures.includes("UNKNOWN")) {
+                return `One or more gestures haven't been identified as a known gesture type, please ensure your image clearly shows the gesture being performed\n${identifiedGestures.join(' ')}`
+            } else {
+                if (!window.confirm(`Identified your unlock gesture combination as the below\nIs this correct?\n${identifiedGestures.join(' ')}`)) {
+                    return "Please chose images for your gesture combination that clearly show the gesture type you wish to use"
+                }
+            }
+        } else {
+            return "Failed to query server on gestures given, please try again later"
+        }
+
         params.append("unlocks", unlockPaths)
 
         // Authenticate user
@@ -108,22 +147,22 @@ export default function AuthenticateComponent({
                 }
             })
             .catch(function (error) {
-                if (error.response.data) {
+                try {
                     return error.response.data
-                } else {
+                } catch {
                     return "Server error in user authentication, please try again later"
                 }
             })
     }
 
-    const handleLockChange = (files) => {
+    const handleLockChange = async (files) => {
         setLockFiles(files)
-        generateFileList(files, null)
+        await generateFileList(files, null)
     }
 
-    const handleUnlockChange = (files) => {
+    const handleUnlockChange = async (files) => {
         setUnlockFiles(files)
-        generateFileList(null, files)
+        await generateFileList(null, files)
     }
 
     const handleSubmit = async e => {
@@ -147,7 +186,7 @@ export default function AuthenticateComponent({
                 } else {
                     alert(`${userCreateRes.TYPE}\n${userCreateRes.MESSAGE}`)
                 }
-                window.location.href = "/register"
+                window.location.reload()
             }
         } else {
             // Send login request to server
@@ -214,15 +253,14 @@ export default function AuthenticateComponent({
 
     function getGestureForms() {
         if (registering) {
-            // If this is a registration page, generate the editable forms depending on the given
             return (
                 <div className="gesture-forms">
                     <Form.Group onChange={(e) => handleLockChange(e.target.files)}>
                         <Form.File
                             id="lock_gesture_form"
                             type="file"
                         >
-                            <Form.File.Label>Chose at least 4 gestures as your lock gesture combination</Form.File.Label>
+                            <Form.File.Label>Chose at least 4 gestures as your lock gesture combination (OPTIONAL)</Form.File.Label>
                             <Form.File.Input multiple/>
                         </Form.File>
                         <Form.Check
@@ -307,11 +345,13 @@ export default function AuthenticateComponent({
         }
     }
 
-    if (authenticated === true) {
+    if (authenticated) {
         window.location.href = "/dashboard"
-    } else if (userExists === true && window.location.pathname === "/register") {
+    } else if (!username || !localStorage.getItem("exists")) {
+        window.location.href = "/"
+    } else if (localStorage.getItem("exists") === 'true' && window.location.pathname === "/register") {
         window.location.href = "/login"
-    } else if (userExists === false && window.location.pathname === "/login") {
+    } else if (localStorage.getItem("exists") === 'false' && window.location.pathname === "/login") {
         window.location.href = "/register"
     } else {
         if (navigator.mediaDevices.getUserMedia !== null) {
@@ -322,10 +362,7 @@ export default function AuthenticateComponent({
                 function (e) {
                     setStreaming(false)
                     if (e.name === "NotAllowedError") {
-                        console.log("Video perms denied")
                         document.getElementById("video_display").hidden = true
-                    } else {
-                        console.log("background error : " + e.name)
                     }
                 }
             )

src/components/views/dashboard.js

@@ -36,14 +36,14 @@ export default function DashboardComponent({ username, authenticated }) {
                 })
                 .catch(function (error) {
                     setDeleting(false)
-                    if (error.response.data) {
+                    try {
                         const responseData = JSON.stringify(error.response.data)
                         if (responseData.TYPE === undefined) {
                             alert(`${responseData}`)
                         } else {
                             alert(`${responseData.TYPE}\n${responseData.MESSAGE}`)
                         }
-                    } else {
+                    } catch {
                         alert("Server error in user deletion, please try again later")
                     }
                     window.location.href = "/dashboard"