/
sketch.js
199 lines (174 loc) · 4.45 KB
/
sketch.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
// ml5 Face Detection Model
let faceapi;
let detections = [];
let dx = 0;
let dy = 0;
let helloRegistered = false;
let square;
let square2;
let serial;
let portName = '/dev/tty.usbmodem14641';
let inData;
let outByte = 0;
let video;
let timestamp = 0;
let capture;
let high = 1;
let low = 0;
let posX = 0;
let posY = 0;
let oldX = 0;
let oldY = 0;
let ardX = 0;
let ardY = 0;
let oldArdX = 0;
let oldArdY = 0;
let poses = [];
let isPerson = false;
let speechRec = new p5.SpeechRec(); // speech recognition object
speechRec.continuous = false; // allow continuous recognition
speechRec.interimResults = true; // allow partial recognition (faster, less accurate)
speechRec.onResult = showResult; // callback function that triggers when speech is recognized
speechRec.onError = showError; // callback function that triggers when an error occurs
speechRec.onEnd = onVoiceRecognitionEnd; // callback function that triggers voice recognition ends
let voice = new p5.Speech(); // speech synthesis object
function setup() {
square = new Square(100, 100);
square2 = new Square(300, 100);
createCanvas(640, 480);
video = createCapture(VIDEO);
video.size(width, height);
// const detectionOptions = {
// withLandmarks: true,
// withDescriptors: false,
// };
// faceapi = ml5.faceApi(video, detectionOptions, faceReady);
// Create a new poseNet method
const poseNet = ml5.poseNet(video, modelLoaded);
// When the model is loaded
function modelLoaded() {
console.log('Model Loaded!');
}
// Listen to new 'pose' events
poseNet.on('pose', function (results) {
poses = results;
});
video.hide();
serial = new p5.SerialPort(); // make a new instance of the serialport library
serial.on('data', serialEvent); // callback for when new data arrives
serial.on('error', serialError); // callback for errors
serial.open(portName);
posX = width / 2;
posY = height / 2;
getAudioContext().suspend();
}
function draw() {
frameRate(10);
image(video, 0, 0, width, height);
square.draw();
square2.draw();
drawKeypoints();
}
function onVoiceRecognitionEnd() {
console.log(
'Voice recognition ended!!!, The message is ' + speechRec.resultString
);
if (speechRec.resultString != undefined) {
if (speechRec.resultString.includes('hello')) {
square.rightWord();
helloRegistered = false;
} else {
square.wrongWord();
}
voice.speak(speechRec.resultString);
}
listen();
}
function listen() {
speechRec.start(); // start listening
console.log("I'm listening...");
}
function showResult() {
// console.log('Transcript: ' + speechRec.resultString); // log the transcript
// console.log('Confidence: ' + speechRec.resultConfidence); // log the confidence
}
function showError() {
console.log('An error occurred!');
}
// function faceReady() {
// faceapi.detect(gotFaces);
// }
// function gotFaces(error, result) {
// if (error) {
// console.log('error' + error);
// return;
// }
// detections = result;
// faceapi.detect(gotFaces);
// }
function serialEvent() {
// read a byte from the serial port:
let inByte = serial.read();
// store it in a global variable:
inData = inByte;
}
function serialError(err) {
console.log('Something went wrong with the serial port. ' + err);
}
// A function to draw ellipses over the detected keypoints
function drawKeypoints() {
if (poses.length > 0) {
if (poses[0].pose.nose != undefined) {
square2.rightWord();
isPerson = true;
}
dx = poses[0].pose.nose.x;
dy = poses[0].pose.nose.y;
} else {
isPerson = false;
square2.wrongWord();
}
oldX = posX;
oldY = posY;
//map posX & posY to new variables for Arduino
posX += (dx - oldX) * 0.08; //was 0.1
posY += (dy - oldY) * 0.08;
//send to Arduino
oldArdX = ardX;
oldArdY = ardY;
ardX = int(map(posY, 0, width, 0, 50));
ardY = int(map(posX, 0, height, 50, 0));
//fill('teal');
stroke('red');
strokeWeight(2);
ellipse(posX, posY, 30, 30);
serial.write(
`${isPerson ? high : low},${
!helloRegistered ? high : low
},${ardX},${ardY}\n`
);
console.log(
`${isPerson ? high : low},${
!helloRegistered ? high : low
},${ardX},${ardY}\n`
);
helloRegistered = true;
}
class Square {
constructor(w, h) {
this.x = w;
this.y = h;
this.fill = 'white';
}
draw() {
fill(this.fill);
noStroke();
rect(this.x, this.y, 40, 40);
}
rightWord() {
this.fill = 'green';
}
wrongWord() {
this.fill = 'red';
}
}