-
Notifications
You must be signed in to change notification settings - Fork 0
/
script.js
217 lines (189 loc) · 6.76 KB
/
script.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
// declare variables
let video;
let poseNet;
let keypointArray = [];
let avgArray = [];
let points = [];
let fr = 60;
let vStart;
let vEnd;
let dst;
let bodyDst;
let noseXL = 0;
let noseYL = 0;
let lEyeXL = 0;
let lEyeYL = 0;
let lWristXL = 0;
let lWristYL = 0;
let rWristXL = 0;
let rWristYL = 0;
let noseX, noseY,
lEyeX, lEyeY, rEyeX, rEyeY,
lEarX, lEarY, rEarX, rEarY,
lShouldX, lShouldY, rShouldX, rShouldY,
lElbowX, lElbowY, rElbowX, rElbowY,
lWristX, lWristY, rWristX, rWristY,
lHipX, lHipY, rHipX, rHipY,
lKneeX, lKneeY, rKneeX, rKneeY,
lAnkleX, lAnkleY, rAnkleX, rAnkleY;
// open WebSocket on port 8080
const socket = io.connect('http://localhost:8080');
function setup() {
createCanvas(640, 475);
video = createCapture(VIDEO);
// hide original unfiltered camera feed
video.hide();
// create an instance of poseNet object
// with default Output Stride and Image Scale values
poseNet = ml5.poseNet(video, { flipHorizontal: false, outputStride: 16, imageScaleFactor: 0.5}, modelReady);
// execute gotPoses whenever new keypoint data is available
poseNet.on('pose', gotPoses);
}
// find the average of 100 console logs
// starting at 50 to avoid skewed values during startup
function average(start, end, name) {
var time = end - start;
var avgArraySecHalf = avgArray.slice(49, 101);
avgArray.push(time);
// log points 25 and 50 to track progress
if (avgArray.length == 25) {
console.log("Array index 25");
} else if (avgArray.length == 50) {
console.log("Array index 50");
} else if (avgArray.length == 100) {
let sum = avgArraySecHalf.reduce((previous, current) => current += previous);
let avg = sum / avgArraySecHalf.length;
console.log("The average " + name + " value is " + avg + " ms.");
}
}
// function executed when new pose data becomes available
function gotPoses(poses) {
// start time for performance testing
var kpStart = window.performance.now();
// assign keypoint coordinates to variables
if (poses.length > 0) {
noseX = poses[0].pose.keypoints[0].position.x;
noseY = poses[0].pose.keypoints[0].position.y;
lEyeX = poses[0].pose.keypoints[1].position.x;
lEyeY = poses[0].pose.keypoints[1].position.y;
rEyeX = poses[0].pose.keypoints[2].position.x;
rEyeY = poses[0].pose.keypoints[2].position.y;
lEarX = poses[0].pose.keypoints[3].position.x;
lEarY = poses[0].pose.keypoints[3].position.y;
rEarX = poses[0].pose.keypoints[4].position.x;
rEarY = poses[0].pose.keypoints[4].position.y;
lShouldX = poses[0].pose.keypoints[5].position.x;
lShouldY = poses[0].pose.keypoints[5].position.y;
rShouldX = poses[0].pose.keypoints[6].position.x;
rShouldY = poses[0].pose.keypoints[6].position.y;
lElbowX = poses[0].pose.keypoints[7].position.x;
lElbowY = poses[0].pose.keypoints[7].position.y;
rElbowX = poses[0].pose.keypoints[8].position.x;
rElbowY = poses[0].pose.keypoints[8].position.y;
lWristX = poses[0].pose.keypoints[9].position.x;
lWristY = poses[0].pose.keypoints[9].position.y;
rWristX = poses[0].pose.keypoints[10].position.x;
rWristY = poses[0].pose.keypoints[10].position.y;
lHipX = poses[0].pose.keypoints[11].position.x;
lHipY = poses[0].pose.keypoints[11].position.y;
rHipX = poses[0].pose.keypoints[12].position.x;
rHipY = poses[0].pose.keypoints[12].position.y;
lKneeX = poses[0].pose.keypoints[13].position.x;
lKneeY = poses[0].pose.keypoints[13].position.y;
rKneeX = poses[0].pose.keypoints[14].position.x;
rKneeY = poses[0].pose.keypoints[14].position.y;
lAnkleX = poses[0].pose.keypoints[15].position.x;
lAnkleY = poses[0].pose.keypoints[15].position.y;
rAnkleX = poses[0].pose.keypoints[16].position.x;
rAnkleY = poses[0].pose.keypoints[16].position.y;
// linear interpolation to smooth transitions
noseXL = lerp(noseXL, noseX, 0.5);
noseYL = lerp(noseYL, noseY, 0.5);
lEyeXL = lerp(lEyeXL, lEyeX, 0.5);
lEyeYL = lerp(lEyeYL, lEyeY, 0.5);
lWristXL = lerp(lWristXL, lWristX, 0.5);
lWristYL = lerp(lWristYL, lWristY, 0.5);
rWristXL = lerp(rWristXL, rWristX, 0.5);
rWristYL = lerp(rWristYL, rWristY, 0.5);
// distance from right to left wrist
bodyDst = dist(rWristX, rWristY, lWristX, lWristY);
// create keypoint variable for animation
keypointArray = poses[0].pose.keypoints;
// create new local array to send
points = [noseXL, noseYL,
lEyeXL, lEyeYL,
lWristXL, lWristYL,
rWristXL, rWristYL,
dst, bodyDst];
// send the array under the identifier pointSender
socket.emit('pointSender', points);
}
// end time for performance testing
var kpEnd = window.performance.now();
// find average performace time
average(kpStart, kpEnd, "keypoint");
}
// executed when poseNet object is instantiated
function modelReady() {
console.log('model ready');
}
function myDraw() {
setInterval(function() {
// start time for performace testing
vStart = window.performance.now();
// draw video
image(video, 0, 0);
// apply artistic filter
filter(POSTERIZE, 2);
// distance between nose and eye for 3D approximation
dst = dist(noseXL, noseYL, lEyeXL, lEyeYL);
// draw nose keypoint
fill(100, 150, 255);
strokeWeight(2);
stroke(0, 0, 0);
ellipse(noseXL, noseYL, dst);
// draw wrist keypoints
fill(255, 0, 0);
ellipse(lWristXL, lWristYL, dst);
ellipse(rWristXL, rWristYL, dst);
drawSkeleton();
// draw skeleton keypoints
fill(0, 255, 0);
drawKeypoints();
// end time for performance testing
vEnd = window.performance.now();
average(vStart, vEnd, "video latency");
}, 1000/fr);
}
myDraw();
// draw lines between keypoints for skeleton
function myLine(x1, y1, x2, y2) {
if (x1 && x2) {
line(x1, y1, x2, y2);
}
}
// A function to draw the skeletons
function drawSkeleton() {
stroke(250, 105, 180);
strokeWeight(4);
myLine(rWristX, rWristY, rElbowX, rElbowY);
myLine(rElbowX, rElbowY, rShouldX, rShouldY);
myLine(rShouldX, rShouldY, lShouldX, lShouldY);
myLine(lShouldX, lShouldY, lElbowX, lElbowY);
myLine(lElbowX, lElbowY, lWristX, lWristY);
myLine(rShouldX, rShouldY, rHipX, rHipY);
myLine(lShouldX, lShouldY, lHipX, lHipY);
myLine(rHipX, rHipY, lHipX, lHipY);
myLine(rHipX, rHipY, rKneeX, rKneeY);
myLine(lHipX, lHipY, lKneeX, lKneeY);
myLine(rKneeX, rKneeY, rAnkleX, rAnkleY);
myLine(lKneeX, lKneeY, lAnkleX, lAnkleY);
}
// loop through array and draw all body keypoints
function drawKeypoints() {
for (i = 5; i < keypointArray.length; i++) {
if (keypointArray[i].position.x && keypointArray[i].position.y) {
ellipse(keypointArray[i].position.x, keypointArray[i].position.y, dst/2);
}
}
}