Permalink
Browse files

successfully streaming joint data to threejs; need to position camera…

… better
  • Loading branch information...
1 parent 2b20494 commit 9ec7fb9b47df2c5ccd51ac4a6ed1e0ba44ba34e2 Greg Borenstein committed Oct 26, 2011
Showing with 255 additions and 41 deletions.
  1. +15 −15 CakeThread.pde
  2. +6 −3 node/three_test.html
  3. +12 −16 node/web.js
  4. +222 −7 skelestreamer.pde
View
@@ -1,21 +1,21 @@
import postdata.*;
class Skeletor {
- PVector head;
- PVector neck;
- PVector rightShoulder;
- PVector rightElbow;
- PVector rightHand;
- PVector leftShoulder;
- PVector leftElbow;
- PVector leftHand;
- PVector torso;
- PVector rightHip;
- PVector rightKnee;
- PVector rightFoot;
- PVector leftHip;
- PVector leftKnee;
- PVector leftFoot;
+ PVector head;
+ PVector neck;
+ PVector rightShoulder;
+ PVector rightElbow;
+ PVector rightHand;
+ PVector leftShoulder;
+ PVector leftElbow;
+ PVector leftHand;
+ PVector torso;
+ PVector rightHip;
+ PVector rightKnee;
+ PVector rightFoot;
+ PVector leftHip;
+ PVector leftKnee;
+ PVector leftFoot;
String uuid;
Skeletor() {
View
@@ -14,19 +14,20 @@
//$(function(){
+
+ //$("#start").click(function(){
var socket = io.connect('http://localhost:5000');
socket.on('data', function (data) {
$.forEach(data, function(value, key, i){
- //console.log($("#" + key));
if(key == "session"){
$("#" + key).html(value);
} else{
jointData[key] = value;
}
});
- //console.log(eval(data));
});
//});
+
var camera, scene, renderer,
geometry, material, mesh;
@@ -65,6 +66,7 @@
renderer.setSize( window.innerWidth, window.innerHeight );
$("body").append( renderer.domElement );
+
}
@@ -77,7 +79,7 @@
}
function render() {
-
+ console.log(jointData.head.x);
$.forEach(jointData, function(value, key, i){
// console.log(key);
@@ -103,5 +105,6 @@
</head>
<body>
<h1>Test</h1>
+ <p><a href="#" id="start">start stream</a></p>
</body>
</html>
View
@@ -20,26 +20,22 @@ var server = net.createServer(function (socket) {
gsock.emit("json", json);
message = message.slice(newlineIndex + 1);
}
-
- })
+ });
});
server.listen(1337, "127.0.0.1");
io.sockets.on('connection', function (socket) {
- console.log("browser connected");
- //socket.emit('data', {message : "welcome browser"});
+ console.log("browser connected");
+ gsock.on("json", function(data) {
+ try {
+ json = JSON.parse(data);
+ socket.emit("data", json);
+
+ } catch (Err) {
+ console.log("skipping: " + Err);
+ return; // continue
+ }
- gsock.on("json", function(data) {
- try {
- json = JSON.parse(data);
- console.log("sending");
- socket.emit("data", json);
-
- } catch (Err) {
- console.log("skipping: " + Err);
- return; // continue
- }
-
- });
+ });
});
View
@@ -1,30 +1,166 @@
import processing.net.*;
+import SimpleOpenNI.*;
+import fsm.*;
+
+SimpleOpenNI kinect;
+FSM app;
CakeThread cakeThread;
String recordingUUID;
+State setupMode = new State(this, "enterSetup", "doSetup", "exitSetup");
+State streamMode = new State(this, "enterStream", "doStream", "exitStream");
+
+boolean readyToStream = false;
+
+int userId;
+IntVector userList;
+
void setup() {
- size(500,500);
+ size(640,480);
+
+ // frameRate(30);
+
+ app = new FSM(setupMode);
+
+ kinect = new SimpleOpenNI(this);
+ kinect.enableDepth();
+ kinect.enableUser(SimpleOpenNI.SKEL_PROFILE_ALL);
+
+ userList = new IntVector();
+ enterSetup(); // FSM Bug: it should start by firing off the enter function for the starting state!
+
+
+}
+
+void draw() {
+ kinect.update();
+ image(kinect.depthImage(), 0, 0);
+ app.update();
+ //text(frameRate, 10,10);
+ // addSkeleton();
+}
+
+void mousePressed() {
+ if (app.isInState(setupMode) && readyToStream) {
+ app.transitionTo(streamMode);
+ }
+}
+
+void enterSetup() {
+ calculateNewUUID();
+ println("entering setup");
+}
+
+void doSetup() {
+ fill(0, 255,0);
+ textSize(30);
+
+ kinect.getUsers(userList);
+
+ if(userList.size() > 0) {
+ userId = userList.get(0);
+ if (kinect.isTrackingSkeleton(userId)) {
+ readyToStream = true;
+ text("Ready to stream.\nUUID: " + recordingUUID, 5, 25);
+ }
+ else {
+ text("Waiting for user calibration.\tUUID: "+ recordingUUID, 5, 25);
+ }
+ }
+ else {
+ text("Waiting for user.\tUUID: " + recordingUUID, 5, 25);
+ }
+}
+
+void exitSetup() {
+}
+
+void enterStream() {
cakeThread = new CakeThread(this);
cakeThread.start();
- frameRate(30);
- calculateNewUUID();
+ println("entering stream");
}
-void draw() {
- background(0);
- text(frameRate, 10,10);
- println(cakeThread.queue.size());
+void doStream() {
+ fill(0, 255,0);
+ textSize(30);
+ text("Streaming: "+ recordingUUID + "\nQueueSize: " + cakeThread.queue.size(), 5, 25);
+
+ drawSkeleton(userId);
+
addSkeleton();
+ //Skeletor last = (Skeletor)cakeThread.queue.get(cakeThread.queue.size() - 1);
+ println(cakeThread.queue.size());
+
}
+void exitStream(){}
+
void calculateNewUUID() {
recordingUUID = "SK" + random(100) + "A" + random(100) + "A" + random(100) + "A";
println(recordingUUID);
}
+
void addSkeleton() {
+ PVector head = new PVector();
+ PVector neck = new PVector();
+ PVector rightShoulder = new PVector();
+ PVector rightElbow = new PVector();
+ PVector rightHand = new PVector();
+ PVector leftShoulder = new PVector();
+ PVector leftElbow = new PVector();
+ PVector leftHand = new PVector();
+ PVector torso = new PVector();
+ PVector rightHip = new PVector();
+ PVector rightKnee = new PVector();
+ PVector rightFoot = new PVector();
+ PVector leftHip = new PVector();
+ PVector leftKnee = new PVector();
+ PVector leftFoot = new PVector();
+
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_HEAD, head);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_NECK, neck);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_RIGHT_SHOULDER, rightShoulder);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_RIGHT_ELBOW, rightElbow);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_RIGHT_HAND, rightHand);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_LEFT_SHOULDER, leftShoulder);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_LEFT_ELBOW, leftElbow);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_LEFT_HAND, leftHand);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_TORSO, torso);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_RIGHT_HIP, rightHip);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_RIGHT_KNEE, rightKnee);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_RIGHT_FOOT, rightFoot);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_LEFT_HIP, leftHip);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_LEFT_KNEE, leftKnee);
+ kinect.getJointPositionSkeleton(userId, kinect.SKEL_LEFT_FOOT, leftFoot);
+
+
+ Skeletor skel = new Skeletor();
+ skel.head = head;
+ skel.neck = neck;
+ skel.rightShoulder = rightShoulder;
+ skel.rightElbow = rightElbow;
+ skel.rightHand = rightHand;
+ skel.leftShoulder = leftShoulder;
+ skel.leftElbow = leftElbow;
+ skel.leftHand = leftHand;
+ skel.torso = torso;
+ skel.rightHip = rightHip;
+ skel.rightKnee = rightKnee;
+ skel.rightFoot = rightFoot;
+ skel.leftHip = leftHip;
+ skel.leftKnee = leftKnee;
+ skel.leftFoot = leftFoot;
+
+ skel.uuid = recordingUUID;
+
+ cakeThread.queue.add(skel);
+}
+
+void fakeSkeleton() {
Skeletor skel = new Skeletor();
skel.head = new PVector(random(1000), random(1000), random(1000));
skel.neck = new PVector(random(1000), random(1000), random(1000));
@@ -47,3 +183,82 @@ void addSkeleton() {
cakeThread.queue.add(skel);
}
+void drawSkeleton(int userId) {
+ stroke(0);
+ strokeWeight(5);
+
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_HEAD, SimpleOpenNI.SKEL_NECK);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_NECK, SimpleOpenNI.SKEL_LEFT_SHOULDER);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_SHOULDER, SimpleOpenNI.SKEL_LEFT_ELBOW);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_ELBOW, SimpleOpenNI.SKEL_LEFT_HAND);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_NECK, SimpleOpenNI.SKEL_RIGHT_SHOULDER);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_SHOULDER, SimpleOpenNI.SKEL_RIGHT_ELBOW);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_ELBOW, SimpleOpenNI.SKEL_RIGHT_HAND);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_SHOULDER, SimpleOpenNI.SKEL_TORSO);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_SHOULDER, SimpleOpenNI.SKEL_TORSO);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_TORSO, SimpleOpenNI.SKEL_LEFT_HIP);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_HIP, SimpleOpenNI.SKEL_LEFT_KNEE);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_KNEE, SimpleOpenNI.SKEL_LEFT_FOOT);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_TORSO, SimpleOpenNI.SKEL_RIGHT_HIP);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_HIP, SimpleOpenNI.SKEL_RIGHT_KNEE);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_KNEE, SimpleOpenNI.SKEL_RIGHT_FOOT);
+ kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_HIP, SimpleOpenNI.SKEL_LEFT_HIP);
+
+ noStroke();
+
+ fill(255, 0, 0);
+ drawJoint(userId, SimpleOpenNI.SKEL_HEAD);
+ drawJoint(userId, SimpleOpenNI.SKEL_NECK);
+ drawJoint(userId, SimpleOpenNI.SKEL_LEFT_SHOULDER);
+ drawJoint(userId, SimpleOpenNI.SKEL_LEFT_ELBOW);
+ drawJoint(userId, SimpleOpenNI.SKEL_NECK);
+ drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_SHOULDER);
+ drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_ELBOW);
+ drawJoint(userId, SimpleOpenNI.SKEL_TORSO);
+ drawJoint(userId, SimpleOpenNI.SKEL_LEFT_HIP);
+ drawJoint(userId, SimpleOpenNI.SKEL_LEFT_KNEE);
+ drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_HIP);
+ drawJoint(userId, SimpleOpenNI.SKEL_LEFT_FOOT);
+ drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_KNEE);
+ drawJoint(userId, SimpleOpenNI.SKEL_LEFT_HIP);
+ drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_FOOT);
+ drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_HAND);
+ drawJoint(userId, SimpleOpenNI.SKEL_LEFT_HAND);
+}
+
+void drawJoint(int userId, int jointID) {
+ PVector joint = new PVector();
+ float confidence = kinect.getJointPositionSkeleton(userId, jointID, joint);
+ if (confidence < 0.5) {
+ return;
+ }
+ PVector convertedJoint = new PVector();
+ kinect.convertRealWorldToProjective(joint, convertedJoint);
+ ellipse(convertedJoint.x, convertedJoint.y, 5, 5);
+}
+
+
+// user-tracking callbacks!
+void onNewUser(int userId) {
+ println("start pose detection");
+ kinect.startPoseDetection("Psi", userId);
+}
+
+void onEndCalibration(int userId, boolean successful) {
+ if (successful) {
+ println(" User calibrated !!!");
+ kinect.startTrackingSkeleton(userId);
+ }
+ else {
+ println(" Failed to calibrate user !!!");
+ kinect.startPoseDetection("Psi", userId);
+ }
+}
+
+void onStartPose(String pose, int userId) {
+ println("Started pose for user");
+ kinect.stopPoseDetection(userId);
+ kinect.requestCalibrationSkeleton(userId, true);
+}
+
+

0 comments on commit 9ec7fb9

Please sign in to comment.