Skip to content

Commit

Permalink
* Add OpenKinectFrameGrabber.grabIR() and stabilize `RealSenseFram…
Browse files Browse the repository at this point in the history 
…eGrabber` (pull #585)

- OpenKinect now have an API for IR video.
- Realsense default RGB is now 720p.
  • Loading branch information
@poqudrof @saudet
poqudrof authored and saudet committed Dec 24, 2016
1 parent f630183 commit 3d3b2b8
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 34 deletions.
23 changes: 21 additions & 2 deletions src/main/java/org/bytedeco/javacv/OpenKinectFrameGrabber.java
View file
Expand Up @@ -96,8 +96,9 @@ public void release() throws Exception {
private int deviceNumber = 0;
private boolean depth = false; // default to "video"
private BytePointer rawDepthImageData = new BytePointer((Pointer)null),
rawVideoImageData = new BytePointer((Pointer)null);
private IplImage rawDepthImage = null, rawVideoImage = null, returnImage = null;
rawVideoImageData = new BytePointer((Pointer)null),
rawIRImageData = new BytePointer((Pointer)null);
private IplImage rawDepthImage = null, rawVideoImage = null, rawIRImage = null, returnImage = null;
private FrameConverter converter = new OpenCVFrameConverter.ToIplImage();
private int[] timestamp = { 0 };
private ByteOrder byteOrder = ByteOrder.BIG_ENDIAN;
Expand Down Expand Up @@ -244,6 +245,24 @@ public IplImage grabVideo() throws Exception {
super.timestamp = timestamp[0];
return rawVideoImage;
}

public IplImage grabIR() throws Exception {
int iplDepth = IPL_DEPTH_8U, channels = 1;

int err = freenect_sync_get_video(rawIRImageData, timestamp, deviceNumber, FREENECT_VIDEO_IR_8BIT);
if (err != 0) {
throw new Exception("freenect_sync_get_video() Error " + err + ": Failed to get video synchronously.");
}

int w = 640, h = 480; // how to get the resolution ??
if (rawIRImage == null || rawIRImage.width() != w || rawIRImage.height() != h) {
rawIRImage = IplImage.createHeader(w, h, iplDepth, channels);
}
cvSetData(rawIRImage, rawIRImageData, w*channels*iplDepth/8);

super.timestamp = timestamp[0];
return rawIRImage;
}

public Frame grab() throws Exception {
IplImage image = depth ? grabDepth() : grabVideo();
Expand Down
119 changes: 87 additions & 32 deletions src/main/java/org/bytedeco/javacv/RealSenseFrameGrabber.java
View file
Expand Up @@ -50,17 +50,18 @@ public static String[] getDeviceDescriptions() throws FrameGrabber.Exception {

public static int DEFAULT_DEPTH_WIDTH = 640;
public static int DEFAULT_DEPTH_HEIGHT = 480;
public static int DEFAULT_COLOR_WIDTH = 640;
public static int DEFAULT_COLOR_HEIGHT = 480;
public static int DEFAULT_COLOR_WIDTH = 1280;
public static int DEFAULT_COLOR_HEIGHT = 720;
public static int DEFAULT_COLOR_FRAMERATE = 30;

private ByteOrder byteOrder = ByteOrder.BIG_ENDIAN;
private int depthImageWidth = DEFAULT_DEPTH_WIDTH;
private int depthImageHeight = DEFAULT_DEPTH_HEIGHT;
private int depthFrameRate = 60;
private int depthFrameRate = 30;

private int IRImageWidth = DEFAULT_DEPTH_WIDTH;
private int IRImageHeight = DEFAULT_DEPTH_HEIGHT;
private int IRFrameRate = 60;
private int IRFrameRate = 30;

public ByteOrder getByteOrder() {
return byteOrder;
Expand Down Expand Up @@ -136,6 +137,9 @@ public void enableColorStream() {
if (imageHeight == 0) {
imageHeight = DEFAULT_COLOR_HEIGHT;
}
if (frameRate == 0) {
frameRate = DEFAULT_COLOR_FRAMERATE;
}
device.enable_stream(RealSense.color, imageWidth, imageHeight, RealSense.rgb8, (int) frameRate);
colorEnabled = true;
}
Expand Down Expand Up @@ -203,8 +207,22 @@ public double getFrameRate() { // TODO: check this.
return super.getFrameRate();
}

private boolean startedOnce = false;
private boolean behaveAsColorFrameGrabber = false;

@Override
public void start() throws FrameGrabber.Exception {
// if no stream is select, just get the color.
if (!colorEnabled && !IREnabled && !depthEnabled) {

if (!startedOnce) {
enableColorStream();
behaveAsColorFrameGrabber = true;
this.setImageMode(ImageMode.GRAY);
}
}

startedOnce = true;
device.start();
}

Expand All @@ -215,14 +233,17 @@ public void start() throws FrameGrabber.Exception {
@Override
public void stop() throws FrameGrabber.Exception {
device.stop();
colorEnabled = false;
IREnabled = false;
depthEnabled = false;
frameNumber = 0;
}
private Pointer rawDepthImageData = new Pointer((Pointer) null),
rawVideoImageData = new Pointer((Pointer) null),
rawIRImageData = new Pointer((Pointer) null);
private IplImage rawDepthImage = null, rawVideoImage = null, rawIRImage = null, returnImage = null;

public IplImage grabDepth() throws Exception {
public IplImage grabDepth() {

if (!depthEnabled) {
System.out.println("Depth stream not enabled, impossible to get the image.");
Expand All @@ -241,7 +262,7 @@ public IplImage grabDepth() throws Exception {
if (rawDepthImage == null || rawDepthImage.width() != deviceWidth || rawDepthImage.height() != deviceHeight) {
rawDepthImage = IplImage.createHeader(deviceWidth, deviceHeight, iplDepth, channels);
}

cvSetData(rawDepthImage, rawDepthImageData, deviceWidth * channels * iplDepth / 8);

if (iplDepth > 8 && !ByteOrder.nativeOrder().equals(byteOrder)) {
Expand All @@ -268,24 +289,23 @@ public IplImage grabVideo() {
rawVideoImageData = device.get_frame_data(RealSense.color);
int deviceWidth = device.get_stream_width(RealSense.color);
int deviceHeight = device.get_stream_height(RealSense.color);

if (rawVideoImage == null || rawVideoImage.width() != deviceWidth || rawVideoImage.height() != deviceHeight) {
rawVideoImage = IplImage.createHeader(deviceWidth, deviceHeight, iplDepth, channels);
}

cvSetData(rawVideoImage, rawVideoImageData, deviceWidth * channels * iplDepth / 8);

if (iplDepth > 8 && !ByteOrder.nativeOrder().equals(byteOrder)) {
// ack, the camera's endianness doesn't correspond to our machine ...
// swap bytes of 16-bit images
ByteBuffer bb = rawVideoImage.getByteBuffer();
ShortBuffer in = bb.order(ByteOrder.BIG_ENDIAN).asShortBuffer();
ShortBuffer out = bb.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer();
out.put(in);
}
cvSetData(rawVideoImage, rawVideoImageData, deviceWidth * channels * iplDepth / 8);

// if (iplDepth > 8 && !ByteOrder.nativeOrder().equals(byteOrder)) {
// // ack, the camera's endianness doesn't correspond to our machine ...
// // swap bytes of 16-bit images
// ByteBuffer bb = rawVideoImage.getByteBuffer();
// ShortBuffer in = bb.order(ByteOrder.BIG_ENDIAN).asShortBuffer();
// ShortBuffer out = bb.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer();
// out.put(in);
// }
// if (channels == 3) {
// cvCvtColor(rawVideoImage, rawVideoImage, CV_RGB2BGR);
// cvCvtColor(rawVideoImage, rawVideoImage, CV_BGR2RGB);
// }
return rawVideoImage;
}
Expand All @@ -304,21 +324,19 @@ public IplImage grabIR() {
int deviceWidth = device.get_stream_width(RealSense.infrared);
int deviceHeight = device.get_stream_height(RealSense.infrared);


if (rawIRImage == null || rawIRImage.width() != deviceWidth || rawIRImage.height() != deviceHeight) {
rawIRImage = IplImage.createHeader(deviceWidth, deviceHeight, iplDepth, channels);
}
cvSetData(rawIRImage, rawIRImageData, deviceWidth * channels * iplDepth / 8);

if (iplDepth > 8 && !ByteOrder.nativeOrder().equals(byteOrder)) {
// ack, the camera's endianness doesn't correspond to our machine ...
// swap bytes of 16-bit images
ByteBuffer bb = rawIRImage.getByteBuffer();
ShortBuffer in = bb.order(ByteOrder.BIG_ENDIAN).asShortBuffer();
ShortBuffer out = bb.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer();
out.put(in);
}

// if (iplDepth > 8 && !ByteOrder.nativeOrder().equals(byteOrder)) {
// // ack, the camera's endianness doesn't correspond to our machine ...
// // swap bytes of 16-bit images
// ByteBuffer bb = rawIRImage.getByteBuffer();
// ShortBuffer in = bb.order(ByteOrder.BIG_ENDIAN).asShortBuffer();
// ShortBuffer out = bb.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer();
// out.put(in);
// }
return rawIRImage;
}

Expand All @@ -331,10 +349,20 @@ public IplImage grabIR() {
public Frame grab() throws Exception {
device.wait_for_frames();

// if (colorEnabled) {
// IplImage image = grabVideo();
// return converter.convert(image);
// }
// frameNumber++;
// For Framegrabber
if (colorEnabled && behaveAsColorFrameGrabber) {
IplImage image = grabVideo();

if (returnImage == null) {
int deviceWidth = device.get_stream_width(RealSense.color);
int deviceHeight = device.get_stream_height(RealSense.color);
// returnImage = IplImage.create(deviceWidth, deviceHeight, IPL_DEPTH_8U, 3);
returnImage = IplImage.create(deviceWidth, deviceHeight, IPL_DEPTH_8U, 1);
}
cvCvtColor(image, returnImage, CV_BGR2GRAY);
return converter.convert(returnImage);
}
return null;
}

Expand Down Expand Up @@ -383,6 +411,32 @@ public void setDepthFrameRate(int frameRate) {
this.depthFrameRate = frameRate;
}

public int getIRFrameRate() {
return IRFrameRate;
}

public void setIRFrameRate(int IRFrameRate) {
this.IRFrameRate = IRFrameRate;
}

@Override
public double getGamma() {
// I guess a default gamma of 2.2 is reasonable...
if (gamma == 0.0) {
return 2.2;
} else {
return gamma;
}
}

// Gamma from the device is not good.
// @Override
// public double getGamma(){
// double gamma = device.get_option(RealSense.RS_OPTION_COLOR_GAMMA);
// System.out.println("Getting cameraGamma " + gamma);
// return gamma;
// }

// --- Presets ---
public void setPreset(int preset) {
/* Provide access to several recommend sets of option presets for ivcam */
Expand Down Expand Up @@ -473,6 +527,7 @@ public void setColorGamma(int value) {
setOption(RealSense.RS_OPTION_COLOR_GAMMA, value);
}


/**
* Color image hue
*/
Expand Down

0 comments on commit 3d3b2b8

Please sign in to comment.