Skip to content

Commit

Permalink
improvement for frame rate on capture and playback
Browse files Browse the repository at this point in the history 
- estimate FPS before video recording begins, and use that in the Encoder() arg
  • Loading branch information
@n8fr8
n8fr8 committed Mar 31, 2015
1 parent 0d264fa commit 6c00738
Show file tree
Hide file tree
Showing 5 changed files with 115 additions and 77 deletions.
1 change: 1 addition & 0 deletions src/info/guardianproject/iocipher/camera/StillCameraActivity.java
View file
Expand Up @@ -44,6 +44,7 @@ protected int getLayout()
public void onPictureTaken(final byte[] data, Camera camera) {
File fileSecurePicture;
try {

long mTime = System.currentTimeMillis();
fileSecurePicture = new File(mFileBasePath,"secureselfie_" + mTime + ".jpg");

Expand Down
142 changes: 80 additions & 62 deletions src/info/guardianproject/iocipher/camera/VideoCameraActivity.java
View file
Expand Up @@ -42,7 +42,7 @@ public class VideoCameraActivity extends CameraBaseActivity {
private String mFileBasePath = null;
private boolean mIsRecording = false;

private ArrayDeque<byte[]> mFrameQ = null;
private ArrayDeque<VideoFrame> mFrameQ = null;

private int mLastWidth = -1;
private int mLastHeight = -1;
Expand All @@ -55,16 +55,16 @@ public class VideoCameraActivity extends CameraBaseActivity {
private byte[] audioData;
private AudioRecord audioRecord;

private int mFramesTotal = 0;
private int mFPS = 0;

private boolean mPreCompressFrames = true;
private OutputStream outputStreamAudio;
private info.guardianproject.iocipher.File fileAudio;

private int frameCounter = 0;
private long start = 0;

private long lastTime = 0;
private int mFramesTotal = 0;
private int mFPS = 15; //default is 15fps

private boolean isRequest = false;

private boolean mInTopHalf = false;
Expand Down Expand Up @@ -115,16 +115,15 @@ public boolean onTouch(View v, MotionEvent ev) {
break;
case MotionEvent.ACTION_CANCEL:
case MotionEvent.ACTION_UP:
if (mIsOnClick) {
if (!mIsRecording) {

//take a picture
mPreviewing = false;
camera.takePicture(null, null, this);
handler.removeCallbacks(mLongPressed);

}

if (mIsRecording)
else
{
stopRecording();
}
Expand All @@ -143,8 +142,6 @@ public boolean onTouch(View v, MotionEvent ev) {
if (mIsOnClick && (Math.abs(mDownX - ev.getX()) > SCROLL_THRESHOLD || Math.abs(mDownY - ev.getY()) > SCROLL_THRESHOLD)) {
mIsOnClick = false;



}
break;
default:
Expand Down Expand Up @@ -187,10 +184,13 @@ public void onClick(View view) {

private void startRecording ()
{
mFrameQ = new ArrayDeque<byte[]>();
mFrameQ = new ArrayDeque<VideoFrame>();

mFramesTotal = 0;

frameCounter = 0;

lastTime = System.currentTimeMillis();

String fileName = "video" + new java.util.Date().getTime() + ".mov";
info.guardianproject.iocipher.File fileOut = new info.guardianproject.iocipher.File(mFileBasePath,fileName);

Expand All @@ -202,7 +202,10 @@ private void startRecording ()
else
initAudio(fileOut.getAbsolutePath()+".pcm");

new Encoder(fileOut).start();
boolean withEmbeddedAudio = false;

Encoder encoder = new Encoder(fileOut,mFPS,withEmbeddedAudio);
encoder.start();
//start capture
startAudioRecording();

Expand Down Expand Up @@ -266,62 +269,67 @@ public void run() {
@Override
public void onPreviewFrame(byte[] data, Camera camera) {

if (mIsRecording && mFrameQ != null)
//even when not recording, we'll compress frames in order to estimate our FPS

Camera.Parameters parameters = camera.getParameters();
mLastWidth = parameters.getPreviewSize().width;
mLastHeight = parameters.getPreviewSize().height;

if (mRotation > 0) //flip height and width
{

Camera.Parameters parameters = camera.getParameters();
mLastWidth = parameters.getPreviewSize().width;
mLastHeight = parameters.getPreviewSize().height;

if (mRotation > 0) //flip height and width
{
mLastWidth =parameters.getPreviewSize().height;
mLastHeight =parameters.getPreviewSize().width;
}

mPreviewFormat = parameters.getPreviewFormat();

byte[] dataResult = data;

if (mPreCompressFrames)
mLastWidth =parameters.getPreviewSize().height;
mLastHeight =parameters.getPreviewSize().width;
}

mPreviewFormat = parameters.getPreviewFormat();

byte[] dataResult = data;

if (mPreCompressFrames)
{
if (mRotation > 0)
{
if (mRotation > 0)
{
dataResult = rotateYUV420Degree90(data,mLastHeight,mLastWidth);

if (getCameraDirection() == CameraInfo.CAMERA_FACING_FRONT)
{
dataResult = rotateYUV420Degree90(dataResult,mLastWidth,mLastHeight);
dataResult = rotateYUV420Degree90(dataResult,mLastHeight,mLastWidth);
}

}
dataResult = rotateYUV420Degree90(data,mLastHeight,mLastWidth);

YuvImage yuv = new YuvImage(dataResult, mPreviewFormat, mLastWidth, mLastHeight, null);
ByteArrayOutputStream out = new ByteArrayOutputStream();
yuv.compressToJpeg(new Rect(0, 0, mLastWidth, mLastHeight), MediaConstants.sJpegQuality, out);
dataResult = out.toByteArray();
}

if (getCameraDirection() == CameraInfo.CAMERA_FACING_FRONT)
{
dataResult = rotateYUV420Degree90(dataResult,mLastWidth,mLastHeight);
dataResult = rotateYUV420Degree90(dataResult,mLastHeight,mLastWidth);
}

}

YuvImage yuv = new YuvImage(dataResult, mPreviewFormat, mLastWidth, mLastHeight, null);
ByteArrayOutputStream out = new ByteArrayOutputStream();
yuv.compressToJpeg(new Rect(0, 0, mLastWidth, mLastHeight), MediaConstants.sJpegQuality, out);

This comment has been minimized.

Sign in to view

Copy link
@0xPoly

0xPoly Apr 4, 2015

A little experimentation shows that compressing a high-res JPEG isn't possible quickly on most phones. This results in a poor frame rate. Consider scaling down the YUV before compressing to Jpeg.

This comment has been minimized.

Sign in to view

Copy link
@n8fr8

n8fr8 via email Apr 5, 2015

Author Owner

This comment has been minimized.

Sign in to view

Copy link
@n8fr8

n8fr8 via email Apr 5, 2015

Author Owner

This comment has been minimized.

Sign in to view

Copy link
@0xPoly

0xPoly Apr 5, 2015

I tried that first, but it didn't work on my device at all. Maybe I invoked it incorrectly?

This comment has been minimized.

Sign in to view

Copy link
@n8fr8

n8fr8 via email Apr 6, 2015

Author Owner

This comment has been minimized.

Sign in to view

Copy link
@0xPoly

0xPoly Apr 6, 2015

or possible it could be part of the automated frame rate test.

I've found that you need about 2 seconds to reliably benchmark a certain frame size. Maybe we could have an activity that runs once at first run and benchmarks these? We could then store three resolutions that work and offer them to the user as "low", "medium" and "high" quality.

This comment has been minimized.

Sign in to view

Copy link
@n8fr8

n8fr8 Apr 6, 2015

Author Owner

That is great idea!
dataResult = out.toByteArray();
}


if (mIsRecording && mFrameQ != null)
synchronized (mFrameQ)
{
if (data != null)
{
mFrameQ.add(dataResult);
mFramesTotal++;

frameCounter++;
if((System.currentTimeMillis() - start) >= 1000) {
mFPS = frameCounter;
frameCounter = 0;
start = System.currentTimeMillis();
}
VideoFrame vf = new VideoFrame();
vf.image = dataResult;
vf.duration = System.currentTimeMillis() - lastTime;
vf.fps = mFPS;

mFrameQ.add(vf);

mFramesTotal++;

}
}


}
frameCounter++;
if((System.currentTimeMillis() - start) >= 1000) {
mFPS = frameCounter;
frameCounter = 0;
start = System.currentTimeMillis();
}

}

Expand Down Expand Up @@ -353,22 +361,32 @@ private byte[] rotateYUV420Degree90(byte[] data, int imageWidth, int imageHeight
return yuv;
}

private class VideoFrame
{
byte[] image;
long fps;
long duration;
}

private class Encoder extends Thread {
private static final String TAG = "ENCODER";

private File fileOut;
private FileOutputStream fos;

public Encoder (File fileOut) throws IOException
public Encoder (File fileOut, int baseFramesPerSecond, boolean withEmbeddedAudio) throws IOException
{
this.fileOut = fileOut;

fos = new info.guardianproject.iocipher.FileOutputStream(fileOut);
SeekableByteChannel sbc = new IOCipherFileChannelWrapper(fos.getChannel());

org.jcodec.common.AudioFormat af = null;//new org.jcodec.common.AudioFormat(org.jcodec.common.AudioFormat.MONO_S16_LE(MediaConstants.sAudioSampleRate));
org.jcodec.common.AudioFormat af = null;

if (withEmbeddedAudio)
af = new org.jcodec.common.AudioFormat(org.jcodec.common.AudioFormat.MONO_S16_LE(MediaConstants.sAudioSampleRate));

muxer = new ImageToMJPEGMOVMuxer(sbc,af);
muxer = new ImageToMJPEGMOVMuxer(sbc,af,baseFramesPerSecond);
}

public void run ()
Expand All @@ -380,9 +398,9 @@ public void run ()
{
if (mFrameQ.peek() != null)
{
byte[] data = mFrameQ.pop();
VideoFrame vf = mFrameQ.pop();

muxer.addFrame(mLastWidth, mLastHeight, ByteBuffer.wrap(data),mFPS);
muxer.addFrame(mLastWidth, mLastHeight, ByteBuffer.wrap(vf.image),vf.fps,vf.duration);

}

Expand Down
26 changes: 14 additions & 12 deletions src/info/guardianproject/iocipher/camera/encoders/ImageToMJPEGMOVMuxer.java
View file
Expand Up @@ -37,36 +37,40 @@ public class ImageToMJPEGMOVMuxer {
private String imageType = "jpeg "; //or "png ";
private AudioFormat af = null;

private int timeScale = 20;
private int framesPerSecond = -1;

public ImageToMJPEGMOVMuxer(SeekableByteChannel ch, AudioFormat af) throws IOException {
this.ch = ch;
private final static String ENCODER_NAME = "JCODEC";

public ImageToMJPEGMOVMuxer(SeekableByteChannel ch, AudioFormat af, int framesPerSecond) throws IOException {

this.ch = ch;
this.af = af;
this.framesPerSecond = framesPerSecond;

// Muxer that will store the encoded frames
muxer = new WebOptimizedMP4Muxer(ch, Brand.MOV, 16000);

// Add video track to muxer
videoTrack = muxer.addTrack(TrackType.VIDEO, timeScale);
videoTrack = muxer.addTrack(TrackType.VIDEO, framesPerSecond);
// videoTrack.setTgtChunkDuration(new Rational(2, 1), Unit.SEC);

if (af != null)
audioTrack = muxer.addPCMAudioTrack(af);

}

public void addFrame(int width, int height, ByteBuffer buff, int timeScaleFPS) throws IOException {
public void addFrame(int width, int height, ByteBuffer buff, long timeScaleFPS, long duration) throws IOException {
if (size == null) {
size = new Size(width,height);
videoTrack.addSampleEntry(MP4Muxer.videoSampleEntry(imageType, size, "JCodec"));
videoTrack.addSampleEntry(MP4Muxer.videoSampleEntry(imageType, size, ENCODER_NAME));

if (af != null)
audioTrack.addSampleEntry(MP4Muxer.audioSampleEntry(af));
}

// Add packet to video track

videoTrack.addFrame(new MP4Packet(buff, frameNo, timeScaleFPS, 1, frameNo, true, null, frameNo, 0));
videoTrack.addFrame(new MP4Packet(buff, frameNo, timeScaleFPS, duration, frameNo, true, null, frameNo, 0));

This comment has been minimized.

Sign in to view

Copy link
@0xPoly

0xPoly Apr 4, 2015

Careful, I think 'duration' here might be a bug. This line change causes the MOV file produced to be way off in time length when played in VLC.

frameNo++;
}
Expand All @@ -77,12 +81,10 @@ public void addAudio (ByteBuffer buffer) throws IOException
}

public void finish() throws IOException {
// Push saved SPS/PPS to a special storage in MP4
// videoTrack.addSampleEntry(MP4Muxer.videoSampleEntry("png ", size, "JCodec"));
videoTrack.addSampleEntry(MP4Muxer.videoSampleEntry(imageType, size, "JCodec"));

videoTrack.addSampleEntry(MP4Muxer.videoSampleEntry(imageType, size, ENCODER_NAME));

// Write MP4 header and finalize recording

// Write MP4 header and finalize recording
if (af != null)
audioTrack.addSampleEntry(MP4Muxer.audioSampleEntry(af));

Expand Down
17 changes: 16 additions & 1 deletion src/info/guardianproject/iocipher/camera/viewer/MjpegView.java
View file
Expand Up @@ -41,8 +41,10 @@ public class MjpegView extends SurfaceView implements SurfaceHolder.Callback {
private int dispWidth;
private int dispHeight;
private int displayMode;

private int frameDelay = 0;

public class MjpegViewThread extends Thread {
public class MjpegViewThread extends Thread {
private SurfaceHolder mSurfaceHolder;
private int frameCounter = 0;
private long start;
Expand Down Expand Up @@ -142,6 +144,10 @@ public void run() {
ovl = makeFpsOverlay(overlayPaint, fps);
}
}

if (frameDelay > 0)
Thread.sleep(frameDelay);

} catch (Exception e) {
e.getStackTrace();
Log.d(TAG, "catch IOException hit in run", e);
Expand Down Expand Up @@ -245,4 +251,13 @@ public void setOverlayPosition(int p) {
public void setDisplayMode(int s) {
displayMode = s;
}

public int getFrameDelay() {
return frameDelay;
}

public void setFrameDelay(int frameDelay) {
this.frameDelay = frameDelay;
}

}
6 changes: 4 additions & 2 deletions src/info/guardianproject/iocipher/camera/viewer/MjpegViewerActivity.java
View file
Expand Up @@ -49,8 +49,9 @@ public void onCreate(Bundle savedInstanceState) {
try {

mv.setDisplayMode(MjpegView.SIZE_BEST_FIT);
// mv.showFps(true);

//mv.showFps(false);
mv.setFrameDelay(5); //we need to better sync each frame to the audio

File fileAudio = null;

if (ioCipherAudioPath == null)
Expand Down Expand Up @@ -117,6 +118,7 @@ public void initAudio(String vfsPath) throws Exception {
at = new AudioTrack(AudioManager.STREAM_MUSIC, MediaConstants.sAudioSampleRate,
MediaConstants.sChannelConfigOut, AudioFormat.ENCODING_PCM_16BIT,
minBufferSize, AudioTrack.MODE_STREAM);

}

}
Expand Down

0 comments on commit 6c00738

Please sign in to comment.