WebcamAndMicrophoneCapture.java

samples/WebcamAndMicrophoneCapture.java

@@ -0,0 +1,232 @@
+/**
+ * @author Ben Davenport
+ * 
+ * This class is a simple example for broadcasting a video capture device (ie, webcam) and an audio capture device (ie, microphone)
+ * using an FFmpegFrameRecorder. 
+ * 
+ * FFmpegFrameRecorder allows the output destination to be either a FILE or an RTMP endpoint (Wowza, FMS, et al)
+ * 
+ * IMPORTANT: There are potential timing issues with audio/video synchronicity across threads, I am working on finding a solution, but
+ * chime in if you can fig it out :o)
+ */
+
+package javacv;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.ShortBuffer;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import javax.sound.sampled.AudioFormat;
+import javax.sound.sampled.AudioSystem;
+import javax.sound.sampled.DataLine;
+import javax.sound.sampled.LineUnavailableException;
+import javax.sound.sampled.Mixer;
+import javax.sound.sampled.TargetDataLine;
+
+import org.bytedeco.javacpp.avcodec;
+import org.bytedeco.javacv.CanvasFrame;
+import org.bytedeco.javacv.FFmpegFrameRecorder;
+import org.bytedeco.javacv.Frame;
+import org.bytedeco.javacv.FrameRecorder.Exception;
+import org.bytedeco.javacv.OpenCVFrameGrabber;
+
+public class WebcamAndMicrophoneCapture
+{
+	final private static int WEBCAM_DEVICE_INDEX = 1;
+	final private static int AUDIO_DEVICE_INDEX = 4;
+
+	final private static int FRAME_RATE = 30;
+	final private static int GOP_LENGTH_IN_FRAMES = 60;
+
+	private static long startTime = 0;
+	private static long videoTS = 0;
+
+	public static void main(String[] args) throws Exception, org.bytedeco.javacv.FrameGrabber.Exception
+	{
+		int captureWidth = 1280;
+		int captureHeight = 720;
+
+		// The available FrameGrabber classes include OpenCVFrameGrabber (opencv_highgui),
+		// DC1394FrameGrabber, FlyCaptureFrameGrabber, OpenKinectFrameGrabber,
+		// PS3EyeFrameGrabber, VideoInputFrameGrabber, and FFmpegFrameGrabber.
+		OpenCVFrameGrabber grabber = new OpenCVFrameGrabber(WEBCAM_DEVICE_INDEX);
+		grabber.setImageWidth(captureWidth);
+		grabber.setImageHeight(captureHeight);
+		grabber.start();
+
+		// org.bytedeco.javacv.FFmpegFrameRecorder.FFmpegFrameRecorder(String
+		// filename, int imageWidth, int imageHeight, int audioChannels)
+		// For each param, we're passing in...
+		// filename = either a path to a local file we wish to create, or an
+		// RTMP url to an FMS / Wowza server
+		// imageWidth = width we specified for the grabber
+		// imageHeight = height we specified for the grabber
+		// audioChannels = 2, because we like stereo
+		FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(
+				"rtmp://my-streaming-server/app_name_here/instance_name/stream_name",
+				captureWidth, captureHeight, 2);
+		recorder.setInterleaved(true);
+
+		// decrease "startup" latency in FFMPEG (see:
+		// https://trac.ffmpeg.org/wiki/StreamingGuide)
+		recorder.setVideoOption("tune", "zerolatency");
+		// tradeoff between quality and encode speed
+		// possible values are ultrafast,superfast, veryfast, faster, fast,
+		// medium, slow, slower, veryslow
+		// ultrafast offers us the least amount of compression (lower encoder
+		// CPU) at the cost of a larger stream size
+		// at the other end, veryslow provides the best compression (high
+		// encoder CPU) while lowering the stream size
+		// (see: https://trac.ffmpeg.org/wiki/Encode/H.264)
+		recorder.setVideoOption("preset", "ultrafast");
+		// Constant Rate Factor (see: https://trac.ffmpeg.org/wiki/Encode/H.264)
+		recorder.setVideoOption("crf", "28");
+		// 2000 kb/s, reasonable "sane" area for 720
+		recorder.setVideoBitrate(2000000);
+		recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
+		recorder.setFormat("flv");
+		// FPS (frames per second)
+		recorder.setFrameRate(FRAME_RATE);
+		// Key frame interval, in our case every 2 seconds -> 30 (fps) * 2 = 60
+		// (gop length)
+		recorder.setGopSize(GOP_LENGTH_IN_FRAMES);
+
+		// We don't want variable bitrate audio
+		recorder.setAudioOption("crf", "0");
+		// Highest quality
+		recorder.setAudioQuality(0);
+		// 192 Kbps
+		recorder.setAudioBitrate(192000);
+		recorder.setSampleRate(44100);
+		recorder.setAudioChannels(2);
+		recorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC);
+
+		// Jack 'n coke... do it...
+		recorder.start();
+
+		// Thread for audio capture, this could be in a nested private class if you prefer...
+		new Thread(new Runnable() {
+			@Override
+			public void run()
+			{
+				// Pick a format...
+				// NOTE: It is better to enumerate the formats that the system supports,
+				// because getLine() can error out with any particular format...
+				// For us: 44.1 sample rate, 16 bits, stereo, signed, little endian
+				AudioFormat audioFormat = new AudioFormat(44100.0F, 16, 2, true, false);
+
+				// Get TargetDataLine with that format
+				Mixer.Info[] minfoSet = AudioSystem.getMixerInfo();
+				Mixer mixer = AudioSystem.getMixer(minfoSet[AUDIO_DEVICE_INDEX]);
+				DataLine.Info dataLineInfo = new DataLine.Info(TargetDataLine.class, audioFormat);
+
+				try
+				{
+					// Open and start capturing audio
+					TargetDataLine line = (TargetDataLine) mixer.getLine(dataLineInfo);
+					line.open(audioFormat);
+					line.start();
+
+					int sampleRate = (int) audioFormat.getSampleRate();
+					int numChannels = audioFormat.getChannels();
+
+					// Let's initialize our audio buffer...
+					int audioBufferSize = sampleRate * numChannels;
+					byte[] audioBytes = new byte[audioBufferSize];
+
+					// Using a ScheduledThreadPoolExecutor vs a while loop with
+					// a Thread.sleep will allow
+					// us to get around some OS specific timing issues, and keep
+					// to a more precise
+					// clock as the fixed rate accounts for garbage collection
+					// time, etc
+					// a similar approach could be used for the webcam capture
+					// as well, if you wish
+					ScheduledThreadPoolExecutor exec = new ScheduledThreadPoolExecutor(1);
+					exec.scheduleAtFixedRate(new Runnable() {
+						@Override
+						public void run()
+						{
+							try
+							{
+								// Read from the line... non-blocking
+								int nBytesRead = line.read(audioBytes, 0, line.available());
+
+								// Since we specified 16 bits in the AudioFormat,
+								// we need to convert our read byte[] to short[]
+								// (see source from FFmpegFrameRecorder.recordSamples for AV_SAMPLE_FMT_S16)
+								// Let's initialize our short[] array
+								int nSamplesRead = nBytesRead / 2;
+								short[] samples = new short[nSamplesRead];
+
+								// Let's wrap our short[] into a ShortBuffer and
+								// pass it to recordSamples
+								ByteBuffer.wrap(audioBytes).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(samples);
+								ShortBuffer sBuff = ShortBuffer.wrap(samples, 0, nSamplesRead);
+
+								// recorder is instance of
+								// org.bytedeco.javacv.FFmpegFrameRecorder
+								recorder.recordSamples(sampleRate, numChannels, sBuff);
+							} 
+							catch (org.bytedeco.javacv.FrameRecorder.Exception e)
+							{
+								e.printStackTrace();
+							}
+						}
+					}, 0, (long) 1000 / FRAME_RATE, TimeUnit.MILLISECONDS);
+				} 
+				catch (LineUnavailableException e1)
+				{
+					e1.printStackTrace();
+				}
+			}
+		}).start();
+
+		// A really nice hardware accelerated component for our preview...
+		CanvasFrame cFrame = new CanvasFrame("Capture Preview", CanvasFrame.getDefaultGamma() / grabber.getGamma());
+
+		Frame capturedFrame = null;
+
+		// While we are capturing...
+		while ((capturedFrame = grabber.grab()) != null)
+		{
+			if (cFrame.isVisible())
+			{
+				// Show our frame in the preview
+				cFrame.showImage(capturedFrame);
+			}
+
+			// Let's define our start time...
+			// This needs to be initialized as close to when we'll use it as
+			// possible,
+			// as the delta from assignment to computed time could be too high
+			if (startTime == 0)
+				startTime = System.currentTimeMillis();
+
+			// Create timestamp for this frame
+			videoTS = 1000 * (System.currentTimeMillis() - startTime);
+
+			// Check for AV drift
+			if (videoTS > recorder.getTimestamp())
+			{
+				System.out.println(
+						"Lip-flap correction: " 
+						+ videoTS + " : "
+						+ recorder.getTimestamp() + " -> "
+						+ (videoTS - recorder.getTimestamp()));
+
+				// We tell the recorder to write this frame at this timestamp
+				recorder.setTimestamp(videoTS);
+			}
+
+			// Send the frame to the org.bytedeco.javacv.FFmpegFrameRecorder
+			recorder.record(capturedFrame);
+		}
+
+		cFrame.dispose();
+		recorder.stop();
+		grabber.stop();
+	}
+}