Skip to content

Commit

Permalink
Implement video capture for Android
Browse files Browse the repository at this point in the history
Add the missing binding to the native Java video capture on Android. The
binding is done with some Java class AndroidCameraInterop based on the
WebRTC project's UnityUtility.java. It enables interop between the Java
camera object and the C++ video track source object. This both avoids
having to build and merge UnityUtility.java, as well as allow extending
it with new functionality like video capture format selection.

Bug: microsoft#246
  • Loading branch information
djee-ms committed Jun 2, 2020
1 parent 081e90c commit d3f5a4f
Show file tree
Hide file tree
Showing 6 changed files with 199 additions and 12 deletions.
65 changes: 61 additions & 4 deletions libs/mrwebrtc/src/interop/interop_api.cpp
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ mrsResult OpenVideoCaptureDevice(
std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
webrtc::VideoCaptureFactory::CreateDeviceInfo());
if (!info) {
RTC_LOG(LS_ERROR)
<< "Could not create device info for video catpure device.";
return Result::kUnknownError;
}

Expand Down Expand Up @@ -751,6 +753,10 @@ mrsResult MRS_CALL mrsLocalVideoTrackCreateFromDevice(
return Result::kInvalidOperation;
}

rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> video_source;

#if !defined(MR_SHARING_ANDROID)

// Open the video capture device
std::unique_ptr<cricket::VideoCapturer> video_capturer;
auto res = OpenVideoCaptureDevice(*config, video_capturer);
Expand Down Expand Up @@ -782,10 +788,57 @@ mrsResult MRS_CALL mrsLocalVideoTrackCreateFromDevice(
}

// Create the video track source
rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> video_source =
pc_factory->CreateVideoSource(std::move(video_capturer),
videoConstraints.get());
video_source = pc_factory->CreateVideoSource(std::move(video_capturer),
videoConstraints.get());

#else // !defined(MR_SHARING_ANDROID)

// Make sure the current thread is attached to the JVM. Since this method
// is often called asynchronously (as it takes some time to initialize the
// video capture device) it is likely to run on a background worker thread.
RTC_DCHECK(webrtc::jni::GetJVM()) << "JavaVM not initialized.";
JNIEnv* env = webrtc::jni::AttachCurrentThreadIfNeeded();

// Create the surface texture helper, which manages the surface texture the camera renders to.
webrtc::ScopedJavaLocalRef<jclass> android_camera_interop_class =
webrtc::GetClass(env, "com/microsoft/mixedreality/webrtc/AndroidCameraInterop");
RTC_DCHECK(android_camera_interop_class.obj() != nullptr) << "Failed to find AndroidCameraInterop Java class.";
jmethodID create_texture_helper_method = webrtc::GetStaticMethodID(
env, android_camera_interop_class.obj(), "CreateSurfaceTextureHelper",
"()Lorg/webrtc/SurfaceTextureHelper;");
jobject texture_helper = env->CallStaticObjectMethod(
android_camera_interop_class.obj(), create_texture_helper_method);
CHECK_EXCEPTION(env);
RTC_DCHECK(texture_helper != nullptr)
<< "Cannot get the Surface Texture Helper.";

// Create the video track source which wraps the Android camera capturer
rtc::scoped_refptr<webrtc::jni::AndroidVideoTrackSource> impl_source(
new rtc::RefCountedObject<webrtc::jni::AndroidVideoTrackSource>(
global_factory->GetSignalingThread(), env, false));
rtc::scoped_refptr<webrtc::VideoTrackSourceProxy> proxy_source =
webrtc::VideoTrackSourceProxy::Create(
global_factory->GetSignalingThread(),
global_factory->GetWorkerThread(), impl_source);

// Create the camera capturer and bind it to the surface texture and the video source, then start capturing.
jmethodID start_capture_method = webrtc::GetStaticMethodID(
env, android_camera_interop_class.obj(), "StartCapture",
"(JLorg/webrtc/SurfaceTextureHelper;)Lorg/webrtc/VideoCapturer;");
jobject camera_tmp =
env->CallStaticObjectMethod(android_camera_interop_class.obj(), start_capture_method,
(jlong)proxy_source.get(), texture_helper);
CHECK_EXCEPTION(env);

// Java objects created are always returned as local references; create a new global reference to keep the camera capturer alive.
jobject java_video_capturer = (jobject)env->NewGlobalRef(camera_tmp);

video_source = proxy_source;

#endif // !defined(MR_SHARING_ANDROID)

if (!video_source) {
RTC_LOG(LS_ERROR) << "Failed to create video track source.";
return Result::kUnknownError;
}

Expand All @@ -799,7 +852,11 @@ mrsResult MRS_CALL mrsLocalVideoTrackCreateFromDevice(

// Create the video track wrapper
RefPtr<LocalVideoTrack> track =
new LocalVideoTrack(std::move(global_factory), std::move(video_track));
new LocalVideoTrack(std::move(global_factory), std::move(video_track)
#if defined(MR_SHARING_ANDROID)
, java_video_capturer
#endif // defined(MR_SHARING_ANDROID)
);
*track_handle_out = track.release();
return Result::kSuccess;
}
Expand Down
30 changes: 28 additions & 2 deletions libs/mrwebrtc/src/media/local_video_track.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,20 @@ namespace WebRTC {

LocalVideoTrack::LocalVideoTrack(
RefPtr<GlobalFactory> global_factory,
rtc::scoped_refptr<webrtc::VideoTrackInterface> track) noexcept
rtc::scoped_refptr<webrtc::VideoTrackInterface> track
#if defined(MR_SHARING_ANDROID)
,
jobject java_video_capturer
#endif // defined(MR_SHARING_ANDROID)
) noexcept
: MediaTrack(std::move(global_factory), ObjectType::kLocalVideoTrack),
track_(std::move(track)),
track_name_(track_->id()) {
track_name_(track_->id())
#if defined(MR_SHARING_ANDROID)
,
java_video_capturer_(java_video_capturer)
#endif // defined(MR_SHARING_ANDROID)
{
RTC_CHECK(track_);
kind_ = mrsTrackKind::kVideoTrack;
rtc::VideoSinkWants sink_settings{};
Expand Down Expand Up @@ -56,6 +66,22 @@ LocalVideoTrack::~LocalVideoTrack() {
}
RTC_CHECK(!transceiver_);
RTC_CHECK(!owner_);
#if defined(MR_SHARING_ANDROID)
// Stop video capture and release Java capturer
if (java_video_capturer_) {
JNIEnv* env = webrtc::jni::GetEnv();
RTC_DCHECK(env);
webrtc::ScopedJavaLocalRef<jclass> pc_factory_class =
webrtc::GetClass(env, "org/webrtc/UnityUtility");
jmethodID stop_camera_method =
webrtc::GetStaticMethodID(env, pc_factory_class.obj(), "StopCamera",
"(Lorg/webrtc/VideoCapturer;)V");
env->CallStaticVoidMethod(pc_factory_class.obj(), stop_camera_method,
java_video_capturer_);
CHECK_EXCEPTION(env);
java_video_capturer_ = nullptr;
}
#endif // defined(MR_SHARING_ANDROID)
}

void LocalVideoTrack::SetEnabled(bool enabled) const noexcept {
Expand Down
26 changes: 20 additions & 6 deletions libs/mrwebrtc/src/media/local_video_track.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,20 @@ class LocalVideoTrack : public VideoFrameObserver, public MediaTrack {
public:
/// Constructor for a track not added to any peer connection.
LocalVideoTrack(RefPtr<GlobalFactory> global_factory,
rtc::scoped_refptr<webrtc::VideoTrackInterface> track) noexcept;
rtc::scoped_refptr<webrtc::VideoTrackInterface> track
#if defined(MR_SHARING_ANDROID)
,
jobject java_video_capturer = nullptr
#endif // defined(MR_SHARING_ANDROID)
) noexcept;

/// Constructor for a track added to a peer connection.
LocalVideoTrack(RefPtr<GlobalFactory> global_factory,
PeerConnection& owner,
Transceiver* transceiver,
rtc::scoped_refptr<webrtc::VideoTrackInterface> track,
rtc::scoped_refptr<webrtc::RtpSenderInterface> sender) noexcept;
LocalVideoTrack(
RefPtr<GlobalFactory> global_factory,
PeerConnection& owner,
Transceiver* transceiver,
rtc::scoped_refptr<webrtc::VideoTrackInterface> track,
rtc::scoped_refptr<webrtc::RtpSenderInterface> sender) noexcept;

~LocalVideoTrack() override;

Expand Down Expand Up @@ -111,6 +117,14 @@ class LocalVideoTrack : public VideoFrameObserver, public MediaTrack {

/// Cached track name, to avoid dispatching on signaling thread.
const std::string track_name_;

#if defined(MR_SHARING_ANDROID)

/// Global reference to the Java video capturer (org.webrtc.VideoCapturer)
/// object.
jobject java_video_capturer_{nullptr};

#endif // defined(MR_SHARING_ANDROID)
};

} // namespace WebRTC
Expand Down
6 changes: 6 additions & 0 deletions libs/mrwebrtc/src/pch.h
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/video_capture/video_capture_factory.h"
#include "rtc_base/memory/aligned_malloc.h"
#if defined(MR_SHARING_ANDROID)
#include "sdk/android/native_api/jni/class_loader.h"
#include "modules/utility/include/helpers_android.h"
#include "sdk/android/src/jni/androidvideotracksource.h"
#include "sdk/android/src/jni/jni_helpers.h"
#endif

// libyuv from WebRTC repository for color conversion
#include "libyuv.h"
Expand Down
6 changes: 6 additions & 0 deletions tools/build/android/webrtc-native/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ android {
dependencies {
implementation fileTree(dir: 'libs', include: ['*.jar'])
implementation 'androidx.appcompat:appcompat:1.1.0'
// See e.g. examples/aarproject/app/build.gradle
// Note: com.google.code.findbugs is for @Nullable,
// see https://stackoverflow.com/questions/19030954/cant-find-nullable-inside-javax-annotation
// FIXME - This uses published packages, not locally-built ones!!!
implementation 'com.google.code.findbugs:jsr305:3.0.2'
implementation 'org.webrtc:google-webrtc:1.0.+'
}

task copyToUnitySample {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

// This file is originaly based on the UnityUtility.java file from the
// WebRTC.org project, modified for the needs of the MixedReality-WebRTC
// project and expanded with additional functionalities.

// UnityUtility.java:
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/

package com.microsoft.mixedreality.webrtc;

import android.content.Context;
import java.util.List;
import javax.annotation.Nullable;
import org.webrtc.CameraEnumerator;
import org.webrtc.Camera2Enumerator;
import org.webrtc.ContextUtils;
import org.webrtc.SurfaceTextureHelper;
import org.webrtc.VideoCapturer;
import org.webrtc.VideoSource;

public class AndroidCameraInterop {
private static final String VIDEO_CAPTURER_THREAD_NAME = "VideoCapturerThread";

public static SurfaceTextureHelper CreateSurfaceTextureHelper() {
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create(VIDEO_CAPTURER_THREAD_NAME, null);
return surfaceTextureHelper;
}

private static boolean useCamera2() {
return Camera2Enumerator.isSupported(ContextUtils.getApplicationContext());
}

private static @Nullable VideoCapturer createCameraCapturer(CameraEnumerator enumerator) {
final String[] deviceNames = enumerator.getDeviceNames();

for (String deviceName : deviceNames) {
if (enumerator.isFrontFacing(deviceName)) {
VideoCapturer videoCapturer = enumerator.createCapturer(deviceName, null);

if (videoCapturer != null) {
return videoCapturer;
}
}
}

return null;
}

public static VideoCapturer StartCapture(
long nativeTrackSource, SurfaceTextureHelper surfaceTextureHelper) {
VideoCapturer capturer =
createCameraCapturer(new Camera2Enumerator(ContextUtils.getApplicationContext()));

VideoSource videoSource = new VideoSource(nativeTrackSource);

capturer.initialize(surfaceTextureHelper, ContextUtils.getApplicationContext(),
videoSource.getCapturerObserver());

capturer.startCapture(720, 480, 30);
return capturer;
}

public static void StopCamera(VideoCapturer camera) throws InterruptedException {
camera.stopCapture();
camera.dispose();
}
}

0 comments on commit d3f5a4f

Please sign in to comment.