Merge pull request #193 from AltspaceVR/eanders/android

Add experimental Android ARM64 support via Unity.

This changes adds support for compiling MixedReality-WebRTC for Android ARM64
for use inside a Unity project and deploy as a Unity build to an Android
device. Support for non-Unity Android apps is out of scope.

The Android build produces 2 archives:
- libwebrtc.aar: the core Google WebRTC implementation for Android ARM64.
- mrwebrtc.aar: the MixedReality-WebRTC API wrapping the Google implementation.

Both archives are copied to the `Assets/Plugins/arm64-v8a` folder of the Unity
sample project, and deployed to the device.

The current change builds over the main WebRTC Google repository instead of
WebRTC UWP, therefore represents a temporary diverging in the
MixedReality-WebRTC. For this reason, it is strongly recommended to use the
`branch-heads/71` Google branch to keep the code as close as possible to the
one used for other platforms (Windows Desktop and UWP). Building from Google's
`master` branch may work but is not supported.

This change is experimental. The `tools/build/libwebrtc` and
`tools/build/android` folders provides some utility scripts to help building
the Google repository for Android ARM64 and the MixedReality-WebRTC project,
respectively (see `tools/README.md`). However this process is involved, and
there is currently no precompiled binaries provided to avoid it, nor any CI in
place to validte changes. This will come in time.

Known limitations and issues:
- Video capture on Android device currently does not work. This requires
  specific interop code to open the device camera from Java, which is not yet
  available. (Bug: #246)
- The two Android archives are huge, much too big for production. Current
  testing shows a deploy at 326 MB on device, including Unity's project and
  without any optimization nor stripping (dev build). (Bug: #247)
- The build should produce a single archive, not two. (Bug: #248)
- Because Android and Windows use different Google repository commits, there is
  a small chance to get discrepancies in behavior and/or incompatibility bugs
  between those. No such bug has been observed so far, but please keep this in
  mind.

Android users are encouraged to try this change and provide feedback via GitHub
issues, be it on the build process itself, on the missing features not listed
above, or anything else related to Android support. The change has been tested
manually on Google Pixel 3A and Occulus Quest. Other Android devices should
work too, but have not been tested explicitly.

A huge thanks to Eric Anderson from AltspaceVR for contributing this change.

Bug: #28

.gitignore

@@ -9,6 +9,7 @@ examples/*/bin
 tests/Microsoft.MixedReality.WebRTC.Tests/bin
 tests/Microsoft.MixedReality.WebRTC.Tests/build
 tests/Microsoft.MixedReality.WebRTC.Tests/obj
+examples/TestNetCoreConsole/bin
 
 # Visual Studio + Code
 .vs
@@ -55,3 +56,26 @@ xdoc.meta
 
 # Profiling data
 *.etl
+
+# CMake
+CMakefiles/
+
+# Android Studio
+.idea/
+
+# MacOS
+.DS_Store
+
+# libwebrtc intermediate build files
+tools/build/libwebrtc/.checkout.sh
+tools/build/libwebrtc/.config.sh
+tools/build/libwebrtc/.build.sh
+tools/build/libwebrtc/.libwebrtc.cmake
+
+# Android intermediate build files
+tools/build/android/**/*.iml
+tools/build/android/.gradle
+tools/build/android/local.properties
+tools/build/android/**/build
+tools/build/android/captures
+tools/build/android/**/.externalNativeBuild

examples/TestAppUwp/Microsoft.MixedReality.WebRTC.TestAppUWP.csproj

@@ -72,12 +72,12 @@
     </AppxManifest>
   </ItemGroup>
   <ItemGroup>
-    <Content Include="..\..\bin\UWP\$(Platform)\$(Configuration)\Microsoft.MixedReality.WebRTC.Native.dll">
-      <Link>Microsoft.MixedReality.WebRTC.Native.dll</Link>
+    <Content Include="..\..\bin\UWP\$(Platform)\$(Configuration)\mrwebrtc.dll">
+      <Link>mrwebrtc.dll</Link>
       <CopyToOutputDirectory>Never</CopyToOutputDirectory>
     </Content>
-    <Content Include="..\..\bin\UWP\$(Platform)\$(Configuration)\Microsoft.MixedReality.WebRTC.Native.pdb">
-      <Link>Microsoft.MixedReality.WebRTC.Native.pdb</Link>
+    <Content Include="..\..\bin\UWP\$(Platform)\$(Configuration)\mrwebrtc.pdb">
+      <Link>mrwebrtc.pdb</Link>
       <CopyToOutputDirectory>Never</CopyToOutputDirectory>
     </Content>
   </ItemGroup>

examples/TestNetCoreConsole/TestNetCoreConsole.csproj

@@ -17,10 +17,10 @@
     <Content Include="..\..\bin\netstandard2.0\$(Configuration)\Microsoft.MixedReality.WebRTC.pdb">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="..\..\bin\Win32\$(Platform)\$(Configuration)\Microsoft.MixedReality.WebRTC.Native.dll">
+    <Content Include="..\..\bin\Win32\$(Platform)\$(Configuration)\mrwebrtc.dll">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="..\..\bin\Win32\$(Platform)\$(Configuration)\Microsoft.MixedReality.WebRTC.Native.pdb">
+    <Content Include="..\..\bin\Win32\$(Platform)\$(Configuration)\mrwebrtc.pdb">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>

libs/Microsoft.MixedReality.WebRTC.Native/src/data_channel.cpp

@@ -92,6 +92,12 @@ void DataChannel::OnStateChange() noexcept {
         owner_->OnDataChannelAdded(*this);
       }
       break;
+    case webrtc::DataChannelInterface::DataState::kClosed:
+      break;
+    case webrtc::DataChannelInterface::DataState::kClosing:
+      break;
+    case webrtc::DataChannelInterface::DataState::kConnecting:
+      break;
   }
 
   // Invoke the StateChanged event

libs/Microsoft.MixedReality.WebRTC.Native/src/media/external_video_track_source_impl.h

@@ -70,7 +70,7 @@ class ExternalVideoTrackSourceImpl : public ExternalVideoTrackSource,
 
   /// Start the video capture. This will begin to produce video frames and start
   /// calling the video frame callback.
-  void StartCapture();
+  void StartCapture() override;
 
   /// Complete a video frame request with a given I420A video frame.
   Result CompleteRequest(uint32_t request_id,
@@ -83,10 +83,10 @@ class ExternalVideoTrackSourceImpl : public ExternalVideoTrackSource,
                          const Argb32VideoFrame& frame) override;
 
   /// Stop the video capture. This will stop producing video frames.
-  void StopCapture();
+  void StopCapture() override;
 
   /// Shutdown the source and release the buffer adapter and its callback.
-  void Shutdown() noexcept;
+  void Shutdown() noexcept override;
 
   webrtc::VideoTrackSourceInterface* impl() const { return track_source_; }
 

libs/Microsoft.MixedReality.WebRTC.Native/src/pch.h

@@ -41,7 +41,6 @@
 #pragma warning(disable : 4244)
 
 // Core WebRTC
-#include "absl/memory/memory.h"
 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
 #include "api/datachannelinterface.h"

libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp

@@ -11,6 +11,7 @@
 #include "peer_connection.h"
 #include "sdp_utils.h"
 #include "video_frame_observer.h"
+#include "sdp_utils.h"
 
 // Internal
 #include "interop/global_factory.h"
@@ -168,7 +169,7 @@ class PeerConnectionImpl : public PeerConnection,
     remote_audio_observer_.reset(new AudioFrameObserver());
   }
 
-  void SetName(std::string_view name) { name_ = name; }
+  void SetName(std::string_view name) override { name_ = name; }
 
   std::string GetName() const override { return name_; }
 
@@ -1009,7 +1010,7 @@ bool PeerConnectionImpl::SetRemoteDescriptionAsync(
     }
   }
   std::string sdp_type_str(type);
-  auto sdp_type = webrtc::SdpTypeFromString(sdp_type_str);
+  auto sdp_type = SdpTypeFromString(sdp_type_str);
   if (!sdp_type.has_value())
     return false;
   std::string remote_desc(sdp);
@@ -1050,6 +1051,8 @@ void PeerConnectionImpl::OnSignalingChange(
       break;
     case webrtc::PeerConnectionInterface::kHaveRemotePrAnswer:
       break;
+    case webrtc::PeerConnectionInterface::kClosed:
+      break;
   }
 }
 

libs/Microsoft.MixedReality.WebRTC.Native/src/sdp_utils.cpp

@@ -209,4 +209,16 @@ std::string EncodeIceServers(const std::string& url,
   return url + "\nusername:" + username + "\npassword:" + password;
 }
 
+std::optional<webrtc::SdpType> SdpTypeFromString(const std::string& type_str) {
+  if (type_str == webrtc::SessionDescriptionInterface::kOffer) {
+    return webrtc::SdpType::kOffer;
+  } else if (type_str == webrtc::SessionDescriptionInterface::kPrAnswer) {
+    return webrtc::SdpType::kPrAnswer;
+  } else if (type_str == webrtc::SessionDescriptionInterface::kAnswer) {
+    return webrtc::SdpType::kAnswer;
+  } else {
+    return std::nullopt;
+  }
+}
+
 }  // namespace Microsoft::MixedReality::WebRTC

libs/Microsoft.MixedReality.WebRTC.Native/src/sdp_utils.h

@@ -60,4 +60,9 @@ std::string EncodeIceServers(const std::string& url,
                              const std::string& username,
                              const std::string& password);
 
+// Copied from src/pc/jsepsessiondescription.cc so that we don't rely on a binary
+// representation of absl::optional as per the Abseil Compatibility Guidelines:
+// https://abseil.io/about/compatibility
+std::optional<webrtc::SdpType> SdpTypeFromString(const std::string& type_str);
+
 }  // namespace Microsoft::MixedReality::WebRTC

libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj

@@ -72,15 +72,15 @@
   </ImportGroup>
   <ImportGroup Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-    <Import Project="..\..\Microsoft.MixedReality.WebRTC.Native.targets" />
+    <Import Project="..\..\mrwebrtc.targets" />
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup>
     <GenerateManifest>false</GenerateManifest>
     <IgnoreImportLibrary>true</IgnoreImportLibrary>
     <OutDir>..\..\..\..\bin\UWP\$(PlatformTarget)\$(Configuration)\</OutDir>
     <IntDir>..\..\..\..\build\Microsoft.MixedReality.WebRTC.Native.UWP\UWP\$(PlatformTarget)\$(Configuration)\</IntDir>
-    <TargetName>Microsoft.MixedReality.WebRTC.Native</TargetName>
+    <TargetName>mrwebrtc</TargetName>
   </PropertyGroup>
   <PropertyGroup Condition="'$(PlatformTarget)' == 'x86'">
     <UnityPluginDirectory>$(ProjectDir)..\..\..\Microsoft.MixedReality.WebRTC.Unity\Assets\Plugins\WSA\x86</UnityPluginDirectory>
@@ -98,7 +98,7 @@
     <LinkIncremental>false</LinkIncremental>
   </PropertyGroup>
   <ItemGroup>
-    <NativeUnityPluginSourceFiles Include="$(OutDir)\Microsoft.MixedReality.WebRTC.Native.dll;$(OutDir)\Microsoft.MixedReality.WebRTC.Native.pdb" />
+    <NativeUnityPluginSourceFiles Include="$(OutDir)\mrwebrtc.dll;$(OutDir)\mrwebrtc.pdb" />
   </ItemGroup>
   <ItemDefinitionGroup>
     <ClCompile>

libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj

@@ -57,7 +57,7 @@
   </ImportGroup>
   <ImportGroup Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-    <Import Project="..\..\Microsoft.MixedReality.WebRTC.Native.targets" />
+    <Import Project="..\..\mrwebrtc.targets" />
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)'=='Debug'">
@@ -69,7 +69,7 @@
   <PropertyGroup>
     <OutDir>..\..\..\..\bin\Win32\$(PlatformTarget)\$(Configuration)\</OutDir>
     <IntDir>..\..\..\..\build\Microsoft.MixedReality.WebRTC.Native\Win32\$(PlatformTarget)\$(Configuration)\</IntDir>
-    <TargetName>Microsoft.MixedReality.WebRTC.Native</TargetName>
+    <TargetName>mrwebrtc</TargetName>
   </PropertyGroup>
   <PropertyGroup Condition="'$(PlatformTarget)' == 'x86'">
     <UnityPluginDirectory>$(ProjectDir)..\..\..\Microsoft.MixedReality.WebRTC.Unity\Assets\Plugins\Win32\x86</UnityPluginDirectory>
@@ -78,7 +78,7 @@
     <UnityPluginDirectory>$(ProjectDir)..\..\..\Microsoft.MixedReality.WebRTC.Unity\Assets\Plugins\Win32\x86_64</UnityPluginDirectory>
   </PropertyGroup>
   <ItemGroup>
-    <NativeUnityPluginSourceFiles Include="$(OutDir)\Microsoft.MixedReality.WebRTC.Native.dll;$(OutDir)\Microsoft.MixedReality.WebRTC.Native.pdb" />
+    <NativeUnityPluginSourceFiles Include="$(OutDir)\mrwebrtc.dll;$(OutDir)\mrwebrtc.pdb" />
   </ItemGroup>
   <ItemDefinitionGroup>
     <ClCompile>

libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj

@@ -41,7 +41,7 @@
   <ImportGroup Label="ExtensionSettings" />
   <ImportGroup Label="Shared" />
   <ImportGroup Label="PropertySheets">
-    <Import Project="..\Microsoft.MixedReality.WebRTC.Native.targets" />
+    <Import Project="..\mrwebrtc.targets" />
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)'=='Debug'">

libs/Microsoft.MixedReality.WebRTC.Unity/Assets/Microsoft.MixedReality.WebRTC.Unity.Tests.Editor/Microsoft.MixedReality.WebRTC.Unity.Tests.Editor.asmdef

@@ -10,7 +10,9 @@
     "excludePlatforms": [],
     "allowUnsafeCode": false,
     "overrideReferences": false,
-    "precompiledReferences": [],
+    "precompiledReferences": [
+        "Microsoft.MixedReality.WebRTC.dll"
+    ],
     "autoReferenced": true,
     "defineConstraints": []
 }

libs/Microsoft.MixedReality.WebRTC.Unity/Assets/Microsoft.MixedReality.WebRTC.Unity.Tests.Runtime/Microsoft.MixedReality.WebRTC.Unity.Tests.Runtime.asmdef

@@ -10,7 +10,9 @@
     "excludePlatforms": [],
     "allowUnsafeCode": false,
     "overrideReferences": false,
-    "precompiledReferences": [],
+    "precompiledReferences": [
+        "Microsoft.MixedReality.WebRTC.dll"
+    ],
     "autoReferenced": true,
     "defineConstraints": []
 }

libs/Microsoft.MixedReality.WebRTC.Unity/Assets/Microsoft.MixedReality.WebRTC.Unity/Scripts/Media/LocalAudioSource.cs

@@ -2,7 +2,8 @@
 // Licensed under the MIT License.
 
 using UnityEngine;
-
+using UnityEngine.XR;
+
 namespace Microsoft.MixedReality.WebRTC.Unity
 {
     /// <summary>
@@ -85,7 +86,13 @@ private void OnPeerInitialized()
         }
 
         private async void DoAutoStartActions(WebRTC.PeerConnection nativePeer)
-        {
+        {
+            // If a headset is active then do not capture local streams.
+            if (XRDevice.isPresent)
+            {
+                return;
+            }
+
             if (AutoStartCapture)
             {
                 //nativePeer.LocalAudioFrameReady += LocalAudioFrameReady;

libs/Microsoft.MixedReality.WebRTC.Unity/Assets/Microsoft.MixedReality.WebRTC.Unity/Scripts/Media/LocalVideoSource.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Threading.Tasks;
 using UnityEngine;
+using UnityEngine.XR;
 
 #if ENABLE_WINMD_SUPPORT
 using global::Windows.Graphics.Holographic;
@@ -192,11 +193,14 @@ protected void OnDisable()
             var nativePeer = PeerConnection.Peer;
             if ((nativePeer != null) && nativePeer.Initialized)
             {
-                VideoStreamStopped.Invoke();
-                Track.I420AVideoFrameReady -= I420ALocalVideoFrameReady;
-                nativePeer.RemoveLocalVideoTrack(Track);
-                Track.Dispose();
-                Track = null;
+                if (Track != null)
+                {
+                    VideoStreamStopped.Invoke();
+                    Track.I420AVideoFrameReady -= I420ALocalVideoFrameReady;
+                    nativePeer.RemoveLocalVideoTrack(Track);
+                    Track.Dispose();
+                    Track = null;
+                }
                 _frameQueue.Clear();
             }
         }
@@ -216,6 +220,12 @@ private void OnPeerInitialized()
 
         private async void DoAutoStartActions(WebRTC.PeerConnection nativePeer)
         {
+            // If a headset is active then do not capture local streams.
+            if (XRDevice.isPresent)
+            {
+                return;
+            }
+
             if (AutoAddTrack)
             {
                 // This needs to be awaited because it will initialize Track, used below

libs/Microsoft.MixedReality.WebRTC.Unity/Assets/Microsoft.MixedReality.WebRTC.Unity/Scripts/Media/SceneVideoSource.cs

@@ -71,7 +71,22 @@ public class SceneVideoSource : CustomVideoSource<Argb32VideoFrameStorage>
         private int _readBackHeight;
 
         protected new void OnEnable()
-        {
+        {
+            // If a headset is active then do not capture scene stream.
+            // Note: this was failing on Oculus Quest, but it looks like a solution exists,
+            // see error message from the Quest:
+            //
+            // "NotSupportedException: Capturing scene content in single - pass instanced stereo
+            // rendering requires blitting from the Texture2DArray render target of the camera,
+            // which is not supported before Unity 2019.1. To use this feature, either upgrade
+            // your project to Unity 2019.1 + or use single-pass non - instanced stereo rendering
+            // (XRSettings.stereoRenderingMode = SinglePass)."
+
+            if (XRDevice.isPresent)
+            {
+                return;
+            }
+
             // If no camera provided, attempt to fallback to main camera
             if (SourceCamera == null)
             {
@@ -96,13 +111,17 @@ protected new void OnDisable()
         {
             base.OnDisable();
 
-            // The camera sometimes goes away before this component.
-            if (SourceCamera != null)
-            {
-                SourceCamera.RemoveCommandBuffer(CameraEvent, _commandBuffer);
+            if (_commandBuffer != null)
+            {
+                // The camera sometimes goes away before this component.
+                if (SourceCamera != null)
+                {
+                    SourceCamera.RemoveCommandBuffer(CameraEvent, _commandBuffer);
+                }
+
+                _commandBuffer.Dispose();
+                _commandBuffer = null;
             }
-            _commandBuffer.Dispose();
-            _commandBuffer = null;
         }
 
         /// <summary>