feat: PiP support for iOS

development.md

@@ -53,6 +53,17 @@
 - [x] Bluetooth fixes (check support of BT media devices in flutter)
 - [x] Switch the earpiece/microphone button *(depends on "Bluetooth fixes")*
 - [x] Align custom event support to other SDKs
+- [x] Picture-in-picture
+- - [x] Android
+- - [x] iOS
+- [x] Call stats component
+- [x] Transcription
+
+### 0.5.0 milestone
+- [ ] Documentation parity
+- - [ ] UI components
+- - [ ] Cookbook
+- - [ ] Advanced guides
 - [ ] Tap to focus (flutter_webrtc)
 - [ ] Video filters / audio filters
 - [ ] Local audio levels (maybe from webrtc)
@@ -61,18 +72,8 @@
 - - [ ] stream_video_flutter (75%)
 - - [ ] stream_video_push_notification
 - - [ ] Coverage check for PRs
-- [ ] Documentation parity
-- - [ ] UI components
-- - [ ] Cookbook
-- - [ ] Advanced guides
-
-### 0.5.0 milestone
-- [ ] Picture-in-picture
-- - [x] Android
-- - [ ] iOS
 - [ ] Dynascale 2.0
 - [ ] SFU switching
 - [ ] Buttons to simulate ICE restarts / SFU switching
 - [ ] Proximity button support
-- [x] Call stats component
-- [ ] Transcription
+- [ ] Reconnection v2

docusaurus/docs/Flutter/05-advanced/03-screen-sharing.mdx

@@ -46,11 +46,11 @@ If you use our UI components you can also add `ToggleScreenShareOption` as one o
 When the method is invoked, ReplayKit will ask for the user's consent that their screen will be shared. Only after the permission is granted, the screensharing starts.
 
 #### Broadcasting
-In most cases, you would need to share your screen while you are in the background, to be able to open other apps. For this, you need to create a Broadcast Upload Extension.
+In most cases, you would need to share your screen while the app is in the background, to be able to open other apps. For this, you need to create a Broadcast Upload Extension.
 
 ##### Toggle screen sharing with broadcast mode
 
-If you want to start screen sharing in broadcast mode on iOS you will need to toggle on using `ScreenShareConstraints`. Just set the flag to true inside `ToggleScreenShareOption` or when you use `call.setScreenShareEnabled()` directly.
+If you want to start screen sharing in broadcast mode on iOS you will need to toggle it by setting `useiOSBroadcastExtension` flag to true in `ScreenShareConstraints`. You can set the constraints inside `ToggleScreenShareOption` or when you use `call.setScreenShareEnabled()` directly.
 
 ```dart
 const constraints = ScreenShareConstraints(
@@ -72,6 +72,8 @@ call.setScreenShareEnabled(enabled: true, constraints: constraints);
 
 ##### Add Broadcast Upload Extension
 
+iOS requires the use of Broadcast Upload Extensions to facilitate screen sharing when your app is in the background. This extension provides the necessary framework to handle capturing and broadcasting the screen content.
+
 Now add the extension, without UI, to your project in Xcode:
 
 ![Screen sharing dashboard](../assets/advanced_assets/broadcast-extension.png)
@@ -85,17 +87,24 @@ After you create the extension, there should be a class called `SampleHandler`,
 
 ![Screen sharing dashboard](../assets/advanced_assets/broadcast-handler-implementation.png)
 
-:::tip
-If Xcode cannot find the `stream_video_screen_sharing` module add this code block to your app's Podfile file:
+To have access to our Handler implementation add `stream_video_screen_sharing` package to your app's `pubspec.yaml` file:
+
+```yaml
+dependencies:
+  stream_video_screen_sharing: ^<latest_version>
+```
+
+Then for native code to see it, add it as a dependency manually for the extension target in the Podfile file:
 
 ```Podfile
-target 'ScreenSharing' do
+target 'YOUR_EXTENSION_NAME' do
   use_frameworks!
-  pod 'stream_video_flutter', :path => File.join('.symlinks', 'plugins', 'stream_video_flutter', 'ios')
+  pod 'stream_video_screen_sharing', :path => File.join('.symlinks', 'plugins', 'stream_video_screen_sharing', 'ios')
 end
 ```
 
-Replace `ScreenSharing` with your extension name.
+:::note
+Replace `YOUR_EXTENSION_NAME` with the name of the extension you created.
 :::
 
 ##### Setup app groups
@@ -178,4 +187,22 @@ StreamBackgroundService.init(
     }
   },
 );
-```
+```
+
+### Screen sharing settings
+
+You can customize the screen sharing behavior by providing `ScreenShareConstraints` to the `ToggleScreenShareOption` widget or `setScreenShareEnabled()` method.
+
+You can specify the following settings:
+
+- `useiOSBroadcastExtension` - Set to `true` to enable broadcast mode on iOS.
+
+- `captureScreenAudio` - Set to `true` to capture audio from the screen.
+
+- `sourceId` - The device ID of an audio source, if you want to capture audio from a specific source.
+
+- `maxFrameRate` - The maximum frame rate for the screen sharing video.
+
+- `params` - The video parameters for the screen sharing video.
+
+For `params` you can use one of our predefined presets in `RtcVideoParametersPresets`.

docusaurus/docs/Flutter/05-advanced/04-picture-in-picture.mdx

@@ -6,41 +6,19 @@ title: Picture in Picture (PiP)
 
 Picture in picture (PIP) keeps the call running and visible while you navigate to other apps.
 
-:::info
-  At the moment Picture in Picture is only supported on Android.
-:::
-
 ### Enable Picture-in-Picture
-You can enable Picture in Picture by setting the `enablePictureInPicture` property to `true` in the `StreamCallContainer` or `StreamCallContent` widget.
+You can enable Picture in Picture by setting the `enablePictureInPicture` property to `true` in the `PictureInPictureConfiguration` provided to `StreamCallContainer` or `StreamCallContent` widget.
 
 ```dart
 StreamCallContainer(
     call: widget.call,
-    enablePictureInPicture: true,
+    pictureInPictureConfiguration: const PictureInPictureConfiguration(
+        enablePictureInPicture: true,
+    ),
 )
 ```
 
-You can customize the widget rendered while app is in Picture-in-Picture mode by providing `callPictureInPictureBuilder` to `StreamCallContent`.
-
-```dart
-  StreamCallContainer(
-    call: widget.call,
-    callContentBuilder: (
-        BuildContext context,
-        Call call,
-        CallState callState,
-        ) {
-      return StreamCallContent(
-        call: call,
-        callState: callState,
-        enablePictureInPicture: true,
-        callPictureInPictureBuilder: (context, call, callState) {
-          // YOUR CUSTOM WIDGET
-        }
-      );
-    },
-  );
-```
+## Android
 
 ### Android Configuration
 To enable Picture in Picture on Android, you need to add the following configuration to your `AndroidManifest.xml` file.
@@ -67,6 +45,85 @@ class MainActivity: FlutterActivity() {
 }
 ```
 
+### Android Customization
+
+For Android, you can customize the widget rendered while app is in Picture-in-Picture mode by providing `callPictureInPictureBuilder` to `PictureInPictureConfiguration`.
+
+```dart
+  StreamCallContainer(
+    call: widget.call,
+    callContentBuilder: (
+        BuildContext context,
+        Call call,
+        CallState callState,
+        ) {
+      return StreamCallContent(
+        call: call,
+        callState: callState,
+        pictureInPictureConfiguration: const PictureInPictureConfiguration(
+          enablePictureInPicture: true,
+          androidPiPConfiguration: AndroidPictureInPictureConfiguration(
+            callPictureInPictureBuilder: (context, call, callState) {
+              // YOUR CUSTOM WIDGET
+            },
+          )
+        ),
+      );
+    },
+  );
+```
+
+## iOS
+
+### Local camera feed in Picture-in-Picture mode
+
+By default iOS is not allowing usage of local camera feed when app is in the background. That includes Picture in picture mode and Split View mode. To enable it you need to request the [multitasking-camera-access](https://developer.apple.com/documentation/bundleresources/entitlements/com_apple_developer_avfoundation_multitasking-camera-access) permission directly from Apple (this will change from iOS 18).
+If you already have the permission you have to enable local feed support in our PiP implementation by setting `ignoreLocalParticipantVideo` to `false`.
+
+```dart
+  StreamCallContainer(
+    call: widget.call,
+    callContentBuilder: (
+        BuildContext context,
+        Call call,
+        CallState callState,
+        ) {
+      return StreamCallContent(
+        call: call,
+        callState: callState,
+        pictureInPictureConfiguration: const PictureInPictureConfiguration(
+          enablePictureInPicture: true,
+          iOSPiPConfiguration: IOSPictureInPictureConfiguration(
+            ignoreLocalParticipantVideo: false,
+          )
+        ),
+      );
+    },
+  );
+```
+
+### Enabling PiP support with custom call content widget
+
+If you are not using our `StreamCallContent` and instead building custom call content widget you can still enable Picture in Picture mode by adding `StreamPictureInPictureUiKitView` anywhere in the widget tree. This widget will handle the Picture in Picture mode in iOS for you.
+
+```dart
+  StreamCallContainer(
+    call: widget.call,
+    callContentBuilder: (
+        BuildContext context,
+        Call call,
+        CallState callState,
+        ) {
+      return Stack(
+        children: [
+          StreamPictureInPictureUiKitView(call: call),
+          // YOUR CUSTOM WIDGET
+        ],
+      );
+    },
+  );
+```
+
 Done. Now after leaving the app, you'll see that the call will be still alive in the background like the one below:
 
 ![Picture in Picture example](../assets/advanced_assets/pip_example.png)

dogfooding/ios/Podfile

@@ -36,7 +36,7 @@ end
 
 target 'ScreenSharing' do
   use_frameworks!
-  pod 'stream_video_flutter', :path => File.join('.symlinks', 'plugins', 'stream_video_flutter', 'ios')
+  pod 'stream_video_screen_sharing', :path => File.join('.symlinks', 'plugins', 'stream_video_screen_sharing', 'ios')
 end
 
 post_install do |installer|
@@ -60,4 +60,4 @@ post_install do |installer|
       end
     end
   end
-end
+end

dogfooding/ios/ScreenSharing/SampleHandler.swift

@@ -1,4 +1,4 @@
 import ReplayKit
-import stream_video_flutter
+import stream_video_screen_sharing
 
 class SampleHandler: BroadcastSampleHandler {}

dogfooding/lib/screens/call_screen.dart

@@ -147,7 +147,10 @@ class _CallScreenState extends State<CallScreen> {
               call: call,
               callState: callState,
               layoutMode: _currentLayoutMode,
-              enablePictureInPicture: true,
+              pictureInPictureConfiguration:
+                  const PictureInPictureConfiguration(
+                enablePictureInPicture: true,
+              ),
               callParticipantsBuilder: (context, call, callState) {
                 return Stack(
                   children: [

dogfooding/pubspec.yaml

@@ -31,6 +31,7 @@ dependencies:
   stream_chat_flutter: ^8.0.0
   stream_video_flutter: ^0.4.4
   stream_video_push_notification: ^0.4.4
+  stream_video_screen_sharing: ^0.4.4
   uni_links: ^0.5.1
 
 dev_dependencies:

packages/stream_video/lib/src/webrtc/rtc_manager.dart

@@ -13,7 +13,6 @@ import 'codecs_helper.dart' as codecs;
 import 'model/rtc_audio_bitrate_preset.dart';
 import 'model/rtc_tracks_info.dart';
 import 'model/rtc_video_encoding.dart';
-import 'model/rtc_video_parameters.dart';
 import 'peer_connection.dart';
 import 'rtc_parser.dart';
 

packages/stream_video/lib/stream_video.dart

@@ -32,6 +32,7 @@ export 'src/utils/string.dart';
 export 'src/utils/subscriptions.dart';
 export 'src/webrtc/media/media_constraints.dart';
 export 'src/webrtc/model/rtc_video_dimension.dart';
+export 'src/webrtc/model/rtc_video_parameters.dart';
 export 'src/webrtc/peer_type.dart';
 export 'src/webrtc/rtc_media_device/rtc_media_device.dart';
 export 'src/webrtc/rtc_media_device/rtc_media_device_notifier.dart';

packages/stream_video/lib/version.g.dart

@@ -1,5 +1,5 @@
 // This file is generated. Do not manually edit.
 /// Current package version.
 const String streamVideoVersion = '0.4.4';
-const String androidWebRTCVersion = 'libwebrtc-m114.5735.10';
-const String iosWebRTCVersion = 'libwebrtc-m114.5735.10';
+const String androidWebRTCVersion = 'libwebrtc-m125.6422.03';
+const String iosWebRTCVersion = 'libwebrtc-m125.6422.04';

packages/stream_video/pubspec.yaml

@@ -14,13 +14,13 @@ dependencies:
   async: ^2.11.0
   collection: ^1.18.0
   connectivity_plus: ^6.0.3
-  dart_webrtc: ^1.4.5
+  dart_webrtc: ^1.4.8
   device_info_plus: ^10.1.0
   equatable: ^2.0.5
   fixnum: ^1.1.0
   flutter:
     sdk: flutter
-  flutter_webrtc: ^0.10.4
+  flutter_webrtc: ^0.11.6
   http: ^1.1.0
   internet_connection_checker: ^1.0.0+1
   intl: ">=0.18.1 <=0.19.0"

packages/stream_video_flutter/example/pubspec.yaml

@@ -21,7 +21,7 @@ dependencies:
     sdk: flutter
   flutter_local_notifications: ^17.1.2
   flutter_svg: ^2.0.7
-  flutter_webrtc: ^0.10.4
+  flutter_webrtc: ^0.11.6
   google_fonts: ^5.1.0
   http: ^1.1.0
   intl: ^0.19.0

packages/stream_video_flutter/ios/Classes/PictureInPicture/SampleBufferVideoCallView.swift

@@ -0,0 +1,52 @@
+//
+// Copyright © 2024 Stream.io Inc. All rights reserved.
+//
+
+import AVKit
+import UIKit
+
+final class SampleBufferVideoCallView: UIView {
+    override class var layerClass: AnyClass {
+        AVSampleBufferDisplayLayer.self
+    }
+
+    private var sampleBufferDisplayLayer: AVSampleBufferDisplayLayer {
+        layer as! AVSampleBufferDisplayLayer
+    }
+
+    var renderingComponent: SampleBufferVideoRendering {
+        #if swift(>=5.9)
+        if #available(iOS 17.0, *) {
+            return sampleBufferDisplayLayer.sampleBufferRenderer
+        } else {
+            return sampleBufferDisplayLayer
+        }
+        #else
+        return sampleBufferDisplayLayer
+        #endif
+    }
+
+    var videoGravity: AVLayerVideoGravity {
+        get { sampleBufferDisplayLayer.videoGravity }
+        set { sampleBufferDisplayLayer.videoGravity = newValue }
+    }
+
+    var preventsDisplaySleepDuringVideoPlayback: Bool {
+        get { sampleBufferDisplayLayer.preventsDisplaySleepDuringVideoPlayback }
+        set { sampleBufferDisplayLayer.preventsDisplaySleepDuringVideoPlayback = newValue }
+    }
+}
+
+protocol SampleBufferVideoRendering {
+    @available(iOS 14.0, *)
+    var requiresFlushToResumeDecoding: Bool { get }
+    var isReadyForMoreMediaData: Bool { get }
+    func flush()
+    func enqueue(_ sampleBuffer: CMSampleBuffer)
+}
+
+extension AVSampleBufferDisplayLayer: SampleBufferVideoRendering {}
+#if swift(>=5.9)
+@available(iOS 17.0, *)
+extension AVSampleBufferVideoRenderer: SampleBufferVideoRendering {}
+#endif