Skip to content

Commit

Permalink
Darwin improvements
Browse files Browse the repository at this point in the history
[Mac/iOS] feat: Add RTCYUVHelper for darwin. (#28)

Cross-platform `RTCMTLVideoView` for both iOS / macOS (#40)

rotationOverride should not be assign (#44)

[ObjC] Expose properties / methods required for AV1 codec support (#60)

Workaround: Render PixelBuffer in RTCMTLVideoView (#58)

Improve iOS/macOS H264 encoder (#70)

fix: fix video encoder not resuming correctly upon foregrounding (#75).

Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com>
  • Loading branch information
cloudwebrtc and hiroshihorie committed Jun 12, 2023
1 parent 378dc0b commit 2abe97d
Show file tree
Hide file tree
Showing 24 changed files with 999 additions and 236 deletions.
17 changes: 7 additions & 10 deletions media/engine/webrtc_video_engine.cc
Original file line number Diff line number Diff line change
Expand Up @@ -642,18 +642,15 @@ WebRtcVideoEngine::GetRtpHeaderExtensions() const {
webrtc::RtpExtension::kVideoContentTypeUri,
webrtc::RtpExtension::kVideoTimingUri,
webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri,
webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) {
webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri,
// "WebRTC-DependencyDescriptorAdvertised"
webrtc::RtpExtension::kDependencyDescriptorUri}) {
result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv);
}
result.emplace_back(webrtc::RtpExtension::kGenericFrameDescriptorUri00, id++,
IsEnabled(trials_, "WebRTC-GenericDescriptorAdvertised")
? webrtc::RtpTransceiverDirection::kSendRecv
: webrtc::RtpTransceiverDirection::kStopped);
result.emplace_back(
webrtc::RtpExtension::kDependencyDescriptorUri, id++,
IsEnabled(trials_, "WebRTC-DependencyDescriptorAdvertised")
? webrtc::RtpTransceiverDirection::kSendRecv
: webrtc::RtpTransceiverDirection::kStopped);

result.emplace_back(
webrtc::RtpExtension::kVideoLayersAllocationUri, id++,
Expand Down Expand Up @@ -931,7 +928,7 @@ void WebRtcVideoChannel::RequestEncoderSwitch(
void WebRtcVideoChannel::StartReceive(uint32_t ssrc) {
RTC_DCHECK_RUN_ON(&thread_checker_);
WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
if(!stream) {
if (!stream) {
return;
}
stream->StartStream();
Expand All @@ -940,7 +937,7 @@ void WebRtcVideoChannel::StartReceive(uint32_t ssrc) {
void WebRtcVideoChannel::StopReceive(uint32_t ssrc) {
RTC_DCHECK_RUN_ON(&thread_checker_);
WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
if(!stream) {
if (!stream) {
return;
}
stream->StopStream();
Expand Down Expand Up @@ -3200,12 +3197,12 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters(
}
}

void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream(){
void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream() {
if (stream_) {
stream_->Start();
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream(){
void WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream() {
if (stream_) {
stream_->Stop();
}
Expand Down
22 changes: 17 additions & 5 deletions sdk/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,15 @@ if (is_ios || is_mac) {
"objc/helpers/RTCDispatcher+Private.h",
"objc/helpers/RTCDispatcher.h",
"objc/helpers/RTCDispatcher.m",
"objc/helpers/RTCYUVHelper.h",
"objc/helpers/RTCYUVHelper.mm",
"objc/helpers/scoped_cftyperef.h",
]

deps = [
":base_objc",
"../rtc_base:checks",
"//third_party/libyuv",
]

absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
Expand Down Expand Up @@ -635,17 +638,13 @@ if (is_ios || is_mac) {
"Metal.framework",
"MetalKit.framework",
]
if (is_ios) {
if (is_ios || is_mac) {
sources += [
"objc/components/renderer/metal/RTCMTLVideoView.h",
"objc/components/renderer/metal/RTCMTLVideoView.m",
]
}
if (is_mac) {
sources += [
"objc/components/renderer/metal/RTCMTLNSVideoView.h",
"objc/components/renderer/metal/RTCMTLNSVideoView.m",
]
frameworks += [ "AppKit.framework" ]
}
deps = [
Expand Down Expand Up @@ -1032,6 +1031,12 @@ if (is_ios || is_mac) {
"objc/api/peerconnection/RTCRtcpParameters+Private.h",
"objc/api/peerconnection/RTCRtcpParameters.h",
"objc/api/peerconnection/RTCRtcpParameters.mm",
"objc/api/peerconnection/RTCRtpCapabilities.h",
"objc/api/peerconnection/RTCRtpCapabilities.mm",
"objc/api/peerconnection/RTCRtpCapabilities+Private.h",
"objc/api/peerconnection/RTCRtpCodecCapability.h",
"objc/api/peerconnection/RTCRtpCodecCapability.mm",
"objc/api/peerconnection/RTCRtpCodecCapability+Private.h",
"objc/api/peerconnection/RTCRtpCodecParameters+Private.h",
"objc/api/peerconnection/RTCRtpCodecParameters.h",
"objc/api/peerconnection/RTCRtpCodecParameters.mm",
Expand Down Expand Up @@ -1329,6 +1334,7 @@ if (is_ios || is_mac) {
"objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
"objc/helpers/RTCCameraPreviewView.h",
"objc/helpers/RTCDispatcher.h",
"objc/helpers/RTCYUVHelper.h",
"objc/helpers/UIDevice+RTCDevice.h",
"objc/api/peerconnection/RTCAudioDeviceModule.h",
"objc/api/peerconnection/RTCIODevice.h",
Expand All @@ -1352,6 +1358,8 @@ if (is_ios || is_mac) {
"objc/api/peerconnection/RTCPeerConnectionFactory.h",
"objc/api/peerconnection/RTCPeerConnectionFactoryOptions.h",
"objc/api/peerconnection/RTCRtcpParameters.h",
"objc/api/peerconnection/RTCRtpCapabilities.h",
"objc/api/peerconnection/RTCRtpCodecCapability.h",
"objc/api/peerconnection/RTCRtpCodecParameters.h",
"objc/api/peerconnection/RTCRtpEncodingParameters.h",
"objc/api/peerconnection/RTCRtpHeaderExtension.h",
Expand Down Expand Up @@ -1474,6 +1482,8 @@ if (is_ios || is_mac) {
"objc/api/peerconnection/RTCPeerConnectionFactory.h",
"objc/api/peerconnection/RTCPeerConnectionFactoryOptions.h",
"objc/api/peerconnection/RTCRtcpParameters.h",
"objc/api/peerconnection/RTCRtpCapabilities.h",
"objc/api/peerconnection/RTCRtpCodecCapability.h",
"objc/api/peerconnection/RTCRtpCodecParameters.h",
"objc/api/peerconnection/RTCRtpEncodingParameters.h",
"objc/api/peerconnection/RTCRtpHeaderExtension.h",
Expand Down Expand Up @@ -1517,6 +1527,7 @@ if (is_ios || is_mac) {
"objc/base/RTCYUVPlanarBuffer.h",
"objc/components/capturer/RTCCameraVideoCapturer.h",
"objc/components/capturer/RTCFileVideoCapturer.h",
"objc/components/renderer/metal/RTCMTLVideoView.h",
"objc/components/renderer/metal/RTCMTLNSVideoView.h",
"objc/components/renderer/opengl/RTCNSGLVideoView.h",
"objc/components/renderer/opengl/RTCVideoViewShading.h",
Expand All @@ -1530,6 +1541,7 @@ if (is_ios || is_mac) {
"objc/components/video_codec/RTCVideoEncoderH264.h",
"objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
"objc/helpers/RTCDispatcher.h",
"objc/helpers/RTCYUVHelper.h",
# Added for Simulcast support
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
Expand Down
8 changes: 8 additions & 0 deletions sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ NS_ASSUME_NONNULL_BEGIN
@class RTC_OBJC_TYPE(RTCVideoTrack);
@class RTC_OBJC_TYPE(RTCPeerConnectionFactoryOptions);
@class RTC_OBJC_TYPE(RTCAudioDeviceModule);
@class RTC_OBJC_TYPE(RTCRtpCapabilities);

typedef NS_ENUM(NSInteger, RTCRtpMediaType);

@protocol RTC_OBJC_TYPE
(RTCPeerConnectionDelegate);
@protocol RTC_OBJC_TYPE
Expand Down Expand Up @@ -61,6 +65,10 @@ RTC_OBJC_EXPORT

@property(nonatomic, readonly) RTCAudioDeviceModule *audioDeviceModule;

- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpSenderCapabilitiesFor:(RTCRtpMediaType)mediaType;

- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpReceiverCapabilitiesFor:(RTCRtpMediaType)mediaType;

/** Initialize an RTCAudioSource with constraints. */
- (RTC_OBJC_TYPE(RTCAudioSource) *)audioSourceWithConstraints:
(nullable RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints;
Expand Down
17 changes: 17 additions & 0 deletions sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
#import "RTCPeerConnection+Private.h"
#import "RTCVideoSource+Private.h"
#import "RTCVideoTrack+Private.h"
#import "RTCRtpReceiver+Private.h"
#import "RTCRtpCapabilities+Private.h"
#import "RTCRtpCodecCapability+Private.h"
#import "base/RTCLogging.h"
#import "base/RTCVideoDecoderFactory.h"
#import "base/RTCVideoEncoderFactory.h"
Expand Down Expand Up @@ -124,6 +127,20 @@ - (instancetype)init {
#endif
}

- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpSenderCapabilitiesFor:(RTCRtpMediaType)mediaType {

webrtc::RtpCapabilities capabilities = _nativeFactory->GetRtpSenderCapabilities([RTCRtpReceiver nativeMediaTypeForMediaType: mediaType]);

return [[RTCRtpCapabilities alloc] initWithNativeCapabilities: capabilities];
}

- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpReceiverCapabilitiesFor:(RTCRtpMediaType)mediaType {

webrtc::RtpCapabilities capabilities = _nativeFactory->GetRtpReceiverCapabilities([RTCRtpReceiver nativeMediaTypeForMediaType: mediaType]);

return [[RTCRtpCapabilities alloc] initWithNativeCapabilities: capabilities];
}

- (instancetype)
initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
Expand Down
33 changes: 33 additions & 0 deletions sdk/objc/api/peerconnection/RTCRtpCapabilities+Private.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Copyright 2023 LiveKit
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#import "RTCRtpCapabilities.h"

#include "api/rtp_parameters.h"

NS_ASSUME_NONNULL_BEGIN

@interface RTC_OBJC_TYPE (RTCRtpCapabilities)
()

@property(nonatomic, readonly) webrtc::RtpCapabilities nativeCapabilities;

- (instancetype)initWithNativeCapabilities:(const webrtc::RtpCapabilities &)nativeCapabilities
NS_DESIGNATED_INITIALIZER;

@end

NS_ASSUME_NONNULL_END
40 changes: 40 additions & 0 deletions sdk/objc/api/peerconnection/RTCRtpCapabilities.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright 2023 LiveKit
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#import <Foundation/Foundation.h>

#import "RTCMacros.h"

@class RTC_OBJC_TYPE(RTCRtpCodecCapability);

NS_ASSUME_NONNULL_BEGIN

RTC_OBJC_EXPORT
@interface RTC_OBJC_TYPE (RTCRtpCapabilities) : NSObject

- (instancetype)init NS_UNAVAILABLE;

@property(nonatomic, readonly) NSArray<RTC_OBJC_TYPE(RTCRtpCodecCapability) *> *codecs;

// Not implemented.
// std::vector<RtpHeaderExtensionCapability> header_extensions;

// Not implemented.
// std::vector<FecMechanism> fec;

@end

NS_ASSUME_NONNULL_END
50 changes: 50 additions & 0 deletions sdk/objc/api/peerconnection/RTCRtpCapabilities.mm
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Copyright 2023 LiveKit
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#import "RTCRtpCapabilities+Private.h"
#import "RTCRtpCodecCapability+Private.h"

#import "RTCMediaStreamTrack.h"
#import "helpers/NSString+StdString.h"

#include "media/base/media_constants.h"
#include "rtc_base/checks.h"

@implementation RTC_OBJC_TYPE (RTCRtpCapabilities)

@synthesize nativeCapabilities = _nativeCapabilities;

- (instancetype)initWithNativeCapabilities:(const webrtc::RtpCapabilities &)nativeCapabilities {
if (self = [super init]) {
_nativeCapabilities = nativeCapabilities;
}

return self;
}

- (NSArray<RTC_OBJC_TYPE(RTCRtpCodecCapability) *> *)codecs {
NSMutableArray *result = [NSMutableArray array];

for (auto &element : _nativeCapabilities.codecs) {
RTCRtpCodecCapability *object =
[[RTCRtpCodecCapability alloc] initWithNativeCodecCapability:element];
[result addObject:object];
}

return result;
}

@end
33 changes: 33 additions & 0 deletions sdk/objc/api/peerconnection/RTCRtpCodecCapability+Private.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Copyright 2023 LiveKit
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#import "RTCRtpCodecCapability.h"

#include "api/rtp_parameters.h"

NS_ASSUME_NONNULL_BEGIN

@interface RTC_OBJC_TYPE (RTCRtpCodecCapability)
()

@property(nonatomic, readonly) webrtc::RtpCodecCapability nativeCodecCapability;

- (instancetype)initWithNativeCodecCapability:
(const webrtc::RtpCodecCapability &)nativeCodecCapability NS_DESIGNATED_INITIALIZER;

@end

NS_ASSUME_NONNULL_END
Loading

0 comments on commit 2abe97d

Please sign in to comment.