Skip to content

Commit

Permalink
Merge pull request #1308 from shogo4405/feature/bitrate-storategy
Browse files Browse the repository at this point in the history 
Support the NetBitRateStrategy.
  • Loading branch information
@shogo4405
shogo4405 committed Sep 30, 2023
2 parents 3050627 + 6b048d3 commit f656089
Show file tree
Hide file tree
Showing 16 changed files with 161 additions and 60 deletions.
15 changes: 4 additions & 11 deletions Examples/iOS/LiveViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,20 +27,19 @@ final class LiveViewController: UIViewController {
private var currentEffect: VideoEffect?
private var currentPosition: AVCaptureDevice.Position = .back
private var retryCount: Int = 0
private var videoBitRate = VideoCodecSettings.default.bitRate
private var preferedStereo = false

override func viewDidLoad() {
super.viewDidLoad()

rtmpConnection.delegate = self

pipIntentView.layer.borderWidth = 1.0
pipIntentView.layer.borderColor = UIColor.white.cgColor
pipIntentView.bounds = MultiCamCaptureSettings.default.regionOfInterest
pipIntentView.isUserInteractionEnabled = true
view.addSubview(pipIntentView)

rtmpConnection.delegate = self

rtmpStream = RTMPStream(connection: rtmpConnection)
if let orientation = DeviceUtil.videoOrientation(by: UIApplication.shared.statusBarOrientation) {
rtmpStream.videoOrientation = orientation
Expand All @@ -62,7 +61,7 @@ final class LiveViewController: UIViewController {
allowFrameReordering: nil,
isHardwareEncoderEnabled: true
)

rtmpStream.bitrateStrategy = VideoAdaptiveNetBitRateStrategy(mamimumVideoBitrate: VideoCodecSettings.default.bitRate)
rtmpStream.mixer.recorder.delegate = self
videoBitrateSlider?.value = Float(VideoCodecSettings.default.bitRate) / 1000
audioBitrateSlider?.value = Float(AudioCodecSettings.default.bitRate) / 1000
Expand Down Expand Up @@ -163,7 +162,7 @@ final class LiveViewController: UIViewController {
}
if slider == videoBitrateSlider {
videoBitrateLabel?.text = "video \(Int(slider.value))/kbps"
rtmpStream.videoSettings.bitRate = UInt32(slider.value * 1000)
rtmpStream.bitrateStrategy = VideoAdaptiveNetBitRateStrategy(mamimumVideoBitrate: Int(slider.value * 1000))
}
if slider == zoomSlider {
let zoomFactor = CGFloat(slider.value)
Expand Down Expand Up @@ -348,21 +347,15 @@ final class LiveViewController: UIViewController {

extension LiveViewController: RTMPConnectionDelegate {
func connection(_ connection: RTMPConnection, publishInsufficientBWOccured stream: RTMPStream) {
// Adaptive bitrate streaming exsample. Please feedback me your good algorithm. :D
videoBitRate -= 32 * 1000
stream.videoSettings.bitRate = max(videoBitRate, 64 * 1000)
}

func connection(_ connection: RTMPConnection, publishSufficientBWOccured stream: RTMPStream) {
videoBitRate += 32 * 1000
stream.videoSettings.bitRate = min(videoBitRate, VideoCodecSettings.default.bitRate)
}

func connection(_ connection: RTMPConnection, updateStats stream: RTMPStream) {
}

func connection(_ connection: RTMPConnection, didClear stream: RTMPStream) {
videoBitRate = VideoCodecSettings.default.bitRate
}
}

Expand Down
4 changes: 0 additions & 4 deletions Examples/iOS/PlaybackViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,10 +141,6 @@ extension PlaybackViewController: NetStreamDelegate {
func stream(_ stream: NetStream, audioCodecErrorOccurred error: HaishinKit.AudioCodec.Error) {
}

func streamWillDropFrame(_ stream: NetStream) -> Bool {
return false
}

func streamDidOpen(_ stream: NetStream) {
}
}
39 changes: 39 additions & 0 deletions Examples/iOS/VideoAdaptiveNetBitRateStrategy.swift
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import Foundation

public final class VideoAdaptiveNetBitRateStrategy: NetBitRateStrategyConvertible {
public weak var stream: NetStream?
public let mamimumVideoBitRate: Int
public let mamimumAudioBitRate: Int = 0
private var zeroBytesOutPerSecondCounts: Int = 0

public init(mamimumVideoBitrate: Int) {
self.mamimumVideoBitRate = mamimumVideoBitrate
}

public func setUp() {
zeroBytesOutPerSecondCounts = 0
stream?.videoSettings.bitRate = mamimumVideoBitRate
}

public func sufficientBWOccured(_ stats: NetBitRateStats) {
logger.info(stats)
guard let stream else {
return
}
stream.videoSettings.bitRate = min(stream.videoSettings.bitRate + 64 * 1000, mamimumVideoBitRate)
}

public func insufficientBWOccured(_ stats: NetBitRateStats) {
logger.info(stats)
guard let stream, 0 < stats.currentBytesOutPerSecond else {
return
}
if 0 < stats.currentBytesOutPerSecond {
let bitRate = Int(stats.currentBytesOutPerSecond * 8) / (zeroBytesOutPerSecondCounts + 1)
stream.videoSettings.bitRate = max(bitRate - stream.audioSettings.bitRate, 64 * 1000)
zeroBytesOutPerSecondCounts = 0
} else {
zeroBytesOutPerSecondCounts += 1
}
}
}
10 changes: 9 additions & 1 deletion HaishinKit.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@
BC11024A2925147300D48035 /* IOCaptureUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1102492925147300D48035 /* IOCaptureUnit.swift */; };
BC110253292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */; };
BC110257292E661E00D48035 /* MultiCamCaptureSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* MultiCamCaptureSettings.swift */; };
BC1BC9042AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1BC9032AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift */; };
BC1DC4A429F4F74F00E928ED /* AVCaptureSession+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC4A329F4F74F00E928ED /* AVCaptureSession+Extension.swift */; };
BC1DC4FB2A02868900E928ED /* FLVVideoFourCC.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC4FA2A02868900E928ED /* FLVVideoFourCC.swift */; };
BC1DC5042A02894D00E928ED /* FLVVideoFourCCTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC5032A02894D00E928ED /* FLVVideoFourCCTests.swift */; };
Expand Down Expand Up @@ -189,6 +190,7 @@
BC562DCB29576D220048D89A /* AVCaptureSession.Preset+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC562DCA29576D220048D89A /* AVCaptureSession.Preset+Extension.swift */; };
BC566F6E25D2ECC500573C4C /* HLSService.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC566F6D25D2ECC500573C4C /* HLSService.swift */; };
BC570B4828E9ACC10098A12C /* IOUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC570B4728E9ACC10098A12C /* IOUnit.swift */; };
BC6692F32AC2F717009EC058 /* NetBitRateStrategyConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC6692F22AC2F717009EC058 /* NetBitRateStrategyConvertible.swift */; };
BC6FC91E29609A6800A746EE /* ShapeFactory.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC6FC91D29609A6800A746EE /* ShapeFactory.swift */; };
BC6FC9222961B3D800A746EE /* vImage_CGImageFormat+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC6FC9212961B3D800A746EE /* vImage_CGImageFormat+Extension.swift */; };
BC701F322AAC676C00C4BEFE /* AVAudioFormatFactory.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC701F312AAC676C00C4BEFE /* AVAudioFormatFactory.swift */; };
Expand Down Expand Up @@ -556,6 +558,7 @@
BC1102492925147300D48035 /* IOCaptureUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOCaptureUnit.swift; sourceTree = "<group>"; };
BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "vImage_Buffer+Extension.swift"; sourceTree = "<group>"; };
BC110256292E661E00D48035 /* MultiCamCaptureSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultiCamCaptureSettings.swift; sourceTree = "<group>"; };
BC1BC9032AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoAdaptiveNetBitRateStrategy.swift; sourceTree = "<group>"; };
BC1DC4A329F4F74F00E928ED /* AVCaptureSession+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureSession+Extension.swift"; sourceTree = "<group>"; };
BC1DC4FA2A02868900E928ED /* FLVVideoFourCC.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FLVVideoFourCC.swift; sourceTree = "<group>"; };
BC1DC5032A02894D00E928ED /* FLVVideoFourCCTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FLVVideoFourCCTests.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -596,6 +599,7 @@
BC562DCA29576D220048D89A /* AVCaptureSession.Preset+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureSession.Preset+Extension.swift"; sourceTree = "<group>"; };
BC566F6D25D2ECC500573C4C /* HLSService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HLSService.swift; sourceTree = "<group>"; };
BC570B4728E9ACC10098A12C /* IOUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOUnit.swift; sourceTree = "<group>"; };
BC6692F22AC2F717009EC058 /* NetBitRateStrategyConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetBitRateStrategyConvertible.swift; sourceTree = "<group>"; };
BC6FC91D29609A6800A746EE /* ShapeFactory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShapeFactory.swift; sourceTree = "<group>"; };
BC6FC9212961B3D800A746EE /* vImage_CGImageFormat+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "vImage_CGImageFormat+Extension.swift"; sourceTree = "<group>"; };
BC701F312AAC676C00C4BEFE /* AVAudioFormatFactory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVAudioFormatFactory.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -965,7 +969,6 @@
2968973F1CDB01AD0074D5F0 /* iOS */ = {
isa = PBXGroup;
children = (
29A39C801D85BEFA007C27E9 /* Screencast */,
296897411CDB01D20074D5F0 /* AppDelegate.swift */,
296897421CDB01D20074D5F0 /* Assets.xcassets */,
291F4E361CF206E200F59C51 /* Icon.png */,
Expand All @@ -977,6 +980,8 @@
BCFB355324FA275600DC5108 /* PlaybackViewController.swift */,
291468161E581C7D00E619BA /* Preference.swift */,
2950742E1E4620B7007F15A4 /* PreferenceViewController.swift */,
29A39C801D85BEFA007C27E9 /* Screencast */,
BC1BC9032AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift */,
296897461CDB01D20074D5F0 /* VisualEffect.swift */,
);
path = iOS;
Expand Down Expand Up @@ -1005,6 +1010,7 @@
isa = PBXGroup;
children = (
29B876971CD70B1100FC07DA /* MIME.swift */,
BC6692F22AC2F717009EC058 /* NetBitRateStrategyConvertible.swift */,
29B876981CD70B1100FC07DA /* NetClient.swift */,
29B876991CD70B1100FC07DA /* NetService.swift */,
29B8769A1CD70B1100FC07DA /* NetSocket.swift */,
Expand Down Expand Up @@ -1708,6 +1714,7 @@
BC1DC5142A05428800E928ED /* HEVCNALUnit.swift in Sources */,
BC6FC9222961B3D800A746EE /* vImage_CGImageFormat+Extension.swift in Sources */,
299B13271D3B751400A1E8F5 /* HKView.swift in Sources */,
BC1BC9042AC80531009005D3 /* VideoAdaptiveNetBitRateStrategy.swift in Sources */,
BC20DF38250377A3007BC608 /* IOUIScreenCaptureUnit.swift in Sources */,
29B876AF1CD70B2800FC07DA /* RTMPChunk.swift in Sources */,
29B876841CD70AE800FC07DA /* AVCDecoderConfigurationRecord.swift in Sources */,
Expand Down Expand Up @@ -1789,6 +1796,7 @@
295891261EEB8EF300CE51E1 /* FLVAACPacket.swift in Sources */,
29B876791CD70ACE00FC07DA /* HTTPStream.swift in Sources */,
BC1DC50A2A039B4400E928ED /* HEVCDecoderConfigurationRecord.swift in Sources */,
BC6692F32AC2F717009EC058 /* NetBitRateStrategyConvertible.swift in Sources */,
BC6FC91E29609A6800A746EE /* ShapeFactory.swift in Sources */,
BC32E88829C9971100051507 /* InstanceHolder.swift in Sources */,
BC7C56B7299E579F00C41A9B /* AudioCodecSettings.swift in Sources */,
Expand Down
3 changes: 1 addition & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@ Project name |Notes |License
- [x] Authentication
- [x] Publish and Recording
- [x] _Playback (Beta)_
- [x] Adaptive bitrate streaming
- [x] Handling (see also [#1153](/../../issues/1153))
- [x] Adaptive bitrate streaming (see also [#1308](/../../issues/1308))
- [ ] Action Message Format
- [x] AMF0
- [ ] AMF3
Expand Down
1 change: 1 addition & 0 deletions Sources/Codec/VTSessionMode.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ enum VTSessionMode {
videoCodec.delegate?.videoCodec(videoCodec, errorOccurred: .failedToPrepare(status: status))
return nil
}
videoCodec.frameInterval = videoCodec.settings.frameInterval
return session
case .decompression:
guard let formatDescription = videoCodec.outputFormat else {
Expand Down
47 changes: 29 additions & 18 deletions Sources/Codec/VideoCodec.swift
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import AVFoundation
import CoreFoundation
import VideoToolbox

#if os(iOS)
#if canImport(UIKit)
import UIKit
#endif

Expand All @@ -16,15 +15,15 @@ public protocol VideoCodecDelegate: AnyObject {
func videoCodec(_ codec: VideoCodec, didOutput sampleBuffer: CMSampleBuffer)
/// Tells the receiver to occured an error.
func videoCodec(_ codec: VideoCodec, errorOccurred error: VideoCodec.Error)
/// Tells the receiver to drop frame.
func videoCodecWillDropFame(_ codec: VideoCodec) -> Bool
}

// MARK: -
/**
* The VideoCodec class provides methods for encode or decode for video.
*/
public class VideoCodec {
public final class VideoCodec {
static let defaultFrameInterval = 0.0

/**
* The VideoCodec error domain codes.
*/
Expand Down Expand Up @@ -61,15 +60,6 @@ public class VideoCodec {
public private(set) var isRunning: Atomic<Bool> = .init(false)

var lockQueue = DispatchQueue(label: "com.haishinkit.HaishinKit.VideoCodec.lock")
var expectedFrameRate = IOMixer.defaultFrameRate
private(set) var outputFormat: CMFormatDescription? {
didSet {
guard !CMFormatDescriptionEqual(outputFormat, otherFormatDescription: oldValue) else {
return
}
delegate?.videoCodec(self, didOutput: outputFormat)
}
}
var needsSync: Atomic<Bool> = .init(true)
var attributes: [NSString: AnyObject]? {
guard VideoCodec.defaultAttributes != nil else {
Expand All @@ -83,17 +73,28 @@ public class VideoCodec {
attributes[kCVPixelBufferHeightKey] = NSNumber(value: settings.videoSize.height)
return attributes
}
var frameInterval = VideoCodec.defaultFrameInterval
var expectedFrameRate = IOMixer.defaultFrameRate
weak var delegate: (any VideoCodecDelegate)?
private(set) var outputFormat: CMFormatDescription? {
didSet {
guard !CMFormatDescriptionEqual(outputFormat, otherFormatDescription: oldValue) else {
return
}
delegate?.videoCodec(self, didOutput: outputFormat)
}
}
private(set) var session: (any VTSessionConvertible)? {
didSet {
oldValue?.invalidate()
invalidateSession = false
}
}
private var invalidateSession = true
private var presentationTimeStamp: CMTime = .invalid

func appendImageBuffer(_ imageBuffer: CVImageBuffer, presentationTimeStamp: CMTime, duration: CMTime) {
guard isRunning.value, !(delegate?.videoCodecWillDropFame(self) ?? false) else {
guard isRunning.value, !willDropFrame(presentationTimeStamp) else {
return
}
if invalidateSession {
Expand All @@ -108,6 +109,7 @@ public class VideoCodec {
delegate?.videoCodec(self, errorOccurred: .failedToFlame(status: status))
return
}
self.presentationTimeStamp = sampleBuffer.presentationTimeStamp
outputFormat = sampleBuffer.formatDescription
delegate?.videoCodec(self, didOutput: sampleBuffer)
}
Expand Down Expand Up @@ -163,7 +165,15 @@ public class VideoCodec {
}
}

#if os(iOS)
private func willDropFrame(_ presentationTimeStamp: CMTime) -> Bool {
guard Self.defaultFrameInterval < frameInterval else {
return false
}
print(presentationTimeStamp.seconds - self.presentationTimeStamp.seconds <= frameInterval)
return presentationTimeStamp.seconds - self.presentationTimeStamp.seconds <= frameInterval
}

#if os(iOS) || os(tvOS) || os(visionOS)
@objc
private func applicationWillEnterForeground(_ notification: Notification) {
invalidateSession = true
Expand Down Expand Up @@ -192,7 +202,7 @@ extension VideoCodec: Running {
public func startRunning() {
lockQueue.async {
self.isRunning.mutate { $0 = true }
#if os(iOS)
#if os(iOS) || os(tvOS) || os(visionOS)
NotificationCenter.default.addObserver(
self,
selector: #selector(self.didAudioSessionInterruption),
Expand All @@ -215,7 +225,8 @@ extension VideoCodec: Running {
self.invalidateSession = true
self.needsSync.mutate { $0 = true }
self.outputFormat = nil
#if os(iOS)
self.presentationTimeStamp = .invalid
#if os(iOS) || os(tvOS) || os(visionOS)
NotificationCenter.default.removeObserver(self, name: AVAudioSession.interruptionNotification, object: nil)
NotificationCenter.default.removeObserver(self, name: UIApplication.willEnterForegroundNotification, object: nil)
#endif
Expand Down
18 changes: 15 additions & 3 deletions Sources/Codec/VideoCodecSettings.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ import VideoToolbox

/// The VideoCodecSettings class specifying video compression settings.
public struct VideoCodecSettings: Codable {
public static let frameInterval30 = (1 / 30) - 0.001
public static let frameInterval10 = (1 / 10) - 0.001
public static let frameInterval05 = (1 / 05) - 0.001
public static let frameInterval01 = (1 / 01) - 0.001

/// The defulat value.
public static let `default` = VideoCodecSettings()

Expand Down Expand Up @@ -76,7 +81,9 @@ public struct VideoCodecSettings: Codable {
/// Specifies the video size of encoding video.
public var videoSize: CGSize
/// Specifies the bitrate.
public var bitRate: UInt32
public var bitRate: Int
/// Specifies the video frame interval.
public var frameInterval: Double
/// Specifies the keyframeInterval.
public var maxKeyFrameIntervalDuration: Int32
/// Specifies the scalingMode.
Expand Down Expand Up @@ -104,7 +111,8 @@ public struct VideoCodecSettings: Codable {
public init(
videoSize: CGSize = .init(width: 854, height: 480),
profileLevel: String = kVTProfileLevel_H264_Baseline_3_1 as String,
bitRate: UInt32 = 640 * 1000,
bitRate: Int = 640 * 1000,
frameInterval: Double = 0.0,
maxKeyFrameIntervalDuration: Int32 = 2,
scalingMode: ScalingMode = .trim,
bitRateMode: BitRateMode = .average,
Expand All @@ -114,14 +122,15 @@ public struct VideoCodecSettings: Codable {
self.videoSize = videoSize
self.profileLevel = profileLevel
self.bitRate = bitRate
self.frameInterval = frameInterval
self.maxKeyFrameIntervalDuration = maxKeyFrameIntervalDuration
self.scalingMode = scalingMode
self.bitRateMode = bitRateMode
self.allowFrameReordering = allowFrameReordering
self.isHardwareEncoderEnabled = isHardwareEncoderEnabled
if profileLevel.contains("HEVC") {
self.format = .hevc
}
self.isHardwareEncoderEnabled = isHardwareEncoderEnabled
}

func invalidateSession(_ rhs: VideoCodecSettings) -> Bool {
Expand All @@ -142,6 +151,9 @@ public struct VideoCodecSettings: Codable {
codec.delegate?.videoCodec(codec, errorOccurred: .failedToSetOption(status: status, option: option))
}
}
if frameInterval != rhs.frameInterval {
codec.frameInterval = frameInterval
}
}

func options(_ codec: VideoCodec) -> Set<VTSessionOption> {
Expand Down
4 changes: 0 additions & 4 deletions Sources/MPEG/TSWriter.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,10 +289,6 @@ extension TSWriter: VideoCodecDelegate {

public func videoCodec(_ codec: VideoCodec, errorOccurred error: VideoCodec.Error) {
}

public func videoCodecWillDropFame(_ codec: VideoCodec) -> Bool {
return false
}
}

class TSFileWriter: TSWriter {
Expand Down

0 comments on commit f656089

Please sign in to comment.