diff --git a/Demo/SwiftUI/Shared/MovieModel.swift b/Demo/SwiftUI/Shared/MovieModel.swift index b43fa0c01..20ef5a067 100644 --- a/Demo/SwiftUI/Shared/MovieModel.swift +++ b/Demo/SwiftUI/Shared/MovieModel.swift @@ -29,9 +29,19 @@ class MEOptions: KSOptions { MEOptions.isUseDisplayLayer && display == .plane } - #if os(tvOS) - override open func preferredDisplayCriteria(refreshRate: Float, videoDynamicRange: Int32) -> AVDisplayCriteria? { - AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange) + #if os(tvOS) || os(xrOS) + override open func preferredDisplayCriteria(track: some MediaPlayerTrack) -> AVDisplayCriteria? { + let refreshRate = track.nominalFrameRate + if #available(tvOS 17.0, *) { + if let formatDescription = track.formatDescription { + return AVDisplayCriteria(refreshRate: refreshRate, formatDescription: formatDescription) + } else { + return nil + } + } else { + let videoDynamicRange = track.dynamicRange(self).rawValue + return AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange) + } } #endif } diff --git a/Demo/demo-iOS/demo-iOS/AppDelegate.swift b/Demo/demo-iOS/demo-iOS/AppDelegate.swift index 09dba8d10..df3ed7d4a 100644 --- a/Demo/demo-iOS/demo-iOS/AppDelegate.swift +++ b/Demo/demo-iOS/demo-iOS/AppDelegate.swift @@ -104,9 +104,19 @@ class MEOptions: KSOptions { } } - #if os(tvOS) - override open func preferredDisplayCriteria(refreshRate: Float, videoDynamicRange: Int32) -> AVDisplayCriteria? { - AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange) + #if os(tvOS) || os(xrOS) + override open func preferredDisplayCriteria(track: some MediaPlayerTrack) -> AVDisplayCriteria? { + let refreshRate = track.nominalFrameRate + if #available(tvOS 17.0, *) { + if let formatDescription = track.formatDescription { + return AVDisplayCriteria(refreshRate: refreshRate, formatDescription: formatDescription) + } else { + return nil + } + } else { + let videoDynamicRange = track.dynamicRange(self).rawValue + return AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange) + } } #endif } diff --git a/Sources/KSPlayer/AVPlayer/KSAVPlayer.swift b/Sources/KSPlayer/AVPlayer/KSAVPlayer.swift index b0c250b91..abf141b4f 100644 --- a/Sources/KSPlayer/AVPlayer/KSAVPlayer.swift +++ b/Sources/KSPlayer/AVPlayer/KSAVPlayer.swift @@ -509,6 +509,7 @@ extension AVAssetTrack { } class AVMediaPlayerTrack: MediaPlayerTrack { + let formatDescription: CMFormatDescription? = nil let description: String private let track: AVPlayerItemTrack let nominalFrameRate: Float diff --git a/Sources/KSPlayer/AVPlayer/KSOptions.swift b/Sources/KSPlayer/AVPlayer/KSOptions.swift index 93895e54f..443e3b3a3 100644 --- a/Sources/KSPlayer/AVPlayer/KSOptions.swift +++ b/Sources/KSPlayer/AVPlayer/KSOptions.swift @@ -226,7 +226,7 @@ open class KSOptions { 16 } - open func audioFrameMaxCount(fps: Float, channels _: Int) -> Int { + open func audioFrameMaxCount(fps: Float, channelCount _: Int) -> Int { Int(fps) >> 2 } @@ -303,10 +303,20 @@ open class KSOptions { */ open func process(assetTrack _: some MediaPlayerTrack) {} - #if os(tvOS) - open func preferredDisplayCriteria(refreshRate _: Float, videoDynamicRange _: Int32) -> AVDisplayCriteria? { -// AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange) - nil + #if os(tvOS) || os(xrOS) + open func preferredDisplayCriteria(track: some MediaPlayerTrack) -> AVDisplayCriteria? { + let refreshRate = track.nominalFrameRate + if #available(tvOS 17.0, *) { + if let formatDescription = track.formatDescription { + return AVDisplayCriteria(refreshRate: refreshRate, formatDescription: formatDescription) + } else { + return nil + } + } else { +// let videoDynamicRange = track.dynamicRange(self).rawValue +// return AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange) + return nil + } } #endif @@ -424,7 +434,7 @@ public extension KSOptions { } } - static func outputNumberOfChannels(channels: AVAudioChannelCount, isUseAudioRenderer: Bool) -> AVAudioChannelCount { + static func outputNumberOfChannels(channelCount: AVAudioChannelCount, isUseAudioRenderer: Bool) -> AVAudioChannelCount { let maximumOutputNumberOfChannels = AVAudioChannelCount(AVAudioSession.sharedInstance().maximumOutputNumberOfChannels) let preferredOutputNumberOfChannels = AVAudioChannelCount(AVAudioSession.sharedInstance().preferredOutputNumberOfChannels) KSLog("[audio] maximumOutputNumberOfChannels: \(maximumOutputNumberOfChannels)") @@ -433,9 +443,9 @@ public extension KSOptions { let isSpatialAudioEnabled = isSpatialAudioEnabled() KSLog("[audio] isSpatialAudioEnabled: \(isSpatialAudioEnabled)") KSLog("[audio] isUseAudioRenderer: \(isUseAudioRenderer)") - var channels = channels - if channels > 2 { - let minChannels = min(maximumOutputNumberOfChannels, channels) + var channelCount = channelCount + if channelCount > 2 { + let minChannels = min(maximumOutputNumberOfChannels, channelCount) if minChannels > preferredOutputNumberOfChannels { try? AVAudioSession.sharedInstance().setPreferredOutputNumberOfChannels(Int(minChannels)) KSLog("[audio] set preferredOutputNumberOfChannels: \(minChannels)") @@ -445,13 +455,13 @@ public extension KSOptions { $0.channels?.count }.max() ?? 2 KSLog("[audio] currentRoute max channels: \(maxRouteChannelsCount)") - channels = AVAudioChannelCount(min(AVAudioSession.sharedInstance().outputNumberOfChannels, maxRouteChannelsCount)) + channelCount = AVAudioChannelCount(min(AVAudioSession.sharedInstance().outputNumberOfChannels, maxRouteChannelsCount)) } } else { - channels = 2 + channelCount = 2 } KSLog("[audio] outputNumberOfChannels: \(AVAudioSession.sharedInstance().outputNumberOfChannels)") - return channels + return channelCount } #endif } diff --git a/Sources/KSPlayer/AVPlayer/KSPlayerLayer.swift b/Sources/KSPlayer/AVPlayer/KSPlayerLayer.swift index b98662eaf..7c7df4fb3 100644 --- a/Sources/KSPlayer/AVPlayer/KSPlayerLayer.swift +++ b/Sources/KSPlayer/AVPlayer/KSPlayerLayer.swift @@ -341,7 +341,7 @@ extension KSPlayerLayer: MediaPlayerDelegate { } #endif for track in player.tracks(mediaType: .video) where track.isEnabled { - #if os(tvOS) + #if os(tvOS) || os(xrOS) setDisplayCriteria(track: track) #endif } @@ -442,7 +442,7 @@ extension KSPlayerLayer: AVPictureInPictureControllerDelegate { // MARK: - private functions extension KSPlayerLayer { - #if os(tvOS) + #if os(tvOS) || os(xrOS) private func setDisplayCriteria(track: some MediaPlayerTrack) { guard let displayManager = UIApplication.shared.windows.first?.avDisplayManager, displayManager.isDisplayCriteriaMatchingEnabled, @@ -450,9 +450,7 @@ extension KSPlayerLayer { else { return } - if let criteria = options.preferredDisplayCriteria(refreshRate: track.nominalFrameRate, - videoDynamicRange: track.dynamicRange(options).rawValue) - { + if let criteria = options.preferredDisplayCriteria(track: track) { displayManager.preferredDisplayCriteria = criteria } } diff --git a/Sources/KSPlayer/AVPlayer/MediaPlayerProtocol.swift b/Sources/KSPlayer/AVPlayer/MediaPlayerProtocol.swift index f810561da..2232b50ae 100644 --- a/Sources/KSPlayer/AVPlayer/MediaPlayerProtocol.swift +++ b/Sources/KSPlayer/AVPlayer/MediaPlayerProtocol.swift @@ -92,6 +92,7 @@ public protocol MediaPlayerTrack: AnyObject, CustomStringConvertible { var yCbCrMatrix: String? { get } var dovi: DOVIDecoderConfigurationRecord? { get } var fieldOrder: FFmpegFieldOrder { get } + var formatDescription: CMFormatDescription? { get } } // public extension MediaPlayerTrack: Identifiable { diff --git a/Sources/KSPlayer/MEPlayer/FFmpegAssetTrack.swift b/Sources/KSPlayer/MEPlayer/FFmpegAssetTrack.swift index 0d1c074b0..63e5a22c9 100644 --- a/Sources/KSPlayer/MEPlayer/FFmpegAssetTrack.swift +++ b/Sources/KSPlayer/MEPlayer/FFmpegAssetTrack.swift @@ -6,8 +6,8 @@ // import AVFoundation +import FFmpegKit import Libavformat - public class FFmpegAssetTrack: MediaPlayerTrack { public private(set) var trackID: Int32 = 0 public var name: String = "" @@ -39,8 +39,9 @@ public class FFmpegAssetTrack: MediaPlayerTrack { public let yCbCrMatrix: String? public var dovi: DOVIDecoderConfigurationRecord? public let fieldOrder: FFmpegFieldOrder + public let formatDescription: CMFormatDescription? var closedCaptionsTrack: FFmpegAssetTrack? - + let isConvertNALSize: Bool convenience init?(stream: UnsafeMutablePointer) { let codecpar = stream.pointee.codecpar.pointee self.init(codecpar: codecpar) @@ -112,8 +113,9 @@ public class FFmpegAssetTrack: MediaPlayerTrack { colorPrimaries = codecpar.color_primaries.colorPrimaries as String? transferFunction = codecpar.color_trc.transferFunction as String? yCbCrMatrix = codecpar.color_space.ycbcrMatrix as String? + let codecType = codecpar.codec_id.mediaSubType // codec_tag byte order is LSB first - mediaSubType = codecpar.codec_tag == 0 ? codecpar.codec_id.mediaSubType : CMFormatDescription.MediaSubType(rawValue: codecpar.codec_tag.bigEndian) + mediaSubType = codecpar.codec_tag == 0 ? codecType : CMFormatDescription.MediaSubType(rawValue: codecpar.codec_tag.bigEndian) var description = "" if let descriptor = avcodec_descriptor_get(codecpar.codec_id) { description += String(cString: descriptor.pointee.name) @@ -121,18 +123,19 @@ public class FFmpegAssetTrack: MediaPlayerTrack { description += " (\(String(cString: profile.pointee.name)))" } } - let sar = codecpar.sample_aspect_ratio.size - naturalSize = CGSize(width: Int(codecpar.width), height: Int(CGFloat(codecpar.height) * sar.height / sar.width)) fieldOrder = FFmpegFieldOrder(rawValue: UInt8(codecpar.field_order.rawValue)) ?? .unknown if codecpar.codec_type == AVMEDIA_TYPE_AUDIO { mediaType = .audio + naturalSize = .zero audioDescriptor = AudioDescriptor(codecpar: codecpar) + formatDescription = nil + isConvertNALSize = false let layout = codecpar.ch_layout let channelsPerFrame = UInt32(layout.nb_channels) let sampleFormat = AVSampleFormat(codecpar.format) let bytesPerSample = UInt32(av_get_bytes_per_sample(sampleFormat)) let formatFlags = ((sampleFormat == AV_SAMPLE_FMT_FLT || sampleFormat == AV_SAMPLE_FMT_DBL) ? kAudioFormatFlagIsFloat : sampleFormat == AV_SAMPLE_FMT_U8 ? 0 : kAudioFormatFlagIsSignedInteger) | kAudioFormatFlagIsPacked - audioStreamBasicDescription = AudioStreamBasicDescription(mSampleRate: Float64(codecpar.sample_rate), mFormatID: codecpar.codec_id.mediaSubType.rawValue, mFormatFlags: formatFlags, mBytesPerPacket: bytesPerSample * channelsPerFrame, mFramesPerPacket: 1, mBytesPerFrame: bytesPerSample * channelsPerFrame, mChannelsPerFrame: channelsPerFrame, mBitsPerChannel: bytesPerSample * 8, mReserved: 0) + audioStreamBasicDescription = AudioStreamBasicDescription(mSampleRate: Float64(codecpar.sample_rate), mFormatID: codecType.rawValue, mFormatFlags: formatFlags, mBytesPerPacket: bytesPerSample * channelsPerFrame, mFramesPerPacket: 1, mBytesPerFrame: bytesPerSample * channelsPerFrame, mChannelsPerFrame: channelsPerFrame, mBitsPerChannel: bytesPerSample * 8, mReserved: 0) description += ", \(codecpar.sample_rate)Hz" description += ", \(codecpar.ch_layout.description)" if let name = av_get_sample_fmt_name(sampleFormat) { @@ -144,6 +147,64 @@ public class FFmpegAssetTrack: MediaPlayerTrack { audioDescriptor = nil mediaType = .video audioStreamBasicDescription = nil + let sar = codecpar.sample_aspect_ratio.size + naturalSize = CGSize(width: Int(codecpar.width), height: Int(CGFloat(codecpar.height) * sar.height / sar.width)) + var extradataSize = Int32(0) + var extradata = codecpar.extradata + let atomsData: Data? + if let extradata { + extradataSize = codecpar.extradata_size + if extradataSize >= 5, extradata[4] == 0xFE { + extradata[4] = 0xFF + isConvertNALSize = true + } else { + isConvertNALSize = false + } + atomsData = Data(bytes: extradata, count: Int(extradataSize)) + } else { + if codecType.rawValue == kCMVideoCodecType_VP9 { + // ff_videotoolbox_vpcc_extradata_create + var ioContext: UnsafeMutablePointer? + guard avio_open_dyn_buf(&ioContext) == 0 else { + return nil + } + ff_isom_write_vpcc(nil, ioContext, nil, 0, &self.codecpar) + extradataSize = avio_close_dyn_buf(ioContext, &extradata) + guard let extradata else { + return nil + } + var data = Data() + var array: [UInt8] = [1, 0, 0, 0] + data.append(&array, count: 4) + data.append(extradata, count: Int(extradataSize)) + atomsData = data + } else { + atomsData = nil + } + isConvertNALSize = false + } + let dic: NSMutableDictionary = [ + kCVImageBufferChromaLocationBottomFieldKey: kCVImageBufferChromaLocation_Left, + kCVImageBufferChromaLocationTopFieldKey: kCVImageBufferChromaLocation_Left, + kCMFormatDescriptionExtension_FullRangeVideo: fullRangeVideo, + codecType.rawValue == kCMVideoCodecType_HEVC ? "EnableHardwareAcceleratedVideoDecoder" : "RequireHardwareAcceleratedVideoDecoder": true, + ] + if let atomsData { + dic[kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms] = [codecType.rawValue.avc: atomsData] + } + dic[kCVImageBufferPixelAspectRatioKey] = sar.aspectRatio + dic[kCVImageBufferColorPrimariesKey] = colorPrimaries + dic[kCVImageBufferTransferFunctionKey] = transferFunction + dic[kCVImageBufferYCbCrMatrixKey] = yCbCrMatrix + // swiftlint:disable line_length + var formatDescriptionOut: CMFormatDescription? + let status = CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault, codecType: codecType.rawValue, width: codecpar.width, height: codecpar.height, extensions: dic, formatDescriptionOut: &formatDescriptionOut) + // swiftlint:enable line_length + if status == noErr { + formatDescription = formatDescriptionOut + } else { + formatDescription = nil + } if let name = av_get_pix_fmt_name(format) { formatName = String(cString: name) } else { @@ -155,6 +216,9 @@ public class FFmpegAssetTrack: MediaPlayerTrack { audioStreamBasicDescription = nil audioDescriptor = nil formatName = nil + formatDescription = nil + isConvertNALSize = false + naturalSize = .zero } else { return nil } diff --git a/Sources/KSPlayer/MEPlayer/MEPlayerItemTrack.swift b/Sources/KSPlayer/MEPlayer/MEPlayerItemTrack.swift index e505062b7..a08b76455 100644 --- a/Sources/KSPlayer/MEPlayer/MEPlayerItemTrack.swift +++ b/Sources/KSPlayer/MEPlayer/MEPlayerItemTrack.swift @@ -52,7 +52,7 @@ class SyncPlayerItemTrack: PlayerItemTrackProtocol, CustomString fps = assetTrack.nominalFrameRate // 默认缓存队列大小跟帧率挂钩,经测试除以4,最优 if mediaType == .audio { - let capacity = options.audioFrameMaxCount(fps: fps, channels: Int(assetTrack.audioDescriptor?.channels ?? 2)) + let capacity = options.audioFrameMaxCount(fps: fps, channelCount: Int(assetTrack.audioDescriptor?.channelCount ?? 2)) outputRenderQueue = CircularBuffer(initialCapacity: capacity, expanding: false) } else if mediaType == .video { outputRenderQueue = CircularBuffer(initialCapacity: options.videoFrameMaxCount(fps: fps, naturalSize: assetTrack.naturalSize), sorted: true, expanding: false) @@ -279,7 +279,7 @@ extension SyncPlayerItemTrack { return SubtitleDecode(assetTrack: assetTrack, options: options) } else { if mediaType == .video, options.asynchronousDecompression, options.hardwareDecode, - let session = DecompressionSession(codecpar: assetTrack.codecpar, options: options) + let session = DecompressionSession(assetTrack: assetTrack, options: options) { return VideoToolboxDecode(assetTrack: assetTrack, options: options, session: session) } else { diff --git a/Sources/KSPlayer/MEPlayer/MetalPlayView.swift b/Sources/KSPlayer/MEPlayer/MetalPlayView.swift index 62546345e..8424c7485 100644 --- a/Sources/KSPlayer/MEPlayer/MetalPlayView.swift +++ b/Sources/KSPlayer/MEPlayer/MetalPlayView.swift @@ -17,7 +17,17 @@ public protocol DisplayLayerDelegate: NSObjectProtocol { } public final class MetalPlayView: UIView { - private var videoInfo: CMVideoFormatDescription? + private var formatDescription: CMFormatDescription? { + didSet { + #if os(tvOS) || os(xrOS) + if let formatDescription { + setDisplayCriteria(formatDescription: formatDescription) + } + #endif + } + } + + private var fps = Float(60) public private(set) var pixelBuffer: CVPixelBuffer? /// 用displayLink会导致锁屏无法draw, /// 用DispatchSourceTimer的话,在播放4k视频的时候repeat的时间会变长, @@ -49,6 +59,7 @@ public final class MetalPlayView: UIView { } func prepare(fps: Float, startPlayTime: TimeInterval = 0) { + self.fps = fps let preferredFramesPerSecond = Int(ceil(fps)) displayLink.preferredFramesPerSecond = preferredFramesPerSecond << 1 #if os(iOS) @@ -157,6 +168,17 @@ extension MetalPlayView { let cmtime = frame.cmtime let par = pixelBuffer.size let sar = pixelBuffer.aspectRatio + if formatDescription == nil || !CMVideoFormatDescriptionMatchesImageBuffer(formatDescription!, imageBuffer: pixelBuffer) { + if formatDescription != nil { + displayView.removeFromSuperview() + displayView = AVSampleBufferDisplayView() + addSubview(displayView) + } + let err = CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: pixelBuffer, formatDescriptionOut: &formatDescription) + if err != noErr { + KSLog("Error at CMVideoFormatDescriptionCreateForImageBuffer \(err)") + } + } if options.isUseDisplayLayer() { if displayView.isHidden { displayView.isHidden = false @@ -190,20 +212,24 @@ extension MetalPlayView { } private func set(pixelBuffer: CVPixelBuffer, time: CMTime) { - if videoInfo == nil || !CMVideoFormatDescriptionMatchesImageBuffer(videoInfo!, imageBuffer: pixelBuffer) { - if videoInfo != nil { - displayView.removeFromSuperview() - displayView = AVSampleBufferDisplayView() - addSubview(displayView) - } - let err = CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: pixelBuffer, formatDescriptionOut: &videoInfo) - if err != noErr { - KSLog("Error at CMVideoFormatDescriptionCreateForImageBuffer \(err)") - } + guard let formatDescription else { return } + displayView.enqueue(imageBuffer: pixelBuffer, formatDescription: formatDescription, time: time) + } + + #if os(tvOS) || os(xrOS) + private func setDisplayCriteria(formatDescription: CMFormatDescription) { + guard let displayManager = UIApplication.shared.windows.first?.avDisplayManager, + displayManager.isDisplayCriteriaMatchingEnabled, + !displayManager.isDisplayModeSwitchInProgress + else { + return + } + if #available(tvOS 17.0, *) { + let criteria = AVDisplayCriteria(refreshRate: fps, formatDescription: formatDescription) + displayManager.preferredDisplayCriteria = criteria } - guard let videoInfo else { return } - displayView.enqueue(imageBuffer: pixelBuffer, formatDescription: videoInfo, time: time) } + #endif } class MetalView: UIView { diff --git a/Sources/KSPlayer/MEPlayer/Resample.swift b/Sources/KSPlayer/MEPlayer/Resample.swift index e2c149653..f5ba87e0a 100644 --- a/Sources/KSPlayer/MEPlayer/Resample.swift +++ b/Sources/KSPlayer/MEPlayer/Resample.swift @@ -232,7 +232,7 @@ public class AudioDescriptor: Equatable { fileprivate var channel: AVChannelLayout fileprivate var outChannel: AVChannelLayout var audioFormat: AVAudioFormat - public var channels: AVAudioChannelCount { + public var channelCount: AVAudioChannelCount { AVAudioChannelCount(channel.nb_channels) } @@ -274,15 +274,15 @@ public class AudioDescriptor: Equatable { lhs.sampleFormat == rhs.sampleFormat && lhs.sampleRate == rhs.sampleRate && lhs.channel == rhs.channel } - private func audioFormat(channels: AVAudioChannelCount, isUseAudioRenderer: Bool) { - if channels != self.channels { - av_channel_layout_default(&outChannel, Int32(channels)) + private func audioFormat(channelCount: AVAudioChannelCount, isUseAudioRenderer: Bool) { + if channelCount != self.channelCount { + av_channel_layout_default(&outChannel, Int32(channelCount)) } let layoutTag: AudioChannelLayoutTag if let tag = outChannel.layoutTag { layoutTag = tag } else { - av_channel_layout_default(&outChannel, Int32(channels)) + av_channel_layout_default(&outChannel, Int32(channelCount)) if let tag = outChannel.layoutTag { layoutTag = tag } else { @@ -332,10 +332,10 @@ public class AudioDescriptor: Equatable { public func setAudioSession(isUseAudioRenderer: Bool) { #if os(macOS) - let channels = AVAudioChannelCount(2) + let channelCount = AVAudioChannelCount(2) #else - let channels = KSOptions.outputNumberOfChannels(channels: channels, isUseAudioRenderer: isUseAudioRenderer) + let channelCount = KSOptions.outputNumberOfChannels(channelCount: channelCount, isUseAudioRenderer: isUseAudioRenderer) #endif - audioFormat(channels: channels, isUseAudioRenderer: isUseAudioRenderer) + audioFormat(channelCount: channelCount, isUseAudioRenderer: isUseAudioRenderer) } } diff --git a/Sources/KSPlayer/MEPlayer/VideoToolboxDecode.swift b/Sources/KSPlayer/MEPlayer/VideoToolboxDecode.swift index bcad27838..9da020e7e 100644 --- a/Sources/KSPlayer/MEPlayer/VideoToolboxDecode.swift +++ b/Sources/KSPlayer/MEPlayer/VideoToolboxDecode.swift @@ -16,7 +16,7 @@ class VideoToolboxDecode: DecodeProtocol { private var startTime = Int64(0) private var lastPosition = Int64(0) required convenience init(assetTrack: FFmpegAssetTrack, options: KSOptions) { - self.init(assetTrack: assetTrack, options: options, session: DecompressionSession(codecpar: assetTrack.codecpar, options: options)) + self.init(assetTrack: assetTrack, options: options, session: DecompressionSession(assetTrack: assetTrack, options: options)) } init(assetTrack: FFmpegAssetTrack, options: KSOptions, session: DecompressionSession?) { @@ -30,7 +30,7 @@ class VideoToolboxDecode: DecodeProtocol { return } do { - let sampleBuffer = try session.formatDescription.getSampleBuffer(isConvertNALSize: session.isConvertNALSize, data: data, size: Int(corePacket.size)) + let sampleBuffer = try session.formatDescription.getSampleBuffer(isConvertNALSize: session.assetTrack.isConvertNALSize, data: data, size: Int(corePacket.size)) let flags: VTDecodeFrameFlags = [ ._EnableAsynchronousDecompression, ] @@ -69,7 +69,7 @@ class VideoToolboxDecode: DecodeProtocol { throw NSError(errorCode: .codecVideoReceiveFrame, avErrorCode: status) } else { // 解决从后台切换到前台,解码失败的问题 - self.session = DecompressionSession(codecpar: session.codecpar, options: options) + self.session = DecompressionSession(assetTrack: session.assetTrack, options: options) doFlushCodec() } } @@ -94,90 +94,33 @@ class VideoToolboxDecode: DecodeProtocol { } class DecompressionSession { - fileprivate let isConvertNALSize: Bool fileprivate let formatDescription: CMFormatDescription fileprivate let decompressionSession: VTDecompressionSession - fileprivate var codecpar: AVCodecParameters - init?(codecpar: AVCodecParameters, options: KSOptions) { - self.codecpar = codecpar - let isFullRangeVideo = codecpar.color_range == AVCOL_RANGE_JPEG - let format = AVPixelFormat(codecpar.format) - guard let pixelFormatType = format.osType(fullRange: isFullRangeVideo) else { + fileprivate var assetTrack: FFmpegAssetTrack + init?(assetTrack: FFmpegAssetTrack, options: KSOptions) { + self.assetTrack = assetTrack + let format = AVPixelFormat(assetTrack.codecpar.format) + guard let pixelFormatType = format.osType(fullRange: assetTrack.fullRangeVideo), let formatDescription = assetTrack.formatDescription else { return nil } - let videoCodecType = codecpar.codec_id.mediaSubType.rawValue + self.formatDescription = formatDescription #if os(macOS) VTRegisterProfessionalVideoWorkflowVideoDecoders() if #available(macOS 11.0, *) { - VTRegisterSupplementalVideoDecoderIfAvailable(videoCodecType) + VTRegisterSupplementalVideoDecoderIfAvailable(formatDescription.mediaSubType.rawValue) } #endif - var extradataSize = Int32(0) - var extradata = codecpar.extradata - let atomsData: Data? - if let extradata { - extradataSize = codecpar.extradata_size - if extradataSize >= 5, extradata[4] == 0xFE { - extradata[4] = 0xFF - isConvertNALSize = true - } else { - isConvertNALSize = false - } - atomsData = Data(bytes: extradata, count: Int(extradataSize)) - } else { - if videoCodecType == kCMVideoCodecType_VP9 { - // ff_videotoolbox_vpcc_extradata_create - var ioContext: UnsafeMutablePointer? - guard avio_open_dyn_buf(&ioContext) == 0 else { - return nil - } - ff_isom_write_vpcc(nil, ioContext, nil, 0, &self.codecpar) - extradataSize = avio_close_dyn_buf(ioContext, &extradata) - guard let extradata else { - return nil - } - var data = Data() - var array: [UInt8] = [1, 0, 0, 0] - data.append(&array, count: 4) - data.append(extradata, count: Int(extradataSize)) - atomsData = data - } else { - atomsData = nil - } - isConvertNALSize = false - } - let dic: NSMutableDictionary = [ - kCVImageBufferChromaLocationBottomFieldKey: kCVImageBufferChromaLocation_Left, - kCVImageBufferChromaLocationTopFieldKey: kCVImageBufferChromaLocation_Left, - kCMFormatDescriptionExtension_FullRangeVideo: isFullRangeVideo, - videoCodecType == kCMVideoCodecType_HEVC ? "EnableHardwareAcceleratedVideoDecoder" : "RequireHardwareAcceleratedVideoDecoder": true, - ] - if let atomsData { - dic[kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms] = [videoCodecType.avc: atomsData] - } - dic[kCVImageBufferPixelAspectRatioKey] = codecpar.sample_aspect_ratio.size.aspectRatio - dic[kCVImageBufferColorPrimariesKey] = codecpar.color_primaries.colorPrimaries - dic[kCVImageBufferTransferFunctionKey] = codecpar.color_trc.transferFunction - dic[kCVImageBufferYCbCrMatrixKey] = codecpar.color_space.ycbcrMatrix - // swiftlint:disable line_length - var description: CMFormatDescription? - var status = CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault, codecType: videoCodecType, width: codecpar.width, height: codecpar.height, extensions: dic, formatDescriptionOut: &description) - // swiftlint:enable line_length - guard status == noErr, let formatDescription = description else { - return nil - } - self.formatDescription = formatDescription // VTDecompressionSessionCanAcceptFormatDescription(<#T##session: VTDecompressionSession##VTDecompressionSession#>, formatDescription: <#T##CMFormatDescription#>) let attributes: NSMutableDictionary = [ kCVPixelBufferPixelFormatTypeKey: pixelFormatType, kCVPixelBufferMetalCompatibilityKey: true, - kCVPixelBufferWidthKey: codecpar.width, - kCVPixelBufferHeightKey: codecpar.height, + kCVPixelBufferWidthKey: assetTrack.codecpar.width, + kCVPixelBufferHeightKey: assetTrack.codecpar.height, kCVPixelBufferIOSurfacePropertiesKey: NSDictionary(), ] var session: VTDecompressionSession? // swiftlint:disable line_length - status = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: formatDescription, decoderSpecification: dic, imageBufferAttributes: attributes, outputCallback: nil, decompressionSessionOut: &session) + let status = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: formatDescription, decoderSpecification: nil, imageBufferAttributes: attributes, outputCallback: nil, decompressionSessionOut: &session) // swiftlint:enable line_length guard status == noErr, let decompressionSession = session else { return nil