fix #563

Demo/SwiftUI/Shared/MovieModel.swift

@@ -29,9 +29,19 @@ class MEOptions: KSOptions {
         MEOptions.isUseDisplayLayer && display == .plane
     }
 
-    #if os(tvOS)
-    override open func preferredDisplayCriteria(refreshRate: Float, videoDynamicRange: Int32) -> AVDisplayCriteria? {
-        AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange)
+    #if os(tvOS) || os(xrOS)
+    override open func preferredDisplayCriteria(track: some MediaPlayerTrack) -> AVDisplayCriteria? {
+        let refreshRate = track.nominalFrameRate
+        if #available(tvOS 17.0, *) {
+            if let formatDescription = track.formatDescription {
+                return AVDisplayCriteria(refreshRate: refreshRate, formatDescription: formatDescription)
+            } else {
+                return nil
+            }
+        } else {
+            let videoDynamicRange = track.dynamicRange(self).rawValue
+            return AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange)
+        }
     }
     #endif
 }

Demo/demo-iOS/demo-iOS/AppDelegate.swift

@@ -104,9 +104,19 @@ class MEOptions: KSOptions {
         }
     }
 
-    #if os(tvOS)
-    override open func preferredDisplayCriteria(refreshRate: Float, videoDynamicRange: Int32) -> AVDisplayCriteria? {
-        AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange)
+    #if os(tvOS) || os(xrOS)
+    override open func preferredDisplayCriteria(track: some MediaPlayerTrack) -> AVDisplayCriteria? {
+        let refreshRate = track.nominalFrameRate
+        if #available(tvOS 17.0, *) {
+            if let formatDescription = track.formatDescription {
+                return AVDisplayCriteria(refreshRate: refreshRate, formatDescription: formatDescription)
+            } else {
+                return nil
+            }
+        } else {
+            let videoDynamicRange = track.dynamicRange(self).rawValue
+            return AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange)
+        }
     }
     #endif
 }

Sources/KSPlayer/AVPlayer/KSAVPlayer.swift

@@ -509,6 +509,7 @@ extension AVAssetTrack {
 }
 
 class AVMediaPlayerTrack: MediaPlayerTrack {
+    let formatDescription: CMFormatDescription? = nil
     let description: String
     private let track: AVPlayerItemTrack
     let nominalFrameRate: Float

Sources/KSPlayer/AVPlayer/KSOptions.swift

@@ -226,7 +226,7 @@ open class KSOptions {
         16
     }
 
-    open func audioFrameMaxCount(fps: Float, channels _: Int) -> Int {
+    open func audioFrameMaxCount(fps: Float, channelCount _: Int) -> Int {
         Int(fps) >> 2
     }
 
@@ -303,10 +303,20 @@ open class KSOptions {
      */
     open func process(assetTrack _: some MediaPlayerTrack) {}
 
-    #if os(tvOS)
-    open func preferredDisplayCriteria(refreshRate _: Float, videoDynamicRange _: Int32) -> AVDisplayCriteria? {
-//        AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange)
-        nil
+    #if os(tvOS) || os(xrOS)
+    open func preferredDisplayCriteria(track: some MediaPlayerTrack) -> AVDisplayCriteria? {
+        let refreshRate = track.nominalFrameRate
+        if #available(tvOS 17.0, *) {
+            if let formatDescription = track.formatDescription {
+                return AVDisplayCriteria(refreshRate: refreshRate, formatDescription: formatDescription)
+            } else {
+                return nil
+            }
+        } else {
+//            let videoDynamicRange = track.dynamicRange(self).rawValue
+//            return AVDisplayCriteria(refreshRate: refreshRate, videoDynamicRange: videoDynamicRange)
+            return nil
+        }
     }
     #endif
 
@@ -424,7 +434,7 @@ public extension KSOptions {
         }
     }
 
-    static func outputNumberOfChannels(channels: AVAudioChannelCount, isUseAudioRenderer: Bool) -> AVAudioChannelCount {
+    static func outputNumberOfChannels(channelCount: AVAudioChannelCount, isUseAudioRenderer: Bool) -> AVAudioChannelCount {
         let maximumOutputNumberOfChannels = AVAudioChannelCount(AVAudioSession.sharedInstance().maximumOutputNumberOfChannels)
         let preferredOutputNumberOfChannels = AVAudioChannelCount(AVAudioSession.sharedInstance().preferredOutputNumberOfChannels)
         KSLog("[audio] maximumOutputNumberOfChannels: \(maximumOutputNumberOfChannels)")
@@ -433,9 +443,9 @@ public extension KSOptions {
         let isSpatialAudioEnabled = isSpatialAudioEnabled()
         KSLog("[audio] isSpatialAudioEnabled: \(isSpatialAudioEnabled)")
         KSLog("[audio] isUseAudioRenderer: \(isUseAudioRenderer)")
-        var channels = channels
-        if channels > 2 {
-            let minChannels = min(maximumOutputNumberOfChannels, channels)
+        var channelCount = channelCount
+        if channelCount > 2 {
+            let minChannels = min(maximumOutputNumberOfChannels, channelCount)
             if minChannels > preferredOutputNumberOfChannels {
                 try? AVAudioSession.sharedInstance().setPreferredOutputNumberOfChannels(Int(minChannels))
                 KSLog("[audio] set preferredOutputNumberOfChannels: \(minChannels)")
@@ -445,13 +455,13 @@ public extension KSOptions {
                     $0.channels?.count
                 }.max() ?? 2
                 KSLog("[audio] currentRoute max channels: \(maxRouteChannelsCount)")
-                channels = AVAudioChannelCount(min(AVAudioSession.sharedInstance().outputNumberOfChannels, maxRouteChannelsCount))
+                channelCount = AVAudioChannelCount(min(AVAudioSession.sharedInstance().outputNumberOfChannels, maxRouteChannelsCount))
             }
         } else {
-            channels = 2
+            channelCount = 2
         }
         KSLog("[audio] outputNumberOfChannels: \(AVAudioSession.sharedInstance().outputNumberOfChannels)")
-        return channels
+        return channelCount
     }
     #endif
 }

Sources/KSPlayer/AVPlayer/KSPlayerLayer.swift

@@ -341,7 +341,7 @@ extension KSPlayerLayer: MediaPlayerDelegate {
         }
         #endif
         for track in player.tracks(mediaType: .video) where track.isEnabled {
-            #if os(tvOS)
+            #if os(tvOS) || os(xrOS)
             setDisplayCriteria(track: track)
             #endif
         }
@@ -442,17 +442,15 @@ extension KSPlayerLayer: AVPictureInPictureControllerDelegate {
 // MARK: - private functions
 
 extension KSPlayerLayer {
-    #if os(tvOS)
+    #if os(tvOS) || os(xrOS)
     private func setDisplayCriteria(track: some MediaPlayerTrack) {
         guard let displayManager = UIApplication.shared.windows.first?.avDisplayManager,
               displayManager.isDisplayCriteriaMatchingEnabled,
               !displayManager.isDisplayModeSwitchInProgress
         else {
             return
         }
-        if let criteria = options.preferredDisplayCriteria(refreshRate: track.nominalFrameRate,
-                                                           videoDynamicRange: track.dynamicRange(options).rawValue)
-        {
+        if let criteria = options.preferredDisplayCriteria(track: track) {
             displayManager.preferredDisplayCriteria = criteria
         }
     }

Sources/KSPlayer/AVPlayer/MediaPlayerProtocol.swift

@@ -92,6 +92,7 @@ public protocol MediaPlayerTrack: AnyObject, CustomStringConvertible {
     var yCbCrMatrix: String? { get }
     var dovi: DOVIDecoderConfigurationRecord? { get }
     var fieldOrder: FFmpegFieldOrder { get }
+    var formatDescription: CMFormatDescription? { get }
 }
 
 // public extension MediaPlayerTrack: Identifiable {

Sources/KSPlayer/MEPlayer/FFmpegAssetTrack.swift

@@ -6,8 +6,8 @@
 //
 
 import AVFoundation
+import FFmpegKit
 import Libavformat
-
 public class FFmpegAssetTrack: MediaPlayerTrack {
     public private(set) var trackID: Int32 = 0
     public var name: String = ""
@@ -39,8 +39,9 @@ public class FFmpegAssetTrack: MediaPlayerTrack {
     public let yCbCrMatrix: String?
     public var dovi: DOVIDecoderConfigurationRecord?
     public let fieldOrder: FFmpegFieldOrder
+    public let formatDescription: CMFormatDescription?
     var closedCaptionsTrack: FFmpegAssetTrack?
-
+    let isConvertNALSize: Bool
     convenience init?(stream: UnsafeMutablePointer<AVStream>) {
         let codecpar = stream.pointee.codecpar.pointee
         self.init(codecpar: codecpar)
@@ -112,27 +113,29 @@ public class FFmpegAssetTrack: MediaPlayerTrack {
         colorPrimaries = codecpar.color_primaries.colorPrimaries as String?
         transferFunction = codecpar.color_trc.transferFunction as String?
         yCbCrMatrix = codecpar.color_space.ycbcrMatrix as String?
+        let codecType = codecpar.codec_id.mediaSubType
         // codec_tag byte order is LSB first
-        mediaSubType = codecpar.codec_tag == 0 ? codecpar.codec_id.mediaSubType : CMFormatDescription.MediaSubType(rawValue: codecpar.codec_tag.bigEndian)
+        mediaSubType = codecpar.codec_tag == 0 ? codecType : CMFormatDescription.MediaSubType(rawValue: codecpar.codec_tag.bigEndian)
         var description = ""
         if let descriptor = avcodec_descriptor_get(codecpar.codec_id) {
             description += String(cString: descriptor.pointee.name)
             if let profile = descriptor.pointee.profiles {
                 description += " (\(String(cString: profile.pointee.name)))"
             }
         }
-        let sar = codecpar.sample_aspect_ratio.size
-        naturalSize = CGSize(width: Int(codecpar.width), height: Int(CGFloat(codecpar.height) * sar.height / sar.width))
         fieldOrder = FFmpegFieldOrder(rawValue: UInt8(codecpar.field_order.rawValue)) ?? .unknown
         if codecpar.codec_type == AVMEDIA_TYPE_AUDIO {
             mediaType = .audio
+            naturalSize = .zero
             audioDescriptor = AudioDescriptor(codecpar: codecpar)
+            formatDescription = nil
+            isConvertNALSize = false
             let layout = codecpar.ch_layout
             let channelsPerFrame = UInt32(layout.nb_channels)
             let sampleFormat = AVSampleFormat(codecpar.format)
             let bytesPerSample = UInt32(av_get_bytes_per_sample(sampleFormat))
             let formatFlags = ((sampleFormat == AV_SAMPLE_FMT_FLT || sampleFormat == AV_SAMPLE_FMT_DBL) ? kAudioFormatFlagIsFloat : sampleFormat == AV_SAMPLE_FMT_U8 ? 0 : kAudioFormatFlagIsSignedInteger) | kAudioFormatFlagIsPacked
-            audioStreamBasicDescription = AudioStreamBasicDescription(mSampleRate: Float64(codecpar.sample_rate), mFormatID: codecpar.codec_id.mediaSubType.rawValue, mFormatFlags: formatFlags, mBytesPerPacket: bytesPerSample * channelsPerFrame, mFramesPerPacket: 1, mBytesPerFrame: bytesPerSample * channelsPerFrame, mChannelsPerFrame: channelsPerFrame, mBitsPerChannel: bytesPerSample * 8, mReserved: 0)
+            audioStreamBasicDescription = AudioStreamBasicDescription(mSampleRate: Float64(codecpar.sample_rate), mFormatID: codecType.rawValue, mFormatFlags: formatFlags, mBytesPerPacket: bytesPerSample * channelsPerFrame, mFramesPerPacket: 1, mBytesPerFrame: bytesPerSample * channelsPerFrame, mChannelsPerFrame: channelsPerFrame, mBitsPerChannel: bytesPerSample * 8, mReserved: 0)
             description += ", \(codecpar.sample_rate)Hz"
             description += ", \(codecpar.ch_layout.description)"
             if let name = av_get_sample_fmt_name(sampleFormat) {
@@ -144,6 +147,64 @@ public class FFmpegAssetTrack: MediaPlayerTrack {
             audioDescriptor = nil
             mediaType = .video
             audioStreamBasicDescription = nil
+            let sar = codecpar.sample_aspect_ratio.size
+            naturalSize = CGSize(width: Int(codecpar.width), height: Int(CGFloat(codecpar.height) * sar.height / sar.width))
+            var extradataSize = Int32(0)
+            var extradata = codecpar.extradata
+            let atomsData: Data?
+            if let extradata {
+                extradataSize = codecpar.extradata_size
+                if extradataSize >= 5, extradata[4] == 0xFE {
+                    extradata[4] = 0xFF
+                    isConvertNALSize = true
+                } else {
+                    isConvertNALSize = false
+                }
+                atomsData = Data(bytes: extradata, count: Int(extradataSize))
+            } else {
+                if codecType.rawValue == kCMVideoCodecType_VP9 {
+                    // ff_videotoolbox_vpcc_extradata_create
+                    var ioContext: UnsafeMutablePointer<AVIOContext>?
+                    guard avio_open_dyn_buf(&ioContext) == 0 else {
+                        return nil
+                    }
+                    ff_isom_write_vpcc(nil, ioContext, nil, 0, &self.codecpar)
+                    extradataSize = avio_close_dyn_buf(ioContext, &extradata)
+                    guard let extradata else {
+                        return nil
+                    }
+                    var data = Data()
+                    var array: [UInt8] = [1, 0, 0, 0]
+                    data.append(&array, count: 4)
+                    data.append(extradata, count: Int(extradataSize))
+                    atomsData = data
+                } else {
+                    atomsData = nil
+                }
+                isConvertNALSize = false
+            }
+            let dic: NSMutableDictionary = [
+                kCVImageBufferChromaLocationBottomFieldKey: kCVImageBufferChromaLocation_Left,
+                kCVImageBufferChromaLocationTopFieldKey: kCVImageBufferChromaLocation_Left,
+                kCMFormatDescriptionExtension_FullRangeVideo: fullRangeVideo,
+                codecType.rawValue == kCMVideoCodecType_HEVC ? "EnableHardwareAcceleratedVideoDecoder" : "RequireHardwareAcceleratedVideoDecoder": true,
+            ]
+            if let atomsData {
+                dic[kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms] = [codecType.rawValue.avc: atomsData]
+            }
+            dic[kCVImageBufferPixelAspectRatioKey] = sar.aspectRatio
+            dic[kCVImageBufferColorPrimariesKey] = colorPrimaries
+            dic[kCVImageBufferTransferFunctionKey] = transferFunction
+            dic[kCVImageBufferYCbCrMatrixKey] = yCbCrMatrix
+            // swiftlint:disable line_length
+            var formatDescriptionOut: CMFormatDescription?
+            let status = CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault, codecType: codecType.rawValue, width: codecpar.width, height: codecpar.height, extensions: dic, formatDescriptionOut: &formatDescriptionOut)
+            // swiftlint:enable line_length
+            if status == noErr {
+                formatDescription = formatDescriptionOut
+            } else {
+                formatDescription = nil
+            }
             if let name = av_get_pix_fmt_name(format) {
                 formatName = String(cString: name)
             } else {
@@ -155,6 +216,9 @@ public class FFmpegAssetTrack: MediaPlayerTrack {
             audioStreamBasicDescription = nil
             audioDescriptor = nil
             formatName = nil
+            formatDescription = nil
+            isConvertNALSize = false
+            naturalSize = .zero
         } else {
             return nil
         }

Sources/KSPlayer/MEPlayer/MEPlayerItemTrack.swift

@@ -52,7 +52,7 @@ class SyncPlayerItemTrack<Frame: MEFrame>: PlayerItemTrackProtocol, CustomString
         fps = assetTrack.nominalFrameRate
         // 默认缓存队列大小跟帧率挂钩,经测试除以4，最优
         if mediaType == .audio {
-            let capacity = options.audioFrameMaxCount(fps: fps, channels: Int(assetTrack.audioDescriptor?.channels ?? 2))
+            let capacity = options.audioFrameMaxCount(fps: fps, channelCount: Int(assetTrack.audioDescriptor?.channelCount ?? 2))
             outputRenderQueue = CircularBuffer(initialCapacity: capacity, expanding: false)
         } else if mediaType == .video {
             outputRenderQueue = CircularBuffer(initialCapacity: options.videoFrameMaxCount(fps: fps, naturalSize: assetTrack.naturalSize), sorted: true, expanding: false)
@@ -279,7 +279,7 @@ extension SyncPlayerItemTrack {
                 return SubtitleDecode(assetTrack: assetTrack, options: options)
             } else {
                 if mediaType == .video, options.asynchronousDecompression, options.hardwareDecode,
-                   let session = DecompressionSession(codecpar: assetTrack.codecpar, options: options)
+                   let session = DecompressionSession(assetTrack: assetTrack, options: options)
                 {
                     return VideoToolboxDecode(assetTrack: assetTrack, options: options, session: session)
                 } else {

Sources/KSPlayer/MEPlayer/MetalPlayView.swift

@@ -17,7 +17,17 @@ public protocol DisplayLayerDelegate: NSObjectProtocol {
 }
 
 public final class MetalPlayView: UIView {
-    private var videoInfo: CMVideoFormatDescription?
+    private var formatDescription: CMFormatDescription? {
+        didSet {
+            #if os(tvOS) || os(xrOS)
+            if let formatDescription {
+                setDisplayCriteria(formatDescription: formatDescription)
+            }
+            #endif
+        }
+    }
+
+    private var fps = Float(60)
     public private(set) var pixelBuffer: CVPixelBuffer?
     /// 用displayLink会导致锁屏无法draw，
     /// 用DispatchSourceTimer的话，在播放4k视频的时候repeat的时间会变长,
@@ -49,6 +59,7 @@ public final class MetalPlayView: UIView {
     }
 
     func prepare(fps: Float, startPlayTime: TimeInterval = 0) {
+        self.fps = fps
         let preferredFramesPerSecond = Int(ceil(fps))
         displayLink.preferredFramesPerSecond = preferredFramesPerSecond << 1
         #if os(iOS)
@@ -157,6 +168,17 @@ extension MetalPlayView {
             let cmtime = frame.cmtime
             let par = pixelBuffer.size
             let sar = pixelBuffer.aspectRatio
+            if formatDescription == nil || !CMVideoFormatDescriptionMatchesImageBuffer(formatDescription!, imageBuffer: pixelBuffer) {
+                if formatDescription != nil {
+                    displayView.removeFromSuperview()
+                    displayView = AVSampleBufferDisplayView()
+                    addSubview(displayView)
+                }
+                let err = CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: pixelBuffer, formatDescriptionOut: &formatDescription)
+                if err != noErr {
+                    KSLog("Error at CMVideoFormatDescriptionCreateForImageBuffer \(err)")
+                }
+            }
             if options.isUseDisplayLayer() {
                 if displayView.isHidden {
                     displayView.isHidden = false
@@ -190,20 +212,24 @@ extension MetalPlayView {
     }
 
     private func set(pixelBuffer: CVPixelBuffer, time: CMTime) {
-        if videoInfo == nil || !CMVideoFormatDescriptionMatchesImageBuffer(videoInfo!, imageBuffer: pixelBuffer) {
-            if videoInfo != nil {
-                displayView.removeFromSuperview()
-                displayView = AVSampleBufferDisplayView()
-                addSubview(displayView)
-            }
-            let err = CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: pixelBuffer, formatDescriptionOut: &videoInfo)
-            if err != noErr {
-                KSLog("Error at CMVideoFormatDescriptionCreateForImageBuffer \(err)")
-            }
+        guard let formatDescription else { return }
+        displayView.enqueue(imageBuffer: pixelBuffer, formatDescription: formatDescription, time: time)
+    }
+
+    #if os(tvOS) || os(xrOS)
+    private func setDisplayCriteria(formatDescription: CMFormatDescription) {
+        guard let displayManager = UIApplication.shared.windows.first?.avDisplayManager,
+              displayManager.isDisplayCriteriaMatchingEnabled,
+              !displayManager.isDisplayModeSwitchInProgress
+        else {
+            return
+        }
+        if #available(tvOS 17.0, *) {
+            let criteria = AVDisplayCriteria(refreshRate: fps, formatDescription: formatDescription)
+            displayManager.preferredDisplayCriteria = criteria
         }
-        guard let videoInfo else { return }
-        displayView.enqueue(imageBuffer: pixelBuffer, formatDescription: videoInfo, time: time)
     }
+    #endif
 }
 
 class MetalView: UIView {