Add WebCodecs HEVC support

https://bugs.webkit.org/show_bug.cgi?id=259102
rdar://112067287

Reviewed by Eric Carlson.

Update H265 RTC encoder to properly handle webcodecs.
In particular, we add some nullptr checks, as well as fixing init data in case AVC format is used.
We also set low latency mode according what webcodecs gives us.

Skipping tests in GLib until it gets implemented.

* LayoutTests/TestExpectations:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h265_annexb-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h265_hevc-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h265_annexb-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h265_hevc-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker_h265_annexb-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker_h265_hevc-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any_h265_annexb-expected.txt:
* LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any_h265_hevc-expected.txt:
* LayoutTests/platform/glib/TestExpectations:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitEncoder.mm:
(-[WK_RTCLocalVideoH264H265Encoder setLowLatency:]):
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm:
(-[RTCVideoDecoderH265 decodeData:size:timeStamp:]):
(-[RTCVideoDecoderH265 setAVCFormat:size:width:height:]):
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH265.h:
* Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm:
(-[RTCVideoEncoderH265 initWithCodecInfo:]):
(-[RTCVideoEncoderH265 setLowLatency:]):
(-[RTCVideoEncoderH265 resetCompressionSession]):
(-[RTCVideoEncoderH265 configureCompressionSession]):
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp:
(WebKit::LibWebRTCCodecs::videoCodecTypeFromWebCodec):

Canonical link: https://commits.webkit.org/266044@main

LayoutTests/TestExpectations

@@ -5083,15 +5083,15 @@ imported/w3c/web-platform-tests/html/canvas/offscreen/wide-gamut-canvas/2d.color
 imported/w3c/web-platform-tests/webcodecs/videoFrame-createImageBitmap.any.worker.html [ Failure ]
 imported/w3c/web-platform-tests/webcodecs/videoFrame-drawImage.any.worker.html [ Failure ]
 
-# HEVC support
-imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?h265_annexb [ Failure ]
-imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?h265_hevc [ Failure ]
-imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h265_annexb [ Failure ]
-imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h265_hevc [ Failure ]
-imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?h265_annexb [ Skip ]
-imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?h265_hevc [ Skip ]
-imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?h265_annexb [ Skip ]
-imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?h265_hevc [ Skip ]
+# HEVC support may not be available to all platforms.
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?h265_annexb [ Pass Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?h265_hevc [ Pass Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h265_annexb [ Pass Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h265_hevc [ Pass Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?h265_annexb [ Pass Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?h265_hevc [ Pass Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?h265_annexb [ Pass Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?h265_hevc [ Pass Failure ]
 
 webkit.org/b/226942 imported/w3c/web-platform-tests/webaudio/the-audio-api/the-audiobuffersourcenode-interface/active-processing.https.html [ Pass Failure ]
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h265_annexb-expected.txt

@@ -1,4 +1,4 @@
 
-FAIL Encoding and decoding cycle promise_test: Unhandled rejection with value: object "InvalidStateError: VideoDecoder is not configured"
-FAIL Encoding and decoding cycle w/ stripped color space promise_test: Unhandled rejection with value: object "InvalidStateError: VideoDecoder is not configured"
+PASS Encoding and decoding cycle
+PASS Encoding and decoding cycle w/ stripped color space
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker_h265_hevc-expected.txt

@@ -1,4 +1,4 @@
 
-FAIL Encoding and decoding cycle promise_test: Unhandled rejection with value: object "InvalidStateError: VideoDecoder is not configured"
-FAIL Encoding and decoding cycle w/ stripped color space promise_test: Unhandled rejection with value: object "InvalidStateError: VideoDecoder is not configured"
+PASS Encoding and decoding cycle
+PASS Encoding and decoding cycle w/ stripped color space
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h265_annexb-expected.txt

@@ -1,4 +1,4 @@
 
-FAIL Encoding and decoding cycle promise_test: Unhandled rejection with value: object "InvalidStateError: VideoDecoder is not configured"
-FAIL Encoding and decoding cycle w/ stripped color space promise_test: Unhandled rejection with value: object "InvalidStateError: VideoDecoder is not configured"
+PASS Encoding and decoding cycle
+PASS Encoding and decoding cycle w/ stripped color space
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any_h265_hevc-expected.txt

@@ -1,4 +1,4 @@
 
-FAIL Encoding and decoding cycle promise_test: Unhandled rejection with value: object "InvalidStateError: VideoDecoder is not configured"
-FAIL Encoding and decoding cycle w/ stripped color space promise_test: Unhandled rejection with value: object "InvalidStateError: VideoDecoder is not configured"
+PASS Encoding and decoding cycle
+PASS Encoding and decoding cycle w/ stripped color space
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker_h265_annexb-expected.txt

@@ -1,20 +1,20 @@
 
-NOTRUN Test isConfigSupported() hev1.1.6.L60.90 unsupported
-NOTRUN Test isConfigSupported() with 1080p crop hev1.1.6.L60.90 unsupported
-NOTRUN Test that isConfigSupported() returns a parsed configuration hev1.1.6.L60.90 unsupported
-NOTRUN Test invalid configs hev1.1.6.L60.90 unsupported
-NOTRUN Test configure() hev1.1.6.L60.90 unsupported
-NOTRUN Decode a key frame hev1.1.6.L60.90 unsupported
-NOTRUN Decode a non key frame first fails hev1.1.6.L60.90 unsupported
-NOTRUN Verify reset() suppresses outputs hev1.1.6.L60.90 unsupported
-NOTRUN Test unconfigured VideoDecoder operations hev1.1.6.L60.90 unsupported
-NOTRUN Test closed VideoDecoder operations hev1.1.6.L60.90 unsupported
-NOTRUN Decode empty frame hev1.1.6.L60.90 unsupported
-NOTRUN Decode corrupt frame hev1.1.6.L60.90 unsupported
-NOTRUN Close while decoding corrupt frame hev1.1.6.L60.90 unsupported
-NOTRUN Test decoding after flush hev1.1.6.L60.90 unsupported
-NOTRUN Test decoding a with negative timestamp hev1.1.6.L60.90 unsupported
-NOTRUN Test reset during flush hev1.1.6.L60.90 unsupported
-NOTRUN Test low-latency decoding hev1.1.6.L60.90 unsupported
-NOTRUN VideoDecoder decodeQueueSize test hev1.1.6.L60.90 unsupported
+PASS Test isConfigSupported()
+PASS Test isConfigSupported() with 1080p crop
+PASS Test that isConfigSupported() returns a parsed configuration
+PASS Test invalid configs
+PASS Test configure()
+PASS Decode a key frame
+PASS Decode a non key frame first fails
+PASS Verify reset() suppresses outputs
+PASS Test unconfigured VideoDecoder operations
+PASS Test closed VideoDecoder operations
+PASS Decode empty frame
+PASS Decode corrupt frame
+PASS Close while decoding corrupt frame
+PASS Test decoding after flush
+PASS Test decoding a with negative timestamp
+PASS Test reset during flush
+PASS Test low-latency decoding
+PASS VideoDecoder decodeQueueSize test
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker_h265_hevc-expected.txt

@@ -1,22 +1,20 @@
 
-Harness Error (TIMEOUT), message = null
-
 PASS Test isConfigSupported()
 PASS Test isConfigSupported() with 1080p crop
 PASS Test that isConfigSupported() returns a parsed configuration
 PASS Test invalid configs
 PASS Test configure()
-FAIL Decode a key frame promise_test: Unhandled rejection with value: object "EncodingError: Decoder failure"
+PASS Decode a key frame
 PASS Decode a non key frame first fails
-TIMEOUT Verify reset() suppresses outputs Test timed out
-NOTRUN Test unconfigured VideoDecoder operations
-NOTRUN Test closed VideoDecoder operations
-NOTRUN Decode empty frame
-NOTRUN Decode corrupt frame
-NOTRUN Close while decoding corrupt frame
-NOTRUN Test decoding after flush
-NOTRUN Test decoding a with negative timestamp
-NOTRUN Test reset during flush
-NOTRUN Test low-latency decoding
-NOTRUN VideoDecoder decodeQueueSize test
+PASS Verify reset() suppresses outputs
+PASS Test unconfigured VideoDecoder operations
+PASS Test closed VideoDecoder operations
+PASS Decode empty frame
+PASS Decode corrupt frame
+PASS Close while decoding corrupt frame
+PASS Test decoding after flush
+PASS Test decoding a with negative timestamp
+PASS Test reset during flush
+PASS Test low-latency decoding
+PASS VideoDecoder decodeQueueSize test
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any_h265_annexb-expected.txt

@@ -1,20 +1,20 @@
 
-NOTRUN Test isConfigSupported() hev1.1.6.L60.90 unsupported
-NOTRUN Test isConfigSupported() with 1080p crop hev1.1.6.L60.90 unsupported
-NOTRUN Test that isConfigSupported() returns a parsed configuration hev1.1.6.L60.90 unsupported
-NOTRUN Test invalid configs hev1.1.6.L60.90 unsupported
-NOTRUN Test configure() hev1.1.6.L60.90 unsupported
-NOTRUN Decode a key frame hev1.1.6.L60.90 unsupported
-NOTRUN Decode a non key frame first fails hev1.1.6.L60.90 unsupported
-NOTRUN Verify reset() suppresses outputs hev1.1.6.L60.90 unsupported
-NOTRUN Test unconfigured VideoDecoder operations hev1.1.6.L60.90 unsupported
-NOTRUN Test closed VideoDecoder operations hev1.1.6.L60.90 unsupported
-NOTRUN Decode empty frame hev1.1.6.L60.90 unsupported
-NOTRUN Decode corrupt frame hev1.1.6.L60.90 unsupported
-NOTRUN Close while decoding corrupt frame hev1.1.6.L60.90 unsupported
-NOTRUN Test decoding after flush hev1.1.6.L60.90 unsupported
-NOTRUN Test decoding a with negative timestamp hev1.1.6.L60.90 unsupported
-NOTRUN Test reset during flush hev1.1.6.L60.90 unsupported
-NOTRUN Test low-latency decoding hev1.1.6.L60.90 unsupported
-NOTRUN VideoDecoder decodeQueueSize test hev1.1.6.L60.90 unsupported
+PASS Test isConfigSupported()
+PASS Test isConfigSupported() with 1080p crop
+PASS Test that isConfigSupported() returns a parsed configuration
+PASS Test invalid configs
+PASS Test configure()
+PASS Decode a key frame
+PASS Decode a non key frame first fails
+PASS Verify reset() suppresses outputs
+PASS Test unconfigured VideoDecoder operations
+PASS Test closed VideoDecoder operations
+PASS Decode empty frame
+PASS Decode corrupt frame
+PASS Close while decoding corrupt frame
+PASS Test decoding after flush
+PASS Test decoding a with negative timestamp
+PASS Test reset during flush
+PASS Test low-latency decoding
+PASS VideoDecoder decodeQueueSize test
 

LayoutTests/imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any_h265_hevc-expected.txt

@@ -1,22 +1,20 @@
 
-Harness Error (TIMEOUT), message = null
-
 PASS Test isConfigSupported()
 PASS Test isConfigSupported() with 1080p crop
 PASS Test that isConfigSupported() returns a parsed configuration
 PASS Test invalid configs
 PASS Test configure()
-FAIL Decode a key frame promise_test: Unhandled rejection with value: object "EncodingError: Decoder failure"
+PASS Decode a key frame
 PASS Decode a non key frame first fails
-TIMEOUT Verify reset() suppresses outputs Test timed out
-NOTRUN Test unconfigured VideoDecoder operations
-NOTRUN Test closed VideoDecoder operations
-NOTRUN Decode empty frame
-NOTRUN Decode corrupt frame
-NOTRUN Close while decoding corrupt frame
-NOTRUN Test decoding after flush
-NOTRUN Test decoding a with negative timestamp
-NOTRUN Test reset during flush
-NOTRUN Test low-latency decoding
-NOTRUN VideoDecoder decodeQueueSize test
+PASS Verify reset() suppresses outputs
+PASS Test unconfigured VideoDecoder operations
+PASS Test closed VideoDecoder operations
+PASS Decode empty frame
+PASS Decode corrupt frame
+PASS Close while decoding corrupt frame
+PASS Test decoding after flush
+PASS Test decoding a with negative timestamp
+PASS Test reset during flush
+PASS Test low-latency decoding
+PASS VideoDecoder decodeQueueSize test
 

LayoutTests/platform/glib/TestExpectations

@@ -1125,6 +1125,16 @@ imported/w3c/web-platform-tests/webcodecs/temporal-svc-encoding.https.any.html?h
 imported/w3c/web-platform-tests/webcodecs/temporal-svc-encoding.https.any.html?vp8 [ Failure ]
 imported/w3c/web-platform-tests/webcodecs/temporal-svc-encoding.https.any.html?vp9 [ Failure ]
 
+# HEVC support
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?h265_annexb [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.html?h265_hevc [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h265_annexb [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h265_hevc [ Failure ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?h265_annexb [ Skip ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.html?h265_hevc [ Skip ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?h265_annexb [ Skip ]
+imported/w3c/web-platform-tests/webcodecs/videoDecoder-codec-specific.https.any.worker.html?h265_hevc [ Skip ]
+
 imported/w3c/web-platform-tests/webcodecs/videoFrame-createImageBitmap.any.worker.html [ Pass ]
 
 # Likely bugs related with dav1ddec.

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitEncoder.mm

@@ -107,6 +107,7 @@ - (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate {
 - (void)setLowLatency:(bool)lowLatencyEnabled {
     if (m_h264Encoder)
         [m_h264Encoder setH264LowLatencyEncoderEnabled:lowLatencyEnabled];
+    [m_h265Encoder setLowLatency:lowLatencyEnabled];
 }
 
 - (void)setUseAnnexB:(bool)useAnnexB {

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm

@@ -138,6 +138,10 @@ - (NSInteger)decodeData:(const uint8_t *)data size:(size_t)size timeStamp:(int64
     _error = noErr;
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
+  if (!data || !size) {
+    RTC_LOG(LS_WARNING) << "Empty frame.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
 
   rtc::ScopedCFTypeRef<CMVideoFormatDescriptionRef> inputFormat =
       rtc::ScopedCF(webrtc::CreateH265VideoFormatDescription(
@@ -223,7 +227,7 @@ - (NSInteger)setAVCFormat:(const uint8_t *)data size:(size_t)size width:(uint16_
     &kCFTypeDictionaryValueCallBacks);
 
   CMVideoFormatDescriptionRef videoFormatDescription = nullptr;
-  auto err = CMVideoFormatDescriptionCreate(NULL, kCMVideoCodecType_H264, width, height, extensionsDict, &videoFormatDescription);
+  auto err = CMVideoFormatDescriptionCreate(NULL, kCMVideoCodecType_HEVC, width, height, extensionsDict, &videoFormatDescription);
   CFRelease(codecConfig);
   CFRelease(atomsDict);
   CFRelease(extensionsDict);

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH265.h

@@ -19,6 +19,7 @@ __attribute__((objc_runtime_name("WK_RTCVideoEncoderH265")))
 @interface RTCVideoEncoderH265 : NSObject <RTCVideoEncoder>
 
 - (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo;
+- (void)setLowLatency:(bool)enabled;
 - (void)setUseAnnexB:(bool)useAnnexB;
 - (void)setDescriptionCallback:(RTCVideoEncoderDescriptionCallback)callback;
 - (void)flush;

Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm

@@ -178,6 +178,7 @@ @implementation RTCVideoEncoderH265 {
   int framesLeft;
   std::vector<uint8_t> _nv12ScaleBuffer;
   bool _useAnnexB;
+  bool _isLowLatencyEnabled;
   bool _needsToSendDescription;
   RTCVideoEncoderDescriptionCallback _descriptionCallback;
 }
@@ -194,8 +195,10 @@ - (instancetype)initWithCodecInfo:(RTCVideoCodecInfo*)codecInfo {
     _codecInfo = codecInfo;
     _bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95));
     _useAnnexB = true;
+    _isLowLatencyEnabled = true;
     RTC_CHECK([codecInfo.name isEqualToString:@"H265"]);
   }
+
   return self;
 }
 
@@ -225,6 +228,11 @@ - (void)setUseAnnexB:(bool)useAnnexB
     _needsToSendDescription = !useAnnexB;
 }
 
+- (void)setLowLatency:(bool)enabled
+{
+    _isLowLatencyEnabled = enabled;
+}
+
 - (void)setDescriptionCallback:(RTCVideoEncoderDescriptionCallback)callback
 {
     _descriptionCallback = callback;
@@ -409,7 +417,8 @@ - (int)resetCompressionSession {
   CFDictionarySetValue(encoder_specs, kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, kCFBooleanTrue);
 #endif
 #if HAVE_VTB_REQUIREDLOWLATENCY
-  CFDictionarySetValue(encoder_specs, kVTVideoEncoderSpecification_RequiredLowLatency, kCFBooleanTrue);
+  if (_isLowLatencyEnabled)
+    CFDictionarySetValue(encoder_specs, kVTVideoEncoderSpecification_RequiredLowLatency, kCFBooleanTrue);
 #endif
   OSStatus status = VTCompressionSessionCreate(
       nullptr,  // use default allocator
@@ -460,7 +469,7 @@ - (int)resetCompressionSession {
 - (void)configureCompressionSession {
   RTC_DCHECK(_compressionSession);
   SetVTSessionProperty(_compressionSession, nullptr, kVTCompressionPropertyKey_RealTime,
-                       false);
+                       _isLowLatencyEnabled);
   // SetVTSessionProperty(_compressionSession,
   // kVTCompressionPropertyKey_ProfileLevel, _profile);
   SetVTSessionProperty(_compressionSession, nullptr,

Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp

@@ -93,7 +93,9 @@ std::optional<VideoCodecType> LibWebRTCCodecs::videoCodecTypeFromWebCodec(const
     if (codec.startsWith("avc1."_s))
         return VideoCodecType::H264;
 
-    // FIXME: Expose H265 if available.
+    if (codec.startsWith("hev1."_s) || codec.startsWith("hvc1."_s))
+        return VideoCodecType::H265;
+
     return { };
 }