Skip to content

Commit

Permalink
Support HEVC with SRTHaishinKit.
Browse files Browse the repository at this point in the history
  • Loading branch information
shogo4405 committed Jun 8, 2024
1 parent e3288d1 commit 3c05a4d
Show file tree
Hide file tree
Showing 10 changed files with 195 additions and 75 deletions.
22 changes: 13 additions & 9 deletions HaishinKit.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@
BC34E00225EBB59C005F975A /* Logboard.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = BC34DFD125EBB12C005F975A /* Logboard.xcframework */; };
BC34FA0B286CB90A00EFAF27 /* PiPHKView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */; };
BC37861D2C0F7B9900D79263 /* CMFormatDescription+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC37861C2C0F7B9900D79263 /* CMFormatDescription+Extension.swift */; };
BC3786232C10CA9B00D79263 /* NALUnitReader.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC3786222C10CA9B00D79263 /* NALUnitReader.swift */; };
BC3802122AB5E770001AE399 /* IOVideoCaptureUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC3802112AB5E770001AE399 /* IOVideoCaptureUnit.swift */; };
BC3802142AB5E7CC001AE399 /* IOAudioCaptureUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC3802132AB5E7CC001AE399 /* IOAudioCaptureUnit.swift */; };
BC3802192AB6AD79001AE399 /* IOAudioMixerTrackTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC3802182AB6AD79001AE399 /* IOAudioMixerTrackTests.swift */; };
Expand Down Expand Up @@ -239,12 +240,12 @@
BCABED1F2BDD097F00CC7E73 /* IOAudioMixer.swift in Sources */ = {isa = PBXBuildFile; fileRef = B3D687812B80302B00E6A28E /* IOAudioMixer.swift */; };
BCABED212BDE23C600CC7E73 /* AudioNode+DebugExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCABED202BDE23C600CC7E73 /* AudioNode+DebugExtension.swift */; };
BCB976DF26107B5600C9A649 /* TSField.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCB976DE26107B5600C9A649 /* TSField.swift */; };
BCB9773F2621812800C9A649 /* AVCFormatStream.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCB9773E2621812800C9A649 /* AVCFormatStream.swift */; };
BCB9773F2621812800C9A649 /* ISOTypeBufferUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCB9773E2621812800C9A649 /* ISOTypeBufferUtil.swift */; };
BCC1A72B264FAC1800661156 /* ESSpecificData.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCC1A72A264FAC1800661156 /* ESSpecificData.swift */; };
BCC4F4152AD6FC1100954EF5 /* IOTellyUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCC4F4142AD6FC1100954EF5 /* IOTellyUnit.swift */; };
BCC4F43D2ADB966800954EF5 /* NetStreamSwitcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCE0E33B2AD369410082C16F /* NetStreamSwitcher.swift */; };
BCC9E9092636FF7400948774 /* DataBufferTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCC9E9082636FF7400948774 /* DataBufferTests.swift */; };
BCCBCE9529A7C9C90095B51C /* AVCFormatStreamTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCCBCE9429A7C9C90095B51C /* AVCFormatStreamTests.swift */; };
BCCBCE9529A7C9C90095B51C /* ISOTypeBufferUtilTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCCBCE9429A7C9C90095B51C /* ISOTypeBufferUtilTests.swift */; };
BCCBCE9729A90D880095B51C /* AVCNALUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCCBCE9629A90D880095B51C /* AVCNALUnit.swift */; };
BCCBCE9B29A9D96A0095B51C /* NALUnitReaderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCCBCE9A29A9D96A0095B51C /* NALUnitReaderTests.swift */; };
BCCC45992AA289FA0016EFE8 /* SRTHaishinKit.h in Headers */ = {isa = PBXBuildFile; fileRef = BCCC45982AA289FA0016EFE8 /* SRTHaishinKit.h */; settings = {ATTRIBUTES = (Public, ); }; };
Expand Down Expand Up @@ -611,6 +612,7 @@
BC34DFD125EBB12C005F975A /* Logboard.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = Logboard.xcframework; path = Carthage/Build/Logboard.xcframework; sourceTree = "<group>"; };
BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PiPHKView.swift; sourceTree = "<group>"; };
BC37861C2C0F7B9900D79263 /* CMFormatDescription+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CMFormatDescription+Extension.swift"; sourceTree = "<group>"; };
BC3786222C10CA9B00D79263 /* NALUnitReader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NALUnitReader.swift; sourceTree = "<group>"; };
BC3802112AB5E770001AE399 /* IOVideoCaptureUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOVideoCaptureUnit.swift; sourceTree = "<group>"; };
BC3802132AB5E7CC001AE399 /* IOAudioCaptureUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioCaptureUnit.swift; sourceTree = "<group>"; };
BC3802182AB6AD79001AE399 /* IOAudioMixerTrackTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioMixerTrackTests.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -667,11 +669,11 @@
BCABED1D2BDCC79000CC7E73 /* SRTMuxer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SRTMuxer.swift; sourceTree = "<group>"; };
BCABED202BDE23C600CC7E73 /* AudioNode+DebugExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AudioNode+DebugExtension.swift"; sourceTree = "<group>"; };
BCB976DE26107B5600C9A649 /* TSField.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TSField.swift; sourceTree = "<group>"; };
BCB9773E2621812800C9A649 /* AVCFormatStream.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVCFormatStream.swift; sourceTree = "<group>"; };
BCB9773E2621812800C9A649 /* ISOTypeBufferUtil.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ISOTypeBufferUtil.swift; sourceTree = "<group>"; };
BCC1A72A264FAC1800661156 /* ESSpecificData.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ESSpecificData.swift; sourceTree = "<group>"; };
BCC4F4142AD6FC1100954EF5 /* IOTellyUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOTellyUnit.swift; sourceTree = "<group>"; };
BCC9E9082636FF7400948774 /* DataBufferTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataBufferTests.swift; sourceTree = "<group>"; };
BCCBCE9429A7C9C90095B51C /* AVCFormatStreamTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVCFormatStreamTests.swift; sourceTree = "<group>"; };
BCCBCE9429A7C9C90095B51C /* ISOTypeBufferUtilTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ISOTypeBufferUtilTests.swift; sourceTree = "<group>"; };
BCCBCE9629A90D880095B51C /* AVCNALUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVCNALUnit.swift; sourceTree = "<group>"; };
BCCBCE9A29A9D96A0095B51C /* NALUnitReaderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NALUnitReaderTests.swift; sourceTree = "<group>"; };
BCCC45962AA289FA0016EFE8 /* SRTHaishinKit.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = SRTHaishinKit.framework; sourceTree = BUILT_PRODUCTS_DIR; };
Expand Down Expand Up @@ -859,16 +861,16 @@
children = (
BC7C56D029A78D4F00C41A9B /* ADTSHeaderTests.swift */,
BC3E384329C216BB007CD972 /* ADTSReaderTests.swift */,
BCCBCE9429A7C9C90095B51C /* AVCFormatStreamTests.swift */,
2917CB652104CA2800F6823A /* AudioSpecificConfigTests.swift */,
BC1720A82C03473200F65941 /* AVCDecoderConfigurationRecordTests.swift */,
BC7C56C629A7701F00C41A9B /* ESSpecificDataTests.swift */,
BC1DC5112A04E46E00E928ED /* HEVCDecoderConfigurationRecordTests.swift */,
BCCBCE9429A7C9C90095B51C /* ISOTypeBufferUtilTests.swift */,
BCCBCE9A29A9D96A0095B51C /* NALUnitReaderTests.swift */,
290EA8951DFB619600053022 /* PacketizedElementaryStreamTests.swift */,
290EA8971DFB619600053022 /* TSPacketTests.swift */,
290EA8961DFB619600053022 /* TSProgramTests.swift */,
BC7C56C229A1F28700C41A9B /* TSReaderTests.swift */,
BC1720A82C03473200F65941 /* AVCDecoderConfigurationRecordTests.swift */,
);
path = ISO;
sourceTree = "<group>";
Expand Down Expand Up @@ -1286,12 +1288,13 @@
BC7C56CC29A786AE00C41A9B /* ADTS.swift */,
29B8767D1CD70AE800FC07DA /* AudioSpecificConfig.swift */,
29B8767E1CD70AE800FC07DA /* AVCDecoderConfigurationRecord.swift */,
BCB9773E2621812800C9A649 /* AVCFormatStream.swift */,
BCCBCE9629A90D880095B51C /* AVCNALUnit.swift */,
29B876B91CD70B3900FC07DA /* CRC32.swift */,
BCC1A72A264FAC1800661156 /* ESSpecificData.swift */,
BC1DC5092A039B4400E928ED /* HEVCDecoderConfigurationRecord.swift */,
BC1DC5132A05428800E928ED /* HEVCNALUnit.swift */,
BCB9773E2621812800C9A649 /* ISOTypeBufferUtil.swift */,
BC3786222C10CA9B00D79263 /* NALUnitReader.swift */,
29B876801CD70AE800FC07DA /* PacketizedElementaryStream.swift */,
BCB976DE26107B5600C9A649 /* TSField.swift */,
29B876821CD70AE800FC07DA /* TSPacket.swift */,
Expand Down Expand Up @@ -1775,7 +1778,7 @@
files = (
BC4914AE28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift in Sources */,
29B876B11CD70B2800FC07DA /* RTMPMessage.swift in Sources */,
BCB9773F2621812800C9A649 /* AVCFormatStream.swift in Sources */,
BCB9773F2621812800C9A649 /* ISOTypeBufferUtil.swift in Sources */,
BC83A4732403D83B006BDE06 /* VTCompressionSession+Extension.swift in Sources */,
BC4914A228DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */,
2915EC4D1D85BB8C00621092 /* RTMPTSocket.swift in Sources */,
Expand Down Expand Up @@ -1866,6 +1869,7 @@
BCA3A5252BC4ED220083BBB1 /* RTMPTimestamp.swift in Sources */,
BC7C56BB299E595000C41A9B /* VideoCodecSettings.swift in Sources */,
29B876881CD70AE800FC07DA /* TSPacket.swift in Sources */,
BC3786232C10CA9B00D79263 /* NALUnitReader.swift in Sources */,
BC22EEEE2AAF50F200E3406D /* Codec.swift in Sources */,
29B876BE1CD70B3900FC07DA /* EventDispatcher.swift in Sources */,
BC2828AF2AA322E400741013 /* AVFrameRateRange+Extension.swift in Sources */,
Expand Down Expand Up @@ -1902,7 +1906,7 @@
buildActionMask = 2147483647;
files = (
290EA89B1DFB619600053022 /* TSPacketTests.swift in Sources */,
BCCBCE9529A7C9C90095B51C /* AVCFormatStreamTests.swift in Sources */,
BCCBCE9529A7C9C90095B51C /* ISOTypeBufferUtilTests.swift in Sources */,
290EA8A91DFB61E700053022 /* ByteArrayTests.swift in Sources */,
BC0587C12BD2A123006751C8 /* IOAudioMixerBySingleTrackTests.swift in Sources */,
295018221FFA1C9D00358E10 /* CMAudioSampleBufferFactory.swift in Sources */,
Expand Down
30 changes: 5 additions & 25 deletions Sources/ISO/AVCNALUnit.swift
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ enum AVCNALUnitType: UInt8, Equatable {
}

// MARK: -
struct AVCNALUnit: Equatable {
struct AVCNALUnit: NALUnit, Equatable {
let refIdc: UInt8
let type: AVCNALUnitType
let payload: Data
Expand All @@ -41,31 +41,11 @@ struct AVCNALUnit: Equatable {
}
}

class AVCNALUnitReader {
static let defaultStartCodeLength: Int = 4
static let defaultNALUnitHeaderLength: Int32 = 4

var nalUnitHeaderLength: Int32 = AVCNALUnitReader.defaultNALUnitHeaderLength

func read(_ data: Data) -> [AVCNALUnit] {
var units: [AVCNALUnit] = []
var lastIndexOf = data.count - 1
for i in (2..<data.count).reversed() {
guard data[i] == 1 && data[i - 1] == 0 && data[i - 2] == 0 else {
continue
}
let startCodeLength = 0 <= i - 3 && data[i - 3] == 0 ? 4 : 3
units.append(.init(data.subdata(in: (i + 1)..<lastIndexOf + 1)))
lastIndexOf = i - startCodeLength
}
return units
}

func makeFormatDescription(_ data: Data) -> CMFormatDescription? {
let units = read(data).filter { $0.type == .pps || $0.type == .sps }
extension [AVCNALUnit] {
func makeFormatDescription(_ nalUnitHeaderLength: Int32 = 4) -> CMFormatDescription? {
guard
let pps = units.first(where: { $0.type == .pps }),
let sps = units.first(where: { $0.type == .sps }) else {
let pps = first(where: { $0.type == .pps }),
let sps = first(where: { $0.type == .sps }) else {
return nil
}
var formatDescription: CMFormatDescription?
Expand Down
82 changes: 81 additions & 1 deletion Sources/ISO/HEVCNALUnit.swift
Original file line number Diff line number Diff line change
@@ -1,8 +1,88 @@
import CoreMedia
import Foundation

enum HEVCNALUnitType: UInt8 {
case unspec = 0
case codedSliceTrailN = 0
case codedSliceTrailR = 1
case codedSliceTsaN = 2
case codedSliceTsaR = 3
case codedSliceStsaN = 4
case codedSliceStsaR = 5
case codedSliceRadlN = 6
case codedSliceRadlR = 7
case codedSliceRaslN = 8
case codedSliceRsslR = 9
/// 10...15 Reserved
case vps = 32
case sps = 33
case pps = 34
case accessUnitDelimiter = 35
case unspec = 0xFF
}

struct HEVCNALUnit: NALUnit, Equatable {
let type: HEVCNALUnitType
let temporalIdPlusOne: UInt8
let payload: Data

init(_ data: Data) {
self.init(data, length: data.count)
}

init(_ data: Data, length: Int) {
self.type = HEVCNALUnitType(rawValue: (data[0] & 0x7e) >> 1) ?? .unspec
self.temporalIdPlusOne = data[1] & 0b00011111
self.payload = data.subdata(in: 2..<length)
}

var data: Data {
var result = Data()
result.append(type.rawValue << 1)
result.append(temporalIdPlusOne)
result.append(payload)
return result
}
}

extension [HEVCNALUnit] {
func makeFormatDescription(_ nalUnitHeaderLength: Int32 = 4) -> CMFormatDescription? {
guard
let vps = first(where: { $0.type == .vps }),
let sps = first(where: { $0.type == .sps }),
let pps = first(where: { $0.type == .pps }) else {
return nil
}
return vps.data.withUnsafeBytes { (vpsBuffer: UnsafeRawBufferPointer) -> CMFormatDescription? in
guard let vpsBaseAddress = vpsBuffer.baseAddress else {
return nil
}
return sps.data.withUnsafeBytes { (spsBuffer: UnsafeRawBufferPointer) -> CMFormatDescription? in
guard let spsBaseAddress = spsBuffer.baseAddress else {
return nil
}
return pps.data.withUnsafeBytes { (ppsBuffer: UnsafeRawBufferPointer) -> CMFormatDescription? in
guard let ppsBaseAddress = ppsBuffer.baseAddress else {
return nil
}
var formatDescriptionOut: CMFormatDescription?
let pointers: [UnsafePointer<UInt8>] = [
vpsBaseAddress.assumingMemoryBound(to: UInt8.self),
spsBaseAddress.assumingMemoryBound(to: UInt8.self),
ppsBaseAddress.assumingMemoryBound(to: UInt8.self)
]
let sizes: [Int] = [vpsBuffer.count, spsBuffer.count, ppsBuffer.count]
let status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(
allocator: kCFAllocatorDefault,
parameterSetCount: pointers.count,
parameterSetPointers: pointers,
parameterSetSizes: sizes,
nalUnitHeaderLength: nalUnitHeaderLength,
extensions: nil,
formatDescriptionOut: &formatDescriptionOut
)
return formatDescriptionOut
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Foundation

struct AVCFormatStream {
struct ISOTypeBufferUtil {
let data: Data

init(data: Data) {
Expand Down Expand Up @@ -33,7 +33,7 @@ struct AVCFormatStream {
return result
}

static func toNALFileFormat(_ data: inout Data) -> Data {
static func toNALFileFormat(_ data: inout Data) {
var lastIndexOf = data.count - 1
for i in (2..<data.count).reversed() {
guard data[i] == 1 && data[i - 1] == 0 && data[i - 2] == 0 else {
Expand All @@ -47,6 +47,5 @@ struct AVCFormatStream {
lastIndexOf = i - startCodeLength
}
}
return data
}
}
38 changes: 38 additions & 0 deletions Sources/ISO/NALUnitReader.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import CoreMedia
import Foundation

protocol NALUnit {
init(_ data: Data)
}

final class NALUnitReader {
static let defaultNALUnitHeaderLength: Int32 = 4
var nalUnitHeaderLength: Int32 = NALUnitReader.defaultNALUnitHeaderLength

func read<T: NALUnit>(_ data: inout Data, type: T.Type) -> [T] {
var units: [T] = .init()
var lastIndexOf = data.count - 1
for i in (2..<data.count).reversed() {
guard data[i] == 1 && data[i - 1] == 0 && data[i - 2] == 0 else {
continue
}
let startCodeLength = 0 <= i - 3 && data[i - 3] == 0 ? 4 : 3
units.append(T.init(data.subdata(in: (i + 1)..<lastIndexOf + 1)))
lastIndexOf = i - startCodeLength
}
return units
}

func makeFormatDescription(_ data: inout Data, type: ESStreamType) -> CMFormatDescription? {
switch type {
case .h264:
let units = read(&data, type: AVCNALUnit.self)
return units.makeFormatDescription(nalUnitHeaderLength)
case .h265:
let units = read(&data, type: HEVCNALUnit.self)
return units.makeFormatDescription(nalUnitHeaderLength)
default:
return nil
}
}
}
20 changes: 16 additions & 4 deletions Sources/ISO/PacketizedElementaryStream.swift
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ enum PESPTSDTSIndicator: UInt8 {
struct PESOptionalHeader {
static let fixedSectionSize: Int = 3
static let defaultMarkerBits: UInt8 = 2
static let offset = CMTime(value: 3, timescale: 30)

var markerBits: UInt8 = PESOptionalHeader.defaultMarkerBits
var scramblingControl: UInt8 = 0
Expand Down Expand Up @@ -58,7 +59,7 @@ struct PESOptionalHeader {
ptsDtsIndicator |= 0x01
}
if (ptsDtsIndicator & 0x02) == 0x02 {
let pts = Int64((presentationTimeStamp.seconds - base) * Double(TSTimestamp.resolution))
let pts = Int64((presentationTimeStamp.seconds + Self.offset.seconds - base) * Double(TSTimestamp.resolution))
optionalFields += TSTimestamp.encode(pts, ptsDtsIndicator << 4)
}
if (ptsDtsIndicator & 0x01) == 0x01 {
Expand Down Expand Up @@ -215,7 +216,18 @@ struct PacketizedElementaryStream: PESPacketHeader {
data.append(contentsOf: [0x00, 0x00, 0x00, 0x01, 0x09, 0x30])
}
if let dataBytes = try? dataBuffer.dataBytes() {
let stream = AVCFormatStream(data: dataBytes)
let stream = ISOTypeBufferUtil(data: dataBytes)
data.append(stream.toByteStream())
}
case .hevc:
if !sampleBuffer.isNotSync {
sampleBuffer.formatDescription?.parameterSets.forEach {
data.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
data.append(contentsOf: $0)
}
}
if let dataBytes = try? dataBuffer.dataBytes() {
let stream = ISOTypeBufferUtil(data: dataBytes)
data.append(stream.toByteStream())
}
default:
Expand Down Expand Up @@ -327,8 +339,8 @@ struct PacketizedElementaryStream: PESPacketHeader {
var blockBuffer: CMBlockBuffer?
var sampleSizes: [Int] = []
switch streamType {
case .h264:
_ = AVCFormatStream.toNALFileFormat(&data)
case .h264, .h265:
ISOTypeBufferUtil.toNALFileFormat(&data)
blockBuffer = data.makeBlockBuffer(advancedBy: 0)
sampleSizes.append(blockBuffer?.dataLength ?? 0)
case .adtsAac:
Expand Down
Loading

0 comments on commit 3c05a4d

Please sign in to comment.