From 519c6ed54ffb73b6d1ea283939522b7c6c0e2041 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 15 Nov 2023 08:42:39 +0100 Subject: [PATCH 1/2] fix offset and size of buffers depend of MediaCodec.BufferInfo --- .../com/pedro/rtmp/flv/audio/AacPacket.kt | 7 +++-- .../com/pedro/rtmp/flv/video/H264Packet.kt | 9 +++--- .../com/pedro/rtmp/flv/video/H265Packet.kt | 9 +++--- .../main/java/com/pedro/rtmp/utils/Utils.kt | 9 ++++++ .../com/pedro/rtsp/rtp/packets/AacPacket.kt | 6 ++-- .../com/pedro/rtsp/rtp/packets/H264Packet.kt | 17 ++++++----- .../com/pedro/rtsp/rtp/packets/H265Packet.kt | 13 +++++---- .../java/com/pedro/rtsp/utils/Extensions.kt | 9 ++++++ .../pedro/srt/mpeg2ts/packets/AacPacket.kt | 7 +++-- .../pedro/srt/mpeg2ts/packets/H26XPacket.kt | 7 +++-- .../java/com/pedro/srt/utils/Extensions.kt | 8 +++++ .../java/com/pedro/srt/utils/ExtensionTest.kt | 29 +++++++++++++++++++ 12 files changed, 97 insertions(+), 33 deletions(-) create mode 100644 srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AacPacket.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AacPacket.kt index fdcc4df2c..817bee7b9 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AacPacket.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AacPacket.kt @@ -19,6 +19,7 @@ package com.pedro.rtmp.flv.audio import android.media.MediaCodec import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType +import com.pedro.rtmp.utils.removeInfo import java.nio.ByteBuffer import kotlin.experimental.or @@ -53,6 +54,7 @@ class AacPacket() { info: MediaCodec.BufferInfo, callback: (FlvPacket) -> Unit ) { + val fixedBuffer = byteBuffer.removeInfo(info) //header is 2 bytes length //4 bits sound format, 2 bits sound rate, 1 bit sound size, 1 bit sound type //8 bits sound data (always 10 because we aer using aac) @@ -76,9 +78,8 @@ class AacPacket() { configSend = true } else { header[1] = Type.RAW.mark - buffer = ByteArray(info.size - info.offset + header.size) - - byteBuffer.get(buffer, header.size, info.size - info.offset) + buffer = ByteArray(fixedBuffer.remaining() + header.size) + fixedBuffer.get(buffer, header.size, fixedBuffer.remaining()) } System.arraycopy(header, 0, buffer, 0, header.size) val ts = info.presentationTimeUs / 1000 diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/H264Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/H264Packet.kt index c63347af2..70b3777aa 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/H264Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/H264Packet.kt @@ -20,6 +20,7 @@ import android.media.MediaCodec import android.util.Log import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType +import com.pedro.rtmp.utils.removeInfo import java.nio.ByteBuffer import kotlin.experimental.and @@ -63,7 +64,7 @@ class H264Packet() { info: MediaCodec.BufferInfo, callback: (FlvPacket) -> Unit ) { - byteBuffer.rewind() + val fixedBuffer = byteBuffer.removeInfo(info) val ts = info.presentationTimeUs / 1000 //header is 5 bytes length: //4 bits FrameType, 4 bits CodecID @@ -94,10 +95,10 @@ class H264Packet() { callback(FlvPacket(buffer, ts, buffer.size, FlvType.VIDEO)) configSend = true } - val headerSize = getHeaderSize(byteBuffer) + val headerSize = getHeaderSize(fixedBuffer) if (headerSize == 0) return //invalid buffer or waiting for sps/pps - byteBuffer.rewind() - val validBuffer = removeHeader(byteBuffer, headerSize) + fixedBuffer.rewind() + val validBuffer = removeHeader(fixedBuffer, headerSize) val size = validBuffer.remaining() buffer = ByteArray(header.size + size + naluSize) diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt index be2f6d25d..6d710b582 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt @@ -20,6 +20,7 @@ import android.media.MediaCodec import android.util.Log import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType +import com.pedro.rtmp.utils.removeInfo import java.nio.ByteBuffer /** @@ -62,7 +63,7 @@ class H265Packet { info: MediaCodec.BufferInfo, callback: (FlvPacket) -> Unit ) { - byteBuffer.rewind() + val fixedBuffer = byteBuffer.removeInfo(info) val ts = info.presentationTimeUs / 1000 //header is 8 bytes length: //mark first byte as extended header (0b10000000) @@ -100,10 +101,10 @@ class H265Packet { callback(FlvPacket(buffer, ts, buffer.size, FlvType.VIDEO)) configSend = true } - val headerSize = getHeaderSize(byteBuffer) + val headerSize = getHeaderSize(fixedBuffer) if (headerSize == 0) return //invalid buffer or waiting for sps/pps - byteBuffer.rewind() - val validBuffer = removeHeader(byteBuffer, headerSize) + fixedBuffer.rewind() + val validBuffer = removeHeader(fixedBuffer, headerSize) val size = validBuffer.remaining() buffer = ByteArray(header.size + size + naluSize) diff --git a/rtmp/src/main/java/com/pedro/rtmp/utils/Utils.kt b/rtmp/src/main/java/com/pedro/rtmp/utils/Utils.kt index 057e5e3b5..7f0b1cce4 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/utils/Utils.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/utils/Utils.kt @@ -16,6 +16,7 @@ package com.pedro.rtmp.utils +import android.media.MediaCodec import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.io.InputStream @@ -27,6 +28,14 @@ import java.util.concurrent.BlockingQueue * Created by pedro on 20/04/21. */ +fun ByteBuffer.removeInfo(info: MediaCodec.BufferInfo): ByteBuffer { + try { + position(info.offset) + limit(info.size) + } catch (ignored: Exception) { } + return slice() +} + inline infix fun BlockingQueue.trySend(item: T): Boolean { return try { this.add(item) diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt index 7ac672890..fc6690342 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt @@ -19,6 +19,7 @@ package com.pedro.rtsp.rtp.packets import android.media.MediaCodec import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants +import com.pedro.rtsp.utils.removeInfo import java.nio.ByteBuffer import kotlin.experimental.and import kotlin.experimental.or @@ -44,10 +45,11 @@ class AacPacket( bufferInfo: MediaCodec.BufferInfo, callback: (RtpFrame) -> Unit ) { - val length = bufferInfo.size - byteBuffer.position() + val fixedBuffer = byteBuffer.removeInfo(bufferInfo) + val length = fixedBuffer.remaining() if (length > 0) { val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 4) - byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 4, length) + fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 4, length) val ts = bufferInfo.presentationTimeUs * 1000 markPacket(buffer) val rtpTs = updateTimeStamp(buffer, ts) diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index a0c4c5d72..9871522e4 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -21,6 +21,7 @@ import android.util.Log import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants import com.pedro.rtsp.utils.getVideoStartCodeSize +import com.pedro.rtsp.utils.removeInfo import java.nio.ByteBuffer import kotlin.experimental.and @@ -51,15 +52,15 @@ class H264Packet( bufferInfo: MediaCodec.BufferInfo, callback: (RtpFrame) -> Unit ) { + val fixedBuffer = byteBuffer.removeInfo(bufferInfo) // We read a NAL units from ByteBuffer and we send them // NAL units are preceded with 0x00000001 - byteBuffer.rewind() - val header = ByteArray(getHeaderSize(byteBuffer) + 1) + val header = ByteArray(getHeaderSize(fixedBuffer) + 1) if (header.size == 1) return //invalid buffer or waiting for sps/pps - byteBuffer.rewind() - byteBuffer.get(header, 0, header.size) + fixedBuffer.rewind() + fixedBuffer.get(header, 0, header.size) val ts = bufferInfo.presentationTimeUs * 1000L - val naluLength = bufferInfo.size - byteBuffer.position() + val naluLength = fixedBuffer.remaining() val type: Int = (header[header.size - 1] and 0x1F).toInt() if (type == RtpConstants.IDR || bufferInfo.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) { stapA?.let { @@ -80,7 +81,7 @@ class H264Packet( if (naluLength <= maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 1) { val buffer = getBuffer(naluLength + RtpConstants.RTP_HEADER_LENGTH + 1) buffer[RtpConstants.RTP_HEADER_LENGTH] = header[header.size - 1] - byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 1, naluLength) + fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 1, naluLength) val rtpTs = updateTimeStamp(buffer, ts) markPacket(buffer) //mark end frame updateSeq(buffer) @@ -98,13 +99,13 @@ class H264Packet( val length = if (naluLength - sum > maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2) { maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2 } else { - bufferInfo.size - byteBuffer.position() + fixedBuffer.remaining() } val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 2) buffer[RtpConstants.RTP_HEADER_LENGTH] = header[0] buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = header[1] val rtpTs = updateTimeStamp(buffer, ts) - byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, length) + fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, length) sum += length // Last packet before next NAL if (sum >= naluLength) { diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt index 08aa78943..59bffa31e 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt @@ -20,6 +20,7 @@ import android.media.MediaCodec import android.util.Log import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants +import com.pedro.rtsp.utils.removeInfo import java.nio.ByteBuffer import kotlin.experimental.and @@ -51,12 +52,12 @@ class H265Packet( bufferInfo: MediaCodec.BufferInfo, callback: (RtpFrame) -> Unit ) { + val fixedBuffer = byteBuffer.removeInfo(bufferInfo) // We read a NAL units from ByteBuffer and we send them // NAL units are preceded with 0x00000001 - byteBuffer.rewind() - byteBuffer.get(header, 0, 6) + fixedBuffer.get(header, 0, 6) val ts = bufferInfo.presentationTimeUs * 1000L - val naluLength = bufferInfo.size - byteBuffer.position() + val naluLength = fixedBuffer.remaining() val type: Int = header[4].toInt().shr(1 and 0x3f) if (type == RtpConstants.IDR_N_LP || type == RtpConstants.IDR_W_DLP || bufferInfo.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) { stapA?.let { @@ -79,7 +80,7 @@ class H265Packet( //Set PayloadHdr (exact copy of nal unit header) buffer[RtpConstants.RTP_HEADER_LENGTH] = header[4] buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = header[5] - byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, naluLength) + fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, naluLength) val rtpTs = updateTimeStamp(buffer, ts) markPacket(buffer) //mark end frame updateSeq(buffer) @@ -102,14 +103,14 @@ class H265Packet( val length = if (naluLength - sum > maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 3) { maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 3 } else { - bufferInfo.size - byteBuffer.position() + fixedBuffer.remaining() } val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 3) buffer[RtpConstants.RTP_HEADER_LENGTH] = header[0] buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = header[1] buffer[RtpConstants.RTP_HEADER_LENGTH + 2] = header[2] val rtpTs = updateTimeStamp(buffer, ts) - byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 3, length) + fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 3, length) sum += length // Last packet before next NAL if (sum >= naluLength) { diff --git a/rtsp/src/main/java/com/pedro/rtsp/utils/Extensions.kt b/rtsp/src/main/java/com/pedro/rtsp/utils/Extensions.kt index 8a85b79ea..3f5e14f51 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/utils/Extensions.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/utils/Extensions.kt @@ -16,12 +16,21 @@ package com.pedro.rtsp.utils +import android.media.MediaCodec import android.util.Base64 import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.nio.ByteBuffer import java.util.concurrent.BlockingQueue +fun ByteBuffer.removeInfo(info: MediaCodec.BufferInfo): ByteBuffer { + try { + position(info.offset) + limit(info.size) + } catch (ignored: Exception) { } + return slice() +} + inline infix fun BlockingQueue.trySend(item: T): Boolean { return try { this.add(item) diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt index 170f91ee3..bb0fa9307 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt @@ -23,6 +23,7 @@ import com.pedro.srt.mpeg2ts.Pes import com.pedro.srt.mpeg2ts.PesType import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.srt.srt.packets.data.PacketPosition +import com.pedro.srt.utils.removeInfo import java.nio.ByteBuffer /** @@ -42,13 +43,13 @@ class AacPacket( info: MediaCodec.BufferInfo, callback: (List) -> Unit ) { - val length = info.size + val fixedBuffer = byteBuffer.removeInfo(info) + val length = fixedBuffer.remaining() if (length < 0) return - byteBuffer.rewind() val payload = ByteArray(length + header.size) writeAdts(payload, payload.size, 0) - byteBuffer.get(payload, header.size, length) + fixedBuffer.get(payload, header.size, length) val pes = Pes(psiManager.getAudioPid().toInt(), false, PesType.AUDIO, info.presentationTimeUs, ByteBuffer.wrap(payload)) val mpeg2tsPackets = mpegTsPacketizer.write(listOf(pes)) diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt index d4a754f54..1bdd3532d 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt @@ -26,6 +26,7 @@ import com.pedro.srt.mpeg2ts.Pes import com.pedro.srt.mpeg2ts.PesType import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.srt.srt.packets.data.PacketPosition +import com.pedro.srt.utils.removeInfo import com.pedro.srt.utils.startWith import com.pedro.srt.utils.toByteArray import java.nio.ByteBuffer @@ -53,7 +54,8 @@ class H26XPacket( info: MediaCodec.BufferInfo, callback: (List) -> Unit ) { - val length = info.size + val fixedBuffer = byteBuffer.removeInfo(info) + val length = fixedBuffer.remaining() if (length < 0) return val isKeyFrame = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME @@ -77,8 +79,7 @@ class H26XPacket( return } } - byteBuffer.rewind() - val validBuffer = fixHeader(byteBuffer, isKeyFrame) + val validBuffer = fixHeader(fixedBuffer, isKeyFrame) val payload = ByteArray(validBuffer.remaining()) validBuffer.get(payload, 0, validBuffer.remaining()) diff --git a/srt/src/main/java/com/pedro/srt/utils/Extensions.kt b/srt/src/main/java/com/pedro/srt/utils/Extensions.kt index 51c290293..beaa619b1 100644 --- a/srt/src/main/java/com/pedro/srt/utils/Extensions.kt +++ b/srt/src/main/java/com/pedro/srt/utils/Extensions.kt @@ -16,12 +16,20 @@ package com.pedro.srt.utils +import android.media.MediaCodec import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.io.InputStream import java.io.OutputStream import java.nio.ByteBuffer import java.util.concurrent.BlockingQueue +import kotlin.math.min + +fun ByteBuffer.removeInfo(info: MediaCodec.BufferInfo): ByteBuffer { + if (info.offset >= 0) position(min(capacity(), info.offset)) + limit(min(capacity(), info.size)) + return slice() +} inline infix fun BlockingQueue.trySend(item: T): Boolean { return try { diff --git a/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt b/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt new file mode 100644 index 000000000..5c5bcc718 --- /dev/null +++ b/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt @@ -0,0 +1,29 @@ +package com.pedro.srt.utils + +import android.media.MediaCodec +import org.junit.Assert.assertEquals +import org.junit.Test +import java.nio.ByteBuffer + +/** + * Created by pedro on 15/11/23. + */ +class ExtensionTest { + + @Test + fun `remove info`() { + val buffer = ByteBuffer.wrap(ByteArray(256) { 0x00 }.mapIndexed { index, byte -> index.toByte() }.toByteArray()) + val info = MediaCodec.BufferInfo() + val offset = 4 + val minusLimit = 2 + info.presentationTimeUs = 0 + info.offset = offset + info.size = buffer.remaining() - minusLimit + info.flags = 0 + + val result = buffer.removeInfo(info) + assertEquals(buffer.remaining() - offset - minusLimit, result.remaining()) + assertEquals(offset.toByte(), result.get(0)) + assertEquals((buffer.remaining() - 1 - minusLimit).toByte(), result.get(result.remaining() - 1)) + } +} \ No newline at end of file From 8c9bd4a75a652302e2205e6404cc1068882589a1 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 15 Nov 2023 08:50:04 +0100 Subject: [PATCH 2/2] fix test --- srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt b/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt index 5c5bcc718..15005cc67 100644 --- a/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt +++ b/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt @@ -22,8 +22,8 @@ class ExtensionTest { info.flags = 0 val result = buffer.removeInfo(info) - assertEquals(buffer.remaining() - offset - minusLimit, result.remaining()) + assertEquals(buffer.capacity() - offset - minusLimit, result.remaining()) assertEquals(offset.toByte(), result.get(0)) - assertEquals((buffer.remaining() - 1 - minusLimit).toByte(), result.get(result.remaining() - 1)) + assertEquals((buffer.capacity() - 1 - minusLimit).toByte(), result.get(result.remaining() - 1)) } } \ No newline at end of file