Merge pull request #1344 from pedroSG94/fix/bufferinfo

fix offset and size of buffers depend of MediaCodec.BufferInfo
pedroSG94 · Nov 15, 2023 · fdd495e · fdd495e
2 parents 9d766ef + 8c9bd4a
commit fdd495e
Show file tree

Hide file tree

Showing 12 changed files with 97 additions and 33 deletions.
diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AacPacket.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AacPacket.kt
@@ -19,6 +19,7 @@ package com.pedro.rtmp.flv.audio
 import android.media.MediaCodec
 import com.pedro.rtmp.flv.FlvPacket
 import com.pedro.rtmp.flv.FlvType
+import com.pedro.rtmp.utils.removeInfo
 import java.nio.ByteBuffer
 import kotlin.experimental.or
 
@@ -53,6 +54,7 @@ class AacPacket() {
     info: MediaCodec.BufferInfo,
     callback: (FlvPacket) -> Unit
   ) {
+    val fixedBuffer = byteBuffer.removeInfo(info)
     //header is 2 bytes length
     //4 bits sound format, 2 bits sound rate, 1 bit sound size, 1 bit sound type
     //8 bits sound data (always 10 because we aer using aac)
@@ -76,9 +78,8 @@ class AacPacket() {
       configSend = true
     } else {
       header[1] = Type.RAW.mark
-      buffer = ByteArray(info.size - info.offset + header.size)
-
-      byteBuffer.get(buffer, header.size, info.size - info.offset)
+      buffer = ByteArray(fixedBuffer.remaining() + header.size)
+      fixedBuffer.get(buffer, header.size, fixedBuffer.remaining())
     }
     System.arraycopy(header, 0, buffer, 0, header.size)
     val ts = info.presentationTimeUs / 1000

diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/H264Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/H264Packet.kt
@@ -20,6 +20,7 @@ import android.media.MediaCodec
 import android.util.Log
 import com.pedro.rtmp.flv.FlvPacket
 import com.pedro.rtmp.flv.FlvType
+import com.pedro.rtmp.utils.removeInfo
 import java.nio.ByteBuffer
 import kotlin.experimental.and
 
@@ -63,7 +64,7 @@ class H264Packet() {
     info: MediaCodec.BufferInfo,
     callback: (FlvPacket) -> Unit
   ) {
-    byteBuffer.rewind()
+    val fixedBuffer = byteBuffer.removeInfo(info)
     val ts = info.presentationTimeUs / 1000
     //header is 5 bytes length:
     //4 bits FrameType, 4 bits CodecID
@@ -94,10 +95,10 @@ class H264Packet() {
       callback(FlvPacket(buffer, ts, buffer.size, FlvType.VIDEO))
       configSend = true
     }
-    val headerSize = getHeaderSize(byteBuffer)
+    val headerSize = getHeaderSize(fixedBuffer)
     if (headerSize == 0) return //invalid buffer or waiting for sps/pps
-    byteBuffer.rewind()
-    val validBuffer = removeHeader(byteBuffer, headerSize)
+    fixedBuffer.rewind()
+    val validBuffer = removeHeader(fixedBuffer, headerSize)
     val size = validBuffer.remaining()
     buffer = ByteArray(header.size + size + naluSize)
 

diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt
@@ -20,6 +20,7 @@ import android.media.MediaCodec
 import android.util.Log
 import com.pedro.rtmp.flv.FlvPacket
 import com.pedro.rtmp.flv.FlvType
+import com.pedro.rtmp.utils.removeInfo
 import java.nio.ByteBuffer
 
 /**
@@ -62,7 +63,7 @@ class H265Packet {
     info: MediaCodec.BufferInfo,
     callback: (FlvPacket) -> Unit
   ) {
-    byteBuffer.rewind()
+    val fixedBuffer = byteBuffer.removeInfo(info)
     val ts = info.presentationTimeUs / 1000
     //header is 8 bytes length:
     //mark first byte as extended header (0b10000000)
@@ -100,10 +101,10 @@ class H265Packet {
       callback(FlvPacket(buffer, ts, buffer.size, FlvType.VIDEO))
       configSend = true
     }
-    val headerSize = getHeaderSize(byteBuffer)
+    val headerSize = getHeaderSize(fixedBuffer)
     if (headerSize == 0) return //invalid buffer or waiting for sps/pps
-    byteBuffer.rewind()
-    val validBuffer = removeHeader(byteBuffer, headerSize)
+    fixedBuffer.rewind()
+    val validBuffer = removeHeader(fixedBuffer, headerSize)
     val size = validBuffer.remaining()
     buffer = ByteArray(header.size + size + naluSize)
 

diff --git a/rtmp/src/main/java/com/pedro/rtmp/utils/Utils.kt b/rtmp/src/main/java/com/pedro/rtmp/utils/Utils.kt
@@ -16,6 +16,7 @@
 
 package com.pedro.rtmp.utils
 
+import android.media.MediaCodec
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.io.InputStream
@@ -27,6 +28,14 @@ import java.util.concurrent.BlockingQueue
  * Created by pedro on 20/04/21.
  */
 
+fun ByteBuffer.removeInfo(info: MediaCodec.BufferInfo): ByteBuffer {
+  try {
+    position(info.offset)
+    limit(info.size)
+  } catch (ignored: Exception) { }
+  return slice()
+}
+
 inline infix fun <reified T: Any> BlockingQueue<T>.trySend(item: T): Boolean {
   return try {
     this.add(item)

diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt
@@ -19,6 +19,7 @@ package com.pedro.rtsp.rtp.packets
 import android.media.MediaCodec
 import com.pedro.rtsp.rtsp.RtpFrame
 import com.pedro.rtsp.utils.RtpConstants
+import com.pedro.rtsp.utils.removeInfo
 import java.nio.ByteBuffer
 import kotlin.experimental.and
 import kotlin.experimental.or
@@ -44,10 +45,11 @@ class AacPacket(
     bufferInfo: MediaCodec.BufferInfo,
     callback: (RtpFrame) -> Unit
   ) {
-    val length = bufferInfo.size - byteBuffer.position()
+    val fixedBuffer = byteBuffer.removeInfo(bufferInfo)
+    val length = fixedBuffer.remaining()
     if (length > 0) {
       val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 4)
-      byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 4, length)
+      fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 4, length)
       val ts = bufferInfo.presentationTimeUs * 1000
       markPacket(buffer)
       val rtpTs = updateTimeStamp(buffer, ts)

diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt
@@ -21,6 +21,7 @@ import android.util.Log
 import com.pedro.rtsp.rtsp.RtpFrame
 import com.pedro.rtsp.utils.RtpConstants
 import com.pedro.rtsp.utils.getVideoStartCodeSize
+import com.pedro.rtsp.utils.removeInfo
 import java.nio.ByteBuffer
 import kotlin.experimental.and
 
@@ -51,15 +52,15 @@ class H264Packet(
     bufferInfo: MediaCodec.BufferInfo,
     callback: (RtpFrame) -> Unit
   ) {
+    val fixedBuffer = byteBuffer.removeInfo(bufferInfo)
     // We read a NAL units from ByteBuffer and we send them
     // NAL units are preceded with 0x00000001
-    byteBuffer.rewind()
-    val header = ByteArray(getHeaderSize(byteBuffer) + 1)
+    val header = ByteArray(getHeaderSize(fixedBuffer) + 1)
     if (header.size == 1) return //invalid buffer or waiting for sps/pps
-    byteBuffer.rewind()
-    byteBuffer.get(header, 0, header.size)
+    fixedBuffer.rewind()
+    fixedBuffer.get(header, 0, header.size)
     val ts = bufferInfo.presentationTimeUs * 1000L
-    val naluLength = bufferInfo.size - byteBuffer.position()
+    val naluLength = fixedBuffer.remaining()
     val type: Int = (header[header.size - 1] and 0x1F).toInt()
     if (type == RtpConstants.IDR || bufferInfo.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) {
       stapA?.let {
@@ -80,7 +81,7 @@ class H264Packet(
       if (naluLength <= maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 1) {
         val buffer = getBuffer(naluLength + RtpConstants.RTP_HEADER_LENGTH + 1)
         buffer[RtpConstants.RTP_HEADER_LENGTH] = header[header.size - 1]
-        byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 1, naluLength)
+        fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 1, naluLength)
         val rtpTs = updateTimeStamp(buffer, ts)
         markPacket(buffer) //mark end frame
         updateSeq(buffer)
@@ -98,13 +99,13 @@ class H264Packet(
           val length = if (naluLength - sum > maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2) {
             maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2
           } else {
-            bufferInfo.size - byteBuffer.position()
+            fixedBuffer.remaining()
           }
           val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 2)
           buffer[RtpConstants.RTP_HEADER_LENGTH] = header[0]
           buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = header[1]
           val rtpTs = updateTimeStamp(buffer, ts)
-          byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, length)
+          fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, length)
           sum += length
           // Last packet before next NAL
           if (sum >= naluLength) {

diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt
@@ -20,6 +20,7 @@ import android.media.MediaCodec
 import android.util.Log
 import com.pedro.rtsp.rtsp.RtpFrame
 import com.pedro.rtsp.utils.RtpConstants
+import com.pedro.rtsp.utils.removeInfo
 import java.nio.ByteBuffer
 import kotlin.experimental.and
 
@@ -51,12 +52,12 @@ class H265Packet(
     bufferInfo: MediaCodec.BufferInfo,
     callback: (RtpFrame) -> Unit
   ) {
+    val fixedBuffer = byteBuffer.removeInfo(bufferInfo)
     // We read a NAL units from ByteBuffer and we send them
     // NAL units are preceded with 0x00000001
-    byteBuffer.rewind()
-    byteBuffer.get(header, 0, 6)
+    fixedBuffer.get(header, 0, 6)
     val ts = bufferInfo.presentationTimeUs * 1000L
-    val naluLength = bufferInfo.size - byteBuffer.position()
+    val naluLength = fixedBuffer.remaining()
     val type: Int = header[4].toInt().shr(1 and 0x3f)
     if (type == RtpConstants.IDR_N_LP || type == RtpConstants.IDR_W_DLP || bufferInfo.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) {
       stapA?.let {
@@ -79,7 +80,7 @@ class H265Packet(
         //Set PayloadHdr (exact copy of nal unit header)
         buffer[RtpConstants.RTP_HEADER_LENGTH] = header[4]
         buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = header[5]
-        byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, naluLength)
+        fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, naluLength)
         val rtpTs = updateTimeStamp(buffer, ts)
         markPacket(buffer) //mark end frame
         updateSeq(buffer)
@@ -102,14 +103,14 @@ class H265Packet(
           val length = if (naluLength - sum > maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 3) {
             maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 3
           } else {
-            bufferInfo.size - byteBuffer.position()
+            fixedBuffer.remaining()
           }
           val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 3)
           buffer[RtpConstants.RTP_HEADER_LENGTH] = header[0]
           buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = header[1]
           buffer[RtpConstants.RTP_HEADER_LENGTH + 2] = header[2]
           val rtpTs = updateTimeStamp(buffer, ts)
-          byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 3, length)
+          fixedBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 3, length)
           sum += length
           // Last packet before next NAL
           if (sum >= naluLength) {

diff --git a/rtsp/src/main/java/com/pedro/rtsp/utils/Extensions.kt b/rtsp/src/main/java/com/pedro/rtsp/utils/Extensions.kt
@@ -16,12 +16,21 @@
 
 package com.pedro.rtsp.utils
 
+import android.media.MediaCodec
 import android.util.Base64
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.nio.ByteBuffer
 import java.util.concurrent.BlockingQueue
 
+fun ByteBuffer.removeInfo(info: MediaCodec.BufferInfo): ByteBuffer {
+  try {
+    position(info.offset)
+    limit(info.size)
+  } catch (ignored: Exception) { }
+  return slice()
+}
+
 inline infix fun <reified T: Any> BlockingQueue<T>.trySend(item: T): Boolean {
   return try {
     this.add(item)

diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt
@@ -23,6 +23,7 @@ import com.pedro.srt.mpeg2ts.Pes
 import com.pedro.srt.mpeg2ts.PesType
 import com.pedro.srt.mpeg2ts.psi.PsiManager
 import com.pedro.srt.srt.packets.data.PacketPosition
+import com.pedro.srt.utils.removeInfo
 import java.nio.ByteBuffer
 
 /**
@@ -42,13 +43,13 @@ class AacPacket(
     info: MediaCodec.BufferInfo,
     callback: (List<MpegTsPacket>) -> Unit
   ) {
-    val length = info.size
+    val fixedBuffer = byteBuffer.removeInfo(info)
+    val length = fixedBuffer.remaining()
     if (length < 0) return
-    byteBuffer.rewind()
 
     val payload = ByteArray(length + header.size)
     writeAdts(payload, payload.size, 0)
-    byteBuffer.get(payload, header.size, length)
+    fixedBuffer.get(payload, header.size, length)
 
     val pes = Pes(psiManager.getAudioPid().toInt(), false, PesType.AUDIO, info.presentationTimeUs, ByteBuffer.wrap(payload))
     val mpeg2tsPackets = mpegTsPacketizer.write(listOf(pes))

diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt
@@ -26,6 +26,7 @@ import com.pedro.srt.mpeg2ts.Pes
 import com.pedro.srt.mpeg2ts.PesType
 import com.pedro.srt.mpeg2ts.psi.PsiManager
 import com.pedro.srt.srt.packets.data.PacketPosition
+import com.pedro.srt.utils.removeInfo
 import com.pedro.srt.utils.startWith
 import com.pedro.srt.utils.toByteArray
 import java.nio.ByteBuffer
@@ -53,7 +54,8 @@ class H26XPacket(
     info: MediaCodec.BufferInfo,
     callback: (List<MpegTsPacket>) -> Unit
   ) {
-    val length = info.size
+    val fixedBuffer = byteBuffer.removeInfo(info)
+    val length = fixedBuffer.remaining()
     if (length < 0) return
     val isKeyFrame = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
       info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME
@@ -77,8 +79,7 @@ class H26XPacket(
         return
       }
     }
-    byteBuffer.rewind()
-    val validBuffer = fixHeader(byteBuffer, isKeyFrame)
+    val validBuffer = fixHeader(fixedBuffer, isKeyFrame)
     val payload = ByteArray(validBuffer.remaining())
     validBuffer.get(payload, 0, validBuffer.remaining())
 

diff --git a/srt/src/main/java/com/pedro/srt/utils/Extensions.kt b/srt/src/main/java/com/pedro/srt/utils/Extensions.kt
@@ -16,12 +16,20 @@
 
 package com.pedro.srt.utils
 
+import android.media.MediaCodec
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.io.InputStream
 import java.io.OutputStream
 import java.nio.ByteBuffer
 import java.util.concurrent.BlockingQueue
+import kotlin.math.min
+
+fun ByteBuffer.removeInfo(info: MediaCodec.BufferInfo): ByteBuffer {
+  if (info.offset >= 0) position(min(capacity(), info.offset))
+  limit(min(capacity(), info.size))
+  return slice()
+}
 
 inline infix fun <reified T: Any> BlockingQueue<T>.trySend(item: T): Boolean {
   return try {

diff --git a/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt b/srt/src/test/java/com/pedro/srt/utils/ExtensionTest.kt
@@ -0,0 +1,29 @@
+package com.pedro.srt.utils
+
+import android.media.MediaCodec
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import java.nio.ByteBuffer
+
+/**
+ * Created by pedro on 15/11/23.
+ */
+class ExtensionTest {
+
+  @Test
+  fun `remove info`() {
+    val buffer = ByteBuffer.wrap(ByteArray(256) { 0x00 }.mapIndexed { index, byte -> index.toByte()  }.toByteArray())
+    val info = MediaCodec.BufferInfo()
+    val offset = 4
+    val minusLimit = 2
+    info.presentationTimeUs = 0
+    info.offset = offset
+    info.size = buffer.remaining() - minusLimit
+    info.flags = 0
+
+    val result = buffer.removeInfo(info)
+    assertEquals(buffer.capacity() - offset - minusLimit, result.remaining())
+    assertEquals(offset.toByte(), result.get(0))
+    assertEquals((buffer.capacity() - 1 - minusLimit).toByte(), result.get(result.remaining() - 1))
+  }
+}