From e4b0f0d2cb4e77a8a72537b63e9227993c12c1ac Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 13:30:43 +0200 Subject: [PATCH 01/18] migrate sender to use mediaframe --- .../java/com/pedro/common/frame/MediaFrame.kt | 10 +++ .../com/pedro/common/frame/MediaFrameType.kt | 5 ++ .../java/com/pedro/rtmp/rtmp/RtmpClient.kt | 8 +- .../java/com/pedro/rtmp/rtmp/RtmpSender.kt | 80 ++++++++++--------- .../com/pedro/rtsp/rtp/packets/AacPacket.kt | 10 ++- .../com/pedro/rtsp/rtp/packets/BasePacket.kt | 6 +- .../com/pedro/rtsp/rtp/packets/G711Packet.kt | 10 ++- .../com/pedro/rtsp/rtp/packets/H264Packet.kt | 8 +- .../com/pedro/rtsp/rtp/packets/OpusPacket.kt | 10 ++- .../java/com/pedro/rtsp/rtsp/RtspClient.kt | 8 +- .../java/com/pedro/rtsp/rtsp/RtspSender.kt | 75 ++++++++++------- .../main/java/com/pedro/srt/srt/SrtClient.kt | 8 +- .../main/java/com/pedro/srt/srt/SrtSender.kt | 80 +++++++++++-------- udp/src/main/java/com/pedro/udp/UdpClient.kt | 8 +- udp/src/main/java/com/pedro/udp/UdpSender.kt | 80 +++++++++++-------- 15 files changed, 237 insertions(+), 169 deletions(-) create mode 100644 common/src/main/java/com/pedro/common/frame/MediaFrame.kt create mode 100644 common/src/main/java/com/pedro/common/frame/MediaFrameType.kt diff --git a/common/src/main/java/com/pedro/common/frame/MediaFrame.kt b/common/src/main/java/com/pedro/common/frame/MediaFrame.kt new file mode 100644 index 000000000..8c0ddaf2f --- /dev/null +++ b/common/src/main/java/com/pedro/common/frame/MediaFrame.kt @@ -0,0 +1,10 @@ +package com.pedro.common.frame + +import android.media.MediaCodec +import java.nio.ByteBuffer + +data class MediaFrame( + val data: ByteBuffer, + val info: MediaCodec.BufferInfo, + val type: MediaFrameType +) diff --git a/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt b/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt new file mode 100644 index 000000000..695f17bce --- /dev/null +++ b/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt @@ -0,0 +1,5 @@ +package com.pedro.common.frame + +enum class MediaFrameType { + VIDEO, AUDIO +} \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt index f72f17901..e853f5b3a 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt @@ -544,15 +544,15 @@ class RtmpClient(private val connectChecker: ConnectChecker) { commandsManager.reset() } - fun sendVideo(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - rtmpSender.sendVideoFrame(h264Buffer, info) + rtmpSender.sendVideoFrame(videoBuffer, info) } } - fun sendAudio(aacBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - rtmpSender.sendAudioFrame(aacBuffer, info) + rtmpSender.sendAudioFrame(audioBuffer, info) } } diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt index 25bc41c86..4932026ac 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt @@ -22,6 +22,8 @@ import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker import com.pedro.common.VideoCodec +import com.pedro.common.frame.MediaFrame +import com.pedro.common.frame.MediaFrameType import com.pedro.common.onMainThread import com.pedro.common.trySend import com.pedro.common.validMessage @@ -65,7 +67,7 @@ class RtmpSender( private var job: Job? = null private val scope = CoroutineScope(Dispatchers.IO) @Volatile - private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) + private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) private var audioFramesSent: Long = 0 private var videoFramesSent: Long = 0 var socket: RtmpSocket? = null @@ -81,60 +83,47 @@ class RtmpSender( } fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { - when (commandsManager.videoCodec) { + videoPacket = when (commandsManager.videoCodec) { VideoCodec.H265 -> { if (vps == null || pps == null) throw IllegalArgumentException("pps or vps can't be null with h265") - videoPacket = H265Packet() - (videoPacket as H265Packet).sendVideoInfo(sps, pps, vps) + H265Packet().apply { sendVideoInfo(sps, pps, vps) } } VideoCodec.AV1 -> { - videoPacket = Av1Packet() - (videoPacket as Av1Packet).sendVideoInfo(sps) + Av1Packet().apply { sendVideoInfo(sps) } } else -> { if (pps == null) throw IllegalArgumentException("pps can't be null with h264") - videoPacket = H264Packet() - (videoPacket as H264Packet).sendVideoInfo(sps, pps) + H264Packet().apply { sendVideoInfo(sps, pps) } } } } fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { - when (commandsManager.audioCodec) { - AudioCodec.G711 -> { - audioPacket = G711Packet() - (audioPacket as G711Packet).sendAudioInfo() - } - AudioCodec.AAC -> { - audioPacket = AacPacket() - (audioPacket as AacPacket).sendAudioInfo(sampleRate, isStereo) - } + audioPacket = when (commandsManager.audioCodec) { + AudioCodec.G711 -> G711Packet().apply { sendAudioInfo() } + AudioCodec.AAC -> AacPacket().apply { sendAudioInfo(sampleRate, isStereo) } AudioCodec.OPUS -> { throw IllegalArgumentException("Unsupported codec: ${commandsManager.audioCodec.name}") } } } - fun sendVideoFrame(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - videoPacket.createFlvPacket(h264Buffer, info) { flvPacket -> - val result = queue.trySend(flvPacket) - if (!result) { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } + val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrameType.VIDEO)) + if (!result) { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ } } } - fun sendAudioFrame(aacBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - audioPacket.createFlvPacket(aacBuffer, info) { flvPacket -> - val result = queue.trySend(flvPacket) - if (!result) { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ - } + val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrameType.AUDIO)) + if (!result) { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ } } } @@ -155,12 +144,9 @@ class RtmpSender( } while (scope.isActive && running) { val error = runCatching { - val flvPacket = runInterruptible { - queue.poll(1, TimeUnit.SECONDS) - } - if (flvPacket == null) { - Log.i(TAG, "Skipping iteration, frame null") - } else { + val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } + val flvPacket = getFlvPacket(mediaFrame) + flvPacket?.let { var size = 0 if (flvPacket.type == FlvType.VIDEO) { videoFramesSent++ @@ -206,6 +192,24 @@ class RtmpSender( queue.clear() } + private fun getFlvPacket(mediaFrame: MediaFrame?): FlvPacket? { + if (mediaFrame == null) return null + var flvPacket: FlvPacket? = null + when (mediaFrame.type) { + MediaFrameType.VIDEO -> { + videoPacket.createFlvPacket(mediaFrame.data, mediaFrame.info) { packet -> + flvPacket = packet + } + } + MediaFrameType.AUDIO -> { + audioPacket.createFlvPacket(mediaFrame.data, mediaFrame.info) { packet -> + flvPacket = packet + } + } + } + return flvPacket + } + @Throws(IllegalArgumentException::class) fun hasCongestion(percentUsed: Float = 20f): Boolean { if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") @@ -219,7 +223,7 @@ class RtmpSender( if (newSize < queue.size - queue.remainingCapacity()) { throw RuntimeException("Can't fit current cache inside new cache size") } - val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) + val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) queue.drainTo(tempQueue) queue = tempQueue } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt index 8c427629c..0941577a0 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt @@ -29,10 +29,8 @@ import kotlin.experimental.or * * RFC 3640. */ -class AacPacket( - sampleRate: Int -): BasePacket( - sampleRate.toLong(), +class AacPacket: BasePacket( + 0, RtpConstants.payloadType + RtpConstants.trackAudio ) { @@ -40,6 +38,10 @@ class AacPacket( channelIdentifier = RtpConstants.trackAudio } + fun setAudioInfo(sampleRate: Int) { + setClock(sampleRate.toLong()) + } + override fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaCodec.BufferInfo, diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt index a932a8faf..fd03d6905 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt @@ -27,7 +27,7 @@ import kotlin.experimental.or /** * Created by pedro on 27/11/18. */ -abstract class BasePacket(private val clock: Long, private val payloadType: Int) { +abstract class BasePacket(private var clock: Long, private val payloadType: Int) { protected var channelIdentifier: Int = 0 private var seq = 0L @@ -50,6 +50,10 @@ abstract class BasePacket(private val clock: Long, private val payloadType: Int) this.ssrc = ssrc } + protected fun setClock(clock: Long) { + this.clock = clock + } + protected fun getBuffer(size: Int): ByteArray { val buffer = ByteArray(size) buffer[0] = 0x80.toByte() diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt index 8bd562fb5..14ffcc1b4 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt @@ -26,10 +26,8 @@ import java.nio.ByteBuffer * RFC 7655. * Valid for G711A and G711U */ -class G711Packet( - sampleRate: Int -): BasePacket( - sampleRate.toLong(), +class G711Packet: BasePacket( + 0, RtpConstants.payloadTypeG711 ) { @@ -37,6 +35,10 @@ class G711Packet( channelIdentifier = RtpConstants.trackAudio } + fun setAudioInfo(sampleRate: Int) { + setClock(sampleRate.toLong()) + } + override fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaCodec.BufferInfo, diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index 7214a15a8..cd0a80c19 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -31,10 +31,7 @@ import kotlin.experimental.and * * RFC 3984 */ -class H264Packet( - sps: ByteArray, - pps: ByteArray -): BasePacket(RtpConstants.clockVideoFrequency, +class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, RtpConstants.payloadType + RtpConstants.trackVideo ) { @@ -45,6 +42,9 @@ class H264Packet( init { channelIdentifier = RtpConstants.trackVideo + } + + fun sendVideoInfo(sps: ByteArray, pps: ByteArray) { setSpsPps(sps, pps) } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt index 98103ca4e..757c728e7 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt @@ -26,10 +26,8 @@ import java.nio.ByteBuffer * * RFC 7587. */ -class OpusPacket( - sampleRate: Int -): BasePacket( - sampleRate.toLong(), +class OpusPacket: BasePacket( + 0, RtpConstants.payloadType + RtpConstants.trackAudio ) { @@ -37,6 +35,10 @@ class OpusPacket( channelIdentifier = RtpConstants.trackAudio } + fun setAudioInfo(sampleRate: Int) { + setClock(sampleRate.toLong()) + } + override fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaCodec.BufferInfo, diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt index 34c90db3f..0d1d36436 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt @@ -427,15 +427,15 @@ class RtspClient(private val connectChecker: ConnectChecker) { scope = CoroutineScope(Dispatchers.IO) } - fun sendVideo(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - rtspSender.sendVideoFrame(h264Buffer, info) + rtspSender.sendVideoFrame(videoBuffer, info) } } - fun sendAudio(aacBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - rtspSender.sendAudioFrame(aacBuffer, info) + rtspSender.sendAudioFrame(audioBuffer, info) } } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt index c365b160f..563a0cc29 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt @@ -21,7 +21,9 @@ import android.util.Log import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker +import com.pedro.common.frame.MediaFrame import com.pedro.common.VideoCodec +import com.pedro.common.frame.MediaFrameType import com.pedro.common.onMainThread import com.pedro.common.socket.TcpStreamSocket import com.pedro.common.trySend @@ -54,8 +56,8 @@ class RtspSender( private val commandsManager: CommandsManager ) { - private var videoPacket: BasePacket? = null - private var audioPacket: BasePacket? = null + private var videoPacket: BasePacket = H264Packet() + private var audioPacket: BasePacket = AacPacket() private var rtpSocket: BaseRtpSocket? = null private var baseSenderReport: BaseSenderReport? = null @@ -66,7 +68,7 @@ class RtspSender( private var job: Job? = null private val scope = CoroutineScope(Dispatchers.IO) @Volatile - private var queue: BlockingQueue> = LinkedBlockingQueue(cacheSize) + private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) private var audioFramesSent: Long = 0 private var videoFramesSent: Long = 0 @@ -95,7 +97,7 @@ class RtspSender( videoPacket = when (commandsManager.videoCodec) { VideoCodec.H264 -> { if (pps == null) throw IllegalArgumentException("pps can't be null with h264") - H264Packet(sps, pps) + H264Packet().apply { sendVideoInfo(sps, pps) } } VideoCodec.H265 -> { if (vps == null || pps == null) throw IllegalArgumentException("pps or vps can't be null with h265") @@ -107,9 +109,9 @@ class RtspSender( fun setAudioInfo(sampleRate: Int) { audioPacket = when (commandsManager.audioCodec) { - AudioCodec.G711 -> G711Packet(sampleRate) - AudioCodec.AAC -> AacPacket(sampleRate) - AudioCodec.OPUS -> OpusPacket(sampleRate) + AudioCodec.G711 -> G711Packet().apply { setAudioInfo(sampleRate) } + AudioCodec.AAC -> AacPacket().apply { setAudioInfo(sampleRate) } + AudioCodec.OPUS -> OpusPacket().apply { setAudioInfo(sampleRate) } } } @@ -119,26 +121,22 @@ class RtspSender( baseSenderReport?.setSocket(socket) } - fun sendVideoFrame(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - videoPacket?.createAndSendPacket(h264Buffer, info) { rtpFrame -> - val result = queue.trySend(rtpFrame) - if (!result) { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } + val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrameType.VIDEO)) + if (!result) { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ } } } - fun sendAudioFrame(aacBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - audioPacket?.createAndSendPacket(aacBuffer, info) { rtpFrame -> - val result = queue.trySend(rtpFrame) - if (!result) { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ - } + val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrameType.AUDIO)) + if (!result) { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ } } } @@ -149,8 +147,8 @@ class RtspSender( val ssrcVideo = Random().nextInt().toLong() val ssrcAudio = Random().nextInt().toLong() baseSenderReport?.setSSRC(ssrcVideo, ssrcAudio) - videoPacket?.setSSRC(ssrcVideo) - audioPacket?.setSSRC(ssrcAudio) + videoPacket.setSSRC(ssrcVideo) + audioPacket.setSSRC(ssrcAudio) running = true job = scope.launch { val isTcp = rtpSocket is RtpSocketTcp @@ -165,12 +163,11 @@ class RtspSender( } while (scope.isActive && running) { val error = runCatching { - val frames = runInterruptible { - queue.poll(1, TimeUnit.SECONDS) - } + val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } + val rtpFrames = getRtpPackets(mediaFrame) var size = 0 var isVideo = false - frames?.forEach { rtpFrame -> + rtpFrames?.forEach { rtpFrame -> rtpSocket?.sendFrame(rtpFrame) //4 is tcp header length val packetSize = if (isTcp) rtpFrame.length + 4 else rtpFrame.length @@ -211,8 +208,8 @@ class RtspSender( baseSenderReport?.reset() baseSenderReport?.close() rtpSocket?.close() - audioPacket?.reset() - videoPacket?.reset() + audioPacket.reset() + videoPacket.reset() resetSentAudioFrames() resetSentVideoFrames() resetDroppedAudioFrames() @@ -222,6 +219,24 @@ class RtspSender( queue.clear() } + private fun getRtpPackets(mediaFrame: MediaFrame?): List? { + if (mediaFrame == null) return null + var rtpPackets: List? = null + when (mediaFrame.type) { + MediaFrameType.VIDEO -> { + videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> + rtpPackets = packets + } + } + MediaFrameType.AUDIO -> { + audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> + rtpPackets = packets + } + } + } + return rtpPackets + } + @Throws(IllegalArgumentException::class) fun hasCongestion(percentUsed: Float = 20f): Boolean { if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") @@ -235,7 +250,7 @@ class RtspSender( if (newSize < queue.size - queue.remainingCapacity()) { throw RuntimeException("Can't fit current cache inside new cache size") } - val tempQueue: BlockingQueue> = LinkedBlockingQueue(newSize) + val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) queue.drainTo(tempQueue) queue = tempQueue } diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt b/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt index 3c852162e..025b35d1a 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt @@ -391,15 +391,15 @@ class SrtClient(private val connectChecker: ConnectChecker) { srtSender.setVideoInfo(sps, pps, vps) } - fun sendVideo(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - srtSender.sendVideoFrame(h264Buffer, info) + srtSender.sendVideoFrame(videoBuffer, info) } } - fun sendAudio(aacBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - srtSender.sendAudioFrame(aacBuffer, info) + srtSender.sendAudioFrame(audioBuffer, info) } } diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt index 023328ea9..181e4c6de 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt @@ -21,6 +21,8 @@ import android.util.Log import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker +import com.pedro.common.frame.MediaFrame +import com.pedro.common.frame.MediaFrameType import com.pedro.common.onMainThread import com.pedro.common.trySend import com.pedro.common.validMessage @@ -67,7 +69,7 @@ class SrtSender( private val limitSize = commandsManager.MTU - SrtPacket.headerSize private val mpegTsPacketizer = MpegTsPacketizer(psiManager) private var audioPacket: BasePacket = AacPacket(limitSize, psiManager) - private val h26XPacket = H26XPacket(limitSize, psiManager) + private val videoPacket = H26XPacket(limitSize, psiManager) @Volatile private var running = false @@ -76,7 +78,7 @@ class SrtSender( private var job: Job? = null private val scope = CoroutineScope(Dispatchers.IO) @Volatile - private var queue: BlockingQueue> = LinkedBlockingQueue(cacheSize) + private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) private var audioFramesSent: Long = 0 private var videoFramesSent: Long = 0 var socket: SrtSocket? = null @@ -102,45 +104,36 @@ class SrtSender( } fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { - h26XPacket.setVideoCodec(commandsManager.videoCodec.toCodec()) - h26XPacket.sendVideoInfo(sps, pps, vps) + videoPacket.setVideoCodec(commandsManager.videoCodec.toCodec()) + videoPacket.sendVideoInfo(sps, pps, vps) } fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { - when (commandsManager.audioCodec) { - AudioCodec.AAC -> { - audioPacket = AacPacket(limitSize, psiManager) - (audioPacket as? AacPacket)?.sendAudioInfo(sampleRate, isStereo) - } - AudioCodec.OPUS -> { - audioPacket = OpusPacket(limitSize, psiManager) - } + audioPacket = when (commandsManager.audioCodec) { + AudioCodec.AAC -> AacPacket(limitSize, psiManager).apply { sendAudioInfo(sampleRate, isStereo) } + AudioCodec.OPUS -> OpusPacket(limitSize, psiManager) AudioCodec.G711 -> { throw IllegalArgumentException("Unsupported codec: ${commandsManager.audioCodec.name}") } } } - fun sendVideoFrame(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - h26XPacket.createAndSendPacket(h264Buffer, info) { mpegTsPackets -> - val result = queue.trySend(mpegTsPackets) - if (!result) { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } + val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrameType.VIDEO)) + if (!result) { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ } } } - fun sendAudioFrame(aacBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - audioPacket.createAndSendPacket(aacBuffer, info) { mpegTsPackets -> - val result = queue.trySend(mpegTsPackets) - if (!result) { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ - } + val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrameType.AUDIO)) + if (!result) { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ } } } @@ -162,13 +155,14 @@ class SrtSender( } while (scope.isActive && running) { val error = runCatching { - val mpegTsPackets = runInterruptible { - queue.poll(1, TimeUnit.SECONDS) + val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } + val mpegTsPackets = getMpegTsPackets(mediaFrame) + mpegTsPackets?.let { + val isKey = mpegTsPackets[0].isKey + val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) + bytesSend += sendPackets(psiPackets) + bytesSend += sendPackets(mpegTsPackets) } - val isKey = mpegTsPackets[0].isKey - val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) - bytesSend += sendPackets(psiPackets) - bytesSend += sendPackets(mpegTsPackets) }.exceptionOrNull() if (error != null) { onMainThread { @@ -200,7 +194,7 @@ class SrtSender( service.clear() mpegTsPacketizer.reset() audioPacket.reset(clear) - h26XPacket.reset(clear) + videoPacket.reset(clear) resetSentAudioFrames() resetSentVideoFrames() resetDroppedAudioFrames() @@ -210,6 +204,24 @@ class SrtSender( queue.clear() } + private fun getMpegTsPackets(mediaFrame: MediaFrame?): List? { + if (mediaFrame == null) return null + var mpegTsPackets: List? = null + when (mediaFrame.type) { + MediaFrameType.VIDEO -> { + videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> + mpegTsPackets = packets + } + } + MediaFrameType.AUDIO -> { + audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> + mpegTsPackets = packets + } + } + } + return mpegTsPackets + } + @Throws(IllegalArgumentException::class) fun hasCongestion(percentUsed: Float = 20f): Boolean { if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") @@ -223,7 +235,7 @@ class SrtSender( if (newSize < queue.size - queue.remainingCapacity()) { throw RuntimeException("Can't fit current cache inside new cache size") } - val tempQueue: BlockingQueue> = LinkedBlockingQueue(newSize) + val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) queue.drainTo(tempQueue) queue = tempQueue } diff --git a/udp/src/main/java/com/pedro/udp/UdpClient.kt b/udp/src/main/java/com/pedro/udp/UdpClient.kt index 5b52faa03..88783af0d 100644 --- a/udp/src/main/java/com/pedro/udp/UdpClient.kt +++ b/udp/src/main/java/com/pedro/udp/UdpClient.kt @@ -229,15 +229,15 @@ class UdpClient(private val connectChecker: ConnectChecker) { udpSender.setVideoInfo(sps, pps, vps) } - fun sendVideo(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandManager.videoDisabled) { - udpSender.sendVideoFrame(h264Buffer, info) + udpSender.sendVideoFrame(videoBuffer, info) } } - fun sendAudio(aacBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandManager.audioDisabled) { - udpSender.sendAudioFrame(aacBuffer, info) + udpSender.sendAudioFrame(audioBuffer, info) } } diff --git a/udp/src/main/java/com/pedro/udp/UdpSender.kt b/udp/src/main/java/com/pedro/udp/UdpSender.kt index b748b0bff..0de22eb31 100644 --- a/udp/src/main/java/com/pedro/udp/UdpSender.kt +++ b/udp/src/main/java/com/pedro/udp/UdpSender.kt @@ -21,6 +21,8 @@ import android.util.Log import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker +import com.pedro.common.frame.MediaFrame +import com.pedro.common.frame.MediaFrameType import com.pedro.common.onMainThread import com.pedro.common.trySend import com.pedro.common.validMessage @@ -67,7 +69,7 @@ class UdpSender( private val limitSize = Constants.MTU private val mpegTsPacketizer = MpegTsPacketizer(psiManager) private var audioPacket: BasePacket = AacPacket(limitSize, psiManager) - private val h26XPacket = H26XPacket(limitSize, psiManager) + private val videoPacket = H26XPacket(limitSize, psiManager) @Volatile private var running = false @@ -76,7 +78,7 @@ class UdpSender( private var job: Job? = null private val scope = CoroutineScope(Dispatchers.IO) @Volatile - private var queue: BlockingQueue> = LinkedBlockingQueue(cacheSize) + private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) private var audioFramesSent: Long = 0 private var videoFramesSent: Long = 0 var socket: UdpSocket? = null @@ -102,45 +104,36 @@ class UdpSender( } fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { - h26XPacket.setVideoCodec(commandManager.videoCodec.toCodec()) - h26XPacket.sendVideoInfo(sps, pps, vps) + videoPacket.setVideoCodec(commandManager.videoCodec.toCodec()) + videoPacket.sendVideoInfo(sps, pps, vps) } fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { - when (commandManager.audioCodec) { - AudioCodec.AAC -> { - audioPacket = AacPacket(limitSize, psiManager) - (audioPacket as? AacPacket)?.sendAudioInfo(sampleRate, isStereo) - } - AudioCodec.OPUS -> { - audioPacket = OpusPacket(limitSize, psiManager) - } + audioPacket = when (commandManager.audioCodec) { + AudioCodec.AAC -> AacPacket(limitSize, psiManager).apply { sendAudioInfo(sampleRate, isStereo) } + AudioCodec.OPUS -> OpusPacket(limitSize, psiManager) AudioCodec.G711 -> { throw IllegalArgumentException("Unsupported codec: ${commandManager.audioCodec.name}") } } } - fun sendVideoFrame(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - h26XPacket.createAndSendPacket(h264Buffer, info) { mpegTsPackets -> - val result = queue.trySend(mpegTsPackets) - if (!result) { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } + val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrameType.VIDEO)) + if (!result) { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ } } } - fun sendAudioFrame(aacBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - audioPacket.createAndSendPacket(aacBuffer, info) { mpegTsPackets -> - val result = queue.trySend(mpegTsPackets) - if (!result) { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ - } + val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrameType.AUDIO)) + if (!result) { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ } } } @@ -162,13 +155,14 @@ class UdpSender( } while (scope.isActive && running) { val error = runCatching { - val mpegTsPackets = runInterruptible { - queue.poll(1, TimeUnit.SECONDS) + val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } + val mpegTsPackets = getMpegTsPackets(mediaFrame) + mpegTsPackets?.let { + val isKey = mpegTsPackets[0].isKey + val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) + bytesSend += sendPackets(psiPackets) + bytesSend += sendPackets(mpegTsPackets) } - val isKey = mpegTsPackets[0].isKey - val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) - bytesSend += sendPackets(psiPackets) - bytesSend += sendPackets(mpegTsPackets) }.exceptionOrNull() if (error != null) { onMainThread { @@ -200,7 +194,7 @@ class UdpSender( service.clear() mpegTsPacketizer.reset() audioPacket.reset(clear) - h26XPacket.reset(clear) + videoPacket.reset(clear) resetSentAudioFrames() resetSentVideoFrames() resetDroppedAudioFrames() @@ -210,6 +204,24 @@ class UdpSender( queue.clear() } + private fun getMpegTsPackets(mediaFrame: MediaFrame?): List? { + if (mediaFrame == null) return null + var mpegTsPackets: List? = null + when (mediaFrame.type) { + MediaFrameType.VIDEO -> { + videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> + mpegTsPackets = packets + } + } + MediaFrameType.AUDIO -> { + audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> + mpegTsPackets = packets + } + } + } + return mpegTsPackets + } + @Throws(IllegalArgumentException::class) fun hasCongestion(percentUsed: Float = 20f): Boolean { if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") @@ -223,7 +235,7 @@ class UdpSender( if (newSize < queue.size - queue.remainingCapacity()) { throw RuntimeException("Can't fit current cache inside new cache size") } - val tempQueue: BlockingQueue> = LinkedBlockingQueue(newSize) + val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) queue.drainTo(tempQueue) queue = tempQueue } From 151636c57db3e0c2a5986340d4fa534121e78e80 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 14:53:49 +0200 Subject: [PATCH 02/18] adding info and type --- .../pedro/streamer/rotation/CameraFragment.kt | 6 ++-- .../main/java/com/pedro/common/Extensions.kt | 9 ++++-- .../java/com/pedro/common/frame/MediaFrame.kt | 18 ++++++++--- .../com/pedro/common/frame/MediaFrameType.kt | 3 -- .../java/com/pedro/rtmp/flv/BasePacket.kt | 4 +-- .../pedro/rtmp/flv/audio/packet/AacPacket.kt | 10 +++--- .../pedro/rtmp/flv/audio/packet/G711Packet.kt | 11 +++---- .../pedro/rtmp/flv/video/packet/Av1Packet.kt | 18 +++++------ .../pedro/rtmp/flv/video/packet/H264Packet.kt | 10 +++--- .../pedro/rtmp/flv/video/packet/H265Packet.kt | 11 ++++--- .../java/com/pedro/rtmp/rtmp/RtmpClient.kt | 5 +-- .../java/com/pedro/rtmp/rtmp/RtmpSender.kt | 31 ++++++++----------- .../com/pedro/rtsp/rtp/packets/AacPacket.kt | 6 ++-- .../com/pedro/rtsp/rtp/packets/Av1Packet.kt | 6 ++-- .../com/pedro/rtsp/rtp/packets/BasePacket.kt | 3 +- .../com/pedro/rtsp/rtp/packets/G711Packet.kt | 6 ++-- .../com/pedro/rtsp/rtp/packets/H264Packet.kt | 6 ++-- .../com/pedro/rtsp/rtp/packets/H265Packet.kt | 6 ++-- .../com/pedro/rtsp/rtp/packets/OpusPacket.kt | 6 ++-- .../java/com/pedro/rtsp/rtsp/RtspClient.kt | 5 +-- .../java/com/pedro/rtsp/rtsp/RtspSender.kt | 16 +++++----- .../pedro/srt/mpeg2ts/packets/AacPacket.kt | 6 ++-- .../pedro/srt/mpeg2ts/packets/BasePacket.kt | 3 +- .../pedro/srt/mpeg2ts/packets/H26XPacket.kt | 6 ++-- .../pedro/srt/mpeg2ts/packets/OpusPacket.kt | 5 +-- .../main/java/com/pedro/srt/srt/SrtClient.kt | 5 +-- .../main/java/com/pedro/srt/srt/SrtSender.kt | 14 ++++----- udp/src/main/java/com/pedro/udp/UdpClient.kt | 5 +-- udp/src/main/java/com/pedro/udp/UdpSender.kt | 14 ++++----- 29 files changed, 130 insertions(+), 124 deletions(-) diff --git a/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt b/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt index c32d311fb..9631ebd14 100644 --- a/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt +++ b/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt @@ -83,9 +83,9 @@ class CameraFragment: Fragment(), ConnectChecker { private lateinit var surfaceView: SurfaceView private lateinit var bStartStop: ImageView private lateinit var txtBitrate: TextView - private val width = 640 - private val height = 480 - private val vBitrate = 1200 * 1000 + private val width = 3840 + private val height = 2160 + private val vBitrate = 20000 * 1000 private var rotation = 0 private val sampleRate = 32000 private val isStereo = true diff --git a/common/src/main/java/com/pedro/common/Extensions.kt b/common/src/main/java/com/pedro/common/Extensions.kt index 0d4dfdf5a..4d36b6666 100644 --- a/common/src/main/java/com/pedro/common/Extensions.kt +++ b/common/src/main/java/com/pedro/common/Extensions.kt @@ -23,6 +23,7 @@ import android.os.Build import android.os.Handler import android.os.Looper import androidx.annotation.RequiresApi +import com.pedro.common.frame.MediaFrame import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.io.UnsupportedEncodingException @@ -41,7 +42,7 @@ import kotlin.coroutines.Continuation */ @Suppress("DEPRECATION") -fun MediaCodec.BufferInfo.isKeyframe(): Boolean { +fun MediaFrame.Info.isKeyframe(): Boolean { return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { this.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME } else { @@ -60,7 +61,7 @@ fun ByteBuffer.toByteArray(): ByteArray { } } -fun ByteBuffer.removeInfo(info: MediaCodec.BufferInfo): ByteBuffer { +fun ByteBuffer.removeInfo(info: MediaFrame.Info): ByteBuffer { try { position(info.offset) limit(info.size) @@ -139,4 +140,6 @@ fun String.getIndexes(char: Char): Array { fun Throwable.validMessage(): String { return (message ?: "").ifEmpty { javaClass.simpleName } -} \ No newline at end of file +} + +fun MediaCodec.BufferInfo.toMediaFrameInfo() = MediaFrame.Info(offset, size, presentationTimeUs, flags) \ No newline at end of file diff --git a/common/src/main/java/com/pedro/common/frame/MediaFrame.kt b/common/src/main/java/com/pedro/common/frame/MediaFrame.kt index 8c0ddaf2f..91accbf43 100644 --- a/common/src/main/java/com/pedro/common/frame/MediaFrame.kt +++ b/common/src/main/java/com/pedro/common/frame/MediaFrame.kt @@ -1,10 +1,20 @@ package com.pedro.common.frame -import android.media.MediaCodec import java.nio.ByteBuffer data class MediaFrame( val data: ByteBuffer, - val info: MediaCodec.BufferInfo, - val type: MediaFrameType -) + val info: MediaFrame.Info, + val type: MediaFrame.Type +) { + data class Info( + val offset: Int, + val size: Int, + val timestamp: Long, + val flags: Int + ) + + enum class Type { + VIDEO, AUDIO + } +} diff --git a/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt b/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt index 695f17bce..4a273691c 100644 --- a/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt +++ b/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt @@ -1,5 +1,2 @@ package com.pedro.common.frame -enum class MediaFrameType { - VIDEO, AUDIO -} \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/BasePacket.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/BasePacket.kt index 3bed71d40..c50412a68 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/BasePacket.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/BasePacket.kt @@ -16,7 +16,7 @@ package com.pedro.rtmp.flv -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import java.nio.ByteBuffer /** @@ -24,6 +24,6 @@ import java.nio.ByteBuffer */ abstract class BasePacket { - abstract fun createFlvPacket(byteBuffer: ByteBuffer, info: MediaCodec.BufferInfo, callback: (FlvPacket) -> Unit) + abstract suspend fun createFlvPacket(byteBuffer: ByteBuffer, info: MediaFrame.Info, callback: suspend (FlvPacket) -> Unit) abstract fun reset(resetInfo: Boolean = true) } \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/AacPacket.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/AacPacket.kt index cbdf62d5e..320d17939 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/AacPacket.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/AacPacket.kt @@ -16,7 +16,7 @@ package com.pedro.rtmp.flv.audio.packet -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.common.removeInfo import com.pedro.rtmp.flv.BasePacket import com.pedro.rtmp.flv.FlvPacket @@ -56,10 +56,10 @@ class AacPacket: BasePacket() { this.audioSize = audioSize } - override fun createFlvPacket( + override suspend fun createFlvPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, - callback: (FlvPacket) -> Unit + info: MediaFrame.Info, + callback: suspend (FlvPacket) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) //header is 2 bytes length @@ -87,7 +87,7 @@ class AacPacket: BasePacket() { fixedBuffer.get(buffer, header.size, fixedBuffer.remaining()) } System.arraycopy(header, 0, buffer, 0, header.size) - val ts = info.presentationTimeUs / 1000 + val ts = info.timestamp / 1000 callback(FlvPacket(buffer, ts, buffer.size, FlvType.AUDIO)) } diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/G711Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/G711Packet.kt index e9074d83d..88dd82d8c 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/G711Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/G711Packet.kt @@ -16,13 +16,12 @@ package com.pedro.rtmp.flv.audio.packet -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.common.removeInfo import com.pedro.rtmp.flv.BasePacket import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.audio.AudioFormat -import com.pedro.rtmp.flv.audio.AudioObjectType import com.pedro.rtmp.flv.audio.AudioSize import com.pedro.rtmp.flv.audio.AudioSoundRate import com.pedro.rtmp.flv.audio.AudioSoundType @@ -42,10 +41,10 @@ class G711Packet: BasePacket() { this.audioSize = audioSize } - override fun createFlvPacket( + override suspend fun createFlvPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, - callback: (FlvPacket) -> Unit + info: MediaFrame.Info, + callback: suspend (FlvPacket) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) //header is 1 byte length @@ -56,7 +55,7 @@ class G711Packet: BasePacket() { val buffer = ByteArray(fixedBuffer.remaining() + header.size) fixedBuffer.get(buffer, header.size, fixedBuffer.remaining()) System.arraycopy(header, 0, buffer, 0, header.size) - val ts = info.presentationTimeUs / 1000 + val ts = info.timestamp / 1000 callback(FlvPacket(buffer, ts, buffer.size, FlvType.AUDIO)) } diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt index 3be88ae5c..c0ad14cc7 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt @@ -16,16 +16,16 @@ package com.pedro.rtmp.flv.video.packet -import android.media.MediaCodec import android.util.Log -import com.pedro.common.removeInfo -import com.pedro.common.toByteArray -import com.pedro.rtmp.flv.FlvPacket -import com.pedro.rtmp.flv.FlvType import com.pedro.common.av1.Av1Parser import com.pedro.common.av1.ObuType +import com.pedro.common.frame.MediaFrame import com.pedro.common.isKeyframe +import com.pedro.common.removeInfo +import com.pedro.common.toByteArray import com.pedro.rtmp.flv.BasePacket +import com.pedro.rtmp.flv.FlvPacket +import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.video.FourCCPacketType import com.pedro.rtmp.flv.video.VideoDataType import com.pedro.rtmp.flv.video.VideoFormat @@ -50,13 +50,13 @@ class Av1Packet: BasePacket() { this.obuSequence = obuSequence.toByteArray() } - override fun createFlvPacket( + override suspend fun createFlvPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, - callback: (FlvPacket) -> Unit + info: MediaFrame.Info, + callback: suspend (FlvPacket) -> Unit ) { var fixedBuffer = byteBuffer.duplicate().removeInfo(info) - val ts = info.presentationTimeUs / 1000 + val ts = info.timestamp / 1000 //header is 8 bytes length: //mark first byte as extended header (0b10000000) diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt index e294319ba..3d5391e87 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt @@ -16,8 +16,8 @@ package com.pedro.rtmp.flv.video.packet -import android.media.MediaCodec import android.util.Log +import com.pedro.common.frame.MediaFrame import com.pedro.common.isKeyframe import com.pedro.common.removeInfo import com.pedro.rtmp.flv.BasePacket @@ -64,13 +64,13 @@ class H264Packet: BasePacket() { this.pps = ppsBytes } - override fun createFlvPacket( + override suspend fun createFlvPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, - callback: (FlvPacket) -> Unit + info: MediaFrame.Info, + callback: suspend (FlvPacket) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) - val ts = info.presentationTimeUs / 1000 + val ts = info.timestamp / 1000 //header is 5 bytes length: //4 bits FrameType, 4 bits CodecID //1 byte AVCPacketType diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt index 575a9c7a2..912ffd4e6 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt @@ -16,8 +16,8 @@ package com.pedro.rtmp.flv.video.packet -import android.media.MediaCodec import android.util.Log +import com.pedro.common.frame.MediaFrame import com.pedro.common.isKeyframe import com.pedro.common.removeInfo import com.pedro.rtmp.flv.BasePacket @@ -64,13 +64,13 @@ class H265Packet: BasePacket() { this.vps = vpsBytes } - override fun createFlvPacket( + override suspend fun createFlvPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, - callback: (FlvPacket) -> Unit + info: MediaFrame.Info, + callback: suspend (FlvPacket) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) - val ts = info.presentationTimeUs / 1000 + val ts = info.timestamp / 1000 //header is 8 bytes length: //mark first byte as extended header (0b10000000) //4 bits data type, 4 bits packet type @@ -87,6 +87,7 @@ class H265Packet: BasePacket() { header[6] = (cts shr 8).toByte() header[7] = cts.toByte() + val packets = mutableListOf() var buffer: ByteArray if (!configSend) { //avoid send cts on sequence start diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt index e853f5b3a..b4e4d4470 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt @@ -25,6 +25,7 @@ import com.pedro.common.TimeUtils import com.pedro.common.UrlParser import com.pedro.common.VideoCodec import com.pedro.common.onMainThread +import com.pedro.common.toMediaFrameInfo import com.pedro.common.validMessage import com.pedro.rtmp.amf.AmfVersion import com.pedro.rtmp.rtmp.message.* @@ -546,13 +547,13 @@ class RtmpClient(private val connectChecker: ConnectChecker) { fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - rtmpSender.sendVideoFrame(videoBuffer, info) + rtmpSender.sendVideoFrame(videoBuffer, info.toMediaFrameInfo()) } } fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - rtmpSender.sendAudioFrame(audioBuffer, info) + rtmpSender.sendAudioFrame(audioBuffer, info.toMediaFrameInfo()) } } diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt index 4932026ac..6ac00d0c1 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt @@ -16,14 +16,12 @@ package com.pedro.rtmp.rtmp -import android.media.MediaCodec import android.util.Log import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker import com.pedro.common.VideoCodec import com.pedro.common.frame.MediaFrame -import com.pedro.common.frame.MediaFrameType import com.pedro.common.onMainThread import com.pedro.common.trySend import com.pedro.common.validMessage @@ -108,9 +106,9 @@ class RtmpSender( } } - fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaFrame.Info) { if (running) { - val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrameType.VIDEO)) + val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrame.Type.VIDEO)) if (!result) { Log.i(TAG, "Video frame discarded") droppedVideoFrames++ @@ -118,9 +116,9 @@ class RtmpSender( } } - fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaFrame.Info) { if (running) { - val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrameType.AUDIO)) + val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrame.Type.AUDIO)) if (!result) { Log.i(TAG, "Audio frame discarded") droppedAudioFrames++ @@ -145,8 +143,7 @@ class RtmpSender( while (scope.isActive && running) { val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } - val flvPacket = getFlvPacket(mediaFrame) - flvPacket?.let { + getFlvPacket(mediaFrame) { flvPacket -> var size = 0 if (flvPacket.type == FlvType.VIDEO) { videoFramesSent++ @@ -192,22 +189,20 @@ class RtmpSender( queue.clear() } - private fun getFlvPacket(mediaFrame: MediaFrame?): FlvPacket? { - if (mediaFrame == null) return null - var flvPacket: FlvPacket? = null + private suspend fun getFlvPacket(mediaFrame: MediaFrame?, callback: suspend (FlvPacket) -> Unit) { + if (mediaFrame == null) return when (mediaFrame.type) { - MediaFrameType.VIDEO -> { - videoPacket.createFlvPacket(mediaFrame.data, mediaFrame.info) { packet -> - flvPacket = packet + MediaFrame.Type.VIDEO -> { + videoPacket.createFlvPacket(mediaFrame.data, mediaFrame.info) { + callback(it) } } - MediaFrameType.AUDIO -> { - audioPacket.createFlvPacket(mediaFrame.data, mediaFrame.info) { packet -> - flvPacket = packet + MediaFrame.Type.AUDIO -> { + audioPacket.createFlvPacket(mediaFrame.data, mediaFrame.info) { + callback(it) } } } - return flvPacket } @Throws(IllegalArgumentException::class) diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt index 0941577a0..194decb39 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt @@ -16,7 +16,7 @@ package com.pedro.rtsp.rtp.packets -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.common.removeInfo import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants @@ -44,13 +44,13 @@ class AacPacket: BasePacket( override fun createAndSendPacket( byteBuffer: ByteBuffer, - bufferInfo: MediaCodec.BufferInfo, + bufferInfo: MediaFrame.Info, callback: (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(bufferInfo) val length = fixedBuffer.remaining() val maxPayload = maxPacketSize - (RtpConstants.RTP_HEADER_LENGTH + 4) - val ts = bufferInfo.presentationTimeUs * 1000 + val ts = bufferInfo.timestamp * 1000 var sum = 0 val frames = mutableListOf() while (sum < length) { diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt index 732b8adc5..06efc5bf3 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt @@ -16,9 +16,9 @@ package com.pedro.rtsp.rtp.packets -import android.media.MediaCodec import com.pedro.common.av1.Av1Parser import com.pedro.common.av1.ObuType +import com.pedro.common.frame.MediaFrame import com.pedro.common.isKeyframe import com.pedro.common.removeInfo import com.pedro.common.toByteArray @@ -51,7 +51,7 @@ class Av1Packet: BasePacket( override fun createAndSendPacket( byteBuffer: ByteBuffer, - bufferInfo: MediaCodec.BufferInfo, + bufferInfo: MediaFrame.Info, callback: (List) -> Unit ) { var fixedBuffer = byteBuffer.removeInfo(bufferInfo) @@ -61,7 +61,7 @@ class Av1Packet: BasePacket( fixedBuffer = fixedBuffer.slice() } val obuList = parser.getObus(fixedBuffer.duplicate().toByteArray()) - val ts = bufferInfo.presentationTimeUs * 1000L + val ts = bufferInfo.timestamp * 1000L if (obuList.isEmpty()) return var data = byteArrayOf() obuList.forEachIndexed { index, obu -> diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt index fd03d6905..74a27fe7e 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt @@ -17,6 +17,7 @@ package com.pedro.rtsp.rtp.packets import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants import com.pedro.rtsp.utils.setLong @@ -37,7 +38,7 @@ abstract class BasePacket(private var clock: Long, private val payloadType: Int) abstract fun createAndSendPacket( byteBuffer: ByteBuffer, - bufferInfo: MediaCodec.BufferInfo, + bufferInfo: MediaFrame.Info, callback: (List) -> Unit ) diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt index 14ffcc1b4..8e23bd4c7 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt @@ -16,7 +16,7 @@ package com.pedro.rtsp.rtp.packets -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants import java.nio.ByteBuffer @@ -41,12 +41,12 @@ class G711Packet: BasePacket( override fun createAndSendPacket( byteBuffer: ByteBuffer, - bufferInfo: MediaCodec.BufferInfo, + bufferInfo: MediaFrame.Info, callback: (List) -> Unit ) { val length = bufferInfo.size - byteBuffer.position() val maxPayload = maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - val ts = bufferInfo.presentationTimeUs * 1000 + val ts = bufferInfo.timestamp * 1000 var sum = 0 val frames = mutableListOf() while (sum < length) { diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index cd0a80c19..fbdb49a4b 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -16,8 +16,8 @@ package com.pedro.rtsp.rtp.packets -import android.media.MediaCodec import android.util.Log +import com.pedro.common.frame.MediaFrame import com.pedro.common.isKeyframe import com.pedro.common.removeInfo import com.pedro.rtsp.rtsp.RtpFrame @@ -50,7 +50,7 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, override fun createAndSendPacket( byteBuffer: ByteBuffer, - bufferInfo: MediaCodec.BufferInfo, + bufferInfo: MediaFrame.Info, callback: (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(bufferInfo) @@ -60,7 +60,7 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, if (header.size == 1) return //invalid buffer or waiting for sps/pps fixedBuffer.rewind() fixedBuffer.get(header, 0, header.size) - val ts = bufferInfo.presentationTimeUs * 1000L + val ts = bufferInfo.timestamp * 1000L val naluLength = fixedBuffer.remaining() val type: Int = (header[header.size - 1] and 0x1F).toInt() val frames = mutableListOf() diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt index 742f03ed5..a4d4d280c 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt @@ -16,7 +16,7 @@ package com.pedro.rtsp.rtp.packets -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.common.removeInfo import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants @@ -40,7 +40,7 @@ class H265Packet: BasePacket( override fun createAndSendPacket( byteBuffer: ByteBuffer, - bufferInfo: MediaCodec.BufferInfo, + bufferInfo: MediaFrame.Info, callback: (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(bufferInfo) @@ -49,7 +49,7 @@ class H265Packet: BasePacket( val header = ByteArray(fixedBuffer.getVideoStartCodeSize() + 2) if (header.size == 2) return //invalid buffer or waiting for sps/pps/vps fixedBuffer.get(header, 0, header.size) - val ts = bufferInfo.presentationTimeUs * 1000L + val ts = bufferInfo.timestamp * 1000L val naluLength = fixedBuffer.remaining() val type: Int = header[header.size - 2].toInt().shr(1 and 0x3f) val frames = mutableListOf() diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt index 757c728e7..97b816272 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt @@ -16,7 +16,7 @@ package com.pedro.rtsp.rtp.packets -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants import java.nio.ByteBuffer @@ -41,12 +41,12 @@ class OpusPacket: BasePacket( override fun createAndSendPacket( byteBuffer: ByteBuffer, - bufferInfo: MediaCodec.BufferInfo, + bufferInfo: MediaFrame.Info, callback: (List) -> Unit ) { val length = bufferInfo.size - byteBuffer.position() val maxPayload = maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - val ts = bufferInfo.presentationTimeUs * 1000 + val ts = bufferInfo.timestamp * 1000 var sum = 0 val frames = mutableListOf() while (sum < length) { diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt index 0d1d36436..9ebe6dfe1 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt @@ -25,6 +25,7 @@ import com.pedro.common.UrlParser import com.pedro.common.VideoCodec import com.pedro.common.onMainThread import com.pedro.common.socket.TcpStreamSocket +import com.pedro.common.toMediaFrameInfo import com.pedro.common.validMessage import com.pedro.rtsp.rtsp.commands.CommandsManager import com.pedro.rtsp.rtsp.commands.Method @@ -429,13 +430,13 @@ class RtspClient(private val connectChecker: ConnectChecker) { fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - rtspSender.sendVideoFrame(videoBuffer, info) + rtspSender.sendVideoFrame(videoBuffer, info.toMediaFrameInfo()) } } fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - rtspSender.sendAudioFrame(audioBuffer, info) + rtspSender.sendAudioFrame(audioBuffer, info.toMediaFrameInfo()) } } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt index 563a0cc29..0be72b78d 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt @@ -16,14 +16,12 @@ package com.pedro.rtsp.rtsp -import android.media.MediaCodec import android.util.Log import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker -import com.pedro.common.frame.MediaFrame import com.pedro.common.VideoCodec -import com.pedro.common.frame.MediaFrameType +import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread import com.pedro.common.socket.TcpStreamSocket import com.pedro.common.trySend @@ -121,9 +119,9 @@ class RtspSender( baseSenderReport?.setSocket(socket) } - fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaFrame.Info) { if (running) { - val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrameType.VIDEO)) + val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrame.Type.VIDEO)) if (!result) { Log.i(TAG, "Video frame discarded") droppedVideoFrames++ @@ -131,9 +129,9 @@ class RtspSender( } } - fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaFrame.Info) { if (running) { - val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrameType.AUDIO)) + val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrame.Type.AUDIO)) if (!result) { Log.i(TAG, "Audio frame discarded") droppedAudioFrames++ @@ -223,12 +221,12 @@ class RtspSender( if (mediaFrame == null) return null var rtpPackets: List? = null when (mediaFrame.type) { - MediaFrameType.VIDEO -> { + MediaFrame.Type.VIDEO -> { videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> rtpPackets = packets } } - MediaFrameType.AUDIO -> { + MediaFrame.Type.AUDIO -> { audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> rtpPackets = packets } diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt index 8fcb13773..5e3fe64ea 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt @@ -16,7 +16,7 @@ package com.pedro.srt.mpeg2ts.packets -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.common.removeInfo import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegType @@ -40,7 +40,7 @@ class AacPacket( override fun createAndSendPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, + info: MediaFrame.Info, callback: (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) @@ -51,7 +51,7 @@ class AacPacket( writeAdts(payload, payload.size, 0) fixedBuffer.get(payload, header.size, length) - val pes = Pes(psiManager.getAudioPid().toInt(), false, PesType.AUDIO, info.presentationTimeUs, ByteBuffer.wrap(payload)) + val pes = Pes(psiManager.getAudioPid().toInt(), false, PesType.AUDIO, info.timestamp, ByteBuffer.wrap(payload)) val mpeg2tsPackets = mpegTsPacketizer.write(listOf(pes)) val chunked = mpeg2tsPackets.chunked(chunkSize) val packets = mutableListOf() diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt index 6cb2e5561..c409a0b58 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt @@ -17,6 +17,7 @@ package com.pedro.srt.mpeg2ts.packets import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegTsPacketizer import com.pedro.srt.mpeg2ts.psi.PsiManager @@ -52,7 +53,7 @@ abstract class BasePacket( abstract fun createAndSendPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, + info: MediaFrame.Info, callback: (List) -> Unit ) diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt index 14db51ce1..4a4c56024 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt @@ -16,8 +16,8 @@ package com.pedro.srt.mpeg2ts.packets -import android.media.MediaCodec import android.util.Log +import com.pedro.common.frame.MediaFrame import com.pedro.common.isKeyframe import com.pedro.common.removeInfo import com.pedro.common.toByteArray @@ -51,7 +51,7 @@ class H26XPacket( override fun createAndSendPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, + info: MediaFrame.Info, callback: (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) @@ -79,7 +79,7 @@ class H26XPacket( val payload = ByteArray(validBuffer.remaining()) validBuffer.get(payload, 0, validBuffer.remaining()) - val pes = Pes(psiManager.getVideoPid().toInt(), isKeyFrame, PesType.VIDEO, info.presentationTimeUs, ByteBuffer.wrap(payload)) + val pes = Pes(psiManager.getVideoPid().toInt(), isKeyFrame, PesType.VIDEO, info.timestamp, ByteBuffer.wrap(payload)) val mpeg2tsPackets = mpegTsPacketizer.write(listOf(pes)) val chunked = mpeg2tsPackets.chunked(chunkSize) val packets = mutableListOf() diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt index 852a2bbee..b147cbb39 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt @@ -17,6 +17,7 @@ package com.pedro.srt.mpeg2ts.packets import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.common.removeInfo import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegType @@ -37,7 +38,7 @@ class OpusPacket( override fun createAndSendPacket( byteBuffer: ByteBuffer, - info: MediaCodec.BufferInfo, + info: MediaFrame.Info, callback: (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) @@ -49,7 +50,7 @@ class OpusPacket( fixedBuffer.get(payload, header.size, length) System.arraycopy(header, 0, payload, 0, header.size) - val pes = Pes(psiManager.getAudioPid().toInt(), true, PesType.PRIVATE_STREAM_1, info.presentationTimeUs, ByteBuffer.wrap(payload)) + val pes = Pes(psiManager.getAudioPid().toInt(), true, PesType.PRIVATE_STREAM_1, info.timestamp, ByteBuffer.wrap(payload)) val mpeg2tsPackets = mpegTsPacketizer.write(listOf(pes)) val chunked = mpeg2tsPackets.chunked(chunkSize) val packets = mutableListOf() diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt b/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt index 025b35d1a..439401c7a 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt @@ -24,6 +24,7 @@ import com.pedro.common.ConnectionFailed import com.pedro.common.UrlParser import com.pedro.common.VideoCodec import com.pedro.common.onMainThread +import com.pedro.common.toMediaFrameInfo import com.pedro.common.validMessage import com.pedro.srt.srt.packets.ControlPacket import com.pedro.srt.srt.packets.DataPacket @@ -393,13 +394,13 @@ class SrtClient(private val connectChecker: ConnectChecker) { fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - srtSender.sendVideoFrame(videoBuffer, info) + srtSender.sendVideoFrame(videoBuffer, info.toMediaFrameInfo()) } } fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - srtSender.sendAudioFrame(audioBuffer, info) + srtSender.sendAudioFrame(audioBuffer, info.toMediaFrameInfo()) } } diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt index 181e4c6de..1b37af0f3 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt @@ -16,13 +16,11 @@ package com.pedro.srt.srt -import android.media.MediaCodec import android.util.Log import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker import com.pedro.common.frame.MediaFrame -import com.pedro.common.frame.MediaFrameType import com.pedro.common.onMainThread import com.pedro.common.trySend import com.pedro.common.validMessage @@ -118,9 +116,9 @@ class SrtSender( } } - fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaFrame.Info) { if (running) { - val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrameType.VIDEO)) + val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrame.Type.VIDEO)) if (!result) { Log.i(TAG, "Video frame discarded") droppedVideoFrames++ @@ -128,9 +126,9 @@ class SrtSender( } } - fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaFrame.Info) { if (running) { - val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrameType.AUDIO)) + val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrame.Type.AUDIO)) if (!result) { Log.i(TAG, "Audio frame discarded") droppedAudioFrames++ @@ -208,12 +206,12 @@ class SrtSender( if (mediaFrame == null) return null var mpegTsPackets: List? = null when (mediaFrame.type) { - MediaFrameType.VIDEO -> { + MediaFrame.Type.VIDEO -> { videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> mpegTsPackets = packets } } - MediaFrameType.AUDIO -> { + MediaFrame.Type.AUDIO -> { audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> mpegTsPackets = packets } diff --git a/udp/src/main/java/com/pedro/udp/UdpClient.kt b/udp/src/main/java/com/pedro/udp/UdpClient.kt index 88783af0d..7e4baf42d 100644 --- a/udp/src/main/java/com/pedro/udp/UdpClient.kt +++ b/udp/src/main/java/com/pedro/udp/UdpClient.kt @@ -23,6 +23,7 @@ import com.pedro.common.ConnectChecker import com.pedro.common.UrlParser import com.pedro.common.VideoCodec import com.pedro.common.onMainThread +import com.pedro.common.toMediaFrameInfo import com.pedro.common.validMessage import com.pedro.udp.utils.UdpSocket import com.pedro.udp.utils.UdpType @@ -231,13 +232,13 @@ class UdpClient(private val connectChecker: ConnectChecker) { fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandManager.videoDisabled) { - udpSender.sendVideoFrame(videoBuffer, info) + udpSender.sendVideoFrame(videoBuffer, info.toMediaFrameInfo()) } } fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandManager.audioDisabled) { - udpSender.sendAudioFrame(audioBuffer, info) + udpSender.sendAudioFrame(audioBuffer, info.toMediaFrameInfo()) } } diff --git a/udp/src/main/java/com/pedro/udp/UdpSender.kt b/udp/src/main/java/com/pedro/udp/UdpSender.kt index 0de22eb31..eaf64c993 100644 --- a/udp/src/main/java/com/pedro/udp/UdpSender.kt +++ b/udp/src/main/java/com/pedro/udp/UdpSender.kt @@ -16,13 +16,11 @@ package com.pedro.udp -import android.media.MediaCodec import android.util.Log import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker import com.pedro.common.frame.MediaFrame -import com.pedro.common.frame.MediaFrameType import com.pedro.common.onMainThread import com.pedro.common.trySend import com.pedro.common.validMessage @@ -118,9 +116,9 @@ class UdpSender( } } - fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaFrame.Info) { if (running) { - val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrameType.VIDEO)) + val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrame.Type.VIDEO)) if (!result) { Log.i(TAG, "Video frame discarded") droppedVideoFrames++ @@ -128,9 +126,9 @@ class UdpSender( } } - fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { + fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaFrame.Info) { if (running) { - val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrameType.AUDIO)) + val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrame.Type.AUDIO)) if (!result) { Log.i(TAG, "Audio frame discarded") droppedAudioFrames++ @@ -208,12 +206,12 @@ class UdpSender( if (mediaFrame == null) return null var mpegTsPackets: List? = null when (mediaFrame.type) { - MediaFrameType.VIDEO -> { + MediaFrame.Type.VIDEO -> { videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> mpegTsPackets = packets } } - MediaFrameType.AUDIO -> { + MediaFrame.Type.AUDIO -> { audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> mpegTsPackets = packets } From f02154b2fc211d30e81f53817dbbc69195860506 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 15:52:57 +0200 Subject: [PATCH 03/18] fix send bytebuffer --- .../main/java/com/pedro/common/Extensions.kt | 4 +- .../java/com/pedro/rtmp/rtmp/RtmpClient.kt | 6 +- .../java/com/pedro/rtmp/rtmp/RtmpSender.kt | 27 +++--- .../com/pedro/rtsp/rtp/packets/AacPacket.kt | 4 +- .../com/pedro/rtsp/rtp/packets/Av1Packet.kt | 4 +- .../com/pedro/rtsp/rtp/packets/BasePacket.kt | 5 +- .../com/pedro/rtsp/rtp/packets/G711Packet.kt | 4 +- .../com/pedro/rtsp/rtp/packets/H264Packet.kt | 4 +- .../com/pedro/rtsp/rtp/packets/H265Packet.kt | 4 +- .../com/pedro/rtsp/rtp/packets/OpusPacket.kt | 4 +- .../java/com/pedro/rtsp/rtsp/RtspClient.kt | 6 +- .../java/com/pedro/rtsp/rtsp/RtspSender.kt | 86 +++++++++---------- .../pedro/srt/mpeg2ts/packets/AacPacket.kt | 4 +- .../pedro/srt/mpeg2ts/packets/BasePacket.kt | 4 +- .../pedro/srt/mpeg2ts/packets/H26XPacket.kt | 4 +- .../pedro/srt/mpeg2ts/packets/OpusPacket.kt | 4 +- .../main/java/com/pedro/srt/srt/SrtClient.kt | 6 +- .../main/java/com/pedro/srt/srt/SrtSender.kt | 40 ++++----- udp/src/main/java/com/pedro/udp/UdpClient.kt | 6 +- udp/src/main/java/com/pedro/udp/UdpSender.kt | 42 ++++----- 20 files changed, 126 insertions(+), 142 deletions(-) diff --git a/common/src/main/java/com/pedro/common/Extensions.kt b/common/src/main/java/com/pedro/common/Extensions.kt index 4d36b6666..559694f1b 100644 --- a/common/src/main/java/com/pedro/common/Extensions.kt +++ b/common/src/main/java/com/pedro/common/Extensions.kt @@ -142,4 +142,6 @@ fun Throwable.validMessage(): String { return (message ?: "").ifEmpty { javaClass.simpleName } } -fun MediaCodec.BufferInfo.toMediaFrameInfo() = MediaFrame.Info(offset, size, presentationTimeUs, flags) \ No newline at end of file +fun MediaCodec.BufferInfo.toMediaFrameInfo() = MediaFrame.Info(offset, size, presentationTimeUs, flags) + +fun ByteBuffer.clone(): ByteBuffer = ByteBuffer.wrap(toByteArray()) \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt index b4e4d4470..566843226 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt @@ -24,6 +24,8 @@ import com.pedro.common.ConnectionFailed import com.pedro.common.TimeUtils import com.pedro.common.UrlParser import com.pedro.common.VideoCodec +import com.pedro.common.clone +import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread import com.pedro.common.toMediaFrameInfo import com.pedro.common.validMessage @@ -547,13 +549,13 @@ class RtmpClient(private val connectChecker: ConnectChecker) { fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - rtmpSender.sendVideoFrame(videoBuffer, info.toMediaFrameInfo()) + rtmpSender.sendMediaFrame(MediaFrame(videoBuffer.clone(), info.toMediaFrameInfo(), MediaFrame.Type.VIDEO)) } } fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - rtmpSender.sendAudioFrame(audioBuffer, info.toMediaFrameInfo()) + rtmpSender.sendMediaFrame(MediaFrame(audioBuffer.clone(), info.toMediaFrameInfo(), MediaFrame.Type.AUDIO)) } } diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt index 6ac00d0c1..8af707888 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt @@ -106,22 +106,17 @@ class RtmpSender( } } - fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaFrame.Info) { - if (running) { - val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrame.Type.VIDEO)) - if (!result) { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } - } - } - - fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaFrame.Info) { - if (running) { - val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrame.Type.AUDIO)) - if (!result) { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ + fun sendMediaFrame(mediaFrame: MediaFrame) { + if (running && !queue.trySend(mediaFrame)) { + when (mediaFrame.type) { + MediaFrame.Type.VIDEO -> { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ + } + MediaFrame.Type.AUDIO -> { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ + } } } } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt index 194decb39..caa2170b9 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AacPacket.kt @@ -42,10 +42,10 @@ class AacPacket: BasePacket( setClock(sampleRate.toLong()) } - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(bufferInfo) val length = fixedBuffer.remaining() diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt index 06efc5bf3..f413b840b 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt @@ -49,10 +49,10 @@ class Av1Packet: BasePacket( channelIdentifier = RtpConstants.trackVideo } - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { var fixedBuffer = byteBuffer.removeInfo(bufferInfo) //remove temporal delimitered OBU if found on start diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt index 74a27fe7e..1369612d4 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/BasePacket.kt @@ -16,7 +16,6 @@ package com.pedro.rtsp.rtp.packets -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants @@ -36,10 +35,10 @@ abstract class BasePacket(private var clock: Long, private val payloadType: Int) protected val maxPacketSize = RtpConstants.MTU - 28 protected val TAG = "BasePacket" - abstract fun createAndSendPacket( + abstract suspend fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) open fun reset() { diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt index 8e23bd4c7..e58980eab 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/G711Packet.kt @@ -39,10 +39,10 @@ class G711Packet: BasePacket( setClock(sampleRate.toLong()) } - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { val length = bufferInfo.size - byteBuffer.position() val maxPayload = maxPacketSize - RtpConstants.RTP_HEADER_LENGTH diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index fbdb49a4b..6b1d5c0a9 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -48,10 +48,10 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, setSpsPps(sps, pps) } - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(bufferInfo) // We read a NAL units from ByteBuffer and we send them diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt index a4d4d280c..360c9fcc9 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt @@ -38,10 +38,10 @@ class H265Packet: BasePacket( channelIdentifier = RtpConstants.trackVideo } - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(bufferInfo) // We read a NAL units from ByteBuffer and we send them diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt index 97b816272..0ffe79ab2 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/OpusPacket.kt @@ -39,10 +39,10 @@ class OpusPacket: BasePacket( setClock(sampleRate.toLong()) } - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, bufferInfo: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { val length = bufferInfo.size - byteBuffer.position() val maxPayload = maxPacketSize - RtpConstants.RTP_HEADER_LENGTH diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt index 9ebe6dfe1..1ebd6ffaf 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt @@ -23,6 +23,8 @@ import com.pedro.common.ConnectChecker import com.pedro.common.ConnectionFailed import com.pedro.common.UrlParser import com.pedro.common.VideoCodec +import com.pedro.common.clone +import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread import com.pedro.common.socket.TcpStreamSocket import com.pedro.common.toMediaFrameInfo @@ -430,13 +432,13 @@ class RtspClient(private val connectChecker: ConnectChecker) { fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - rtspSender.sendVideoFrame(videoBuffer, info.toMediaFrameInfo()) + rtspSender.sendMediaFrame(MediaFrame(videoBuffer.clone(), info.toMediaFrameInfo(), MediaFrame.Type.VIDEO)) } } fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - rtspSender.sendAudioFrame(audioBuffer, info.toMediaFrameInfo()) + rtspSender.sendMediaFrame(MediaFrame(audioBuffer.clone(), info.toMediaFrameInfo(), MediaFrame.Type.AUDIO)) } } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt index 0be72b78d..bd517ce77 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt @@ -119,22 +119,17 @@ class RtspSender( baseSenderReport?.setSocket(socket) } - fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaFrame.Info) { - if (running) { - val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrame.Type.VIDEO)) - if (!result) { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } - } - } - - fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaFrame.Info) { - if (running) { - val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrame.Type.AUDIO)) - if (!result) { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ + fun sendMediaFrame(mediaFrame: MediaFrame) { + if (running && !queue.trySend(mediaFrame)) { + when (mediaFrame.type) { + MediaFrame.Type.VIDEO -> { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ + } + MediaFrame.Type.AUDIO -> { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ + } } } } @@ -162,32 +157,33 @@ class RtspSender( while (scope.isActive && running) { val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } - val rtpFrames = getRtpPackets(mediaFrame) - var size = 0 - var isVideo = false - rtpFrames?.forEach { rtpFrame -> - rtpSocket?.sendFrame(rtpFrame) - //4 is tcp header length - val packetSize = if (isTcp) rtpFrame.length + 4 else rtpFrame.length - bytesSend += packetSize - size += packetSize - isVideo = rtpFrame.isVideoFrame() - if (isVideo) { - videoFramesSent++ - } else { - audioFramesSent++ - } - if (baseSenderReport?.update(rtpFrame) == true) { + getRtpPackets(mediaFrame) { rtpFrames -> + var size = 0 + var isVideo = false + rtpFrames.forEach { rtpFrame -> + rtpSocket?.sendFrame(rtpFrame) //4 is tcp header length - val reportSize = if (isTcp) RtpConstants.REPORT_PACKET_LENGTH + 4 else RtpConstants.REPORT_PACKET_LENGTH - bytesSend += reportSize - if (isEnableLogs) Log.i(TAG, "wrote report") + val packetSize = if (isTcp) rtpFrame.length + 4 else rtpFrame.length + bytesSend += packetSize + size += packetSize + isVideo = rtpFrame.isVideoFrame() + if (isVideo) { + videoFramesSent++ + } else { + audioFramesSent++ + } + if (baseSenderReport?.update(rtpFrame) == true) { + //4 is tcp header length + val reportSize = if (isTcp) RtpConstants.REPORT_PACKET_LENGTH + 4 else RtpConstants.REPORT_PACKET_LENGTH + bytesSend += reportSize + if (isEnableLogs) Log.i(TAG, "wrote report") + } + } + rtpSocket?.flush() + if (isEnableLogs) { + val type = if (isVideo) "Video" else "Audio" + Log.i(TAG, "wrote $type packet, size $size") } - } - rtpSocket?.flush() - if (isEnableLogs) { - val type = if (isVideo) "Video" else "Audio" - Log.i(TAG, "wrote $type packet, size $size") } }.exceptionOrNull() if (error != null) { @@ -217,22 +213,20 @@ class RtspSender( queue.clear() } - private fun getRtpPackets(mediaFrame: MediaFrame?): List? { - if (mediaFrame == null) return null - var rtpPackets: List? = null + private suspend fun getRtpPackets(mediaFrame: MediaFrame?, callback: suspend (List) -> Unit) { + if (mediaFrame == null) return when (mediaFrame.type) { MediaFrame.Type.VIDEO -> { videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - rtpPackets = packets + callback(packets) } } MediaFrame.Type.AUDIO -> { audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - rtpPackets = packets + callback(packets) } } } - return rtpPackets } @Throws(IllegalArgumentException::class) diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt index 5e3fe64ea..d116a2650 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt @@ -38,10 +38,10 @@ class AacPacket( private var sampleRate = 44100 private var isStereo = true - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, info: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) val length = fixedBuffer.remaining() diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt index c409a0b58..ae6995755 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt @@ -51,10 +51,10 @@ abstract class BasePacket( protected val mpegTsPacketizer = MpegTsPacketizer(psiManager) protected var chunkSize = limitSize / MpegTsPacketizer.packetSize //max number of ts packets per srtpacket - abstract fun createAndSendPacket( + abstract suspend fun createAndSendPacket( byteBuffer: ByteBuffer, info: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) abstract fun resetPacket(resetInfo: Boolean) diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt index 4a4c56024..4ac489bc5 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt @@ -49,10 +49,10 @@ class H26XPacket( private var codec = Codec.AVC private var configSend = false - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, info: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) val length = fixedBuffer.remaining() diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt index b147cbb39..8a236fbe8 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt @@ -36,10 +36,10 @@ class OpusPacket( psiManager: PsiManager, ): BasePacket(psiManager, limitSize) { - override fun createAndSendPacket( + override suspend fun createAndSendPacket( byteBuffer: ByteBuffer, info: MediaFrame.Info, - callback: (List) -> Unit + callback: suspend (List) -> Unit ) { val fixedBuffer = byteBuffer.removeInfo(info) val length = fixedBuffer.remaining() diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt b/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt index 439401c7a..1d4740fb1 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtClient.kt @@ -23,6 +23,8 @@ import com.pedro.common.ConnectChecker import com.pedro.common.ConnectionFailed import com.pedro.common.UrlParser import com.pedro.common.VideoCodec +import com.pedro.common.clone +import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread import com.pedro.common.toMediaFrameInfo import com.pedro.common.validMessage @@ -394,13 +396,13 @@ class SrtClient(private val connectChecker: ConnectChecker) { fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.videoDisabled) { - srtSender.sendVideoFrame(videoBuffer, info.toMediaFrameInfo()) + srtSender.sendMediaFrame(MediaFrame(videoBuffer.clone(), info.toMediaFrameInfo(), MediaFrame.Type.VIDEO)) } } fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandsManager.audioDisabled) { - srtSender.sendAudioFrame(audioBuffer, info.toMediaFrameInfo()) + srtSender.sendMediaFrame(MediaFrame(audioBuffer.clone(), info.toMediaFrameInfo(), MediaFrame.Type.AUDIO)) } } diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt index 1b37af0f3..df0531fea 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt @@ -116,22 +116,17 @@ class SrtSender( } } - fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaFrame.Info) { - if (running) { - val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrame.Type.VIDEO)) - if (!result) { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } - } - } - - fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaFrame.Info) { - if (running) { - val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrame.Type.AUDIO)) - if (!result) { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ + fun sendMediaFrame(mediaFrame: MediaFrame) { + if (running && !queue.trySend(mediaFrame)) { + when (mediaFrame.type) { + MediaFrame.Type.VIDEO -> { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ + } + MediaFrame.Type.AUDIO -> { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ + } } } } @@ -154,8 +149,7 @@ class SrtSender( while (scope.isActive && running) { val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } - val mpegTsPackets = getMpegTsPackets(mediaFrame) - mpegTsPackets?.let { + getMpegTsPackets(mediaFrame) { mpegTsPackets -> val isKey = mpegTsPackets[0].isKey val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) bytesSend += sendPackets(psiPackets) @@ -202,22 +196,20 @@ class SrtSender( queue.clear() } - private fun getMpegTsPackets(mediaFrame: MediaFrame?): List? { - if (mediaFrame == null) return null - var mpegTsPackets: List? = null + private suspend fun getMpegTsPackets(mediaFrame: MediaFrame?, callback: suspend (List) -> Unit) { + if (mediaFrame == null) return when (mediaFrame.type) { MediaFrame.Type.VIDEO -> { videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - mpegTsPackets = packets + callback(packets) } } MediaFrame.Type.AUDIO -> { audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - mpegTsPackets = packets + callback(packets) } } } - return mpegTsPackets } @Throws(IllegalArgumentException::class) diff --git a/udp/src/main/java/com/pedro/udp/UdpClient.kt b/udp/src/main/java/com/pedro/udp/UdpClient.kt index 7e4baf42d..a06dc0df5 100644 --- a/udp/src/main/java/com/pedro/udp/UdpClient.kt +++ b/udp/src/main/java/com/pedro/udp/UdpClient.kt @@ -22,6 +22,8 @@ import com.pedro.common.AudioCodec import com.pedro.common.ConnectChecker import com.pedro.common.UrlParser import com.pedro.common.VideoCodec +import com.pedro.common.clone +import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread import com.pedro.common.toMediaFrameInfo import com.pedro.common.validMessage @@ -232,13 +234,13 @@ class UdpClient(private val connectChecker: ConnectChecker) { fun sendVideo(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandManager.videoDisabled) { - udpSender.sendVideoFrame(videoBuffer, info.toMediaFrameInfo()) + udpSender.sendMediaFrame(MediaFrame(videoBuffer.clone(), info.toMediaFrameInfo(), MediaFrame.Type.VIDEO)) } } fun sendAudio(audioBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (!commandManager.audioDisabled) { - udpSender.sendAudioFrame(audioBuffer, info.toMediaFrameInfo()) + udpSender.sendMediaFrame(MediaFrame(audioBuffer.clone(), info.toMediaFrameInfo(), MediaFrame.Type.AUDIO)) } } diff --git a/udp/src/main/java/com/pedro/udp/UdpSender.kt b/udp/src/main/java/com/pedro/udp/UdpSender.kt index eaf64c993..86c0dff11 100644 --- a/udp/src/main/java/com/pedro/udp/UdpSender.kt +++ b/udp/src/main/java/com/pedro/udp/UdpSender.kt @@ -33,6 +33,8 @@ import com.pedro.srt.mpeg2ts.packets.H26XPacket import com.pedro.srt.mpeg2ts.packets.OpusPacket import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.srt.mpeg2ts.service.Mpeg2TsService +import com.pedro.srt.srt.SrtSender +import com.pedro.srt.srt.SrtSender.Companion import com.pedro.srt.utils.Constants import com.pedro.srt.utils.toCodec import com.pedro.udp.utils.UdpSocket @@ -116,22 +118,17 @@ class UdpSender( } } - fun sendVideoFrame(videoBuffer: ByteBuffer, info: MediaFrame.Info) { - if (running) { - val result = queue.trySend(MediaFrame(videoBuffer, info, MediaFrame.Type.VIDEO)) - if (!result) { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } - } - } - - fun sendAudioFrame(audioBuffer: ByteBuffer, info: MediaFrame.Info) { - if (running) { - val result = queue.trySend(MediaFrame(audioBuffer, info, MediaFrame.Type.AUDIO)) - if (!result) { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ + fun sendMediaFrame(mediaFrame: MediaFrame) { + if (running && !queue.trySend(mediaFrame)) { + when (mediaFrame.type) { + MediaFrame.Type.VIDEO -> { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ + } + MediaFrame.Type.AUDIO -> { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ + } } } } @@ -154,8 +151,7 @@ class UdpSender( while (scope.isActive && running) { val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } - val mpegTsPackets = getMpegTsPackets(mediaFrame) - mpegTsPackets?.let { + getMpegTsPackets(mediaFrame) { mpegTsPackets -> val isKey = mpegTsPackets[0].isKey val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) bytesSend += sendPackets(psiPackets) @@ -202,22 +198,20 @@ class UdpSender( queue.clear() } - private fun getMpegTsPackets(mediaFrame: MediaFrame?): List? { - if (mediaFrame == null) return null - var mpegTsPackets: List? = null + private suspend fun getMpegTsPackets(mediaFrame: MediaFrame?, callback: suspend (List) -> Unit) { + if (mediaFrame == null) return when (mediaFrame.type) { MediaFrame.Type.VIDEO -> { videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - mpegTsPackets = packets + callback(packets) } } MediaFrame.Type.AUDIO -> { audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - mpegTsPackets = packets + callback(packets) } } } - return mpegTsPackets } @Throws(IllegalArgumentException::class) From 7a578d9d26fc11893c9b546607413edbddf62d78 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 16:32:52 +0200 Subject: [PATCH 04/18] refactor senders --- .../java/com/pedro/common/base/BaseSender.kt | 132 ++++++++++ .../java/com/pedro/common/frame/MediaFrame.kt | 4 +- .../com/pedro/common/frame/MediaFrameType.kt | 2 - .../java/com/pedro/rtmp/rtmp/RtmpSender.kt | 198 ++++----------- .../com/pedro/rtsp/rtp/packets/H264Packet.kt | 5 +- .../java/com/pedro/rtsp/rtsp/RtspClient.kt | 2 +- .../java/com/pedro/rtsp/rtsp/RtspSender.kt | 228 +++++------------- .../rtsp/rtsp/commands/CommandsManager.kt | 18 +- .../main/java/com/pedro/srt/srt/SrtSender.kt | 192 +++------------ udp/src/main/java/com/pedro/udp/UdpSender.kt | 195 +++------------ 10 files changed, 318 insertions(+), 658 deletions(-) create mode 100644 common/src/main/java/com/pedro/common/base/BaseSender.kt delete mode 100644 common/src/main/java/com/pedro/common/frame/MediaFrameType.kt diff --git a/common/src/main/java/com/pedro/common/base/BaseSender.kt b/common/src/main/java/com/pedro/common/base/BaseSender.kt new file mode 100644 index 000000000..811c57621 --- /dev/null +++ b/common/src/main/java/com/pedro/common/base/BaseSender.kt @@ -0,0 +1,132 @@ +package com.pedro.common.base + +import android.util.Log +import com.pedro.common.BitrateManager +import com.pedro.common.ConnectChecker +import com.pedro.common.frame.MediaFrame +import com.pedro.common.trySend +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch +import java.nio.ByteBuffer +import java.util.concurrent.BlockingQueue +import java.util.concurrent.LinkedBlockingQueue + +abstract class BaseSender( + protected val connectChecker: ConnectChecker, + protected val TAG: String +) { + + @Volatile + protected var running = false + private var cacheSize = 200 + @Volatile + protected var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) + protected var audioFramesSent: Long = 0 + protected var videoFramesSent: Long = 0 + var droppedAudioFrames: Long = 0 + protected set + var droppedVideoFrames: Long = 0 + protected set + protected val bitrateManager: BitrateManager = BitrateManager(connectChecker) + protected var isEnableLogs = true + private var job: Job? = null + protected val scope = CoroutineScope(Dispatchers.IO) + + abstract fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) + abstract fun setAudioInfo(sampleRate: Int, isStereo: Boolean) + protected abstract suspend fun onRun() + protected abstract suspend fun stopImp(clear: Boolean = true) + + fun sendMediaFrame(mediaFrame: MediaFrame) { + if (running && !queue.trySend(mediaFrame)) { + when (mediaFrame.type) { + MediaFrame.Type.VIDEO -> { + Log.i(TAG, "Video frame discarded") + droppedVideoFrames++ + } + MediaFrame.Type.AUDIO -> { + Log.i(TAG, "Audio frame discarded") + droppedAudioFrames++ + } + } + } + } + + fun start() { + bitrateManager.reset() + queue.clear() + running = true + scope.launch { onRun() } + } + + suspend fun stop(clear: Boolean = true) { + running = false + stopImp(clear) + resetSentAudioFrames() + resetSentVideoFrames() + resetDroppedAudioFrames() + resetDroppedVideoFrames() + job?.cancelAndJoin() + job = null + queue.clear() + } + + @Throws(IllegalArgumentException::class) + fun hasCongestion(percentUsed: Float = 20f): Boolean { + if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") + val size = queue.size.toFloat() + val remaining = queue.remainingCapacity().toFloat() + val capacity = size + remaining + return size >= capacity * (percentUsed / 100f) + } + + fun resizeCache(newSize: Int) { + if (newSize < queue.size - queue.remainingCapacity()) { + throw RuntimeException("Can't fit current cache inside new cache size") + } + val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) + queue.drainTo(tempQueue) + queue = tempQueue + } + + fun getCacheSize(): Int = cacheSize + + fun getItemsInCache(): Int = queue.size + + fun clearCache() { + queue.clear() + } + + fun getSentAudioFrames(): Long = audioFramesSent + + fun getSentVideoFrames(): Long = videoFramesSent + + fun resetSentAudioFrames() { + audioFramesSent = 0 + } + + fun resetSentVideoFrames() { + videoFramesSent = 0 + } + + fun resetDroppedAudioFrames() { + droppedAudioFrames = 0 + } + + fun resetDroppedVideoFrames() { + droppedVideoFrames = 0 + } + + fun setLogs(enable: Boolean) { + isEnableLogs = enable + } + + fun setBitrateExponentialFactor(factor: Float) { + bitrateManager.exponentialFactor = factor + } + + fun getBitrateExponentialFactor() = bitrateManager.exponentialFactor +} \ No newline at end of file diff --git a/common/src/main/java/com/pedro/common/frame/MediaFrame.kt b/common/src/main/java/com/pedro/common/frame/MediaFrame.kt index 91accbf43..2959ef47f 100644 --- a/common/src/main/java/com/pedro/common/frame/MediaFrame.kt +++ b/common/src/main/java/com/pedro/common/frame/MediaFrame.kt @@ -4,8 +4,8 @@ import java.nio.ByteBuffer data class MediaFrame( val data: ByteBuffer, - val info: MediaFrame.Info, - val type: MediaFrame.Type + val info: Info, + val type: Type ) { data class Info( val offset: Int, diff --git a/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt b/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt deleted file mode 100644 index 4a273691c..000000000 --- a/common/src/main/java/com/pedro/common/frame/MediaFrameType.kt +++ /dev/null @@ -1,2 +0,0 @@ -package com.pedro.common.frame - diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt index 8af707888..f737144b3 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt @@ -18,12 +18,11 @@ package com.pedro.rtmp.rtmp import android.util.Log import com.pedro.common.AudioCodec -import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker import com.pedro.common.VideoCodec +import com.pedro.common.base.BaseSender import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread -import com.pedro.common.trySend import com.pedro.common.validMessage import com.pedro.rtmp.flv.BasePacket import com.pedro.rtmp.flv.FlvPacket @@ -34,53 +33,28 @@ import com.pedro.rtmp.flv.video.packet.Av1Packet import com.pedro.rtmp.flv.video.packet.H264Packet import com.pedro.rtmp.flv.video.packet.H265Packet import com.pedro.rtmp.utils.socket.RtmpSocket -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job import kotlinx.coroutines.async -import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.delay import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withContext import java.nio.ByteBuffer -import java.util.concurrent.BlockingQueue -import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit /** * Created by pedro on 8/04/21. */ class RtmpSender( - private val connectChecker: ConnectChecker, + connectChecker: ConnectChecker, private val commandsManager: CommandsManager -) { +): BaseSender(connectChecker, "RtmpSender") { private var audioPacket: BasePacket = AacPacket() private var videoPacket: BasePacket = H264Packet() - @Volatile - private var running = false - private var cacheSize = 200 - - private var job: Job? = null - private val scope = CoroutineScope(Dispatchers.IO) - @Volatile - private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) - private var audioFramesSent: Long = 0 - private var videoFramesSent: Long = 0 var socket: RtmpSocket? = null - var droppedAudioFrames: Long = 0 - private set - var droppedVideoFrames: Long = 0 - private set - private val bitrateManager: BitrateManager = BitrateManager(connectChecker) - private var isEnableLogs = true - - companion object { - private const val TAG = "RtmpSender" - } - fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { + override fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { videoPacket = when (commandsManager.videoCodec) { VideoCodec.H265 -> { if (vps == null || pps == null) throw IllegalArgumentException("pps or vps can't be null with h265") @@ -96,7 +70,7 @@ class RtmpSender( } } - fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { + override fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { audioPacket = when (commandsManager.audioCodec) { AudioCodec.G711 -> G711Packet().apply { sendAudioInfo() } AudioCodec.AAC -> AacPacket().apply { sendAudioInfo(sampleRate, isStereo) } @@ -106,82 +80,54 @@ class RtmpSender( } } - fun sendMediaFrame(mediaFrame: MediaFrame) { - if (running && !queue.trySend(mediaFrame)) { - when (mediaFrame.type) { - MediaFrame.Type.VIDEO -> { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } - MediaFrame.Type.AUDIO -> { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ - } + override suspend fun onRun() = withContext(Dispatchers.IO) { + var bytesSend = 0L + val bitrateTask = async { + while (scope.isActive && running) { + //bytes to bits + bitrateManager.calculateBitrate(bytesSend * 8) + bytesSend = 0 + delay(timeMillis = 1000) } } - } - - fun start() { - bitrateManager.reset() - queue.clear() - running = true - job = scope.launch { - var bytesSend = 0L - val bitrateTask = async { - while (scope.isActive && running) { - //bytes to bits - bitrateManager.calculateBitrate(bytesSend * 8) - bytesSend = 0 - delay(timeMillis = 1000) - } - } - while (scope.isActive && running) { - val error = runCatching { - val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } - getFlvPacket(mediaFrame) { flvPacket -> - var size = 0 - if (flvPacket.type == FlvType.VIDEO) { - videoFramesSent++ - socket?.let { socket -> - size = commandsManager.sendVideoPacket(flvPacket, socket) - if (isEnableLogs) { - Log.i(TAG, "wrote Video packet, size $size") - } + while (scope.isActive && running) { + val error = runCatching { + val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } + getFlvPacket(mediaFrame) { flvPacket -> + var size = 0 + if (flvPacket.type == FlvType.VIDEO) { + videoFramesSent++ + socket?.let { socket -> + size = commandsManager.sendVideoPacket(flvPacket, socket) + if (isEnableLogs) { + Log.i(TAG, "wrote Video packet, size $size") } - } else { - audioFramesSent++ - socket?.let { socket -> - size = commandsManager.sendAudioPacket(flvPacket, socket) - if (isEnableLogs) { - Log.i(TAG, "wrote Audio packet, size $size") - } + } + } else { + audioFramesSent++ + socket?.let { socket -> + size = commandsManager.sendAudioPacket(flvPacket, socket) + if (isEnableLogs) { + Log.i(TAG, "wrote Audio packet, size $size") } } - bytesSend += size - } - }.exceptionOrNull() - if (error != null) { - onMainThread { - connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") } - Log.e(TAG, "send error: ", error) - return@launch + bytesSend += size + } + }.exceptionOrNull() + if (error != null) { + onMainThread { + connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") } + Log.e(TAG, "send error: ", error) + return@withContext } } } - suspend fun stop(clear: Boolean = true) { - running = false + override suspend fun stopImp(clear: Boolean) { audioPacket.reset(clear) videoPacket.reset(clear) - resetSentAudioFrames() - resetSentVideoFrames() - resetDroppedAudioFrames() - resetDroppedVideoFrames() - job?.cancelAndJoin() - job = null - queue.clear() } private suspend fun getFlvPacket(mediaFrame: MediaFrame?, callback: suspend (FlvPacket) -> Unit) { @@ -199,66 +145,4 @@ class RtmpSender( } } } - - @Throws(IllegalArgumentException::class) - fun hasCongestion(percentUsed: Float = 20f): Boolean { - if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") - val size = queue.size.toFloat() - val remaining = queue.remainingCapacity().toFloat() - val capacity = size + remaining - return size >= capacity * (percentUsed / 100f) - } - - fun resizeCache(newSize: Int) { - if (newSize < queue.size - queue.remainingCapacity()) { - throw RuntimeException("Can't fit current cache inside new cache size") - } - val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) - queue.drainTo(tempQueue) - queue = tempQueue - } - - fun getCacheSize(): Int { - return cacheSize - } - - fun getItemsInCache(): Int = queue.size - - fun clearCache() { - queue.clear() - } - - fun getSentAudioFrames(): Long { - return audioFramesSent - } - - fun getSentVideoFrames(): Long { - return videoFramesSent - } - - fun resetSentAudioFrames() { - audioFramesSent = 0 - } - - fun resetSentVideoFrames() { - videoFramesSent = 0 - } - - fun resetDroppedAudioFrames() { - droppedAudioFrames = 0 - } - - fun resetDroppedVideoFrames() { - droppedVideoFrames = 0 - } - - fun setLogs(enable: Boolean) { - isEnableLogs = enable - } - - fun setBitrateExponentialFactor(factor: Float) { - bitrateManager.exponentialFactor = factor - } - - fun getBitrateExponentialFactor() = bitrateManager.exponentialFactor } \ No newline at end of file diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index 6b1d5c0a9..26e479485 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -22,6 +22,7 @@ import com.pedro.common.isKeyframe import com.pedro.common.removeInfo import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants +import com.pedro.rtsp.utils.getData import com.pedro.rtsp.utils.getVideoStartCodeSize import java.nio.ByteBuffer import kotlin.experimental.and @@ -44,8 +45,8 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, channelIdentifier = RtpConstants.trackVideo } - fun sendVideoInfo(sps: ByteArray, pps: ByteArray) { - setSpsPps(sps, pps) + fun sendVideoInfo(sps: ByteBuffer, pps: ByteBuffer) { + setSpsPps(sps.getData(), pps.getData()) } override suspend fun createAndSendPacket( diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt index 1ebd6ffaf..7b844ecea 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt @@ -227,7 +227,7 @@ class RtspClient(private val connectChecker: ConnectChecker) { commandsManager.audioServerPorts ) if (!commandsManager.audioDisabled) { - rtspSender.setAudioInfo(commandsManager.sampleRate) + rtspSender.setAudioInfo(commandsManager.sampleRate, commandsManager.isStereo) } if (!commandsManager.videoDisabled) { if (!commandsManager.videoInfoReady()) { diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt index bd517ce77..81522261e 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt @@ -18,13 +18,12 @@ package com.pedro.rtsp.rtsp import android.util.Log import com.pedro.common.AudioCodec -import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker import com.pedro.common.VideoCodec +import com.pedro.common.base.BaseSender import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread import com.pedro.common.socket.TcpStreamSocket -import com.pedro.common.trySend import com.pedro.common.validMessage import com.pedro.rtsp.rtcp.BaseSenderReport import com.pedro.rtsp.rtp.packets.* @@ -32,15 +31,12 @@ import com.pedro.rtsp.rtp.sockets.BaseRtpSocket import com.pedro.rtsp.rtp.sockets.RtpSocketTcp import com.pedro.rtsp.rtsp.commands.CommandsManager import com.pedro.rtsp.utils.RtpConstants -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job import kotlinx.coroutines.async -import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.delay import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withContext import java.io.IOException import java.nio.ByteBuffer import java.util.* @@ -50,37 +46,15 @@ import java.util.concurrent.* * Created by pedro on 7/11/18. */ class RtspSender( - private val connectChecker: ConnectChecker, + connectChecker: ConnectChecker, private val commandsManager: CommandsManager -) { +): BaseSender(connectChecker, "RtspSender") { private var videoPacket: BasePacket = H264Packet() private var audioPacket: BasePacket = AacPacket() private var rtpSocket: BaseRtpSocket? = null private var baseSenderReport: BaseSenderReport? = null - private var cacheSize = 200 - @Volatile - private var running = false - - private var job: Job? = null - private val scope = CoroutineScope(Dispatchers.IO) - @Volatile - private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) - - private var audioFramesSent: Long = 0 - private var videoFramesSent: Long = 0 - var droppedAudioFrames: Long = 0 - private set - var droppedVideoFrames: Long = 0 - private set - private val bitrateManager: BitrateManager = BitrateManager(connectChecker) - private var isEnableLogs = true - - companion object { - private const val TAG = "RtspSender" - } - @Throws(IOException::class) fun setSocketsInfo( protocol: Protocol, host: String, @@ -91,7 +65,13 @@ class RtspSender( baseSenderReport = BaseSenderReport.getInstance(protocol, host, videoSourcePorts[1], audioSourcePorts[1], videoServerPorts[1], audioServerPorts[1]) } - fun setVideoInfo(sps: ByteArray, pps: ByteArray?, vps: ByteArray?) { + @Throws(IOException::class) + suspend fun setSocket(socket: TcpStreamSocket) { + rtpSocket?.setSocket(socket) + baseSenderReport?.setSocket(socket) + } + + override fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { videoPacket = when (commandsManager.videoCodec) { VideoCodec.H264 -> { if (pps == null) throw IllegalArgumentException("pps can't be null with h264") @@ -105,7 +85,7 @@ class RtspSender( } } - fun setAudioInfo(sampleRate: Int) { + override fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { audioPacket = when (commandsManager.audioCodec) { AudioCodec.G711 -> G711Packet().apply { setAudioInfo(sampleRate) } AudioCodec.AAC -> AacPacket().apply { setAudioInfo(sampleRate) } @@ -113,104 +93,70 @@ class RtspSender( } } - @Throws(IOException::class) - suspend fun setSocket(socket: TcpStreamSocket) { - rtpSocket?.setSocket(socket) - baseSenderReport?.setSocket(socket) - } - - fun sendMediaFrame(mediaFrame: MediaFrame) { - if (running && !queue.trySend(mediaFrame)) { - when (mediaFrame.type) { - MediaFrame.Type.VIDEO -> { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } - MediaFrame.Type.AUDIO -> { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ - } - } - } - } - - fun start() { - bitrateManager.reset() - queue.clear() + override suspend fun onRun() = withContext(Dispatchers.IO) { val ssrcVideo = Random().nextInt().toLong() val ssrcAudio = Random().nextInt().toLong() baseSenderReport?.setSSRC(ssrcVideo, ssrcAudio) videoPacket.setSSRC(ssrcVideo) audioPacket.setSSRC(ssrcAudio) - running = true - job = scope.launch { - val isTcp = rtpSocket is RtpSocketTcp - var bytesSend = 0L - val bitrateTask = async { - while (scope.isActive && running) { - //bytes to bits - bitrateManager.calculateBitrate(bytesSend * 8) - bytesSend = 0 - delay(timeMillis = 1000) - } - } + val isTcp = rtpSocket is RtpSocketTcp + var bytesSend = 0L + val bitrateTask = async { while (scope.isActive && running) { - val error = runCatching { - val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } - getRtpPackets(mediaFrame) { rtpFrames -> - var size = 0 - var isVideo = false - rtpFrames.forEach { rtpFrame -> - rtpSocket?.sendFrame(rtpFrame) - //4 is tcp header length - val packetSize = if (isTcp) rtpFrame.length + 4 else rtpFrame.length - bytesSend += packetSize - size += packetSize - isVideo = rtpFrame.isVideoFrame() - if (isVideo) { - videoFramesSent++ - } else { - audioFramesSent++ - } - if (baseSenderReport?.update(rtpFrame) == true) { - //4 is tcp header length - val reportSize = if (isTcp) RtpConstants.REPORT_PACKET_LENGTH + 4 else RtpConstants.REPORT_PACKET_LENGTH - bytesSend += reportSize - if (isEnableLogs) Log.i(TAG, "wrote report") - } + //bytes to bits + bitrateManager.calculateBitrate(bytesSend * 8) + bytesSend = 0 + delay(timeMillis = 1000) + } + } + while (scope.isActive && running) { + val error = runCatching { + val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } + getRtpPackets(mediaFrame) { rtpFrames -> + var size = 0 + var isVideo = false + rtpFrames.forEach { rtpFrame -> + rtpSocket?.sendFrame(rtpFrame) + //4 is tcp header length + val packetSize = if (isTcp) rtpFrame.length + 4 else rtpFrame.length + bytesSend += packetSize + size += packetSize + isVideo = rtpFrame.isVideoFrame() + if (isVideo) { + videoFramesSent++ + } else { + audioFramesSent++ } - rtpSocket?.flush() - if (isEnableLogs) { - val type = if (isVideo) "Video" else "Audio" - Log.i(TAG, "wrote $type packet, size $size") + if (baseSenderReport?.update(rtpFrame) == true) { + //4 is tcp header length + val reportSize = if (isTcp) RtpConstants.REPORT_PACKET_LENGTH + 4 else RtpConstants.REPORT_PACKET_LENGTH + bytesSend += reportSize + if (isEnableLogs) Log.i(TAG, "wrote report") } } - }.exceptionOrNull() - if (error != null) { - onMainThread { - connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") + rtpSocket?.flush() + if (isEnableLogs) { + val type = if (isVideo) "Video" else "Audio" + Log.i(TAG, "wrote $type packet, size $size") } - Log.e(TAG, "send error: ", error) - return@launch } + }.exceptionOrNull() + if (error != null) { + onMainThread { + connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") + } + Log.e(TAG, "send error: ", error) + return@withContext } } } - suspend fun stop() { - running = false + override suspend fun stopImp(clear: Boolean) { baseSenderReport?.reset() baseSenderReport?.close() rtpSocket?.close() audioPacket.reset() videoPacket.reset() - resetSentAudioFrames() - resetSentVideoFrames() - resetDroppedAudioFrames() - resetDroppedVideoFrames() - job?.cancelAndJoin() - job = null - queue.clear() } private suspend fun getRtpPackets(mediaFrame: MediaFrame?, callback: suspend (List) -> Unit) { @@ -228,66 +174,4 @@ class RtspSender( } } } - - @Throws(IllegalArgumentException::class) - fun hasCongestion(percentUsed: Float = 20f): Boolean { - if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") - val size = queue.size.toFloat() - val remaining = queue.remainingCapacity().toFloat() - val capacity = size + remaining - return size >= capacity * (percentUsed / 100f) - } - - fun resizeCache(newSize: Int) { - if (newSize < queue.size - queue.remainingCapacity()) { - throw RuntimeException("Can't fit current cache inside new cache size") - } - val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) - queue.drainTo(tempQueue) - queue = tempQueue - } - - fun getCacheSize(): Int { - return cacheSize - } - - fun getItemsInCache(): Int = queue.size - - fun clearCache() { - queue.clear() - } - - fun getSentAudioFrames(): Long { - return audioFramesSent - } - - fun getSentVideoFrames(): Long { - return videoFramesSent - } - - fun resetSentAudioFrames() { - audioFramesSent = 0 - } - - fun resetSentVideoFrames() { - videoFramesSent = 0 - } - - fun resetDroppedAudioFrames() { - droppedAudioFrames = 0 - } - - fun resetDroppedVideoFrames() { - droppedVideoFrames = 0 - } - - fun setLogs(enable: Boolean) { - isEnableLogs = enable - } - - fun setBitrateExponentialFactor(factor: Float) { - bitrateManager.exponentialFactor = factor - } - - fun getBitrateExponentialFactor() = bitrateManager.exponentialFactor } \ No newline at end of file diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/commands/CommandsManager.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/commands/CommandsManager.kt index ab77ea259..5c44325e7 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/commands/CommandsManager.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/commands/CommandsManager.kt @@ -49,9 +49,9 @@ open class CommandsManager { private set var path: String? = null private set - var sps: ByteArray? = null + var sps: ByteBuffer? = null private set - var pps: ByteArray? = null + var pps: ByteBuffer? = null private set private var cSeq = 0 private var sessionId: String? = null @@ -71,7 +71,7 @@ open class CommandsManager { val videoServerPorts = intArrayOf(5006, 5007) //For H265 - var vps: ByteArray? = null + var vps: ByteBuffer? = null private set //For auth @@ -100,9 +100,9 @@ open class CommandsManager { } fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { - this.sps = sps.getData() - this.pps = pps?.getData() - this.vps = vps?.getData() + this.sps = sps + this.pps = pps + this.vps = vps } fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { @@ -134,11 +134,11 @@ open class CommandsManager { } private val spsString: String - get() = sps?.encodeToString() ?: "" + get() = sps?.getData()?.encodeToString() ?: "" private val ppsString: String - get() = pps?.encodeToString() ?: "" + get() = pps?.getData()?.encodeToString() ?: "" private val vpsString: String - get() = vps?.encodeToString() ?: "" + get() = vps?.getData()?.encodeToString() ?: "" private fun addHeaders(): String { return "CSeq: ${++cSeq}\r\n" + diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt index df0531fea..a4c838de6 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt @@ -18,11 +18,10 @@ package com.pedro.srt.srt import android.util.Log import com.pedro.common.AudioCodec -import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker +import com.pedro.common.base.BaseSender import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread -import com.pedro.common.trySend import com.pedro.common.validMessage import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegTsPacketizer @@ -36,27 +35,22 @@ import com.pedro.srt.mpeg2ts.service.Mpeg2TsService import com.pedro.srt.srt.packets.SrtPacket import com.pedro.srt.utils.SrtSocket import com.pedro.srt.utils.toCodec -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job import kotlinx.coroutines.async -import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.delay import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withContext import java.nio.ByteBuffer -import java.util.concurrent.BlockingQueue -import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit /** * Created by pedro on 20/8/23. */ class SrtSender( - private val connectChecker: ConnectChecker, + connectChecker: ConnectChecker, private val commandsManager: CommandsManager -) { +): BaseSender(connectChecker, "SrtSender") { private val service = Mpeg2TsService() @@ -68,29 +62,7 @@ class SrtSender( private val mpegTsPacketizer = MpegTsPacketizer(psiManager) private var audioPacket: BasePacket = AacPacket(limitSize, psiManager) private val videoPacket = H26XPacket(limitSize, psiManager) - - @Volatile - private var running = false - private var cacheSize = 200 - - private var job: Job? = null - private val scope = CoroutineScope(Dispatchers.IO) - @Volatile - private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) - private var audioFramesSent: Long = 0 - private var videoFramesSent: Long = 0 var socket: SrtSocket? = null - var droppedAudioFrames: Long = 0 - private set - var droppedVideoFrames: Long = 0 - private set - - private val bitrateManager: BitrateManager = BitrateManager(connectChecker) - private var isEnableLogs = true - - companion object { - private const val TAG = "SrtSender" - } private fun setTrackConfig(videoEnabled: Boolean, audioEnabled: Boolean) { Pid.reset() @@ -101,12 +73,12 @@ class SrtSender( psiManager.updateService(service) } - fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { + override fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { videoPacket.setVideoCodec(commandsManager.videoCodec.toCodec()) videoPacket.sendVideoInfo(sps, pps, vps) } - fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { + override fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { audioPacket = when (commandsManager.audioCodec) { AudioCodec.AAC -> AacPacket(limitSize, psiManager).apply { sendAudioInfo(sampleRate, isStereo) } AudioCodec.OPUS -> OpusPacket(limitSize, psiManager) @@ -116,57 +88,45 @@ class SrtSender( } } - fun sendMediaFrame(mediaFrame: MediaFrame) { - if (running && !queue.trySend(mediaFrame)) { - when (mediaFrame.type) { - MediaFrame.Type.VIDEO -> { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } - MediaFrame.Type.AUDIO -> { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ - } + override suspend fun onRun() = withContext(Dispatchers.IO) { + setTrackConfig(!commandsManager.videoDisabled, !commandsManager.audioDisabled) + var bytesSend = 0L + val bitrateTask = async { + while (scope.isActive && running) { + //bytes to bits + bitrateManager.calculateBitrate(bytesSend * 8) + bytesSend = 0 + delay(timeMillis = 1000) } } - } - - fun start() { - bitrateManager.reset() - queue.clear() - setTrackConfig(!commandsManager.videoDisabled, !commandsManager.audioDisabled) - running = true - job = scope.launch { - var bytesSend = 0L - val bitrateTask = async { - while (scope.isActive && running) { - //bytes to bits - bitrateManager.calculateBitrate(bytesSend * 8) - bytesSend = 0 - delay(timeMillis = 1000) + while (scope.isActive && running) { + val error = runCatching { + val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } + getMpegTsPackets(mediaFrame) { mpegTsPackets -> + val isKey = mpegTsPackets[0].isKey + val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) + bytesSend += sendPackets(psiPackets) + bytesSend += sendPackets(mpegTsPackets) } - } - while (scope.isActive && running) { - val error = runCatching { - val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } - getMpegTsPackets(mediaFrame) { mpegTsPackets -> - val isKey = mpegTsPackets[0].isKey - val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) - bytesSend += sendPackets(psiPackets) - bytesSend += sendPackets(mpegTsPackets) - } - }.exceptionOrNull() - if (error != null) { - onMainThread { - connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") - } - Log.e(TAG, "send error: ", error) - return@launch + }.exceptionOrNull() + if (error != null) { + onMainThread { + connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") } + Log.e(TAG, "send error: ", error) + return@withContext } } } + override suspend fun stopImp(clear: Boolean) { + psiManager.reset() + service.clear() + mpegTsPacketizer.reset() + audioPacket.reset(clear) + videoPacket.reset(clear) + } + private suspend fun sendPackets(packets: List): Long { var bytesSend = 0L packets.forEach { mpegTsPacket -> @@ -180,22 +140,6 @@ class SrtSender( return bytesSend } - suspend fun stop(clear: Boolean) { - running = false - psiManager.reset() - service.clear() - mpegTsPacketizer.reset() - audioPacket.reset(clear) - videoPacket.reset(clear) - resetSentAudioFrames() - resetSentVideoFrames() - resetDroppedAudioFrames() - resetDroppedVideoFrames() - job?.cancelAndJoin() - job = null - queue.clear() - } - private suspend fun getMpegTsPackets(mediaFrame: MediaFrame?, callback: suspend (List) -> Unit) { if (mediaFrame == null) return when (mediaFrame.type) { @@ -211,66 +155,4 @@ class SrtSender( } } } - - @Throws(IllegalArgumentException::class) - fun hasCongestion(percentUsed: Float = 20f): Boolean { - if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") - val size = queue.size.toFloat() - val remaining = queue.remainingCapacity().toFloat() - val capacity = size + remaining - return size >= capacity * (percentUsed / 100f) - } - - fun resizeCache(newSize: Int) { - if (newSize < queue.size - queue.remainingCapacity()) { - throw RuntimeException("Can't fit current cache inside new cache size") - } - val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) - queue.drainTo(tempQueue) - queue = tempQueue - } - - fun getCacheSize(): Int { - return cacheSize - } - - fun getItemsInCache(): Int = queue.size - - fun clearCache() { - queue.clear() - } - - fun getSentAudioFrames(): Long { - return audioFramesSent - } - - fun getSentVideoFrames(): Long { - return videoFramesSent - } - - fun resetSentAudioFrames() { - audioFramesSent = 0 - } - - fun resetSentVideoFrames() { - videoFramesSent = 0 - } - - fun resetDroppedAudioFrames() { - droppedAudioFrames = 0 - } - - fun resetDroppedVideoFrames() { - droppedVideoFrames = 0 - } - - fun setLogs(enable: Boolean) { - isEnableLogs = enable - } - - fun setBitrateExponentialFactor(factor: Float) { - bitrateManager.exponentialFactor = factor - } - - fun getBitrateExponentialFactor() = bitrateManager.exponentialFactor } \ No newline at end of file diff --git a/udp/src/main/java/com/pedro/udp/UdpSender.kt b/udp/src/main/java/com/pedro/udp/UdpSender.kt index 86c0dff11..524c9fa4e 100644 --- a/udp/src/main/java/com/pedro/udp/UdpSender.kt +++ b/udp/src/main/java/com/pedro/udp/UdpSender.kt @@ -18,11 +18,10 @@ package com.pedro.udp import android.util.Log import com.pedro.common.AudioCodec -import com.pedro.common.BitrateManager import com.pedro.common.ConnectChecker +import com.pedro.common.base.BaseSender import com.pedro.common.frame.MediaFrame import com.pedro.common.onMainThread -import com.pedro.common.trySend import com.pedro.common.validMessage import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegTsPacketizer @@ -33,35 +32,27 @@ import com.pedro.srt.mpeg2ts.packets.H26XPacket import com.pedro.srt.mpeg2ts.packets.OpusPacket import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.srt.mpeg2ts.service.Mpeg2TsService -import com.pedro.srt.srt.SrtSender -import com.pedro.srt.srt.SrtSender.Companion import com.pedro.srt.utils.Constants import com.pedro.srt.utils.toCodec import com.pedro.udp.utils.UdpSocket -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job import kotlinx.coroutines.async -import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.delay import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withContext import java.nio.ByteBuffer -import java.util.concurrent.BlockingQueue -import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit /** * Created by pedro on 6/3/24. */ class UdpSender( - private val connectChecker: ConnectChecker, + connectChecker: ConnectChecker, private val commandManager: CommandManager -) { +): BaseSender(connectChecker, "SrtSender") { private val service = Mpeg2TsService() - private val psiManager = PsiManager(service).apply { upgradePatVersion() upgradeSdtVersion() @@ -70,29 +61,7 @@ class UdpSender( private val mpegTsPacketizer = MpegTsPacketizer(psiManager) private var audioPacket: BasePacket = AacPacket(limitSize, psiManager) private val videoPacket = H26XPacket(limitSize, psiManager) - - @Volatile - private var running = false - private var cacheSize = 200 - - private var job: Job? = null - private val scope = CoroutineScope(Dispatchers.IO) - @Volatile - private var queue: BlockingQueue = LinkedBlockingQueue(cacheSize) - private var audioFramesSent: Long = 0 - private var videoFramesSent: Long = 0 var socket: UdpSocket? = null - var droppedAudioFrames: Long = 0 - private set - var droppedVideoFrames: Long = 0 - private set - - private val bitrateManager: BitrateManager = BitrateManager(connectChecker) - private var isEnableLogs = true - - companion object { - private const val TAG = "SrtSender" - } private fun setTrackConfig(videoEnabled: Boolean, audioEnabled: Boolean) { Pid.reset() @@ -103,12 +72,12 @@ class UdpSender( psiManager.updateService(service) } - fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { + override fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) { videoPacket.setVideoCodec(commandManager.videoCodec.toCodec()) videoPacket.sendVideoInfo(sps, pps, vps) } - fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { + override fun setAudioInfo(sampleRate: Int, isStereo: Boolean) { audioPacket = when (commandManager.audioCodec) { AudioCodec.AAC -> AacPacket(limitSize, psiManager).apply { sendAudioInfo(sampleRate, isStereo) } AudioCodec.OPUS -> OpusPacket(limitSize, psiManager) @@ -118,57 +87,45 @@ class UdpSender( } } - fun sendMediaFrame(mediaFrame: MediaFrame) { - if (running && !queue.trySend(mediaFrame)) { - when (mediaFrame.type) { - MediaFrame.Type.VIDEO -> { - Log.i(TAG, "Video frame discarded") - droppedVideoFrames++ - } - MediaFrame.Type.AUDIO -> { - Log.i(TAG, "Audio frame discarded") - droppedAudioFrames++ - } + override suspend fun onRun() = withContext(Dispatchers.IO) { + setTrackConfig(!commandManager.videoDisabled, !commandManager.audioDisabled) + var bytesSend = 0L + val bitrateTask = async { + while (scope.isActive && running) { + //bytes to bits + bitrateManager.calculateBitrate(bytesSend * 8) + bytesSend = 0 + delay(timeMillis = 1000) } } - } - - fun start() { - bitrateManager.reset() - queue.clear() - setTrackConfig(!commandManager.videoDisabled, !commandManager.audioDisabled) - running = true - job = scope.launch { - var bytesSend = 0L - val bitrateTask = async { - while (scope.isActive && running) { - //bytes to bits - bitrateManager.calculateBitrate(bytesSend * 8) - bytesSend = 0 - delay(timeMillis = 1000) + while (scope.isActive && running) { + val error = runCatching { + val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } + getMpegTsPackets(mediaFrame) { mpegTsPackets -> + val isKey = mpegTsPackets[0].isKey + val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) + bytesSend += sendPackets(psiPackets) + bytesSend += sendPackets(mpegTsPackets) } - } - while (scope.isActive && running) { - val error = runCatching { - val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } - getMpegTsPackets(mediaFrame) { mpegTsPackets -> - val isKey = mpegTsPackets[0].isKey - val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) - bytesSend += sendPackets(psiPackets) - bytesSend += sendPackets(mpegTsPackets) - } - }.exceptionOrNull() - if (error != null) { - onMainThread { - connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") - } - Log.e(TAG, "send error: ", error) - return@launch + }.exceptionOrNull() + if (error != null) { + onMainThread { + connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") } + Log.e(TAG, "send error: ", error) + return@withContext } } } + override suspend fun stopImp(clear: Boolean) { + psiManager.reset() + service.clear() + mpegTsPacketizer.reset() + audioPacket.reset(clear) + videoPacket.reset(clear) + } + private suspend fun sendPackets(packets: List): Long { var bytesSend = 0L packets.forEach { mpegTsPacket -> @@ -182,22 +139,6 @@ class UdpSender( return bytesSend } - suspend fun stop(clear: Boolean) { - running = false - psiManager.reset() - service.clear() - mpegTsPacketizer.reset() - audioPacket.reset(clear) - videoPacket.reset(clear) - resetSentAudioFrames() - resetSentVideoFrames() - resetDroppedAudioFrames() - resetDroppedVideoFrames() - job?.cancelAndJoin() - job = null - queue.clear() - } - private suspend fun getMpegTsPackets(mediaFrame: MediaFrame?, callback: suspend (List) -> Unit) { if (mediaFrame == null) return when (mediaFrame.type) { @@ -213,66 +154,4 @@ class UdpSender( } } } - - @Throws(IllegalArgumentException::class) - fun hasCongestion(percentUsed: Float = 20f): Boolean { - if (percentUsed < 0 || percentUsed > 100) throw IllegalArgumentException("the value must be in range 0 to 100") - val size = queue.size.toFloat() - val remaining = queue.remainingCapacity().toFloat() - val capacity = size + remaining - return size >= capacity * (percentUsed / 100f) - } - - fun resizeCache(newSize: Int) { - if (newSize < queue.size - queue.remainingCapacity()) { - throw RuntimeException("Can't fit current cache inside new cache size") - } - val tempQueue: BlockingQueue = LinkedBlockingQueue(newSize) - queue.drainTo(tempQueue) - queue = tempQueue - } - - fun getCacheSize(): Int { - return cacheSize - } - - fun getItemsInCache(): Int = queue.size - - fun clearCache() { - queue.clear() - } - - fun getSentAudioFrames(): Long { - return audioFramesSent - } - - fun getSentVideoFrames(): Long { - return videoFramesSent - } - - fun resetSentAudioFrames() { - audioFramesSent = 0 - } - - fun resetSentVideoFrames() { - videoFramesSent = 0 - } - - fun resetDroppedAudioFrames() { - droppedAudioFrames = 0 - } - - fun resetDroppedVideoFrames() { - droppedVideoFrames = 0 - } - - fun setLogs(enable: Boolean) { - isEnableLogs = enable - } - - fun setBitrateExponentialFactor(factor: Float) { - bitrateManager.exponentialFactor = factor - } - - fun getBitrateExponentialFactor() = bitrateManager.exponentialFactor } \ No newline at end of file From 32add6a0d84a1f761b392ff6edda2fbf20632541 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 16:43:22 +0200 Subject: [PATCH 05/18] move bitrateTask to base --- .../java/com/pedro/common/base/BaseSender.kt | 17 ++++++++++++++++- .../java/com/pedro/rtmp/rtmp/RtmpSender.kt | 18 +++--------------- .../java/com/pedro/rtsp/rtsp/RtspSender.kt | 18 +++--------------- .../main/java/com/pedro/srt/srt/SrtSender.kt | 18 +++--------------- udp/src/main/java/com/pedro/udp/UdpSender.kt | 18 +++--------------- 5 files changed, 28 insertions(+), 61 deletions(-) diff --git a/common/src/main/java/com/pedro/common/base/BaseSender.kt b/common/src/main/java/com/pedro/common/base/BaseSender.kt index 811c57621..95e6f97b2 100644 --- a/common/src/main/java/com/pedro/common/base/BaseSender.kt +++ b/common/src/main/java/com/pedro/common/base/BaseSender.kt @@ -8,7 +8,10 @@ import com.pedro.common.trySend import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job +import kotlinx.coroutines.async import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import java.nio.ByteBuffer import java.util.concurrent.BlockingQueue @@ -34,6 +37,8 @@ abstract class BaseSender( protected var isEnableLogs = true private var job: Job? = null protected val scope = CoroutineScope(Dispatchers.IO) + @Volatile + protected var bytesSend = 0L abstract fun setVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) abstract fun setAudioInfo(sampleRate: Int, isStereo: Boolean) @@ -59,7 +64,17 @@ abstract class BaseSender( bitrateManager.reset() queue.clear() running = true - scope.launch { onRun() } + scope.launch { + val bitrateTask = async { + while (scope.isActive && running) { + //bytes to bits + bitrateManager.calculateBitrate(bytesSend * 8) + bytesSend = 0 + delay(timeMillis = 1000) + } + } + onRun() + } } suspend fun stop(clear: Boolean = true) { diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt index f737144b3..044d0967b 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt @@ -33,12 +33,8 @@ import com.pedro.rtmp.flv.video.packet.Av1Packet import com.pedro.rtmp.flv.video.packet.H264Packet import com.pedro.rtmp.flv.video.packet.H265Packet import com.pedro.rtmp.utils.socket.RtmpSocket -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.runInterruptible -import kotlinx.coroutines.withContext import java.nio.ByteBuffer import java.util.concurrent.TimeUnit @@ -80,16 +76,7 @@ class RtmpSender( } } - override suspend fun onRun() = withContext(Dispatchers.IO) { - var bytesSend = 0L - val bitrateTask = async { - while (scope.isActive && running) { - //bytes to bits - bitrateManager.calculateBitrate(bytesSend * 8) - bytesSend = 0 - delay(timeMillis = 1000) - } - } + override suspend fun onRun() { while (scope.isActive && running) { val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } @@ -120,7 +107,8 @@ class RtmpSender( connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") } Log.e(TAG, "send error: ", error) - return@withContext + running = false + return } } } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt index 81522261e..5265be4ad 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt @@ -31,12 +31,8 @@ import com.pedro.rtsp.rtp.sockets.BaseRtpSocket import com.pedro.rtsp.rtp.sockets.RtpSocketTcp import com.pedro.rtsp.rtsp.commands.CommandsManager import com.pedro.rtsp.utils.RtpConstants -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.runInterruptible -import kotlinx.coroutines.withContext import java.io.IOException import java.nio.ByteBuffer import java.util.* @@ -93,22 +89,13 @@ class RtspSender( } } - override suspend fun onRun() = withContext(Dispatchers.IO) { + override suspend fun onRun() { val ssrcVideo = Random().nextInt().toLong() val ssrcAudio = Random().nextInt().toLong() baseSenderReport?.setSSRC(ssrcVideo, ssrcAudio) videoPacket.setSSRC(ssrcVideo) audioPacket.setSSRC(ssrcAudio) val isTcp = rtpSocket is RtpSocketTcp - var bytesSend = 0L - val bitrateTask = async { - while (scope.isActive && running) { - //bytes to bits - bitrateManager.calculateBitrate(bytesSend * 8) - bytesSend = 0 - delay(timeMillis = 1000) - } - } while (scope.isActive && running) { val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } @@ -146,7 +133,8 @@ class RtspSender( connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") } Log.e(TAG, "send error: ", error) - return@withContext + running = false + return } } } diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt index a4c838de6..3a36b521e 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt @@ -35,12 +35,8 @@ import com.pedro.srt.mpeg2ts.service.Mpeg2TsService import com.pedro.srt.srt.packets.SrtPacket import com.pedro.srt.utils.SrtSocket import com.pedro.srt.utils.toCodec -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.runInterruptible -import kotlinx.coroutines.withContext import java.nio.ByteBuffer import java.util.concurrent.TimeUnit @@ -88,17 +84,8 @@ class SrtSender( } } - override suspend fun onRun() = withContext(Dispatchers.IO) { + override suspend fun onRun() { setTrackConfig(!commandsManager.videoDisabled, !commandsManager.audioDisabled) - var bytesSend = 0L - val bitrateTask = async { - while (scope.isActive && running) { - //bytes to bits - bitrateManager.calculateBitrate(bytesSend * 8) - bytesSend = 0 - delay(timeMillis = 1000) - } - } while (scope.isActive && running) { val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } @@ -114,7 +101,8 @@ class SrtSender( connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") } Log.e(TAG, "send error: ", error) - return@withContext + running = false + return } } } diff --git a/udp/src/main/java/com/pedro/udp/UdpSender.kt b/udp/src/main/java/com/pedro/udp/UdpSender.kt index 524c9fa4e..d85b2a8c5 100644 --- a/udp/src/main/java/com/pedro/udp/UdpSender.kt +++ b/udp/src/main/java/com/pedro/udp/UdpSender.kt @@ -35,12 +35,8 @@ import com.pedro.srt.mpeg2ts.service.Mpeg2TsService import com.pedro.srt.utils.Constants import com.pedro.srt.utils.toCodec import com.pedro.udp.utils.UdpSocket -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.runInterruptible -import kotlinx.coroutines.withContext import java.nio.ByteBuffer import java.util.concurrent.TimeUnit @@ -87,17 +83,8 @@ class UdpSender( } } - override suspend fun onRun() = withContext(Dispatchers.IO) { + override suspend fun onRun() { setTrackConfig(!commandManager.videoDisabled, !commandManager.audioDisabled) - var bytesSend = 0L - val bitrateTask = async { - while (scope.isActive && running) { - //bytes to bits - bitrateManager.calculateBitrate(bytesSend * 8) - bytesSend = 0 - delay(timeMillis = 1000) - } - } while (scope.isActive && running) { val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } @@ -113,7 +100,8 @@ class UdpSender( connectChecker.onConnectionFailed("Error send packet, ${error.validMessage()}") } Log.e(TAG, "send error: ", error) - return@withContext + running = false + return } } } From 3ce4201777277d0800d59fa71d948a358f7283c9 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 17:05:57 +0200 Subject: [PATCH 06/18] refactor mpegts wrote packet logcat --- .../pedro/streamer/rotation/CameraFragment.kt | 6 +++--- .../com/pedro/srt/mpeg2ts/packets/AacPacket.kt | 2 +- .../com/pedro/srt/mpeg2ts/packets/H26XPacket.kt | 2 +- .../com/pedro/srt/mpeg2ts/packets/OpusPacket.kt | 2 +- .../main/java/com/pedro/srt/srt/SrtSender.kt | 17 ++++++++++------- udp/src/main/java/com/pedro/udp/UdpSender.kt | 17 ++++++++++------- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt b/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt index 9631ebd14..c32d311fb 100644 --- a/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt +++ b/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt @@ -83,9 +83,9 @@ class CameraFragment: Fragment(), ConnectChecker { private lateinit var surfaceView: SurfaceView private lateinit var bStartStop: ImageView private lateinit var txtBitrate: TextView - private val width = 3840 - private val height = 2160 - private val vBitrate = 20000 * 1000 + private val width = 640 + private val height = 480 + private val vBitrate = 1200 * 1000 private var rotation = 0 private val sampleRate = 32000 private val isStereo = true diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt index d116a2650..3d4b21cf0 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/AacPacket.kt @@ -64,7 +64,7 @@ class AacPacket( val packetPosition = PacketPosition.SINGLE packets.add(MpegTsPacket(buffer.array(), MpegType.AUDIO, packetPosition, false)) } - callback(packets) + if (packets.isNotEmpty()) callback(packets) } override fun resetPacket(resetInfo: Boolean) { diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt index 4ac489bc5..baf3b10d8 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt @@ -92,7 +92,7 @@ class H26XPacket( val packetPosition = PacketPosition.SINGLE packets.add(MpegTsPacket(buffer.array(), MpegType.VIDEO, packetPosition, isKeyFrame)) } - callback(packets) + if (packets.isNotEmpty()) callback(packets) } override fun resetPacket(resetInfo: Boolean) { diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt index 8a236fbe8..817db1b44 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt @@ -63,7 +63,7 @@ class OpusPacket( val packetPosition = PacketPosition.SINGLE packets.add(MpegTsPacket(buffer.array(), MpegType.AUDIO, packetPosition, true)) } - callback(packets) + if (packets.isNotEmpty()) callback(packets) } override fun resetPacket(resetInfo: Boolean) { } diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt index 3a36b521e..b84fe03eb 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt @@ -25,6 +25,7 @@ import com.pedro.common.onMainThread import com.pedro.common.validMessage import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegTsPacketizer +import com.pedro.srt.mpeg2ts.MpegType import com.pedro.srt.mpeg2ts.Pid import com.pedro.srt.mpeg2ts.packets.AacPacket import com.pedro.srt.mpeg2ts.packets.BasePacket @@ -90,10 +91,11 @@ class SrtSender( val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } getMpegTsPackets(mediaFrame) { mpegTsPackets -> - val isKey = mpegTsPackets[0].isKey + val firstPacket = mpegTsPackets[0] + val isKey = firstPacket.isKey val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) - bytesSend += sendPackets(psiPackets) - bytesSend += sendPackets(mpegTsPackets) + bytesSend += sendPackets(psiPackets, MpegType.PSI) + bytesSend += sendPackets(mpegTsPackets, firstPacket.type) } }.exceptionOrNull() if (error != null) { @@ -115,16 +117,17 @@ class SrtSender( videoPacket.reset(clear) } - private suspend fun sendPackets(packets: List): Long { + private suspend fun sendPackets(packets: List, type: MpegType): Long { + if (packets.isEmpty()) return 0 var bytesSend = 0L packets.forEach { mpegTsPacket -> var size = 0 size += commandsManager.writeData(mpegTsPacket, socket) - if (isEnableLogs) { - Log.i(TAG, "wrote ${mpegTsPacket.type.name} packet, size $size") - } bytesSend += size } + if (isEnableLogs) { + Log.i(TAG, "wrote ${type.name} packet, size $bytesSend") + } return bytesSend } diff --git a/udp/src/main/java/com/pedro/udp/UdpSender.kt b/udp/src/main/java/com/pedro/udp/UdpSender.kt index d85b2a8c5..22308340a 100644 --- a/udp/src/main/java/com/pedro/udp/UdpSender.kt +++ b/udp/src/main/java/com/pedro/udp/UdpSender.kt @@ -25,6 +25,7 @@ import com.pedro.common.onMainThread import com.pedro.common.validMessage import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegTsPacketizer +import com.pedro.srt.mpeg2ts.MpegType import com.pedro.srt.mpeg2ts.Pid import com.pedro.srt.mpeg2ts.packets.AacPacket import com.pedro.srt.mpeg2ts.packets.BasePacket @@ -89,10 +90,11 @@ class UdpSender( val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } getMpegTsPackets(mediaFrame) { mpegTsPackets -> - val isKey = mpegTsPackets[0].isKey + val firstPacket = mpegTsPackets[0] + val isKey = firstPacket.isKey val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) - bytesSend += sendPackets(psiPackets) - bytesSend += sendPackets(mpegTsPackets) + bytesSend += sendPackets(psiPackets, MpegType.PSI) + bytesSend += sendPackets(mpegTsPackets, firstPacket.type) } }.exceptionOrNull() if (error != null) { @@ -114,16 +116,17 @@ class UdpSender( videoPacket.reset(clear) } - private suspend fun sendPackets(packets: List): Long { + private suspend fun sendPackets(packets: List, type: MpegType): Long { + if (packets.isEmpty()) return 0 var bytesSend = 0L packets.forEach { mpegTsPacket -> var size = 0 size += commandManager.writeData(mpegTsPacket, socket) - if (isEnableLogs) { - Log.i(TAG, "wrote ${mpegTsPacket.type.name} packet, size $size") - } bytesSend += size } + if (isEnableLogs) { + Log.i(TAG, "wrote ${type.name} packet, size $bytesSend") + } return bytesSend } From 6a240fb5e2d113e68c8ea711928f525fd7570129 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 19:43:56 +0200 Subject: [PATCH 07/18] update tests --- .../com/pedro/rtmp/flv/audio/AacPacketTest.kt | 11 ++++----- .../pedro/rtmp/flv/audio/G711PacketTest.kt | 10 ++++---- .../com/pedro/rtmp/flv/video/Av1PacketTest.kt | 10 ++++---- .../pedro/rtmp/flv/video/H264PacketTest.kt | 10 ++++---- .../pedro/rtmp/flv/video/H265PacketTest.kt | 10 ++++---- .../com/pedro/rtsp/rtp/packets/H264Packet.kt | 5 ++-- .../java/com/pedro/rtsp/rtp/AacPacketTest.kt | 13 ++++------ .../java/com/pedro/rtsp/rtp/Av1PacketTest.kt | 14 ++++------- .../java/com/pedro/rtsp/rtp/G711PacketTest.kt | 12 ++++------ .../java/com/pedro/rtsp/rtp/H264PacketTest.kt | 24 +++++++------------ .../java/com/pedro/rtsp/rtp/H265PacketTest.kt | 19 +++++---------- .../java/com/pedro/rtsp/rtp/OpusPacketTest.kt | 12 ++++------ 12 files changed, 56 insertions(+), 94 deletions(-) diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt index 2b0937b3b..53b6c7c03 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt @@ -16,9 +16,10 @@ package com.pedro.rtmp.flv.audio -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.audio.packet.AacPacket +import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals import org.junit.Test import java.nio.ByteBuffer @@ -29,14 +30,10 @@ import java.nio.ByteBuffer class AacPacketTest { @Test - fun `GIVEN a aac buffer WHEN call create a aac packet 2 times THEN return config and expected buffer`() { + fun `GIVEN a aac buffer WHEN call create a aac packet 2 times THEN return config and expected buffer`() = runTest { val timestamp = 123456789L val buffer = ByteArray(256) { 0x00 } - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = buffer.size - info.flags = 1 + val info = MediaFrame.Info(0, buffer.size, timestamp, 1) val aacPacket = AacPacket() aacPacket.sendAudioInfo(32000, true) aacPacket.createFlvPacket(ByteBuffer.wrap(buffer), info) { flvPacket -> diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt index 075b4b253..e93c5bab8 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt @@ -17,9 +17,11 @@ package com.pedro.rtmp.flv.audio import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.audio.packet.G711Packet import junit.framework.TestCase.assertEquals +import kotlinx.coroutines.test.runTest import org.junit.Test import java.nio.ByteBuffer @@ -29,14 +31,10 @@ import java.nio.ByteBuffer class G711PacketTest { @Test - fun `GIVEN a G711 buffer WHEN call create a G711 packet THEN expected buffer`() { + fun `GIVEN a G711 buffer WHEN call create a G711 packet THEN expected buffer`() = runTest { val timestamp = 123456789L val buffer = ByteArray(256) { 0x00 } - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = buffer.size - info.flags = 1 + val info = MediaFrame.Info(0, buffer.size, timestamp, 1) val g711Packet = G711Packet() g711Packet.sendAudioInfo() g711Packet.createFlvPacket(ByteBuffer.wrap(buffer), info) { flvPacket -> diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt index 55c33d5be..31c6b85b5 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt @@ -17,10 +17,12 @@ package com.pedro.rtmp.flv.video import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.video.packet.Av1Packet import junit.framework.TestCase.assertEquals +import kotlinx.coroutines.test.runTest import org.junit.Assert.assertArrayEquals import org.junit.Test import java.nio.ByteBuffer @@ -31,17 +33,13 @@ import java.nio.ByteBuffer class Av1PacketTest { @Test - fun `GIVEN av1data WHEN create flv packet THEN get expected FlvPacket`() { + fun `GIVEN av1data WHEN create flv packet THEN get expected FlvPacket`() = runTest { val timestamp = 123456789L val av1data = byteArrayOf(0x0a, 0x0d, 0x00, 0x00, 0x00, 0x24, 0x4f, 0x7e, 0x7f, 0x00, 0x68, 0x83.toByte(), 0x00, 0x83.toByte(), 0x02) val expectedConfig = byteArrayOf(-112, 97, 118, 48, 49, -127, 4, 12, 0, 10, 13, 0, 0, 0, 36, 79, 126, 127, 0, 104, -125, 0, -125, 2) val expectedFlvPacket = byteArrayOf(-111, 97, 118, 48, 49, 10, 13, 0, 0, 0, 36, 79, 126, 127, 0, 104, -125, 0, -125, 2) - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = av1data.size - info.flags = 1 + val info = MediaFrame.Info(0, av1data.size, timestamp, 1) val frames = mutableListOf() val av1Packet = Av1Packet() diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt index e003b2f86..ed9d93adf 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt @@ -17,9 +17,11 @@ package com.pedro.rtmp.flv.video import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.video.packet.H264Packet +import kotlinx.coroutines.test.runTest import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals import org.junit.Test @@ -31,18 +33,14 @@ import java.nio.ByteBuffer class H264PacketTest { @Test - fun `GIVEN a h264 buffer WHEN call create a h264 packet 1 time THEN return config and expected buffer`() { + fun `GIVEN a h264 buffer WHEN call create a h264 packet 1 time THEN return config and expected buffer`() = runTest { val timestamp = 123456789L val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05) val fakeH264 = header.plus(ByteArray(300) { 0x00 }) val expectedConfig = byteArrayOf(23, 0, 0, 0, 0, 1, 100, 0, 30, -1, -31, 0, 17, 103, 100, 0, 30, -84, -76, 15, 2, -115, 53, 2, 2, 2, 7, -117, 23, 8, 1, 0, 4, 104, -18, 13, -117) val expectedFlvPacket = byteArrayOf(23, 1, 0, 0, 0, 0, 0, 1, 45, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeH264.size - info.flags = 1 + val info = MediaFrame.Info(0, fakeH264.size, timestamp, 1) val h264Packet = H264Packet() val sps = byteArrayOf(103, 100, 0, 30, -84, -76, 15, 2, -115, 53, 2, 2, 2, 7, -117, 23, 8) val pps = byteArrayOf(104, -18, 13, -117) diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt index 7644e576f..3fbeb2e38 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt @@ -17,9 +17,11 @@ package com.pedro.rtmp.flv.video import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.video.packet.H265Packet +import kotlinx.coroutines.test.runTest import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals import org.junit.Test @@ -31,18 +33,14 @@ import java.nio.ByteBuffer class H265PacketTest { @Test - fun `GIVEN a h265 buffer WHEN call create a h265 packet 1 time THEN return config and expected buffer`() { + fun `GIVEN a h265 buffer WHEN call create a h265 packet 1 time THEN return config and expected buffer`() = runTest { val timestamp = 123456789L val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05) val fakeH264 = header.plus(ByteArray(300) { 0x00 }) val expectedConfig = byteArrayOf(-112, 104, 118, 99, 49, 1, 1, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, -103, -16, 0, -4, -3, -8, -8, 0, 0, 3, 3, -96, 0, 1, 0, 24, 64, 1, 12, 1, -1, -1, 1, 96, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, -103, 44, 9, -95, 0, 1, 0, 35, 66, 1, 1, 1, 96, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, -103, -96, 15, 8, 2, -127, 104, -76, -82, -55, 46, -26, -96, -64, -64, -64, 16, -94, 0, 1, 0, 8, 68, 1, -64, 102, 124, 12, -58, 64) val expectedFlvPacket = byteArrayOf(-111, 104, 118, 99, 49, 0, 0, 0, 0, 0, 1, 45, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeH264.size - info.flags = 1 + val info = MediaFrame.Info(0, fakeH264.size, timestamp, 1) val h265Packet = H265Packet() val vps = byteArrayOf(64, 1, 12, 1, -1, -1, 1, 96, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, -103, 44, 9) val sps = byteArrayOf(66, 1, 1, 1, 96, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, -103, -96, 15, 8, 2, -127, 104, -76, -82, -55, 46, -26, -96, -64, -64, -64, 16) diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index 26e479485..202749b6b 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -20,9 +20,9 @@ import android.util.Log import com.pedro.common.frame.MediaFrame import com.pedro.common.isKeyframe import com.pedro.common.removeInfo +import com.pedro.common.toByteArray import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants -import com.pedro.rtsp.utils.getData import com.pedro.rtsp.utils.getVideoStartCodeSize import java.nio.ByteBuffer import kotlin.experimental.and @@ -46,7 +46,7 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, } fun sendVideoInfo(sps: ByteBuffer, pps: ByteBuffer) { - setSpsPps(sps.getData(), pps.getData()) + setSpsPps(sps.toByteArray(), pps.toByteArray()) } override suspend fun createAndSendPacket( @@ -132,6 +132,7 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, private fun setSpsPps(sps: ByteArray, pps: ByteArray) { this.sps = sps this.pps = pps + Log.e("Pedro", "sps: ${sps.size}, pps: ${pps.size}") stapA = ByteArray(sps.size + pps.size + 5) stapA?.let { // STAP-A NAL header is 24 diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt index 28dacf4aa..ab33a1a72 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt @@ -16,11 +16,12 @@ package com.pedro.rtsp.rtp -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.AacPacket import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants import junit.framework.TestCase.assertEquals +import kotlinx.coroutines.test.runTest import org.junit.Test import java.nio.ByteBuffer @@ -30,16 +31,12 @@ import java.nio.ByteBuffer class AacPacketTest { @Test - fun `GIVEN a ByteBuffer raw aac WHEN create a packet THEN get a RTP aac packet`() { + fun `GIVEN a ByteBuffer raw aac WHEN create a packet THEN get a RTP aac packet`() = runTest { val timestamp = 123456789L val fakeAac = ByteArray(300) { 0x00 } - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeAac.size - info.flags = 1 - val aacPacket = AacPacket(44100) + val info = MediaFrame.Info(0, fakeAac.size, timestamp, 1) + val aacPacket = AacPacket().apply { setAudioInfo(44100) } aacPacket.setSSRC(123456789) val frames = mutableListOf() aacPacket.createAndSendPacket(ByteBuffer.wrap(fakeAac), info) { diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt index 29e6b1cc3..593db66ad 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt @@ -16,14 +16,12 @@ package com.pedro.rtsp.rtp -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.Av1Packet import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants -import junit.framework.TestCase import junit.framework.TestCase.assertEquals -import org.junit.Assert -import org.junit.Assert.assertArrayEquals +import kotlinx.coroutines.test.runTest import org.junit.Test import java.nio.ByteBuffer @@ -33,15 +31,11 @@ import java.nio.ByteBuffer class Av1PacketTest { @Test - fun `GIVEN av1data WHEN create rtp packet THEN get expected packet`() { + fun `GIVEN av1data WHEN create rtp packet THEN get expected packet`() = runTest { val timestamp = 123456789L val av1data = byteArrayOf(0x0a, 0x0d, 0x00, 0x00, 0x00, 0x24, 0x4f, 0x7e, 0x7f, 0x00, 0x68, 0x83.toByte(), 0x00, 0x83.toByte(), 0x02) - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = av1data.size - info.flags = 1 + val info = MediaFrame.Info(0, av1data.size, timestamp, 1) val frames = mutableListOf() val av1Packet = Av1Packet() diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt index 0262cc074..f5fe0c785 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt @@ -17,11 +17,13 @@ package com.pedro.rtsp.rtp import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.G711Packet import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants import junit.framework.TestCase import junit.framework.TestCase.assertEquals +import kotlinx.coroutines.test.runTest import org.junit.Test import java.nio.ByteBuffer @@ -31,16 +33,12 @@ import java.nio.ByteBuffer class G711PacketTest { @Test - fun `GIVEN g711 data WHEN create rtp packet THEN get expected packet`() { + fun `GIVEN g711 data WHEN create rtp packet THEN get expected packet`() = runTest { val timestamp = 123456789L val fakeG711 = ByteArray(30) { 0x05 } - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeG711.size - info.flags = 1 - val g711Packet = G711Packet(8000) + val info = MediaFrame.Info(0, fakeG711.size, timestamp, 1) + val g711Packet = G711Packet().apply { setAudioInfo(8000) } g711Packet.setSSRC(123456789) val frames = mutableListOf() g711Packet.createAndSendPacket(ByteBuffer.wrap(fakeG711), info) { diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt index 48fbea38a..c8baa4dd6 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt @@ -17,9 +17,11 @@ package com.pedro.rtsp.rtp import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.H264Packet import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants +import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals import org.junit.Assert.assertNotEquals import org.junit.Assert.assertTrue @@ -32,20 +34,15 @@ import java.nio.ByteBuffer class H264PacketTest { @Test - fun `GIVEN a small ByteBuffer raw h264 WHEN create a packet THEN get a RTP h264 packet`() { + fun `GIVEN a small ByteBuffer raw h264 WHEN create a packet THEN get a RTP h264 packet`() = runTest { val timestamp = 123456789L val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05) val fakeH264 = header.plus(ByteArray(300) { 0x00 }) - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeH264.size - info.flags = 1 - + val info = MediaFrame.Info(0, fakeH264.size, timestamp, 1) val fakeSps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04) val fakePps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0A, 0x0B, 0x0C) - val h264Packet = H264Packet(fakeSps, fakePps) + val h264Packet = H264Packet().apply { sendVideoInfo(ByteBuffer.wrap(fakeSps), ByteBuffer.wrap(fakePps)) } h264Packet.setSSRC(123456789) val frames = mutableListOf() h264Packet.createAndSendPacket(ByteBuffer.wrap(fakeH264), info) { @@ -66,20 +63,15 @@ class H264PacketTest { } @Test - fun `GIVEN a big ByteBuffer raw h264 WHEN create a packet THEN get a RTP h264 packet`() { + fun `GIVEN a big ByteBuffer raw h264 WHEN create a packet THEN get a RTP h264 packet`() = runTest { val timestamp = 123456789L val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05) val fakeH264 = header.plus(ByteArray(2500) { 0x00 }) - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeH264.size - info.flags = 1 - + val info = MediaFrame.Info(0, fakeH264.size, timestamp, 1) val fakeSps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04) val fakePps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0A, 0x0B, 0x0C) - val h264Packet = H264Packet(fakeSps, fakePps) + val h264Packet = H264Packet().apply { sendVideoInfo(ByteBuffer.wrap(fakeSps), ByteBuffer.wrap(fakePps)) } h264Packet.setSSRC(123456789) val frames = mutableListOf() h264Packet.createAndSendPacket(ByteBuffer.wrap(fakeH264), info) { diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt index 42b996a6b..a964ed79d 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt @@ -17,9 +17,11 @@ package com.pedro.rtsp.rtp import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.H265Packet import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants +import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals import org.junit.Assert.assertNotEquals import org.junit.Assert.assertTrue @@ -32,16 +34,12 @@ import java.nio.ByteBuffer class H265PacketTest { @Test - fun `GIVEN a small ByteBuffer raw h265 WHEN create a packet THEN get a RTP h265 packet`() { + fun `GIVEN a small ByteBuffer raw h265 WHEN create a packet THEN get a RTP h265 packet`() = runTest { val timestamp = 123456789L val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05, 0x00) val fakeH265 = header.plus(ByteArray(300) { 0x00 }) - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeH265.size - info.flags = 1 + val info = MediaFrame.Info(0, fakeH265.size, timestamp, 1) val h265Packet = H265Packet() h265Packet.setSSRC(123456789) @@ -61,17 +59,12 @@ class H265PacketTest { } @Test - fun `GIVEN a big ByteBuffer raw h265 WHEN create a packet THEN get a RTP h265 packet`() { + fun `GIVEN a big ByteBuffer raw h265 WHEN create a packet THEN get a RTP h265 packet`() = runTest { val timestamp = 123456789L val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05, 0x00) val fakeH265 = header.plus(ByteArray(2500) { 0x00 }) - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeH265.size - info.flags = 1 - + val info = MediaFrame.Info(0, fakeH265.size, timestamp, 1) val h265Packet = H265Packet() h265Packet.setSSRC(123456789) val frames = mutableListOf() diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt index 41806c6d5..abda14d68 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt @@ -17,10 +17,12 @@ package com.pedro.rtsp.rtp import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.OpusPacket import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants import junit.framework.TestCase.assertEquals +import kotlinx.coroutines.test.runTest import org.junit.Test import java.nio.ByteBuffer @@ -30,16 +32,12 @@ import java.nio.ByteBuffer class OpusPacketTest { @Test - fun `GIVEN opus data WHEN create rtp packet THEN get expected packet`() { + fun `GIVEN opus data WHEN create rtp packet THEN get expected packet`() = runTest { val timestamp = 123456789L val fakeOpus = ByteArray(30) { 0x05 } - val info = MediaCodec.BufferInfo() - info.presentationTimeUs = timestamp - info.offset = 0 - info.size = fakeOpus.size - info.flags = 1 - val opusPacket = OpusPacket(8000) + val info = MediaFrame.Info(0, fakeOpus.size, timestamp, 1) + val opusPacket = OpusPacket().apply { setAudioInfo(8000) } opusPacket.setSSRC(123456789) val frames = mutableListOf() opusPacket.createAndSendPacket(ByteBuffer.wrap(fakeOpus), info) { From b99341cdeedaaa406afa658af43dd68016558333 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 19:46:38 +0200 Subject: [PATCH 08/18] optmize imports --- common/src/main/AndroidManifest.xml | 2 +- rtmp/src/main/java/com/pedro/rtmp/rtmp/Handshake.kt | 3 --- .../src/main/java/com/pedro/rtmp/rtmp/message/BasicHeader.kt | 1 - rtmp/src/main/java/com/pedro/rtmp/rtmp/message/RtmpHeader.kt | 2 -- .../src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt | 1 - rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt | 1 - .../src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt | 1 - .../src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt | 1 - rtsp/src/test/java/com/pedro/rtsp/rtcp/RtcpReportTest.kt | 3 --- rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt | 2 -- rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt | 1 - rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt | 1 - rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt | 1 - srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt | 4 ++-- .../main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt | 1 - .../main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt | 1 - srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt | 5 ----- srt/src/main/java/com/pedro/srt/srt/CommandsManager.kt | 2 +- .../control/handshake/extension/KeyMaterialMessage.kt | 1 - srt/src/test/java/com/pedro/srt/mpeg2ts/PesTest.kt | 2 +- 20 files changed, 5 insertions(+), 31 deletions(-) diff --git a/common/src/main/AndroidManifest.xml b/common/src/main/AndroidManifest.xml index a5918e68a..44008a433 100644 --- a/common/src/main/AndroidManifest.xml +++ b/common/src/main/AndroidManifest.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/Handshake.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/Handshake.kt index f2e7b4509..b17da8390 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/Handshake.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/Handshake.kt @@ -18,11 +18,8 @@ package com.pedro.rtmp.rtmp import android.util.Log import com.pedro.common.TimeUtils -import com.pedro.rtmp.utils.readUntil import com.pedro.rtmp.utils.socket.RtmpSocket import java.io.IOException -import java.io.InputStream -import java.io.OutputStream import kotlin.random.Random /** diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/message/BasicHeader.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/message/BasicHeader.kt index 73e0a6f77..75b573085 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/message/BasicHeader.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/message/BasicHeader.kt @@ -19,7 +19,6 @@ package com.pedro.rtmp.rtmp.message import com.pedro.rtmp.rtmp.chunk.ChunkType import com.pedro.rtmp.utils.socket.RtmpSocket import java.io.IOException -import java.io.InputStream import kotlin.experimental.and /** diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/message/RtmpHeader.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/message/RtmpHeader.kt index 7d0eb86c4..cdaad6b79 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/message/RtmpHeader.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/message/RtmpHeader.kt @@ -20,8 +20,6 @@ import com.pedro.rtmp.rtmp.chunk.ChunkType import com.pedro.rtmp.utils.* import com.pedro.rtmp.utils.socket.RtmpSocket import java.io.IOException -import java.io.InputStream -import java.io.OutputStream import kotlin.math.min /** diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt index e93c5bab8..5c16ee8d8 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt @@ -16,7 +16,6 @@ package com.pedro.rtmp.flv.audio -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.audio.packet.G711Packet diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt index 31c6b85b5..80528f18b 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt @@ -16,7 +16,6 @@ package com.pedro.rtmp.flv.video -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt index ed9d93adf..30c91257c 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt @@ -16,7 +16,6 @@ package com.pedro.rtmp.flv.video -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt index 3fbeb2e38..4345c7e93 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt @@ -16,7 +16,6 @@ package com.pedro.rtmp.flv.video -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtcp/RtcpReportTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtcp/RtcpReportTest.kt index ef7cef06a..0432b2b7c 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtcp/RtcpReportTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtcp/RtcpReportTest.kt @@ -36,9 +36,6 @@ import org.mockito.junit.MockitoJUnitRunner import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.times import org.mockito.kotlin.verify -import java.io.OutputStream -import java.net.DatagramPacket -import java.net.MulticastSocket /** * Created by pedro on 9/9/23. diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt index f5fe0c785..6b6f208cf 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt @@ -16,12 +16,10 @@ package com.pedro.rtsp.rtp -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.G711Packet import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants -import junit.framework.TestCase import junit.framework.TestCase.assertEquals import kotlinx.coroutines.test.runTest import org.junit.Test diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt index c8baa4dd6..375ce4e79 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt @@ -16,7 +16,6 @@ package com.pedro.rtsp.rtp -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.H264Packet import com.pedro.rtsp.rtsp.RtpFrame diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt index a964ed79d..fa01f324b 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt @@ -16,7 +16,6 @@ package com.pedro.rtsp.rtp -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.H265Packet import com.pedro.rtsp.rtsp.RtpFrame diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt index abda14d68..f0195435c 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt @@ -16,7 +16,6 @@ package com.pedro.rtsp.rtp -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.rtsp.rtp.packets.OpusPacket import com.pedro.rtsp.rtsp.RtpFrame diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt index 2b4a105a5..4ceea14ef 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt @@ -16,10 +16,10 @@ package com.pedro.srt.mpeg2ts -import com.pedro.srt.mpeg2ts.psi.Psi -import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.common.TimeUtils import com.pedro.common.toByteArray +import com.pedro.srt.mpeg2ts.psi.Psi +import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.srt.utils.toInt import java.nio.ByteBuffer diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt index ae6995755..c527a24d0 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/BasePacket.kt @@ -16,7 +16,6 @@ package com.pedro.srt.mpeg2ts.packets -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegTsPacketizer diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt index 817db1b44..da1fbe4f2 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/OpusPacket.kt @@ -16,7 +16,6 @@ package com.pedro.srt.mpeg2ts.packets -import android.media.MediaCodec import com.pedro.common.frame.MediaFrame import com.pedro.common.removeInfo import com.pedro.srt.mpeg2ts.MpegTsPacket diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt index 8faf6884f..8c73cef44 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt @@ -16,12 +16,7 @@ package com.pedro.srt.mpeg2ts.psi -import com.pedro.common.TimeUtils -import com.pedro.srt.mpeg2ts.MpegTsPacket -import com.pedro.srt.mpeg2ts.MpegTsPacketizer -import com.pedro.srt.mpeg2ts.MpegType import com.pedro.srt.mpeg2ts.service.Mpeg2TsService -import com.pedro.srt.srt.packets.data.PacketPosition import kotlin.random.Random /** diff --git a/srt/src/main/java/com/pedro/srt/srt/CommandsManager.kt b/srt/src/main/java/com/pedro/srt/srt/CommandsManager.kt index 28a5e310d..5292875f8 100644 --- a/srt/src/main/java/com/pedro/srt/srt/CommandsManager.kt +++ b/srt/src/main/java/com/pedro/srt/srt/CommandsManager.kt @@ -28,9 +28,9 @@ import com.pedro.srt.srt.packets.control.KeepAlive import com.pedro.srt.srt.packets.control.Shutdown import com.pedro.srt.srt.packets.control.handshake.EncryptionType import com.pedro.srt.srt.packets.control.handshake.Handshake -import com.pedro.srt.utils.EncryptInfo import com.pedro.srt.srt.packets.data.KeyBasedEncryption import com.pedro.srt.utils.Constants +import com.pedro.srt.utils.EncryptInfo import com.pedro.srt.utils.EncryptionUtil import com.pedro.srt.utils.SrtSocket import kotlinx.coroutines.sync.Mutex diff --git a/srt/src/main/java/com/pedro/srt/srt/packets/control/handshake/extension/KeyMaterialMessage.kt b/srt/src/main/java/com/pedro/srt/srt/packets/control/handshake/extension/KeyMaterialMessage.kt index 593c654e2..429c16a9e 100644 --- a/srt/src/main/java/com/pedro/srt/srt/packets/control/handshake/extension/KeyMaterialMessage.kt +++ b/srt/src/main/java/com/pedro/srt/srt/packets/control/handshake/extension/KeyMaterialMessage.kt @@ -16,7 +16,6 @@ package com.pedro.srt.srt.packets.control.handshake.extension -import com.pedro.srt.srt.packets.data.KeyBasedEncryption import com.pedro.srt.utils.EncryptInfo import com.pedro.srt.utils.writeUInt16 import com.pedro.srt.utils.writeUInt32 diff --git a/srt/src/test/java/com/pedro/srt/mpeg2ts/PesTest.kt b/srt/src/test/java/com/pedro/srt/mpeg2ts/PesTest.kt index 95ade9389..cea35dbc5 100644 --- a/srt/src/test/java/com/pedro/srt/mpeg2ts/PesTest.kt +++ b/srt/src/test/java/com/pedro/srt/mpeg2ts/PesTest.kt @@ -16,10 +16,10 @@ package com.pedro.srt.mpeg2ts +import com.pedro.common.TimeUtils import com.pedro.srt.Utils import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.srt.mpeg2ts.service.Mpeg2TsService -import com.pedro.common.TimeUtils import kotlinx.coroutines.test.runTest import org.junit.Assert.assertArrayEquals import org.junit.Before From 897f06960857ad69baf705c26c7ca228263d8b3f Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 19:49:22 +0200 Subject: [PATCH 09/18] update common test --- common/src/test/java/com/pedro/common/ExtensionTest.kt | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/common/src/test/java/com/pedro/common/ExtensionTest.kt b/common/src/test/java/com/pedro/common/ExtensionTest.kt index b8920456a..776cfdb97 100644 --- a/common/src/test/java/com/pedro/common/ExtensionTest.kt +++ b/common/src/test/java/com/pedro/common/ExtensionTest.kt @@ -16,7 +16,7 @@ package com.pedro.common -import android.media.MediaCodec +import com.pedro.common.frame.MediaFrame import org.junit.Assert.assertEquals import org.junit.Test import java.nio.ByteBuffer @@ -29,14 +29,9 @@ class ExtensionTest { @Test fun `remove info`() { val buffer = ByteBuffer.wrap(ByteArray(256) { 0x00 }.mapIndexed { index, byte -> index.toByte() }.toByteArray()) - val info = MediaCodec.BufferInfo() val offset = 4 val minusLimit = 2 - info.presentationTimeUs = 0 - info.offset = offset - info.size = buffer.remaining() - minusLimit - info.flags = 0 - + val info = MediaFrame.Info(4, buffer.remaining() - minusLimit, 0, 0) val result = buffer.removeInfo(info) assertEquals(buffer.capacity() - offset - minusLimit, result.remaining()) assertEquals(offset.toByte(), result.get(0)) From 9e1f8be4cd835ad39e54a570503a027ef255c37e Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 19:54:45 +0200 Subject: [PATCH 10/18] remove logcat --- rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index 202749b6b..3d23f21a5 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -132,7 +132,6 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, private fun setSpsPps(sps: ByteArray, pps: ByteArray) { this.sps = sps this.pps = pps - Log.e("Pedro", "sps: ${sps.size}, pps: ${pps.size}") stapA = ByteArray(sps.size + pps.size + 5) stapA?.let { // STAP-A NAL header is 24 From d90cfc62550f4bb05e97b1a042c97fd310852353 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 23:16:22 +0200 Subject: [PATCH 11/18] add SrtSender test --- .../java/com/pedro/srt/srt/SrtSenderTest.kt | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt diff --git a/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt b/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt new file mode 100644 index 000000000..f295bdb92 --- /dev/null +++ b/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt @@ -0,0 +1,72 @@ +package com.pedro.srt.srt + +import com.pedro.common.AudioCodec +import com.pedro.common.ConnectChecker +import com.pedro.common.VideoCodec +import com.pedro.common.frame.MediaFrame +import com.pedro.srt.mpeg2ts.MpegTsPacket +import com.pedro.srt.utils.Constants +import com.pedro.srt.utils.SrtSocket +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mock +import org.mockito.Mockito +import org.mockito.junit.MockitoJUnitRunner +import org.mockito.kotlin.any +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +@RunWith(MockitoJUnitRunner::class) +class SrtSenderTest { + + @Mock + lateinit var connectChecker: ConnectChecker + @Mock + lateinit var socket: SrtSocket + @Mock + lateinit var commandsManager: CommandsManager + private val output = ByteArrayOutputStream() + private var latch = CountDownLatch(7) + + @Before + fun setup() = runTest { + output.reset() + Mockito.`when`(commandsManager.audioCodec).thenReturn(AudioCodec.AAC) + Mockito.`when`(commandsManager.videoCodec).thenReturn(VideoCodec.H264) + Mockito.`when`(commandsManager.MTU).thenReturn(Constants.MTU) + Mockito.lenient().`when`(commandsManager.writeData(any(), any())).then { + val packet = it.arguments[0] as MpegTsPacket + val size = packet.buffer.size + output.write(packet.buffer) + latch.countDown().let { size } + } + } + + @Test + fun `GIVEN video and audio mediaFrames WHEN send to sender THEN write the expected packets`() = runTest { + latch = CountDownLatch(7) //writeData must be called 4 times + val srtSender = SrtSender(connectChecker, commandsManager) + srtSender.setAudioInfo(44100, true) + val sps = ByteBuffer.wrap(byteArrayOf(0, 0, 0, 1, 103, 100, 0, 30, -84, -76, 15, 2, -115, 53, 2, 2, 2, 7, -117, 23, 8)) + val pps = ByteBuffer.wrap(byteArrayOf(0, 0, 0, 1, 104, -18, 13, -117)) + srtSender.setVideoInfo(sps, pps, null) + srtSender.socket = socket + srtSender.start() + + val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05) + val videoData = ByteBuffer.wrap(header.plus(ByteArray(300) { 0x00 })) + val audioData = ByteBuffer.wrap(ByteArray(256) { 0x00 }) + + val videoFrame = MediaFrame(videoData, MediaFrame.Info(0, videoData.remaining(), 0, 1), MediaFrame.Type.VIDEO) + val audioFrame = MediaFrame(audioData, MediaFrame.Info(0, audioData.remaining(), 0, 1), MediaFrame.Type.AUDIO) + srtSender.sendMediaFrame(videoFrame) + srtSender.sendMediaFrame(audioFrame) + latch.await(1000, TimeUnit.MILLISECONDS) + assertEquals(1692, output.toByteArray().size) + } +} \ No newline at end of file From 109d375b597f3cc6fe23ceaece5c1679de7dd148 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 23:20:12 +0200 Subject: [PATCH 12/18] add stop sender to tests --- srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt b/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt index f295bdb92..af1fd0d57 100644 --- a/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt +++ b/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt @@ -67,6 +67,8 @@ class SrtSenderTest { srtSender.sendMediaFrame(videoFrame) srtSender.sendMediaFrame(audioFrame) latch.await(1000, TimeUnit.MILLISECONDS) + srtSender.stop() + assertEquals(1692, output.toByteArray().size) } } \ No newline at end of file From c0ed321d7bc06f9a1cf61dd31ba51a1e78b45922 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Mon, 30 Sep 2024 23:47:31 +0200 Subject: [PATCH 13/18] refactor mpeg2ts psi --- .../com/pedro/srt/mpeg2ts/psi/PsiManager.kt | 21 ++++++++--- .../main/java/com/pedro/srt/srt/SrtSender.kt | 37 ++----------------- udp/src/main/java/com/pedro/udp/UdpSender.kt | 36 ++---------------- 3 files changed, 23 insertions(+), 71 deletions(-) diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt index 8c73cef44..94cd0bd40 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/PsiManager.kt @@ -16,7 +16,12 @@ package com.pedro.srt.mpeg2ts.psi +import com.pedro.srt.mpeg2ts.MpegTsPacket +import com.pedro.srt.mpeg2ts.MpegTsPacketizer +import com.pedro.srt.mpeg2ts.MpegTsPayload +import com.pedro.srt.mpeg2ts.MpegType import com.pedro.srt.mpeg2ts.service.Mpeg2TsService +import com.pedro.srt.srt.packets.data.PacketPosition import kotlin.random.Random /** @@ -29,6 +34,7 @@ class PsiManager( private val idExtension = Random.nextInt(Byte.MIN_VALUE.toInt(), Byte.MAX_VALUE.toInt()).toShort() private var sdtCount = 0 private var patCount = 0 + companion object { const val sdtPeriod = 200 const val patPeriod = 40 @@ -46,22 +52,25 @@ class PsiManager( service = service ) - fun shouldSend(isKey: Boolean = false): TableToSend { - var value = TableToSend.NONE + fun checkSendInfo(isKey: Boolean = false, mpegTsPacketizer: MpegTsPacketizer): List { + val pmt = service.pmt ?: return arrayListOf() + val psiPackets = mutableListOf() if (sdtCount >= sdtPeriod && patCount >= patPeriod) { - value = TableToSend.ALL + psiPackets.addAll(listOf(pmt, sdt, pat)) sdtCount = 0 patCount = 0 } else if (patCount >= patPeriod || isKey) { - value = TableToSend.PAT_PMT + psiPackets.addAll(listOf(pat, pmt)) patCount = 0 } else if (sdtCount >= sdtPeriod) { - value = TableToSend.SDT + psiPackets.add(sdt) sdtCount = 0 } sdtCount++ patCount++ - return value + return mpegTsPacketizer.write(psiPackets, increasePsiContinuity = true).map { b -> + MpegTsPacket(b, MpegType.PSI, PacketPosition.SINGLE, isKey = false) + } } fun upgradeSdtVersion() { diff --git a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt index f0da012c5..6a10adb6d 100644 --- a/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt +++ b/srt/src/main/java/com/pedro/srt/srt/SrtSender.kt @@ -32,7 +32,6 @@ import com.pedro.srt.mpeg2ts.packets.BasePacket import com.pedro.srt.mpeg2ts.packets.H26XPacket import com.pedro.srt.mpeg2ts.packets.OpusPacket import com.pedro.srt.mpeg2ts.psi.PsiManager -import com.pedro.srt.mpeg2ts.psi.TableToSend import com.pedro.srt.mpeg2ts.service.Mpeg2TsService import com.pedro.srt.srt.packets.SrtPacket import com.pedro.srt.srt.packets.data.PacketPosition @@ -99,9 +98,10 @@ class SrtSender( val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } getMpegTsPackets(mediaFrame) { mpegTsPackets -> - if (mpegTsPackets.isNotEmpty()) { - bytesSend += sendPackets(mpegTsPackets, mpegTsPackets[0].type) - } + val isKey = mpegTsPackets[0].isKey + val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) + bytesSend += sendPackets(psiPackets, MpegType.PSI) + bytesSend += sendPackets(mpegTsPackets, mpegTsPackets[0].type) } }.exceptionOrNull() if (error != null) { @@ -142,43 +142,14 @@ class SrtSender( when (mediaFrame.type) { MediaFrame.Type.VIDEO -> { videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - val isKey = packets[0].isKey - checkSendInfo(isKey, callback) callback(packets) } } MediaFrame.Type.AUDIO -> { audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - val isKey = packets[0].isKey - checkSendInfo(isKey, callback) callback(packets) } } } } - - private suspend fun checkSendInfo(isKey: Boolean = false, callback: suspend (List) -> Unit) { - val pmt = psiManager.getPmt() ?: return - when (psiManager.shouldSend(isKey)) { - TableToSend.PAT_PMT -> { - val psiPackets = mpegTsPacketizer.write(listOf(psiManager.getPat(), pmt), increasePsiContinuity = true).map { b -> - MpegTsPacket(b, MpegType.PSI, PacketPosition.SINGLE, isKey = false) - } - callback(psiPackets) - } - TableToSend.SDT -> { - val psiPackets = mpegTsPacketizer.write(listOf(psiManager.getSdt()), increasePsiContinuity = true).map { b -> - MpegTsPacket(b, MpegType.PSI, PacketPosition.SINGLE, isKey = false) - } - callback(psiPackets) - } - TableToSend.NONE -> {} - TableToSend.ALL -> { - val psiPackets = mpegTsPacketizer.write(listOf(pmt, psiManager.getSdt(), psiManager.getPat()), increasePsiContinuity = true).map { b -> - MpegTsPacket(b, MpegType.PSI, PacketPosition.SINGLE, isKey = false) - } - callback(psiPackets) - } - } - } } \ No newline at end of file diff --git a/udp/src/main/java/com/pedro/udp/UdpSender.kt b/udp/src/main/java/com/pedro/udp/UdpSender.kt index 0575a5afb..6a112c26c 100644 --- a/udp/src/main/java/com/pedro/udp/UdpSender.kt +++ b/udp/src/main/java/com/pedro/udp/UdpSender.kt @@ -99,9 +99,10 @@ class UdpSender( val error = runCatching { val mediaFrame = runInterruptible { queue.poll(1, TimeUnit.SECONDS) } getMpegTsPackets(mediaFrame) { mpegTsPackets -> - if (mpegTsPackets.isNotEmpty()) { - bytesSend += sendPackets(mpegTsPackets, mpegTsPackets[0].type) - } + val isKey = mpegTsPackets[0].isKey + val psiPackets = psiManager.checkSendInfo(isKey, mpegTsPacketizer) + bytesSend += sendPackets(psiPackets, MpegType.PSI) + bytesSend += sendPackets(mpegTsPackets, mpegTsPackets[0].type) } }.exceptionOrNull() if (error != null) { @@ -142,43 +143,14 @@ class UdpSender( when (mediaFrame.type) { MediaFrame.Type.VIDEO -> { videoPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - val isKey = packets[0].isKey - checkSendInfo(isKey, callback) callback(packets) } } MediaFrame.Type.AUDIO -> { audioPacket.createAndSendPacket(mediaFrame.data, mediaFrame.info) { packets -> - val isKey = packets[0].isKey - checkSendInfo(isKey, callback) callback(packets) } } } } - - private suspend fun checkSendInfo(isKey: Boolean = false, callback: suspend (List) -> Unit) { - val pmt = psiManager.getPmt() ?: return - when (psiManager.shouldSend(isKey)) { - TableToSend.PAT_PMT -> { - val psiPackets = mpegTsPacketizer.write(listOf(psiManager.getPat(), pmt), increasePsiContinuity = true).map { b -> - MpegTsPacket(b, MpegType.PSI, PacketPosition.SINGLE, isKey = false) - } - callback(psiPackets) - } - TableToSend.SDT -> { - val psiPackets = mpegTsPacketizer.write(listOf(psiManager.getSdt()), increasePsiContinuity = true).map { b -> - MpegTsPacket(b, MpegType.PSI, PacketPosition.SINGLE, isKey = false) - } - callback(psiPackets) - } - TableToSend.NONE -> {} - TableToSend.ALL -> { - val psiPackets = mpegTsPacketizer.write(listOf(pmt, psiManager.getSdt(), psiManager.getPat()), increasePsiContinuity = true).map { b -> - MpegTsPacket(b, MpegType.PSI, PacketPosition.SINGLE, isKey = false) - } - callback(psiPackets) - } - } - } } \ No newline at end of file From 5bd40f999570a80138509fb8823728a5a679b0bf Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Tue, 1 Oct 2024 00:02:18 +0200 Subject: [PATCH 14/18] fix psi tests --- .../com/pedro/srt/mpeg2ts/psi/TableToSend.kt | 25 ------------------- .../com/pedro/srt/mpeg2ts/PsiManagerTest.kt | 20 +++++++++------ udp/src/main/java/com/pedro/udp/UdpSender.kt | 1 - 3 files changed, 12 insertions(+), 34 deletions(-) delete mode 100644 srt/src/main/java/com/pedro/srt/mpeg2ts/psi/TableToSend.kt diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/TableToSend.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/TableToSend.kt deleted file mode 100644 index b430d38e7..000000000 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/psi/TableToSend.kt +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2024 pedroSG94. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.pedro.srt.mpeg2ts.psi - -/** - * Created by pedro on 28/8/23. - * - */ -enum class TableToSend { - PAT_PMT, SDT, NONE, ALL -} \ No newline at end of file diff --git a/srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt b/srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt index e682477e8..474da8eff 100644 --- a/srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt +++ b/srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt @@ -17,7 +17,6 @@ package com.pedro.srt.mpeg2ts import com.pedro.srt.mpeg2ts.psi.PsiManager -import com.pedro.srt.mpeg2ts.psi.TableToSend import com.pedro.srt.mpeg2ts.service.Mpeg2TsService import org.junit.Assert.assertEquals import org.junit.Test @@ -27,26 +26,31 @@ import org.junit.Test */ class PsiManagerTest { - private val service = Mpeg2TsService() + private val service = Mpeg2TsService().apply { generatePmt() } + private val psiManager = PsiManager(service).apply { + upgradePatVersion() + upgradeSdtVersion() + } + private val mpegTsPacketizer = MpegTsPacketizer(psiManager) @Test fun `GIVEN a psiManager WHEN call should send is key false patPeriod times THEN return TableToSend PAT_PMT`() { val psiManager = PsiManager(service) - var sendValue = TableToSend.NONE + var packets = listOf() (0..PsiManager.patPeriod).forEach { _ -> - sendValue = psiManager.shouldSend(false) + packets = psiManager.checkSendInfo(false, mpegTsPacketizer) } - assertEquals(TableToSend.PAT_PMT, sendValue) + assertEquals(2, packets.size) } @Test fun `GIVEN a psiManager WHEN call should send is key false sdtPeriod times THEN return TableToSend ALL`() { val psiManager = PsiManager(service) - var sendValue = TableToSend.NONE + var packets = listOf() (0..PsiManager.sdtPeriod).forEach { _ -> - sendValue = psiManager.shouldSend(false) + packets = psiManager.checkSendInfo(false, mpegTsPacketizer) } - assertEquals(TableToSend.ALL, sendValue) + assertEquals(3, packets.size) } @Test diff --git a/udp/src/main/java/com/pedro/udp/UdpSender.kt b/udp/src/main/java/com/pedro/udp/UdpSender.kt index 6a112c26c..23c696f2f 100644 --- a/udp/src/main/java/com/pedro/udp/UdpSender.kt +++ b/udp/src/main/java/com/pedro/udp/UdpSender.kt @@ -32,7 +32,6 @@ import com.pedro.srt.mpeg2ts.packets.BasePacket import com.pedro.srt.mpeg2ts.packets.H26XPacket import com.pedro.srt.mpeg2ts.packets.OpusPacket import com.pedro.srt.mpeg2ts.psi.PsiManager -import com.pedro.srt.mpeg2ts.psi.TableToSend import com.pedro.srt.mpeg2ts.service.Mpeg2TsService import com.pedro.srt.srt.packets.data.PacketPosition import com.pedro.srt.utils.Constants From 1bc2b1faccbce670c99f01f4ae43481faa0b1528 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Tue, 1 Oct 2024 00:14:34 +0200 Subject: [PATCH 15/18] rename psi tests --- srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt b/srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt index 474da8eff..aa5c87bdd 100644 --- a/srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt +++ b/srt/src/test/java/com/pedro/srt/mpeg2ts/PsiManagerTest.kt @@ -34,7 +34,7 @@ class PsiManagerTest { private val mpegTsPacketizer = MpegTsPacketizer(psiManager) @Test - fun `GIVEN a psiManager WHEN call should send is key false patPeriod times THEN return TableToSend PAT_PMT`() { + fun `GIVEN a psiManager WHEN call should send is key false patPeriod times THEN return PAT and PMT packets`() { val psiManager = PsiManager(service) var packets = listOf() (0..PsiManager.patPeriod).forEach { _ -> @@ -44,7 +44,7 @@ class PsiManagerTest { } @Test - fun `GIVEN a psiManager WHEN call should send is key false sdtPeriod times THEN return TableToSend ALL`() { + fun `GIVEN a psiManager WHEN call should send is key false sdtPeriod times THEN return PMT, SDT and PAT packets`() { val psiManager = PsiManager(service) var packets = listOf() (0..PsiManager.sdtPeriod).forEach { _ -> From 5e0db879e89c6dc4866db0df0644e24bc723e043 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Tue, 1 Oct 2024 12:11:41 +0200 Subject: [PATCH 16/18] fix NPE on decoders --- .../src/main/java/com/pedro/common/Extensions.kt | 11 ++++++++++- .../pedro/encoder/input/decoder/AudioDecoder.java | 11 ++++++++--- .../pedro/encoder/input/decoder/BaseDecoder.java | 1 + .../pedro/encoder/input/decoder/VideoDecoder.java | 14 ++++++++++---- .../java/com/pedro/library/base/FromFileBase.java | 4 ++-- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/common/src/main/java/com/pedro/common/Extensions.kt b/common/src/main/java/com/pedro/common/Extensions.kt index 559694f1b..8b60f991c 100644 --- a/common/src/main/java/com/pedro/common/Extensions.kt +++ b/common/src/main/java/com/pedro/common/Extensions.kt @@ -19,6 +19,7 @@ package com.pedro.common import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CaptureRequest import android.media.MediaCodec +import android.media.MediaFormat import android.os.Build import android.os.Handler import android.os.Looper @@ -144,4 +145,12 @@ fun Throwable.validMessage(): String { fun MediaCodec.BufferInfo.toMediaFrameInfo() = MediaFrame.Info(offset, size, presentationTimeUs, flags) -fun ByteBuffer.clone(): ByteBuffer = ByteBuffer.wrap(toByteArray()) \ No newline at end of file +fun ByteBuffer.clone(): ByteBuffer = ByteBuffer.wrap(toByteArray()) + +fun MediaFormat.getIntegerSafe(name: String): Int? { + return try { getInteger(name) } catch (e: Exception) { null } +} + +fun MediaFormat.getLongSafe(name: String): Long? { + return try { getLong(name) } catch (e: Exception) { null } +} \ No newline at end of file diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioDecoder.java b/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioDecoder.java index 53b949e80..8d54b7741 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioDecoder.java +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioDecoder.java @@ -20,6 +20,7 @@ import android.media.MediaFormat; import android.util.Log; +import com.pedro.common.ExtensionsKt; import com.pedro.encoder.Frame; import com.pedro.encoder.input.audio.GetMicrophoneData; import com.pedro.encoder.utils.CodecUtil; @@ -63,10 +64,14 @@ protected boolean extract(MediaExtractor audioExtractor) { } } if (mediaFormat != null) { - channels = mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + final Integer channels = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_CHANNEL_COUNT); + final Integer sampleRate = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_SAMPLE_RATE); + final Long duration = ExtensionsKt.getLongSafe(mediaFormat, MediaFormat.KEY_DURATION); + if (channels == null || sampleRate == null) return false; + this.channels = channels; isStereo = channels >= 2; - sampleRate = mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); - duration = mediaFormat.getLong(MediaFormat.KEY_DURATION); + this.sampleRate = sampleRate; + this.duration = duration != null ? duration : -1; fixBuffer(); return true; //audio decoder not supported diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/BaseDecoder.java b/encoder/src/main/java/com/pedro/encoder/input/decoder/BaseDecoder.java index 31d9e9adc..9357af1e2 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/decoder/BaseDecoder.java +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/BaseDecoder.java @@ -210,6 +210,7 @@ public boolean isLoopMode() { } public double getDuration() { + if (duration < 0) return duration; //fail to extract duration from file. return duration / 10E5; } diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoDecoder.java b/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoDecoder.java index f915e33f1..be2eaf0e1 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoDecoder.java +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoDecoder.java @@ -21,6 +21,7 @@ import android.os.Build; import android.view.Surface; +import com.pedro.common.ExtensionsKt; import java.nio.ByteBuffer; /** @@ -51,10 +52,15 @@ protected boolean extract(MediaExtractor videoExtractor) { } } if (mediaFormat != null) { - width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH); - height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); - duration = mediaFormat.getLong(MediaFormat.KEY_DURATION); - fps = mediaFormat.getInteger(MediaFormat.KEY_FRAME_RATE); + final Integer width = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_WIDTH); + final Integer height = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_HEIGHT); + final Long duration = ExtensionsKt.getLongSafe(mediaFormat, MediaFormat.KEY_DURATION); + final Integer fps = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_FRAME_RATE); + if (width == null || height == null) return false; + this.width = width; + this.height = height; + this.duration = duration != null ? duration : -1; + this.fps = fps != null ? fps : 30; return true; //video decoder not supported } else { diff --git a/library/src/main/java/com/pedro/library/base/FromFileBase.java b/library/src/main/java/com/pedro/library/base/FromFileBase.java index 8ae321999..68abe3221 100644 --- a/library/src/main/java/com/pedro/library/base/FromFileBase.java +++ b/library/src/main/java/com/pedro/library/base/FromFileBase.java @@ -619,14 +619,14 @@ public double getAudioTime() { } /** - * @return return duration in seconds. 0 if no streaming + * @return return duration in seconds. 0 if no streaming, -1 if can't extract it from file */ public double getVideoDuration() { return videoDecoder.getDuration(); } /** - * @return return duration in seconds. 0 if no streaming + * @return return duration in seconds. 0 if no streaming, -1 if can't extract it from file */ public double getAudioDuration() { return audioDecoder.getDuration(); From 740db747a951b668b57373bb0aa831a76236dfdc Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Tue, 1 Oct 2024 20:55:49 +0200 Subject: [PATCH 17/18] change flags to isKeyFrame --- common/src/main/java/com/pedro/common/Extensions.kt | 4 ++-- common/src/main/java/com/pedro/common/frame/MediaFrame.kt | 2 +- .../main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt | 2 +- .../main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt | 2 +- .../main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt | 2 +- rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt | 2 +- rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt | 2 +- srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/common/src/main/java/com/pedro/common/Extensions.kt b/common/src/main/java/com/pedro/common/Extensions.kt index 8b60f991c..b6074a071 100644 --- a/common/src/main/java/com/pedro/common/Extensions.kt +++ b/common/src/main/java/com/pedro/common/Extensions.kt @@ -43,7 +43,7 @@ import kotlin.coroutines.Continuation */ @Suppress("DEPRECATION") -fun MediaFrame.Info.isKeyframe(): Boolean { +fun MediaCodec.BufferInfo.isKeyframe(): Boolean { return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { this.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME } else { @@ -143,7 +143,7 @@ fun Throwable.validMessage(): String { return (message ?: "").ifEmpty { javaClass.simpleName } } -fun MediaCodec.BufferInfo.toMediaFrameInfo() = MediaFrame.Info(offset, size, presentationTimeUs, flags) +fun MediaCodec.BufferInfo.toMediaFrameInfo() = MediaFrame.Info(offset, size, presentationTimeUs, isKeyframe()) fun ByteBuffer.clone(): ByteBuffer = ByteBuffer.wrap(toByteArray()) diff --git a/common/src/main/java/com/pedro/common/frame/MediaFrame.kt b/common/src/main/java/com/pedro/common/frame/MediaFrame.kt index 2959ef47f..32529c870 100644 --- a/common/src/main/java/com/pedro/common/frame/MediaFrame.kt +++ b/common/src/main/java/com/pedro/common/frame/MediaFrame.kt @@ -11,7 +11,7 @@ data class MediaFrame( val offset: Int, val size: Int, val timestamp: Long, - val flags: Int + val isKeyFrame: Boolean ) enum class Type { diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt index c0ad14cc7..1c122ef81 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt @@ -96,7 +96,7 @@ class Av1Packet: BasePacket() { val size = fixedBuffer.remaining() buffer = ByteArray(header.size + size) - val nalType = if (info.isKeyframe()) VideoDataType.KEYFRAME.value else VideoDataType.INTER_FRAME.value + val nalType = if (info.isKeyFrame) VideoDataType.KEYFRAME.value else VideoDataType.INTER_FRAME.value header[0] = (0b10000000 or (nalType shl 4) or FourCCPacketType.CODED_FRAMES.value).toByte() fixedBuffer.get(buffer, header.size, size) diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt index 3d5391e87..ac6359bd3 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H264Packet.kt @@ -109,7 +109,7 @@ class H264Packet: BasePacket() { val type: Int = (validBuffer.get(0) and 0x1F).toInt() var nalType = VideoDataType.INTER_FRAME.value - if (type == VideoNalType.IDR.value || info.isKeyframe()) { + if (type == VideoNalType.IDR.value || info.isKeyFrame) { nalType = VideoDataType.KEYFRAME.value } else if (type == VideoNalType.SPS.value || type == VideoNalType.PPS.value) { // we don't need send it because we already do it in video config diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt index 912ffd4e6..61c440842 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt @@ -117,7 +117,7 @@ class H265Packet: BasePacket() { val type: Int = validBuffer.get(0).toInt().shr(1 and 0x3f) var nalType = VideoDataType.INTER_FRAME.value - if (type == VideoNalType.IDR_N_LP.value || type == VideoNalType.IDR_W_DLP.value || info.isKeyframe()) { + if (type == VideoNalType.IDR_N_LP.value || type == VideoNalType.IDR_W_DLP.value || info.isKeyFrame) { nalType = VideoDataType.KEYFRAME.value } else if (type == VideoNalType.HEVC_VPS.value || type == VideoNalType.HEVC_SPS.value || type == VideoNalType.HEVC_PPS.value) { // we don't need send it because we already do it in video config diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt index f413b840b..977b9a78c 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/Av1Packet.kt @@ -94,7 +94,7 @@ class Av1Packet: BasePacket( markPacket(buffer) //mark end frame } val oSize = if (isFirstPacket) obuList.size else 1 - buffer[RtpConstants.RTP_HEADER_LENGTH] = generateAv1AggregationHeader(bufferInfo.isKeyframe(), isFirstPacket, isLastPacket, oSize) + buffer[RtpConstants.RTP_HEADER_LENGTH] = generateAv1AggregationHeader(bufferInfo.isKeyFrame, isFirstPacket, isLastPacket, oSize) updateSeq(buffer) val rtpFrame = RtpFrame(buffer, rtpTs, buffer.size, channelIdentifier) frames.add(rtpFrame) diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index 3d23f21a5..65e71059a 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -65,7 +65,7 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, val naluLength = fixedBuffer.remaining() val type: Int = (header[header.size - 1] and 0x1F).toInt() val frames = mutableListOf() - if (type == RtpConstants.IDR || bufferInfo.isKeyframe()) { + if (type == RtpConstants.IDR || bufferInfo.isKeyFrame) { stapA?.let { val buffer = getBuffer(it.size + RtpConstants.RTP_HEADER_LENGTH) val rtpTs = updateTimeStamp(buffer, ts) diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt index baf3b10d8..ecee92f69 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt @@ -57,7 +57,7 @@ class H26XPacket( val fixedBuffer = byteBuffer.removeInfo(info) val length = fixedBuffer.remaining() if (length < 0) return - val isKeyFrame = info.isKeyframe() + val isKeyFrame = info.isKeyFrame if (codec == Codec.HEVC) { val sps = this.sps From de0458faf7ce6298a90def490bd0e20c31db5af5 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Tue, 1 Oct 2024 21:05:02 +0200 Subject: [PATCH 18/18] update keyframe in tests --- common/src/test/java/com/pedro/common/ExtensionTest.kt | 2 +- rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt | 2 +- rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt | 2 +- rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt | 2 +- rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt | 2 +- rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt | 2 +- rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt | 2 +- rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt | 2 +- rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt | 2 +- rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt | 4 ++-- rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt | 4 ++-- rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt | 2 +- srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt | 4 ++-- 13 files changed, 16 insertions(+), 16 deletions(-) diff --git a/common/src/test/java/com/pedro/common/ExtensionTest.kt b/common/src/test/java/com/pedro/common/ExtensionTest.kt index 776cfdb97..55fcc1f6a 100644 --- a/common/src/test/java/com/pedro/common/ExtensionTest.kt +++ b/common/src/test/java/com/pedro/common/ExtensionTest.kt @@ -31,7 +31,7 @@ class ExtensionTest { val buffer = ByteBuffer.wrap(ByteArray(256) { 0x00 }.mapIndexed { index, byte -> index.toByte() }.toByteArray()) val offset = 4 val minusLimit = 2 - val info = MediaFrame.Info(4, buffer.remaining() - minusLimit, 0, 0) + val info = MediaFrame.Info(4, buffer.remaining() - minusLimit, 0, false) val result = buffer.removeInfo(info) assertEquals(buffer.capacity() - offset - minusLimit, result.remaining()) assertEquals(offset.toByte(), result.get(0)) diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt index 53b6c7c03..8c40ba093 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt @@ -33,7 +33,7 @@ class AacPacketTest { fun `GIVEN a aac buffer WHEN call create a aac packet 2 times THEN return config and expected buffer`() = runTest { val timestamp = 123456789L val buffer = ByteArray(256) { 0x00 } - val info = MediaFrame.Info(0, buffer.size, timestamp, 1) + val info = MediaFrame.Info(0, buffer.size, timestamp, false) val aacPacket = AacPacket() aacPacket.sendAudioInfo(32000, true) aacPacket.createFlvPacket(ByteBuffer.wrap(buffer), info) { flvPacket -> diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt index 5c16ee8d8..6b2ad6656 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/G711PacketTest.kt @@ -33,7 +33,7 @@ class G711PacketTest { fun `GIVEN a G711 buffer WHEN call create a G711 packet THEN expected buffer`() = runTest { val timestamp = 123456789L val buffer = ByteArray(256) { 0x00 } - val info = MediaFrame.Info(0, buffer.size, timestamp, 1) + val info = MediaFrame.Info(0, buffer.size, timestamp, false) val g711Packet = G711Packet() g711Packet.sendAudioInfo() g711Packet.createFlvPacket(ByteBuffer.wrap(buffer), info) { flvPacket -> diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt index 80528f18b..ef8cce402 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/Av1PacketTest.kt @@ -38,7 +38,7 @@ class Av1PacketTest { val expectedConfig = byteArrayOf(-112, 97, 118, 48, 49, -127, 4, 12, 0, 10, 13, 0, 0, 0, 36, 79, 126, 127, 0, 104, -125, 0, -125, 2) val expectedFlvPacket = byteArrayOf(-111, 97, 118, 48, 49, 10, 13, 0, 0, 0, 36, 79, 126, 127, 0, 104, -125, 0, -125, 2) - val info = MediaFrame.Info(0, av1data.size, timestamp, 1) + val info = MediaFrame.Info(0, av1data.size, timestamp, true) val frames = mutableListOf() val av1Packet = Av1Packet() diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt index 30c91257c..1a9bbb4e1 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H264PacketTest.kt @@ -39,7 +39,7 @@ class H264PacketTest { val expectedConfig = byteArrayOf(23, 0, 0, 0, 0, 1, 100, 0, 30, -1, -31, 0, 17, 103, 100, 0, 30, -84, -76, 15, 2, -115, 53, 2, 2, 2, 7, -117, 23, 8, 1, 0, 4, 104, -18, 13, -117) val expectedFlvPacket = byteArrayOf(23, 1, 0, 0, 0, 0, 0, 1, 45, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - val info = MediaFrame.Info(0, fakeH264.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeH264.size, timestamp, true) val h264Packet = H264Packet() val sps = byteArrayOf(103, 100, 0, 30, -84, -76, 15, 2, -115, 53, 2, 2, 2, 7, -117, 23, 8) val pps = byteArrayOf(104, -18, 13, -117) diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt index 4345c7e93..7c4b9a44a 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/video/H265PacketTest.kt @@ -39,7 +39,7 @@ class H265PacketTest { val expectedConfig = byteArrayOf(-112, 104, 118, 99, 49, 1, 1, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, -103, -16, 0, -4, -3, -8, -8, 0, 0, 3, 3, -96, 0, 1, 0, 24, 64, 1, 12, 1, -1, -1, 1, 96, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, -103, 44, 9, -95, 0, 1, 0, 35, 66, 1, 1, 1, 96, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, -103, -96, 15, 8, 2, -127, 104, -76, -82, -55, 46, -26, -96, -64, -64, -64, 16, -94, 0, 1, 0, 8, 68, 1, -64, 102, 124, 12, -58, 64) val expectedFlvPacket = byteArrayOf(-111, 104, 118, 99, 49, 0, 0, 0, 0, 0, 1, 45, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - val info = MediaFrame.Info(0, fakeH264.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeH264.size, timestamp, true) val h265Packet = H265Packet() val vps = byteArrayOf(64, 1, 12, 1, -1, -1, 1, 96, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, -103, 44, 9) val sps = byteArrayOf(66, 1, 1, 1, 96, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, -103, -96, 15, 8, 2, -127, 104, -76, -82, -55, 46, -26, -96, -64, -64, -64, 16) diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt index ab33a1a72..ebff9d0c6 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/AacPacketTest.kt @@ -35,7 +35,7 @@ class AacPacketTest { val timestamp = 123456789L val fakeAac = ByteArray(300) { 0x00 } - val info = MediaFrame.Info(0, fakeAac.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeAac.size, timestamp, false) val aacPacket = AacPacket().apply { setAudioInfo(44100) } aacPacket.setSSRC(123456789) val frames = mutableListOf() diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt index 593db66ad..92737f1f4 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/Av1PacketTest.kt @@ -35,7 +35,7 @@ class Av1PacketTest { val timestamp = 123456789L val av1data = byteArrayOf(0x0a, 0x0d, 0x00, 0x00, 0x00, 0x24, 0x4f, 0x7e, 0x7f, 0x00, 0x68, 0x83.toByte(), 0x00, 0x83.toByte(), 0x02) - val info = MediaFrame.Info(0, av1data.size, timestamp, 1) + val info = MediaFrame.Info(0, av1data.size, timestamp, true) val frames = mutableListOf() val av1Packet = Av1Packet() diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt index 6b6f208cf..993f5038c 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/G711PacketTest.kt @@ -35,7 +35,7 @@ class G711PacketTest { val timestamp = 123456789L val fakeG711 = ByteArray(30) { 0x05 } - val info = MediaFrame.Info(0, fakeG711.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeG711.size, timestamp, false) val g711Packet = G711Packet().apply { setAudioInfo(8000) } g711Packet.setSSRC(123456789) val frames = mutableListOf() diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt index 375ce4e79..62854d23b 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt @@ -38,7 +38,7 @@ class H264PacketTest { val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05) val fakeH264 = header.plus(ByteArray(300) { 0x00 }) - val info = MediaFrame.Info(0, fakeH264.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeH264.size, timestamp, true) val fakeSps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04) val fakePps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0A, 0x0B, 0x0C) val h264Packet = H264Packet().apply { sendVideoInfo(ByteBuffer.wrap(fakeSps), ByteBuffer.wrap(fakePps)) } @@ -67,7 +67,7 @@ class H264PacketTest { val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05) val fakeH264 = header.plus(ByteArray(2500) { 0x00 }) - val info = MediaFrame.Info(0, fakeH264.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeH264.size, timestamp, true) val fakeSps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04) val fakePps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0A, 0x0B, 0x0C) val h264Packet = H264Packet().apply { sendVideoInfo(ByteBuffer.wrap(fakeSps), ByteBuffer.wrap(fakePps)) } diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt index fa01f324b..32e46ea5e 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt @@ -38,7 +38,7 @@ class H265PacketTest { val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05, 0x00) val fakeH265 = header.plus(ByteArray(300) { 0x00 }) - val info = MediaFrame.Info(0, fakeH265.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeH265.size, timestamp, true) val h265Packet = H265Packet() h265Packet.setSSRC(123456789) @@ -63,7 +63,7 @@ class H265PacketTest { val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05, 0x00) val fakeH265 = header.plus(ByteArray(2500) { 0x00 }) - val info = MediaFrame.Info(0, fakeH265.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeH265.size, timestamp, true) val h265Packet = H265Packet() h265Packet.setSSRC(123456789) val frames = mutableListOf() diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt index f0195435c..185f95035 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/OpusPacketTest.kt @@ -35,7 +35,7 @@ class OpusPacketTest { val timestamp = 123456789L val fakeOpus = ByteArray(30) { 0x05 } - val info = MediaFrame.Info(0, fakeOpus.size, timestamp, 1) + val info = MediaFrame.Info(0, fakeOpus.size, timestamp, false) val opusPacket = OpusPacket().apply { setAudioInfo(8000) } opusPacket.setSSRC(123456789) val frames = mutableListOf() diff --git a/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt b/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt index af1fd0d57..b27b5998d 100644 --- a/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt +++ b/srt/src/test/java/com/pedro/srt/srt/SrtSenderTest.kt @@ -62,8 +62,8 @@ class SrtSenderTest { val videoData = ByteBuffer.wrap(header.plus(ByteArray(300) { 0x00 })) val audioData = ByteBuffer.wrap(ByteArray(256) { 0x00 }) - val videoFrame = MediaFrame(videoData, MediaFrame.Info(0, videoData.remaining(), 0, 1), MediaFrame.Type.VIDEO) - val audioFrame = MediaFrame(audioData, MediaFrame.Info(0, audioData.remaining(), 0, 1), MediaFrame.Type.AUDIO) + val videoFrame = MediaFrame(videoData, MediaFrame.Info(0, videoData.remaining(), 0, true), MediaFrame.Type.VIDEO) + val audioFrame = MediaFrame(audioData, MediaFrame.Info(0, audioData.remaining(), 0, false), MediaFrame.Type.AUDIO) srtSender.sendMediaFrame(videoFrame) srtSender.sendMediaFrame(audioFrame) latch.await(1000, TimeUnit.MILLISECONDS)