From 952e336554ab6ad2384d6447d3d63f7b37657d40 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 5 Dec 2023 21:10:23 +0100 Subject: [PATCH] adding av1 support to rtmp --- .../openglexample/OpenGlRtmpActivity.java | 2 + .../main/java/com/pedro/common/Extensions.kt | 11 ++ .../com/pedro/rtmp/flv/video/Av1Packet.kt | 171 ++++++++++++++++++ .../com/pedro/rtmp/flv/video/H265Packet.kt | 2 +- .../com/pedro/rtmp/flv/video/VideoNalType.kt | 4 +- .../pedro/rtmp/rtmp/CommandsManagerAmf0.kt | 14 +- .../java/com/pedro/rtmp/rtmp/RtmpSender.kt | 10 +- .../com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt | 2 +- .../pedro/srt/mpeg2ts/packets/H26XPacket.kt | 2 +- .../java/com/pedro/srt/utils/Extensions.kt | 11 -- 10 files changed, 209 insertions(+), 20 deletions(-) create mode 100644 rtmp/src/main/java/com/pedro/rtmp/flv/video/Av1Packet.kt diff --git a/app/src/main/java/com/pedro/streamer/openglexample/OpenGlRtmpActivity.java b/app/src/main/java/com/pedro/streamer/openglexample/OpenGlRtmpActivity.java index f6ef6ad01..d7f719206 100644 --- a/app/src/main/java/com/pedro/streamer/openglexample/OpenGlRtmpActivity.java +++ b/app/src/main/java/com/pedro/streamer/openglexample/OpenGlRtmpActivity.java @@ -38,6 +38,7 @@ import androidx.appcompat.app.AppCompatActivity; import com.pedro.common.ConnectChecker; +import com.pedro.common.VideoCodec; import com.pedro.encoder.input.gl.SpriteGestureController; import com.pedro.encoder.input.gl.render.filters.AnalogTVFilterRender; import com.pedro.encoder.input.gl.render.filters.AndroidViewFilterRender; @@ -131,6 +132,7 @@ protected void onCreate(Bundle savedInstanceState) { etUrl = findViewById(R.id.et_rtp_url); etUrl.setHint(R.string.hint_rtmp); rtmpCamera1 = new RtmpCamera1(openGlView, this); + rtmpCamera1.setVideoCodec(VideoCodec.AV1); openGlView.getHolder().addCallback(this); openGlView.setOnTouchListener(this); } diff --git a/common/src/main/java/com/pedro/common/Extensions.kt b/common/src/main/java/com/pedro/common/Extensions.kt index 1710d8538..7145fc8b0 100644 --- a/common/src/main/java/com/pedro/common/Extensions.kt +++ b/common/src/main/java/com/pedro/common/Extensions.kt @@ -26,6 +26,17 @@ import java.util.concurrent.BlockingQueue * Created by pedro on 3/11/23. */ +fun ByteBuffer.toByteArray(): ByteArray { + return if (this.hasArray() && !isDirect) { + this.array() + } else { + this.rewind() + val byteArray = ByteArray(this.remaining()) + this.get(byteArray) + byteArray + } +} + fun ByteBuffer.removeInfo(info: MediaCodec.BufferInfo): ByteBuffer { try { position(info.offset) diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/Av1Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/Av1Packet.kt new file mode 100644 index 000000000..afdea9dd6 --- /dev/null +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/Av1Packet.kt @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2023 pedroSG94. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedro.rtmp.flv.video + +import android.media.MediaCodec +import android.util.Log +import com.pedro.common.removeInfo +import com.pedro.common.toByteArray +import com.pedro.rtmp.flv.FlvPacket +import com.pedro.rtmp.flv.FlvType +import java.nio.ByteBuffer + +/** + * Created by pedro on 05/12/23. + * + */ +class Av1Packet { + + private val TAG = "AV1Packet" + + private val header = ByteArray(8) + private val naluSize = 4 + //first time we need send video config + private var configSend = false + + private var av1ConfigurationRecord: ByteArray? = null + var profileIop = ProfileIop.BASELINE + + fun sendVideoInfo(av1ConfigurationRecord: ByteBuffer) { + val mAv1ConfigurationRecord = removeHeader(av1ConfigurationRecord) + this.av1ConfigurationRecord = mAv1ConfigurationRecord.toByteArray() + } + + fun createFlvVideoPacket( + byteBuffer: ByteBuffer, + info: MediaCodec.BufferInfo, + callback: (FlvPacket) -> Unit + ) { + val fixedBuffer = byteBuffer.removeInfo(info) + val ts = info.presentationTimeUs / 1000 + + //header is 8 bytes length: + //mark first byte as extended header (0b10000000) + //4 bits data type, 4 bits packet type + //4 bytes extended codec type (in this case av01) + //3 bytes CompositionTime, the cts. + val codec = VideoFormat.AV1.value // { "a", "v", "0", "1" } + header[1] = (codec shr 24).toByte() + header[2] = (codec shr 16).toByte() + header[3] = (codec shr 8).toByte() + header[4] = codec.toByte() + val cts = 0 + val ctsLength = 3 + header[5] = (cts shr 16).toByte() + header[6] = (cts shr 8).toByte() + header[7] = cts.toByte() + + var buffer: ByteArray + if (!configSend) { + //avoid send cts on sequence start + header[0] = (0b10000000 or (VideoDataType.KEYFRAME.value shl 4) or FourCCPacketType.SEQUENCE_START.value).toByte() + val sps = this.av1ConfigurationRecord + if (sps != null) { + val config = sps + buffer = ByteArray(config.size + header.size - ctsLength) + val b = ByteBuffer.wrap(buffer, header.size - ctsLength, sps.size) + b.put(sps) + } else { + Log.e(TAG, "waiting for a valid sps and pps") + return + } + + System.arraycopy(header, 0, buffer, 0, header.size - ctsLength) + callback(FlvPacket(buffer, ts, buffer.size, FlvType.VIDEO)) + configSend = true + } + val headerSize = getHeaderSize(fixedBuffer) + if (headerSize == 0) return //invalid buffer or waiting for sps/pps + fixedBuffer.rewind() + val validBuffer = removeHeader(fixedBuffer, headerSize) + val size = validBuffer.remaining() + buffer = ByteArray(header.size + size + naluSize) + + val type: Int = validBuffer.get(0).toInt().shr(1 and 0x3f) + var nalType = VideoDataType.INTER_FRAME.value + if (type == VideoNalType.KEY.value || info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) { + nalType = VideoDataType.KEYFRAME.value + } else if (type == VideoNalType.CONFIG.value) { + // we don't need send it because we already do it in video config + return + } + header[0] = (0b10000000 or (nalType shl 4) or FourCCPacketType.CODED_FRAMES.value).toByte() + writeNaluSize(buffer, header.size, size) + validBuffer.get(buffer, header.size + naluSize, size) + + System.arraycopy(header, 0, buffer, 0, header.size) + callback(FlvPacket(buffer, ts, buffer.size, FlvType.VIDEO)) + } + + //naluSize = UInt32 + private fun writeNaluSize(buffer: ByteArray, offset: Int, size: Int) { + buffer[offset] = (size ushr 24).toByte() + buffer[offset + 1] = (size ushr 16).toByte() + buffer[offset + 2] = (size ushr 8).toByte() + buffer[offset + 3] = size.toByte() + } + + private fun removeHeader(byteBuffer: ByteBuffer, size: Int = -1): ByteBuffer { + val position = if (size == -1) getStartCodeSize(byteBuffer) else size + byteBuffer.position(position) + return byteBuffer.slice() + } + + private fun getHeaderSize(byteBuffer: ByteBuffer): Int { + if (byteBuffer.remaining() < 4) return 0 + + val av1ConfigurationRecord = this.av1ConfigurationRecord + if (av1ConfigurationRecord != null) { + val startCodeSize = getStartCodeSize(byteBuffer) + if (startCodeSize == 0) return 0 + val startCode = ByteArray(startCodeSize) { 0x00 } + startCode[startCodeSize - 1] = 0x01 + val avHeader = startCode.plus(av1ConfigurationRecord) + if (byteBuffer.remaining() < avHeader.size) return startCodeSize + + val possibleAvcHeader = ByteArray(avHeader.size) + byteBuffer.get(possibleAvcHeader, 0, possibleAvcHeader.size) + return if (avHeader.contentEquals(possibleAvcHeader)) { + avHeader.size + } else { + startCodeSize + } + } + return 0 + } + + private fun getStartCodeSize(byteBuffer: ByteBuffer): Int { + var startCodeSize = 0 + if (byteBuffer.get(0).toInt() == 0x00 && byteBuffer.get(1).toInt() == 0x00 + && byteBuffer.get(2).toInt() == 0x00 && byteBuffer.get(3).toInt() == 0x01) { + //match 00 00 00 01 + startCodeSize = 4 + } else if (byteBuffer.get(0).toInt() == 0x00 && byteBuffer.get(1).toInt() == 0x00 + && byteBuffer.get(2).toInt() == 0x01) { + //match 00 00 01 + startCodeSize = 3 + } + return startCodeSize + } + + fun reset(resetInfo: Boolean = true) { + if (resetInfo) { + av1ConfigurationRecord = null + } + configSend = false + } +} \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt index ed44eefb8..06ef60231 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/H265Packet.kt @@ -29,7 +29,7 @@ import java.nio.ByteBuffer */ class H265Packet { - private val TAG = "H264Packet" + private val TAG = "H265Packet" private val header = ByteArray(8) private val naluSize = 4 diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoNalType.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoNalType.kt index 8f4eb723c..8feaa6804 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoNalType.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoNalType.kt @@ -24,5 +24,7 @@ enum class VideoNalType(val value: Int) { SPS(7), PPS(8), AUD(9), EO_SEQ(10), EO_STREAM(11), FILL(12), HEVC_VPS(32), HEVC_SPS(33), HEVC_PPS(34), //H265 IDR - IDR_N_LP(20), IDR_W_DLP(19) + IDR_N_LP(20), IDR_W_DLP(19), + //AV01 + KEY(50), CONFIG(51) } \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/CommandsManagerAmf0.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/CommandsManagerAmf0.kt index 45262b965..478f593ea 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/CommandsManagerAmf0.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/CommandsManagerAmf0.kt @@ -55,6 +55,11 @@ class CommandsManagerAmf0: CommandsManager() { list.add(AmfString("hvc1")) val array = AmfStrictArray(list) connectInfo.setProperty("fourCcList", array) + } else if (videoCodec == VideoCodec.AV1) { + val list = mutableListOf() + list.add(AmfString("av01")) + val array = AmfStrictArray(list) + connectInfo.setProperty("fourCcList", array) } } connectInfo.setProperty("pageUrl", "") @@ -108,9 +113,12 @@ class CommandsManagerAmf0: CommandsManager() { amfEcmaArray.setProperty("width", width.toDouble()) amfEcmaArray.setProperty("height", height.toDouble()) //few servers don't support it even if it is in the standard rtmp enhanced - //val codecValue = if (videoCodec == VideoCodec.H265) VideoFormat.HEVC.value else VideoFormat.AVC.value - //amfEcmaArray.setProperty("videocodecid", codecValue.toDouble()) - amfEcmaArray.setProperty("videocodecid", VideoFormat.AVC.value.toDouble()) + val codecValue = when (videoCodec) { + VideoCodec.H264 -> VideoFormat.AVC.value + VideoCodec.H265 -> VideoFormat.HEVC.value + VideoCodec.AV1 -> VideoFormat.AV1.value + } + amfEcmaArray.setProperty("videocodecid", codecValue.toDouble()) amfEcmaArray.setProperty("framerate", fps.toDouble()) amfEcmaArray.setProperty("videodatarate", 0.0) } diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt index 0c270de13..bd59e45d6 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt @@ -26,6 +26,7 @@ import com.pedro.common.trySend import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.audio.AacPacket +import com.pedro.rtmp.flv.video.Av1Packet import com.pedro.rtmp.flv.video.H264Packet import com.pedro.rtmp.flv.video.H265Packet import com.pedro.rtmp.flv.video.ProfileIop @@ -55,6 +56,7 @@ class RtmpSender( private var aacPacket = AacPacket() private var h264Packet = H264Packet() private var h265Packet = H265Packet() + private var av1Packet = Av1Packet() @Volatile private var running = false private var cacheSize = 200 @@ -83,7 +85,7 @@ class RtmpSender( if (vps == null || pps == null) throw IllegalArgumentException("pps or vps can't be null with h265") h265Packet.sendVideoInfo(sps, pps, vps) } else if (videoCodec == VideoCodec.AV1) { - //TODO send info to av1 packet + av1Packet.sendVideoInfo(sps) } else { if (pps == null) throw IllegalArgumentException("pps can't be null with h264") h264Packet.sendVideoInfo(sps, pps) @@ -108,7 +110,11 @@ class RtmpSender( fun sendVideoFrame(h264Buffer: ByteBuffer, info: MediaCodec.BufferInfo) { if (running) { - if (videoCodec == VideoCodec.H265) { + if (videoCodec == VideoCodec.AV1) { + av1Packet.createFlvVideoPacket(h264Buffer, info) { flvPacket -> + enqueueVideoFrame(flvPacket) + } + } else if (videoCodec == VideoCodec.H265) { h265Packet.createFlvVideoPacket(h264Buffer, info) { flvPacket -> enqueueVideoFrame(flvPacket) } diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt index 1d36340c2..e41cf1b7d 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/MpegTsPacketizer.kt @@ -19,7 +19,7 @@ package com.pedro.srt.mpeg2ts import com.pedro.srt.mpeg2ts.psi.Psi import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.common.TimeUtils -import com.pedro.srt.utils.toByteArray +import com.pedro.common.toByteArray import com.pedro.srt.utils.toInt import java.nio.ByteBuffer diff --git a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt index 97d9e29f4..7b49d7048 100644 --- a/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt +++ b/srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt @@ -20,6 +20,7 @@ import android.media.MediaCodec import android.os.Build import android.util.Log import com.pedro.common.removeInfo +import com.pedro.common.toByteArray import com.pedro.srt.mpeg2ts.Codec import com.pedro.srt.mpeg2ts.MpegTsPacket import com.pedro.srt.mpeg2ts.MpegType @@ -28,7 +29,6 @@ import com.pedro.srt.mpeg2ts.PesType import com.pedro.srt.mpeg2ts.psi.PsiManager import com.pedro.srt.srt.packets.data.PacketPosition import com.pedro.srt.utils.startWith -import com.pedro.srt.utils.toByteArray import java.nio.ByteBuffer /** diff --git a/srt/src/main/java/com/pedro/srt/utils/Extensions.kt b/srt/src/main/java/com/pedro/srt/utils/Extensions.kt index b94b56ddd..aa7d5e63f 100644 --- a/srt/src/main/java/com/pedro/srt/utils/Extensions.kt +++ b/srt/src/main/java/com/pedro/srt/utils/Extensions.kt @@ -21,17 +21,6 @@ import java.io.OutputStream import java.nio.ByteBuffer -fun ByteBuffer.toByteArray(): ByteArray { - return if (this.hasArray() && !isDirect) { - this.array() - } else { - this.rewind() - val byteArray = ByteArray(this.remaining()) - this.get(byteArray) - byteArray - } -} - fun ByteBuffer.startWith(byteArray: ByteArray): Boolean { val startData = ByteArray(byteArray.size) this.rewind()