diff --git a/common/src/main/java/com/pedro/common/socket/StreamSocket.kt b/common/src/main/java/com/pedro/common/socket/StreamSocket.kt index be5dfd7e7..cff021dc7 100644 --- a/common/src/main/java/com/pedro/common/socket/StreamSocket.kt +++ b/common/src/main/java/com/pedro/common/socket/StreamSocket.kt @@ -18,46 +18,12 @@ package com.pedro.common.socket -import io.ktor.network.selector.SelectorManager -import io.ktor.network.sockets.ReadWriteSocket -import io.ktor.network.sockets.isClosed -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import java.net.InetAddress -import java.net.InetSocketAddress - /** * Created by pedro on 22/9/24. */ -abstract class StreamSocket( - private val host: String, - private val port: Int -) { - - private var selectorManager = SelectorManager(Dispatchers.IO) - protected var socket: ReadWriteSocket? = null - private var address: InetAddress? = null - - abstract suspend fun buildSocketConfigAndConnect(selectorManager: SelectorManager): ReadWriteSocket - abstract suspend fun closeResources() - - suspend fun connect() { - selectorManager = SelectorManager(Dispatchers.IO) - val socket = buildSocketConfigAndConnect(selectorManager) - address = InetSocketAddress(host, port).address - this.socket = socket - } - - suspend fun close() = withContext(Dispatchers.IO) { - try { - address = null - closeResources() - socket?.close() - selectorManager.close() - } catch (ignored: Exception) {} - } - - fun isConnected(): Boolean = socket?.isClosed != true - - fun isReachable(): Boolean = address?.isReachable(5000) ?: false +interface StreamSocket { + suspend fun connect() + suspend fun close() + fun isConnected(): Boolean + fun isReachable(): Boolean } \ No newline at end of file diff --git a/common/src/main/java/com/pedro/common/socket/TcpStreamSocket.kt b/common/src/main/java/com/pedro/common/socket/TcpStreamSocket.kt index c85604d00..b6dd9effd 100644 --- a/common/src/main/java/com/pedro/common/socket/TcpStreamSocket.kt +++ b/common/src/main/java/com/pedro/common/socket/TcpStreamSocket.kt @@ -22,21 +22,25 @@ import io.ktor.network.selector.SelectorManager import io.ktor.network.sockets.InetSocketAddress import io.ktor.network.sockets.ReadWriteSocket import io.ktor.network.sockets.aSocket +import io.ktor.network.sockets.isClosed import io.ktor.network.sockets.openReadChannel import io.ktor.network.sockets.openWriteChannel import io.ktor.network.tls.tls import io.ktor.utils.io.ByteReadChannel import io.ktor.utils.io.ByteWriteChannel +import io.ktor.utils.io.readByte import io.ktor.utils.io.readFully import io.ktor.utils.io.readUTF8Line import io.ktor.utils.io.writeByte import io.ktor.utils.io.writeFully import io.ktor.utils.io.writeStringUtf8 import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout import java.net.ConnectException import java.security.SecureRandom import javax.net.ssl.TrustManager +import java.net.InetAddress /** * Created by pedro on 22/9/24. @@ -46,38 +50,50 @@ class TcpStreamSocket( private val port: Int, private val secured: Boolean = false, private val certificate: TrustManager? = null -): StreamSocket(host, port) { +): StreamSocket { private val timeout = 5000L private var input: ByteReadChannel? = null private var output: ByteWriteChannel? = null + private var selectorManager = SelectorManager(Dispatchers.IO) + private var socket: ReadWriteSocket? = null + private var address: InetAddress? = null - override suspend fun buildSocketConfigAndConnect(selectorManager: SelectorManager): ReadWriteSocket { - val builder = aSocket(selectorManager).tcp() + override suspend fun connect() { + selectorManager = SelectorManager(Dispatchers.IO) + val builder = aSocket(selectorManager).tcp().connect(remoteAddress = InetSocketAddress(host, port)) val socket = if (secured) { - builder.connect(remoteAddress = InetSocketAddress(host, port)).tls(Dispatchers.Default) { + builder.tls(Dispatchers.Default) { trustManager = certificate random = SecureRandom() } - } else { - builder.connect(host, port) - } + } else builder input = socket.openReadChannel() output = socket.openWriteChannel(autoFlush = false) - return socket + address = java.net.InetSocketAddress(host, port).address + this.socket = socket } - override suspend fun closeResources() { - input = null - output = null + override suspend fun close() = withContext(Dispatchers.IO) { + try { + address = null + input = null + output = null + socket?.close() + selectorManager.close() + } catch (ignored: Exception) {} } + override fun isConnected(): Boolean = socket?.isClosed != true + + override fun isReachable(): Boolean = address?.isReachable(5000) ?: false + suspend fun flush() { output?.flush() } suspend fun write(b: Int) = withTimeout(timeout) { - output?.writeByte(b) + output?.writeByte(b.toByte()) } suspend fun write(b: ByteArray) = withTimeout(timeout) { @@ -85,7 +101,7 @@ class TcpStreamSocket( } suspend fun write(b: ByteArray, offset: Int, size: Int) = withTimeout(timeout) { - output?.writeFully(b, offset, size) + output?.writeFully(b, offset, offset + size) } suspend fun writeUInt16(b: Int) { diff --git a/common/src/main/java/com/pedro/common/socket/UdpStreamSocket.kt b/common/src/main/java/com/pedro/common/socket/UdpStreamSocket.kt index aaf16b725..ce8cb1b41 100644 --- a/common/src/main/java/com/pedro/common/socket/UdpStreamSocket.kt +++ b/common/src/main/java/com/pedro/common/socket/UdpStreamSocket.kt @@ -22,49 +22,67 @@ import io.ktor.network.selector.SelectorManager import io.ktor.network.sockets.ConnectedDatagramSocket import io.ktor.network.sockets.Datagram import io.ktor.network.sockets.InetSocketAddress -import io.ktor.network.sockets.ReadWriteSocket import io.ktor.network.sockets.aSocket +import io.ktor.network.sockets.isClosed import io.ktor.utils.io.core.ByteReadPacket -import io.ktor.utils.io.core.readBytes +import io.ktor.utils.io.core.remaining +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.io.readByteArray +import java.net.ConnectException +import java.net.InetAddress /** * Created by pedro on 22/9/24. */ class UdpStreamSocket( - host: String, - port: Int, + private val host: String, + private val port: Int, private val sourcePort: Int? = null, private val receiveSize: Int? = null, private val broadcastMode: Boolean = false -): StreamSocket(host, port) { +): StreamSocket { private val address = InetSocketAddress(host, port) - private val udpSocket by lazy { - socket as ConnectedDatagramSocket - } + private var selectorManager = SelectorManager(Dispatchers.IO) + private var socket: ConnectedDatagramSocket? = null + private var myAddress: InetAddress? = null - override suspend fun buildSocketConfigAndConnect(selectorManager: SelectorManager): ReadWriteSocket { + override suspend fun connect() { + selectorManager = SelectorManager(Dispatchers.IO) val builder = aSocket(selectorManager).udp() val localAddress = if (sourcePort == null) null else InetSocketAddress("0.0.0.0", sourcePort) - return builder.connect( + val socket = builder.connect( remoteAddress = address, localAddress = localAddress ) { broadcast = broadcastMode receiveBufferSize = receiveSize ?: 0 } + myAddress = java.net.InetSocketAddress(host, port).address + this.socket = socket + } + + override suspend fun close() = withContext(Dispatchers.IO) { + try { + socket?.close() + selectorManager.close() + } catch (ignored: Exception) {} } - override suspend fun closeResources() { } + override fun isConnected(): Boolean = socket?.isClosed != true + + override fun isReachable(): Boolean = myAddress?.isReachable(5000) ?: false suspend fun readPacket(): ByteArray { - val packet = udpSocket.receive().packet + val socket = socket ?: throw ConnectException("Read with socket closed, broken pipe") + val packet = socket.receive().packet val length = packet.remaining.toInt() - return packet.readBytes().sliceArray(0 until length) + return packet.readByteArray().sliceArray(0 until length) } suspend fun writePacket(bytes: ByteArray) { val datagram = Datagram(ByteReadPacket(bytes), address) - udpSocket.send(datagram) + socket?.send(datagram) } } \ No newline at end of file diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/AndroidExtractor.kt b/encoder/src/main/java/com/pedro/encoder/input/decoder/AndroidExtractor.kt new file mode 100644 index 000000000..d4b7cb0a4 --- /dev/null +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/AndroidExtractor.kt @@ -0,0 +1,151 @@ +/* + * + * * Copyright (C) 2024 pedroSG94. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.pedro.encoder.input.decoder + +import android.content.Context +import android.media.MediaExtractor +import android.media.MediaFormat +import android.net.Uri +import com.pedro.common.frame.MediaFrame +import com.pedro.common.getIntegerSafe +import com.pedro.common.getLongSafe +import com.pedro.common.validMessage +import java.io.FileDescriptor +import java.io.IOException +import java.nio.ByteBuffer + +/** + * Created by pedro on 18/10/24. + */ +class AndroidExtractor: Extractor { + + private var mediaExtractor = MediaExtractor() + private var sleepTime: Long = 0 + private var accumulativeTs: Long = 0 + @Volatile + private var lastExtractorTs: Long = 0 + private var format: MediaFormat? = null + + override fun selectTrack(type: MediaFrame.Type): String { + return when (type) { + MediaFrame.Type.VIDEO -> selectTrack("video/") + MediaFrame.Type.AUDIO -> selectTrack("audio/") + } + } + + override fun initialize(path: String) { + try { + reset() + mediaExtractor = MediaExtractor() + mediaExtractor.setDataSource(path) + } catch (e: Exception) { + throw IOException(e.validMessage()) + } + } + + override fun initialize(context: Context, uri: Uri) { + try { + reset() + mediaExtractor = MediaExtractor() + mediaExtractor.setDataSource(context, uri, null) + } catch (e: Exception) { + throw IOException(e.validMessage()) + } + } + + override fun initialize(fileDescriptor: FileDescriptor) { + try { + reset() + mediaExtractor = MediaExtractor() + mediaExtractor.setDataSource(fileDescriptor) + } catch (e: Exception) { + throw IOException(e.validMessage()) + } + } + + override fun readFrame(buffer: ByteBuffer): Int { + return mediaExtractor.readSampleData(buffer, 0) + } + + override fun advance(): Boolean { + return mediaExtractor.advance() + } + + override fun getTimeStamp(): Long { + return mediaExtractor.sampleTime + } + + override fun getSleepTime(ts: Long): Long { + val extractorTs = getTimeStamp() + accumulativeTs += extractorTs - lastExtractorTs + lastExtractorTs = getTimeStamp() + sleepTime = if (accumulativeTs > ts) (accumulativeTs - ts) / 1000 else 0 + return sleepTime + } + + override fun seekTo(time: Long) { + mediaExtractor.seekTo(time, MediaExtractor.SEEK_TO_PREVIOUS_SYNC) + lastExtractorTs = getTimeStamp() + } + + override fun release() { + mediaExtractor.release() + } + + override fun getVideoInfo(): VideoInfo { + val format = this.format ?: throw IOException("Extractor track not selected") + val width = format.getIntegerSafe(MediaFormat.KEY_WIDTH) ?: throw IOException("Width info is required") + val height = format.getIntegerSafe(MediaFormat.KEY_HEIGHT) ?: throw IOException("Height info is required") + val duration = format.getLongSafe(MediaFormat.KEY_DURATION) ?: throw IOException("Duration info is required") + val fps = format.getIntegerSafe(MediaFormat.KEY_FRAME_RATE) ?: 30 + return VideoInfo(width, height, fps, duration) + } + + override fun getAudioInfo(): AudioInfo { + val format = this.format ?: throw IOException("Extractor track not selected") + val sampleRate = format.getIntegerSafe(MediaFormat.KEY_SAMPLE_RATE) ?: throw IOException("Channels info is required") + val channels = format.getIntegerSafe(MediaFormat.KEY_CHANNEL_COUNT) ?: throw IOException("SampleRate info is required") + val duration = format.getLongSafe(MediaFormat.KEY_DURATION) ?: throw IOException("Duration info is required") + return AudioInfo(sampleRate, channels, duration) + } + + override fun getFormat(): MediaFormat { + return format ?: throw IOException("Extractor track not selected") + } + + private fun selectTrack(type: String): String { + for (i in 0 until mediaExtractor.trackCount) { + val format = mediaExtractor.getTrackFormat(i) + val mime = format.getString(MediaFormat.KEY_MIME) ?: continue + if (mime.startsWith(type, ignoreCase = true)) { + mediaExtractor.selectTrack(i) + this.format = format + return mime + } + } + throw IOException("track not found") + } + + private fun reset() { + sleepTime = 0 + accumulativeTs = 0 + lastExtractorTs = 0 + format = null + } +} \ No newline at end of file diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioDecoder.java b/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioDecoder.java index 8d54b7741..0b03f4266 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioDecoder.java +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioDecoder.java @@ -21,6 +21,7 @@ import android.util.Log; import com.pedro.common.ExtensionsKt; +import com.pedro.common.frame.MediaFrame; import com.pedro.encoder.Frame; import com.pedro.encoder.input.audio.GetMicrophoneData; import com.pedro.encoder.utils.CodecUtil; @@ -52,39 +53,26 @@ public AudioDecoder(GetMicrophoneData getMicrophoneData, } @Override - protected boolean extract(MediaExtractor audioExtractor) { - size = 2048; - for (int i = 0; i < audioExtractor.getTrackCount() && !mime.startsWith("audio/"); i++) { - mediaFormat = audioExtractor.getTrackFormat(i); - mime = mediaFormat.getString(MediaFormat.KEY_MIME); - if (mime.startsWith("audio/")) { - audioExtractor.selectTrack(i); - } else { - mediaFormat = null; - } - } - if (mediaFormat != null) { - final Integer channels = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_CHANNEL_COUNT); - final Integer sampleRate = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_SAMPLE_RATE); - final Long duration = ExtensionsKt.getLongSafe(mediaFormat, MediaFormat.KEY_DURATION); - if (channels == null || sampleRate == null) return false; - this.channels = channels; + protected boolean extract(Extractor extractor) { + try { + size = 2048; + mime = extractor.selectTrack(MediaFrame.Type.AUDIO); + AudioInfo audioInfo = extractor.getAudioInfo(); + mediaFormat = extractor.getFormat(); + this.channels = audioInfo.getChannels(); isStereo = channels >= 2; - this.sampleRate = sampleRate; - this.duration = duration != null ? duration : -1; + this.sampleRate = audioInfo.getSampleRate(); + this.duration = audioInfo.getSampleRate(); fixBuffer(); return true; - //audio decoder not supported - } else { + } catch (Exception e) { mime = ""; return false; } } private void fixBuffer() { - if (channels >= 2) { - size *= channels; - } + if (channels >= 2) size *= channels; pcmBuffer = new byte[size]; } @@ -121,35 +109,6 @@ protected void finished() { audioDecoderInterface.onAudioDecoderFinished(); } - /** - * This method should be called after prepare. - * Get max output size to set max input size in encoder. - */ - public int getOutsize() { - if (!(mime.equals(CodecUtil.AAC_MIME) || mime.equals(CodecUtil.OPUS_MIME) - || mime.equals(CodecUtil.VORBIS_MIME) || mime.equals(CodecUtil.G711_MIME))) { - Log.i(TAG, "fixing input size"); - try { - if (running) { - return codec.getOutputBuffers()[0].remaining(); - } else { - if (codec != null) { - codec.start(); - int outSize = codec.getOutputBuffers()[0].remaining(); - stopDecoder(); - if (prepare(null)) return outSize; - } - return 0; - } - } catch (Exception e) { - return 0; - } - } else { - Log.i(TAG, "default input size"); - return 0; - } - } - public void mute() { muted = true; } diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioInfo.kt b/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioInfo.kt new file mode 100644 index 000000000..ebc132b55 --- /dev/null +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/AudioInfo.kt @@ -0,0 +1,28 @@ +/* + * + * * Copyright (C) 2024 pedroSG94. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.pedro.encoder.input.decoder + +/** + * Created by pedro on 18/10/24. + */ +data class AudioInfo( + val sampleRate: Int, + val channels: Int, + val duration: Long +) diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/BaseDecoder.java b/encoder/src/main/java/com/pedro/encoder/input/decoder/BaseDecoder.java index 9357af1e2..c8d78cce5 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/decoder/BaseDecoder.java +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/BaseDecoder.java @@ -17,10 +17,7 @@ package com.pedro.encoder.input.decoder; import android.content.Context; -import android.content.res.AssetFileDescriptor; import android.media.MediaCodec; -import android.media.MediaDataSource; -import android.media.MediaExtractor; import android.media.MediaFormat; import android.net.Uri; import android.os.Build; @@ -29,19 +26,15 @@ import android.util.Log; import android.view.Surface; -import androidx.annotation.RequiresApi; - import java.io.FileDescriptor; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseDecoder { protected String TAG = "BaseDecoder"; protected MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); - protected MediaExtractor extractor; protected MediaCodec codec; protected volatile boolean running = false; protected MediaFormat mediaFormat; @@ -51,59 +44,27 @@ public abstract class BaseDecoder { private volatile long startTs = 0; protected long duration; protected final Object sync = new Object(); - private volatile long lastExtractorTs = 0; - //Avoid decode while change output surface protected AtomicBoolean pause = new AtomicBoolean(false); protected volatile boolean looped = false; private final DecoderInterface decoderInterface; + private Extractor extractor = new AndroidExtractor(); public BaseDecoder(DecoderInterface decoderInterface) { this.decoderInterface = decoderInterface; } public boolean initExtractor(String filePath) throws IOException { - extractor = new MediaExtractor(); - extractor.setDataSource(filePath); + extractor.initialize(filePath); return extract(extractor); } public boolean initExtractor(FileDescriptor fileDescriptor) throws IOException { - extractor = new MediaExtractor(); - extractor.setDataSource(fileDescriptor); - return extract(extractor); - } - - @RequiresApi(api = Build.VERSION_CODES.N) - public boolean initExtractor(AssetFileDescriptor assetFileDescriptor) throws IOException { - extractor = new MediaExtractor(); - extractor.setDataSource(assetFileDescriptor); - return extract(extractor); - } - - @RequiresApi(api = Build.VERSION_CODES.M) - public boolean initExtractor(MediaDataSource mediaDataSource) throws IOException { - extractor = new MediaExtractor(); - extractor.setDataSource(mediaDataSource); - return extract(extractor); - } - - public boolean initExtractor(String filePath, Map headers) throws IOException { - extractor = new MediaExtractor(); - extractor.setDataSource(filePath, headers); - return extract(extractor); - } - - public boolean initExtractor(FileDescriptor fileDescriptor, long offset, long length) - throws IOException { - extractor = new MediaExtractor(); - extractor.setDataSource(fileDescriptor, offset, length); + extractor.initialize(fileDescriptor); return extract(extractor); } - public boolean initExtractor(Context context, Uri uri, Map headers) - throws IOException { - extractor = new MediaExtractor(); - extractor.setDataSource(context, uri, headers); + public boolean initExtractor(Context context, Uri uri) throws IOException { + extractor.initialize(context, uri); return extract(extractor); } @@ -130,13 +91,8 @@ public void stop() { Log.i(TAG, "stop decoder"); running = false; stopDecoder(); - lastExtractorTs = 0; startTs = 0; - if (extractor != null) { - extractor.release(); - extractor = null; - mime = ""; - } + extractor.release(); } protected boolean prepare(Surface surface) { @@ -196,8 +152,7 @@ protected void stopDecoder(boolean clearTs) { public void moveTo(double time) { synchronized (sync) { - extractor.seekTo((long) (time * 10E5), MediaExtractor.SEEK_TO_PREVIOUS_SYNC); - lastExtractorTs = extractor.getSampleTime(); + extractor.seekTo((long) (time * 10E5)); } } @@ -216,7 +171,7 @@ public double getDuration() { public double getTime() { if (running) { - return extractor.getSampleTime() / 10E5; + return extractor.getTimeStamp() / 10E5; } else { return 0; } @@ -226,7 +181,7 @@ public boolean isRunning() { return running; } - protected abstract boolean extract(MediaExtractor extractor); + protected abstract boolean extract(Extractor extractor) throws IOException; protected abstract boolean decodeOutput(ByteBuffer outputBuffer, long timeStamp); @@ -238,7 +193,6 @@ private void decode() { startTs = System.nanoTime() / 1000; } long sleepTime = 0; - long accumulativeTs = 0; while (running) { synchronized (sync) { if (pause.get()) continue; @@ -255,6 +209,7 @@ private void decode() { int inIndex = codec.dequeueInputBuffer(10000); int sampleSize; long timeStamp = System.nanoTime() / 1000; + boolean finished = false; if (inIndex >= 0) { ByteBuffer input; if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { @@ -263,23 +218,16 @@ private void decode() { input = codec.getInputBuffers()[inIndex]; } if (input == null) continue; - sampleSize = extractor.readSampleData(input, 0); - + sampleSize = extractor.readFrame(input); long ts = System.nanoTime() / 1000 - startTs; - long extractorTs = extractor.getSampleTime(); - accumulativeTs += extractorTs - lastExtractorTs; - lastExtractorTs = extractor.getSampleTime(); - - if (accumulativeTs > ts) sleepTime = (accumulativeTs - ts) / 1000; - else sleepTime = 0; - - if (sampleSize < 0) { + sleepTime = extractor.getSleepTime(ts); + finished = !extractor.advance(); + if (finished) { if (!loopMode) { codec.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); } } else { codec.queueInputBuffer(inIndex, 0, sampleSize, ts + sleepTime, 0); - extractor.advance(); } } int outIndex = codec.dequeueOutputBuffer(bufferInfo, 10000); @@ -293,7 +241,6 @@ private void decode() { } boolean render = decodeOutput(output, timeStamp); codec.releaseOutputBuffer(outIndex, render && bufferInfo.size != 0); - boolean finished = extractor.getSampleTime() < 0; if (finished) { if (loopMode) { moveTo(0); @@ -317,4 +264,12 @@ private boolean sleep(long sleepTime) { return false; } } + + public void setExtractor(Extractor extractor) { + this.extractor = extractor; + } + + public Extractor getExtractor() { + return extractor; + } } diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/Extractor.kt b/encoder/src/main/java/com/pedro/encoder/input/decoder/Extractor.kt new file mode 100644 index 000000000..fe7f95ef1 --- /dev/null +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/Extractor.kt @@ -0,0 +1,58 @@ +/* + * + * * Copyright (C) 2024 pedroSG94. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.pedro.encoder.input.decoder + +import android.content.Context +import android.media.MediaFormat +import android.net.Uri +import com.pedro.common.frame.MediaFrame +import java.io.FileDescriptor +import java.nio.ByteBuffer + +/** + * Created by pedro on 18/10/24. + */ +interface Extractor { + + fun selectTrack(type: MediaFrame.Type): String + + fun initialize(path: String) + + fun initialize(context: Context, uri: Uri) + + fun initialize(fileDescriptor: FileDescriptor) + + fun readFrame(buffer: ByteBuffer): Int + + fun advance(): Boolean + + fun getTimeStamp(): Long + + fun getSleepTime(ts: Long): Long + + fun seekTo(time: Long) + + fun release() + + fun getVideoInfo(): VideoInfo + + fun getAudioInfo(): AudioInfo + + fun getFormat(): MediaFormat +} \ No newline at end of file diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoDecoder.java b/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoDecoder.java index be2eaf0e1..6a737b129 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoDecoder.java +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoDecoder.java @@ -22,6 +22,8 @@ import android.view.Surface; import com.pedro.common.ExtensionsKt; +import com.pedro.common.frame.MediaFrame; + import java.nio.ByteBuffer; /** @@ -41,29 +43,17 @@ public VideoDecoder(VideoDecoderInterface videoDecoderInterface, DecoderInterfac } @Override - protected boolean extract(MediaExtractor videoExtractor) { - for (int i = 0; i < videoExtractor.getTrackCount() && !mime.startsWith("video/"); i++) { - mediaFormat = videoExtractor.getTrackFormat(i); - mime = mediaFormat.getString(MediaFormat.KEY_MIME); - if (mime.startsWith("video/")) { - videoExtractor.selectTrack(i); - } else { - mediaFormat = null; - } - } - if (mediaFormat != null) { - final Integer width = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_WIDTH); - final Integer height = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_HEIGHT); - final Long duration = ExtensionsKt.getLongSafe(mediaFormat, MediaFormat.KEY_DURATION); - final Integer fps = ExtensionsKt.getIntegerSafe(mediaFormat, MediaFormat.KEY_FRAME_RATE); - if (width == null || height == null) return false; - this.width = width; - this.height = height; - this.duration = duration != null ? duration : -1; - this.fps = fps != null ? fps : 30; + protected boolean extract(Extractor extractor) { + try { + mime = extractor.selectTrack(MediaFrame.Type.VIDEO); + VideoInfo info = extractor.getVideoInfo(); + mediaFormat = extractor.getFormat(); + this.width = info.getWidth(); + this.height = info.getHeight(); + this.duration = info.getDuration(); + this.fps = info.getFps(); return true; - //video decoder not supported - } else { + } catch (Exception e) { mime = ""; return false; } diff --git a/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoInfo.kt b/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoInfo.kt new file mode 100644 index 000000000..4f893cdd8 --- /dev/null +++ b/encoder/src/main/java/com/pedro/encoder/input/decoder/VideoInfo.kt @@ -0,0 +1,29 @@ +/* + * + * * Copyright (C) 2024 pedroSG94. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.pedro.encoder.input.decoder + +/** + * Created by pedro on 18/10/24. + */ +data class VideoInfo( + val width: Int, + val height: Int, + val fps: Int, + val duration: Long +) diff --git a/encoder/src/main/java/com/pedro/encoder/input/sources/audio/AudioFileSource.kt b/encoder/src/main/java/com/pedro/encoder/input/sources/audio/AudioFileSource.kt index a2677c9c8..d14484b5f 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/sources/audio/AudioFileSource.kt +++ b/encoder/src/main/java/com/pedro/encoder/input/sources/audio/AudioFileSource.kt @@ -25,6 +25,7 @@ import com.pedro.encoder.Frame import com.pedro.encoder.input.audio.GetMicrophoneData import com.pedro.encoder.input.decoder.AudioDecoder import com.pedro.encoder.input.decoder.DecoderInterface +import com.pedro.encoder.input.decoder.Extractor import java.io.IOException /** @@ -62,7 +63,7 @@ class AudioFileSource( override fun create(sampleRate: Int, isStereo: Boolean, echoCanceler: Boolean, noiseSuppressor: Boolean): Boolean { //create extractor to confirm valid parameters - val result = audioDecoder.initExtractor(context, path, null) + val result = audioDecoder.initExtractor(context, path) if (!result) { throw IllegalArgumentException("Audio file track not found") } @@ -125,7 +126,8 @@ class AudioFileSource( val isStereo = audioDecoder.isStereo val wasRunning = audioDecoder.isRunning val audioDecoder = AudioDecoder(getMicrophoneData, audioDecoderInterface, decoderInterface) - if (!audioDecoder.initExtractor(context, uri, null)) throw IOException("Extraction failed") + audioDecoder.extractor = this.audioDecoder.extractor + if (!audioDecoder.initExtractor(context, uri)) throw IOException("Extraction failed") if (sampleRate != audioDecoder.sampleRate) throw IOException("SampleRate must be the same that the previous file") if (isStereo != audioDecoder.isStereo) throw IOException("Channels must be the same that the previous file") this.audioDecoder.stop() @@ -161,4 +163,8 @@ class AudioFileSource( } fun isAudioDeviceEnabled(): Boolean = audioTrackPlayer?.playState == AudioTrack.PLAYSTATE_PLAYING + + fun setExtractor(extractor: Extractor) { + audioDecoder.extractor = extractor + } } \ No newline at end of file diff --git a/encoder/src/main/java/com/pedro/encoder/input/sources/video/VideoFileSource.kt b/encoder/src/main/java/com/pedro/encoder/input/sources/video/VideoFileSource.kt index 5f0657aab..b7ede3bf7 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/sources/video/VideoFileSource.kt +++ b/encoder/src/main/java/com/pedro/encoder/input/sources/video/VideoFileSource.kt @@ -21,6 +21,7 @@ import android.graphics.SurfaceTexture import android.net.Uri import android.view.Surface import com.pedro.encoder.input.decoder.DecoderInterface +import com.pedro.encoder.input.decoder.Extractor import com.pedro.encoder.input.decoder.VideoDecoder import com.pedro.encoder.input.sources.OrientationForced import java.io.IOException @@ -51,7 +52,7 @@ class VideoFileSource( } override fun create(width: Int, height: Int, fps: Int, rotation: Int): Boolean { - val result = videoDecoder.initExtractor(context, path, null) + val result = videoDecoder.initExtractor(context, path) if (!result) { throw IllegalArgumentException("Video file track not found") } @@ -96,7 +97,8 @@ class VideoFileSource( val height = videoDecoder.height val wasRunning = videoDecoder.isRunning val videoDecoder = VideoDecoder(videoDecoderInterface, decoderInterface) - if (!videoDecoder.initExtractor(context, uri, null)) throw IOException("Extraction failed") + videoDecoder.extractor = this.videoDecoder.extractor + if (!videoDecoder.initExtractor(context, uri)) throw IOException("Extraction failed") if (width != videoDecoder.width || height != videoDecoder.height) throw IOException("Resolution must be the same that the previous file") this.videoDecoder.stop() this.videoDecoder = videoDecoder @@ -104,5 +106,9 @@ class VideoFileSource( videoDecoder.prepareVideo(Surface(surfaceTexture)) videoDecoder.start() } + + fun setExtractor(extractor: Extractor) { + videoDecoder.extractor = extractor + } } } \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f07d2d363..1d4892dcc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -18,7 +18,7 @@ annotation = "1.9.0" coroutines = "1.9.0" junit = "4.13.2" mockito = "5.4.0" -ktor = "2.3.12" +ktor = "3.0.0" uvcandroid = "1.0.7" [libraries] diff --git a/library/src/main/java/com/pedro/library/base/FromFileBase.java b/library/src/main/java/com/pedro/library/base/FromFileBase.java index 68abe3221..000220522 100644 --- a/library/src/main/java/com/pedro/library/base/FromFileBase.java +++ b/library/src/main/java/com/pedro/library/base/FromFileBase.java @@ -40,6 +40,7 @@ import com.pedro.encoder.input.decoder.AudioDecoderInterface; import com.pedro.encoder.input.decoder.BaseDecoder; import com.pedro.encoder.input.decoder.DecoderInterface; +import com.pedro.encoder.input.decoder.Extractor; import com.pedro.encoder.input.decoder.VideoDecoder; import com.pedro.encoder.input.decoder.VideoDecoderInterface; import com.pedro.encoder.utils.CodecUtil; @@ -179,7 +180,7 @@ public boolean prepareVideo(FileDescriptor fileDescriptor, int bitRate, int rota */ public boolean prepareVideo(Context context, Uri uri, int bitRate, int rotation, int profile, int level) throws IOException { - if (!videoDecoder.initExtractor(context, uri, null)) return false; + if (!videoDecoder.initExtractor(context, uri)) return false; return finishPrepareVideo(bitRate, rotation, profile, level); } @@ -248,7 +249,7 @@ public boolean prepareAudio(FileDescriptor fileDescriptor, int bitRate) throws I * @throws IOException Normally file not found. */ public boolean prepareAudio(Context context, Uri uri, int bitRate) throws IOException { - if (!audioDecoder.initExtractor(context, uri, null)) return false; + if (!audioDecoder.initExtractor(context, uri)) return false; return finishPrepareAudio(bitRate); } @@ -640,7 +641,7 @@ public void replaceAudioFile(String filePath) throws IOException { public void replaceAudioFile(Context context, Uri uri) throws IOException { resetAudioDecoder((BaseDecoder decoder) -> { - if (!decoder.initExtractor(context, uri, null)) throw new IOException("Extraction failed"); + if (!decoder.initExtractor(context, uri)) throw new IOException("Extraction failed"); }); } @@ -658,7 +659,7 @@ public void replaceVideoFile(String filePath) throws IOException { public void replaceVideoFile(Context context, Uri uri) throws IOException { resetVideoDecoder((BaseDecoder decoder) -> { - if (!decoder.initExtractor(context, uri, null)) throw new IOException("Extraction failed"); + if (!decoder.initExtractor(context, uri)) throw new IOException("Extraction failed"); }); } @@ -673,6 +674,7 @@ private void resetVideoDecoder(IORunnable runnable) throws IOException { int height = videoDecoder.getHeight(); boolean wasRunning = videoDecoder.isRunning(); VideoDecoder videoDecoder = new VideoDecoder(videoDecoderInterface, decoderInterface); + videoDecoder.setExtractor(this.videoDecoder.getExtractor()); runnable.run(videoDecoder); if (width != videoDecoder.getWidth() || height != videoDecoder.getHeight()) throw new IOException("Resolution must be the same that the previous file"); this.videoDecoder.stop(); @@ -686,6 +688,7 @@ private void resetAudioDecoder(IORunnable runnable) throws IOException { boolean isStereo = audioDecoder.isStereo(); boolean wasRunning = audioDecoder.isRunning(); AudioDecoder audioDecoder = new AudioDecoder(getMicrophoneData, audioDecoderInterface, decoderInterface); + audioDecoder.setExtractor(this.audioDecoder.getExtractor()); runnable.run(audioDecoder); if (sampleRate != audioDecoder.getSampleRate()) throw new IOException("SampleRate must be the same that the previous file"); if (isStereo != audioDecoder.isStereo()) throw new IOException("Channels must be the same that the previous file"); @@ -705,6 +708,14 @@ public void moveTo(double time) { if (audioEnabled) audioDecoder.moveTo(time); } + public void setVideoExtractor(Extractor extractor) { + videoDecoder.setExtractor(extractor); + } + + public void setAudioExtractor(Extractor extractor) { + audioDecoder.setExtractor(extractor); + } + protected abstract void onVideoInfoImp(ByteBuffer sps, ByteBuffer pps, ByteBuffer vps); protected abstract void getVideoDataImp(ByteBuffer videoBuffer, MediaCodec.BufferInfo info); diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt index 7b844ecea..750f3b9b1 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.kt @@ -219,13 +219,6 @@ class RtspClient(private val connectChecker: ConnectChecker) { val error = runCatching { commandsManager.setUrl(host, port, "/$path") - rtspSender.setSocketsInfo(commandsManager.protocol, - host, - commandsManager.videoClientPorts, - commandsManager.audioClientPorts, - commandsManager.videoServerPorts, - commandsManager.audioServerPorts - ) if (!commandsManager.audioDisabled) { rtspSender.setAudioInfo(commandsManager.sampleRate, commandsManager.isStereo) } @@ -333,6 +326,13 @@ class RtspClient(private val connectChecker: ConnectChecker) { } return@launch } + rtspSender.setSocketsInfo(commandsManager.protocol, + host, + commandsManager.videoClientPorts, + commandsManager.audioClientPorts, + commandsManager.videoServerPorts, + commandsManager.audioServerPorts + ) rtspSender.setSocket(socket) rtspSender.start() reTries = numRetry