diff --git a/encoder/src/main/java/com/pedro/encoder/BaseEncoder.java b/encoder/src/main/java/com/pedro/encoder/BaseEncoder.java index dcc9c904d..88a0dfd10 100644 --- a/encoder/src/main/java/com/pedro/encoder/BaseEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/BaseEncoder.java @@ -57,6 +57,7 @@ public abstract class BaseEncoder implements EncoderCallback { private EncoderErrorCallback encoderErrorCallback; protected String type; protected CodecUtil.CodecTypeError typeError; + protected TimestampMode timestampMode = TimestampMode.CLOCK; public void setEncoderErrorCallback(EncoderErrorCallback encoderErrorCallback) { this.encoderErrorCallback = encoderErrorCallback; @@ -70,6 +71,11 @@ public void setType(String type) { this.type = type; } + public void setTimestampMode(TimestampMode timestampMode) { + if (isRunning()) return; + this.timestampMode = timestampMode; + } + public void restart() { start(false); initCodec(); diff --git a/encoder/src/main/java/com/pedro/encoder/TimestampMode.kt b/encoder/src/main/java/com/pedro/encoder/TimestampMode.kt new file mode 100644 index 000000000..8ee08aacf --- /dev/null +++ b/encoder/src/main/java/com/pedro/encoder/TimestampMode.kt @@ -0,0 +1,5 @@ +package com.pedro.encoder + +enum class TimestampMode { + CLOCK, BUFFER +} \ No newline at end of file diff --git a/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java b/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java index 5ba3f8bbf..a89b24bcb 100644 --- a/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java @@ -26,6 +26,7 @@ import com.pedro.encoder.BaseEncoder; import com.pedro.encoder.Frame; import com.pedro.encoder.GetFrame; +import com.pedro.encoder.TimestampMode; import com.pedro.encoder.input.audio.GetMicrophoneData; import com.pedro.encoder.utils.CodecUtil; @@ -46,8 +47,7 @@ public class AudioEncoder extends BaseEncoder implements GetMicrophoneData { public static final int inputSize = 8192; private boolean isStereo = true; private GetFrame getFrame; - private long bytesRead = 0; - private boolean tsModeBuffer = false; + private float tsBuffer = 0; public AudioEncoder(GetAudioData getAudioData) { this.getAudioData = getAudioData; @@ -117,13 +117,13 @@ public boolean prepareAudioEncoder() { @Override public void start(boolean resetTs) { + if (resetTs) tsBuffer = 0; shouldReset = resetTs; Log.i(TAG, "started"); } @Override protected void stopImp() { - bytesRead = 0; Log.i(TAG, "stopped"); } @@ -144,12 +144,12 @@ protected Frame getInputFrame() throws InterruptedException { @Override protected long calculatePts(Frame frame, long presentTimeUs) { long pts; - if (tsModeBuffer) { - int channels = isStereo ? 2 : 1; - pts = 1000000 * bytesRead / 2 / channels / sampleRate; - bytesRead += frame.getSize(); - } else { + if (timestampMode == TimestampMode.CLOCK) { pts = Math.max(0, frame.getTimeStamp() - presentTimeUs); + } else { + int channels = isStereo ? 2 : 1; + tsBuffer += (long) ((frame.getSize() / (channels * 2f) / sampleRate) * 1_000_000f); + pts = (long) tsBuffer; } return pts; } @@ -199,16 +199,6 @@ public void setSampleRate(int sampleRate) { this.sampleRate = sampleRate; } - public boolean isTsModeBuffer() { - return tsModeBuffer; - } - - public void setTsModeBuffer(boolean tsModeBuffer) { - if (!isRunning()) { - this.tsModeBuffer = tsModeBuffer; - } - } - @Override public void formatChanged(@NonNull MediaCodec mediaCodec, @NonNull MediaFormat mediaFormat) { getAudioData.onAudioFormat(mediaFormat); diff --git a/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneMode.kt b/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneMode.kt deleted file mode 100644 index 28a66fb48..000000000 --- a/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneMode.kt +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2024 pedroSG94. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.pedro.encoder.input.audio - -enum class MicrophoneMode { - SYNC, ASYNC, BUFFER -} \ No newline at end of file diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java index 56a2dfaaf..9313970d7 100644 --- a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java @@ -34,6 +34,7 @@ import com.pedro.common.av1.ObuType; import com.pedro.encoder.BaseEncoder; import com.pedro.encoder.Frame; +import com.pedro.encoder.TimestampMode; import com.pedro.encoder.input.video.FpsLimiter; import com.pedro.encoder.input.video.GetCameraData; import com.pedro.encoder.utils.CodecUtil; @@ -65,6 +66,7 @@ public class VideoEncoder extends BaseEncoder implements GetCameraData { private int bitRate = 1200 * 1024; //in kbps private int rotation = 90; private int iFrameInterval = 2; + private long firstTimestamp = 0; //for disable video private final FpsLimiter fpsLimiter = new FpsLimiter(); private FormatVideoEncoder formatVideoEncoder = FormatVideoEncoder.YUV420Dynamical; @@ -176,6 +178,7 @@ public boolean prepareVideoEncoder(int width, int height, int fps, int bitRate, @Override public void start(boolean resetTs) { + if (resetTs) firstTimestamp = 0; forceKey = false; shouldReset = resetTs; spsPpsSetted = false; @@ -544,8 +547,13 @@ protected void checkBuffer(@NonNull ByteBuffer byteBuffer, Log.e(TAG, "manual av1 extraction failed"); } } - if (formatVideoEncoder == FormatVideoEncoder.SURFACE) { - bufferInfo.presentationTimeUs = System.nanoTime() / 1000 - presentTimeUs; + if (timestampMode == TimestampMode.CLOCK) { + if (formatVideoEncoder == FormatVideoEncoder.SURFACE) { + bufferInfo.presentationTimeUs = System.nanoTime() / 1000 - presentTimeUs; + } + } else { + if (firstTimestamp == 0) firstTimestamp = bufferInfo.presentationTimeUs; + bufferInfo.presentationTimeUs -= firstTimestamp; } } diff --git a/library/src/main/java/com/pedro/library/base/Camera1Base.java b/library/src/main/java/com/pedro/library/base/Camera1Base.java index 6129416c5..49ce8e92e 100644 --- a/library/src/main/java/com/pedro/library/base/Camera1Base.java +++ b/library/src/main/java/com/pedro/library/base/Camera1Base.java @@ -36,13 +36,12 @@ import com.pedro.common.AudioCodec; import com.pedro.common.VideoCodec; import com.pedro.encoder.EncoderErrorCallback; +import com.pedro.encoder.TimestampMode; import com.pedro.encoder.audio.AudioEncoder; import com.pedro.encoder.audio.GetAudioData; import com.pedro.encoder.input.audio.CustomAudioEffect; import com.pedro.encoder.input.audio.GetMicrophoneData; import com.pedro.encoder.input.audio.MicrophoneManager; -import com.pedro.encoder.input.audio.MicrophoneManagerManual; -import com.pedro.encoder.input.audio.MicrophoneMode; import com.pedro.encoder.input.video.Camera1ApiManager; import com.pedro.encoder.input.video.CameraCallbacks; import com.pedro.encoder.input.video.CameraHelper; @@ -126,36 +125,11 @@ public Camera1Base(Context context) { } private void init() { + microphoneManager = new MicrophoneManager(getMicrophoneData); videoEncoder = new VideoEncoder(getVideoData); - setMicrophoneMode(MicrophoneMode.ASYNC); - recordController = new AndroidMuxerRecordController(); - } - - /** - * Must be called before prepareAudio. - * - * @param microphoneMode mode to work accord to audioEncoder. By default ASYNC: - * SYNC using same thread. This mode could solve choppy audio or audio frame discarded. - * ASYNC using other thread. - */ - public void setMicrophoneMode(MicrophoneMode microphoneMode) { - switch (microphoneMode) { - case SYNC: - microphoneManager = new MicrophoneManagerManual(); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setGetFrame(((MicrophoneManagerManual) microphoneManager).getGetFrame()); - audioEncoder.setTsModeBuffer(false); - break; - case ASYNC: - microphoneManager = new MicrophoneManager(getMicrophoneData); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setTsModeBuffer(false); - break; - case BUFFER: - microphoneManager = new MicrophoneManager(getMicrophoneData); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setTsModeBuffer(true); - break; + audioEncoder = new AudioEncoder(getAudioData); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) { + recordController = new AndroidMuxerRecordController(); } } @@ -163,6 +137,15 @@ public void setCameraCallbacks(CameraCallbacks callbacks) { cameraManager.setCameraCallbacks(callbacks); } + /** + * Set the mode to calculate timestamp. By default CLOCK. + * Must be called before startRecord/startStream or it will be ignored. + */ + public void setTimestampMode(TimestampMode timestampModeVideo, TimestampMode timestampModeAudio) { + videoEncoder.setTimestampMode(timestampModeVideo); + audioEncoder.setTimestampMode(timestampModeAudio); + } + /** * Set a callback to know errors related with Video/Audio encoders * @param encoderErrorCallback callback to use, null to remove @@ -433,9 +416,7 @@ public void stopRecord() { @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) public void replaceView(Context context) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) { - replaceGlInterface(new GlStreamInterface(context)); - } + replaceGlInterface(new GlStreamInterface(context)); } @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) diff --git a/library/src/main/java/com/pedro/library/base/Camera2Base.java b/library/src/main/java/com/pedro/library/base/Camera2Base.java index e143e8130..2dc286479 100644 --- a/library/src/main/java/com/pedro/library/base/Camera2Base.java +++ b/library/src/main/java/com/pedro/library/base/Camera2Base.java @@ -35,13 +35,12 @@ import com.pedro.common.AudioCodec; import com.pedro.common.VideoCodec; import com.pedro.encoder.EncoderErrorCallback; +import com.pedro.encoder.TimestampMode; import com.pedro.encoder.audio.AudioEncoder; import com.pedro.encoder.audio.GetAudioData; import com.pedro.encoder.input.audio.CustomAudioEffect; import com.pedro.encoder.input.audio.GetMicrophoneData; import com.pedro.encoder.input.audio.MicrophoneManager; -import com.pedro.encoder.input.audio.MicrophoneManagerManual; -import com.pedro.encoder.input.audio.MicrophoneMode; import com.pedro.encoder.input.video.Camera2ApiManager; import com.pedro.encoder.input.video.CameraCallbacks; import com.pedro.encoder.input.video.CameraHelper; @@ -110,43 +109,25 @@ public Camera2Base(Context context) { private void init(Context context) { cameraManager = new Camera2ApiManager(context); + microphoneManager = new MicrophoneManager(getMicrophoneData); videoEncoder = new VideoEncoder(getVideoData); - setMicrophoneMode(MicrophoneMode.ASYNC); + audioEncoder = new AudioEncoder(getAudioData); recordController = new AndroidMuxerRecordController(); } - /** - * Must be called before prepareAudio. - * - * @param microphoneMode mode to work accord to audioEncoder. By default ASYNC: - * SYNC using same thread. This mode could solve choppy audio or AudioEncoder frame discarded. - * ASYNC using other thread. - */ - public void setMicrophoneMode(MicrophoneMode microphoneMode) { - switch (microphoneMode) { - case SYNC: - microphoneManager = new MicrophoneManagerManual(); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setGetFrame(((MicrophoneManagerManual) microphoneManager).getGetFrame()); - audioEncoder.setTsModeBuffer(false); - break; - case ASYNC: - microphoneManager = new MicrophoneManager(getMicrophoneData); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setTsModeBuffer(false); - break; - case BUFFER: - microphoneManager = new MicrophoneManager(getMicrophoneData); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setTsModeBuffer(true); - break; - } - } - public void setCameraCallbacks(CameraCallbacks callbacks) { cameraManager.setCameraCallbacks(callbacks); } + /** + * Set the mode to calculate timestamp. By default CLOCK. + * Must be called before startRecord/startStream or it will be ignored. + */ + public void setTimestampMode(TimestampMode timestampModeVideo, TimestampMode timestampModeAudio) { + videoEncoder.setTimestampMode(timestampModeVideo); + audioEncoder.setTimestampMode(timestampModeAudio); + } + /** * Set a callback to know errors related with Video/Audio encoders * @param encoderErrorCallback callback to use, null to remove diff --git a/library/src/main/java/com/pedro/library/base/DisplayBase.java b/library/src/main/java/com/pedro/library/base/DisplayBase.java index f558a8ea9..23413418e 100644 --- a/library/src/main/java/com/pedro/library/base/DisplayBase.java +++ b/library/src/main/java/com/pedro/library/base/DisplayBase.java @@ -30,7 +30,6 @@ import android.media.projection.MediaProjectionManager; import android.os.Build; import android.view.Surface; -import android.view.SurfaceView; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -39,13 +38,12 @@ import com.pedro.common.AudioCodec; import com.pedro.common.VideoCodec; import com.pedro.encoder.EncoderErrorCallback; +import com.pedro.encoder.TimestampMode; import com.pedro.encoder.audio.AudioEncoder; import com.pedro.encoder.audio.GetAudioData; import com.pedro.encoder.input.audio.CustomAudioEffect; import com.pedro.encoder.input.audio.GetMicrophoneData; import com.pedro.encoder.input.audio.MicrophoneManager; -import com.pedro.encoder.input.audio.MicrophoneManagerManual; -import com.pedro.encoder.input.audio.MicrophoneMode; import com.pedro.encoder.utils.CodecUtil; import com.pedro.encoder.video.FormatVideoEncoder; import com.pedro.encoder.video.GetVideoData; @@ -78,10 +76,9 @@ public abstract class DisplayBase { private MediaProjection mediaProjection; private final MediaProjectionManager mediaProjectionManager; protected VideoEncoder videoEncoder; - private MicrophoneManager microphoneManager; + private final MicrophoneManager microphoneManager; private AudioEncoder audioEncoder; private boolean streaming = false; - protected SurfaceView surfaceView; private VirtualDisplay virtualDisplay; private int dpi = 320; private int resultCode = -1; @@ -98,33 +95,19 @@ public DisplayBase(Context context, boolean useOpengl) { } mediaProjectionManager = ((MediaProjectionManager) context.getSystemService(MEDIA_PROJECTION_SERVICE)); - this.surfaceView = null; + microphoneManager = new MicrophoneManager(getMicrophoneData); videoEncoder = new VideoEncoder(getVideoData); audioEncoder = new AudioEncoder(getAudioData); - //Necessary use same thread to read input buffer and encode it with internal audio or audio is choppy. - setMicrophoneMode(MicrophoneMode.SYNC); recordController = new AndroidMuxerRecordController(); } /** - * Must be called before prepareAudio. - * - * @param microphoneMode mode to work accord to audioEncoder. By default SYNC: - * SYNC using same thread. This mode could solve choppy audio or audio frame discarded. - * ASYNC using other thread. + * Set the mode to calculate timestamp. By default CLOCK. + * Must be called before startRecord/startStream or it will be ignored. */ - public void setMicrophoneMode(MicrophoneMode microphoneMode) { - switch (microphoneMode) { - case SYNC: - microphoneManager = new MicrophoneManagerManual(); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setGetFrame(((MicrophoneManagerManual) microphoneManager).getGetFrame()); - break; - case ASYNC: - microphoneManager = new MicrophoneManager(getMicrophoneData); - audioEncoder = new AudioEncoder(getAudioData); - break; - } + public void setTimestampMode(TimestampMode timestampModeVideo, TimestampMode timestampModeAudio) { + videoEncoder.setTimestampMode(timestampModeVideo); + audioEncoder.setTimestampMode(timestampModeAudio); } /** diff --git a/library/src/main/java/com/pedro/library/base/FromFileBase.java b/library/src/main/java/com/pedro/library/base/FromFileBase.java index 171088335..8ae321999 100644 --- a/library/src/main/java/com/pedro/library/base/FromFileBase.java +++ b/library/src/main/java/com/pedro/library/base/FromFileBase.java @@ -32,6 +32,7 @@ import com.pedro.common.AudioCodec; import com.pedro.common.VideoCodec; import com.pedro.encoder.EncoderErrorCallback; +import com.pedro.encoder.TimestampMode; import com.pedro.encoder.audio.AudioEncoder; import com.pedro.encoder.audio.GetAudioData; import com.pedro.encoder.input.audio.GetMicrophoneData; @@ -121,6 +122,15 @@ private void init(VideoDecoderInterface videoDecoderInterface, recordController = new AndroidMuxerRecordController(); } + /** + * Set the mode to calculate timestamp. By default CLOCK. + * Must be called before startRecord/startStream or it will be ignored. + */ + public void setTimestampMode(TimestampMode timestampModeVideo, TimestampMode timestampModeAudio) { + videoEncoder.setTimestampMode(timestampModeVideo); + audioEncoder.setTimestampMode(timestampModeAudio); + } + /** * @param callback get fps while record or stream */ diff --git a/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java b/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java index 1783fcead..8524bd018 100644 --- a/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java +++ b/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java @@ -27,13 +27,12 @@ import com.pedro.common.AudioCodec; import com.pedro.encoder.EncoderErrorCallback; +import com.pedro.encoder.TimestampMode; import com.pedro.encoder.audio.AudioEncoder; import com.pedro.encoder.audio.GetAudioData; import com.pedro.encoder.input.audio.CustomAudioEffect; import com.pedro.encoder.input.audio.GetMicrophoneData; import com.pedro.encoder.input.audio.MicrophoneManager; -import com.pedro.encoder.input.audio.MicrophoneManagerManual; -import com.pedro.encoder.input.audio.MicrophoneMode; import com.pedro.encoder.utils.CodecUtil; import com.pedro.library.base.recording.BaseRecordController; import com.pedro.library.base.recording.RecordController; @@ -52,41 +51,22 @@ public abstract class OnlyAudioBase { protected BaseRecordController recordController; - private MicrophoneManager microphoneManager; + private final MicrophoneManager microphoneManager; private AudioEncoder audioEncoder; private boolean streaming = false; public OnlyAudioBase() { - setMicrophoneMode(MicrophoneMode.ASYNC); + microphoneManager = new MicrophoneManager(getMicrophoneData); + audioEncoder = new AudioEncoder(getAudioData); recordController = new AacMuxerRecordController(); } /** - * Must be called before prepareAudio. - * - * @param microphoneMode mode to work accord to audioEncoder. By default ASYNC: - * SYNC using same thread. This mode could solve choppy audio or audio frame discarded. - * ASYNC using other thread. + * Set the mode to calculate timestamp. By default CLOCK. + * Must be called before startRecord/startStream or it will be ignored. */ - public void setMicrophoneMode(MicrophoneMode microphoneMode) { - switch (microphoneMode) { - case SYNC: - microphoneManager = new MicrophoneManagerManual(); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setGetFrame(((MicrophoneManagerManual) microphoneManager).getGetFrame()); - audioEncoder.setTsModeBuffer(false); - break; - case ASYNC: - microphoneManager = new MicrophoneManager(getMicrophoneData); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setTsModeBuffer(false); - break; - case BUFFER: - microphoneManager = new MicrophoneManager(getMicrophoneData); - audioEncoder = new AudioEncoder(getAudioData); - audioEncoder.setTsModeBuffer(true); - break; - } + public void setTimestampMode(TimestampMode timestampModeAudio) { + audioEncoder.setTimestampMode(timestampModeAudio); } /** diff --git a/library/src/main/java/com/pedro/library/base/StreamBase.kt b/library/src/main/java/com/pedro/library/base/StreamBase.kt index 57d7303b1..67e550ba9 100644 --- a/library/src/main/java/com/pedro/library/base/StreamBase.kt +++ b/library/src/main/java/com/pedro/library/base/StreamBase.kt @@ -30,9 +30,13 @@ import com.pedro.common.AudioCodec import com.pedro.common.VideoCodec import com.pedro.encoder.EncoderErrorCallback import com.pedro.encoder.Frame +import com.pedro.encoder.TimestampMode import com.pedro.encoder.audio.AudioEncoder import com.pedro.encoder.audio.GetAudioData import com.pedro.encoder.input.audio.GetMicrophoneData +import com.pedro.encoder.input.sources.audio.AudioSource +import com.pedro.encoder.input.sources.video.NoVideoSource +import com.pedro.encoder.input.sources.video.VideoSource import com.pedro.encoder.utils.CodecUtil import com.pedro.encoder.video.FormatVideoEncoder import com.pedro.encoder.video.GetVideoData @@ -41,9 +45,6 @@ import com.pedro.library.base.recording.BaseRecordController import com.pedro.library.base.recording.RecordController import com.pedro.library.util.AndroidMuxerRecordController import com.pedro.library.util.FpsListener -import com.pedro.encoder.input.sources.audio.AudioSource -import com.pedro.encoder.input.sources.video.NoVideoSource -import com.pedro.encoder.input.sources.video.VideoSource import com.pedro.library.util.streamclient.StreamBaseClient import com.pedro.library.view.GlStreamInterface import java.nio.ByteBuffer @@ -335,6 +336,15 @@ abstract class StreamBase( audioSource = source } + /** + * Set the mode to calculate timestamp. By default CLOCK. + * Must be called before startRecord/startStream or it will be ignored. + */ + fun setTimestampMode(timestampModeVideo: TimestampMode, timestampModeAudio: TimestampMode) { + videoEncoder.setTimestampMode(timestampModeVideo) + audioEncoder.setTimestampMode(timestampModeAudio) + } + /** * Set a callback to know errors related with Video/Audio encoders * @param encoderErrorCallback callback to use, null to remove