Skip to content

Commit

Permalink
Merge pull request #1591 from pedroSG94/feature/ts-mode-buffer
Browse files Browse the repository at this point in the history
allow clock or buffer timestamp calculation
  • Loading branch information
pedroSG94 authored Sep 29, 2024
2 parents 6959ac1 + 5cb9677 commit 52c31f9
Show file tree
Hide file tree
Showing 11 changed files with 95 additions and 161 deletions.
6 changes: 6 additions & 0 deletions encoder/src/main/java/com/pedro/encoder/BaseEncoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public abstract class BaseEncoder implements EncoderCallback {
private EncoderErrorCallback encoderErrorCallback;
protected String type;
protected CodecUtil.CodecTypeError typeError;
protected TimestampMode timestampMode = TimestampMode.CLOCK;

public void setEncoderErrorCallback(EncoderErrorCallback encoderErrorCallback) {
this.encoderErrorCallback = encoderErrorCallback;
Expand All @@ -70,6 +71,11 @@ public void setType(String type) {
this.type = type;
}

public void setTimestampMode(TimestampMode timestampMode) {
if (isRunning()) return;
this.timestampMode = timestampMode;
}

public void restart() {
start(false);
initCodec();
Expand Down
5 changes: 5 additions & 0 deletions encoder/src/main/java/com/pedro/encoder/TimestampMode.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package com.pedro.encoder

enum class TimestampMode {
CLOCK, BUFFER
}
26 changes: 8 additions & 18 deletions encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.pedro.encoder.BaseEncoder;
import com.pedro.encoder.Frame;
import com.pedro.encoder.GetFrame;
import com.pedro.encoder.TimestampMode;
import com.pedro.encoder.input.audio.GetMicrophoneData;
import com.pedro.encoder.utils.CodecUtil;

Expand All @@ -46,8 +47,7 @@ public class AudioEncoder extends BaseEncoder implements GetMicrophoneData {
public static final int inputSize = 8192;
private boolean isStereo = true;
private GetFrame getFrame;
private long bytesRead = 0;
private boolean tsModeBuffer = false;
private float tsBuffer = 0;

public AudioEncoder(GetAudioData getAudioData) {
this.getAudioData = getAudioData;
Expand Down Expand Up @@ -117,13 +117,13 @@ public boolean prepareAudioEncoder() {

@Override
public void start(boolean resetTs) {
if (resetTs) tsBuffer = 0;
shouldReset = resetTs;
Log.i(TAG, "started");
}

@Override
protected void stopImp() {
bytesRead = 0;
Log.i(TAG, "stopped");
}

Expand All @@ -144,12 +144,12 @@ protected Frame getInputFrame() throws InterruptedException {
@Override
protected long calculatePts(Frame frame, long presentTimeUs) {
long pts;
if (tsModeBuffer) {
int channels = isStereo ? 2 : 1;
pts = 1000000 * bytesRead / 2 / channels / sampleRate;
bytesRead += frame.getSize();
} else {
if (timestampMode == TimestampMode.CLOCK) {
pts = Math.max(0, frame.getTimeStamp() - presentTimeUs);
} else {
int channels = isStereo ? 2 : 1;
tsBuffer += (long) ((frame.getSize() / (channels * 2f) / sampleRate) * 1_000_000f);
pts = (long) tsBuffer;
}
return pts;
}
Expand Down Expand Up @@ -199,16 +199,6 @@ public void setSampleRate(int sampleRate) {
this.sampleRate = sampleRate;
}

public boolean isTsModeBuffer() {
return tsModeBuffer;
}

public void setTsModeBuffer(boolean tsModeBuffer) {
if (!isRunning()) {
this.tsModeBuffer = tsModeBuffer;
}
}

@Override
public void formatChanged(@NonNull MediaCodec mediaCodec, @NonNull MediaFormat mediaFormat) {
getAudioData.onAudioFormat(mediaFormat);
Expand Down

This file was deleted.

12 changes: 10 additions & 2 deletions encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.pedro.common.av1.ObuType;
import com.pedro.encoder.BaseEncoder;
import com.pedro.encoder.Frame;
import com.pedro.encoder.TimestampMode;
import com.pedro.encoder.input.video.FpsLimiter;
import com.pedro.encoder.input.video.GetCameraData;
import com.pedro.encoder.utils.CodecUtil;
Expand Down Expand Up @@ -65,6 +66,7 @@ public class VideoEncoder extends BaseEncoder implements GetCameraData {
private int bitRate = 1200 * 1024; //in kbps
private int rotation = 90;
private int iFrameInterval = 2;
private long firstTimestamp = 0;
//for disable video
private final FpsLimiter fpsLimiter = new FpsLimiter();
private FormatVideoEncoder formatVideoEncoder = FormatVideoEncoder.YUV420Dynamical;
Expand Down Expand Up @@ -176,6 +178,7 @@ public boolean prepareVideoEncoder(int width, int height, int fps, int bitRate,

@Override
public void start(boolean resetTs) {
if (resetTs) firstTimestamp = 0;
forceKey = false;
shouldReset = resetTs;
spsPpsSetted = false;
Expand Down Expand Up @@ -544,8 +547,13 @@ protected void checkBuffer(@NonNull ByteBuffer byteBuffer,
Log.e(TAG, "manual av1 extraction failed");
}
}
if (formatVideoEncoder == FormatVideoEncoder.SURFACE) {
bufferInfo.presentationTimeUs = System.nanoTime() / 1000 - presentTimeUs;
if (timestampMode == TimestampMode.CLOCK) {
if (formatVideoEncoder == FormatVideoEncoder.SURFACE) {
bufferInfo.presentationTimeUs = System.nanoTime() / 1000 - presentTimeUs;
}
} else {
if (firstTimestamp == 0) firstTimestamp = bufferInfo.presentationTimeUs;
bufferInfo.presentationTimeUs -= firstTimestamp;
}
}

Expand Down
49 changes: 15 additions & 34 deletions library/src/main/java/com/pedro/library/base/Camera1Base.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,12 @@
import com.pedro.common.AudioCodec;
import com.pedro.common.VideoCodec;
import com.pedro.encoder.EncoderErrorCallback;
import com.pedro.encoder.TimestampMode;
import com.pedro.encoder.audio.AudioEncoder;
import com.pedro.encoder.audio.GetAudioData;
import com.pedro.encoder.input.audio.CustomAudioEffect;
import com.pedro.encoder.input.audio.GetMicrophoneData;
import com.pedro.encoder.input.audio.MicrophoneManager;
import com.pedro.encoder.input.audio.MicrophoneManagerManual;
import com.pedro.encoder.input.audio.MicrophoneMode;
import com.pedro.encoder.input.video.Camera1ApiManager;
import com.pedro.encoder.input.video.CameraCallbacks;
import com.pedro.encoder.input.video.CameraHelper;
Expand Down Expand Up @@ -126,43 +125,27 @@ public Camera1Base(Context context) {
}

private void init() {
microphoneManager = new MicrophoneManager(getMicrophoneData);
videoEncoder = new VideoEncoder(getVideoData);
setMicrophoneMode(MicrophoneMode.ASYNC);
recordController = new AndroidMuxerRecordController();
}

/**
* Must be called before prepareAudio.
*
* @param microphoneMode mode to work accord to audioEncoder. By default ASYNC:
* SYNC using same thread. This mode could solve choppy audio or audio frame discarded.
* ASYNC using other thread.
*/
public void setMicrophoneMode(MicrophoneMode microphoneMode) {
switch (microphoneMode) {
case SYNC:
microphoneManager = new MicrophoneManagerManual();
audioEncoder = new AudioEncoder(getAudioData);
audioEncoder.setGetFrame(((MicrophoneManagerManual) microphoneManager).getGetFrame());
audioEncoder.setTsModeBuffer(false);
break;
case ASYNC:
microphoneManager = new MicrophoneManager(getMicrophoneData);
audioEncoder = new AudioEncoder(getAudioData);
audioEncoder.setTsModeBuffer(false);
break;
case BUFFER:
microphoneManager = new MicrophoneManager(getMicrophoneData);
audioEncoder = new AudioEncoder(getAudioData);
audioEncoder.setTsModeBuffer(true);
break;
audioEncoder = new AudioEncoder(getAudioData);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
recordController = new AndroidMuxerRecordController();
}
}

public void setCameraCallbacks(CameraCallbacks callbacks) {
cameraManager.setCameraCallbacks(callbacks);
}

/**
* Set the mode to calculate timestamp. By default CLOCK.
* Must be called before startRecord/startStream or it will be ignored.
*/
public void setTimestampMode(TimestampMode timestampModeVideo, TimestampMode timestampModeAudio) {
videoEncoder.setTimestampMode(timestampModeVideo);
audioEncoder.setTimestampMode(timestampModeAudio);
}

/**
* Set a callback to know errors related with Video/Audio encoders
* @param encoderErrorCallback callback to use, null to remove
Expand Down Expand Up @@ -433,9 +416,7 @@ public void stopRecord() {

@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public void replaceView(Context context) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
replaceGlInterface(new GlStreamInterface(context));
}
replaceGlInterface(new GlStreamInterface(context));
}

@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
Expand Down
43 changes: 12 additions & 31 deletions library/src/main/java/com/pedro/library/base/Camera2Base.java
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,12 @@
import com.pedro.common.AudioCodec;
import com.pedro.common.VideoCodec;
import com.pedro.encoder.EncoderErrorCallback;
import com.pedro.encoder.TimestampMode;
import com.pedro.encoder.audio.AudioEncoder;
import com.pedro.encoder.audio.GetAudioData;
import com.pedro.encoder.input.audio.CustomAudioEffect;
import com.pedro.encoder.input.audio.GetMicrophoneData;
import com.pedro.encoder.input.audio.MicrophoneManager;
import com.pedro.encoder.input.audio.MicrophoneManagerManual;
import com.pedro.encoder.input.audio.MicrophoneMode;
import com.pedro.encoder.input.video.Camera2ApiManager;
import com.pedro.encoder.input.video.CameraCallbacks;
import com.pedro.encoder.input.video.CameraHelper;
Expand Down Expand Up @@ -110,43 +109,25 @@ public Camera2Base(Context context) {

private void init(Context context) {
cameraManager = new Camera2ApiManager(context);
microphoneManager = new MicrophoneManager(getMicrophoneData);
videoEncoder = new VideoEncoder(getVideoData);
setMicrophoneMode(MicrophoneMode.ASYNC);
audioEncoder = new AudioEncoder(getAudioData);
recordController = new AndroidMuxerRecordController();
}

/**
* Must be called before prepareAudio.
*
* @param microphoneMode mode to work accord to audioEncoder. By default ASYNC:
* SYNC using same thread. This mode could solve choppy audio or AudioEncoder frame discarded.
* ASYNC using other thread.
*/
public void setMicrophoneMode(MicrophoneMode microphoneMode) {
switch (microphoneMode) {
case SYNC:
microphoneManager = new MicrophoneManagerManual();
audioEncoder = new AudioEncoder(getAudioData);
audioEncoder.setGetFrame(((MicrophoneManagerManual) microphoneManager).getGetFrame());
audioEncoder.setTsModeBuffer(false);
break;
case ASYNC:
microphoneManager = new MicrophoneManager(getMicrophoneData);
audioEncoder = new AudioEncoder(getAudioData);
audioEncoder.setTsModeBuffer(false);
break;
case BUFFER:
microphoneManager = new MicrophoneManager(getMicrophoneData);
audioEncoder = new AudioEncoder(getAudioData);
audioEncoder.setTsModeBuffer(true);
break;
}
}

public void setCameraCallbacks(CameraCallbacks callbacks) {
cameraManager.setCameraCallbacks(callbacks);
}

/**
* Set the mode to calculate timestamp. By default CLOCK.
* Must be called before startRecord/startStream or it will be ignored.
*/
public void setTimestampMode(TimestampMode timestampModeVideo, TimestampMode timestampModeAudio) {
videoEncoder.setTimestampMode(timestampModeVideo);
audioEncoder.setTimestampMode(timestampModeAudio);
}

/**
* Set a callback to know errors related with Video/Audio encoders
* @param encoderErrorCallback callback to use, null to remove
Expand Down
33 changes: 8 additions & 25 deletions library/src/main/java/com/pedro/library/base/DisplayBase.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import android.media.projection.MediaProjectionManager;
import android.os.Build;
import android.view.Surface;
import android.view.SurfaceView;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
Expand All @@ -39,13 +38,12 @@
import com.pedro.common.AudioCodec;
import com.pedro.common.VideoCodec;
import com.pedro.encoder.EncoderErrorCallback;
import com.pedro.encoder.TimestampMode;
import com.pedro.encoder.audio.AudioEncoder;
import com.pedro.encoder.audio.GetAudioData;
import com.pedro.encoder.input.audio.CustomAudioEffect;
import com.pedro.encoder.input.audio.GetMicrophoneData;
import com.pedro.encoder.input.audio.MicrophoneManager;
import com.pedro.encoder.input.audio.MicrophoneManagerManual;
import com.pedro.encoder.input.audio.MicrophoneMode;
import com.pedro.encoder.utils.CodecUtil;
import com.pedro.encoder.video.FormatVideoEncoder;
import com.pedro.encoder.video.GetVideoData;
Expand Down Expand Up @@ -78,10 +76,9 @@ public abstract class DisplayBase {
private MediaProjection mediaProjection;
private final MediaProjectionManager mediaProjectionManager;
protected VideoEncoder videoEncoder;
private MicrophoneManager microphoneManager;
private final MicrophoneManager microphoneManager;
private AudioEncoder audioEncoder;
private boolean streaming = false;
protected SurfaceView surfaceView;
private VirtualDisplay virtualDisplay;
private int dpi = 320;
private int resultCode = -1;
Expand All @@ -98,33 +95,19 @@ public DisplayBase(Context context, boolean useOpengl) {
}
mediaProjectionManager =
((MediaProjectionManager) context.getSystemService(MEDIA_PROJECTION_SERVICE));
this.surfaceView = null;
microphoneManager = new MicrophoneManager(getMicrophoneData);
videoEncoder = new VideoEncoder(getVideoData);
audioEncoder = new AudioEncoder(getAudioData);
//Necessary use same thread to read input buffer and encode it with internal audio or audio is choppy.
setMicrophoneMode(MicrophoneMode.SYNC);
recordController = new AndroidMuxerRecordController();
}

/**
* Must be called before prepareAudio.
*
* @param microphoneMode mode to work accord to audioEncoder. By default SYNC:
* SYNC using same thread. This mode could solve choppy audio or audio frame discarded.
* ASYNC using other thread.
* Set the mode to calculate timestamp. By default CLOCK.
* Must be called before startRecord/startStream or it will be ignored.
*/
public void setMicrophoneMode(MicrophoneMode microphoneMode) {
switch (microphoneMode) {
case SYNC:
microphoneManager = new MicrophoneManagerManual();
audioEncoder = new AudioEncoder(getAudioData);
audioEncoder.setGetFrame(((MicrophoneManagerManual) microphoneManager).getGetFrame());
break;
case ASYNC:
microphoneManager = new MicrophoneManager(getMicrophoneData);
audioEncoder = new AudioEncoder(getAudioData);
break;
}
public void setTimestampMode(TimestampMode timestampModeVideo, TimestampMode timestampModeAudio) {
videoEncoder.setTimestampMode(timestampModeVideo);
audioEncoder.setTimestampMode(timestampModeAudio);
}

/**
Expand Down
Loading

0 comments on commit 52c31f9

Please sign in to comment.