Skip to content

Commit

Permalink
* Fix FFmpegFrameRecorder dropped frame issues with audio samples …
Browse files Browse the repository at this point in the history
…(pull #2307)
  • Loading branch information
joshpkware authored Dec 15, 2024
1 parent 2b64c33 commit dedec82
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 13 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@

* Fix `FFmpegFrameRecorder` dropped frame issues with audio samples ([pull #2307](https://github.com/bytedeco/javacv/pull/2307))
* Add `FrameFilter.videoFilterArgs/audioFilterArgs` properties to support multiple different inputs ([pull #2304](https://github.com/bytedeco/javacv/pull/2304))
* Ensure `FFmpegFrameGrabber.start()` skips over streams with no codecs ([issue #2299](https://github.com/bytedeco/javacv/issues/2299))
* Add `FFmpegLogCallback.logRejectedOptions()` for debugging purposes ([pull #2301](https://github.com/bytedeco/javacv/pull/2301))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,11 @@ public void testFFmpegFrameFilterMultipleInputs() {
}
if (frame3.samples != null) {
d++;
assertEquals(2, frame3.audioChannels);
assertEquals(4, frame3.audioChannels);
assertEquals(1, frame3.samples.length);
assertTrue(frame3.samples[0] instanceof ByteBuffer);
assertTrue(frame3.samples[0] instanceof ShortBuffer);
assertEquals(frame2.samples.length, frame3.samples.length);
assertEquals(frame2.samples[0].limit(), frame3.samples[0].limit());
assertEquals(2 * frame2.samples[0].limit(), frame3.samples[0].limit());
}
}
}
Expand Down
33 changes: 23 additions & 10 deletions src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@
import org.bytedeco.javacpp.ShortPointer;

import org.bytedeco.ffmpeg.avcodec.*;
import org.bytedeco.ffmpeg.avdevice.*;
import org.bytedeco.ffmpeg.avformat.*;
import org.bytedeco.ffmpeg.avutil.*;
import org.bytedeco.ffmpeg.swresample.*;
Expand Down Expand Up @@ -397,6 +396,7 @@ static class SeekCallback extends Seek_Pointer_long_int {
private PointerPointer plane_ptr, plane_ptr2;
private AVPacket video_pkt, audio_pkt;
private int[] got_video_packet, got_audio_packet;
private boolean wrote_samples = false;
private AVFormatContext ifmt_ctx;
private IntPointer display_matrix;
private AVChannelLayout default_layout;
Expand Down Expand Up @@ -468,6 +468,7 @@ public synchronized void startUnsafe() throws Exception {
plane_ptr2 = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS).retainReference();
video_pkt = new AVPacket().retainReference();
audio_pkt = new AVPacket().retainReference();
wrote_samples = false;
got_video_packet = new int[1];
got_audio_packet = new int[1];
default_layout = new AVChannelLayout().retainReference();
Expand Down Expand Up @@ -1169,11 +1170,9 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf
throw new Exception("start() was not called successfully!");
}

if (samples == null && samples_out[0].position() > 0) {
// Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2
double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size);
writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor));
return writeFrame((AVFrame)null);
if (samples == null && samples_convert_ctx == null) {
// We haven't tried to record any samples yet so we don't need to flush.
return false;
}

int ret;
Expand Down Expand Up @@ -1265,7 +1264,8 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf
throw new Exception("Audio samples Buffer has unsupported type: " + samples);
}

if (samples_convert_ctx == null || samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate) {
boolean formatChanged = samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate;
if (samples != null && (samples_convert_ctx == null || formatChanged)) {
if (samples_convert_ctx == null) {
samples_convert_ctx = new SwrContext().retainReference();
}
Expand Down Expand Up @@ -1294,9 +1294,14 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf
for (int i = 0; i < samples_out.length; i++) {
plane_ptr2.put(i, samples_out[i]);
}
if (samples == null && inputCount == 0 && plane_ptr != null) {
plane_ptr.releaseReference();
// needs to be null to flush swr context.
plane_ptr = null;
}
if ((ret = swr_convert(samples_convert_ctx, plane_ptr2, outputCount, plane_ptr, inputCount)) < 0) {
throw new Exception("swr_convert() error " + ret + ": Cannot convert audio samples.");
} else if (ret == 0) {
} else if (ret == 0 && inputCount == 0) {
break;
}
for (int i = 0; samples != null && i < samples.length; i++) {
Expand All @@ -1310,6 +1315,14 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf
writeSamples(audio_input_frame_size);
}
}

if (samples == null && samples_out[0].position() > 0) {
// Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2
double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size);
writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor));
return writeFrame((AVFrame)null);
}

return samples != null ? frame.key_frame() != 0 : writeFrame((AVFrame)null);

}
Expand Down Expand Up @@ -1338,6 +1351,7 @@ private void writeSamples(int nb_samples) throws Exception {
frame.format(audio_c.sample_fmt());
frame.quality(audio_c.global_quality());
writeFrame(frame);
wrote_samples = true;
}

private boolean writeFrame(AVFrame frame) throws Exception {
Expand Down Expand Up @@ -1376,8 +1390,7 @@ private boolean writeFrame(AVFrame frame) throws Exception {
/* write the compressed frame in the media file */
writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt);

if (frame == null) {
// avoid infinite loop with buggy codecs on flush
if (frame == null && !wrote_samples) {
break;
}
}
Expand Down

0 comments on commit dedec82

Please sign in to comment.