[FFmpeg-devel] [PATCH v2 1/2] libavdevice/avfoundation: add buffer fifo and output packets in order they arrive
mindmark at gmail.com
mindmark at gmail.com
Mon Apr 12 05:31:41 EEST 2021
From: Mark Reid <mindmark at gmail.com>
This fixes audio issues I've had with some capture devices. The audio
gets really choppy and stops working. This seems to be because avf_read_packet
stops outputting the audio frames because a video frame happens to be available first.
It base on the approach used in a patch from #4437
https://trac.ffmpeg.org/ticket/4437
My approach uses an AVFifoBuffer instead of NSMutableArray and also
outputs the packets in the same order they arrive from AVFFoundation.
should fix ticket #4437 and #4513
---
libavdevice/avfoundation.m | 160 ++++++++++++++++++++++++++++---------
1 file changed, 124 insertions(+), 36 deletions(-)
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 59d5b0af4f..5ac6ec4183 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -31,13 +31,17 @@
#include "libavutil/pixdesc.h"
#include "libavutil/opt.h"
#include "libavutil/avstring.h"
+#include "libavutil/avassert.h"
#include "libavformat/internal.h"
#include "libavutil/internal.h"
#include "libavutil/parseutils.h"
#include "libavutil/time.h"
#include "libavutil/imgutils.h"
+#include "libavutil/fifo.h"
#include "avdevice.h"
+#define FIFO_SIZE 4
+
static const int avf_time_base = 1000000;
static const AVRational avf_time_base_q = {
@@ -128,8 +132,8 @@ typedef struct
AVCaptureSession *capture_session;
AVCaptureVideoDataOutput *video_output;
AVCaptureAudioDataOutput *audio_output;
- CMSampleBufferRef current_frame;
- CMSampleBufferRef current_audio_frame;
+ AVFifoBuffer *video_fifo;
+ AVFifoBuffer *audio_fifo;
AVCaptureDevice *observed_device;
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -138,6 +142,11 @@ typedef struct
int observed_quit;
} AVFContext;
+typedef struct {
+ int64_t ts;
+ CMSampleBufferRef frame;
+} BufferRef;
+
static void lock_frames(AVFContext* ctx)
{
pthread_mutex_lock(&ctx->frame_lock);
@@ -148,6 +157,48 @@ static void unlock_frames(AVFContext* ctx)
pthread_mutex_unlock(&ctx->frame_lock);
}
+static inline void fifo_write(AVFifoBuffer* f, int64_t ts, CMSampleBufferRef frame)
+{
+ BufferRef buf = {
+ .ts = ts,
+ .frame = frame,
+ };
+
+ CFRetain(frame);
+ av_fifo_generic_write(f, &buf, sizeof(BufferRef), NULL);
+}
+
+static inline void fifo_peek(AVFifoBuffer* f, BufferRef *buf)
+{
+ if (av_fifo_size(f)) {
+ av_fifo_generic_peek(f, buf, sizeof(BufferRef), NULL);
+ return;
+ }
+ buf->frame = nil;
+ return;
+}
+
+static inline void fifo_drain(AVFifoBuffer* f, int release)
+{
+ av_assert2(av_fifo_size(f) >= sizeof(BufferRef));
+ if (release) {
+ BufferRef buf;
+ fifo_peek(f, &buf);
+ CFRelease(buf.frame);
+ }
+ av_fifo_drain(f, sizeof(BufferRef));
+}
+
+static inline void fifo_freep(AVFifoBuffer **f)
+{
+ if (f) {
+ while (av_fifo_size(*f)) {
+ fifo_drain(*f, 1);
+ }
+ av_fifo_freep(f);
+ }
+}
+
/** FrameReciever class - delegate for AVCaptureSession
*/
@interface AVFFrameReceiver : NSObject
@@ -225,13 +276,16 @@ static void unlock_frames(AVFContext* ctx)
didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
fromConnection:(AVCaptureConnection *)connection
{
+ AVFifoBuffer *fifo = _context->video_fifo;
+ int64_t ts = av_gettime_relative();
lock_frames(_context);
- if (_context->current_frame != nil) {
- CFRelease(_context->current_frame);
+ if (av_fifo_space(fifo) == 0) {
+ av_log(_context, AV_LOG_DEBUG, "video fifo is full, the oldest frame has been dropped\n");
+ fifo_drain(fifo, 1);
}
- _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
+ fifo_write(fifo, ts, videoFrame);
unlock_frames(_context);
@@ -269,13 +323,16 @@ static void unlock_frames(AVFContext* ctx)
didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
fromConnection:(AVCaptureConnection *)connection
{
+ AVFifoBuffer *fifo = _context->audio_fifo;
+ int64_t ts = av_gettime_relative();
lock_frames(_context);
- if (_context->current_audio_frame != nil) {
- CFRelease(_context->current_audio_frame);
+ if (!av_fifo_space(fifo)) {
+ av_log(_context, AV_LOG_DEBUG, "audio fifo is full, the oldest frame has been dropped\n");
+ fifo_drain(fifo, 1);
}
- _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
+ fifo_write(fifo, ts, audioFrame);
unlock_frames(_context);
@@ -301,12 +358,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_audio_delegate = NULL;
av_freep(&ctx->audio_buffer);
+ fifo_freep(&ctx->video_fifo);
+ fifo_freep(&ctx->audio_fifo);
pthread_mutex_destroy(&ctx->frame_lock);
-
- if (ctx->current_frame) {
- CFRelease(ctx->current_frame);
- }
}
static void parse_device_name(AVFormatContext *s)
@@ -624,6 +679,7 @@ static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
static int get_video_config(AVFormatContext *s)
{
AVFContext *ctx = (AVFContext*)s->priv_data;
+ BufferRef buf;
CVImageBufferRef image_buffer;
CMBlockBufferRef block_buffer;
CGSize image_buffer_size;
@@ -644,8 +700,13 @@ static int get_video_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);
- image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
- block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
+ fifo_peek(ctx->video_fifo, &buf);
+ if (buf.frame == nil) {
+ return 1;
+ }
+
+ image_buffer = CMSampleBufferGetImageBuffer(buf.frame);
+ block_buffer = CMSampleBufferGetDataBuffer(buf.frame);
if (image_buffer) {
image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
@@ -661,9 +722,6 @@ static int get_video_config(AVFormatContext *s)
stream->codecpar->format = ctx->pixel_format;
}
- CFRelease(ctx->current_frame);
- ctx->current_frame = nil;
-
unlock_frames(ctx);
return 0;
@@ -672,6 +730,7 @@ static int get_video_config(AVFormatContext *s)
static int get_audio_config(AVFormatContext *s)
{
AVFContext *ctx = (AVFContext*)s->priv_data;
+ BufferRef buf;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
@@ -690,7 +749,12 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);
- format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
+ fifo_peek(ctx->audio_fifo, &buf);
+ if (buf.frame == nil) {
+ return 1;
+ }
+
+ format_desc = CMSampleBufferGetFormatDescription(buf.frame);
const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
if (!basic_desc) {
@@ -737,7 +801,7 @@ static int get_audio_config(AVFormatContext *s)
}
if (ctx->audio_non_interleaved) {
- CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
+ CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(buf.frame);
ctx->audio_buffer_size = CMBlockBufferGetDataLength(block_buffer);
ctx->audio_buffer = av_malloc(ctx->audio_buffer_size);
if (!ctx->audio_buffer) {
@@ -746,9 +810,6 @@ static int get_audio_config(AVFormatContext *s)
}
}
- CFRelease(ctx->current_audio_frame);
- ctx->current_audio_frame = nil;
-
unlock_frames(ctx);
return 0;
@@ -771,6 +832,9 @@ static int avf_read_header(AVFormatContext *s)
pthread_mutex_init(&ctx->frame_lock, NULL);
+ ctx->video_fifo = av_fifo_alloc_array(FIFO_SIZE, sizeof(BufferRef));
+ ctx->audio_fifo = av_fifo_alloc_array(FIFO_SIZE, sizeof(BufferRef));
+
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
CGGetActiveDisplayList(0, NULL, &num_screens);
#endif
@@ -1051,33 +1115,52 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
AVFContext* ctx = (AVFContext*)s->priv_data;
do {
+ BufferRef video;
+ BufferRef audio;
CVImageBufferRef image_buffer;
CMBlockBufferRef block_buffer;
lock_frames(ctx);
- if (ctx->current_frame != nil) {
+ fifo_peek(ctx->video_fifo, &video);
+ fifo_peek(ctx->audio_fifo, &audio);
+
+ if (video.frame != nil && audio.frame != nil) {
+ // process oldest CMSampleBufferRef first
+ if (audio.ts <= video.ts) {
+ video.frame = nil;
+ } else {
+ audio.frame = nil;
+ }
+ }
+
+ if (video.frame != nil) {
int status;
int length = 0;
- image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
- block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);
+ fifo_drain(ctx->video_fifo, 0);
+ unlock_frames(ctx);
+
+ image_buffer = CMSampleBufferGetImageBuffer(video.frame);
+ block_buffer = CMSampleBufferGetDataBuffer(video.frame);
if (image_buffer != nil) {
length = (int)CVPixelBufferGetDataSize(image_buffer);
} else if (block_buffer != nil) {
length = (int)CMBlockBufferGetDataLength(block_buffer);
} else {
+ CFRelease(video.frame);
return AVERROR(EINVAL);
}
if (av_new_packet(pkt, length) < 0) {
+ CFRelease(video.frame);
return AVERROR(EIO);
}
CMItemCount count;
CMSampleTimingInfo timing_info;
- if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
+ if (CMSampleBufferGetOutputSampleTimingInfoArray(video.frame, 1, &timing_info, &count) == noErr) {
AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
}
@@ -1094,31 +1177,37 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
status = AVERROR(EIO);
}
}
- CFRelease(ctx->current_frame);
- ctx->current_frame = nil;
+ CFRelease(video.frame);
- if (status < 0)
+ if (status < 0) {
return status;
- } else if (ctx->current_audio_frame != nil) {
- CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
+ }
+ } else if (audio.frame != nil) {
+ CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(audio.frame);
int block_buffer_size = CMBlockBufferGetDataLength(block_buffer);
+ fifo_drain(ctx->audio_fifo, 0);
+ unlock_frames(ctx);
+
if (!block_buffer || !block_buffer_size) {
+ CFRelease(audio.frame);
return AVERROR(EIO);
}
if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
+ CFRelease(audio.frame);
return AVERROR_BUFFER_TOO_SMALL;
}
if (av_new_packet(pkt, block_buffer_size) < 0) {
+ CFRelease(audio.frame);
return AVERROR(EIO);
}
CMItemCount count;
CMSampleTimingInfo timing_info;
- if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
+ if (CMSampleBufferGetOutputSampleTimingInfoArray(audio.frame, 1, &timing_info, &count) == noErr) {
AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
}
@@ -1131,6 +1220,7 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
if (ret != kCMBlockBufferNoErr) {
+ CFRelease(audio.frame);
return AVERROR(EIO);
}
@@ -1162,12 +1252,12 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
} else {
OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
if (ret != kCMBlockBufferNoErr) {
+ CFRelease(audio.frame);
return AVERROR(EIO);
}
}
- CFRelease(ctx->current_audio_frame);
- ctx->current_audio_frame = nil;
+ CFRelease(audio.frame);
} else {
pkt->data = NULL;
unlock_frames(ctx);
@@ -1177,8 +1267,6 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
return AVERROR(EAGAIN);
}
}
-
- unlock_frames(ctx);
} while (!pkt->data);
return 0;
--
2.29.2
More information about the ffmpeg-devel
mailing list