[FFmpeg-devel] [PATCH 1/3] ffplay: implement separete audio decoder thread
Marton Balint
cus at passwd.hu
Fri Oct 31 02:35:54 CET 2014
On Thu, 30 Oct 2014, wm4 wrote:
> On Thu, 30 Oct 2014 00:31:25 +0100
> Marton Balint <cus at passwd.hu> wrote:
>
>> Signed-off-by: Marton Balint <cus at passwd.hu>
>> ---
>> ffplay.c | 265 ++++++++++++++++++++++++++++++++++++---------------------------
>> 1 file changed, 153 insertions(+), 112 deletions(-)
>>
>> diff --git a/ffplay.c b/ffplay.c
>> index a979164..24bcae2 100644
>> --- a/ffplay.c
>> +++ b/ffplay.c
>> @@ -121,7 +121,8 @@ typedef struct PacketQueue {
>>
>> #define VIDEO_PICTURE_QUEUE_SIZE 3
>> #define SUBPICTURE_QUEUE_SIZE 16
>> -#define FRAME_QUEUE_SIZE FFMAX(VIDEO_PICTURE_QUEUE_SIZE, SUBPICTURE_QUEUE_SIZE)
>> +#define SAMPLE_QUEUE_SIZE 9
>> +#define FRAME_QUEUE_SIZE FFMAX(SAMPLE_QUEUE_SIZE, FFMAX(VIDEO_PICTURE_QUEUE_SIZE, SUBPICTURE_QUEUE_SIZE))
>>
>> typedef struct AudioParams {
>> int freq;
>> @@ -196,6 +197,7 @@ typedef struct Decoder {
>> typedef struct VideoState {
>> SDL_Thread *read_tid;
>> SDL_Thread *video_tid;
>> + SDL_Thread *audio_tid;
>> AVInputFormat *iformat;
>> int no_background;
>> int abort_request;
>> @@ -217,6 +219,7 @@ typedef struct VideoState {
>>
>> FrameQueue pictq;
>> FrameQueue subpq;
>> + FrameQueue sampq;
>>
>> Decoder auddec;
>> Decoder viddec;
>> @@ -242,8 +245,6 @@ typedef struct VideoState {
>> unsigned int audio_buf1_size;
>> int audio_buf_index; /* in bytes */
>> int audio_write_buf_size;
>> - int audio_buf_frames_pending;
>> - int audio_last_serial;
>> struct AudioParams audio_src;
>> #if CONFIG_AVFILTER
>> struct AudioParams audio_filter_src;
>> @@ -252,7 +253,6 @@ typedef struct VideoState {
>> struct SwrContext *swr_ctx;
>> int frame_drops_early;
>> int frame_drops_late;
>> - AVFrame *frame;
>>
>> enum ShowMode {
>> SHOW_MODE_NONE = -1, SHOW_MODE_VIDEO = 0, SHOW_MODE_WAVES, SHOW_MODE_RDFT, SHOW_MODE_NB
>> @@ -712,12 +712,29 @@ static Frame *frame_queue_peek_writable(FrameQueue *f)
>> return &f->queue[f->windex];
>> }
>>
>> +static Frame *frame_queue_peek_readable(FrameQueue *f)
>> +{
>> + /* wait until we have a readable a new frame */
>> + SDL_LockMutex(f->mutex);
>> + while (f->size - f->rindex_shown <= 0 &&
>> + !f->pktq->abort_request) {
>> + SDL_CondWait(f->cond, f->mutex);
>> + }
>> + SDL_UnlockMutex(f->mutex);
>> +
>> + if (f->pktq->abort_request)
>> + return NULL;
>> +
>> + return &f->queue[(f->rindex + f->rindex_shown) % f->max_size];
>> +}
>> +
>> static void frame_queue_push(FrameQueue *f)
>> {
>> if (++f->windex == f->max_size)
>> f->windex = 0;
>> SDL_LockMutex(f->mutex);
>> f->size++;
>> + SDL_CondSignal(f->cond);
>> SDL_UnlockMutex(f->mutex);
>> }
>>
>> @@ -1280,6 +1297,7 @@ static void stream_close(VideoState *is)
>>
>> /* free all pictures */
>> frame_queue_destory(&is->pictq);
>> + frame_queue_destory(&is->sampq);
>> frame_queue_destory(&is->subpq);
>> SDL_DestroyCond(is->continue_read_thread);
>> #if !CONFIG_AVFILTER
>> @@ -2100,6 +2118,93 @@ end:
>> }
>> #endif /* CONFIG_AVFILTER */
>>
>> +static int audio_thread(void *arg)
>> +{
>> + VideoState *is = arg;
>> + AVFrame *frame = av_frame_alloc();
>> + Frame *af;
>> +#if CONFIG_AVFILTER
>> + int last_serial = -1;
>> + int64_t dec_channel_layout;
>> + int reconfigure;
>> +#endif
>> + int got_frame = 0;
>> + AVRational tb;
>> + int ret = 0;
>> +
>> + if (!frame)
>> + return AVERROR(ENOMEM);
>> +
>> + do {
>> + if ((got_frame = decoder_decode_frame(&is->auddec, frame, NULL)) < 0)
>> + goto the_end;
>> +
>> + if (got_frame) {
>> + tb = (AVRational){1, frame->sample_rate};
>> +
>> +#if CONFIG_AVFILTER
>> + dec_channel_layout = get_valid_channel_layout(frame->channel_layout, av_frame_get_channels(frame));
>> +
>> + reconfigure =
>> + cmp_audio_fmts(is->audio_filter_src.fmt, is->audio_filter_src.channels,
>> + frame->format, av_frame_get_channels(frame)) ||
>> + is->audio_filter_src.channel_layout != dec_channel_layout ||
>> + is->audio_filter_src.freq != frame->sample_rate ||
>> + is->auddec.pkt_serial != last_serial;
>> +
>> + if (reconfigure) {
>> + char buf1[1024], buf2[1024];
>> + av_get_channel_layout_string(buf1, sizeof(buf1), -1, is->audio_filter_src.channel_layout);
>> + av_get_channel_layout_string(buf2, sizeof(buf2), -1, dec_channel_layout);
>> + av_log(NULL, AV_LOG_DEBUG,
>> + "Audio frame changed from rate:%d ch:%d fmt:%s layout:%s serial:%d to rate:%d ch:%d fmt:%s layout:%s serial:%d\n",
>> + is->audio_filter_src.freq, is->audio_filter_src.channels, av_get_sample_fmt_name(is->audio_filter_src.fmt), buf1, last_serial,
>> + frame->sample_rate, av_frame_get_channels(frame), av_get_sample_fmt_name(frame->format), buf2, is->auddec.pkt_serial);
>> +
>> + is->audio_filter_src.fmt = frame->format;
>> + is->audio_filter_src.channels = av_frame_get_channels(frame);
>> + is->audio_filter_src.channel_layout = dec_channel_layout;
>> + is->audio_filter_src.freq = frame->sample_rate;
>> + last_serial = is->auddec.pkt_serial;
>> +
>> + if ((ret = configure_audio_filters(is, afilters, 1)) < 0)
>> + goto the_end;
>> + }
>> +
>> + if ((ret = av_buffersrc_add_frame(is->in_audio_filter, frame)) < 0)
>> + goto the_end;
>> +
>> + while ((ret = av_buffersink_get_frame_flags(is->out_audio_filter, frame, 0)) >= 0) {
>> + tb = is->out_audio_filter->inputs[0]->time_base;
>> +#endif
>> + if (!(af = frame_queue_peek_writable(&is->sampq)))
>> + goto the_end;
>> +
>> + af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
>> + af->pos = av_frame_get_pkt_pos(frame);
>> + af->serial = is->auddec.pkt_serial;
>> + af->duration = av_q2d((AVRational){frame->nb_samples, frame->sample_rate});
>> +
>> + av_frame_move_ref(af->frame, frame);
>> + frame_queue_push(&is->sampq);
>> +
>> +#if CONFIG_AVFILTER
>> + if (is->audioq.serial != is->auddec.pkt_serial)
>> + break;
>> + }
>> + if (ret == AVERROR_EOF)
>> + is->auddec.finished = is->auddec.pkt_serial;
>> +#endif
>> + }
>> + } while (ret >= 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF);
>> + the_end:
>> +#if CONFIG_AVFILTER
>> + avfilter_graph_free(&is->agraph);
>> +#endif
>> + av_frame_free(&frame);
>> + return ret;
>> +}
>> +
>> static int video_thread(void *arg)
>> {
>> VideoState *is = arg;
>> @@ -2315,135 +2420,77 @@ static int audio_decode_frame(VideoState *is)
>> {
>> int data_size, resampled_data_size;
>> int64_t dec_channel_layout;
>> - int got_frame = 0;
>> av_unused double audio_clock0;
>> int wanted_nb_samples;
>> - AVRational tb;
>> - int ret;
>> - int reconfigure;
>> -
>> - if (!is->frame)
>> - if (!(is->frame = av_frame_alloc()))
>> - return AVERROR(ENOMEM);
>> -
>> - for (;;) {
>> - if (is->audioq.serial != is->auddec.pkt_serial)
>> - is->audio_buf_frames_pending = got_frame = 0;
>> -
>> - if (!got_frame)
>> - av_frame_unref(is->frame);
>> + Frame *af;
>>
>> + {
>> if (is->paused)
>> return -1;
>>
>> - while (is->audio_buf_frames_pending || got_frame) {
>> - if (!is->audio_buf_frames_pending) {
>> - got_frame = 0;
>> - tb = (AVRational){1, is->frame->sample_rate};
>> -
>> -#if CONFIG_AVFILTER
>> - dec_channel_layout = get_valid_channel_layout(is->frame->channel_layout, av_frame_get_channels(is->frame));
>> -
>> - reconfigure =
>> - cmp_audio_fmts(is->audio_filter_src.fmt, is->audio_filter_src.channels,
>> - is->frame->format, av_frame_get_channels(is->frame)) ||
>> - is->audio_filter_src.channel_layout != dec_channel_layout ||
>> - is->audio_filter_src.freq != is->frame->sample_rate ||
>> - is->auddec.pkt_serial != is->audio_last_serial;
>> -
>> - if (reconfigure) {
>> - char buf1[1024], buf2[1024];
>> - av_get_channel_layout_string(buf1, sizeof(buf1), -1, is->audio_filter_src.channel_layout);
>> - av_get_channel_layout_string(buf2, sizeof(buf2), -1, dec_channel_layout);
>> - av_log(NULL, AV_LOG_DEBUG,
>> - "Audio frame changed from rate:%d ch:%d fmt:%s layout:%s serial:%d to rate:%d ch:%d fmt:%s layout:%s serial:%d\n",
>> - is->audio_filter_src.freq, is->audio_filter_src.channels, av_get_sample_fmt_name(is->audio_filter_src.fmt), buf1, is->audio_last_serial,
>> - is->frame->sample_rate, av_frame_get_channels(is->frame), av_get_sample_fmt_name(is->frame->format), buf2, is->auddec.pkt_serial);
>> -
>> - is->audio_filter_src.fmt = is->frame->format;
>> - is->audio_filter_src.channels = av_frame_get_channels(is->frame);
>> - is->audio_filter_src.channel_layout = dec_channel_layout;
>> - is->audio_filter_src.freq = is->frame->sample_rate;
>> - is->audio_last_serial = is->auddec.pkt_serial;
>> -
>> - if ((ret = configure_audio_filters(is, afilters, 1)) < 0)
>> - return ret;
>> - }
>> -
>> - if ((ret = av_buffersrc_add_frame(is->in_audio_filter, is->frame)) < 0)
>> - return ret;
>> -#endif
>> - }
>> -#if CONFIG_AVFILTER
>> - if ((ret = av_buffersink_get_frame_flags(is->out_audio_filter, is->frame, 0)) < 0) {
>> - if (ret == AVERROR(EAGAIN)) {
>> - is->audio_buf_frames_pending = 0;
>> - continue;
>> - }
>> - if (ret == AVERROR_EOF)
>> - is->auddec.finished = is->auddec.pkt_serial;
>> - return ret;
>> - }
>> - is->audio_buf_frames_pending = 1;
>> - tb = is->out_audio_filter->inputs[0]->time_base;
>> -#endif
>> + do {
>> + if (!(af = frame_queue_peek_readable(&is->sampq)))
>> + return -1;
>> + frame_queue_next(&is->sampq);
>> + } while (af->serial != is->audioq.serial);
>>
>> - data_size = av_samples_get_buffer_size(NULL, av_frame_get_channels(is->frame),
>> - is->frame->nb_samples,
>> - is->frame->format, 1);
>> + {
>> + data_size = av_samples_get_buffer_size(NULL, av_frame_get_channels(af->frame),
>> + af->frame->nb_samples,
>> + af->frame->format, 1);
>>
>> dec_channel_layout =
>> - (is->frame->channel_layout && av_frame_get_channels(is->frame) == av_get_channel_layout_nb_channels(is->frame->channel_layout)) ?
>> - is->frame->channel_layout : av_get_default_channel_layout(av_frame_get_channels(is->frame));
>> - wanted_nb_samples = synchronize_audio(is, is->frame->nb_samples);
>> + (af->frame->channel_layout && av_frame_get_channels(af->frame) == av_get_channel_layout_nb_channels(af->frame->channel_layout)) ?
>> + af->frame->channel_layout : av_get_default_channel_layout(av_frame_get_channels(af->frame));
>> + wanted_nb_samples = synchronize_audio(is, af->frame->nb_samples);
>>
>> - if (is->frame->format != is->audio_src.fmt ||
>> + if (af->frame->format != is->audio_src.fmt ||
>> dec_channel_layout != is->audio_src.channel_layout ||
>> - is->frame->sample_rate != is->audio_src.freq ||
>> - (wanted_nb_samples != is->frame->nb_samples && !is->swr_ctx)) {
>> + af->frame->sample_rate != is->audio_src.freq ||
>> + (wanted_nb_samples != af->frame->nb_samples && !is->swr_ctx)) {
>> swr_free(&is->swr_ctx);
>> is->swr_ctx = swr_alloc_set_opts(NULL,
>> is->audio_tgt.channel_layout, is->audio_tgt.fmt, is->audio_tgt.freq,
>> - dec_channel_layout, is->frame->format, is->frame->sample_rate,
>> + dec_channel_layout, af->frame->format, af->frame->sample_rate,
>> 0, NULL);
>> if (!is->swr_ctx || swr_init(is->swr_ctx) < 0) {
>> av_log(NULL, AV_LOG_ERROR,
>> "Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!\n",
>> - is->frame->sample_rate, av_get_sample_fmt_name(is->frame->format), av_frame_get_channels(is->frame),
>> + af->frame->sample_rate, av_get_sample_fmt_name(af->frame->format), av_frame_get_channels(af->frame),
>> is->audio_tgt.freq, av_get_sample_fmt_name(is->audio_tgt.fmt), is->audio_tgt.channels);
>> swr_free(&is->swr_ctx);
>> - break;
>> + return -1;
>> }
>> is->audio_src.channel_layout = dec_channel_layout;
>> - is->audio_src.channels = av_frame_get_channels(is->frame);
>> - is->audio_src.freq = is->frame->sample_rate;
>> - is->audio_src.fmt = is->frame->format;
>> + is->audio_src.channels = av_frame_get_channels(af->frame);
>> + is->audio_src.freq = af->frame->sample_rate;
>> + is->audio_src.fmt = af->frame->format;
>> }
>>
>> if (is->swr_ctx) {
>> - const uint8_t **in = (const uint8_t **)is->frame->extended_data;
>> + const uint8_t **in = (const uint8_t **)af->frame->extended_data;
>> uint8_t **out = &is->audio_buf1;
>> - int out_count = (int64_t)wanted_nb_samples * is->audio_tgt.freq / is->frame->sample_rate + 256;
>> + int out_count = (int64_t)wanted_nb_samples * is->audio_tgt.freq / af->frame->sample_rate + 256;
>> int out_size = av_samples_get_buffer_size(NULL, is->audio_tgt.channels, out_count, is->audio_tgt.fmt, 0);
>> int len2;
>> if (out_size < 0) {
>> av_log(NULL, AV_LOG_ERROR, "av_samples_get_buffer_size() failed\n");
>> - break;
>> + return -1;
>> }
>> - if (wanted_nb_samples != is->frame->nb_samples) {
>> - if (swr_set_compensation(is->swr_ctx, (wanted_nb_samples - is->frame->nb_samples) * is->audio_tgt.freq / is->frame->sample_rate,
>> - wanted_nb_samples * is->audio_tgt.freq / is->frame->sample_rate) < 0) {
>> + if (wanted_nb_samples != af->frame->nb_samples) {
>> + if (swr_set_compensation(is->swr_ctx, (wanted_nb_samples - af->frame->nb_samples) * is->audio_tgt.freq / af->frame->sample_rate,
>> + wanted_nb_samples * is->audio_tgt.freq / af->frame->sample_rate) < 0) {
>> av_log(NULL, AV_LOG_ERROR, "swr_set_compensation() failed\n");
>> - break;
>> + return -1;
>> }
>> }
>> av_fast_malloc(&is->audio_buf1, &is->audio_buf1_size, out_size);
>> if (!is->audio_buf1)
>> return AVERROR(ENOMEM);
>> - len2 = swr_convert(is->swr_ctx, out, out_count, in, is->frame->nb_samples);
>> + len2 = swr_convert(is->swr_ctx, out, out_count, in, af->frame->nb_samples);
>> if (len2 < 0) {
>> av_log(NULL, AV_LOG_ERROR, "swr_convert() failed\n");
>> - break;
>> + return -1;
>> }
>> if (len2 == out_count) {
>> av_log(NULL, AV_LOG_WARNING, "audio buffer is probably too small\n");
>> @@ -2453,17 +2500,17 @@ static int audio_decode_frame(VideoState *is)
>> is->audio_buf = is->audio_buf1;
>> resampled_data_size = len2 * is->audio_tgt.channels * av_get_bytes_per_sample(is->audio_tgt.fmt);
>> } else {
>> - is->audio_buf = is->frame->data[0];
>> + is->audio_buf = af->frame->data[0];
>> resampled_data_size = data_size;
>> }
>>
>> audio_clock0 = is->audio_clock;
>> /* update the audio clock with the pts */
>> - if (is->frame->pts != AV_NOPTS_VALUE)
>> - is->audio_clock = is->frame->pts * av_q2d(tb) + (double) is->frame->nb_samples / is->frame->sample_rate;
>> + if (!isnan(af->pts))
>> + is->audio_clock = af->pts + (double) af->frame->nb_samples / af->frame->sample_rate;
>> else
>> is->audio_clock = NAN;
>> - is->audio_clock_serial = is->auddec.pkt_serial;
>> + is->audio_clock_serial = af->serial;
>> #ifdef DEBUG
>> {
>> static double last_clock;
>> @@ -2475,12 +2522,6 @@ static int audio_decode_frame(VideoState *is)
>> #endif
>> return resampled_data_size;
>> }
>> -
>> - if ((got_frame = decoder_decode_frame(&is->auddec, is->frame, NULL)) < 0)
>> - return -1;
>> -
>> - if (is->auddec.flushed)
>> - is->audio_buf_frames_pending = 0;
>> }
>> }
>>
>> @@ -2707,6 +2748,7 @@ static int stream_component_open(VideoState *is, int stream_index)
>> is->auddec.start_pts = is->audio_st->start_time;
>> is->auddec.start_pts_tb = is->audio_st->time_base;
>> }
>> + is->audio_tid = SDL_CreateThread(audio_thread, is);
>> SDL_PauseAudio(0);
>> break;
>> case AVMEDIA_TYPE_VIDEO:
>> @@ -2750,6 +2792,8 @@ static void stream_component_close(VideoState *is, int stream_index)
>> packet_queue_abort(&is->audioq);
>>
>> SDL_CloseAudio();
>> + frame_queue_signal(&is->sampq);
>> + SDL_WaitThread(is->audio_tid, NULL);
>>
>> decoder_destroy(&is->auddec);
>> packet_queue_flush(&is->audioq);
>> @@ -2757,7 +2801,6 @@ static void stream_component_close(VideoState *is, int stream_index)
>> av_freep(&is->audio_buf1);
>> is->audio_buf1_size = 0;
>> is->audio_buf = NULL;
>> - av_frame_free(&is->frame);
>>
>> if (is->rdft) {
>> av_rdft_end(is->rdft);
>> @@ -2765,9 +2808,6 @@ static void stream_component_close(VideoState *is, int stream_index)
>> is->rdft = NULL;
>> is->rdft_bits = 0;
>> }
>> -#if CONFIG_AVFILTER
>> - avfilter_graph_free(&is->agraph);
>> -#endif
>> break;
>> case AVMEDIA_TYPE_VIDEO:
>> packet_queue_abort(&is->videoq);
>> @@ -3065,7 +3105,7 @@ static int read_thread(void *arg)
>> continue;
>> }
>> if (!is->paused &&
>> - (!is->audio_st || is->auddec.finished == is->audioq.serial) &&
>> + (!is->audio_st || (is->auddec.finished == is->audioq.serial && frame_queue_nb_remaining(&is->sampq) == 0)) &&
>> (!is->video_st || (is->viddec.finished == is->videoq.serial && frame_queue_nb_remaining(&is->pictq) == 0))) {
>> if (loop != 1 && (!loop || --loop)) {
>> stream_seek(is, start_time != AV_NOPTS_VALUE ? start_time : 0, 0, 0);
>> @@ -3160,6 +3200,8 @@ static VideoState *stream_open(const char *filename, AVInputFormat *iformat)
>> goto fail;
>> if (frame_queue_init(&is->subpq, &is->subtitleq, SUBPICTURE_QUEUE_SIZE, 0) < 0)
>> goto fail;
>> + if (frame_queue_init(&is->sampq, &is->audioq, SAMPLE_QUEUE_SIZE, 1) < 0)
>> + goto fail;
>>
>> packet_queue_init(&is->videoq);
>> packet_queue_init(&is->audioq);
>> @@ -3171,7 +3213,6 @@ static VideoState *stream_open(const char *filename, AVInputFormat *iformat)
>> init_clock(&is->audclk, &is->audioq.serial);
>> init_clock(&is->extclk, &is->extclk.serial);
>> is->audio_clock_serial = -1;
>> - is->audio_last_serial = -1;
>> is->av_sync_type = av_sync_type;
>> is->read_tid = SDL_CreateThread(read_thread, is);
>> if (!is->read_tid) {
>> @@ -3421,8 +3462,8 @@ static void event_loop(VideoState *cur_stream)
>> pos = -1;
>> if (pos < 0 && cur_stream->video_stream >= 0)
>> pos = frame_queue_last_pos(&cur_stream->pictq);
>> - if (pos < 0 && cur_stream->audio_stream >= 0 && cur_stream->frame)
>> - pos = av_frame_get_pkt_pos(cur_stream->frame);
>> + if (pos < 0 && cur_stream->audio_stream >= 0)
>> + pos = frame_queue_last_pos(&cur_stream->sampq);
>> if (pos < 0)
>> pos = avio_tell(cur_stream->ic->pb);
>> if (cur_stream->ic->bit_rate)
>
> Is there any actual advantage to this? Also, ffmpeg does support
> multithreaded audio decoding; only for some codecs though.
Not just the decoding, but the filtering as well will be done in
the audio decoder thread, so if that consumes a lot of CPU, ffplay should
handle it better and less audio buffer underruns should occur.
Another benefit might be the future implementation of a non-blocking audio
callback where ffplay returns silence if no frames are available at the
time of the callback. (this would be useful on platforms where SDL or the
underlying audio driver API does not automatically reset the audio buffer
to avoid repeated sound on a buffer underrun)
Obviously another reason is unification of the three decoding code paths
to allow further factorizations and extensions.
Regards,
Marton
More information about the ffmpeg-devel
mailing list