[Libav-user] Audio sync issue when resampling audio with libavfilter

[Rubén Sánchez]

Hi everyone!

The error was in another part of the code. This audio filter just works fine.

Rubén Sánchez Castellano
Senior Programmer
Telescope Inc Barcelona


From: Libav-user <libav-user-bounces at ffmpeg.org> on behalf of Rubén Sánchez <ruben.sanchez at telescope.tv>
Reply-To: "This list is about using libavcodec, libavformat, libavutil, libavdevice and libavfilter." <libav-user at ffmpeg.org>
Date: Friday, 11 August 2017 at 14:11
To: "libav-user at ffmpeg.org" <libav-user at ffmpeg.org>
Subject: [Libav-user] Audio sync issue when resampling audio with libavfilter

Hi!

I'm developing an app that among other features, must resample 48000KHz audio input into 44100 audio output and encode it using AAC encoder. I'm able to resample the audio with the code attached but after 30 minutes the audio gets out of sync and the audio goes after the video. This particular case has the following context:

Input: Stereo, FLTP, 48KHz
Output: Stereo, FLTP, 44.1KHz
FFmpeg version 3.3.3

This is the method I use to initialize the filter:

================= CODE ========================

AVFilterGraph *filter_graph = NULL;
AVFilterContext *buffersrc_ctx = NULL;
AVFilterContext *buffersink_ctx = NULL;

int initialize_audio_filter(AVStream *inputStream) {
                char args[512];
                int ret;
                AVFilter *buffersrc = avfilter_get_by_name("abuffer");
                AVFilter *buffersink = avfilter_get_by_name("abuffersink");
                AVFilterInOut *outputs = avfilter_inout_alloc();
                AVFilterInOut *inputs = avfilter_inout_alloc();
                filter_graph = avfilter_graph_alloc();
                const enum AVSampleFormat out_sample_fmts[] = {AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE};
                const int64_t out_channel_layouts[] = {AV_CH_LAYOUT_STEREO, -1};
                const int out_sample_rates[] = {44100, -1};

                snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%" PRIx64,
                inputStream->codec->time_base.num, inputStream->codec->time_base.den,
                inputStream->codec->sample_rate,
                av_get_sample_fmt_name(inputStream->codec->sample_fmt),
                inputStream->codec->channel_layout);
                ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in", args, NULL, filter_graph);

                if (ret < 0) {
                svsCritical("", QString("Could not create filter graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
                return -1;
                }

                ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out", NULL, NULL, filter_graph);

                if (ret < 0) {
                svsCritical("", QString("Cannot create buffer sink, error: %1").arg(svsAvErrorToFormattedString(ret)))
                return ret;
                }

                ret = av_opt_set_int_list(buffersink_ctx, "sample_fmts", out_sample_fmts, -1,
                                AV_OPT_SEARCH_CHILDREN);

                if (ret < 0) {
                svsCritical("", QString("Cannot set output sample format, error: %1").arg(svsAvErrorToFormattedString(ret)))
                return ret;
                }

                ret = av_opt_set_int_list(buffersink_ctx, "channel_layouts", out_channel_layouts, -1,
                                AV_OPT_SEARCH_CHILDREN);

                if (ret < 0) {
                svsCritical("", QString("Cannot set output channel layout, error: %1").arg(svsAvErrorToFormattedString(ret)))
                return ret;
                }

                ret = av_opt_set_int_list(buffersink_ctx, "sample_rates", out_sample_rates, -1,
                                AV_OPT_SEARCH_CHILDREN);

                if (ret < 0) {
                svsCritical("", QString("Cannot set output sample rate, error: %1").arg(svsAvErrorToFormattedString(ret)))
                return ret;
                }

                /* Endpoints for the filter graph. */
                outputs -> name = av_strdup("in");
                outputs -> filter_ctx = buffersrc_ctx;
                outputs -> pad_idx = 0;
                outputs -> next = NULL;
                /* Endpoints for the filter graph. */
                inputs -> name = av_strdup("out");
                inputs -> filter_ctx = buffersink_ctx;
                inputs -> pad_idx = 0;
                inputs -> next = NULL;

   QString filter_description = "aresample=44100:async=100,asetnsamples=n=1024:p=0,aformat=sample_fmts=fltp:channel_layouts=stereo";
   if ((ret = avfilter_graph_parse_ptr(filter_graph, filter_description.toStdString().c_str(), &inputs, &outputs, NULL)) < 0) {
                svsCritical("", QString("Could not add the filter to graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
                }

                if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0) {
                svsCritical("", QString("Could not configure the graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
                }

                /* Print summary of the sink buffer
                * Note: args buffer is reused to store channel layout string */
                AVFilterLink *outlink = buffersink_ctx->inputs[0];
                av_get_channel_layout_string(args, sizeof(args), -1, outlink->channel_layout);
                svsInfo("", QString::asprintf("Output: srate:%dHz fmt:%s chlayout:%s\n",
                                (int) outlink->sample_rate,
                                (char *) av_x_if_null(av_get_sample_fmt_name((AVSampleFormat) outlink->format), "?"),
                                args))
                return 0;
}
================= END CODE ========================

Notice I configure the audio filter to generate the output with 1024 samples since this is the amount of samples AAC needs to encode an audio frame.

And this is how I use it:

================= CODE ========================

AVFrame* StreamFeedController::resampleAudio(const QString& key, AVFrame *frame) {

                /* Push the decoded frame into the filtergraph */
                qint32 ret;
                ret = av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
                if(ret < 0) {
                svsWarning(key, QString("Error adding frame to buffer: %1").arg(svsAvErrorToFormattedString(ret)))
                // Delete input frame and return null
                av_frame_unref(frame);
                return nullptr;
                }

                AVFrame *resampled_frame = av_frame_alloc();

                /* Pull filtered frames from the filtergraph */
                ret = av_buffersink_get_frame(buffersink_ctx, resampled_frame);

                /* Set the timestamp on the resampled frame */
//             resampled_frame->pts = frame->pts;
                resampled_frame->best_effort_timestamp = resampled_frame->pts;

                if(ret < 0) {
                // This is very common. For 48KHz -> 44.1KHz for some input frames the
                // filter has not data enough to generate another output.
                av_frame_unref(frame);
                av_frame_unref(resampled_frame);
                return nullptr;
                }
                av_frame_unref(frame);
                return resampled_frame;
}
================= END CODE ========================

Any idea why the filter stops working after 30 minutes? I could reproduce this always.



Rubén Sánchez Castellano
Senior Programmer
Telescope Inc Barcelona

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://ffmpeg.org/pipermail/libav-user/attachments/20170828/7324a733/attachment.html>