[FFmpeg-devel] [PATCH] FFMPEG: add graph reconfig capability
softworkz .
softworkz at hotmail.com
Wed Mar 26 09:28:50 EET 2025
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces at ffmpeg.org> On Behalf Of
> yangyalei050 via ffmpeg-devel
> Sent: Dienstag, 25. März 2025 12:25
> To: ffmpeg-devel at ffmpeg.org
> Cc: 269032741 at qq.com
> Subject: [FFmpeg-devel] [PATCH] FFMPEG: add graph reconfig capability
>
> From: yangyalei <269032741 at qq.com>
>
> 1. Support reuse the graph to play different audio.
> 2. Support config part of the graph.
>
> Signed-off-by: yangyalei <269032741 at qq.com>
> ---
> libavfilter/avfilter.c | 8 +-
> libavfilter/avfiltergraph.c | 201 ++++++++++++++++++++++++++++++++----
> 2 files changed, 187 insertions(+), 22 deletions(-)
>
> diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
> index e732556ffa..71396674fe 100644
> --- a/libavfilter/avfilter.c
> +++ b/libavfilter/avfilter.c
> @@ -345,7 +345,8 @@ int ff_filter_config_links(AVFilterContext *filter)
> FilterLinkInternal *li = ff_link_internal(link);
> FilterLinkInternal *li_in;
>
> - if (!link) continue;
> + if (!link || !link->incfg.formats || !link->outcfg.formats)
> + continue;
> if (!link->src || !link->dst) {
> av_log(filter, AV_LOG_ERROR,
> "Not all input and output are properly linked
> (%d).\n", i);
> @@ -359,6 +360,11 @@ int ff_filter_config_links(AVFilterContext *filter)
>
> switch (li->init_state) {
> case AVLINK_INIT:
> + /* for part graph re-negotiation.
> + * For example: output-filter link has config_props,
> + * but the input-filter still need config_props. */
> + if ((ret = ff_filter_config_links(link->src)) < 0)
> + return ret;
> continue;
> case AVLINK_STARTINIT:
> av_log(filter, AV_LOG_INFO, "circular filter chain
> detected\n");
> diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c
> index 5e93f93aab..8565fe6a24 100644
> --- a/libavfilter/avfiltergraph.c
> +++ b/libavfilter/avfiltergraph.c
> @@ -23,6 +23,7 @@
> #include "config.h"
>
> #include <string.h>
> +#include <stdbool.h>
>
> #include "libavutil/avassert.h"
> #include "libavutil/bprint.h"
> @@ -348,7 +349,7 @@ static int filter_query_formats(AVFilterContext
> *ctx)
>
> if (filter->formats_state == FF_FILTER_FORMATS_QUERY_FUNC) {
> if ((ret = filter->formats.query_func(ctx)) < 0) {
> - if (ret != AVERROR(EAGAIN))
> + if (ret != AVERROR(EAGAIN) && ret != FFERROR_NOT_READY)
> av_log(ctx, AV_LOG_ERROR, "Query format failed for
> '%s': %s\n",
> ctx->name, av_err2str(ret));
> return ret;
> @@ -390,7 +391,7 @@ static int filter_query_formats(AVFilterContext
> *ctx)
> av_freep(&cfg_in_dyn);
> av_freep(&cfg_out_dyn);
> if (ret < 0) {
> - if (ret != AVERROR(EAGAIN))
> + if (ret != AVERROR(EAGAIN) && ret != FFERROR_NOT_READY)
> av_log(ctx, AV_LOG_ERROR, "Query format failed for
> '%s': %s\n",
> ctx->name, av_err2str(ret));
> return ret;
> @@ -438,6 +439,81 @@ static int formats_declared(AVFilterContext *f)
> return 1;
> }
>
> +static void ff_avfilter_link_unref_formats(AVFilterLink *link)
> +{
> + ff_formats_unref(&link->incfg.formats);
> + ff_formats_unref(&link->outcfg.formats);
> + ff_formats_unref(&link->incfg.samplerates);
> + ff_formats_unref(&link->outcfg.samplerates);
> + ff_channel_layouts_unref(&link->incfg.channel_layouts);
> + ff_channel_layouts_unref(&link->outcfg.channel_layouts);
What about color_spaces and color_ranges?
Below you are removing the unref for those, but you don't add them here or elsewhere again.
> +}
> +
> +static bool formats_useless(AVFilterContext *f)
> +{
> + bool in, out;
> + int i;
> +
> + if (!f->nb_inputs || !f->nb_outputs)
> + return false;
> +
> + in = out = true;
> +
> + for (i = 0; i < f->nb_inputs; i++)
> + if (f->inputs[i]->outcfg.formats) {
> + in = false;
> + break;
> + }
> +
> + for (i = 0; i < f->nb_outputs; i++)
> + if (f->outputs[i]->incfg.formats) {
> + out = false;
> + break;
> + }
> +
> + return in ^ out;
> +}
> +
> +
> +static bool sanitize_formats(AVFilterGraph *graph)
> +{
> + AVFilterContext *f;
> + bool changed = false;
> + unsigned i, j;
> +
> + for (i = 0; i < graph->nb_filters; i++) {
> + f = graph->filters[i];
> +
> + for (j = 0; j < f->nb_inputs; j++) {
> + if ((!f->inputs[j]->outcfg.formats && f->inputs[j]-
> >incfg.formats)
> + || (f->inputs[j]->outcfg.formats && !f->inputs[j]-
> >incfg.formats)) {
> + ff_avfilter_link_unref_formats(f->inputs[j]);
> + changed = true;
> + }
> + }
> +
> + for (j = 0; j < f->nb_outputs; j++) {
> + if ((!f->outputs[j]->incfg.formats && f->outputs[j]-
> >outcfg.formats)
> + || (f->outputs[j]->incfg.formats && !f->outputs[j]-
> >outcfg.formats)) {
> + ff_avfilter_link_unref_formats(f->outputs[j]);
> + changed = true;
> + }
> + }
> +
> + if (formats_useless(f)) {
> + for (j = 0; j < f->nb_inputs; j++)
> + ff_avfilter_link_unref_formats(f->inputs[j]);
> +
> + for (j = 0; j < f->nb_outputs; j++)
> + ff_avfilter_link_unref_formats(f->outputs[j]);
> +
> + changed = true;
> + }
> + }
> +
> + return changed;
> +}
> +
> /**
> * Perform one round of query_formats() and merging formats lists on
> the
> * filter graph.
> @@ -462,7 +538,7 @@ static int query_formats(AVFilterGraph *graph, void
> *log_ctx)
> if (formats_declared(f))
> continue;
> ret = filter_query_formats(f);
> - if (ret < 0 && ret != AVERROR(EAGAIN))
> + if (ret < 0 && ret != AVERROR(EAGAIN) && ret !=
> FFERROR_NOT_READY)
> return ret;
> /* note: EAGAIN could indicate a partial success, not counted
> yet */
> count_queried += ret >= 0;
> @@ -478,7 +554,7 @@ static int query_formats(AVFilterGraph *graph, void
> *log_ctx)
> unsigned neg_step;
> int convert_needed = 0;
>
> - if (!link)
> + if (!link || !link->incfg.formats)
> continue;
>
> neg = ff_filter_get_negotiation(link);
> @@ -492,6 +568,7 @@ static int query_formats(AVFilterGraph *graph, void
> *log_ctx)
> break;
> }
> }
> +
> for (neg_step = 0; neg_step < neg->nb_mergers; neg_step++)
> {
> const AVFilterFormatsMerger *m = &neg-
> >mergers[neg_step];
> void *a = FF_FIELD_AT(void *, m->offset, link->incfg);
> @@ -789,17 +866,6 @@ static int pick_format(AVFilterLink *link,
> AVFilterLink *ref)
> return ret;
> }
>
> - ff_formats_unref(&link->incfg.formats);
> - ff_formats_unref(&link->outcfg.formats);
> - ff_formats_unref(&link->incfg.samplerates);
> - ff_formats_unref(&link->outcfg.samplerates);
> - ff_channel_layouts_unref(&link->incfg.channel_layouts);
> - ff_channel_layouts_unref(&link->outcfg.channel_layouts);
> - ff_formats_unref(&link->incfg.color_spaces);
> - ff_formats_unref(&link->outcfg.color_spaces);
> - ff_formats_unref(&link->incfg.color_ranges);
> - ff_formats_unref(&link->outcfg.color_ranges);
The latter 4 are removed without replacement.
> return 0;
> }
>
> @@ -818,6 +884,7 @@ do {
> \
> list_type *fmts;
> \
>
> \
> if (link->type != out_link->type ||
> \
> + !out_link->incfg.list ||
> \
> out_link->incfg.list->nb == 1)
> \
> continue;
> \
> fmts = out_link->incfg.list;
> \
> @@ -925,6 +992,7 @@ static void
> swap_samplerates_on_filter(AVFilterContext *filter)
> link = filter->inputs[i];
>
> if (link->type == AVMEDIA_TYPE_AUDIO &&
> + link->outcfg.samplerates &&
> link->outcfg.samplerates->nb_formats== 1)
> break;
> }
> @@ -938,6 +1006,7 @@ static void
> swap_samplerates_on_filter(AVFilterContext *filter)
> int best_idx, best_diff = INT_MAX;
>
> if (outlink->type != AVMEDIA_TYPE_AUDIO ||
> + !outlink->incfg.samplerates ||
> outlink->incfg.samplerates->nb_formats < 2)
> continue;
>
> @@ -1007,6 +1076,7 @@ static void
> swap_channel_layouts_on_filter(AVFilterContext *filter)
> link = filter->inputs[i];
>
> if (link->type == AVMEDIA_TYPE_AUDIO &&
> + link->outcfg.channel_layouts &&
> link->outcfg.channel_layouts->nb_channel_layouts == 1)
> break;
> }
> @@ -1018,6 +1088,7 @@ static void
> swap_channel_layouts_on_filter(AVFilterContext *filter)
> int best_idx = -1, best_score = INT_MIN, best_count_diff =
> INT_MAX;
>
> if (outlink->type != AVMEDIA_TYPE_AUDIO ||
> + !outlink->incfg.channel_layouts ||
> outlink->incfg.channel_layouts->nb_channel_layouts < 2)
> continue;
>
> @@ -1110,6 +1181,7 @@ static void
> swap_sample_fmts_on_filter(AVFilterContext *filter)
> link = filter->inputs[i];
>
> if (link->type == AVMEDIA_TYPE_AUDIO &&
> + link->outcfg.formats &&
> link->outcfg.formats->nb_formats == 1)
> break;
> }
> @@ -1124,6 +1196,7 @@ static void
> swap_sample_fmts_on_filter(AVFilterContext *filter)
> int best_idx = -1, best_score = INT_MIN;
>
> if (outlink->type != AVMEDIA_TYPE_AUDIO ||
> + !outlink->incfg.formats ||
> outlink->incfg.formats->nb_formats < 2)
> continue;
>
> @@ -1180,7 +1253,8 @@ static int pick_formats(AVFilterGraph *graph)
> AVFilterContext *filter = graph->filters[i];
> if (filter->nb_inputs){
> for (j = 0; j < filter->nb_inputs; j++){
> - if (filter->inputs[j]->incfg.formats && filter-
> >inputs[j]->incfg.formats->nb_formats == 1) {
> + if (filter->inputs[j]->format < 0 &&
> + filter->inputs[j]->incfg.formats && filter-
> >inputs[j]->incfg.formats->nb_formats == 1) {
> if ((ret = pick_format(filter->inputs[j],
> NULL)) < 0)
> return ret;
> change = 1;
> @@ -1189,7 +1263,8 @@ static int pick_formats(AVFilterGraph *graph)
> }
> if (filter->nb_outputs){
> for (j = 0; j < filter->nb_outputs; j++){
> - if (filter->outputs[j]->incfg.formats && filter-
> >outputs[j]->incfg.formats->nb_formats == 1) {
> + if (filter->outputs[j]->format < 0 &&
> + filter->outputs[j]->incfg.formats && filter-
> >outputs[j]->incfg.formats->nb_formats == 1) {
> if ((ret = pick_format(filter->outputs[j],
> NULL)) < 0)
> return ret;
> change = 1;
> @@ -1198,7 +1273,7 @@ static int pick_formats(AVFilterGraph *graph)
> }
> if (filter->nb_inputs && filter->nb_outputs && filter-
> >inputs[0]->format>=0) {
> for (j = 0; j < filter->nb_outputs; j++) {
> - if (filter->outputs[j]->format<0) {
> + if (filter->outputs[j]->format < 0 && filter-
> >outputs[j]->incfg.formats) {
> if ((ret = pick_format(filter->outputs[j],
> filter->inputs[0])) < 0)
> return ret;
> change = 1;
> @@ -1234,6 +1309,8 @@ static int graph_config_formats(AVFilterGraph
> *graph, void *log_ctx)
> if (ret < 0)
> return ret;
>
> + while (sanitize_formats(graph));
> +
> /* Once everything is merged, it's possible that we'll still have
> * multiple valid media format choices. We try to minimize the
> amount
> * of format conversion inside filters */
> @@ -1292,12 +1369,94 @@ static int graph_config_pointers(AVFilterGraph
> *graph, void *log_ctx)
> return 0;
> }
>
> +static void graph_clear_formats(AVFilterGraph *graph, void *log_ctx)
> +{
> + int status_in, status_out;
> + AVRational time_base;
> + AVFilterContext *f;
> + FilterLinkInternal *ilinki;
> + FilterLinkInternal *olinki;
> + int i, j;
> +
> + for (i = graph->nb_filters - 1; i >= 0; i--) {
> + f = graph->filters[i];
> +
> + if (!strncmp(f->name, "auto_", strlen("auto_"))) {
> + ilinki = ff_link_internal(f->inputs[0]);
> + olinki = ff_link_internal(f->outputs[0]);
> + if (ilinki->status_out || ilinki->status_in ||
> + olinki->status_out || olinki->status_in) {
> + AVFilterContext *src = f->inputs[0]->src;
> + AVFilterContext *dst = f->outputs[0]->dst;
> + unsigned srcpad = FF_OUTLINK_IDX(f->inputs[0]);
> + unsigned dstpad = FF_INLINK_IDX(f->outputs[0]);
> +
> + status_in = ilinki->status_in;
> + status_out = ilinki->status_out;
> + time_base = f->inputs[0]->time_base;
> +
> + avfilter_free(f);
> + avfilter_link(src, srcpad, dst, dstpad);
> +
> + olinki->status_in = status_in;
> + olinki->status_out = status_out;
> + dst->inputs[0]->time_base = time_base;
> +
> + ff_filter_set_ready(dst, 200);
> + continue;
> + }
> + }
> +
> + for (j = 0; j < f->nb_inputs; j++) {
> + ilinki = ff_link_internal(f->inputs[j]);
> + if (ilinki->status_out) {
> + ff_formats_unref(&f->inputs[j]->outcfg.formats);
> + ff_formats_unref(&f->inputs[j]->outcfg.samplerates);
> + ff_channel_layouts_unref(&f->inputs[j]-
> >outcfg.channel_layouts);
> +
> + if (ilinki->status_in) {
> + f->inputs[j]->format = -1;
> + f->inputs[j]->sample_rate = 0;
> + ilinki->init_state = AVLINK_UNINIT;
> + f->inputs[j]->time_base.num = 0;
> + f->inputs[j]->time_base.den = 0;
> + av_channel_layout_uninit(&f->inputs[j]->ch_layout);
> + }
> + }
> + }
> +
> + for (j = 0; j < f->nb_outputs; j++) {
> + olinki = ff_link_internal(f->outputs[j]);
> + if (olinki->status_in) {
> + ff_formats_unref(&f->outputs[j]->incfg.formats);
> + ff_formats_unref(&f->outputs[j]->incfg.samplerates);
> + ff_channel_layouts_unref(&f->outputs[j]-
> >incfg.channel_layouts);
> +
> + if (olinki->status_out) {
> + f->outputs[j]->format = -1;
> + f->outputs[j]->sample_rate = 0;
> + olinki->init_state = AVLINK_UNINIT;
> + f->outputs[j]->time_base.num = 0;
> + f->outputs[j]->time_base.den = 0;
> + av_channel_layout_uninit(&f->outputs[j]-
> >ch_layout);
> + }
The code above is executed for audio and video filters but it seems to be handling audio filters only.
> + }
> + }
> + }
> +}
> +
> int avfilter_graph_config(AVFilterGraph *graphctx, void *log_ctx)
> {
> - int ret;
> + AVClass *avc = log_ctx ? *(AVClass **)log_ctx : NULL;
> + int ret, i;
>
> - if ((ret = graph_check_validity(graphctx, log_ctx)))
> - return ret;
This is the only usage of graph_check_validity(), so you would need to remove that function as well, explaining why it is no longer needed.
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces at ffmpeg.org> On Behalf Of
> yangyalei via ffmpeg-devel
> Sent: Mittwoch, 26. März 2025 04:10
> To: FFmpeg development discussions and patches <ffmpeg-devel at ffmpeg.org>
> Cc: yangyalei <269032741 at qq.com>
> Subject: Re: [FFmpeg-devel] [PATCH] FFMPEG: add graph reconfig
> capability
> (abuff_src at Music)(abuff_src at Ring)---amix---abuff_sink
>
>
> In the graph above, We support three playback scenarios at the same
> time:
>
>
> 1. Only abuff_src at Music needs to be played, and abuff_src at Ring has no
> data. abuff_src at Ring is ignored during negotiation, "abuff_src at Music --
> amix -- abuff_sink" link negotiation is successful, and playback can be
> performed;
>
>
> 2. Only abuff_src at Ring needs to be played, and abuff_src at Music has no
> data. abuff_src at Music is ignored during negotiation, "abuff_src at ring --
> > amix --> abuff_sink" link negotiation is successful, and
> playback can be performed;
>
>
> 3. Both abuff_src at Music and abuff_src at Ring need to be played, and all
> filters are negotiated successfully and mixed playback is performed.
Is this case about the ffmpeg CLI tool or are you using libavfilter directly?
In the latter case: Why don't you just remove the source filter which has no data, what's the purpose of keeping a filter without data flow in the graph?
Your patch enables a filter to remain in the graph without having any formats negotiated but is still connected to other filters. What's not quite clear to me: How can this work in your example with the amix filter?
If the amix filter has two connected inputs but one of those inputs does not deliver any data, I would expect that the graph processing gets stuck at the amix filter input where it is waiting for data from the second input, isn't that the case?
Thanks
sw
More information about the ffmpeg-devel
mailing list