[FFmpeg-devel] [PATCH] lavfi: fix merging of formats and clarify exception.

[Nicolas George]

The following commit:
b97d61f avfilter/ff_merge_formats: only merge if doing so does not loose chroma or alpha
introduced an exception to avoid lossy conversions.
Add a comment to explain the logic.
Fix the call to avoid applying it on audio formats.

Signed-off-by: Nicolas George <nicolas.george at normalesup.org>
---
 libavfilter/avfiltergraph.c |    8 ++++----
 libavfilter/formats.c       |   11 ++++++++++-
 libavfilter/formats.h       |    3 ++-
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c
index 0bc8464..d0b6b16 100644
--- a/libavfilter/avfiltergraph.c
+++ b/libavfilter/avfiltergraph.c
@@ -272,8 +272,8 @@ static int query_formats(AVFilterGraph *graph, AVClass *log_ctx)
                 continue;
 
             if (link->in_formats != link->out_formats &&
-                !ff_merge_formats(link->in_formats,
-                                        link->out_formats))
+                !ff_merge_formats(link->in_formats, link->out_formats,
+                                  link->type))
                 convert_needed = 1;
             if (link->type == AVMEDIA_TYPE_AUDIO) {
                 if (link->in_channel_layouts != link->out_channel_layouts &&
@@ -337,8 +337,8 @@ static int query_formats(AVFilterGraph *graph, AVClass *log_ctx)
                 filter_query_formats(convert);
                 inlink  = convert->inputs[0];
                 outlink = convert->outputs[0];
-                if (!ff_merge_formats( inlink->in_formats,  inlink->out_formats) ||
-                    !ff_merge_formats(outlink->in_formats, outlink->out_formats))
+                if (!ff_merge_formats( inlink->in_formats,  inlink->out_formats,  inlink->type) ||
+                    !ff_merge_formats(outlink->in_formats, outlink->out_formats, outlink->type))
                     ret |= AVERROR(ENOSYS);
                 if (inlink->type == AVMEDIA_TYPE_AUDIO &&
                     (!ff_merge_samplerates(inlink->in_samplerates,
diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index eb9d940..5876280 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -89,7 +89,8 @@ do {
     MERGE_REF(ret, b, fmts, type, fail);                                        \
 } while (0)
 
-AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b)
+AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b,
+                                  enum AVMediaType type)
 {
     AVFilterFormats *ret = NULL;
     int i, j;
@@ -99,6 +100,14 @@ AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b)
     if (a == b)
         return a;
 
+    /* Do not lose chroma or alpha in merging.
+       It happens if both lists have formats with chroma (resp. alpha), but
+       the only formats in common do not have it (e.g. YUV+gray vs.
+       RGB+gray): in that case, the merging would select the gray format,
+       possibly causing a lossy conversion elsewhere in the graph.
+       To avoid that, pretend that there are no common formats to force the
+       insertion of a conversion filter. */
+    if (type == AVMEDIA_TYPE_VIDEO)
     for (i = 0; i < a->format_count; i++)
         for (j = 0; j < b->format_count; j++) {
             const AVPixFmtDescriptor *adesc = av_pix_fmt_desc_get(a->formats[i]);
diff --git a/libavfilter/formats.h b/libavfilter/formats.h
index a476e70..c06f6df 100644
--- a/libavfilter/formats.h
+++ b/libavfilter/formats.h
@@ -218,7 +218,8 @@ AVFilterFormats *ff_planar_sample_fmts(void);
  * If a and b do not share any common formats, neither is modified, and NULL
  * is returned.
  */
-AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b);
+AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b,
+                                  enum AVMediaType type);
 
 /**
  * Add *ref as a new reference to formats.
-- 
1.7.10.4