[FFmpeg-devel] [PATCH v2] avfilter/vf_scale: translate and verify swscale internal range flag

[Jan Ekström]

This value - while it looks like the actual range of the content -
is nothing but the internal value of swscale.

Thus, if we have RGB content, force the value to 1. Swscale will
ignore it, but at least the value of the output AVFrame will now
properly be "full range" instead of "limited range", as it is right
now.

Additionally, after calling sws_setColorspaceDetails double-check
the configured internal flag for the color range. Warn if this is
different to the requested value.

Finally, utilize the translated configured output value into the
output AVFrame's color_range.
---
 libavfilter/vf_scale.c | 51 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c
index 58eee96744..592e4a344e 100644
--- a/libavfilter/vf_scale.c
+++ b/libavfilter/vf_scale.c
@@ -750,11 +750,30 @@ scale:
         || in_range != AVCOL_RANGE_UNSPECIFIED
         || scale->out_range != AVCOL_RANGE_UNSPECIFIED) {
         int in_full, out_full, brightness, contrast, saturation;
+        int configured_in_full_range_flag, configured_out_full_range_flag;
         const int *inv_table, *table;
+        const AVPixFmtDescriptor *in_desc  = av_pix_fmt_desc_get(in->format);
+        const AVPixFmtDescriptor *out_desc = av_pix_fmt_desc_get(out->format);
+        if (!in_desc || !out_desc) {
+            av_log(ctx, AV_LOG_ERROR,
+                   "Failed to get one or more of the pixel format descriptors "
+                   "for formats - in: %d (%s), out: %d (%s)!\n",
+                   in->format, in_desc ? "OK" : "bad",
+                   out->format, out_desc ? "OK": "bad");
+            av_frame_free(&in);
+            av_frame_free(frame_out);
+            return AVERROR_INVALIDDATA;
+        }
 
         sws_getColorspaceDetails(scale->sws, (int **)&inv_table, &in_full,
                                  (int **)&table, &out_full,
                                  &brightness, &contrast, &saturation);
+        // translate the swscale internal range flags to hold true
+        // for RGB
+        in_full = in_desc->flags & AV_PIX_FMT_FLAG_RGB ?
+                  1 : in_full;
+        out_full = out_desc->flags & AV_PIX_FMT_FLAG_RGB ?
+                   1 : out_full;
 
         if (scale->in_color_matrix)
             inv_table = parse_yuv_type(scale->in_color_matrix, in->colorspace);
@@ -782,7 +801,37 @@ scale:
                                      table, out_full,
                                      brightness, contrast, saturation);
 
-        out->color_range = out_full ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG;
+        // double-check what was actually just configured,
+        // since swscale can silently ignore the color range
+        // value in sws_setColorspaceDetails.
+        sws_getColorspaceDetails(scale->sws, (int **)&inv_table,
+                                 &configured_in_full_range_flag,
+                                 (int **)&table,
+                                 &configured_out_full_range_flag,
+                                 &brightness, &contrast, &saturation);
+
+        // translate the actually configured internal range flags to hold true
+        // for RGB as well.
+        configured_in_full_range_flag = in_desc->flags & AV_PIX_FMT_FLAG_RGB ?
+                                        1 : configured_in_full_range_flag;
+        configured_out_full_range_flag = out_desc->flags & AV_PIX_FMT_FLAG_RGB ?
+                                         1 : configured_out_full_range_flag;
+
+        if (in_full != configured_in_full_range_flag ||
+            out_full != configured_out_full_range_flag) {
+            av_log(ctx, AV_LOG_WARNING,
+                   "swscale overrode set input/output range value as it "
+                   "considered it an invalid configuration! "
+                   "(input: requested: %s, configured: %s), "
+                   "(output: requested: %s, configured: %s)!\n",
+                   in_full ? "full" : "limited",
+                   configured_in_full_range_flag ? "full" : "limited",
+                   out_full ? "full" : "limited",
+                   configured_out_full_range_flag ? "full" : "limited");
+        }
+
+        out->color_range = configured_out_full_range_flag ?
+                           AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG;
     }
 
     av_reduce(&out->sample_aspect_ratio.num, &out->sample_aspect_ratio.den,
-- 
2.26.2