[FFmpeg-devel] [PATCH] lavfi/overlay: add format option

[Stefano Sabatini]

In particular, fix misbehavior in case main and overlay input formats
mismatch (e.g. YUV420 and YUV444).

TODO: bump micro
---
 doc/filters.texi         |   20 ++++++++++++++++-
 libavfilter/vf_overlay.c |   56 ++++++++++++++++++++++++++++++++++------------
 2 files changed, 61 insertions(+), 15 deletions(-)

diff --git a/doc/filters.texi b/doc/filters.texi
index 5dd7404..da03a69 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -3862,9 +3862,27 @@ frame. Default value is "0" for both expressions. In case the
 expression is invalid, it is set to a huge value (meaning that the
 overlay will not be displayed within the output visible area).
 
+ at item format
+Set the format for the output video.
+
+It accepts the following values:
+ at table @samp
+ at item yuv420
+force YUV420 output
+
+ at item yuv444
+force YUV444 output
+
 @item rgb
+force RGB output
+ at end table
+
+Default value is @samp{yuv420}.
+
+ at item rgb @emph{(deprecated)}
 If set to 1, force the filter to accept inputs in the RGB
-color space. Default value is 0.
+color space. Default value is 0. This option is deprecated, use
+ at option{format} instead.
 
 @item shortest
 If set to 1, force the output to terminate when the shortest input
diff --git a/libavfilter/vf_overlay.c b/libavfilter/vf_overlay.c
index 4b7fbe6..0bfb5f4 100644
--- a/libavfilter/vf_overlay.c
+++ b/libavfilter/vf_overlay.c
@@ -106,7 +106,7 @@ typedef struct {
     int overlay_pix_step[4];    ///< steps per pixel for each plane of the overlay
     int hsub, vsub;             ///< chroma subsampling values
     int shortest;               ///< terminate stream when the shortest input terminates
-
+    enum OverlayFormat { OVERLAY_FORMAT_YUV420, OVERLAY_FORMAT_YUV444, OVERLAY_FORMAT_RGB, OVERLAY_FORMAT_NB} format;
     double var_values[VAR_VARS_NB];
     char *x_expr, *y_expr;
     AVExpr *x_pexpr, *y_pexpr;
@@ -118,7 +118,13 @@ typedef struct {
 static const AVOption overlay_options[] = {
     { "x", "set the x expression", OFFSET(x_expr), AV_OPT_TYPE_STRING, {.str = "0"}, CHAR_MIN, CHAR_MAX, FLAGS },
     { "y", "set the y expression", OFFSET(y_expr), AV_OPT_TYPE_STRING, {.str = "0"}, CHAR_MIN, CHAR_MAX, FLAGS },
-    { "rgb", "force packed RGB in input and output", OFFSET(allow_packed_rgb), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, FLAGS },
+    { "rgb", "force packed RGB in input and output (deprecated)", OFFSET(allow_packed_rgb), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, FLAGS },
+
+    { "format", "set output format", OFFSET(format), AV_OPT_TYPE_INT, {.i64=OVERLAY_FORMAT_YUV420}, 0, OVERLAY_FORMAT_NB-1, FLAGS, "format" },
+    { "yuv420", "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_YUV420}, .flags = FLAGS, .unit = "format" },
+    { "yuv444", "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_YUV444}, .flags = FLAGS, .unit = "format" },
+    { "rgb",    "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_RGB},    .flags = FLAGS, .unit = "format" },
+
     { "shortest", "force termination when the shortest input terminates", OFFSET(shortest), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS },
     { NULL },
 };
@@ -129,11 +135,21 @@ static av_cold int init(AVFilterContext *ctx, const char *args)
 {
     OverlayContext *over = ctx->priv;
     static const char *shorthand[] = { "x", "y", NULL };
+    int ret;
 
     over->class = &overlay_class;
     av_opt_set_defaults(over);
 
-    return av_opt_set_from_string(over, args, shorthand, "=", ":");
+    ret = av_opt_set_from_string(over, args, shorthand, "=", ":");
+    if (ret < 0)
+        return ret;
+
+    if (over->allow_packed_rgb) {
+        av_log(ctx, AV_LOG_WARNING,
+               "The rgb option is deprecated and is overriding the format option, use format instead\n");
+        over->format = OVERLAY_FORMAT_RGB;
+    }
+    return 0;
 }
 
 static av_cold void uninit(AVFilterContext *ctx)
@@ -155,15 +171,20 @@ static int query_formats(AVFilterContext *ctx)
     OverlayContext *over = ctx->priv;
 
     /* overlay formats contains alpha, for avoiding conversion with alpha information loss */
-    static const enum AVPixelFormat main_pix_fmts_yuv[] = {
-        AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVA420P,
-        AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVA444P,
-        AV_PIX_FMT_NONE
+    static const enum AVPixelFormat main_pix_fmts_yuv420[] = {
+        AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVA420P, AV_PIX_FMT_NONE
     };
-    static const enum AVPixelFormat overlay_pix_fmts_yuv[] = {
-        AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA444P,
-        AV_PIX_FMT_NONE
+    static const enum AVPixelFormat overlay_pix_fmts_yuv420[] = {
+        AV_PIX_FMT_YUVA420P, AV_PIX_FMT_NONE
+    };
+
+    static const enum AVPixelFormat main_pix_fmts_yuv444[] = {
+        AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVA444P, AV_PIX_FMT_NONE
     };
+    static const enum AVPixelFormat overlay_pix_fmts_yuv444[] = {
+        AV_PIX_FMT_YUVA444P, AV_PIX_FMT_NONE
+    };
+
     static const enum AVPixelFormat main_pix_fmts_rgb[] = {
         AV_PIX_FMT_ARGB,  AV_PIX_FMT_RGBA,
         AV_PIX_FMT_ABGR,  AV_PIX_FMT_BGRA,
@@ -179,12 +200,19 @@ static int query_formats(AVFilterContext *ctx)
     AVFilterFormats *main_formats;
     AVFilterFormats *overlay_formats;
 
-    if (over->allow_packed_rgb) {
+    switch (over->format) {
+    case OVERLAY_FORMAT_YUV420:
+        main_formats    = ff_make_format_list(main_pix_fmts_yuv420);
+        overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420);
+        break;
+    case OVERLAY_FORMAT_YUV444:
+        main_formats    = ff_make_format_list(main_pix_fmts_yuv444);
+        overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444);
+        break;
+    case OVERLAY_FORMAT_RGB:
         main_formats    = ff_make_format_list(main_pix_fmts_rgb);
         overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb);
-    } else {
-        main_formats    = ff_make_format_list(main_pix_fmts_yuv);
-        overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv);
+        break;
     }
 
     ff_formats_ref(main_formats,    &ctx->inputs [MAIN   ]->out_formats);
-- 
1.7.9.5