[FFmpeg-devel] [PATCH v2] libavfilter/vf_overlay_qsv: Use format of first input to set output format for overlay_qsv

[Wenbin Chen]

overlay_qsv hard coded to use nv12 as output format. Now use the format
of the first input to set output format.

For detailed information of supported format on different platform,
please see the "composition" rows in "Video Processing Features" at
below link:
https://www.intel.com/content/www/us/en/develop/documentation/media-capabilities-of-intel-hardware/top.html

Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
---
 libavfilter/vf_overlay_qsv.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libavfilter/vf_overlay_qsv.c b/libavfilter/vf_overlay_qsv.c
index 7e76b39aa9..d947a1faa1 100644
--- a/libavfilter/vf_overlay_qsv.c
+++ b/libavfilter/vf_overlay_qsv.c
@@ -276,6 +276,7 @@ static int config_output(AVFilterLink *outlink)
     int ret;
 
     av_log(ctx, AV_LOG_DEBUG, "Output is of %s.\n", av_get_pix_fmt_name(outlink->format));
+    vpp->qsv_param.out_sw_format = in0->format;
     if ((in0->format == AV_PIX_FMT_QSV && in1->format != AV_PIX_FMT_QSV) ||
         (in0->format != AV_PIX_FMT_QSV && in1->format == AV_PIX_FMT_QSV)) {
         av_log(ctx, AV_LOG_ERROR, "Mixing hardware and software pixel formats is not supported.\n");
@@ -288,6 +289,7 @@ static int config_output(AVFilterLink *outlink)
             av_log(ctx, AV_LOG_ERROR, "Inputs with different underlying QSV devices are forbidden.\n");
             return AVERROR(EINVAL);
         }
+        vpp->qsv_param.out_sw_format = hw_frame0->sw_format;
     }
 
     outlink->w          = vpp->var_values[VAR_MW];
-- 
2.32.0