[FFmpeg-devel] [PATCH] libavfilter/vf_overlay_qsv: Use format of first input to set output format for overlay_qsv

[Wenbin Chen]

overlay_qsv hard coded to use nv12 as output format. Now use the format
of the first input to set output format. Now the following command
works:

ffmpeg -hwaccel qsv -c:v hevc_qsv -i input_p010.265 -hwaccel qsv -c:v hevc_qsv \
-i input2_p010.265 -filter_complex "[0:v][1:v]overlay_qsv=x=0:y=0:alpha=255, \
hwdownload,format=p010le" -f rawvideo -y output_p010.yuv

Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
---
 libavfilter/vf_overlay_qsv.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libavfilter/vf_overlay_qsv.c b/libavfilter/vf_overlay_qsv.c
index 7e76b39aa9..d947a1faa1 100644
--- a/libavfilter/vf_overlay_qsv.c
+++ b/libavfilter/vf_overlay_qsv.c
@@ -276,6 +276,7 @@ static int config_output(AVFilterLink *outlink)
     int ret;
 
     av_log(ctx, AV_LOG_DEBUG, "Output is of %s.\n", av_get_pix_fmt_name(outlink->format));
+    vpp->qsv_param.out_sw_format = in0->format;
     if ((in0->format == AV_PIX_FMT_QSV && in1->format != AV_PIX_FMT_QSV) ||
         (in0->format != AV_PIX_FMT_QSV && in1->format == AV_PIX_FMT_QSV)) {
         av_log(ctx, AV_LOG_ERROR, "Mixing hardware and software pixel formats is not supported.\n");
@@ -288,6 +289,7 @@ static int config_output(AVFilterLink *outlink)
             av_log(ctx, AV_LOG_ERROR, "Inputs with different underlying QSV devices are forbidden.\n");
             return AVERROR(EINVAL);
         }
+        vpp->qsv_param.out_sw_format = hw_frame0->sw_format;
     }
 
     outlink->w          = vpp->var_values[VAR_MW];
-- 
2.25.1