[FFmpeg-devel] [PATCH 2/4] libavutil/hwcontext_qsv: fix a bug when malloc handle_pairs_internal

[Wenbin Chen]

This commandline cause core dumped:
ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 \
-hwaccel_output_format vaapi -i input.264 \
-vf "hwmap=derive_device=qsv,format=qsv" \
-c:v h264_qsv output.264

reason: We use nb_surfaces to assign surface to handle_pairs_internal
but handle_pairs_internal is alloced with the size of init_pool_size.
This lead to access to illegal address.

Now change it to use nb_surfaces to allocate handle_pairs_internal and the
core dumped error is unseen. Also change D3D11VA to use nb_surfaces
to align to VAAPI and DXVA2.

Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
---
 libavutil/hwcontext_qsv.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/libavutil/hwcontext_qsv.c b/libavutil/hwcontext_qsv.c
index c18747f7eb..5a285fd25b 100644
--- a/libavutil/hwcontext_qsv.c
+++ b/libavutil/hwcontext_qsv.c
@@ -1123,8 +1123,7 @@ static int qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
     case AV_HWDEVICE_TYPE_VAAPI:
         {
             AVVAAPIFramesContext *src_hwctx = src_ctx->hwctx;
-            s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
-                                                 sizeof(*s->handle_pairs_internal));
+            s->handle_pairs_internal = av_calloc(src_hwctx->nb_surfaces, sizeof(*s->handle_pairs_internal));
             if (!s->handle_pairs_internal)
                 return AVERROR(ENOMEM);
             s->surfaces_internal = av_calloc(src_hwctx->nb_surfaces,
@@ -1146,15 +1145,15 @@ static int qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
     case AV_HWDEVICE_TYPE_D3D11VA:
         {
             AVD3D11VAFramesContext *src_hwctx = src_ctx->hwctx;
-            s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
+            s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces,
                                                  sizeof(*s->handle_pairs_internal));
             if (!s->handle_pairs_internal)
                 return AVERROR(ENOMEM);
-            s->surfaces_internal = av_calloc(src_ctx->initial_pool_size,
+            s->surfaces_internal = av_calloc(src_ctx->nb_surfaces,
                                              sizeof(*s->surfaces_internal));
             if (!s->surfaces_internal)
                 return AVERROR(ENOMEM);
-            for (i = 0; i < src_ctx->initial_pool_size; i++) {
+            for (i = 0; i < src_ctx->nb_surfaces; i++) {
                 qsv_init_surface(dst_ctx, &s->surfaces_internal[i]);
                 s->handle_pairs_internal[i].first = (mfxMemId)src_hwctx->texture_infos[i].texture;
                 if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) {
@@ -1164,7 +1163,7 @@ static int qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
                 }
                 s->surfaces_internal[i].Data.MemId = (mfxMemId)&s->handle_pairs_internal[i];
             }
-            dst_hwctx->nb_surfaces = src_ctx->initial_pool_size;
+            dst_hwctx->nb_surfaces = src_ctx->nb_surfaces;
             if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) {
                 dst_hwctx->frame_type |= MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET;
             } else {
@@ -1177,7 +1176,7 @@ static int qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
     case AV_HWDEVICE_TYPE_DXVA2:
         {
             AVDXVA2FramesContext *src_hwctx = src_ctx->hwctx;
-            s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
+            s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces,
                                                  sizeof(*s->handle_pairs_internal));
             if (!s->handle_pairs_internal)
                 return AVERROR(ENOMEM);
-- 
2.25.1