[FFmpeg-devel] [PATCH v2] avutil/hwcontext_vaapi: fix SEGV in vaTerminate when vaInitialize fails

[Aman Gupta]

From: Aman Gupta <aman at tmm1.net>

Program terminated with signal SIGSEGV, Segmentation fault.
  #0  0x0000000000aff8a4 in vaTerminate ()
  #1  0x0000000000ae50ce in vaapi_device_free (ctx=<optimized out>) at libavutil/hwcontext_vaapi.c:882
  #2  0x0000000000ae1f9e in hwdevice_ctx_free (opaque=<optimized out>, data=<optimized out>) at libavutil/hwcontext.c:66
  #3  0x0000000000ad856f in buffer_replace (src=0x0, dst=0x7fffa26ef1b8) at libavutil/buffer.c:119
  #4  av_buffer_unref (buf=buf at entry=0x7fffa26ef1f8) at libavutil/buffer.c:129
  #5  0x0000000000ae299f in av_hwdevice_ctx_create (pdevice_ref=0x170ac50 <hw_device_ctx>, type=type at entry=AV_HWDEVICE_TYPE_VAAPI, device=<optimized out>,
      opts=opts at entry=0x0, flags=flags at entry=0) at libavutil/hwcontext.c:494
  #6  0x0000000000400968 in vaapi_device_init (device=<optimized out>) at ffmpeg_vaapi.c:223
---
 libavutil/hwcontext_vaapi.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 6176bdc..0051acb 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -961,14 +961,13 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
         return AVERROR(EINVAL);
     }
 
-    hwctx->display = display;
-
     vas = vaInitialize(display, &major, &minor);
     if (vas != VA_STATUS_SUCCESS) {
         av_log(ctx, AV_LOG_ERROR, "Failed to initialise VAAPI "
                "connection: %d (%s).\n", vas, vaErrorStr(vas));
         return AVERROR(EIO);
     }
+    hwctx->display = display;
     av_log(ctx, AV_LOG_VERBOSE, "Initialised VAAPI connection: "
            "version %d.%d\n", major, minor);
 
-- 
2.10.1 (Apple Git-78)