[FFmpeg-devel] [PATCH v1] hwcontext_vaapi: Fix VAAPI initialization failure.

[zhoupeng]

Fix VA-API initialization failure caused by defaulting to VAAPI_DRM when hardware drivers only support VAAPI_X11.
Each VAAPI scheme checks driver support through the successful return of vaInitialize.If unsupported, it continues to the next scheme without immediate failure.
---
 libavutil/hwcontext_vaapi.c | 104 +++++++++++++++++++-----------------
 1 file changed, 56 insertions(+), 48 deletions(-)

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 95aa38d9d2..eacb7ae00f 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -1666,12 +1666,23 @@ static void vaapi_device_log_info(void *context, const char *message)
 #endif
 
 static int vaapi_device_connect(AVHWDeviceContext *ctx,
-                                VADisplay display)
+                                VADisplay display, char *driver_name)
 {
     AVVAAPIDeviceContext *hwctx = ctx->hwctx;
     int major, minor;
     VAStatus vas;
 
+#if VA_CHECK_VERSION(0, 38, 0)
+    if (driver_name) {
+        vas = vaSetDriverName(display, driver_name);
+        if (vas != VA_STATUS_SUCCESS) {
+            av_log(ctx, AV_LOG_ERROR, "Failed to set driver name to "
+                   "%s: %d (%s).\n", driver_name, vas, vaErrorStr(vas));
+            return AVERROR_EXTERNAL;
+        }
+    }
+#endif
+
 #if CONFIG_VAAPI_1
     vaSetErrorCallback(display, &vaapi_device_log_error, ctx);
     vaSetInfoCallback (display, &vaapi_device_log_info,  ctx);
@@ -1698,6 +1709,8 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
     VADisplay display = NULL;
     const AVDictionaryEntry *ent;
     int try_drm, try_x11, try_win32, try_all;
+    char *driver_name = NULL;
+    int ret = -1;
 
     priv = av_mallocz(sizeof(*priv));
     if (!priv)
@@ -1729,8 +1742,14 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
         try_win32 = HAVE_VAAPI_WIN32;
     }
 
+#if VA_CHECK_VERSION(0, 38, 0)
+    ent = av_dict_get(opts, "driver", NULL, 0);
+    if (ent)
+        driver_name = ent->value;
+#endif
+
 #if HAVE_VAAPI_DRM
-    while (!display && try_drm) {
+    while ((ret < 0) && try_drm) {
         // If the device is specified, try to open it as a DRM device node.
         // If not, look for a usable render node, possibly restricted to those
         // using a specified kernel driver.
@@ -1844,17 +1863,23 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
         }
 
         display = vaGetDisplayDRM(priv->drm_fd);
-        if (!display) {
+        if (display) {
+            ret = vaapi_device_connect(ctx, display, driver_name);
+            if (ret < 0) {
+                close(priv->drm_fd);
+                priv->drm_fd = -1;
+                vaTerminate(display);
+                display = NULL;
+            }
+        } else
             av_log(ctx, AV_LOG_VERBOSE, "Cannot open a VA display "
                    "from DRM device %s.\n", device);
-            return AVERROR_EXTERNAL;
-        }
         break;
     }
 #endif
 
 #if HAVE_VAAPI_X11
-    if (!display && try_x11) {
+    if ((ret < 0) && try_x11) {
         // Try to open the device as an X11 display.
         priv->x11_display = XOpenDisplay(device);
         if (!priv->x11_display) {
@@ -1862,20 +1887,26 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
                    "%s.\n", XDisplayName(device));
         } else {
             display = vaGetDisplay(priv->x11_display);
-            if (!display) {
+            if (display) {
+                av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
+                       "X11 display %s.\n", XDisplayName(device));
+
+                ret = vaapi_device_connect(ctx, display, driver_name);
+                if (ret < 0) {
+                    vaTerminate(display);
+                    XCloseDisplay(priv->x11_display);
+                    display = NULL;
+                    priv->x11_display = NULL;
+                }
+            } else
                 av_log(ctx, AV_LOG_ERROR, "Cannot open a VA display "
                        "from X11 display %s.\n", XDisplayName(device));
-                return AVERROR_UNKNOWN;
-            }
-
-            av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
-                   "X11 display %s.\n", XDisplayName(device));
         }
     }
 #endif
 
 #if HAVE_VAAPI_WIN32
-    if (!display && try_win32) {
+    if ((ret < 0) && try_win32) {
         // Try to create a display from the specified device, if any.
         if (!device) {
             display = vaGetDisplayWin32(NULL);
@@ -1925,45 +1956,22 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
 #endif
         }
 
-        if (!display) {
+        if (display) {
+            av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
+                    "Win32 display.\n");
+
+            ret = vaapi_device_connect(ctx, display, driver_name);
+            if (ret < 0) {
+                vaTerminate(display);
+                display = NULL;
+            }
+        } else
             av_log(ctx, AV_LOG_ERROR, "Cannot open a VA display "
                     "from Win32 display.\n");
-            return AVERROR_UNKNOWN;
-        }
-
-        av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
-                "Win32 display.\n");
-    }
-#endif
-
-    if (!display) {
-        if (device)
-            av_log(ctx, AV_LOG_ERROR, "No VA display found for "
-                   "device %s.\n", device);
-        else
-            av_log(ctx, AV_LOG_ERROR, "No VA display found for "
-                   "any default device.\n");
-        return AVERROR(EINVAL);
     }
-
-    ent = av_dict_get(opts, "driver", NULL, 0);
-    if (ent) {
-#if VA_CHECK_VERSION(0, 38, 0)
-        VAStatus vas;
-        vas = vaSetDriverName(display, ent->value);
-        if (vas != VA_STATUS_SUCCESS) {
-            av_log(ctx, AV_LOG_ERROR, "Failed to set driver name to "
-                   "%s: %d (%s).\n", ent->value, vas, vaErrorStr(vas));
-            vaTerminate(display);
-            return AVERROR_EXTERNAL;
-        }
-#else
-        av_log(ctx, AV_LOG_WARNING, "Driver name setting is not "
-               "supported with this VAAPI version.\n");
 #endif
-    }
 
-    return vaapi_device_connect(ctx, display);
+    return ret;
 }
 
 static int vaapi_device_derive(AVHWDeviceContext *ctx,
@@ -2048,7 +2056,7 @@ static int vaapi_device_derive(AVHWDeviceContext *ctx,
             return AVERROR(EIO);
         }
 
-        return vaapi_device_connect(ctx, display);
+        return vaapi_device_connect(ctx, display, NULL);
     }
 #endif
     return AVERROR(ENOSYS);
-- 
2.20.1