[FFmpeg-devel] [PATCH] hwcontext_vaapi: implement auto connecting to a wayland display

[Rostislav Pehlivanov]

Allows Wayland users to use vaapi without having to specify a
vaapi drm device path.

Signed-off-by: Rostislav Pehlivanov <atomnuker at gmail.com>
---
 configure                   |  6 +++++-
 libavutil/hwcontext_vaapi.c | 29 +++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/configure b/configure
index 1d2fffa132..900abbbd11 100755
--- a/configure
+++ b/configure
@@ -2063,6 +2063,7 @@ SYSTEM_FUNCS="
 SYSTEM_LIBRARIES="
     vaapi_drm
     vaapi_x11
+    vaapi_wayland
     vdpau_x11
     wincrypt
 "
@@ -3347,7 +3348,7 @@ swscale_suggest="libm"
 
 avcodec_extralibs="pthreads_extralibs iconv_extralibs"
 avfilter_extralibs="pthreads_extralibs"
-avutil_extralibs="d3d11va_extralibs nanosleep_extralibs pthreads_extralibs vaapi_drm_extralibs vaapi_x11_extralibs vdpau_x11_extralibs"
+avutil_extralibs="d3d11va_extralibs nanosleep_extralibs pthreads_extralibs vaapi_drm_extralibs vaapi_x11_extralibs vaapi_wayland_extralibs vdpau_x11_extralibs"
 
 # programs
 ffmpeg_deps="avcodec avfilter avformat swresample"
@@ -6162,6 +6163,9 @@ enabled vaapi &&
 enabled vaapi &&
     check_lib vaapi_x11 "va/va.h va/va_x11.h" vaGetDisplay -lva -lva-x11 -lX11
 
+enabled vaapi &&
+    check_lib vaapi_wayland "va/va.h va/va_wayland.h" vaGetDisplayWl -lva -lva-wayland -lwayland-client
+
 enabled vaapi &&
     check_cpp_condition "va/va.h" "VA_CHECK_VERSION(1, 0, 0)" &&
     enable vaapi_1
diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 29698d1b27..4039753dcb 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -24,6 +24,9 @@
 #if HAVE_VAAPI_DRM
 #   include <va/va_drm.h>
 #endif
+#if HAVE_VAAPI_WAYLAND
+#   include <va/va_wayland.h>
+#endif
 
 #if CONFIG_LIBDRM
 #   include <va/va_drmcommon.h>
@@ -54,6 +57,9 @@ typedef struct VAAPIDevicePriv {
 #if HAVE_VAAPI_X11
     Display *x11_display;
 #endif
+#if HAVE_VAAPI_WAYLAND
+    struct wl_display *wayland_display;
+#endif
 
     int drm_fd;
 } VAAPIDevicePriv;
@@ -1196,6 +1202,10 @@ static void vaapi_device_free(AVHWDeviceContext *ctx)
     if (hwctx->display)
         vaTerminate(hwctx->display);
 
+#if HAVE_VAAPI_WAYLAND
+    if (priv->wayland_display)
+        wl_display_disconnect(priv->wayland_display);
+#endif
 #if HAVE_VAAPI_X11
     if (priv->x11_display)
         XCloseDisplay(priv->x11_display);
@@ -1264,6 +1274,25 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
     ctx->user_opaque = priv;
     ctx->free        = vaapi_device_free;
 
+#if HAVE_VAAPI_WAYLAND
+    if (!display && !(device && device[0] == '/')) {
+        priv->wayland_display = wl_display_connect(NULL);
+        if (!priv->wayland_display) {
+            av_log(ctx, AV_LOG_VERBOSE, "Cannot connect to a Wayland display\n");
+        } else {
+            display = vaGetDisplayWl(priv->wayland_display);
+            if (!display) {
+                av_log(ctx, AV_LOG_ERROR, "Cannot open a VA display "
+                       "from a Wayland display connection\n");
+                return AVERROR_UNKNOWN;
+            }
+
+            av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
+                   "a Wayland display connection\n");
+        }
+    }
+#endif
+
 #if HAVE_VAAPI_X11
     if (!display && !(device && device[0] == '/')) {
         // Try to open the device as an X11 display.
-- 
2.16.0.rc0.223.g4a4ac83678