[FFmpeg-devel] [PATCH 1/2] libavutil/hwcontext_vulkan: Fix VK_FORMAT_R8G8_UNORM and VK_FORMAT_R16G16_UNORM map problem on Vulkan

[Wenbin Chen]

Vulkan map both DRM_FORMAT_RG88 and DRM_FORMAT_GR88 to
VK_FORMAT_R8G8_UNORM. This cannot distinguish nv12/nv24 and nv21/nv42.
Vulkan also map both DRM_FORMAT_RG1616 and DRM_FORMAT_GR1616 to
VK_FORMAT_R16G16_UNORM. This causes issue when map back.
VK_FORMAT_R16G16 will be mapped to DRM_FORMAT_GR1616, while p010 need
DRM_FORMAT_RG1616.
Add sw_format check to vulkan_fmt_to_drm() to fix this problem.

Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
---
 libavutil/hwcontext_vulkan.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
index 237caa4bc0..791649001f 100644
--- a/libavutil/hwcontext_vulkan.c
+++ b/libavutil/hwcontext_vulkan.c
@@ -3290,8 +3290,17 @@ static void vulkan_unmap_to_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
     av_free(drm_desc);
 }
 
-static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt)
+static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt,
+                                         enum AVPixelFormat sw_format)
 {
+    if (vkfmt == VK_FORMAT_R8G8_UNORM &&
+       (sw_format == AV_PIX_FMT_NV12 || sw_format == AV_PIX_FMT_NV24))
+            return DRM_FORMAT_RG88;
+
+    if (vkfmt == VK_FORMAT_R16G16_UNORM &&
+       (sw_format == AV_PIX_FMT_P010 || sw_format == AV_PIX_FMT_P016))
+            return DRM_FORMAT_RG1616;
+
     for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
         if (vulkan_drm_format_map[i].vk_format == vkfmt)
             return vulkan_drm_format_map[i].drm_fourcc;
@@ -3373,7 +3382,7 @@ static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
         };
         VkFormat plane_vkfmt = av_vkfmt_from_pixfmt(hwfc->sw_format)[i];
 
-        drm_desc->layers[i].format    = vulkan_fmt_to_drm(plane_vkfmt);
+        drm_desc->layers[i].format    = vulkan_fmt_to_drm(plane_vkfmt, hwfc->sw_format);
         drm_desc->layers[i].nb_planes = 1;
 
         if (drm_desc->layers[i].format == DRM_FORMAT_INVALID) {
-- 
2.32.0