[FFmpeg-devel] [PATCH] Allow exporting of RGB and BGR images to CUDA.

[dpeeters at MRU.MEDICAL.CANON]

Use the step size when calculating the number of channels to allow for more than two channels per plane.  This allows the use of AV_PIX_FMT_0BGR32 when using av_hwframe_transfer_data to transfer data from a Vulkan frame to a CUDA frame.

Signed-off-by: David Peeters <dpeeters at MRU.MEDICAL.CANON>
---
 libavutil/hwcontext_vulkan.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
index 237caa4bc0..05e8fc5268 100644
--- a/libavutil/hwcontext_vulkan.c
+++ b/libavutil/hwcontext_vulkan.c
@@ -2992,6 +2992,8 @@ static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
     VulkanDevicePriv *p = ctx->internal->priv;
     FFVulkanFunctions *vk = &p->vkfn;
+    int max_pixsteps[4];
+    int max_pixstep_comps[4];
 
     AVHWFramesContext *cuda_fc = (AVHWFramesContext*)cuda_hwfc->data;
     AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
@@ -3001,6 +3003,8 @@ static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
     CUarray_format cufmt = desc->comp[0].depth > 8 ? CU_AD_FORMAT_UNSIGNED_INT16 :
                                                      CU_AD_FORMAT_UNSIGNED_INT8;
 
+    av_image_fill_max_pixsteps(max_pixsteps, max_pixstep_comps, desc);
+
     dst_f = (AVVkFrame *)frame->data[0];
 
     dst_int = dst_f->internal;
@@ -3023,7 +3027,9 @@ static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
                 .arrayDesc = {
                     .Depth = 0,
                     .Format = cufmt,
-                    .NumChannels = 1 + ((planes == 2) && i),
+                    .NumChannels = desc->comp[max_pixstep_comps[i]].depth > 8
+                        ? max_pixsteps[i] / 2
+                        : max_pixsteps[i],
                     .Flags = 0,
                 },
                 .numLevels = 1,
-- 
2.36.1