[FFmpeg-devel] [PATCH V5 3/5] libavutil/hwcontext_vulkan: Allocate vkFrame in one memory
Lynne
dev at lynne.ee
Thu Dec 2 12:22:00 EET 2021
2 Dec 2021, 02:53 by wenbin.chen at intel.com:
>> The vaapi can import external frame, but the planes of the external
>> frames should be in the same drm object. A new option "contiguous_planes"
>> is added to device. This flag tells device to allocate places in one
>> memory. When device is derived from vaapi this flag will be enabled.
>> A new flag frame_flag is also added to AVVulkanFramesContext. User
>> can use this flag to force enable or disable this behaviour.
>> A new variable "offset "is added to AVVKFrame. It describe describe the
>> offset from the memory currently bound to the VkImage.
>>
>> Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
>> ---
>> libavutil/hwcontext_vulkan.c | 68
>> +++++++++++++++++++++++++++++++++++-
>> libavutil/hwcontext_vulkan.h | 24 +++++++++++++
>> 2 files changed, 91 insertions(+), 1 deletion(-)
>>
>> diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
>> index a0437c9661..eef9009ae1 100644
>> --- a/libavutil/hwcontext_vulkan.c
>> +++ b/libavutil/hwcontext_vulkan.c
>> @@ -103,8 +103,14 @@ typedef struct VulkanDevicePriv {
>> /* Settings */
>> int use_linear_images;
>>
>> + /* allocate planes in a contiguous memory */
>> + int contiguous_planes;
>> +
>> /* Nvidia */
>> int dev_is_nvidia;
>> +
>> + /* Intel */
>> + int dev_is_intel;
>> } VulkanDevicePriv;
>>
>> typedef struct VulkanFramesPriv {
>> @@ -153,6 +159,8 @@ typedef struct AVVkFrameInternal {
>> av_free((void *)props); \
>> }
>>
>> +#define VKF_FLAG(x, f) (((x) & (~AV_VK_FRAME_FLAG_NONE)) & (f))
>> +
>> static const struct {
>> enum AVPixelFormat pixfmt;
>> const VkFormat vkfmts[4];
>> @@ -1374,6 +1382,13 @@ static int
>> vulkan_device_create_internal(AVHWDeviceContext *ctx,
>> if (opt_d)
>> p->use_linear_images = strtol(opt_d->value, NULL, 10);
>>
>> + opt_d = av_dict_get(opts, "contiguous_planes", NULL, 0);
>> + if (opt_d)
>> + p->contiguous_planes = strtol(opt_d->value, NULL, 10);
>> + else
>> + p->contiguous_planes = -1;
>> +
>> +
>> hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames;
>> hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount;
>>
>> @@ -1425,6 +1440,8 @@ static int vulkan_device_init(AVHWDeviceContext
>> *ctx)
>>
>> p->dev_is_nvidia = (p->props.properties.vendorID == 0x10de);
>>
>> + p->dev_is_intel = (p->props.properties.vendorID == 0x8086);
>> +
>> vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev,
>> &queue_num, NULL);
>> if (!queue_num) {
>> av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
>> @@ -1742,8 +1759,12 @@ static int alloc_bind_mem(AVHWFramesContext
>> *hwfc, AVVkFrame *f,
>> AVHWDeviceContext *ctx = hwfc->device_ctx;
>> VulkanDevicePriv *p = ctx->internal->priv;
>> FFVulkanFunctions *vk = &p->vkfn;
>> + AVVulkanFramesContext *hwfctx = hwfc->hwctx;
>> const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
>> VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } };
>> + VkMemoryRequirements memory_requirements = { 0 };
>> + int mem_size = 0;
>> + int mem_size_list[AV_NUM_DATA_POINTERS] = { 0 };
>>
>> AVVulkanDeviceContext *hwctx = ctx->hwctx;
>>
>> @@ -1771,6 +1792,19 @@ static int alloc_bind_mem(AVHWFramesContext
>> *hwfc, AVVkFrame *f,
>> req.memoryRequirements.size =
>> FFALIGN(req.memoryRequirements.size,
>> p-
>> >props.properties.limits.minMemoryMapAlignment);
>>
>> + if (VKF_FLAG(hwfctx->flags,
>> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) {
>> + if (memory_requirements.size == 0) {
>> + memory_requirements = req.memoryRequirements;
>> + } else if (memory_requirements.memoryTypeBits !=
>> req.memoryRequirements.memoryTypeBits) {
>> + av_log(hwfc, AV_LOG_ERROR, "the param for each planes are not
>> the same\n");
>> + return AVERROR(EINVAL);
>> + }
>> +
>> + mem_size_list[i] = req.memoryRequirements.size;
>> + mem_size += mem_size_list[i];
>> + continue;
>> + }
>> +
>> /* In case the implementation prefers/requires dedicated allocation */
>> use_ded_mem = ded_req.prefersDedicatedAllocation |
>> ded_req.requiresDedicatedAllocation;
>> @@ -1792,6 +1826,29 @@ static int alloc_bind_mem(AVHWFramesContext
>> *hwfc, AVVkFrame *f,
>> bind_info[i].memory = f->mem[i];
>> }
>>
>> + if (VKF_FLAG(hwfctx->flags,
>> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) {
>> + memory_requirements.size = mem_size;
>> +
>> + /* Allocate memory */
>> + if ((err = alloc_mem(ctx, &memory_requirements,
>> + f->tiling == VK_IMAGE_TILING_LINEAR ?
>> + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT :
>> + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
>> + (void *)(((uint8_t *)alloc_pnext)),
>> + &f->flags, &f->mem[0])))
>> + return err;
>> +
>> + f->size[0] = memory_requirements.size;
>> +
>> + for (int i = 0; i < planes; i++) {
>> + bind_info[i].sType =
>> VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
>> + bind_info[i].image = f->img[i];
>> + bind_info[i].memory = f->mem[0];
>> + bind_info[i].memoryOffset = i == 0 ? 0 : mem_size_list[i-1];
>> + f->offset[i] = bind_info[i].memoryOffset;
>> + }
>> + }
>> +
>> /* Bind the allocated memory to the images */
>> ret = vk->BindImageMemory2(hwctx->act_dev, planes, bind_info);
>> if (ret != VK_SUCCESS) {
>> @@ -2154,6 +2211,12 @@ static int
>> vulkan_frames_init(AVHWFramesContext *hwfc)
>> if (!hwctx->usage)
>> hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS;
>>
>> + if (!(hwctx->flags & AV_VK_FRAME_FLAG_NONE)) {
>> + if (p->contiguous_planes == 1 ||
>> + ((p->contiguous_planes == -1) && p->dev_is_intel))
>> + hwctx->flags |= AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY;
>> + }
>> +
>> err = create_exec_ctx(hwfc, &fp->conv_ctx,
>> dev_hwctx->queue_family_comp_index,
>> dev_hwctx->nb_comp_queues);
>> @@ -3074,6 +3137,7 @@ static int
>> vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
>> FFVulkanFunctions *vk = &p->vkfn;
>> VulkanFramesPriv *fp = hwfc->internal->priv;
>> AVVulkanDeviceContext *hwctx = hwfc->device_ctx->hwctx;
>> + AVVulkanFramesContext *hwfctx = hwfc->hwctx;
>> const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
>> VkImageDrmFormatModifierPropertiesEXT drm_mod = {
>> .sType =
>> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
>> @@ -3142,7 +3206,9 @@ static int
>> vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
>> continue;
>>
>> vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub,
>> &layout);
>> - drm_desc->layers[i].planes[0].offset = layout.offset;
>> + drm_desc->layers[i].planes[0].offset = layout.offset +
>> + VKF_FLAG(hwfctx->flags,
>> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY) ?
>> + f->offset[i] : 0;
>> drm_desc->layers[i].planes[0].pitch = layout.rowPitch;
>> }
>>
>> diff --git a/libavutil/hwcontext_vulkan.h b/libavutil/hwcontext_vulkan.h
>> index fdf2a60156..acb4073e3d 100644
>> --- a/libavutil/hwcontext_vulkan.h
>> +++ b/libavutil/hwcontext_vulkan.h
>> @@ -35,6 +35,17 @@
>> * with the data pointer set to an AVVkFrame.
>> */
>>
>> +/**
>> + * Defines the behaviour of frame allocation
>> + * AV_VK_FRAME_FLAG_NONE: planes will be allocated in separte memory
>> + * AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY: planes will be allocated in
>> a
>> + * contiguous memory.
>> + */
>> +typedef enum {
>> + AV_VK_FRAME_FLAG_NONE = (1ULL << 0),
>> + AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY = (1ULL << 1) | 1ULL
>> +} AVVkFrameFlags;
>> +
>> /**
>> * Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
>> * All of these can be set before init to change what the context uses
>> @@ -165,6 +176,14 @@ typedef struct AVVulkanFramesContext {
>> * extensions are present in enabled_dev_extensions.
>> */
>> void *alloc_pnext[AV_NUM_DATA_POINTERS];
>> +
>> + /**
>> + * Is a combination of AVVkFrameFlags. Defines the behaviour of frame
>> + * allocation.
>> + * If no flag is set, then the flags are automatically determined
>> + * based on the device.
>> + */
>> + int flags;
>> } AVVulkanFramesContext;
>>
>> /*
>> @@ -230,6 +249,11 @@ typedef struct AVVkFrame {
>> * Internal data.
>> */
>> struct AVVkFrameInternal *internal;
>> +
>> + /**
>> + * Describe the offset from the memory currently bound to the VkImage.
>> + */
>> + size_t offset[AV_NUM_DATA_POINTERS];
>> } AVVkFrame;
>>
>> /**
>> --
>> 2.25.1
>>
>
> Hi Lynn:
> (I move my comment from patch V4 to here, in case you miss it)
> I made a mistake. I should use "layout.offset + f->offset[i]". I resubmit
> the patchset. Let's discuss under this patch.
>
> There are two offsets.
> The first offset relates to the base address of the memory. This offset is the
> memoryOffset we pass to vulkan when we bind memory.
> The second offset relates to the base address of the image. This offset is set by
> vulkan when we create vkImage.
>
> The offset we get from vkGetImageSubresourceLayout is the second offset.
> The offset drm_object need is the first offset.
> If one image is bind to one memory, these two offsets should be the same, because the
> start of the image is the start of the memory. If we bind multiple image to one memory,
> these two offsets are different.
> I don't find an API to get the second kind of offset, so I create a new variable to store them.
>
> In practice, these two offsets are different. When I allocate planes to one memory. The
> offsets I get from vkGetImageSubresourceLayout are all 0. However, I need to set it to 2088960
> to make command works. 2088960 is the memoryOffset value I pass to vulkan when I bind
> memory.
>
My question still stands: why can't the driver output the binding offset
for DRM-tiled images?
The spec explicitly allows for it, and RADV already outputs non-zero offset
via vkGetImageSubresourceLayout.
I'd rather not commit any driver-specific quirks because 2 internal
teams don't communicate with one another.
More information about the ffmpeg-devel
mailing list