[FFmpeg-devel] [PATCH V5 5/5] libavutil/hwcontext_vulkan: specify the modifier to create VKImage
Chen, Wenbin
wenbin.chen at intel.com
Mon Dec 6 07:48:42 EET 2021
> 2 Dec 2021, 02:43 by wenbin.chen at intel.com:
>
> > When vulkan image exports to drm, the tilling need to be
> > VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to
> create vulkan
> > image using this format.
> >
> > Now the following command line works:
> >
> > ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -
> hwaccel_output_format \
> > vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan,
> \
> > scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v
> h264_vaapi output.264
> >
> > Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
> > ---
> > libavutil/hwcontext_vulkan.c | 127
> +++++++++++++++++++++++++++++++++--
> > 1 file changed, 121 insertions(+), 6 deletions(-)
> >
> > diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
> > index f980b72720..8224c0d4e4 100644
> > --- a/libavutil/hwcontext_vulkan.c
> > +++ b/libavutil/hwcontext_vulkan.c
> > @@ -120,6 +120,9 @@ typedef struct VulkanFramesPriv {
> > /* Image transfers */
> > VulkanExecCtx upload_ctx;
> > VulkanExecCtx download_ctx;
> > +
> > + /*modifier info*/
> > + VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
> > } VulkanFramesPriv;
> >
> > typedef struct AVVkFrameInternal {
> > @@ -242,6 +245,28 @@ const VkFormat *av_vkfmt_from_pixfmt(enum
> AVPixelFormat p)
> > return NULL;
> > }
> >
> > +static void *find_in_structure_list(VkBaseOutStructure *stru_list,
> VkStructureType sType) {
> > + if (!stru_list)
> > + return NULL;
> > +
> > + for(;stru_list;stru_list = stru_list->pNext)
> > + if (stru_list->sType == sType)
> > + return stru_list;
> > +
> > + return NULL;
> > +}
> > +
> > +static void append_to_structure_list(VkBaseOutStructure **stru_list,
> VkBaseOutStructure *added_stru) {
> > + VkBaseOutStructure *p;
> > + if (!*stru_list) {
> > + *stru_list = added_stru;
> > + return;
> > + }
> > + for(p = *stru_list; p->pNext; p = p->pNext);
> > + p->pNext = added_stru;
> > + return;
> > +}
> >
>
> Could you tidy these 2 functions up? Also, code style issues, as usual.
> Take a look at libplacebo's functions or copy them, they're much
> neater: https://0x1.st/tx.txt
>
>
> > static int pixfmt_is_supported(AVHWDeviceContext *dev_ctx, enum
> AVPixelFormat p,
> > int linear)
> > {
> > @@ -2094,6 +2119,10 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> > AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
> > VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> > FFVulkanFunctions *vk = &p->vkfn;
> > + const int has_modifiers = hwctx->tiling ==
> VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
> > + int loop_count;
> > + VkImageDrmFormatModifierListCreateInfoEXT *modifier_info =
> find_in_structure_list(hwctx->create_pnext,
> > +
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT);
> > VkExternalImageFormatProperties eprops = {
> > .sType =
> VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
> > };
> > @@ -2101,9 +2130,18 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> > .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
> > .pNext = &eprops,
> > };
> > + VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info
> = {
> > + .sType =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER
> _INFO_EXT,
> > + .pNext = NULL,
> > + .pQueueFamilyIndices = p->qfs,
> > + .queueFamilyIndexCount = p->num_qfs,
> > + .sharingMode = p->num_qfs > 1 ?
> VK_SHARING_MODE_CONCURRENT :
> > + VK_SHARING_MODE_EXCLUSIVE,
> > + };
> > VkPhysicalDeviceExternalImageFormatInfo enext = {
> > .sType =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
> > .handleType = exp,
> > + .pNext = has_modifiers && modifier_info ? &phy_dev_mod_info :
> NULL,
> > };
> > VkPhysicalDeviceImageFormatInfo2 pinfo = {
> > .sType =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
> > @@ -2115,11 +2153,16 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> > .flags = VK_IMAGE_CREATE_ALIAS_BIT,
> > };
> >
> > - ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx-
> >phys_dev,
> > - &pinfo, &props);
> > - if (ret == VK_SUCCESS) {
> > - *iexp |= exp;
> > - *comp_handle_types |=
> eprops.externalMemoryProperties.compatibleHandleTypes;
> > + loop_count = has_modifiers && modifier_info ? modifier_info-
> >drmFormatModifierCount : 1;
> > + for (int i = 0; i < loop_count; i++) {
> > + if (has_modifiers && modifier_info)
> > + phy_dev_mod_info.drmFormatModifier = modifier_info-
> >pDrmFormatModifiers[i];
> > + ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx-
> >phys_dev,
> > + &pinfo, &props);
> > + if (ret == VK_SUCCESS) {
> > + *iexp |= exp;
> > + *comp_handle_types |=
> eprops.externalMemoryProperties.compatibleHandleTypes;
> > + }
> > }
> > }
> >
> > @@ -2190,6 +2233,12 @@ static void
> vulkan_frames_uninit(AVHWFramesContext *hwfc)
> > {
> > VulkanFramesPriv *fp = hwfc->internal->priv;
> >
> > + if (fp->modifier_info) {
> > + if (fp->modifier_info->pDrmFormatModifiers)
> > + av_freep(&fp->modifier_info->pDrmFormatModifiers);
> > + av_freep(&fp->modifier_info);
> > + }
> > +
> > free_exec_ctx(hwfc, &fp->conv_ctx);
> > free_exec_ctx(hwfc, &fp->upload_ctx);
> > free_exec_ctx(hwfc, &fp->download_ctx);
> > @@ -2203,11 +2252,77 @@ static int
> vulkan_frames_init(AVHWFramesContext *hwfc)
> > VulkanFramesPriv *fp = hwfc->internal->priv;
> > AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
> > VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> > + VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
> > + const int has_modifiers = !!(p->extensions &
> FF_VK_EXT_DRM_MODIFIER_FLAGS);
> >
> > /* Default pool flags */
> > - hwctx->tiling = hwctx->tiling ? hwctx->tiling : p->use_linear_images ?
> > + hwctx->tiling = hwctx->tiling ? hwctx->tiling : has_modifiers ?
> > + VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT : p-
> >use_linear_images ?
> > VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL;
> >
> > + modifier_info = find_in_structure_list(hwctx->create_pnext,
> > +
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT);
> > + /* get the supported modifier */
> > + if (has_modifiers && !modifier_info) {
> > + const VkFormat *fmt = av_vkfmt_from_pixfmt(hwfc->sw_format);
> > + FFVulkanFunctions *vk = &p->vkfn;
> > + VkDrmFormatModifierPropertiesEXT *mod_props;
> > + uint64_t *modifiers;
> > + int modifier_count = 0;
> > +
> > + VkDrmFormatModifierPropertiesListEXT mod_props_list = {
> > + .sType =
> VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
> > + .pNext = NULL,
> > + .drmFormatModifierCount = 0,
> > + .pDrmFormatModifierProperties = NULL,
> > + };
> > + VkFormatProperties2 prop = {
> > + .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
> > + .pNext = &mod_props_list,
> > + };
> > + vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev,
> fmt[0], &prop);
> > + if (!mod_props_list.drmFormatModifierCount) {
> > + av_log(hwfc, AV_LOG_ERROR, "There are not supported modifiers
> for sw_format\n");
> >
>
> Grammar issues: should be something like
> "There are no supported modifiers for given sw_format\n".
>
>
> > + return AVERROR(EINVAL);
> > + }
> > +
> > + modifier_info = av_malloc(sizeof(*modifier_info));
> > + if (!modifier_info)
> > + return AVERROR(ENOMEM);
> > + append_to_structure_list((VkBaseOutStructure **)&hwctx-
> >create_pnext,
> > + (VkBaseOutStructure *)modifier_info);
> > + fp->modifier_info = modifier_info;
> > + modifier_info->pNext = NULL;
> > + modifier_info->sType =
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT;
> > +
> > + modifiers = av_malloc(mod_props_list.drmFormatModifierCount *
> > + sizeof(*modifiers));
> > + if (!modifiers)
> > + return AVERROR(ENOMEM);
> > + modifier_info->pDrmFormatModifiers = modifiers;
> > +
> > + mod_props = av_malloc(mod_props_list.drmFormatModifierCount *
> > + sizeof(*mod_props));
> > + if (!mod_props)
> > + return AVERROR(ENOMEM);
> > + mod_props_list.pDrmFormatModifierProperties = mod_props;
> > + vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev,
> fmt[0], &prop);
> > + for (int i = 0; i < mod_props_list.drmFormatModifierCount; i++) {
> > + if (!(mod_props[i].drmFormatModifierTilingFeatures &
> FF_VK_DEFAULT_USAGE_FLAGS))
> >
>
> You should use the usage flags given by the user in AVVulkanFramesContext,
> not the default ones (unless none were specified by the user, but in that case,
> that field is already set to the default usage flags).
>
>
> > + continue;
> > + modifiers[modifier_count++] = mod_props[i].drmFormatModifier;
> > + }
> > + if (!modifier_count) {
> > + av_log(hwfc, AV_LOG_ERROR, "The supported modifiers doesn't
> support "
> > + "default usage\n");
> >
>
> Same, should be something like:
> "None of the given modifiers supports the usage flags!"
>
>
> > + av_freep(&mod_props);
> > + return AVERROR(EINVAL);
> > + }
> > + modifier_info->drmFormatModifierCount = modifier_count;
> > + av_freep(&mod_props);
> > + }
> > +
> > +
> > if (!hwctx->usage)
> > hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS;
> >
> > --
> > 2.25.1
> >
Thanks for review
I will fix this problem in next patchset.
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel at ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
> >
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
More information about the ffmpeg-devel
mailing list