[FFmpeg-devel] [PATCH v2 04/12] vulkan_decode: support software-defined decoders
Lynne
dev at lynne.ee
Mon Feb 24 10:04:17 EET 2025
---
libavcodec/vulkan_decode.c | 191 +++++++++++++++++++++++++++----------
libavcodec/vulkan_decode.h | 10 ++
2 files changed, 150 insertions(+), 51 deletions(-)
diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c
index c57998108c..084563e8e9 100644
--- a/libavcodec/vulkan_decode.c
+++ b/libavcodec/vulkan_decode.c
@@ -24,6 +24,9 @@
#include "libavutil/mem.h"
#include "libavutil/vulkan_loader.h"
+#define DECODER_IS_SDR(codec_id) \
+ ((codec_id) == AV_CODEC_ID_FFV1)
+
#if CONFIG_H264_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc;
#endif
@@ -63,7 +66,9 @@ static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx,
codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :
- 0;
+ VK_STRUCTURE_TYPE_MAX_ENUM;
+ if (profile_struct_type == VK_STRUCTURE_TYPE_MAX_ENUM)
+ return NULL;
profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
@@ -119,13 +124,26 @@ static AVFrame *vk_get_dpb_pool(FFVulkanDecodeShared *ctx)
return avf;
}
+static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic)
+{
+ FFVulkanDecodeShared *ctx = dec->shared_ctx;
+ FFVulkanFunctions *vk = &ctx->s.vkfn;
+
+ vkpic->dpb_frame = NULL;
+ vkpic->img_view_ref = VK_NULL_HANDLE;
+ vkpic->img_view_out = VK_NULL_HANDLE;
+ vkpic->img_view_dest = VK_NULL_HANDLE;
+
+ vkpic->destroy_image_view = vk->DestroyImageView;
+ vkpic->wait_semaphores = vk->WaitSemaphores;
+}
+
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
FFVulkanDecodePicture *vkpic, int is_current,
int alloc_dpb)
{
int err;
FFVulkanDecodeShared *ctx = dec->shared_ctx;
- FFVulkanFunctions *vk = &ctx->s.vkfn;
vkpic->slices_size = 0;
@@ -134,13 +152,7 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
if (vkpic->img_view_ref)
return 0;
- vkpic->dpb_frame = NULL;
- vkpic->img_view_ref = VK_NULL_HANDLE;
- vkpic->img_view_out = VK_NULL_HANDLE;
- vkpic->img_view_dest = VK_NULL_HANDLE;
-
- vkpic->destroy_image_view = vk->DestroyImageView;
- vkpic->wait_semaphores = vk->WaitSemaphores;
+ init_frame(dec, vkpic);
if (ctx->common.layered_dpb && alloc_dpb) {
vkpic->img_view_ref = ctx->common.layered_view;
@@ -183,6 +195,53 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
return 0;
}
+int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic,
+ FFVulkanDecodePicture *vkpic, int is_current,
+ enum FFVkShaderRepFormat rep_fmt, int alloc_dpb)
+{
+ int err;
+ FFVulkanDecodeShared *ctx = dec->shared_ctx;
+
+ vkpic->slices_size = 0;
+
+ if (vkpic->img_view_ref)
+ return 0;
+
+ init_frame(dec, vkpic);
+
+ if (ctx->common.layered_dpb && alloc_dpb) {
+ vkpic->img_view_ref = ctx->common.layered_view;
+ vkpic->img_aspect_ref = ctx->common.layered_aspect;
+ } else if (alloc_dpb) {
+ vkpic->dpb_frame = vk_get_dpb_pool(ctx);
+ if (!vkpic->dpb_frame)
+ return AVERROR(ENOMEM);
+
+ err = ff_vk_create_imageview(&ctx->s,
+ &vkpic->img_view_ref, &vkpic->img_aspect_ref,
+ vkpic->dpb_frame, 0, rep_fmt);
+ if (err < 0)
+ return err;
+
+ vkpic->img_view_dest = vkpic->img_view_ref;
+ }
+
+ if (!alloc_dpb || is_current) {
+ err = ff_vk_create_imageview(&ctx->s,
+ &vkpic->img_view_out, &vkpic->img_aspect,
+ pic, 0, rep_fmt);
+ if (err < 0)
+ return err;
+
+ if (!alloc_dpb) {
+ vkpic->img_view_ref = vkpic->img_view_out;
+ vkpic->img_aspect_ref = vkpic->img_aspect;
+ }
+ }
+
+ return 0;
+}
+
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,
const uint8_t *data, size_t size, int add_startcode,
uint32_t *nb_slices, const uint32_t **offsets)
@@ -223,9 +282,14 @@ int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,
buf_size = 2 << av_log2(buf_size);
err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref,
+ DECODER_IS_SDR(avctx->codec_id) ?
+ (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
+ VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) :
VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,
ctx->s.hwfc->create_pnext, buf_size,
- VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
+ VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
+ (DECODER_IS_SDR(avctx->codec_id) ?
+ VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : 0x0));
if (err < 0)
return err;
@@ -276,6 +340,10 @@ void ff_vk_decode_flush(AVCodecContext *avctx)
VkCommandBuffer cmd_buf;
FFVkExecContext *exec;
+ /* Non-video queues do not need to be reset */
+ if (!(get_codecdesc(avctx->codec_id)->decode_op))
+ return;
+
exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
ff_vk_exec_start(&ctx->s, exec);
cmd_buf = exec->buf;
@@ -551,6 +619,7 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re
{
int err;
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
+ const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
AVVulkanDeviceContext *hwctx = device->hwctx;
@@ -569,11 +638,13 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re
ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
hwctx->nb_enabled_dev_extensions);
- if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
- av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
- VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
- av_refstruct_unref(&dec->shared_ctx);
- return AVERROR(ENOSYS);
+ if (vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
+ if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
+ av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
+ VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
+ av_refstruct_unref(&dec->shared_ctx);
+ return AVERROR(ENOSYS);
+ }
}
err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
@@ -927,53 +998,61 @@ static void free_profile_data(AVHWFramesContext *hwfc)
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
{
- VkFormat vkfmt;
+ VkFormat vkfmt = VK_FORMAT_UNDEFINED;
int err, dedicated_dpb;
AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
AVVulkanFramesContext *hwfc = frames_ctx->hwctx;
FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
- FFVulkanDecodeProfileData *prof;
- FFVulkanDecodeShared *ctx;
-
- frames_ctx->sw_format = AV_PIX_FMT_NONE;
+ FFVulkanDecodeProfileData *prof = NULL;
err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);
if (err < 0)
return err;
- prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
- if (!prof)
- return AVERROR(ENOMEM);
+ frames_ctx->sw_format = avctx->sw_pix_fmt;
- err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
- &frames_ctx->sw_format, &vkfmt,
- prof, &dedicated_dpb);
- if (err < 0) {
- av_free(prof);
- return err;
- }
+ if (!DECODER_IS_SDR(avctx->codec_id)) {
+ prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
+ if (!prof)
+ return AVERROR(ENOMEM);
+
+ err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
+ &frames_ctx->sw_format, &vkfmt,
+ prof, &dedicated_dpb);
+ if (err < 0) {
+ av_free(prof);
+ return err;
+ }
+
+ frames_ctx->user_opaque = prof;
+ frames_ctx->free = free_profile_data;
- frames_ctx->user_opaque = prof;
- frames_ctx->free = free_profile_data;
+ hwfc->create_pnext = &prof->profile_list;
+ }
frames_ctx->width = avctx->coded_width;
frames_ctx->height = avctx->coded_height;
frames_ctx->format = AV_PIX_FMT_VULKAN;
hwfc->format[0] = vkfmt;
- hwfc->create_pnext = &prof->profile_list;
hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
- VK_IMAGE_USAGE_SAMPLED_BIT |
- VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
+ VK_IMAGE_USAGE_SAMPLED_BIT;
- if (!dec->dedicated_dpb)
- hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
+ if (prof) {
+ FFVulkanDecodeShared *ctx;
- ctx = dec->shared_ctx;
- if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
- FF_VK_EXT_VIDEO_MAINTENANCE_1))
- hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
+ hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
+ if (!dec->dedicated_dpb)
+ hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
+
+ ctx = dec->shared_ctx;
+ if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
+ FF_VK_EXT_VIDEO_MAINTENANCE_1))
+ hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
+ } else if (DECODER_IS_SDR(avctx->codec_id)) {
+ hwfc->usage |= VK_IMAGE_USAGE_STORAGE_BIT;
+ }
return err;
}
@@ -1075,8 +1154,10 @@ int ff_vk_decode_init(AVCodecContext *avctx)
if (err < 0)
return err;
+ vk_desc = get_codecdesc(avctx->codec_id);
+
profile = get_video_profile(ctx, avctx->codec_id);
- if (!profile) {
+ if ((vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) && !profile) {
av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");
return AVERROR(EINVAL);
}
@@ -1109,9 +1190,11 @@ int ff_vk_decode_init(AVCodecContext *avctx)
if (err < 0)
goto fail;
- err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
- if (err < 0)
- goto fail;
+ if (profile) {
+ err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
+ if (err < 0)
+ goto fail;
+ }
/* If doing an out-of-place decoding, create a DPB pool */
if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) {
@@ -1163,12 +1246,18 @@ int ff_vk_decode_init(AVCodecContext *avctx)
}
session_params_create.videoSession = ctx->common.session;
- ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
- s->hwctx->alloc, &ctx->empty_session_params);
- if (ret != VK_SUCCESS) {
- av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
- ff_vk_ret2str(ret));
- return AVERROR_EXTERNAL;
+ if (profile) {
+ ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
+ s->hwctx->alloc, &ctx->empty_session_params);
+ if (ret != VK_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
+ ff_vk_ret2str(ret));
+ return AVERROR_EXTERNAL;
+ }
+ } else {
+ /* For SDR decoders, this alignment value will be 0. Since this will make
+ * add_slice() malfunction, set it to a sane default value. */
+ ctx->caps.minBitstreamBufferSizeAlignment = AV_INPUT_BUFFER_PADDING_SIZE;
}
driver_props = &dec->shared_ctx->s.driver_props;
diff --git a/libavcodec/vulkan_decode.h b/libavcodec/vulkan_decode.h
index 1d89db323f..9a11a80f95 100644
--- a/libavcodec/vulkan_decode.h
+++ b/libavcodec/vulkan_decode.h
@@ -56,6 +56,9 @@ typedef struct FFVulkanDecodeShared {
VkVideoDecodeCapabilitiesKHR dec_caps;
VkVideoSessionParametersKHR empty_session_params;
+
+ /* Software-defined decoder context */
+ void *sd_ctx;
} FFVulkanDecodeShared;
typedef struct FFVulkanDecodeContext {
@@ -141,6 +144,13 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
FFVulkanDecodePicture *vkpic, int is_current,
int alloc_dpb);
+/**
+ * Software-defined decoder version of ff_vk_decode_prepare_frame.
+ */
+int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic,
+ FFVulkanDecodePicture *vkpic, int is_current,
+ enum FFVkShaderRepFormat rep_fmt, int alloc_dpb);
+
/**
* Add slice data to frame.
*/
--
2.47.2
More information about the ffmpeg-devel
mailing list