[Libav-user] How to use Vulkan encoder correctly?

[vytskalt]

Hi everyone, I'm looking to use the libav h264_vulkan encoder to render some videos on my GPU. After some trial and error I figured out how to encode a simple video, but the application leaks VkImageView objects and throws other validation errors, and I don't know whether it's a bug in libav or I'm just using it wrong.

Here's the source code of a minimal application that triggers the errors:
```
#include "vulkan/vulkan_core.h"
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/error.h>
#include <libavutil/frame.h>
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_vulkan.h>
#include <libavutil/imgutils.h>
#include <libavutil/opt.h>
#include <libavutil/pixdesc.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

const VkExtent2D IMAGE_SIZE = {1920, 1080};

static const char *validationLayers[] = {
"VK_LAYER_KHRONOS_validation",
};
static const size_t validationLayersCount = 1;

static const char *instanceExtensions[] = {
VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME,
};
static const size_t instanceExtensionsCount = 1;

static const char *deviceExtensions[] = {
VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME,
VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME,
VK_KHR_VIDEO_ENCODE_H264_EXTENSION_NAME,
};
static const size_t deviceExtensionsCount = 4;

typedef struct {
uint32_t transferAndComputeFamily;
uint32_t encodeFamily;
} QueueFamilyIndices;

VkInstance instance;
VkPhysicalDevice physicalDevice = VK_NULL_HANDLE;
QueueFamilyIndices queueFamilyIndices;
VkPhysicalDeviceFeatures2 deviceFeatures;
VkDevice device;

AVBufferRef *ff_hw_dev;
AVBufferRef *ff_hw_frames_ctx;
AVFrame *ff_frame;
AVVkFrame *ff_vk_frame;

VkImage imageOut;

void check(VkResult result, const char *msg) {
if (result != VK_SUCCESS) {
fprintf(stderr, "%s\n", msg);
exit(1);
}
}

QueueFamilyIndices findQueueFamilies(VkPhysicalDevice device) {
QueueFamilyIndices indices = {
.transferAndComputeFamily = UINT32_MAX,
.encodeFamily = UINT32_MAX,
};

uint32_t familyCount;
vkGetPhysicalDeviceQueueFamilyProperties(device, &familyCount, NULL);
VkQueueFamilyProperties *families =
malloc(familyCount * sizeof(VkQueueFamilyProperties));
if (!families) {
fprintf(stderr, "Failed to allocate memory for queue families\n");
exit(1);
}
vkGetPhysicalDeviceQueueFamilyProperties(device, &familyCount, families);

for (uint32_t i = 0; i < familyCount; i++) {
if (families[i].queueFlags & VK_QUEUE_COMPUTE_BIT &&
families[i].queueFlags & VK_QUEUE_TRANSFER_BIT) {
indices.transferAndComputeFamily = i;
}

if (families[i].queueFlags & VK_QUEUE_VIDEO_ENCODE_BIT_KHR) {
indices.encodeFamily = i;
}

if (indices.transferAndComputeFamily != UINT32_MAX &&
indices.encodeFamily != UINT32_MAX) {
break;
}
}

free(families);
return indices;
}

void encodeVideo() {
ff_hw_dev = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VULKAN);
AVHWDeviceContext *hwctx = (AVHWDeviceContext *)ff_hw_dev->data;
AVVulkanDeviceContext *vk = (AVVulkanDeviceContext *)hwctx->hwctx;

vk->get_proc_addr = vkGetInstanceProcAddr;
vk->inst = instance;
vk->phys_dev = physicalDevice;
vk->act_dev = device;
vk->device_features = deviceFeatures;
vk->enabled_inst_extensions = instanceExtensions;
vk->nb_enabled_inst_extensions = instanceExtensionsCount;
vk->enabled_dev_extensions = deviceExtensions;
vk->nb_enabled_dev_extensions = deviceExtensionsCount;

vk->qf[0].idx = queueFamilyIndices.transferAndComputeFamily;
vk->qf[0].num = 1;
vk->qf[0].flags =
(VkQueueFlagBits)(VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT);

vk->qf[1].idx = queueFamilyIndices.encodeFamily;
vk->qf[1].num = 1;
vk->qf[1].flags = VK_QUEUE_VIDEO_ENCODE_BIT_KHR;
vk->qf[1].video_caps = VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR;

vk->nb_qf = 2;

int result = av_hwdevice_ctx_init(ff_hw_dev);
if (result < 0) {
fprintf(stderr, "failed to init ffmpeg vulkan hwdevice\n");
exit(1);
}

ff_hw_frames_ctx = av_hwframe_ctx_alloc(ff_hw_dev);
if (!ff_hw_frames_ctx) {
fprintf(stderr, "Failed to allocate hardware frames context\n");
exit(1);
}

AVHWFramesContext *frames_ctx = (AVHWFramesContext *)ff_hw_frames_ctx->data;
frames_ctx->format = AV_PIX_FMT_VULKAN;
frames_ctx->sw_format = AV_PIX_FMT_NV12;
frames_ctx->width = IMAGE_SIZE.width;
frames_ctx->height = IMAGE_SIZE.height;

AVVulkanFramesContext *vk_frames_ctx =
(AVVulkanFramesContext *)frames_ctx->hwctx;
vk_frames_ctx->tiling = VK_IMAGE_TILING_OPTIMAL;
vk_frames_ctx->img_flags = VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
vk_frames_ctx->flags = AV_VK_FRAME_FLAG_NONE;
vk_frames_ctx->usage =
(VkImageUsageFlagBits)(VK_IMAGE_USAGE_SAMPLED_BIT |
VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
VK_IMAGE_USAGE_TRANSFER_DST_BIT |
VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR);

result = av_hwframe_ctx_init(ff_hw_frames_ctx);
if (result < 0) {
fprintf(stderr, "Failed to initialize hardware frames context\n");
exit(1);
}

ff_frame = av_frame_alloc();
if (!ff_frame) {
fprintf(stderr, "Failed to allocate Vulkan frame\n");
exit(1);
}

result = av_hwframe_get_buffer(ff_hw_frames_ctx, ff_frame, 0);
if (result < 0) {
fprintf(stderr, "Failed to allocate Vulkan frame buffer\n");
exit(1);
}

ff_vk_frame = (AVVkFrame *)ff_frame->data[0];
if (!ff_vk_frame) {
fprintf(stderr, "Failed to get VkFrame from AVFrame\n");
exit(1);
}

imageOut = ff_vk_frame->img[0];

const AVCodec *codec = avcodec_find_encoder_by_name("h264_vulkan");
if (!codec) {
fprintf(stderr, "h264_vulkan encoder not found\n");
exit(1);
}

AVCodecContext *codec_ctx = avcodec_alloc_context3(codec);
if (!codec_ctx) {
fprintf(stderr, "Failed to allocate codec context\n");
exit(1);
}

codec_ctx->width = IMAGE_SIZE.width;
codec_ctx->height = IMAGE_SIZE.height;
codec_ctx->time_base = (AVRational){1, 30}; // 30 FPS
codec_ctx->framerate = (AVRational){30, 1};
codec_ctx->pix_fmt = AV_PIX_FMT_VULKAN;
codec_ctx->bit_rate = 5000000; // 5 Mbps
codec_ctx->hw_frames_ctx = ff_hw_frames_ctx;

result = avcodec_open2(codec_ctx, codec, NULL);
if (result < 0) {
fprintf(stderr, "Failed to open codec\n");
exit(1);
}

AVFormatContext *fmt_ctx;

result = avformat_alloc_output_context2(&fmt_ctx, NULL, NULL, "output.mp4");
if (result < 0) {
fprintf(stderr, "Failed to create output context\n");
exit(1);
}

AVStream *stream = avformat_new_stream(fmt_ctx, NULL);
if (!stream) {
fprintf(stderr, "Failed to create stream\n");
exit(1);
}

result = avcodec_parameters_from_context(stream->codecpar, codec_ctx);
if (result < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
exit(1);
}

if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) {
result = avio_open(&fmt_ctx->pb, "output.mp4", AVIO_FLAG_WRITE);
if (result < 0) {
fprintf(stderr, "Failed to open output file\n");
exit(1);
}
}

result = avformat_write_header(fmt_ctx, NULL);
if (result < 0) {
fprintf(stderr, "Failed to write header\n");
exit(1);
}

AVPacket *pkt = av_packet_alloc();
if (!pkt) {
fprintf(stderr, "Failet to allocate frame or packet\n");
exit(1);
}

uint32_t totalFrames = 30;
for (uint32_t i = 0; i < totalFrames; i++) {
ff_frame->pts = i;

result = avcodec_send_frame(codec_ctx, ff_frame);
if (result < 0) {
fprintf(stderr, "Failed to send frame to encoder\n");
exit(1);
}

while (result >= 0) {
result = avcodec_receive_packet(codec_ctx, pkt);
if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) {
break;
} else if (result < 0) {
fprintf(stderr, "Failed to receive packet\n");
exit(1);
}

pkt->stream_index = stream->index;
av_packet_rescale_ts(pkt, codec_ctx->time_base, stream->time_base);
result = av_interleaved_write_frame(fmt_ctx, pkt);
av_packet_unref(pkt);

if (result < 0) {
fprintf(stderr, "Failed to write packet\n");
exit(1);
}
}
}

result = avcodec_send_frame(codec_ctx, NULL);
while (result >= 0) {
result = avcodec_receive_packet(codec_ctx, pkt);
if (result == AVERROR_EOF) {
break;
} else if (result < 0) {
break;
}

pkt->stream_index = stream->index;
av_packet_rescale_ts(pkt, codec_ctx->time_base, stream->time_base);
av_interleaved_write_frame(fmt_ctx, pkt);
av_packet_unref(pkt);
}

av_write_trailer(fmt_ctx);
printf("Encoding completed successfully!\n");

// Cleanup
vkDeviceWaitIdle(device); // unsure if needed
avcodec_free_context(&codec_ctx);
if (fmt_ctx) {
if (fmt_ctx->pb) {
avio_closep(&fmt_ctx->pb);
}
avformat_free_context(fmt_ctx);
}

av_packet_free(&pkt);
av_frame_free(&ff_frame);
av_buffer_unref(&ff_hw_dev);
}

void cleanupVulkan() {
vkDestroyDevice(device, NULL);
vkDestroyInstance(instance, NULL);
}

void createInstance() {
VkApplicationInfo appInfo = {0};
appInfo.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
appInfo.pApplicationName = "FFmpeg";
appInfo.applicationVersion = VK_MAKE_VERSION(1, 0, 0);
appInfo.pEngineName = "No Engine";
appInfo.engineVersion = VK_MAKE_VERSION(1, 0, 0);
appInfo.apiVersion = VK_API_VERSION_1_3;

VkInstanceCreateInfo createInfo = {0};
createInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
createInfo.pApplicationInfo = &appInfo;
createInfo.flags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR;

createInfo.enabledLayerCount = validationLayersCount;
createInfo.ppEnabledLayerNames = validationLayers;

createInfo.enabledExtensionCount = instanceExtensionsCount;
createInfo.ppEnabledExtensionNames = instanceExtensions;

check(vkCreateInstance(&createInfo, NULL, &instance),
"failed to create instance");
}

bool checkDeviceExtensionSupport(VkPhysicalDevice device) {
uint32_t extensionCount;
vkEnumerateDeviceExtensionProperties(device, NULL, &extensionCount, NULL);
VkExtensionProperties *supportedExtensions =
malloc(extensionCount * sizeof(VkExtensionProperties));
if (!supportedExtensions) {
fprintf(stderr, "Failed to allocate memory for device extensions\n");
exit(1);
}
vkEnumerateDeviceExtensionProperties(device, NULL, &extensionCount,
supportedExtensions);

for (size_t i = 0; i < deviceExtensionsCount; i++) {
const char *requiredExtension = deviceExtensions[i];
bool isFound = false;
for (uint32_t j = 0; j < extensionCount; j++) {
if (strcmp(requiredExtension, supportedExtensions[j].extensionName) ==
0) {
isFound = true;
break;
}
}

if (!isFound) {
free(supportedExtensions);
return false;
}
}

free(supportedExtensions);
return true;
}

void pickPhysicalDevice() {
uint32_t deviceCount;
vkEnumeratePhysicalDevices(instance, &deviceCount, NULL);
if (deviceCount == 0) {
fprintf(stderr, "no physical devices found\n");
exit(1);
}

VkPhysicalDevice *devices = malloc(deviceCount * sizeof(VkPhysicalDevice));
if (!devices) {
fprintf(stderr, "Failed to allocate memory for devices\n");
exit(1);
}
vkEnumeratePhysicalDevices(instance, &deviceCount, devices);

for (uint32_t i = 0; i < deviceCount; i++) {
VkPhysicalDevice device = devices[i];
VkPhysicalDeviceProperties props;
vkGetPhysicalDeviceProperties(device, &props);

if (!checkDeviceExtensionSupport(device)) {
continue;
}

QueueFamilyIndices indices = findQueueFamilies(device);
if (indices.transferAndComputeFamily == UINT32_MAX ||
indices.encodeFamily == UINT32_MAX) {
continue;
}

physicalDevice = device;
queueFamilyIndices = indices;
break;
}

free(devices);

if (physicalDevice == VK_NULL_HANDLE) {
fprintf(stderr, "no suitable physical device found\n");
exit(1);
}
}

void createLogicalDevice() {
VkDeviceQueueCreateInfo queueCreateInfos[2] = {0};
float queuePriority = 1.0f;

queueCreateInfos[0].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
queueCreateInfos[0].queueFamilyIndex =
queueFamilyIndices.transferAndComputeFamily;
queueCreateInfos[0].queueCount = 1;
queueCreateInfos[0].pQueuePriorities = &queuePriority;

queueCreateInfos[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
queueCreateInfos[1].queueFamilyIndex = queueFamilyIndices.encodeFamily;
queueCreateInfos[1].queueCount = 1;
queueCreateInfos[1].pQueuePriorities = &queuePriority;

VkPhysicalDeviceFeatures coreDeviceFeatures = {0};
coreDeviceFeatures.shaderImageGatherExtended = VK_TRUE;
coreDeviceFeatures.fragmentStoresAndAtomics = VK_TRUE;
coreDeviceFeatures.shaderInt64 = VK_TRUE;
coreDeviceFeatures.vertexPipelineStoresAndAtomics = VK_TRUE;

VkPhysicalDeviceVideoMaintenance1FeaturesKHR videoMaint1Features = {0};
videoMaint1Features.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_1_FEATURES_KHR;
videoMaint1Features.videoMaintenance1 = VK_TRUE;

VkPhysicalDeviceSamplerYcbcrConversionFeatures ycbcrFeatures = {0};
ycbcrFeatures.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES;
ycbcrFeatures.samplerYcbcrConversion = VK_TRUE;
ycbcrFeatures.pNext = &videoMaint1Features;

VkPhysicalDeviceSynchronization2Features enabledSync2Features = {0};
enabledSync2Features.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SYNCHRONIZATION_2_FEATURES;
enabledSync2Features.synchronization2 = VK_TRUE;
enabledSync2Features.pNext = &ycbcrFeatures;

VkPhysicalDeviceTimelineSemaphoreFeatures timelineSemaphoreFeatures = {0};
timelineSemaphoreFeatures.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES;
timelineSemaphoreFeatures.timelineSemaphore = VK_TRUE;
timelineSemaphoreFeatures.pNext = &enabledSync2Features;

deviceFeatures = (VkPhysicalDeviceFeatures2){0};
deviceFeatures.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
deviceFeatures.features = coreDeviceFeatures;
deviceFeatures.pNext = &timelineSemaphoreFeatures;

VkDeviceCreateInfo createInfo = {0};
createInfo.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
createInfo.queueCreateInfoCount = 2;
createInfo.pQueueCreateInfos = queueCreateInfos;
createInfo.pNext = &deviceFeatures;

createInfo.enabledLayerCount = validationLayersCount;
createInfo.ppEnabledLayerNames = validationLayers;

createInfo.enabledExtensionCount = deviceExtensionsCount;
createInfo.ppEnabledExtensionNames = deviceExtensions;

check(vkCreateDevice(physicalDevice, &createInfo, NULL, &device),
"failed to create logical device");
}

int main() {
createInstance();
pickPhysicalDevice();
createLogicalDevice();
encodeVideo();
cleanupVulkan();
return 0;
}
```

Interesting code is in the encodeVideo function. It creates an empty frame and sends it to the h264_vulkan encoder to create a video.

When running it, there are two different validation layer errors, which I don't know how to fix:
```
Validation Error: [ VUID-VkImageCreateInfo-pNext-06811 ] | MessageID = 0x30f4ac70
vkCreateImage(): pCreateInfo specifies flags (VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT), format (VK_FORMAT_G8_B8R8_2PLANE_420_UNORM), imageType (VK_IMAGE_TYPE_2D), and tiling (VK_IMAGE_TILING_OPTIMAL) which are not supported by any of the supported video format properties for the video profiles specified in the VkVideoProfileListInfoKHR structure included in the pCreateInfo->pNext chain, as reported by vkGetPhysicalDeviceVideoFormatPropertiesKHR for the same video profiles and the image usage flags specified in pCreateInfo->usage (VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR).
The Vulkan spec states: If the pNext chain includes a VkVideoProfileListInfoKHR structure with profileCount greater than 0, then supportedVideoFormat must be VK_TRUE (https://docs.vulkan.org/spec/latest/chapters/resources.html#VUID-VkImageCreateInfo-pNext-06811)

Validation Error: [ VUID-VkImageCreateInfo-pNext-06811 ] | MessageID = 0x30f4ac70
vkCreateImage(): pCreateInfo specifies flags (VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT), format (VK_FORMAT_G8_B8R8_2PLANE_420_UNORM), imageType (VK_IMAGE_TYPE_2D), and tiling (VK_IMAGE_TILING_OPTIMAL) which are not supported by any of the supported video format properties for the video profiles specified in the VkVideoProfileListInfoKHR structure included in the pCreateInfo->pNext chain, as reported by vkGetPhysicalDeviceVideoFormatPropertiesKHR for the same video profiles and the image usage flags specified in pCreateInfo->usage (VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR).
The Vulkan spec states: If the pNext chain includes a VkVideoProfileListInfoKHR structure with profileCount greater than 0, then supportedVideoFormat must be VK_TRUE (https://docs.vulkan.org/spec/latest/chapters/resources.html#VUID-VkImageCreateInfo-pNext-06811)

Encoding completed successfully!
Validation Error: [ VUID-vkDestroyDevice-device-05137 ] | MessageID = 0x4872eaa0
vkDestroyDevice(): Object Tracking - For VkDevice 0x237f16b0, VkImageView 0x450000000045 has not been destroyed.
The Vulkan spec states: All child objects created on device that can be destroyed or freed must have been destroyed or freed prior to destroying device (https://docs.vulkan.org/spec/latest/chapters/devsandqueues.html#VUID-vkDestroyDevice-device-05137)
Objects: 1
[0] VkImageView 0x450000000045

Validation Error: [ VUID-vkDestroyDevice-device-05137 ] | MessageID = 0x4872eaa0
vkDestroyDevice(): Object Tracking - For VkDevice 0x237f16b0, VkImageView 0x460000000046 has not been destroyed.
The Vulkan spec states: All child objects created on device that can be destroyed or freed must have been destroyed or freed prior to destroying device (https://docs.vulkan.org/spec/latest/chapters/devsandqueues.html#VUID-vkDestroyDevice-device-05137)
Objects: 1
[0] VkImageView 0x460000000046

Validation Error: [ VUID-vkDestroyDevice-device-05137 ] | MessageID = 0x4872eaa0
vkDestroyDevice(): Object Tracking - For VkDevice 0x237f16b0, VkImageView 0x470000000047 has not been destroyed.
The Vulkan spec states: All child objects created on device that can be destroyed or freed must have been destroyed or freed prior to destroying device (https://docs.vulkan.org/spec/latest/chapters/devsandqueues.html#VUID-vkDestroyDevice-device-05137)
Objects: 1 [0] VkImageView 0x470000000047
```

I'm using FFmpeg commit da18c2a373, compiled with `--enable-vulkan`, and the application compiled with `gcc main.c -Iinclude -lm -Llib -lvulkan -lavformat -lavcodec -lavutil -lswresample -o main`.

Could anyone familiar with Vulkan encoding take a look at my code and see if there any obvious mistakes in it?

Also something to note is that the first validation errors disappear when using the FFmpeg stable 7.1.1 release instead of master branch, but the VkImageView leaks are still there.

Thanks!
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://ffmpeg.org/pipermail/libav-user/attachments/20250723/f21c41a9/attachment.htm>