[FFmpeg-user] Encoding with vaapi: The same frame is outputted
hao hu
superarhow at gmail.com
Tue Jul 25 04:31:29 EEST 2023
I am attempting to encode video using VAAPI's hardware acceleration. I have
managed to obtain NV12 format data and used the hwupload filter or
av_hwframe_transfer_data API to upload them to the GPU. Finally, I obtained
an H.264 format .mp4 file. However, the video appears to only have the
first 3-4 frames from my input data, with the following frames being
identical.
My hardware setup consists of a Linux PC with an AMD Radeon graphics card,
and vainfo reports no issues (I have tested ffmpeg command line output
using VAAPI, and it works perfectly).
My ffmpeg is configured with the following command line:
./configure --enable-shared --enable-libvpx --enable-libvorbis
--enable-vaapi --enable-debug
I have attached my source file and Makefile. Could you please help me
identify what I am doing wrong?
-------------- next part --------------
#include <stdio.h>
#include <string.h>
#include <errno.h>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/pixdesc.h>
#include <libavutil/hwcontext.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
#include <libavfilter/avfilter.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/opt.h>
#include <libavutil/timestamp.h>
}
#include <iostream>
#include <chrono>
#include <thread>
static int width, height;
static AVBufferRef *hw_device_ctx = NULL;
static std::string av_error_desc(int rc) {
char errbuf[256];
av_strerror(rc, errbuf, sizeof(errbuf));
return std::string(errbuf);
}
#define CHECK_ERROR(r, reason) if (r < 0) { \
std::cout << "Failed to " << reason << ":" << r << av_error_desc(r) << std::endl; \
}
static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx)
{
AVBufferRef *hw_frames_ref;
AVHWFramesContext *frames_ctx = NULL;
int err = 0;
if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) {
fprintf(stderr, "Failed to create VAAPI frame context.\n");
return -1;
}
frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
frames_ctx->format = AV_PIX_FMT_VAAPI;
frames_ctx->sw_format = AV_PIX_FMT_NV12;
frames_ctx->width = width;
frames_ctx->height = height;
frames_ctx->initial_pool_size = 20;
if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) {
fprintf(stderr, "Failed to initialize VAAPI frame context."
"Error code: %s\n",av_error_desc(err).c_str());
av_buffer_unref(&hw_frames_ref);
return err;
}
ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
if (!ctx->hw_frames_ctx)
err = AVERROR(ENOMEM);
av_buffer_unref(&hw_frames_ref);
return err;
}
static int encode_write(AVCodecContext *avctx, AVFrame *frame, AVFormatContext* fmtctx, AVStream* stream)
{
int ret = 0;
AVPacket *enc_pkt;
const char *type_desc = av_get_media_type_string(avctx->codec_type);
if (!(enc_pkt = av_packet_alloc()))
return AVERROR(ENOMEM);
if ((ret = avcodec_send_frame(avctx, frame)) < 0) {
fprintf(stderr, "Error code: %s\n", av_error_desc(ret).c_str());
goto end;
}
while (1) {
ret = avcodec_receive_packet(avctx, enc_pkt);
if ((ret >= 0 || ret == AVERROR_EOF) && avctx->stats_out)
fprintf(stdout, "%s\n", avctx->stats_out);
if (ret)
break;
enc_pkt->pts = frame->pts;
enc_pkt->duration = frame->duration;
enc_pkt->time_base = stream->time_base;
enc_pkt->stream_index = 0;
av_packet_rescale_ts(enc_pkt, avctx->time_base, stream->time_base);
AVPacket* pkt = enc_pkt;
fprintf(stdout, "encoder -> type:%s "
"pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s "
"duration:%s duration_time:%s size:%d\n",
type_desc,
av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, &avctx->time_base),
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, &avctx->time_base),
av_ts2str(pkt->duration), av_ts2timestr(pkt->duration, &avctx->time_base), pkt->size);
//ret = fwrite(enc_pkt->data, enc_pkt->size, 1, fout);
//ret = av_write_frame(fmtctx, enc_pkt);
ret = av_interleaved_write_frame(fmtctx, enc_pkt);
CHECK_ERROR(ret, "write_frame");
av_packet_unref(enc_pkt);
}
end:
av_packet_free(&enc_pkt);
ret = ((ret == AVERROR(EAGAIN)) ? 0 : -1);
return ret;
}
void convertRGBAtoNV12(const uint8_t* rgbaData, uint8_t* nv12Data, uint8_t* nv12UVData, int width, int height) {
// Calculate the size of the NV12 data
// Convert RGBA to NV12
for (int y = 0; y < height; ++y) {
for (int x = 0; x < width; ++x) {
// Get the RGBA components
uint8_t r = rgbaData[(y * width + x) * 4];
uint8_t g = rgbaData[(y * width + x) * 4 + 1];
uint8_t b = rgbaData[(y * width + x) * 4 + 2];
//uint8_t a = rgbaData[(y * width + x) * 4 + 3];
// Convert to 12-bit NV12
uint8_t yValue = (uint8_t)(((66 * r + 129 * g + 25 * b + 128) >> 8) + 16);
uint8_t uValue = (uint8_t)(((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128);
uint8_t vValue = (uint8_t)(((112 * r - 94 * g - 18 * b + 128) >> 8) + 128);
// Set the Y value
nv12Data[y * width + x] = yValue;
/*
// Set the U and V values for even rows
if (y % 2 == 0) {
nv12UVData[(y / 2) * width + x * 2] = uValue;
nv12UVData[(y / 2) * width + x * 2 + 1] = vValue;
}*/
nv12UVData[(y/2)*width + x/2]=0;
}
}
}
#pragma pack(push, 1)
struct BMPHeader {
uint16_t signature;
uint32_t file_size;
uint32_t reserved;
uint32_t data_offset;
uint32_t header_size;
uint32_t width;
uint32_t height;
uint16_t planes;
uint16_t bits_per_pixel;
uint32_t compression;
uint32_t image_size;
uint32_t x_pixels_per_meter;
uint32_t y_pixels_per_meter;
uint32_t total_colors;
uint32_t important_colors;
};
#pragma pack(pop)
int main(int argc, char *argv[])
{
int size, err;
//, *fout = NULL;
AVFrame *sw_frame = NULL, *hw_frame = NULL;
AVCodecContext *avctx = NULL;
const AVCodec *codec = NULL;
const char *enc_name = "h264_vaapi";
AVFormatContext* formatContext = NULL;
AVStream* stream = NULL;
SwsContext* swsContext = nullptr;
uint8_t* frame_buf = NULL;
char args[512];
//int frame_count = 696;
int frame_count = 100;
width = 1280;
height = 720;
size = width * height * 4;
frame_buf = (uint8_t*)malloc(size);
swsContext = sws_getContext(width, height, AV_PIX_FMT_RGBA, width, height, AV_PIX_FMT_NV12,
SWS_BICUBIC, nullptr, nullptr, nullptr);
if (swsContext == nullptr) {
fprintf(stderr, "Failed to alloc sws context\n");
return -1;
}
// Create a filter graph
AVFilterGraph* filterGraph = avfilter_graph_alloc();
// Create the buffersrc and buffersink filters
const AVFilter* buffersrc = avfilter_get_by_name("buffer");
AVFilterContext* buffersrc_ctx = nullptr;
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
width, height, AV_PIX_FMT_NV12,
1, 25,
16, 9);
err = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, nullptr, filterGraph);
CHECK_ERROR(err, "avfilter_graph_create src");
AVFilterContext* buffersink_ctx = nullptr;
const AVFilter* buffersink = avfilter_get_by_name("buffersink");
err = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
"", nullptr, filterGraph);
CHECK_ERROR(err, "avfilter_graph_create sink");
AVFilterContext* hwupload_filter_ctx = nullptr;
// Create hwupload filter
err = avfilter_graph_create_filter(&hwupload_filter_ctx, avfilter_get_by_name("hwupload"), "hwupload", NULL, NULL, filterGraph);
CHECK_ERROR(err, "avfilter_graph_create hwupload");
// Set filter parameters
AVBufferSrcParameters* buffersrc_params = av_buffersrc_parameters_alloc();
avformat_alloc_output_context2(&formatContext, NULL, NULL, "output.mp4");
err = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI,
"/dev/dri/renderD128", NULL, 0);
if (err < 0) {
fprintf(stderr, "Failed to create a VAAPI device. Error code: %s\n", av_error_desc(err).c_str());
goto close;
}
if (!(codec = avcodec_find_encoder_by_name(enc_name))) {
fprintf(stderr, "Could not find encoder.\n");
err = -1;
goto close;
}
if (!(avctx = avcodec_alloc_context3(codec))) {
err = AVERROR(ENOMEM);
goto close;
}
avctx->codec_id = codec->id;
avctx->width = width;
avctx->height = height;
avctx->time_base = (AVRational){1, 25};
avctx->framerate = (AVRational){25, 1};
avctx->sample_aspect_ratio = (AVRational){1, 1};
avctx->pix_fmt = AV_PIX_FMT_VAAPI;
//avctx->bit_rate = 1024*1024*8;
avctx->gop_size = 120;
avctx->keyint_min = 25;
avctx->colorspace = AVCOL_SPC_BT470BG;
avctx->color_range = AVCOL_RANGE_JPEG;
avctx->chroma_sample_location = AVCHROMA_LOC_CENTER;
avctx->field_order = AV_FIELD_PROGRESSIVE;
avctx->sw_pix_fmt = AV_PIX_FMT_NONE;
avctx->max_b_frames = 2;
avctx->profile = FF_PROFILE_H264_BASELINE | FF_PROFILE_H264_CONSTRAINED;
av_opt_set(avctx->priv_data, "b", "0", 0);
av_opt_set(avctx->priv_data, "bf", "2", 0);
av_opt_set(avctx->priv_data, "g", "120", 0);
av_opt_set(avctx->priv_data, "i_qfactor", "1", 0);
av_opt_set(avctx->priv_data, "i_qoffset", "0", 0);
av_opt_set(avctx->priv_data, "b_qfactor", "6/5", 0);
av_opt_set(avctx->priv_data, "b_qoffset", "0", 0);
av_opt_set(avctx->priv_data, "qmin", "-1", 0);
av_opt_set(avctx->priv_data, "qmax", "-1", 0);
if (formatContext->oformat->flags & AVFMT_GLOBALHEADER)
avctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
stream = avformat_new_stream(formatContext, codec);
stream->id = (int)(formatContext->nb_streams - 1);
if ((err = avcodec_parameters_from_context(stream->codecpar, avctx)) < 0) {
fprintf(stderr, "Failed to avcodec_parameters_from_context.\n");
goto close;
}
stream->time_base = avctx->time_base;
//stream->r_frame_rate = { 25, 1 };
//stream->avg_frame_rate = { 25, 1 };
/* set hw_frames_ctx for encoder's AVCodecContext */
if ((err = set_hwframe_ctx(avctx, hw_device_ctx)) < 0) {
fprintf(stderr, "Failed to set hwframe context.\n");
goto close;
}
if ((err = avcodec_open2(avctx, codec, NULL)) < 0) {
fprintf(stderr, "Cannot open video encoder codec. Error code: %s\n", av_error_desc(err).c_str());
goto close;
}
if ((err = avio_open(&formatContext->pb, "output.mp4", AVIO_FLAG_WRITE)) < 0) {
fprintf(stderr, "Failed to avio open.\n");
goto close;
}
av_dump_format(formatContext, 0, "output.mp4", 1);
if ((err = avformat_write_header(formatContext, nullptr)) < 0) {
fprintf(stderr, "Failed to writer header.\n");
goto close;
}
buffersrc_params->format = AV_PIX_FMT_NV12; // Set the input pixel format
//buffersrc_params->hw_frames_ctx = av_hwframe_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI); // Create the VAAPI hardware context
//buffersrc_params->hw_frames_ctx = avctx->hw_device_ctx;// av_hwframe_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI); // Create the VAAPI hardware context
// Set other parameters as needed
// Initialize the buffersrc filter
err = av_buffersrc_parameters_set(buffersrc_ctx, buffersrc_params);
CHECK_ERROR(err, "av_buffersrc_parameters_set");
hwupload_filter_ctx->hw_device_ctx = hw_device_ctx;
// Initialize the filter graph
err = avfilter_link(buffersrc_ctx, 0, hwupload_filter_ctx, 0);
CHECK_ERROR(err, "avfilter_link src->upload");
err = avfilter_link(hwupload_filter_ctx, 0, buffersink_ctx, 0);
CHECK_ERROR(err, "avfilter_link upload->sink");
err = avfilter_graph_config(filterGraph, NULL);
CHECK_ERROR(err, "avfilter_graph_config");
for (int i = 0; i < frame_count; i++) {
if (!(sw_frame = av_frame_alloc())) {
err = AVERROR(ENOMEM);
goto close;
}
/* read data into software frame, and transfer them into hw frame */
sw_frame->width = width;
sw_frame->height = height;
sw_frame->format = AV_PIX_FMT_NV12;
if ((err = av_frame_get_buffer(sw_frame, 0)) < 0)
goto close;
// Convert the pixel format to the format required by the codec
/*
int in_linesize[1] = { width * 4 };
err = sws_scale(swsContext, &frame_buf, in_linesize, 0, height, sw_frame->data, sw_frame->linesize);
if (err < 0) {
goto close;
}*/
//convertRGBAtoNV12(frame_buf, sw_frame->data[0], sw_frame->data[1], width, height);
// fill random data/test
int cx = width / 2;
int cy = height / 2;
for (int y = 0; y < height; ++y) {
for (int x = 0; x < width; x++) {
int dist = std::sqrt((cx - x) * (cx - x) + (cy - y) * (cy - y));
sw_frame->data[0][y * width + x] = (dist + i * 4) % 255;
//sw_frame->data[1][y * width / 2 + x / 2] = (i+dist)%255;
sw_frame->data[1][y * width / 2 + x / 2] = 0;
}
}
if (!(hw_frame = av_frame_alloc())) {
err = AVERROR(ENOMEM);
goto close;
}
/*
if ((err = av_hwframe_get_buffer(avctx->hw_frames_ctx, hw_frame, 0)) < 0) {
fprintf(stderr, "Error code: %s.\n", av_error_desc(err).c_str());
goto close;
}
if (!hw_frame->hw_frames_ctx) {
err = AVERROR(ENOMEM);
goto close;
}
*/
/*
if ((err = av_hwframe_transfer_data(hw_frame, sw_frame, 0)) < 0) {
fprintf(stderr, "Error while transferring frame data to surface."
"Error code: %s.\n", av_error_desc(err).c_str());
goto close;
}
*/
// Push the input frame into the filtergraph
err = av_buffersrc_add_frame_flags(buffersrc_ctx, sw_frame, AV_BUFFERSRC_FLAG_KEEP_REF);
CHECK_ERROR(err, "add_frame");
/*
// Pull the output frame from the filtergraph
AVFrame* outputFrame = av_frame_alloc();
av_buffersink_get_frame(buffersink_ctx, outputFrame);
// Access the output hardware frame object
AVHWFramesContext* hw_frames_ctx = (AVHWFramesContext*)outputFrame->hw_frames_ctx->data;
// Access the hardware-specific data as needed
*/
using namespace std::chrono_literals;
std::this_thread::sleep_for(30ms);
err = av_buffersink_get_frame(buffersink_ctx, hw_frame);
CHECK_ERROR(err, "get_frame");
hw_frame->width = width;
hw_frame->height = height;
hw_frame->format = avctx->pix_fmt;
hw_frame->duration = 1;
hw_frame->pts = i;
hw_frame->sample_aspect_ratio = (AVRational){ 1, 1 };
hw_frame->pkt_dts = hw_frame->pts;
hw_frame->time_base = (AVRational){ 1, 25 };
hw_frame->color_range = AVCOL_RANGE_JPEG;
hw_frame->best_effort_timestamp = hw_frame->pts;
hw_frame->pkt_duration = 1;
hw_frame->colorspace = AVCOL_SPC_BT470BG;
hw_frame->chroma_location = AVCHROMA_LOC_CENTER;
if ((err = (encode_write(avctx, hw_frame, formatContext, stream))) < 0) {
fprintf(stderr, "Failed to encode.\n");
goto close;
}
av_frame_free(&hw_frame);
av_frame_free(&sw_frame);
}
/* flush encoder */
err = encode_write(avctx, NULL, formatContext, stream);
if (err == AVERROR_EOF)
err = 0;
av_write_trailer(formatContext);
avio_closep(&formatContext->pb);
close:
//if (fout)
// fclose(fout);
free(frame_buf);
av_frame_free(&sw_frame);
av_frame_free(&hw_frame);
avcodec_free_context(&avctx);
av_buffer_unref(&hw_device_ctx);
return err;
}
-------------- next part --------------
A non-text attachment was scrubbed...
Name: Makefile
Type: application/octet-stream
Size: 992 bytes
Desc: not available
URL: <https://ffmpeg.org/pipermail/ffmpeg-user/attachments/20230725/e08cb849/attachment.obj>
More information about the ffmpeg-user
mailing list