[Libav-user] Problem with setting up overlay_cuda

[Tomislav Timic]

I am trying to overlay dvb subtitles with overlay cuda, i am transforming
dvb subtitle rect from pal8 to rgba and sending rgba frame to subtitle
buffer source context, and i want to overlay those subtitles on gpu, now
overlay cuda accepts nv12 pixel format and i am trying to make
configuration so that I convert rgba to nv12 upload it to nvidia device,
scale image and overlay, but I get this error Impossible to convert between
the formats supported by the filter 'overlay_cuda' and the filter
'auto_scale_1'. If anyone has experience with this I would appreciate your
help.

std::string subtitleArgs = StringFmtUtil::format(
        "video_size=%dx%d:pix_fmt=rgba:time_base=%d/%d:frame_rate=%d/%d:pixel_aspect=%d/%d",
        width,
        height,
        inputStreamTb.num,
        inputStreamTb.den,
        inputFps.num,
        inputFps.den,
        sampleAspectRatio.num,
        sampleAspectRatio.den);
avfilter_graph_create_filter(&subBufferSrcCtx,
                             avfilter_get_by_name("buffer"),
                             "sub_in",
                             subtitleArgs.c_str(),
                             nullptr,
                             vFilterGraph);

AVBufferSrcParameters *inputParSub;
inputParSub = av_buffersrc_parameters_alloc();
if (!inputParSub)
{
    return false;
}
inputParSub->time_base = inputStreamTb;
inputParSub->width = width;
inputParSub->height = height;
inputParSub->sample_aspect_ratio = sampleAspectRatio;
inputParSub->frame_rate = inputFps;
inputParSub->format = AV_PIX_FMT_RGBA;
inputParSub->hw_frames_ctx = av_buffer_ref(hwFrameRef);

ret = av_buffersrc_parameters_set(subBufferSrcCtx, inputParSub);
av_freep(&inputParSub);
if (ret < 0)
{
    return false;
}

if (avfilter_graph_create_filter(&hwUploadCudaCtx,
                                 avfilter_get_by_name("hwupload_cuda"),
                                 "upload_cuda",
                                 nullptr,
                                 nullptr,
                                 vFilterGraph) < 0)
{
    release();
    return false;
}

if (avfilter_graph_create_filter(&nv12FmtCtx,
                                 avfilter_get_by_name("format"),
                                 "nv12_fmt",
                                 "pix_fmts=nv12",
                                 nullptr,
                                 vFilterGraph) < 0)
{
    release();
    return false;
}

auto subScaleArgs = StringFmtUtil::format(
        "h=%d:w=%d",
        height, width);
if (avfilter_graph_create_filter(&scaleSubNppCtx,
                                 avfilter_get_by_name("scale_npp"),
                                 "scale_sub_npp",
                                 subScaleArgs.c_str(),
                                 nullptr,
                                 vFilterGraph) < 0)
{
    release();
    return false;
}

if (avfilter_graph_create_filter(&overlayCtx,
                                 avfilter_get_by_name("overlay_cuda"),
                                 "overlay_cuda",
                                 nullptr,
                                 nullptr,
                                 vFilterGraph) < 0)
{
    LOG(ERROR) << LOG_TAG << " failed to overlay cuda filter";
    release();
    return false;
}

// And linking them

avfilter_link(subBufferSrcCtx, 0, nv12FmtCtx, 0);
avfilter_link(nv12FmtCtx, 0, hwUploadCudaCtx, 0);
avfilter_link(hwUploadCudaCtx, 0, scaleSubNppCtx, 0);
avfilter_link(scaleSubNppCtx, 0, overlayCtx, 1);
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://ffmpeg.org/pipermail/libav-user/attachments/20250804/7e18431d/attachment.htm>