[Libav-user] Using GPU Filters with C

[David Franco]

I have a fork of ffmpeg 5.4 where I implemented the scale_vt filte from ffmpeg 6r:
https://pastebin.com/ZCe9T0ZzI am trying to pass 4k frames using gpu (videotoolbox) to this scale filter to be able to use them later.
However I am noticing that my CPU usage is very high so the frames are probably downloaded to the cpu and then reuploaded again.
What is wrong with my current implementation?
codec_context is the decoder codec context wich is already opened, however it seems like no hw_device_ctx is assigned to it so I had to allocate it myself.
I had to use AVBufferSrcParameters  because the scale_vt filter would crash on configuration if the input link didn’t have a hw_frames_ctx.

 if (!use_hw_resize_)
    {
      return false;
    }
    AVBufferRef *hw_frames_ref = av_hwframe_ctx_alloc(codec_context_->hw_device_ctx);
    codec_context_->hw_frames_ctx = hw_frames_ref;
    AVPixelFormat input_format = AV_PIX_FMT_CUDA;
    // Create new graph
    filter_graph_ = avfilter_graph_alloc();
    AVFilterContext *filter_ctx = nullptr;
    if (this->hw_type_ == AV_HWDEVICE_TYPE_VIDEOTOOLBOX)
    {
      const AVFilter *filter = avfilter_get_by_name("scale_vt");
      if (filter)
      {
        snprintf(strbuf_, sizeof(strbuf_), "w=%d:h=%d",
                 output_hw_width_, output_hw_height_);
        if (avfilter_graph_create_filter(&filter_ctx, filter, NULL,
                                         strbuf_, NULL, filter_graph_) < 0)
        {
          return false;
        }
        input_format = AV_PIX_FMT_VIDEOTOOLBOX;
      }
    }
    if (!filter_ctx)
    {
      return false;
    }
    const AVFilter *buffer = avfilter_get_by_name("buffer");
    const AVFilter *buffersink = avfilter_get_by_name("buffersink");
    int err;
    // Create buffer filter
    snprintf(strbuf_, sizeof(strbuf_), "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d",
             input_hw_width_, input_hw_height_,
             input_format,
             1,           // TODO -> codec_context_->time_base.num
             AV_TIME_BASE // TODO
    );
    err = avfilter_graph_create_filter(&buffer_ctx_, buffer,
                                       "in", strbuf_, NULL, filter_graph_);
    if (err < 0)
    {
      // Unable to create buffer filter
      return false;
    }
    AVBufferSrcParameters *src_params = av_buffersrc_parameters_alloc();
    src_params->hw_frames_ctx = av_buffer_ref(hw_frames_ref); // Set GPU context
    src_params->format = input_format;
    src_params->width = input_hw_width_;
    src_params->height = input_hw_height_;
    src_params->time_base = input_timebase;
    if (av_buffersrc_parameters_set(buffer_ctx_, src_params) < 0)
    {
      av_freep(&src_params);
      return false;
    }
    av_freep(&src_params);
    // Create buffersink filter
    err = avfilter_graph_create_filter(&buffersink_ctx_, buffersink,
                                       "out", NULL, NULL, filter_graph_);
    if (err < 0)
    {
      // Unable to create buffersink filter
      return false;
    }
    this->filter_ctx_list_.push_back(filter_ctx);

    // Connect inputs and outputs
    AVFilterContext *last_context;
    for (int x = 0; x < filter_ctx_list_.size(); x++)
    {
      AVFilterContext *current_context = filter_ctx_list_.at(x);
      if (x == 0)
      {
        err = avfilter_link(buffer_ctx_, 0, current_context, 0);
        if (err < 0)
        {
          // Linking error
          return false;
        }
        last_context = current_context;
        continue;
      }
      err = avfilter_link(last_context, 0, current_context, 0);
      if (err < 0)
      {
        // Linking error
        return false;
      }
      last_context = current_context;
    }

    err = avfilter_link(last_context, 0, buffersink_ctx_, 0);
    if (err < 0)
    {
      // Linking error
      return false;
    }

    err = avfilter_graph_config(filter_graph_, NULL);
    if (err < 0)
    {
      // Error configuring the filter graph
      return false;
    }

    hw_filter_is_valid_ = true;

-- 
This email and any files transmitted with it are confidential and intended 
solely for the use of the individual or entity to whom they are addressed. 
If you have received this email in error please notify the system manager. 
This message contains confidential information and is intended only for the 
individual named. If you are not the named addressee you should not 
disseminate, distribute or copy this e-mail. Please notify the sender 
immediately by e-mail if you have received this e-mail by mistake and 
delete this e-mail from your system. If you are not the intended recipient 
you are notified that disclosing, copying, distributing or taking any 
action in reliance on the contents of this information is strictly 
prohibited.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://ffmpeg.org/pipermail/libav-user/attachments/20250228/033a0e69/attachment.htm>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: favicon.ico
Type: image/vnd.microsoft.icon
Size: 318 bytes
Desc: not available
URL: <https://ffmpeg.org/pipermail/libav-user/attachments/20250228/033a0e69/attachment.ico>