[FFmpeg-devel] [PATCH v3] avfilter: add overlay vaapi filter
Xinpeng Sun
xinpeng.sun at intel.com
Wed Feb 26 07:53:34 EET 2020
Overlay one video on the top of another.
It takes two inputs and has one output. The first input is the "main" video on which the
second input is overlaid. This filter requires same memory layout for all the inputs.
An example command to use this filter to overlay an image LOGO at the top-left corner of
the INPUT video and both inputs are yuv420p format:
FFMPEG -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -hwaccel_output_format vaapi \
-i INPUT -i LOGO -filter_complex \
"[0:v]hwupload[a], [1:v]format=yuv420p, hwupload[b], [a][b]overlay_vaapi, hwdownload" \
OUTPUT
Signed-off-by: Xinpeng Sun <xinpeng.sun at intel.com>
Signed-off-by: Zachary Zhou <zachary.zhou at intel.com>
---
configure | 3 +
doc/filters.texi | 51 ++++
libavfilter/Makefile | 1 +
libavfilter/allfilters.c | 1 +
libavfilter/vf_overlay_vaapi.c | 426 +++++++++++++++++++++++++++++++++
5 files changed, 482 insertions(+)
create mode 100644 libavfilter/vf_overlay_vaapi.c
diff --git a/configure b/configure
index ab761c7183..19fe94729f 100755
--- a/configure
+++ b/configure
@@ -3533,6 +3533,7 @@ openclsrc_filter_deps="opencl"
overlay_opencl_filter_deps="opencl"
overlay_qsv_filter_deps="libmfx"
overlay_qsv_filter_select="qsvvpp"
+overlay_vaapi_filter_deps="vaapi"
overlay_vulkan_filter_deps="vulkan libglslang"
owdenoise_filter_deps="gpl"
pad_opencl_filter_deps="opencl"
@@ -3592,6 +3593,7 @@ tonemap_vaapi_filter_deps="vaapi VAProcFilterParameterBufferHDRToneMapping"
tonemap_opencl_filter_deps="opencl const_nan"
transpose_opencl_filter_deps="opencl"
transpose_vaapi_filter_deps="vaapi VAProcPipelineCaps_rotation_flags"
+overlay_vaapi_filter_deps="vaapi VAProcPipelineCaps_blend_flags"
unsharp_opencl_filter_deps="opencl"
uspp_filter_deps="gpl avcodec"
vaguedenoiser_filter_deps="gpl"
@@ -6599,6 +6601,7 @@ if enabled vaapi; then
check_struct "va/va.h" "VADecPictureParameterBufferVP9" bit_depth
check_type "va/va.h va/va_vpp.h" "VAProcFilterParameterBufferHDRToneMapping"
check_struct "va/va.h va/va_vpp.h" "VAProcPipelineCaps" rotation_flags
+ check_struct "va/va.h va/va_vpp.h" "VAProcPipelineCaps" blend_flags
check_type "va/va.h va/va_enc_hevc.h" "VAEncPictureParameterBufferHEVC"
check_type "va/va.h va/va_enc_jpeg.h" "VAEncPictureParameterBufferJPEG"
check_type "va/va.h va/va_enc_vp8.h" "VAEncPictureParameterBufferVP8"
diff --git a/doc/filters.texi b/doc/filters.texi
index 70fd7a4cc7..fbf7264b94 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -21732,6 +21732,57 @@ To enable compilation of these filters you need to configure FFmpeg with
To use vaapi filters, you need to setup the vaapi device correctly. For more information, please read @url{https://trac.ffmpeg.org/wiki/Hardware/VAAPI}
+ at section overlay_vaapi
+
+Overlay one video on the top of another.
+
+It takes two inputs and has one output. The first input is the "main" video on which the second input is overlaid.
+This filter requires same memory layout for all the inputs. So, format conversion may be needed.
+
+The filter accepts the following options:
+
+ at table @option
+
+ at item x
+Set the x coordinate of the overlaid video on the main video.
+Default value is @code{0}.
+
+ at item y
+Set the y coordinate of the overlaid video on the main video.
+Default value is @code{0}.
+
+ at item w
+Set the width of the overlaid video on the main video.
+Default value is the width of input overlay video.
+
+ at item h
+Set the height of the overlaid video on the main video.
+Default value is the height of input overlay video.
+
+ at item alpha
+Set blocking detection thresholds. Allowed range is 0.0 to 1.0, it
+requires an input video with alpha channel.
+Default value is @code{0.0}.
+
+ at end table
+
+ at subsection Examples
+
+ at itemize
+ at item
+Overlay an image LOGO at the top-left corner of the INPUT video. Both inputs for this filter are yuv420p format.
+ at example
+-i INPUT -i LOGO -filter_complex "[0:v]hwupload[a], [1:v]format=yuv420p, hwupload[b], [a][b]overlay_vaapi" OUTPUT
+ at end example
+ at item
+Overlay an image LOGO at the offset (200, 100) from the top-left corner of the INPUT video.
+The inputs have same memory layout for color channels, the overlay has additional alpha plane, like INPUT is yuv420p, and the LOGO is yuva420p.
+ at example
+-i INPUT -i LOGO -filter_complex "[0:v]hwupload[a], [1:v]format=yuva420p, hwupload[b], [a][b]overlay_vaapi=x=200:y=100:w=400:h=300:alpha=1.0, hwdownload, format=nv12" OUTPUT
+ at end example
+
+ at end itemize
+
@section tonemap_vaapi
Perform HDR(High Dynamic Range) to SDR(Standard Dynamic Range) conversion with tone-mapping.
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 089880a39d..96fce9e84f 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -326,6 +326,7 @@ OBJS-$(CONFIG_OVERLAY_FILTER) += vf_overlay.o framesync.o
OBJS-$(CONFIG_OVERLAY_OPENCL_FILTER) += vf_overlay_opencl.o opencl.o \
opencl/overlay.o framesync.o
OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o framesync.o
+OBJS-$(CONFIG_OVERLAY_VAAPI_FILTER) += vf_overlay_vaapi.o framesync.o vaapi_vpp.o
OBJS-$(CONFIG_OVERLAY_VULKAN_FILTER) += vf_overlay_vulkan.o vulkan.o
OBJS-$(CONFIG_OWDENOISE_FILTER) += vf_owdenoise.o
OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 88ebd121ad..dfea838f96 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -310,6 +310,7 @@ extern AVFilter ff_vf_oscilloscope;
extern AVFilter ff_vf_overlay;
extern AVFilter ff_vf_overlay_opencl;
extern AVFilter ff_vf_overlay_qsv;
+extern AVFilter ff_vf_overlay_vaapi;
extern AVFilter ff_vf_overlay_vulkan;
extern AVFilter ff_vf_owdenoise;
extern AVFilter ff_vf_pad;
diff --git a/libavfilter/vf_overlay_vaapi.c b/libavfilter/vf_overlay_vaapi.c
new file mode 100644
index 0000000000..993db9148e
--- /dev/null
+++ b/libavfilter/vf_overlay_vaapi.c
@@ -0,0 +1,426 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <string.h>
+
+#include "libavutil/avassert.h"
+#include "libavutil/mem.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+
+#include "avfilter.h"
+#include "framesync.h"
+#include "formats.h"
+#include "internal.h"
+#include "vaapi_vpp.h"
+
+typedef struct OverlayVAAPIContext {
+ VAAPIVPPContext vpp_ctx; // must be the first field
+ FFFrameSync fs;
+ int overlay_ox;
+ int overlay_oy;
+ int overlay_ow;
+ int overlay_oh;
+ float alpha;
+} OverlayVAAPIContext;
+
+static int overlay_vaapi_query_formats(AVFilterContext *ctx)
+{
+ int ret;
+ enum {
+ MAIN = 0,
+ OVERLAY = 1,
+ };
+
+ static const enum AVPixelFormat pix_fmts[] = {
+ AV_PIX_FMT_VAAPI,
+ AV_PIX_FMT_NONE
+ };
+
+ ret = ff_formats_ref(ff_make_format_list(pix_fmts), &ctx->inputs[MAIN]->out_formats);
+ if (ret < 0)
+ return ret;
+
+ ret = ff_formats_ref(ff_make_format_list(pix_fmts), &ctx->inputs[OVERLAY]->out_formats);
+ if (ret < 0)
+ return ret;
+
+ ret = ff_formats_ref(ff_make_format_list(pix_fmts), &ctx->outputs[0]->in_formats);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static int overlay_vaapi_build_filter_params(AVFilterContext *avctx)
+{
+ VAAPIVPPContext *vpp_ctx = avctx->priv;
+ VAStatus vas;
+ int support_flag;
+ VAProcPipelineCaps pipeline_caps;
+
+ memset(&pipeline_caps, 0, sizeof(pipeline_caps));
+ vas = vaQueryVideoProcPipelineCaps(vpp_ctx->hwctx->display,
+ vpp_ctx->va_context,
+ NULL, 0,
+ &pipeline_caps);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to query pipeline "
+ "caps: %d (%s).\n", vas, vaErrorStr(vas));
+ return AVERROR(EIO);
+ }
+
+ if (!pipeline_caps.blend_flags) {
+ av_log(avctx, AV_LOG_ERROR, "VAAPI driver doesn't support overlay\n");
+ return AVERROR(EINVAL);
+ }
+
+ support_flag = pipeline_caps.blend_flags & VA_BLEND_GLOBAL_ALPHA;
+ if (!support_flag) {
+ av_log(avctx, AV_LOG_ERROR, "VAAPI driver doesn't support global alpha blending\n");
+ return AVERROR(EINVAL);
+ }
+
+ return 0;
+}
+
+
+static int overlay_vaapi_render_picture(AVFilterContext *avctx,
+ VAProcPipelineParameterBuffer *params,
+ VAProcPipelineParameterBuffer *subpic_params,
+ AVFrame *output_frame)
+{
+ VAAPIVPPContext *ctx = avctx->priv;
+ VASurfaceID output_surface;
+ VABufferID params_id;
+ VABufferID subpic_params_id;
+ VAStatus vas;
+ int err = 0;
+
+ output_surface = (VASurfaceID)(uintptr_t)output_frame->data[3];
+
+ vas = vaBeginPicture(ctx->hwctx->display,
+ ctx->va_context, output_surface);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to attach new picture: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+ err = AVERROR(EIO);
+ goto fail;
+ }
+
+ vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context,
+ VAProcPipelineParameterBufferType,
+ sizeof(*params), 1, params, ¶ms_id);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to create parameter buffer: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+ err = AVERROR(EIO);
+ goto fail_after_begin;
+ }
+ av_log(avctx, AV_LOG_DEBUG, "Pipeline parameter buffer is %#x.\n",
+ params_id);
+
+
+ vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context,
+ VAProcPipelineParameterBufferType,
+ sizeof(*subpic_params), 1, subpic_params, &subpic_params_id);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to create parameter buffer: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+ err = AVERROR(EIO);
+ goto fail_after_begin;
+ }
+ av_log(avctx, AV_LOG_DEBUG, "Pipeline subpic parameter buffer is %#x.\n",
+ subpic_params_id);
+
+ vas = vaRenderPicture(ctx->hwctx->display, ctx->va_context,
+ ¶ms_id, 1);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to render parameter buffer: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+ err = AVERROR(EIO);
+ goto fail_after_begin;
+ }
+
+ vas = vaRenderPicture(ctx->hwctx->display, ctx->va_context,
+ &subpic_params_id, 1);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to render subpic parameter buffer: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+ err = AVERROR(EIO);
+ goto fail_after_begin;
+ }
+
+ vas = vaEndPicture(ctx->hwctx->display, ctx->va_context);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to start picture processing: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+ err = AVERROR(EIO);
+ goto fail_after_render;
+ }
+
+ if (CONFIG_VAAPI_1 || ctx->hwctx->driver_quirks &
+ AV_VAAPI_DRIVER_QUIRK_RENDER_PARAM_BUFFERS) {
+ vas = vaDestroyBuffer(ctx->hwctx->display, params_id);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to free parameter buffer: "
+ "%d (%s).\n", vas, vaErrorStr(vas));
+ // And ignore.
+ }
+ }
+
+ return 0;
+
+ // We want to make sure that if vaBeginPicture has been called, we also
+ // call vaRenderPicture and vaEndPicture. These calls may well fail or
+ // do something else nasty, but once we're in this failure case there
+ // isn't much else we can do.
+fail_after_begin:
+ vaRenderPicture(ctx->hwctx->display, ctx->va_context, ¶ms_id, 1);
+fail_after_render:
+ vaEndPicture(ctx->hwctx->display, ctx->va_context);
+fail:
+ return err;
+}
+
+static int overlay_vaapi_blend(FFFrameSync *fs)
+{
+ AVFilterContext *avctx = fs->parent;
+ AVFilterLink *outlink = avctx->outputs[0];
+ OverlayVAAPIContext *ctx = avctx->priv;
+ VAAPIVPPContext *vpp_ctx = avctx->priv;
+ AVFrame *input_main, *input_overlay;
+ AVFrame *output;
+ VAProcPipelineParameterBuffer params, subpic_params;
+ VABlendState blend_state; // Blend State
+ VARectangle overlay_region, output_region;
+ int err;
+
+ err = overlay_vaapi_build_filter_params(avctx);
+ if (err < 0)
+ return err;
+
+ err = ff_framesync_get_frame(fs, 0, &input_main, 0);
+ if (err < 0)
+ return err;
+ err = ff_framesync_get_frame(fs, 1, &input_overlay, 0);
+ if (err < 0)
+ return err;
+
+ av_log(avctx, AV_LOG_DEBUG, "Filter main: %s, %ux%u (%"PRId64").\n",
+ av_get_pix_fmt_name(input_main->format),
+ input_main->width, input_main->height, input_main->pts);
+
+ av_log(avctx, AV_LOG_DEBUG, "Filter overlay: %s, %ux%u (%"PRId64").\n",
+ av_get_pix_fmt_name(input_overlay->format),
+ input_overlay->width, input_overlay->height, input_overlay->pts);
+
+ if (vpp_ctx->va_context == VA_INVALID_ID)
+ return AVERROR(EINVAL);
+
+ output = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+ if (!output) {
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+
+ err = av_frame_copy_props(output, input_main);
+ if (err < 0)
+ goto fail;
+
+ err = ff_vaapi_vpp_init_params(avctx, ¶ms,
+ input_main, output);
+ if (err < 0)
+ goto fail;
+
+ overlay_region = (VARectangle) {
+ .x = ctx->overlay_ox,
+ .y = ctx->overlay_oy,
+ .width = ctx->overlay_ow ? ctx->overlay_ow : input_overlay->width,
+ .height = ctx->overlay_oh ? ctx->overlay_oh : input_overlay->height,
+ };
+
+ output_region = (VARectangle) {
+ .x = 0,
+ .y = 0,
+ .width = output->width,
+ .height = output->height,
+ };
+
+ if (overlay_region.x + overlay_region.width > input_main->width ||
+ overlay_region.y + overlay_region.height > input_main->height) {
+ av_log(ctx, AV_LOG_WARNING,
+ "The overlay image exceeds the scope of the main image, "
+ "will crop the overlay image according based on the main image.\n");
+ }
+
+ params.filters = &vpp_ctx->filter_buffers[0];
+ params.num_filters = vpp_ctx->nb_filter_buffers;
+
+ params.output_region = &output_region;
+ params.output_background_color = VAAPI_VPP_BACKGROUND_BLACK;
+
+ memcpy(&subpic_params, ¶ms, sizeof(subpic_params));
+
+ blend_state.flags = VA_BLEND_GLOBAL_ALPHA;
+ blend_state.global_alpha = ctx->alpha;
+ subpic_params.blend_state = &blend_state;
+
+ subpic_params.surface = (VASurfaceID)(uintptr_t)input_overlay->data[3];
+ subpic_params.output_region = &overlay_region;
+
+ err = overlay_vaapi_render_picture(avctx, ¶ms, &subpic_params, output);
+ if (err < 0)
+ goto fail;
+
+ av_log(avctx, AV_LOG_DEBUG, "Filter output: %s, %ux%u (%"PRId64").\n",
+ av_get_pix_fmt_name(output->format),
+ output->width, output->height, output->pts);
+
+ return ff_filter_frame(outlink, output);
+
+fail:
+ av_frame_free(&output);
+ return err;
+}
+
+static int overlay_vaapi_init_framesync(AVFilterContext *avctx)
+{
+ OverlayVAAPIContext *ctx = avctx->priv;
+ int ret, i;
+
+ ctx->fs.on_event = overlay_vaapi_blend;
+ ctx->fs.opaque = ctx;
+ ret = ff_framesync_init(&ctx->fs, avctx, avctx->nb_inputs);
+ if (ret < 0)
+ return ret;
+
+ for (i = 0; i < avctx->nb_inputs; i++) {
+ FFFrameSyncIn *in = &ctx->fs.in[i];
+ in->before = EXT_STOP;
+ in->after = EXT_INFINITY;
+ in->sync = i ? 1 : 2;
+ in->time_base = avctx->inputs[i]->time_base;
+ }
+
+ return ff_framesync_configure(&ctx->fs);
+}
+
+static int overlay_vaapi_config_output(AVFilterLink *outlink)
+{
+ AVFilterContext *avctx = outlink->src;
+ OverlayVAAPIContext *ctx = avctx->priv;
+ VAAPIVPPContext *vpp_ctx = avctx->priv;
+ int err;
+
+ err = overlay_vaapi_init_framesync(avctx);
+ if (err < 0)
+ return err;
+
+ vpp_ctx->output_width = avctx->inputs[0]->w;
+ vpp_ctx->output_height = avctx->inputs[0]->h;
+
+ err = ff_vaapi_vpp_config_output(outlink);
+ if (err < 0)
+ return err;
+
+ err = ff_framesync_init_dualinput(&ctx->fs, avctx);
+ if (err < 0)
+ return err;
+
+ return ff_framesync_configure(&ctx->fs);
+}
+
+static av_cold int overlay_vaapi_init(AVFilterContext *avctx)
+{
+ VAAPIVPPContext *vpp_ctx = avctx->priv;
+
+ ff_vaapi_vpp_ctx_init(avctx);
+ vpp_ctx->output_format = AV_PIX_FMT_NONE;
+
+ return 0;
+}
+
+static int overlay_vaapi_activate(AVFilterContext *avctx)
+{
+ OverlayVAAPIContext *ctx = avctx->priv;
+
+ return ff_framesync_activate(&ctx->fs);
+}
+
+static av_cold void overlay_vaapi_uninit(AVFilterContext *avctx)
+{
+ OverlayVAAPIContext *ctx = avctx->priv;
+
+ ff_framesync_uninit(&ctx->fs);
+}
+
+#define OFFSET(x) offsetof(OverlayVAAPIContext, x)
+#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
+static const AVOption overlay_vaapi_options[] = {
+ { "x", "Overlay x position",
+ OFFSET(overlay_ox), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, .flags = FLAGS },
+ { "y", "Overlay y position",
+ OFFSET(overlay_oy), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, .flags = FLAGS },
+ { "w", "Overlay width",
+ OFFSET(overlay_ow), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, .flags = FLAGS },
+ { "h", "Overlay height",
+ OFFSET(overlay_oh), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, .flags = FLAGS },
+ { "alpha", "Overlay global alpha",
+ OFFSET(alpha), AV_OPT_TYPE_FLOAT, { .dbl = 0.0}, 0.0, 1.0, .flags = FLAGS},
+ { NULL },
+};
+
+AVFILTER_DEFINE_CLASS(overlay_vaapi);
+
+static const AVFilterPad overlay_vaapi_inputs[] = {
+ {
+ .name = "main",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .get_video_buffer = ff_default_get_video_buffer,
+ .config_props = &ff_vaapi_vpp_config_input,
+ },
+ {
+ .name = "overlay",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .get_video_buffer = ff_default_get_video_buffer,
+ },
+ { NULL }
+};
+
+static const AVFilterPad overlay_vaapi_outputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .config_props = &overlay_vaapi_config_output,
+ },
+ { NULL }
+};
+
+AVFilter ff_vf_overlay_vaapi = {
+ .name = "overlay_vaapi",
+ .description = NULL_IF_CONFIG_SMALL("Overlay one video on top of another"),
+ .priv_size = sizeof(OverlayVAAPIContext),
+ .priv_class = &overlay_vaapi_class,
+ .init = &overlay_vaapi_init,
+ .uninit = &overlay_vaapi_uninit,
+ .query_formats = &overlay_vaapi_query_formats,
+ .activate = &overlay_vaapi_activate,
+ .inputs = overlay_vaapi_inputs,
+ .outputs = overlay_vaapi_outputs,
+ .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
+};
--
2.17.1
More information about the ffmpeg-devel
mailing list