[FFmpeg-devel] [PATCH 2/2] avfilter: add QSV variants of the stack filters
Xiang, Haihao
haihao.xiang at intel.com
Tue Feb 7 05:59:07 EET 2023
From: Haihao Xiang <haihao.xiang at intel.com>
Include hstack_qsv, vstack_qsv and xstack_qsv. They may accept input
streams with different sizes.
Examples:
$ ffmpeg -hwaccel qsv -hwaccel_output_format qsv -i input.mp4 \
-filter_complex "[0:v][0:v]hstack_qsv" -f null -
$ ffmpeg \
-hwaccel qsv -hwaccel_output_format qsv -i input.mp4 \
-hwaccel qsv -hwaccel_output_format qsv -i input.mp4 \
-hwaccel qsv -hwaccel_output_format qsv -i input.mp4 \
-hwaccel qsv -hwaccel_output_format qsv -i input.mp4 \
-filter_complex "[0:v][1:v][2:v][3:v]xstack_qsv=inputs=4:fill=0x000000:layout=0_0_1920x1080|w0_0_1920x1080|0_h0_1920x1080|w0_h0_1920x1080" \
-f null -
Signed-off-by: Haihao Xiang <haihao.xiang at intel.com>
---
Changelog | 1 +
configure | 6 +
doc/filters.texi | 88 +++++++++++++
libavfilter/Makefile | 3 +
libavfilter/allfilters.c | 3 +
libavfilter/version.h | 2 +-
libavfilter/vf_stack_qsv.c | 254 +++++++++++++++++++++++++++++++++++++
7 files changed, 356 insertions(+), 1 deletion(-)
create mode 100644 libavfilter/vf_stack_qsv.c
diff --git a/Changelog b/Changelog
index df9cd69da2..8955934c59 100644
--- a/Changelog
+++ b/Changelog
@@ -39,6 +39,7 @@ version <next>:
- ffmpeg CLI new option: -fix_sub_duration_heartbeat
- WavArc decoder and demuxer
- CrystalHD decoders deprecated
+- hstack_qsv, vstack_qsv and xstack_qsv filters
version 5.1:
diff --git a/configure b/configure
index d67855c729..1d535ab425 100755
--- a/configure
+++ b/configure
@@ -3772,6 +3772,12 @@ yadif_videotoolbox_filter_deps="metal corevideo videotoolbox"
hstack_vaapi_filter_deps="vaapi_1"
vstack_vaapi_filter_deps="vaapi_1"
xstack_vaapi_filter_deps="vaapi_1"
+hstack_qsv_filter_deps="libmfx"
+hstack_qsv_filter_select="qsvvpp"
+vstack_qsv_filter_deps="libmfx"
+vstack_qsv_filter_select="qsvvpp"
+xstack_qsv_filter_deps="libmfx"
+xstack_qsv_filter_select="qsvvpp"
# examples
avio_list_dir_deps="avformat avutil"
diff --git a/doc/filters.texi b/doc/filters.texi
index 3a54c68f3e..43c77dc041 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -26772,6 +26772,94 @@ See @ref{xstack}.
@c man end VAAPI VIDEO FILTERS
+ at chapter QSV Video Filters
+ at c man begin QSV VIDEO FILTERS
+
+Below is a description of the currently available QSV video filters.
+
+To enable compilation of these filters you need to configure FFmpeg with
+ at code{--enable-libmfx} or @code{--enable-libvpl}.
+
+To use QSV filters, you need to setup the QSV device correctly. For more information, please read @url{https://trac.ffmpeg.org/wiki/Hardware/QuickSync}
+
+ at section hstack_qsv
+Stack input videos horizontally.
+
+This is the QSV variant of the @ref{hstack} filter, each input stream may
+have different height, this filter will scale down/up each input stream while
+keeping the orignal aspect.
+
+It accepts the following options:
+
+ at table @option
+ at item inputs
+See @ref{hstack}.
+
+ at item shortest
+See @ref{hstack}.
+
+ at item height
+Set height of output. If set to 0, this filter will set height of output to
+height of the first input stream. Default value is 0.
+ at end table
+
+ at section vstack_qsv
+Stack input videos vertically.
+
+This is the QSV variant of the @ref{vstack} filter, each input stream may
+have different width, this filter will scale down/up each input stream while
+keeping the orignal aspect.
+
+It accepts the following options:
+
+ at table @option
+ at item inputs
+See @ref{vstack}.
+
+ at item shortest
+See @ref{vstack}.
+
+ at item width
+Set width of output. If set to 0, this filter will set width of output to
+width of the first input stream. Default value is 0.
+ at end table
+
+ at section xstack_qsv
+Stack video inputs into custom layout.
+
+This is the QSV variant of the @ref{xstack} filter.
+
+It accepts the following options:
+
+ at table @option
+ at item inputs
+See @ref{xstack}.
+
+ at item shortest
+See @ref{xstack}.
+
+ at item layout
+See @ref{xstack}.
+Moreover, this permits the user to supply output size for each input stream.
+ at example
+xstack_qsv=inputs=4:layout=0_0_1920x1080|0_h0_1920x1080|w0_0_1920x1080|w0_h0_1920x1080
+ at end example
+
+ at item grid
+See @ref{xstack}.
+
+ at item grid_tile_size
+Set output size for each input stream when @option{grid} is set. If this option
+is not set, this filter will set output size by default to the size of the
+first input stream. For the syntax of this option, check the
+ at ref{video size syntax,,"Video size" section in the ffmpeg-utils manual,ffmpeg-utils}.
+
+ at item fill
+See @ref{xstack}.
+ at end table
+
+ at c man end QSV VIDEO FILTERS
+
@chapter Video Sources
@c man begin VIDEO SOURCES
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 0173b11870..b3d3d981dd 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -561,6 +561,9 @@ OBJS-$(CONFIG_ZSCALE_FILTER) += vf_zscale.o
OBJS-$(CONFIG_HSTACK_VAAPI_FILTER) += vf_stack_vaapi.o framesync.o vaapi_vpp.o
OBJS-$(CONFIG_VSTACK_VAAPI_FILTER) += vf_stack_vaapi.o framesync.o vaapi_vpp.o
OBJS-$(CONFIG_XSTACK_VAAPI_FILTER) += vf_stack_vaapi.o framesync.o vaapi_vpp.o
+OBJS-$(CONFIG_HSTACK_QSV_FILTER) += vf_stack_qsv.o framesync.o
+OBJS-$(CONFIG_VSTACK_QSV_FILTER) += vf_stack_qsv.o framesync.o
+OBJS-$(CONFIG_XSTACK_QSV_FILTER) += vf_stack_qsv.o framesync.o
OBJS-$(CONFIG_ALLRGB_FILTER) += vsrc_testsrc.o
OBJS-$(CONFIG_ALLYUV_FILTER) += vsrc_testsrc.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 9cdcca4853..d7db46c2af 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -526,6 +526,9 @@ extern const AVFilter ff_vf_zscale;
extern const AVFilter ff_vf_hstack_vaapi;
extern const AVFilter ff_vf_vstack_vaapi;
extern const AVFilter ff_vf_xstack_vaapi;
+extern const AVFilter ff_vf_hstack_qsv;
+extern const AVFilter ff_vf_vstack_qsv;
+extern const AVFilter ff_vf_xstack_qsv;
extern const AVFilter ff_vsrc_allrgb;
extern const AVFilter ff_vsrc_allyuv;
diff --git a/libavfilter/version.h b/libavfilter/version.h
index 057ab63415..93036a615d 100644
--- a/libavfilter/version.h
+++ b/libavfilter/version.h
@@ -31,7 +31,7 @@
#include "version_major.h"
-#define LIBAVFILTER_VERSION_MINOR 56
+#define LIBAVFILTER_VERSION_MINOR 57
#define LIBAVFILTER_VERSION_MICRO 100
diff --git a/libavfilter/vf_stack_qsv.c b/libavfilter/vf_stack_qsv.c
new file mode 100644
index 0000000000..9eb0748bd6
--- /dev/null
+++ b/libavfilter/vf_stack_qsv.c
@@ -0,0 +1,254 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * Hardware accelerated hstack, vstack and xstack filters based on Intel Quick Sync Video VPP
+ */
+
+#include "config_components.h"
+
+#include "libavutil/opt.h"
+#include "libavutil/common.h"
+#include "libavutil/pixdesc.h"
+#include "libavutil/eval.h"
+#include "libavutil/hwcontext.h"
+#include "libavutil/avstring.h"
+#include "libavutil/avassert.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/mathematics.h"
+#include "libavutil/parseutils.h"
+
+#include "internal.h"
+#include "filters.h"
+#include "formats.h"
+#include "video.h"
+
+#include "framesync.h"
+#include "qsvvpp.h"
+
+#define HSTACK_NAME "hstack_qsv"
+#define VSTACK_NAME "vstack_qsv"
+#define XSTACK_NAME "xstack_qsv"
+#define HWContext QSVVPPContext
+#define StackHWContext StackQSVContext
+#include "stack_internal.h"
+
+typedef struct StackQSVContext {
+ StackBaseContext base;
+
+ QSVVPPParam qsv_param;
+ mfxExtVPPComposite comp_conf;
+} StackQSVContext;
+
+static void rgb2yuv(float r, float g, float b, int *y, int *u, int *v, int depth)
+{
+ *y = ((0.21260*219.0/255.0) * r + (0.71520*219.0/255.0) * g +
+ (0.07220*219.0/255.0) * b) * ((1 << depth) - 1);
+ *u = (-(0.11457*224.0/255.0) * r - (0.38543*224.0/255.0) * g +
+ (0.50000*224.0/255.0) * b + 0.5) * ((1 << depth) - 1);
+ *v = ((0.50000*224.0/255.0) * r - (0.45415*224.0/255.0) * g -
+ (0.04585*224.0/255.0) * b + 0.5) * ((1 << depth) - 1);
+}
+
+static int process_frame(FFFrameSync *fs)
+{
+ AVFilterContext *ctx = fs->parent;
+ QSVVPPContext *qsv = fs->opaque;
+ AVFrame *frame = NULL;
+ int ret = 0;
+
+ for (int i = 0; i < ctx->nb_inputs; i++) {
+ ret = ff_framesync_get_frame(fs, i, &frame, 0);
+ if (ret == 0)
+ ret = ff_qsvvpp_filter_frame(qsv, ctx->inputs[i], frame);
+ if (ret < 0 && ret != AVERROR(EAGAIN))
+ break;
+ }
+
+ if (ret == 0 && qsv->got_frame == 0) {
+ for (int i = 0; i < ctx->nb_inputs; i++)
+ FF_FILTER_FORWARD_WANTED(ctx->outputs[0], ctx->inputs[i]);
+
+ ret = FFERROR_NOT_READY;
+ }
+
+ return ret;
+}
+
+static int config_output(AVFilterLink *outlink)
+{
+ AVFilterContext *ctx = outlink->src;
+ StackQSVContext *sctx = ctx->priv;
+ AVFilterLink *inlink0 = ctx->inputs[0];
+ enum AVPixelFormat in_format;
+ int depth = 8, ret;
+ mfxVPPCompInputStream *is = sctx->comp_conf.InputStream;
+
+ if (inlink0->format == AV_PIX_FMT_QSV) {
+ if (!inlink0->hw_frames_ctx || !inlink0->hw_frames_ctx->data)
+ return AVERROR(EINVAL);
+
+ in_format = ((AVHWFramesContext*)inlink0->hw_frames_ctx->data)->sw_format;
+ } else
+ in_format = inlink0->format;
+
+ sctx->qsv_param.out_sw_format = in_format;
+
+ for (int i = 1; i < sctx->base.nb_inputs; i++) {
+ AVFilterLink *inlink = ctx->inputs[i];
+
+ if (inlink0->format == AV_PIX_FMT_QSV) {
+ AVHWFramesContext *hwfc0 = (AVHWFramesContext *)inlink0->hw_frames_ctx->data;
+ AVHWFramesContext *hwfc = (AVHWFramesContext *)inlink->hw_frames_ctx->data;
+
+ if (inlink0->format != inlink->format) {
+ av_log(ctx, AV_LOG_ERROR, "Mixing hardware and software pixel formats is not supported.\n");
+
+ return AVERROR(EINVAL);
+ } else if (hwfc0->device_ctx != hwfc->device_ctx) {
+ av_log(ctx, AV_LOG_ERROR, "Inputs with different underlying QSV devices are forbidden.\n");
+
+ return AVERROR(EINVAL);
+ }
+ }
+ }
+
+ if (in_format == AV_PIX_FMT_P010)
+ depth = 10;
+
+ if (sctx->base.fillcolor_enable) {
+ int Y, U, V;
+
+ rgb2yuv(sctx->base.fillcolor[0] / 255.0, sctx->base.fillcolor[1] / 255.0,
+ sctx->base.fillcolor[2] / 255.0, &Y, &U, &V, depth);
+ sctx->comp_conf.Y = Y;
+ sctx->comp_conf.U = U;
+ sctx->comp_conf.V = V;
+ }
+
+ ret = config_comm_output(outlink);
+ if (ret < 0)
+ return ret;
+
+ for (int i = 0; i < sctx->base.nb_inputs; i++) {
+ is[i].DstX = sctx->base.regions[i].x;
+ is[i].DstY = sctx->base.regions[i].y;
+ is[i].DstW = sctx->base.regions[i].width;
+ is[i].DstH = sctx->base.regions[i].height;
+ is[i].GlobalAlpha = 255;
+ is[i].GlobalAlphaEnable = 0;
+ is[i].PixelAlphaEnable = 0;
+ }
+
+ return ff_qsvvpp_init(ctx, &sctx->qsv_param);
+}
+
+/*
+ * Callback for qsvvpp
+ * @Note: qsvvpp composition does not generate PTS for result frame.
+ * so we assign the PTS from framesync to the output frame.
+ */
+
+static int filter_callback(AVFilterLink *outlink, AVFrame *frame)
+{
+ StackQSVContext *sctx = outlink->src->priv;
+
+ frame->pts = av_rescale_q(sctx->base.fs.pts,
+ sctx->base.fs.time_base, outlink->time_base);
+ return ff_filter_frame(outlink, frame);
+}
+
+
+static int qsv_stack_init(AVFilterContext *ctx)
+{
+ StackQSVContext *sctx = ctx->priv;
+ int ret;
+
+ ret = stack_init(ctx);
+ if (ret)
+ return ret;
+
+ /* fill composite config */
+ sctx->comp_conf.Header.BufferId = MFX_EXTBUFF_VPP_COMPOSITE;
+ sctx->comp_conf.Header.BufferSz = sizeof(sctx->comp_conf);
+ sctx->comp_conf.NumInputStream = sctx->base.nb_inputs;
+ sctx->comp_conf.InputStream = av_calloc(sctx->base.nb_inputs,
+ sizeof(*sctx->comp_conf.InputStream));
+ if (!sctx->comp_conf.InputStream)
+ return AVERROR(ENOMEM);
+
+ /* initialize QSVVPP params */
+ sctx->qsv_param.filter_frame = filter_callback;
+ sctx->qsv_param.ext_buf = av_mallocz(sizeof(*sctx->qsv_param.ext_buf));
+
+ if (!sctx->qsv_param.ext_buf)
+ return AVERROR(ENOMEM);
+
+ sctx->qsv_param.ext_buf[0] = (mfxExtBuffer *)&sctx->comp_conf;
+ sctx->qsv_param.num_ext_buf = 1;
+ sctx->qsv_param.num_crop = 0;
+
+ return 0;
+}
+
+static av_cold void qsv_stack_uninit(AVFilterContext *ctx)
+{
+ StackQSVContext *sctx = ctx->priv;
+
+ stack_uninit(ctx);
+
+ ff_qsvvpp_close(ctx);
+ av_freep(&sctx->comp_conf.InputStream);
+ av_freep(&sctx->qsv_param.ext_buf);
+}
+
+static int qsv_stack_query_formats(AVFilterContext *ctx)
+{
+ static const enum AVPixelFormat pixel_formats[] = {
+ AV_PIX_FMT_NV12,
+ AV_PIX_FMT_P010,
+ AV_PIX_FMT_QSV,
+ AV_PIX_FMT_NONE,
+ };
+
+ return ff_set_common_formats_from_list(ctx, pixel_formats);
+}
+
+#include "stack_internal.c"
+
+#if CONFIG_HSTACK_QSV_FILTER
+
+DEFINE_HSTACK_OPTIONS(qsv);
+DEFINE_STACK_FILTER(hstack, qsv, "Quick Sync Video");
+
+#endif
+
+#if CONFIG_VSTACK_QSV_FILTER
+
+DEFINE_VSTACK_OPTIONS(qsv);
+DEFINE_STACK_FILTER(vstack, qsv, "Quick Sync Video");
+
+#endif
+
+#if CONFIG_XSTACK_QSV_FILTER
+
+DEFINE_XSTACK_OPTIONS(qsv);
+DEFINE_STACK_FILTER(xstack, qsv, "Quick Sync Video");
+
+#endif
--
2.25.1
More information about the ffmpeg-devel
mailing list