[FFmpeg-devel] [PATCH v1] avfilter: Add tonemap vaapi filter for H2S
Song, Ruiling
ruiling.song at intel.com
Wed Nov 13 03:04:36 EET 2019
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces at ffmpeg.org> On Behalf Of
> Xinpeng Sun
> Sent: Tuesday, November 12, 2019 5:00 PM
> To: ffmpeg-devel at ffmpeg.org
> Cc: Sun, Xinpeng <xinpeng.sun at intel.com>; Zhou, Zachary
> <zachary.zhou at intel.com>
> Subject: [FFmpeg-devel] [PATCH v1] avfilter: Add tonemap vaapi filter for
> H2S
>
> It performs HDR(High Dynamic Range) to SDR(Standard Dynamic Range)
> conversion
> with tone-mapping. It supports HDR10 only as input temporarily.
>
> H2S: P010 -> NV12
Have you tried P010 HDR to P010 SDR? Does it work? I think people may like use 10bit SDR because it has more color details.
>
> An example command to use this filter with vaapi codecs:
> FFMPEG -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -
> hwaccel_output_format vaapi \
> -i INPUT -vf 'tonemap_vaapi=h2s,hwdownload,format=nv12' -pix_fmt nv12 \
> -f rawvideo -y OUTPUT
>
> Signed-off-by: Xinpeng Sun <xinpeng.sun at intel.com>
> Signed-off-by: Zachary Zhou <zachary.zhou at intel.com>
> ---
> doc/filters.texi | 30 ++++
> libavfilter/Makefile | 1 +
> libavfilter/allfilters.c | 1 +
> libavfilter/vaapi_vpp.c | 5 +
> libavfilter/vf_tonemap_vaapi.c | 272
> +++++++++++++++++++++++++++++++++
> 5 files changed, 309 insertions(+)
> create mode 100644 libavfilter/vf_tonemap_vaapi.c
>
> diff --git a/doc/filters.texi b/doc/filters.texi
> index 6800124574..b1c466ba24 100644
> --- a/doc/filters.texi
> +++ b/doc/filters.texi
> @@ -20754,6 +20754,36 @@ Convert HDR(PQ/HLG) video to bt2020-transfer-
> characteristic p010 format using li
> @end example
> @end itemize
>
This should not be here. Please move above opencl video filters or start another chapter dedicated for vaapi accelerated video filters somewhere.
> + at section tonemap_vappi
> +
> +Perform HDR(High Dynamic Range) to SDR(Standard Dynamic Range)
> conversion with tone-mapping.
> +It maps the dynamic range of HDR10 content to the SDR content.
> +It only accepts HDR10 as input temporarilly.
> +
> +It accepts the following parameters:
> +
> + at table @option
> + at item type
> +Specify the tone-mapping operator to be used.
> +
> +Possible values are:
> + at table @var
> + at item h2s
> +Perform H2S(HDR to SDR), convert from p010 to nv12
> + at end table
> +
> + at end table
> +
> + at subsection Example
> +
> + at itemize
> + at item
> +Convert HDR video to SDR video from p010 format to nv12 format.
> + at example
> +-i INPUT -vf "tonemap_vaapi=h2s" OUTPUT
> + at end example
> + at end itemize
> +
> @section unsharp_opencl
>
> Sharpen or blur the input video.
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index fce930360d..90a0e9945e 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -410,6 +410,7 @@ OBJS-$(CONFIG_TMIX_FILTER) += vf_mix.o
> framesync.o
> OBJS-$(CONFIG_TONEMAP_FILTER) += vf_tonemap.o colorspace.o
> OBJS-$(CONFIG_TONEMAP_OPENCL_FILTER) += vf_tonemap_opencl.o
> colorspace.o opencl.o \
> opencl/tonemap.o opencl/colorspace_common.o
> +OBJS-$(CONFIG_TONEMAP_VAAPI_FILTER) += vf_tonemap_vaapi.o
> vaapi_vpp.o
> OBJS-$(CONFIG_TPAD_FILTER) += vf_tpad.o
> OBJS-$(CONFIG_TRANSPOSE_FILTER) += vf_transpose.o
> OBJS-$(CONFIG_TRANSPOSE_NPP_FILTER) += vf_transpose_npp.o
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index 7c1e19e1da..b2fb1f8a98 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -390,6 +390,7 @@ extern AVFilter ff_vf_tlut2;
> extern AVFilter ff_vf_tmix;
> extern AVFilter ff_vf_tonemap;
> extern AVFilter ff_vf_tonemap_opencl;
> +extern AVFilter ff_vf_tonemap_vaapi;
> extern AVFilter ff_vf_tpad;
> extern AVFilter ff_vf_transpose;
> extern AVFilter ff_vf_transpose_npp;
> diff --git a/libavfilter/vaapi_vpp.c b/libavfilter/vaapi_vpp.c
> index b5b245c8af..5776243fa0 100644
> --- a/libavfilter/vaapi_vpp.c
> +++ b/libavfilter/vaapi_vpp.c
> @@ -257,6 +257,11 @@ static const VAAPIColourProperties
> vaapi_colour_standard_map[] = {
> { VAProcColorStandardSMPTE170M, 6, 6, 6 },
> { VAProcColorStandardSMPTE240M, 7, 7, 7 },
> { VAProcColorStandardGenericFilm, 8, 1, 1 },
> +
> +#if VA_CHECK_VERSION(2, 3, 0)
> + { VAProcColorStandardExplicit, 9, 16, AVCOL_SPC_BT2020_NCL},
> +#endif
> +
> #if VA_CHECK_VERSION(1, 1, 0)
> { VAProcColorStandardSRGB, 1, 13, 0 },
> { VAProcColorStandardXVYCC601, 1, 11, 5 },
> diff --git a/libavfilter/vf_tonemap_vaapi.c b/libavfilter/vf_tonemap_vaapi.c
> new file mode 100644
> index 0000000000..27ee17bf00
> --- /dev/null
> +++ b/libavfilter/vf_tonemap_vaapi.c
> @@ -0,0 +1,272 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
> USA
> + */
> +#include <string.h>
> +
> +#include "libavutil/avassert.h"
> +#include "libavutil/mem.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/pixdesc.h"
> +#include "libavutil/mastering_display_metadata.h"
> +
> +#include "avfilter.h"
> +#include "formats.h"
> +#include "internal.h"
> +#include "vaapi_vpp.h"
> +
> +typedef enum {
> + HDR_VAAPI_H2S,
> +} HDRType;
> +
> +typedef struct HDRVAAPIContext {
> + VAAPIVPPContext vpp_ctx; // must be the first field
> +
> + int hdr_type;
> +
> + char *master_display;
> + char *content_light;
> +
> + VAHdrMetaDataHDR10 in_metadata;
> +
> + AVFrameSideData *src_display;
> + AVFrameSideData *src_light;
> +} HDRVAAPIContext;
> +
> +static int tonemap_vaapi_set_filter_params(AVFilterContext *avctx,
> AVFrame *input_frame)
> +{
> + VAAPIVPPContext *vpp_ctx = avctx->priv;
> + HDRVAAPIContext *ctx = avctx->priv;
> + VAStatus vas;
> + VAProcFilterParameterBufferHDRToneMapping *hdrtm_param;
> +
> + vas = vaMapBuffer(vpp_ctx->hwctx->display, vpp_ctx->filter_buffers[0],
> + (void**)&hdrtm_param);
> + if (vas != VA_STATUS_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to map "
> + "buffer (%d): %d (%s).\n",
> + vpp_ctx->filter_buffers[0], vas, vaErrorStr(vas));
> + return AVERROR(EIO);
> + }
> +
> + memcpy(hdrtm_param->data.metadata, &ctx->in_metadata,
> sizeof(VAHdrMetaDataHDR10));
> +
> + vas = vaUnmapBuffer(vpp_ctx->hwctx->display, vpp_ctx-
> >filter_buffers[0]);
> + if (vas != VA_STATUS_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to unmap output buffers: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + return AVERROR(EIO);
> + }
> +
> + return 0;
> +}
> +
> +static int tonemap_vaapi_build_filter_params(AVFilterContext *avctx)
> +{
> + VAAPIVPPContext *vpp_ctx = avctx->priv;
> + HDRVAAPIContext *ctx = avctx->priv;
> + VAStatus vas;
> + VAProcFilterCapHighDynamicRange hdr_cap;
> + int num_query_caps;
> + VAProcFilterParameterBufferHDRToneMapping hdrtm_param;
> +
> + vas = vaQueryVideoProcFilterCaps(vpp_ctx->hwctx->display,
> + vpp_ctx->va_context,
> + VAProcFilterHighDynamicRangeToneMapping,
> + &hdr_cap, &num_query_caps);
> + if (vas != VA_STATUS_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to query HDR caps "
> + "context: %d (%s).\n", vas, vaErrorStr(vas));
> + return AVERROR(EIO);
> + }
> +
> + if (hdr_cap.metadata_type == VAProcHighDynamicRangeMetadataNone)
> {
> + av_log(avctx, AV_LOG_ERROR, "VAAPI driver doesn't support HDR\n");
> + return AVERROR(EINVAL);
> + }
> +
> + switch (ctx->hdr_type) {
> + case HDR_VAAPI_H2S:
> + if (!(VA_TONE_MAPPING_HDR_TO_SDR & hdr_cap.caps_flag)) {
> + av_log(avctx, AV_LOG_ERROR,
> + "VAAPI driver doesn't support H2S\n");
> + return AVERROR(EINVAL);
> + }
> + break;
> + default:
> + av_assert0(0);
> + }
> +
> + memset(&hdrtm_param, 0, sizeof(hdrtm_param));
> + memset(&ctx->in_metadata, 0, sizeof(ctx->in_metadata));
> + hdrtm_param.type = VAProcFilterHighDynamicRangeToneMapping;
> + hdrtm_param.data.metadata_type =
> VAProcHighDynamicRangeMetadataHDR10;
I think you need to check whether the input video is HDR10. If not, you need to return some error code.
And output an error message that this filter only support HDR10.
> + hdrtm_param.data.metadata = &ctx->in_metadata;
> + hdrtm_param.data.metadata_size = sizeof(VAHdrMetaDataHDR10);
> +
> + ff_vaapi_vpp_make_param_buffers(avctx,
> + VAProcFilterParameterBufferType,
> + &hdrtm_param, sizeof(hdrtm_param), 1);
> +
> + return 0;
> +}
> +
> +static int tonemap_vaapi_filter_frame(AVFilterLink *inlink, AVFrame
> *input_frame)
> +{
> + AVFilterContext *avctx = inlink->dst;
> + AVFilterLink *outlink = avctx->outputs[0];
> + VAAPIVPPContext *vpp_ctx = avctx->priv;
> + HDRVAAPIContext *ctx = avctx->priv;
> + AVFrame *output_frame = NULL;
> + VASurfaceID input_surface, output_surface;
> +
> + VAProcPipelineParameterBuffer params;
> + int err;
> +
> + av_log(avctx, AV_LOG_DEBUG, "Filter input: %s, %ux%u (%"PRId64").\n",
> + av_get_pix_fmt_name(input_frame->format),
> + input_frame->width, input_frame->height, input_frame->pts);
> +
> + if (vpp_ctx->va_context == VA_INVALID_ID){
> + av_frame_free(&input_frame);
> + return AVERROR(EINVAL);
> + }
> +
> + err = tonemap_vaapi_set_filter_params(avctx, input_frame);
> + if (err < 0)
> + goto fail;
> +
> + input_surface = (VASurfaceID)(uintptr_t)input_frame->data[3];
> + av_log(avctx, AV_LOG_DEBUG, "Using surface %#x for tonemap vpp
> input.\n",
> + input_surface);
> +
> + output_frame = ff_get_video_buffer(outlink, vpp_ctx->output_width,
> + vpp_ctx->output_height);
> + if (!output_frame) {
> + err = AVERROR(ENOMEM);
> + goto fail;
> + }
> +
> + output_surface = (VASurfaceID)(uintptr_t)output_frame->data[3];
> + av_log(avctx, AV_LOG_DEBUG, "Using surface %#x for tonemap vpp
> output.\n",
> + output_surface);
> + memset(¶ms, 0, sizeof(params));
> +
> + err = av_frame_copy_props(output_frame, input_frame);
> + if (err < 0)
> + return err;
> +
> + switch (ctx->hdr_type)
> + {
> + case HDR_VAAPI_H2S:
> + params.output_color_standard = VAProcColorStandardBT709;
> + params.output_color_properties.colour_primaries = AVCOL_PRI_BT709;
> + params.output_color_properties.transfer_characteristics =
> AVCOL_TRC_BT709;
I think it would be much better to allow setting these parameters through filter options. Please take a look at vf_scale_vaapi.c
Ruiling
> + break;
> + default:
> + av_assert0(0);
> + }
> +
> + err = ff_vaapi_vpp_init_params(avctx, ¶ms,
> + input_frame, output_frame);
> + if (err < 0)
> + goto fail;
> +
> + err = ff_vaapi_vpp_render_picture(avctx, ¶ms, output_frame);
> + if (err < 0)
> + goto fail;
> +
> + av_frame_free(&input_frame);
> +
> + av_log(avctx, AV_LOG_DEBUG, "Filter output: %s, %ux%u
> (%"PRId64").\n",
> + av_get_pix_fmt_name(output_frame->format),
> + output_frame->width, output_frame->height, output_frame->pts);
> +
> + return ff_filter_frame(outlink, output_frame);
> +
> +fail:
> + av_frame_free(&input_frame);
> + av_frame_free(&output_frame);
> + return err;
> +}
> +
> +static av_cold int tonemap_vaapi_init(AVFilterContext *avctx)
> +{
> + VAAPIVPPContext *vpp_ctx = avctx->priv;
> + HDRVAAPIContext *ctx = avctx->priv;
> +
> + ff_vaapi_vpp_ctx_init(avctx);
> + vpp_ctx->build_filter_params = tonemap_vaapi_build_filter_params;
> + vpp_ctx->pipeline_uninit = ff_vaapi_vpp_pipeline_uninit;
> +
> + if (ctx->hdr_type == HDR_VAAPI_H2S) {
> + vpp_ctx->output_format = AV_PIX_FMT_NV12;
> + } else {
> + av_assert0(0);
> + }
> +
> + return 0;
> +}
> +
> +static int tonemap_vaapi_vpp_query_formats(AVFilterContext *avctx)
> +{
> + return ff_vaapi_vpp_query_formats(avctx);
> +}
> +
> +#define OFFSET(x) offsetof(HDRVAAPIContext, x)
> +#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM |
> AV_OPT_FLAG_FILTERING_PARAM)
> +static const AVOption tonemap_vaapi_options[] = {
> + { "type", "hdr type", OFFSET(hdr_type), AV_OPT_TYPE_INT, { .i64
> = HDR_VAAPI_H2S }, 0, 1, FLAGS, "type" },
> + { "h2s", "vaapi P010 to NV12", 0, AV_OPT_TYPE_CONST,
> {.i64=HDR_VAAPI_H2S}, INT_MIN, INT_MAX, FLAGS, "type" },
> + { "display", "set master display", OFFSET(master_display),
> AV_OPT_TYPE_STRING, {.str=NULL}, CHAR_MIN, CHAR_MAX, FLAGS },
> + { "light", "set content light", OFFSET(content_light),
> AV_OPT_TYPE_STRING, {.str=NULL}, CHAR_MIN, CHAR_MAX, FLAGS },
> + { NULL }
> +};
> +
> +
> +AVFILTER_DEFINE_CLASS(tonemap_vaapi);
> +
> +static const AVFilterPad tonemap_vaapi_inputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_VIDEO,
> + .filter_frame = &tonemap_vaapi_filter_frame,
> + .config_props = &ff_vaapi_vpp_config_input,
> + },
> + { NULL }
> +};
> +
> +static const AVFilterPad tonemap_vaapi_outputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_VIDEO,
> + .config_props = &ff_vaapi_vpp_config_output,
> + },
> + { NULL }
> +};
> +
> +AVFilter ff_vf_tonemap_vaapi = {
> + .name = "tonemap_vaapi",
> + .description = NULL_IF_CONFIG_SMALL("VAAPI VPP for tonemap"),
> + .priv_size = sizeof(HDRVAAPIContext),
> + .init = &tonemap_vaapi_init,
> + .uninit = &ff_vaapi_vpp_ctx_uninit,
> + .query_formats = &tonemap_vaapi_vpp_query_formats,
> + .inputs = tonemap_vaapi_inputs,
> + .outputs = tonemap_vaapi_outputs,
> + .priv_class = &tonemap_vaapi_class,
> + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
> +};
> --
> 2.17.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
More information about the ffmpeg-devel
mailing list