[FFmpeg-devel] [PATCH] qsvenc: Add ROI encoding support for h264_qsv and hevc_qsv
Zhong Li
lizhong1008 at gmail.com
Wed Dec 25 17:02:07 EET 2019
Guo, Yejun <yejun.guo at intel.com> 于2019年12月25日周三 下午7:10写道：
>
> command examples:
> ./ffmpeg -s 1920x1080 -i input.yuv -vf addroi=0:0:640:320:0.5 -c:v hevc_qsv -b:v 5M -y qsv.b5m.roi.h265
> ./ffmpeg -s 1920x1080 -i input.yuv -vf addroi=0:0:640:320:-0.5 -c:v h264_qsv -mfmode 1 -b:v 4M -y qsv.b4m.roi.h264
>
> Signed-off-by: Guo, Yejun <yejun.guo at intel.com>
> ---
> libavcodec/qsvenc.c | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> libavcodec/qsvenc.h | 12 +++++-
> 2 files changed, 124 insertions(+), 1 deletion(-)
>
> diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
> index 9e41650..aa42c9b 100644
> --- a/libavcodec/qsvenc.c
> +++ b/libavcodec/qsvenc.c
> @@ -482,6 +482,7 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q)
> int target_bitrate_kbps, max_bitrate_kbps, brc_param_multiplier;
> int buffer_size_in_kilobytes, initial_delay_in_kilobytes;
> int ret;
> + av_unused int mfe_enabled = 0;
>
> ret = ff_qsv_codec_id_to_mfx(avctx->codec_id);
> if (ret < 0)
> @@ -755,6 +756,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
> q->extmfp.Header.BufferSz = sizeof(q->extmfp);
>
> q->extmfp.MFMode = q->mfmode;
> + mfe_enabled = q->mfmode != MFX_MF_DISABLED;
> av_log(avctx,AV_LOG_VERBOSE,"MFMode:%d\n", q->extmfp.MFMode);
> q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extmfp;
> }
> @@ -790,6 +792,25 @@ FF_ENABLE_DEPRECATION_WARNINGS
> }
> #endif
>
> +#if QSV_HAVE_ROI_ENCODING
> + // mfe and roi could not be enabled together, they even could not be queried together,
> + // otherwise, gpu hang.
This should be fixed in MSDK instead of workaround here.
> + if (!mfe_enabled) {
> + q->extroi.Header.BufferId = MFX_EXTBUFF_ENCODER_ROI;
> + q->extroi.Header.BufferSz = sizeof(q->extroi);
> + q->extroi.ROIMode = MFX_ROI_MODE_QP_DELTA;
> + // 256 to query the maximum supported value
> + q->extroi.NumROI = 256;
> + // due to the requirement of msdk, we must set non_empty rect for query
> + for (int i = 0; i < sizeof(q->extroi.ROI)/sizeof(q->extroi.ROI[0]); ++i) {
Should be good to use FF_ARRAY_ELEMS()?
> + q->extroi.ROI[i].Right = 16;
> + q->extroi.ROI[i].Bottom = 16;
> + }
> + q->roi_index_in_internal_buffer = q->nb_extparam_internal;
> + q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extroi;
> + }
> +#endif
> +
> if (!check_enc_param(avctx,q)) {
> av_log(avctx, AV_LOG_ERROR,
> "some encoding parameters are not supported by the QSV "
> @@ -1178,6 +1199,10 @@ int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q)
> "Error querying encoder params");
> }
>
> +#if QSV_HAVE_ROI_ENCODING
> + q->roi_max_regions = q->extroi.NumROI;
> +#endif
> +
> ret = MFXVideoENCODE_QueryIOSurf(q->session, &q->param, &q->req);
> if (ret < 0)
> return ff_qsv_print_error(avctx, ret,
> @@ -1375,6 +1400,88 @@ static void print_interlace_msg(AVCodecContext *avctx, QSVEncContext *q)
> }
> }
>
> +static int qsv_encode_set_roi(AVCodecContext *avctx, QSVEncContext *q, const AVFrame *frame, mfxEncodeCtrl *enc_ctrl)
> +{
> +#if QSV_HAVE_ROI_ENCODING
> + // in qsv, the min_delta_qp is -51, and the max_delta_qp is 51
> + int qp_range = 51;
This is only true for HEVC and H264, not ture for vp8 and vp9.
Though I see ROI is just supported for HEVC and H264 now.
> + int nb_roi;
> + const AVRegionOfInterest *roi;
> + uint32_t roi_size;
> +
> + AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
> + if (!sd) {
> + // need to reset for the case: with roi side data --> no roi side data
> + enc_ctrl->NumExtParam = 0;
> + return 0;
> + }
> +
> +#if QSV_HAVE_MF
> + if (avctx->codec_id == AV_CODEC_ID_H264) {
> + if (q->mfmode != MFX_MF_DISABLED) {
> + if (!q->roi_warned) {
> + q->roi_warned = 1;
> + av_log(avctx, AV_LOG_WARNING, "to enable ROI, "
> + "please make sure Multi-Frame Mode is disabled.\n");
> + }
> + return 0;
> + }
> + }
> +#endif
> +
> + // msdk currently just supports roi encoding for h264 and h265
> + if (avctx->codec_id != AV_CODEC_ID_H264 && avctx->codec_id != AV_CODEC_ID_HEVC) {
I belive you should also check MSDK run-time libary version which can
support ROI.
> + if (!q->roi_warned) {
> + q->roi_warned = 1;
> + av_log(avctx, AV_LOG_WARNING, "ROI encoding is only enabled for h264_qsv and hevc_qsv.\n");
> + }
> + return 0;
> + }
> +
> + roi = (const AVRegionOfInterest*)sd->data;
> + roi_size = roi->self_size;
> + if (!roi_size || sd->size % roi_size != 0) {
> + av_log(avctx, AV_LOG_ERROR, "Invalid AVRegionOfInterest.self_size.\n");
> + return AVERROR(EINVAL);
> + }
> +
> + nb_roi = sd->size / roi_size;
> + if (nb_roi > q->roi_max_regions) {
> + if (!q->roi_warned) {
> + av_log(avctx, AV_LOG_WARNING, "More ROIs set than supported (%d > %d).\n",
> + nb_roi, q->roi_max_regions);
> + q->roi_warned = 1;
> + }
> + nb_roi = q->roi_max_regions;
> + }
> + q->extroi.NumROI = nb_roi;
> +
> + // For overlapping regions, the first in the array takes priority.
> + for (int i = 0; i < nb_roi; i++) {
> + float qoffset;
> +
> + roi = (const AVRegionOfInterest*)(sd->data + roi_size * i);
> + if (roi->qoffset.den == 0) {
> + av_log(avctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must not be zero.\n");
> + return AVERROR(EINVAL);
> + }
> + qoffset = roi->qoffset.num * 1.0f / roi->qoffset.den;
> + qoffset = av_clipf(qoffset * qp_range, -qp_range, +qp_range);
> +
> + q->extroi.ROI[i].Left = roi->left;
> + q->extroi.ROI[i].Right = roi->right;
> + q->extroi.ROI[i].Top = roi->top;
> + q->extroi.ROI[i].Bottom = roi->bottom;
> + q->extroi.ROI[i].DeltaQP = qoffset;
> + }
> +
> + enc_ctrl->NumExtParam = 1;
> + enc_ctrl->ExtParam = &q->extparam_internal[q->roi_index_in_internal_buffer];
> +#endif
> +
> + return 0;
> +}
> +
> static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
> const AVFrame *frame)
> {
> @@ -1445,6 +1552,12 @@ static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
> q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl);
> }
>
> + if (frame) {
> + ret = qsv_encode_set_roi(avctx, q, frame, &qsv_frame->enc_ctrl);
> + if (ret < 0)
> + return ret;
> + }
> +
> sync = av_mallocz(sizeof(*sync));
> if (!sync) {
> av_freep(&bs);
> diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h
> index 6609171..9c6caf1 100644
> --- a/libavcodec/qsvenc.h
> +++ b/libavcodec/qsvenc.h
> @@ -40,6 +40,7 @@
>
> #define QSV_HAVE_EXT_HEVC_TILES QSV_VERSION_ATLEAST(1, 13)
> #define QSV_HAVE_EXT_VP9_PARAM QSV_VERSION_ATLEAST(1, 26)
> +#define QSV_HAVE_ROI_ENCODING QSV_VERSION_ATLEAST(1, 22)
>
> #define QSV_HAVE_TRELLIS QSV_VERSION_ATLEAST(1, 8)
> #define QSV_HAVE_MAX_SLICE_SIZE QSV_VERSION_ATLEAST(1, 9)
> @@ -131,12 +132,21 @@ typedef struct QSVEncContext {
> #if QSV_HAVE_EXT_VP9_PARAM
> mfxExtVP9Param extvp9param;
> #endif
> +#if QSV_HAVE_ROI_ENCODING
> + mfxExtEncoderROI extroi;
> + // Maximum number of regions supported
> + int roi_max_regions;
> + int roi_index_in_internal_buffer;
> + // If the driver does not support ROI then warn the first time we
> + // encounter a frame with ROI side data.
> + int roi_warned;
> +#endif
>
> mfxExtOpaqueSurfaceAlloc opaque_alloc;
> mfxFrameSurface1 **opaque_surfaces;
> AVBufferRef *opaque_alloc_buf;
>
> - mfxExtBuffer *extparam_internal[2 + QSV_HAVE_CO2 + QSV_HAVE_CO3 + (QSV_HAVE_MF * 2)];
> + mfxExtBuffer *extparam_internal[2 + QSV_HAVE_CO2 + QSV_HAVE_CO3 + (QSV_HAVE_MF * 2) + QSV_HAVE_ROI_ENCODING];
> int nb_extparam_internal;
>
> mfxExtBuffer **extparam;
> --
> 2.7.4
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
More information about the ffmpeg-devel
mailing list